Release 0.16.1

Refs #86, #88, #89, #90

.github/workflows/publish.yml

@@ -4,54 +4,49 @@ on:
   release:
     types: [created]
 
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        cache: pip
+        cache-dependency-path: setup.py
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install '.[test]'
     - name: Run tests
       run: |
         pytest
   deploy:
     runs-on: ubuntu-latest
     needs: [test]
+    environment: release
+    permissions:
+      id-token: write
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-publish-pip-
+        python-version: "3.12"
+        cache: pip
+        cache-dependency-path: setup.py
     - name: Install dependencies
       run: |
-        pip install setuptools wheel twine build
-    - name: Publish
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        pip install setuptools wheel build
+    - name: Build
       run: |
         python -m build
-        twine upload dist/*
+    - name: Publish
+      uses: pypa/gh-action-pypi-publish@release/v1
+

.github/workflows/test.yml

@@ -1,35 +1,31 @@
 name: Test
 
-on: [push]
+on: [push, pull_request]
+
+permissions:
+  contents: read
 
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: actions/cache@v2
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        cache: pip
+        cache-dependency-path: setup.py
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
-        # Show dependency versions
-        pip freeze
+        pip install '.[test]'
     - name: Run tests
       run: |
         pytest
     - name: Check if cog needs to run
       run: |
         cog --check README.md
-        cog --check docs/*.md
+        cog --check docs/*.md

.readthedocs.yaml

@@ -0,0 +1,17 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: docs/conf.py
+
+formats:
+   - pdf
+   - epub
+
+python:
+   install:
+   - requirements: docs/requirements.txt

docs/create.md

@@ -58,7 +58,7 @@ The `create` command has a number of options:
 - `--username TEXT`: The username to use for the user that is created by the command (or the username of an existing user if you do not want to create a new one). If ommitted a default such as `s3.read-write.static.niche-museums.com` will be used.
 - `-c, --create-bucket`: Create the buckets if they do not exist. Without this any missing buckets will be treated as an error.
 - `--prefix my-prefix/`: Credentials should only allow access to keys in the S3 bucket that start with this prefix.
-- `--public`: When creating a bucket, set it so that any file uploaded to that bucket can be downloaded by anyone who knows its filename. This attaches the {ref}`public_bucket_policy`.
+- `--public`: When creating a bucket, set it so that any file uploaded to that bucket can be downloaded by anyone who knows its filename. This attaches the {ref}`public_bucket_policy` and sets the `PublicAccessBlockConfiguration` to `false` for [every option](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PublicAccessBlockConfiguration.html).
 - `--website`: Sets the bucket to public and configures it to act as a website, with `index.html` treated as an index page and `error.html` used to display custom errors. The URL for the website will be `http://<bucket-name>.s3-website.<region>.amazonaws.com/` - the region defaults to `us-east-1` unless you specify a `--bucket-region`.
 - `--read-only`: The user should only be allowed to read files from the bucket.
 - `--write-only`: The user should only be allowed to write files to the bucket, but not read them. This can be useful for logging and backups.

docs/help.md

@@ -34,6 +34,7 @@ Options:
 
 Commands:
   create              Create and return new AWS credentials for specified...
+  debug-bucket        Run a bunch of diagnostics to help debug a bucket
   delete-objects      Delete one or more object from an S3 bucket
   delete-user         Delete specified users, their access keys and their...
   get-cors-policy     Get CORS policy for a bucket
@@ -103,6 +104,23 @@ Options:
   -a, --auth FILENAME             Path to JSON/INI file containing credentials
   --help                          Show this message and exit.
 ```
+## s3-credentials debug-bucket --help
+
+```
+Usage: s3-credentials debug-bucket [OPTIONS] BUCKET
+
+  Run a bunch of diagnostics to help debug a bucket
+
+     s3-credentials debug-bucket my-bucket
+
+Options:
+  --access-key TEXT     AWS access key ID
+  --secret-key TEXT     AWS secret access key
+  --session-token TEXT  AWS session token
+  --endpoint-url TEXT   Custom endpoint URL
+  -a, --auth FILENAME   Path to JSON/INI file containing credentials
+  --help                Show this message and exit.
+```
 ## s3-credentials delete-objects --help
 
 ```
@@ -238,8 +256,13 @@ Usage: s3-credentials list-bucket [OPTIONS] BUCKET
 
       s3-credentials list-bucket my-bucket --csv
 
+  Add --urls to get an extra URL field for each key:
+
+      s3-credentials list-bucket my-bucket --urls
+
 Options:
   --prefix TEXT         List keys starting with this prefix
+  --urls                Show URLs for each key
   --nl                  Output newline-delimited JSON
   --csv                 Output CSV
   --tsv                 Output TSV

docs/other-commands.md

@@ -180,6 +180,8 @@ You can use the `--prefix myprefix/` option to list only keys that start with a
 
 The commmand accepts the same `--nl`, `--csv` and `--tsv` options as `list-users`.
 
+Add `--urls` to include a `URL` field in the output providing the full URL to each object.
+
 ## list-user-policies
 
 To see a list of inline policies belonging to users:
@@ -503,3 +505,45 @@ The following example allows GET and PUT methods from code running on `https://w
       --allowed-origin https://www.example.com/ \
       --expose-header ETag \
       --max-age-seconds 60
+
+## debug-bucket
+
+The `debug-bucket` command is useful for diagnosing issues with a bucket:
+
+    s3-credentials debug-bucket my-bucket
+
+Example output:
+```
+Bucket ACL:
+{
+    "Owner": {
+        "DisplayName": "username",
+        "ID": "cc8ca3a037c6a7c1fa7580076bf7cd1949b3f2f58f01c9df9e53c51f6a249910"
+    },
+    "Grants": [
+        {
+            "Grantee": {
+                "DisplayName": "username",
+                "ID": "cc8ca3a037c6a7c1fa7580076bf7cd1949b3f2f58f01c9df9e53c51f6a249910",
+                "Type": "CanonicalUser"
+            },
+            "Permission": "FULL_CONTROL"
+        }
+    ]
+}
+Bucket policy status:
+{
+    "PolicyStatus": {
+        "IsPublic": true
+    }
+}
+Bucket public access block:
+{
+    "PublicAccessBlockConfiguration": {
+        "BlockPublicAcls": false,
+        "IgnorePublicAcls": false,
+        "BlockPublicPolicy": false,
+        "RestrictPublicBuckets": false
+    }
+}
+```

s3_credentials/cli.py

@@ -16,6 +16,13 @@ import sys
 import textwrap
 from . import policies
 
+PUBLIC_ACCESS_BLOCK_CONFIGURATION = {
+    "BlockPublicAcls": False,
+    "IgnorePublicAcls": False,
+    "BlockPublicPolicy": False,
+    "RestrictPublicBuckets": False,
+}
+
 
 def bucket_exists(s3, bucket):
     try:
@@ -306,7 +313,7 @@ def create(
     user_permissions_boundary,
     silent,
     dry_run,
-    **boto_options
+    **boto_options,
 ):
     """
     Create and return new AWS credentials for specified S3 buckets - optionally
@@ -388,6 +395,12 @@ def create(
                             ),
                         )
                     )
+                    if public:
+                        click.echo(
+                            "... then add this public access block configuration:"
+                        )
+                        click.echo(json.dumps(PUBLIC_ACCESS_BLOCK_CONFIGURATION))
+
                     if bucket_policy:
                         click.echo("... then attach the following bucket policy to it:")
                         click.echo(json.dumps(bucket_policy, indent=4))
@@ -402,6 +415,13 @@ def create(
                         info += " in region: {}".format(bucket_region)
                     log(info)
 
+                    if public:
+                        s3.put_public_access_block(
+                            Bucket=bucket,
+                            PublicAccessBlockConfiguration=PUBLIC_ACCESS_BLOCK_CONFIGURATION,
+                        )
+                        log("Set public access block configuration")
+
                     if bucket_policy:
                         s3.put_bucket_policy(
                             Bucket=bucket, Policy=json.dumps(bucket_policy)
@@ -781,10 +801,10 @@ def list_buckets(buckets, details, nl, csv, tsv, **boto_options):
                         ).items()
                         if key != "ResponseMetadata"
                     )
-                    bucket_website[
-                        "url"
-                    ] = "http://{}.s3-website.{}.amazonaws.com/".format(
-                        bucket["Name"], region
+                    bucket_website["url"] = (
+                        "http://{}.s3-website.{}.amazonaws.com/".format(
+                            bucket["Name"], region
+                        )
                     )
                 except s3.exceptions.ClientError:
                     bucket_website = None
@@ -916,9 +936,10 @@ def ensure_s3_role_exists(iam, sts):
 @cli.command()
 @click.argument("bucket")
 @click.option("--prefix", help="List keys starting with this prefix")
+@click.option("--urls", is_flag=True, help="Show URLs for each key")
 @common_output_options
 @common_boto3_options
-def list_bucket(bucket, prefix, nl, csv, tsv, **boto_options):
+def list_bucket(bucket, prefix, urls, nl, csv, tsv, **boto_options):
     """
     List contents of bucket
 
@@ -929,16 +950,31 @@ def list_bucket(bucket, prefix, nl, csv, tsv, **boto_options):
     Add --csv or --csv for CSV or TSV format:
 
         s3-credentials list-bucket my-bucket --csv
+
+    Add --urls to get an extra URL field for each key:
+
+        s3-credentials list-bucket my-bucket --urls
     """
     s3 = make_client("s3", **boto_options)
     kwargs = {"Bucket": bucket}
     if prefix:
         kwargs["Prefix"] = prefix
 
+    fields = ["Key", "LastModified", "ETag", "Size", "StorageClass", "Owner"]
+    if urls:
+        fields.append("URL")
+
+    items = paginate(s3, "list_objects_v2", "Contents", **kwargs)
+    if urls:
+        items = (
+            dict(item, URL="https://s3.amazonaws.com/{}/{}".format(bucket, item["Key"]))
+            for item in items
+        )
+
     try:
         output(
-            paginate(s3, "list_objects_v2", "Contents", **kwargs),
-            ("Key", "LastModified", "ETag", "Size", "StorageClass", "Owner"),
+            items,
+            fields,
             nl,
             csv,
             tsv,
@@ -1272,7 +1308,7 @@ def set_cors_policy(
     allowed_origins,
     expose_headers,
     max_age_seconds,
-    **boto_options
+    **boto_options,
 ):
     """
     Set CORS policy for a bucket
@@ -1329,6 +1365,49 @@ def get_cors_policy(bucket, **boto_options):
     click.echo(json.dumps(response["CORSRules"], indent=4, default=str))
 
 
+def without_response_metadata(data):
+    return dict(
+        (key, value) for key, value in data.items() if key != "ResponseMetadata"
+    )
+
+
+@cli.command()
+@click.argument("bucket")
+@common_boto3_options
+def debug_bucket(bucket, **boto_options):
+    """
+    Run a bunch of diagnostics to help debug a bucket
+
+       s3-credentials debug-bucket my-bucket
+    """
+    s3 = make_client("s3", **boto_options)
+
+    try:
+        bucket_acl = s3.get_bucket_acl(Bucket=bucket)
+        click.echo("Bucket ACL:")
+        click.echo(json.dumps(without_response_metadata(bucket_acl), indent=4))
+    except Exception as ex:
+        print(f"Error checking bucket ACL: {ex}")
+
+    try:
+        bucket_policy_status = s3.get_bucket_policy_status(Bucket=bucket)
+        click.echo("Bucket policy status:")
+        click.echo(
+            json.dumps(without_response_metadata(bucket_policy_status), indent=4)
+        )
+    except Exception as ex:
+        print(f"Error checking bucket policy status: {ex}")
+
+    try:
+        bucket_public_access_block = s3.get_public_access_block(Bucket=bucket)
+        click.echo("Bucket public access block:")
+        click.echo(
+            json.dumps(without_response_metadata(bucket_public_access_block), indent=4)
+        )
+    except Exception as ex:
+        print(f"Error checking bucket public access block: {ex}")
+
+
 @cli.command()
 @click.argument("bucket")
 @click.argument(
@@ -1450,9 +1529,11 @@ def fix_json(row):
         [
             (
                 key,
-                json.dumps(value, indent=2, default=str)
-                if isinstance(value, (dict, list, tuple))
-                else value,
+                (
+                    json.dumps(value, indent=2, default=str)
+                    if isinstance(value, (dict, list, tuple))
+                    else value
+                ),
             )
             for key, value in row.items()
         ]

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 import os
 
-VERSION = "0.15"
+VERSION = "0.16.1"
 
 
 def get_long_description():
@@ -32,6 +32,6 @@ setup(
         s3-credentials=s3_credentials.cli:cli
     """,
     install_requires=["click", "boto3"],
-    extras_require={"test": ["pytest", "pytest-mock", "cogapp", "moto[s3]"]},
+    extras_require={"test": ["pytest", "pytest-mock", "cogapp", "moto>=5.0.4"]},
     python_requires=">=3.7",
 )

tests/conftest.py

@@ -2,7 +2,7 @@ import boto3
 import logging
 import os
 import pytest
-from moto import mock_s3
+from moto import mock_aws
 
 
 def pytest_addoption(parser):
@@ -51,7 +51,7 @@ def aws_credentials():
 
 @pytest.fixture(scope="function")
 def moto_s3(aws_credentials):
-    with mock_s3():
+    with mock_aws():
         client = boto3.client("s3", region_name="us-east-1")
         client.create_bucket(Bucket="my-bucket")
         yield client

tests/test_dry_run.py

@@ -55,6 +55,8 @@ Would assume role using following policy for 1200 seconds:*"""
             ["--public"],
             (
                 """Would create bucket: 'my-bucket'
+... then add this public access block configuration:
+{"BlockPublicAcls": false, "IgnorePublicAcls": false, "BlockPublicPolicy": false, "RestrictPublicBuckets": false}
 ... then attach the following bucket policy to it:*
 Would create user: 's3.read-write.my-bucket' with permissions boundary: 'arn:aws:iam::aws:policy/AmazonS3FullAccess'
 Would attach policy called 's3.read-write.my-bucket' to user 's3.read-write.my-bucket', details:*

tests/test_s3_credentials.py

@@ -476,6 +476,7 @@ def test_create_public(mocker):
         assert result.exit_code == 0
         assert result.output == (
             "Created bucket: pytest-bucket-simonw-1\n"
+            "Set public access block configuration\n"
             "Attached bucket policy allowing public access\n"
             "Attached policy s3.read-write.pytest-bucket-simonw-1 to user s3.read-write.pytest-bucket-simonw-1\n"
             "Created access key for user: s3.read-write.pytest-bucket-simonw-1\n"
@@ -490,11 +491,8 @@ def test_create_public(mocker):
             "call('sts')",
             "call().head_bucket(Bucket='pytest-bucket-simonw-1')",
             "call().create_bucket(Bucket='pytest-bucket-simonw-1')",
-            "call().put_bucket_policy(Bucket='pytest-bucket-simonw-1', "
-            'Policy=\'{"Version": "2012-10-17", "Statement": [{"Sid": '
-            '"AllowAllGetObject", "Effect": "Allow", "Principal": "*", "Action": '
-            '["s3:GetObject"], "Resource": '
-            '["arn:aws:s3:::pytest-bucket-simonw-1/*"]}]}\')',
+            "call().put_public_access_block(Bucket='pytest-bucket-simonw-1', PublicAccessBlockConfiguration={'BlockPublicAcls': False, 'IgnorePublicAcls': False, 'BlockPublicPolicy': False, 'RestrictPublicBuckets': False})",
+            'call().put_bucket_policy(Bucket=\'pytest-bucket-simonw-1\', Policy=\'{"Version": "2012-10-17", "Statement": [{"Sid": "AllowAllGetObject", "Effect": "Allow", "Principal": "*", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::pytest-bucket-simonw-1/*"]}]}\')',
             "call().get_user(UserName='s3.read-write.pytest-bucket-simonw-1')",
             "call().put_user_policy(PolicyDocument='{}', PolicyName='s3.read-write.pytest-bucket-simonw-1', UserName='s3.read-write.pytest-bucket-simonw-1')".format(
                 READ_WRITE_POLICY.replace("$!BUCKET_NAME!$", "pytest-bucket-simonw-1"),
@@ -520,6 +518,7 @@ def test_create_website(mocker):
         assert result.exit_code == 0
         assert result.output == (
             "Created bucket: pytest-bucket-simonw-1\n"
+            "Set public access block configuration\n"
             "Attached bucket policy allowing public access\n"
             "Configured website: IndexDocument=index.html, ErrorDocument=error.html\n"
             "Attached policy s3.read-write.pytest-bucket-simonw-1 to user s3.read-write.pytest-bucket-simonw-1\n"
@@ -535,14 +534,9 @@ def test_create_website(mocker):
             "call('sts')",
             "call().head_bucket(Bucket='pytest-bucket-simonw-1')",
             "call().create_bucket(Bucket='pytest-bucket-simonw-1')",
-            "call().put_bucket_policy(Bucket='pytest-bucket-simonw-1', "
-            'Policy=\'{"Version": "2012-10-17", "Statement": [{"Sid": '
-            '"AllowAllGetObject", "Effect": "Allow", "Principal": "*", "Action": '
-            '["s3:GetObject"], "Resource": '
-            '["arn:aws:s3:::pytest-bucket-simonw-1/*"]}]}\')',
-            "call().put_bucket_website(Bucket='pytest-bucket-simonw-1', "
-            "WebsiteConfiguration={'ErrorDocument': {'Key': 'error.html'}, "
-            "'IndexDocument': {'Suffix': 'index.html'}})",
+            "call().put_public_access_block(Bucket='pytest-bucket-simonw-1', PublicAccessBlockConfiguration={'BlockPublicAcls': False, 'IgnorePublicAcls': False, 'BlockPublicPolicy': False, 'RestrictPublicBuckets': False})",
+            'call().put_bucket_policy(Bucket=\'pytest-bucket-simonw-1\', Policy=\'{"Version": "2012-10-17", "Statement": [{"Sid": "AllowAllGetObject", "Effect": "Allow", "Principal": "*", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::pytest-bucket-simonw-1/*"]}]}\')',
+            "call().put_bucket_website(Bucket='pytest-bucket-simonw-1', WebsiteConfiguration={'ErrorDocument': {'Key': 'error.html'}, 'IndexDocument': {'Suffix': 'index.html'}})",
             "call().get_user(UserName='s3.read-write.pytest-bucket-simonw-1')",
             "call().put_user_policy(PolicyDocument='{}', PolicyName='s3.read-write.pytest-bucket-simonw-1', UserName='s3.read-write.pytest-bucket-simonw-1')".format(
                 READ_WRITE_POLICY.replace("$!BUCKET_NAME!$", "pytest-bucket-simonw-1"),