delete-objects command

Closes #83

docs/help.md

@@ -34,6 +34,7 @@ Options:
 
 Commands:
   create              Create and return new AWS credentials for specified...
+  delete-objects      Delete one or more object from an S3 bucket
   delete-user         Delete specified users, their access keys and their...
   get-cors-policy     Get CORS policy for a bucket
   get-object          Download an object from an S3 bucket
@@ -102,6 +103,32 @@ Options:
   -a, --auth FILENAME             Path to JSON/INI file containing credentials
   --help                          Show this message and exit.
 ```
+## s3-credentials delete-objects --help
+
+```
+Usage: s3-credentials delete-objects [OPTIONS] BUCKET [KEYS]...
+
+  Delete one or more object from an S3 bucket
+
+  Pass one or more keys to delete them:
+
+      s3-credentials delete-objects my-bucket one.txt two.txt
+
+  To delete all files matching a prefix, pass --prefix:
+
+      s3-credentials delete-objects my-bucket --prefix my-folder/
+
+Options:
+  --prefix TEXT         Delete everything with this prefix
+  -s, --silent          Don't show informational output
+  -d, --dry-run         Show keys that would be deleted without deleting them
+  --access-key TEXT     AWS access key ID
+  --secret-key TEXT     AWS secret access key
+  --session-token TEXT  AWS session token
+  --endpoint-url TEXT   Custom endpoint URL
+  -a, --auth FILENAME   Path to JSON/INI file containing credentials
+  --help                Show this message and exit.
+```
 ## s3-credentials delete-user --help
 
 ```

docs/other-commands.md

@@ -1,5 +1,12 @@
 # Other commands
 
+```{contents}
+---
+local:
+class: this-will-duplicate-information-and-it-is-still-useful-here
+---
+```
+
 ## policy
 
 You can use the `s3-credentials policy` command to generate the JSON policy document that would be used without applying it. The command takes one or more required bucket names and a subset of the options available on the `create` command:
@@ -399,6 +406,22 @@ out/alverstone-mead-2.jpg => s3://my-bucket/out/alverstone-mead-2.jpg
 out/alverstone-mead-1.jpg => s3://my-bucket/out/alverstone-mead-1.jpg
 ```
 
+## delete-objects
+
+`s3-credentials delete-objects` can be used to delete one or more keys from the bucket.
+
+Pass one or more keys to delete them:
+
+    s3-credentials delete-objects my-bucket one.txt two.txt three.txt
+
+Use `--prefix my-prefix` to delete all keys with the specified prefix:
+
+    s3-credentials delete-objects my-bucket --prefix my-prefix
+
+Pass `-d` or `--dry-run` to perform a dry-run of the deletion, which will list the keys that would be deleted without actually deleting them.
+
+    s3-credentials delete-objects my-bucket --prefix my-prefix --dry-run
+
 ## get-object
 
 To download a file from a bucket use `s3-credentials get-object`:

s3_credentials/cli.py

@@ -1329,6 +1329,75 @@ def get_cors_policy(bucket, **boto_options):
     click.echo(json.dumps(response["CORSRules"], indent=4, default=str))
 
 
+@cli.command()
+@click.argument("bucket")
+@click.argument(
+    "keys",
+    nargs=-1,
+)
+@click.option(
+    "--prefix",
+    help="Delete everything with this prefix",
+)
+@click.option(
+    "silent", "-s", "--silent", is_flag=True, help="Don't show informational output"
+)
+@click.option(
+    "dry_run",
+    "-d",
+    "--dry-run",
+    is_flag=True,
+    help="Show keys that would be deleted without deleting them",
+)
+@common_boto3_options
+def delete_objects(bucket, keys, prefix, silent, dry_run, **boto_options):
+    """
+    Delete one or more object from an S3 bucket
+
+    Pass one or more keys to delete them:
+
+        s3-credentials delete-objects my-bucket one.txt two.txt
+
+    To delete all files matching a prefix, pass --prefix:
+
+        s3-credentials delete-objects my-bucket --prefix my-folder/
+    """
+    s3 = make_client("s3", **boto_options)
+    if keys and prefix:
+        raise click.ClickException("Cannot pass both keys and --prefix")
+    if not keys and not prefix:
+        raise click.ClickException("Specify one or more keys or use --prefix")
+    if prefix:
+        # List all keys with this prefix
+        paginator = s3.get_paginator("list_objects_v2")
+        response_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
+        keys = []
+        for response in response_iterator:
+            keys.extend([obj["Key"] for obj in response.get("Contents", [])])
+    if not silent:
+        click.echo(
+            "Deleting {} object{} from {}".format(
+                len(keys), "s" if len(keys) != 1 else "", bucket
+            ),
+            err=True,
+        )
+    if dry_run:
+        click.echo("The following keys would be deleted:")
+        for key in keys:
+            click.echo(key)
+        return
+    for batch in batches(keys, 1000):
+        # Remove any rogue \r characters:
+        batch = [k.strip() for k in batch]
+        response = s3.delete_objects(
+            Bucket=bucket, Delete={"Objects": [{"Key": key} for key in batch]}
+        )
+        if response.get("Errors"):
+            click.echo(
+                "Errors deleting objects: {}".format(response["Errors"]), err=True
+            )
+
+
 def output(iterator, headers, nl, csv, tsv):
     if nl:
         for item in iterator:
@@ -1397,3 +1466,7 @@ def format_bytes(size):
         size /= 1024
 
     return size
+
+
+def batches(all, batch_size):
+    return [all[i : i + batch_size] for i in range(0, len(all), batch_size)]

tests/test_s3_credentials.py

@@ -1212,3 +1212,67 @@ def test_put_objects(moto_s3, args, expected, expected_output):
             for obj in moto_s3.list_objects(Bucket="my-bucket").get("Contents") or []
         }
         assert keys == (expected or set())
+
+
+@pytest.mark.parametrize(
+    "args,expected,expected_error",
+    (
+        ([], None, "Error: Specify one or more keys or use --prefix"),
+        (
+            ["one.txt", "--prefix", "directory/"],
+            None,
+            "Cannot pass both keys and --prefix",
+        ),
+        (["one.txt"], ["directory/two.txt", "directory/three.json"], None),
+        (["one.txt", "directory/two.txt"], ["directory/three.json"], None),
+        (["--prefix", "directory/"], ["one.txt"], None),
+    ),
+)
+def test_delete_objects(moto_s3_populated, args, expected, expected_error):
+    runner = CliRunner(mix_stderr=False)
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            cli, ["delete-objects", "my-bucket"] + args, catch_exceptions=False
+        )
+        if expected_error:
+            assert result.exit_code != 0
+            assert expected_error in result.stderr
+        else:
+            assert result.exit_code == 0, result.output
+            # Check expected files are left in bucket
+            keys = {
+                obj["Key"]
+                for obj in moto_s3_populated.list_objects(Bucket="my-bucket").get(
+                    "Contents"
+                )
+                or []
+            }
+            assert keys == set(expected)
+
+
+@pytest.mark.parametrize("arg", ("-d", "--dry-run"))
+def test_delete_objects_dry_run(moto_s3_populated, arg):
+    runner = CliRunner(mix_stderr=False)
+
+    def get_keys():
+        return {
+            obj["Key"]
+            for obj in moto_s3_populated.list_objects(Bucket="my-bucket").get(
+                "Contents"
+            )
+            or []
+        }
+
+    with runner.isolated_filesystem():
+        before_keys = get_keys()
+        result = runner.invoke(
+            cli, ["delete-objects", "my-bucket", "--prefix", "directory/", arg]
+        )
+        assert result.exit_code == 0
+        assert result.output == (
+            "The following keys would be deleted:\n"
+            "directory/three.json\n"
+            "directory/two.txt\n"
+        )
+        after_keys = get_keys()
+        assert before_keys == after_keys