diff --git a/docs/help.md b/docs/help.md index 138b546..cdf0b67 100644 --- a/docs/help.md +++ b/docs/help.md @@ -34,6 +34,7 @@ Options: Commands: create Create and return new AWS credentials for specified... + delete-objects Delete one or more object from an S3 bucket delete-user Delete specified users, their access keys and their... get-cors-policy Get CORS policy for a bucket get-object Download an object from an S3 bucket @@ -102,6 +103,32 @@ Options: -a, --auth FILENAME Path to JSON/INI file containing credentials --help Show this message and exit. ``` +## s3-credentials delete-objects --help + +``` +Usage: s3-credentials delete-objects [OPTIONS] BUCKET [KEYS]... + + Delete one or more object from an S3 bucket + + Pass one or more keys to delete them: + + s3-credentials delete-objects my-bucket one.txt two.txt + + To delete all files matching a prefix, pass --prefix: + + s3-credentials delete-objects my-bucket --prefix my-folder/ + +Options: + --prefix TEXT Delete everything with this prefix + -s, --silent Don't show informational output + -d, --dry-run Show keys that would be deleted without deleting them + --access-key TEXT AWS access key ID + --secret-key TEXT AWS secret access key + --session-token TEXT AWS session token + --endpoint-url TEXT Custom endpoint URL + -a, --auth FILENAME Path to JSON/INI file containing credentials + --help Show this message and exit. +``` ## s3-credentials delete-user --help ``` diff --git a/docs/other-commands.md b/docs/other-commands.md index d91939c..548c1f3 100644 --- a/docs/other-commands.md +++ b/docs/other-commands.md @@ -1,5 +1,12 @@ # Other commands +```{contents} +--- +local: +class: this-will-duplicate-information-and-it-is-still-useful-here +--- +``` + ## policy You can use the `s3-credentials policy` command to generate the JSON policy document that would be used without applying it. The command takes one or more required bucket names and a subset of the options available on the `create` command: @@ -399,6 +406,22 @@ out/alverstone-mead-2.jpg => s3://my-bucket/out/alverstone-mead-2.jpg out/alverstone-mead-1.jpg => s3://my-bucket/out/alverstone-mead-1.jpg ``` +## delete-objects + +`s3-credentials delete-objects` can be used to delete one or more keys from the bucket. + +Pass one or more keys to delete them: + + s3-credentials delete-objects my-bucket one.txt two.txt three.txt + +Use `--prefix my-prefix` to delete all keys with the specified prefix: + + s3-credentials delete-objects my-bucket --prefix my-prefix + +Pass `-d` or `--dry-run` to perform a dry-run of the deletion, which will list the keys that would be deleted without actually deleting them. + + s3-credentials delete-objects my-bucket --prefix my-prefix --dry-run + ## get-object To download a file from a bucket use `s3-credentials get-object`: diff --git a/s3_credentials/cli.py b/s3_credentials/cli.py index d57833e..ebe5b65 100644 --- a/s3_credentials/cli.py +++ b/s3_credentials/cli.py @@ -1329,6 +1329,75 @@ def get_cors_policy(bucket, **boto_options): click.echo(json.dumps(response["CORSRules"], indent=4, default=str)) +@cli.command() +@click.argument("bucket") +@click.argument( + "keys", + nargs=-1, +) +@click.option( + "--prefix", + help="Delete everything with this prefix", +) +@click.option( + "silent", "-s", "--silent", is_flag=True, help="Don't show informational output" +) +@click.option( + "dry_run", + "-d", + "--dry-run", + is_flag=True, + help="Show keys that would be deleted without deleting them", +) +@common_boto3_options +def delete_objects(bucket, keys, prefix, silent, dry_run, **boto_options): + """ + Delete one or more object from an S3 bucket + + Pass one or more keys to delete them: + + s3-credentials delete-objects my-bucket one.txt two.txt + + To delete all files matching a prefix, pass --prefix: + + s3-credentials delete-objects my-bucket --prefix my-folder/ + """ + s3 = make_client("s3", **boto_options) + if keys and prefix: + raise click.ClickException("Cannot pass both keys and --prefix") + if not keys and not prefix: + raise click.ClickException("Specify one or more keys or use --prefix") + if prefix: + # List all keys with this prefix + paginator = s3.get_paginator("list_objects_v2") + response_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix) + keys = [] + for response in response_iterator: + keys.extend([obj["Key"] for obj in response.get("Contents", [])]) + if not silent: + click.echo( + "Deleting {} object{} from {}".format( + len(keys), "s" if len(keys) != 1 else "", bucket + ), + err=True, + ) + if dry_run: + click.echo("The following keys would be deleted:") + for key in keys: + click.echo(key) + return + for batch in batches(keys, 1000): + # Remove any rogue \r characters: + batch = [k.strip() for k in batch] + response = s3.delete_objects( + Bucket=bucket, Delete={"Objects": [{"Key": key} for key in batch]} + ) + if response.get("Errors"): + click.echo( + "Errors deleting objects: {}".format(response["Errors"]), err=True + ) + + def output(iterator, headers, nl, csv, tsv): if nl: for item in iterator: @@ -1397,3 +1466,7 @@ def format_bytes(size): size /= 1024 return size + + +def batches(all, batch_size): + return [all[i : i + batch_size] for i in range(0, len(all), batch_size)] diff --git a/tests/test_s3_credentials.py b/tests/test_s3_credentials.py index 720fd76..f8d0316 100644 --- a/tests/test_s3_credentials.py +++ b/tests/test_s3_credentials.py @@ -1212,3 +1212,67 @@ def test_put_objects(moto_s3, args, expected, expected_output): for obj in moto_s3.list_objects(Bucket="my-bucket").get("Contents") or [] } assert keys == (expected or set()) + + +@pytest.mark.parametrize( + "args,expected,expected_error", + ( + ([], None, "Error: Specify one or more keys or use --prefix"), + ( + ["one.txt", "--prefix", "directory/"], + None, + "Cannot pass both keys and --prefix", + ), + (["one.txt"], ["directory/two.txt", "directory/three.json"], None), + (["one.txt", "directory/two.txt"], ["directory/three.json"], None), + (["--prefix", "directory/"], ["one.txt"], None), + ), +) +def test_delete_objects(moto_s3_populated, args, expected, expected_error): + runner = CliRunner(mix_stderr=False) + with runner.isolated_filesystem(): + result = runner.invoke( + cli, ["delete-objects", "my-bucket"] + args, catch_exceptions=False + ) + if expected_error: + assert result.exit_code != 0 + assert expected_error in result.stderr + else: + assert result.exit_code == 0, result.output + # Check expected files are left in bucket + keys = { + obj["Key"] + for obj in moto_s3_populated.list_objects(Bucket="my-bucket").get( + "Contents" + ) + or [] + } + assert keys == set(expected) + + +@pytest.mark.parametrize("arg", ("-d", "--dry-run")) +def test_delete_objects_dry_run(moto_s3_populated, arg): + runner = CliRunner(mix_stderr=False) + + def get_keys(): + return { + obj["Key"] + for obj in moto_s3_populated.list_objects(Bucket="my-bucket").get( + "Contents" + ) + or [] + } + + with runner.isolated_filesystem(): + before_keys = get_keys() + result = runner.invoke( + cli, ["delete-objects", "my-bucket", "--prefix", "directory/", arg] + ) + assert result.exit_code == 0 + assert result.output == ( + "The following keys would be deleted:\n" + "directory/three.json\n" + "directory/two.txt\n" + ) + after_keys = get_keys() + assert before_keys == after_keys