diff --git a/docs/help.md b/docs/help.md index 670e0fb..138b546 100644 --- a/docs/help.md +++ b/docs/help.md @@ -45,6 +45,7 @@ Commands: list-users List all users for this account policy Output generated JSON policy for one or more buckets put-object Upload an object to an S3 bucket + put-objects Upload multiple objects to an S3 bucket set-cors-policy Set CORS policy for a bucket whoami Identify currently authenticated user ``` @@ -374,6 +375,45 @@ Options: -a, --auth FILENAME Path to JSON/INI file containing credentials --help Show this message and exit. ``` +## s3-credentials put-objects --help + +``` +Usage: s3-credentials put-objects [OPTIONS] BUCKET OBJECTS... + + Upload multiple objects to an S3 bucket + + Pass one or more files to upload them: + + s3-credentials put-objects my-bucket one.txt two.txt + + These will be saved to the root of the bucket. To save to a different location + use the --prefix option: + + s3-credentials put-objects my-bucket one.txt two.txt --prefix my-folder + + This will upload them my-folder/one.txt and my-folder/two.txt. + + If you pass a directory it will be uploaded recursively: + + s3-credentials put-objects my-bucket my-folder + + This will create keys in my-folder/... in the S3 bucket. + + To upload all files in a folder to the root of the bucket instead use this: + + s3-credentials put-objects my-bucket my-folder/* + +Options: + --prefix TEXT Prefix to add to the files within the bucket + -s, --silent Don't show progress bar + --dry-run Show steps without executing them + --access-key TEXT AWS access key ID + --secret-key TEXT AWS secret access key + --session-token TEXT AWS session token + --endpoint-url TEXT Custom endpoint URL + -a, --auth FILENAME Path to JSON/INI file containing credentials + --help Show this message and exit. +``` ## s3-credentials set-cors-policy --help ``` diff --git a/docs/other-commands.md b/docs/other-commands.md index 81eb632..d91939c 100644 --- a/docs/other-commands.md +++ b/docs/other-commands.md @@ -359,6 +359,46 @@ The `Content-Type` on the uploaded object will be automatically set based on the echo "

Hello World

" | \ s3-credentials put-object my-bucket hello.html - --content-type "text/html" +## put-objects + +`s3-credentials put-objects` can be used to upload more than one file at once. + +Pass one or more filenames to upload them to the root of your bucket: + + s3-credentials put-objects my-bucket one.txt two.txt three.txt + +Use `--prefix my-prefix` to upload them to the specified prefix: + + s3-credentials put-objects my-bucket one.txt --prefix my-prefix + +This will upload the file to `my-prefix/one.txt`. + +Pass one or more directories to upload the contents of those directories. +`.` uploads everything in your current directory: + + s3-credentials put-objects my-bucket . + +Passing directory names will upload the directory and all of its contents: + + s3-credentials put-objects my-bucket my-directory + +If `my-directory` had files `one.txt` and `two.txt` in it, the result would be: + + my-directory/one.txt + my-directory/two.txt + +A progress bar will be shown by default. Use `-s` or `--silent` to hide it. + +Add `--dry-run` to get a preview of what would be uploaded without uploading anything: + + s3-credentials put-objects my-bucket . --dry-run + +``` +out/IMG_1254.jpeg => s3://my-bucket/out/IMG_1254.jpeg +out/alverstone-mead-2.jpg => s3://my-bucket/out/alverstone-mead-2.jpg +out/alverstone-mead-1.jpg => s3://my-bucket/out/alverstone-mead-1.jpg +``` + ## get-object To download a file from a bucket use `s3-credentials get-object`: diff --git a/s3_credentials/cli.py b/s3_credentials/cli.py index 2c92430..d57833e 100644 --- a/s3_credentials/cli.py +++ b/s3_credentials/cli.py @@ -992,7 +992,7 @@ def put_object(bucket, key, path, content_type, silent, **boto_options): extra_args["ContentType"] = content_type if not silent: # Show progress bar - with click.progressbar(length=size, label="Uploading") as bar: + with click.progressbar(length=size, label="Uploading", file=sys.stderr) as bar: s3.upload_fileobj( fp, bucket, key, Callback=bar.update, ExtraArgs=extra_args ) @@ -1000,6 +1000,91 @@ def put_object(bucket, key, path, content_type, silent, **boto_options): s3.upload_fileobj(fp, bucket, key, ExtraArgs=extra_args) +@cli.command() +@click.argument("bucket") +@click.argument( + "objects", + nargs=-1, + required=True, +) +@click.option( + "--prefix", + help="Prefix to add to the files within the bucket", +) +@click.option("silent", "-s", "--silent", is_flag=True, help="Don't show progress bar") +@click.option("--dry-run", help="Show steps without executing them", is_flag=True) +@common_boto3_options +def put_objects(bucket, objects, prefix, silent, dry_run, **boto_options): + """ + Upload multiple objects to an S3 bucket + + Pass one or more files to upload them: + + s3-credentials put-objects my-bucket one.txt two.txt + + These will be saved to the root of the bucket. To save to a different location + use the --prefix option: + + s3-credentials put-objects my-bucket one.txt two.txt --prefix my-folder + + This will upload them my-folder/one.txt and my-folder/two.txt. + + If you pass a directory it will be uploaded recursively: + + s3-credentials put-objects my-bucket my-folder + + This will create keys in my-folder/... in the S3 bucket. + + To upload all files in a folder to the root of the bucket instead use this: + + s3-credentials put-objects my-bucket my-folder/* + """ + s3 = make_client("s3", **boto_options) + if prefix and not prefix.endswith("/"): + prefix = prefix + "/" + total_size = 0 + # Figure out files to upload and their keys + paths = [] # (path, key) + for obj in objects: + path = pathlib.Path(obj) + if path.is_file(): + # Just use the filename as the key + paths.append((path, path.name)) + elif path.is_dir(): + # Key is the relative path within the directory + for p in path.glob("**/*"): + if p.is_file(): + paths.append((p, str(p.relative_to(path.parent)))) + + def upload(path, key, callback=None): + final_key = key + if prefix: + final_key = prefix + key + if dry_run: + click.echo("{} => s3://{}/{}".format(path, bucket, final_key)) + else: + s3.upload_file( + Filename=str(path), Bucket=bucket, Key=final_key, Callback=callback + ) + + if not silent and not dry_run: + total_size = sum(p[0].stat().st_size for p in paths) + with click.progressbar( + length=total_size, + label="Uploading {} ({} file{})".format( + format_bytes(total_size), + len(paths), + "s" if len(paths) != 1 else "", + ), + file=sys.stderr, + ) as bar: + for path, key in paths: + upload(path, key, bar.update) + else: + for path, key in paths: + upload(path, key) + + @cli.command() @click.argument("bucket") @click.argument("key") @@ -1131,6 +1216,7 @@ def get_objects(bucket, keys, output, patterns, silent, **boto_options): len(key_sizes), "s" if len(key_sizes) != 1 else "", ), + file=sys.stderr, ) as bar: for key in keys_to_download: download(key, bar.update) diff --git a/tests/conftest.py b/tests/conftest.py index 16641bd..fd8042e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,6 +54,11 @@ def moto_s3(aws_credentials): with mock_s3(): client = boto3.client("s3", region_name="us-east-1") client.create_bucket(Bucket="my-bucket") - for key in ("one.txt", "directory/two.txt", "directory/three.json"): - client.put_object(Bucket="my-bucket", Key=key, Body=key.encode("utf-8")) yield client + + +@pytest.fixture(scope="function") +def moto_s3_populated(moto_s3): + for key in ("one.txt", "directory/two.txt", "directory/three.json"): + moto_s3.put_object(Bucket="my-bucket", Key=key, Body=key.encode("utf-8")) + yield moto_s3 diff --git a/tests/test_s3_credentials.py b/tests/test_s3_credentials.py index a225be7..fb92356 100644 --- a/tests/test_s3_credentials.py +++ b/tests/test_s3_credentials.py @@ -1,5 +1,6 @@ import botocore from click.testing import CliRunner +import s3_credentials from s3_credentials.cli import cli import json import os @@ -1137,7 +1138,7 @@ def test_list_roles_csv(stub_iam_for_list_roles): ), ) @pytest.mark.parametrize("output", (None, "out")) -def test_get_objects(moto_s3, output, files, patterns, expected, error): +def test_get_objects(moto_s3_populated, output, files, patterns, expected, error): runner = CliRunner() with runner.isolated_filesystem(): args = ["get-objects", "my-bucket"] + (files or []) @@ -1161,3 +1162,51 @@ def test_get_objects(moto_s3, output, files, patterns, expected, error): assert all_files == expected if error: assert error in result.output + + +@pytest.mark.parametrize( + "args,expected,expected_output", + ( + (["."], {"one.txt", "directory/two.txt", "directory/three.json"}, None), + (["one.txt"], {"one.txt"}, None), + (["directory"], {"directory/two.txt", "directory/three.json"}, None), + ( + ["directory", "--prefix", "o"], + {"o/directory/two.txt", "o/directory/three.json"}, + None, + ), + # --dry-run tests + ( + ["directory", "--prefix", "o", "--dry-run"], + None, + ( + "directory/two.txt => s3://my-bucket/o/directory/two.txt\n" + "directory/three.json => s3://my-bucket/o/directory/three.json\n" + ), + ), + ( + [".", "--prefix", "p"], + {"p/one.txt", "p/directory/two.txt", "p/directory/three.json"}, + None, + ), + ), +) +def test_put_objects(moto_s3, args, expected, expected_output): + runner = CliRunner(mix_stderr=False) + with runner.isolated_filesystem(): + # Create files + pathlib.Path("one.txt").write_text("one") + pathlib.Path("directory").mkdir() + pathlib.Path("directory/two.txt").write_text("two") + pathlib.Path("directory/three.json").write_text('{"three": 3}') + result = runner.invoke( + cli, ["put-objects", "my-bucket"] + args, catch_exceptions=False + ) + assert result.exit_code == 0, result.output + assert result.output == (expected_output or "") + # Check files were uploaded + keys = { + obj["Key"] + for obj in moto_s3.list_objects(Bucket="my-bucket").get("Contents") or [] + } + assert keys == (expected or set())