kopia lustrzana https://github.com/simonw/s3-credentials
list-users/list-buckets --nl, --csv, --tsv - refs #48
Simon Willison
rodzic
cc10c7f2b5
commit
917d575c73
|
|
@ -202,11 +202,11 @@ To see a list of all users that exist for your AWS account:
|
|||
|
||||
s3-credentials list-users
|
||||
|
||||
This will return pretty-printed JSON objects by default.
|
||||
This will a pretty-printed array of JSON objects by default.
|
||||
|
||||
Add `--nl` to collapse these to single lines as valid newline-delimited JSON.
|
||||
|
||||
Add `--array` to output a valid JSON array of objects instead.
|
||||
Add `--csv` or `--tsv` to get back CSV or TSV data.
|
||||
|
||||
### list-buckets
|
||||
|
||||
|
|
@ -230,7 +230,7 @@ With no extra arguments this will show all available buckets - you can also add
|
|||
"CreationDate": "2021-11-03 21:46:12+00:00"
|
||||
}
|
||||
|
||||
This accepts the same `--nl` and `--array` options as `list-users`.
|
||||
This accepts the same `--nl`, `--csv` and `--tsv` options as `list-users`.
|
||||
|
||||
Add `--details` to include details of the bucket ACL, website configuration and public access block settings. This is useful for running a security audit of your buckets.
|
||||
|
||||
|
|
|
|||
|
|
@ -480,25 +480,35 @@ def whoami(**boto_options):
|
|||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--array", help="Output a valid JSON array", is_flag=True)
|
||||
@click.option("--nl", help="Output newline-delimited JSON", is_flag=True)
|
||||
@common_output_options
|
||||
@common_boto3_options
|
||||
def list_users(array, nl, **boto_options):
|
||||
def list_users(nl, csv, tsv, **boto_options):
|
||||
"List all users"
|
||||
iam = make_client("iam", **boto_options)
|
||||
paginator = iam.get_paginator("list_users")
|
||||
gathered = []
|
||||
for response in paginator.paginate():
|
||||
for user in response["Users"]:
|
||||
if array:
|
||||
gathered.append(user)
|
||||
else:
|
||||
if nl:
|
||||
click.echo(json.dumps(user, default=str))
|
||||
else:
|
||||
click.echo(json.dumps(user, indent=4, default=str))
|
||||
if gathered:
|
||||
click.echo(json.dumps(gathered, indent=4, default=str))
|
||||
|
||||
def iterate():
|
||||
for response in paginator.paginate():
|
||||
for user in response["Users"]:
|
||||
yield user
|
||||
|
||||
output(
|
||||
iterate(),
|
||||
(
|
||||
"UserName",
|
||||
"UserId",
|
||||
"Arn",
|
||||
"Path",
|
||||
"CreateDate",
|
||||
"PasswordLastUsed",
|
||||
"PermissionsBoundary",
|
||||
"Tags",
|
||||
),
|
||||
nl,
|
||||
csv,
|
||||
tsv,
|
||||
)
|
||||
|
||||
|
||||
@cli.command()
|
||||
|
|
@ -531,52 +541,50 @@ def list_user_policies(usernames, **boto_options):
|
|||
@cli.command()
|
||||
@click.argument("buckets", nargs=-1)
|
||||
@click.option("--details", help="Include extra bucket details (slower)", is_flag=True)
|
||||
@click.option("--array", help="Output a valid JSON array", is_flag=True)
|
||||
@click.option("--nl", help="Output newline-delimited JSON", is_flag=True)
|
||||
@common_output_options
|
||||
@common_boto3_options
|
||||
def list_buckets(buckets, details, array, nl, **boto_options):
|
||||
def list_buckets(buckets, details, nl, csv, tsv, **boto_options):
|
||||
"List buckets - defaults to all, or pass one or more bucket names"
|
||||
s3 = make_client("s3", **boto_options)
|
||||
gathered = []
|
||||
for bucket in s3.list_buckets()["Buckets"]:
|
||||
if buckets and (bucket["Name"] not in buckets):
|
||||
continue
|
||||
if details:
|
||||
bucket_acl = dict(
|
||||
(key, value)
|
||||
for key, value in s3.get_bucket_acl(
|
||||
Bucket=bucket["Name"],
|
||||
).items()
|
||||
if key != "ResponseMetadata"
|
||||
)
|
||||
try:
|
||||
pab = s3.get_public_access_block(
|
||||
Bucket=bucket["Name"],
|
||||
)["PublicAccessBlockConfiguration"]
|
||||
except s3.exceptions.ClientError:
|
||||
pab = None
|
||||
try:
|
||||
bucket_website = dict(
|
||||
|
||||
headers = ["Name", "CreationDate"]
|
||||
if details:
|
||||
headers += ["bucket_acl", "public_access_block", "bucket_website"]
|
||||
|
||||
def iterator():
|
||||
for bucket in s3.list_buckets()["Buckets"]:
|
||||
if buckets and (bucket["Name"] not in buckets):
|
||||
continue
|
||||
if details:
|
||||
bucket_acl = dict(
|
||||
(key, value)
|
||||
for key, value in s3.get_bucket_website(
|
||||
for key, value in s3.get_bucket_acl(
|
||||
Bucket=bucket["Name"],
|
||||
).items()
|
||||
if key != "ResponseMetadata"
|
||||
)
|
||||
except s3.exceptions.ClientError:
|
||||
bucket_website = None
|
||||
bucket["bucket_acl"] = bucket_acl
|
||||
bucket["public_access_block"] = pab
|
||||
bucket["bucket_website"] = bucket_website
|
||||
if array:
|
||||
gathered.append(bucket)
|
||||
else:
|
||||
if nl:
|
||||
click.echo(json.dumps(bucket, default=str))
|
||||
else:
|
||||
click.echo(json.dumps(bucket, indent=4, default=str))
|
||||
if gathered:
|
||||
click.echo(json.dumps(gathered, indent=4, default=str))
|
||||
try:
|
||||
pab = s3.get_public_access_block(
|
||||
Bucket=bucket["Name"],
|
||||
)["PublicAccessBlockConfiguration"]
|
||||
except s3.exceptions.ClientError:
|
||||
pab = None
|
||||
try:
|
||||
bucket_website = dict(
|
||||
(key, value)
|
||||
for key, value in s3.get_bucket_website(
|
||||
Bucket=bucket["Name"],
|
||||
).items()
|
||||
if key != "ResponseMetadata"
|
||||
)
|
||||
except s3.exceptions.ClientError:
|
||||
bucket_website = None
|
||||
bucket["bucket_acl"] = bucket_acl
|
||||
bucket["public_access_block"] = pab
|
||||
bucket["bucket_website"] = bucket_website
|
||||
yield bucket
|
||||
|
||||
output(iterator(), headers, nl, csv, tsv)
|
||||
|
||||
|
||||
@cli.command()
|
||||
|
|
@ -712,7 +720,13 @@ def list_bucket(bucket, prefix, nl, csv, tsv, **boto_options):
|
|||
except botocore.exceptions.ClientError as e:
|
||||
raise click.ClickException(e)
|
||||
|
||||
output(iterate(), nl, csv, tsv)
|
||||
output(
|
||||
iterate(),
|
||||
("Key", "LastModified", "ETag", "Size", "StorageClass", "Owner"),
|
||||
nl,
|
||||
csv,
|
||||
tsv,
|
||||
)
|
||||
|
||||
|
||||
@cli.command()
|
||||
|
|
@ -779,20 +793,16 @@ def get_object(bucket, key, output, **boto_options):
|
|||
s3.download_fileobj(bucket, key, fp)
|
||||
|
||||
|
||||
def output(iterator, nl, csv, tsv):
|
||||
def output(iterator, headers, nl, csv, tsv):
|
||||
if nl:
|
||||
for item in iterator:
|
||||
click.echo(json.dumps(item, default=repr))
|
||||
click.echo(json.dumps(item, default=str))
|
||||
elif csv or tsv:
|
||||
first = next(iterator, None)
|
||||
if first is None:
|
||||
return
|
||||
headers = first.keys()
|
||||
writer = DictWriter(
|
||||
sys.stdout, headers, dialect="excel-tab" if tsv else "excel"
|
||||
)
|
||||
writer.writeheader()
|
||||
writer.writerows(itertools.chain([first], iterator))
|
||||
writer.writerows(iterator)
|
||||
else:
|
||||
for line in stream_indented_json(iterator):
|
||||
click.echo(line)
|
||||
|
|
@ -811,7 +821,7 @@ def stream_indented_json(iterator, indent=2):
|
|||
line = "{first}{serialized}{separator}{last}".format(
|
||||
first="[\n" if first else "",
|
||||
serialized=textwrap.indent(
|
||||
json.dumps(data, indent=indent, default=repr), " " * indent
|
||||
json.dumps(data, indent=indent, default=str), " " * indent
|
||||
),
|
||||
separator="," if not is_last else "",
|
||||
last="\n]" if is_last else "",
|
||||
|
|
|
|||
|
|
@ -232,7 +232,7 @@ def read_file(s3, bucket, path):
|
|||
|
||||
def cleanup_any_resources():
|
||||
# Delete any users beginning s3-credentials-tests.
|
||||
users = json.loads(get_output("list-users", "--array"))
|
||||
users = json.loads(get_output("list-users"))
|
||||
users_to_delete = [
|
||||
user["UserName"]
|
||||
for user in users
|
||||
|
|
@ -243,7 +243,7 @@ def cleanup_any_resources():
|
|||
get_output("delete-user", *users_to_delete)
|
||||
s3 = boto3.client("s3")
|
||||
# Delete any buckets beginning s3-credentials-tests.
|
||||
buckets = json.loads(get_output("list-buckets", "--array"))
|
||||
buckets = json.loads(get_output("list-buckets"))
|
||||
buckets_to_delete = [
|
||||
bucket["Name"]
|
||||
for bucket in buckets
|
||||
|
|
|
|||
|
|
@ -61,46 +61,44 @@ def test_whoami(mocker, stub_sts):
|
|||
(
|
||||
(
|
||||
"",
|
||||
"{\n"
|
||||
"[\n"
|
||||
" {\n"
|
||||
' "Path": "/",\n'
|
||||
' "UserName": "NameA",\n'
|
||||
' "UserId": "AID000000000000000001",\n'
|
||||
' "Arn": "arn:aws:iam::000000000000:user/NameB",\n'
|
||||
' "CreateDate": "2020-01-01 00:00:00+00:00"\n'
|
||||
"}\n"
|
||||
"{\n"
|
||||
" },\n"
|
||||
" {\n"
|
||||
' "Path": "/",\n'
|
||||
' "UserName": "NameA",\n'
|
||||
' "UserId": "AID000000000000000000",\n'
|
||||
' "Arn": "arn:aws:iam::000000000000:user/NameB",\n'
|
||||
' "CreateDate": "2020-01-01 00:00:00+00:00"\n'
|
||||
"}\n",
|
||||
),
|
||||
(
|
||||
"--array",
|
||||
"[\n"
|
||||
" {\n"
|
||||
' "Path": "/",\n'
|
||||
' "UserName": "NameA",\n'
|
||||
' "UserId": "AID000000000000000001",\n'
|
||||
' "Arn": "arn:aws:iam::000000000000:user/NameB",\n'
|
||||
' "CreateDate": "2020-01-01 00:00:00+00:00"\n'
|
||||
" },\n"
|
||||
" {\n"
|
||||
' "Path": "/",\n'
|
||||
' "UserName": "NameA",\n'
|
||||
' "UserId": "AID000000000000000000",\n'
|
||||
' "Arn": "arn:aws:iam::000000000000:user/NameB",\n'
|
||||
' "CreateDate": "2020-01-01 00:00:00+00:00"\n'
|
||||
" }\n"
|
||||
"]\n"
|
||||
"",
|
||||
" }\n"
|
||||
"]\n",
|
||||
),
|
||||
(
|
||||
"--nl",
|
||||
'{"Path": "/", "UserName": "NameA", "UserId": "AID000000000000000001", "Arn": "arn:aws:iam::000000000000:user/NameB", "CreateDate": "2020-01-01 00:00:00+00:00"}\n'
|
||||
'{"Path": "/", "UserName": "NameA", "UserId": "AID000000000000000000", "Arn": "arn:aws:iam::000000000000:user/NameB", "CreateDate": "2020-01-01 00:00:00+00:00"}\n',
|
||||
),
|
||||
(
|
||||
"--csv",
|
||||
(
|
||||
"UserName,UserId,Arn,Path,CreateDate,PasswordLastUsed,PermissionsBoundary,Tags\n"
|
||||
"NameA,AID000000000000000001,arn:aws:iam::000000000000:user/NameB,/,2020-01-01 00:00:00+00:00,,,\n"
|
||||
"NameA,AID000000000000000000,arn:aws:iam::000000000000:user/NameB,/,2020-01-01 00:00:00+00:00,,,\n"
|
||||
),
|
||||
),
|
||||
(
|
||||
"--tsv",
|
||||
(
|
||||
"UserName\tUserId\tArn\tPath\tCreateDate\tPasswordLastUsed\tPermissionsBoundary\tTags\n"
|
||||
"NameA\tAID000000000000000001\tarn:aws:iam::000000000000:user/NameB\t/\t2020-01-01 00:00:00+00:00\t\t\t\n"
|
||||
"NameA\tAID000000000000000000\tarn:aws:iam::000000000000:user/NameB\t/\t2020-01-01 00:00:00+00:00\t\t\t\n"
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_list_users(option, expected, stub_iam):
|
||||
|
|
@ -138,27 +136,18 @@ def test_list_users(option, expected, stub_iam):
|
|||
(
|
||||
(
|
||||
[],
|
||||
"{\n"
|
||||
' "Name": "bucket-one",\n'
|
||||
' "CreationDate": "2020-01-01 00:00:00+00:00"\n'
|
||||
"}\n"
|
||||
"{\n"
|
||||
' "Name": "bucket-two",\n'
|
||||
' "CreationDate": "2020-02-01 00:00:00+00:00"\n'
|
||||
"}\n",
|
||||
),
|
||||
(
|
||||
["--array"],
|
||||
"[\n"
|
||||
" {\n"
|
||||
' "Name": "bucket-one",\n'
|
||||
' "CreationDate": "2020-01-01 00:00:00+00:00"\n'
|
||||
" },\n"
|
||||
" {\n"
|
||||
' "Name": "bucket-two",\n'
|
||||
' "CreationDate": "2020-02-01 00:00:00+00:00"\n'
|
||||
" }"
|
||||
"\n]\n",
|
||||
(
|
||||
"[\n"
|
||||
" {\n"
|
||||
' "Name": "bucket-one",\n'
|
||||
' "CreationDate": "2020-01-01 00:00:00+00:00"\n'
|
||||
" },\n"
|
||||
" {\n"
|
||||
' "Name": "bucket-two",\n'
|
||||
' "CreationDate": "2020-02-01 00:00:00+00:00"\n'
|
||||
" }\n"
|
||||
"]\n"
|
||||
),
|
||||
),
|
||||
(
|
||||
["--nl"],
|
||||
|
|
@ -249,40 +238,42 @@ def test_list_buckets_details(stub_s3):
|
|||
result = runner.invoke(cli, ["list-buckets", "--details"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == (
|
||||
"{\n"
|
||||
"[\n"
|
||||
" {\n"
|
||||
' "Name": "bucket-one",\n'
|
||||
' "CreationDate": "2020-01-01 00:00:00+00:00",\n'
|
||||
' "bucket_acl": {\n'
|
||||
' "Owner": {\n'
|
||||
' "Owner": {\n'
|
||||
' "DisplayName": "swillison",\n'
|
||||
' "ID": "36b2eeee501c5952a8ac119f9e5212277a4c01eccfa8d6a9d670bba1e2d5f441"\n'
|
||||
" },\n"
|
||||
' "Grants": [\n'
|
||||
" {\n"
|
||||
' "Grantee": {\n'
|
||||
' "DisplayName": "swillison",\n'
|
||||
' "ID": "36b2eeee501c5952a8ac119f9e5212277a4c01eccfa8d6a9d670bba1e2d5f441"\n'
|
||||
" },\n"
|
||||
' "Grants": [\n'
|
||||
" {\n"
|
||||
' "Grantee": {\n'
|
||||
' "DisplayName": "swillison",\n'
|
||||
' "ID": "36b2eeee501c5952a8ac119f9e5212277a4c01eccfa8d6a9d670bba1e2d5f441",\n'
|
||||
' "Type": "CanonicalUser"\n'
|
||||
" },\n"
|
||||
' "Permission": "FULL_CONTROL"\n'
|
||||
" }\n"
|
||||
" ]\n"
|
||||
' "ID": "36b2eeee501c5952a8ac119f9e5212277a4c01eccfa8d6a9d670bba1e2d5f441",\n'
|
||||
' "Type": "CanonicalUser"\n'
|
||||
" },\n"
|
||||
' "Permission": "FULL_CONTROL"\n'
|
||||
" }\n"
|
||||
" ]\n"
|
||||
" },\n"
|
||||
' "public_access_block": {\n'
|
||||
' "BlockPublicAcls": true,\n'
|
||||
' "IgnorePublicAcls": true,\n'
|
||||
' "BlockPublicPolicy": true,\n'
|
||||
' "RestrictPublicBuckets": true\n'
|
||||
' "BlockPublicAcls": true,\n'
|
||||
' "IgnorePublicAcls": true,\n'
|
||||
' "BlockPublicPolicy": true,\n'
|
||||
' "RestrictPublicBuckets": true\n'
|
||||
" },\n"
|
||||
' "bucket_website": {\n'
|
||||
' "IndexDocument": {\n'
|
||||
' "Suffix": "index.html"\n'
|
||||
" },\n"
|
||||
' "ErrorDocument": {\n'
|
||||
' "Key": "error.html"\n'
|
||||
" }\n"
|
||||
' "IndexDocument": {\n'
|
||||
' "Suffix": "index.html"\n'
|
||||
" },\n"
|
||||
' "ErrorDocument": {\n'
|
||||
' "Key": "error.html"\n'
|
||||
" }\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
" }\n"
|
||||
"]\n"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -771,17 +762,17 @@ def test_policy(options, expected):
|
|||
(
|
||||
["--tsv"],
|
||||
(
|
||||
"Key\tLastModified\tETag\tSize\tStorageClass\n"
|
||||
'yolo-causeway-1.jpg\t2019-12-26 17:00:22+00:00\t"""87abea888b22089cabe93a0e17cf34a4"""\t5923104\tSTANDARD\n'
|
||||
'yolo-causeway-2.jpg\t2019-12-26 17:00:22+00:00\t"""87abea888b22089cabe93a0e17cf34a4"""\t5923104\tSTANDARD\n'
|
||||
"Key\tLastModified\tETag\tSize\tStorageClass\tOwner\n"
|
||||
'yolo-causeway-1.jpg\t2019-12-26 17:00:22+00:00\t"""87abea888b22089cabe93a0e17cf34a4"""\t5923104\tSTANDARD\t\n'
|
||||
'yolo-causeway-2.jpg\t2019-12-26 17:00:22+00:00\t"""87abea888b22089cabe93a0e17cf34a4"""\t5923104\tSTANDARD\t\n'
|
||||
),
|
||||
),
|
||||
(
|
||||
["--csv"],
|
||||
(
|
||||
"Key,LastModified,ETag,Size,StorageClass\n"
|
||||
'yolo-causeway-1.jpg,2019-12-26 17:00:22+00:00,"""87abea888b22089cabe93a0e17cf34a4""",5923104,STANDARD\n'
|
||||
'yolo-causeway-2.jpg,2019-12-26 17:00:22+00:00,"""87abea888b22089cabe93a0e17cf34a4""",5923104,STANDARD\n'
|
||||
"Key,LastModified,ETag,Size,StorageClass,Owner\n"
|
||||
'yolo-causeway-1.jpg,2019-12-26 17:00:22+00:00,"""87abea888b22089cabe93a0e17cf34a4""",5923104,STANDARD,\n'
|
||||
'yolo-causeway-2.jpg,2019-12-26 17:00:22+00:00,"""87abea888b22089cabe93a0e17cf34a4""",5923104,STANDARD,\n'
|
||||
),
|
||||
),
|
||||
),
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue