From 3cb1c38eccd86ed3f1f1a8e7a07aa81a48749fa5 Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Fri, 11 Aug 2017 15:30:50 -0400 Subject: [PATCH] [cloud] Allow S3 module to correctly delete objects by groups of <= 1000 (#28068) --- lib/ansible/modules/cloud/amazon/s3.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/ansible/modules/cloud/amazon/s3.py b/lib/ansible/modules/cloud/amazon/s3.py index 6979f613a54..dd9c84ecbfa 100644 --- a/lib/ansible/modules/cloud/amazon/s3.py +++ b/lib/ansible/modules/cloud/amazon/s3.py @@ -327,8 +327,7 @@ def create_bucket(module, s3, bucket, location=None): def paginated_list(s3, **pagination_params): pg = s3.get_paginator('list_objects_v2') for page in pg.paginate(**pagination_params): - for data in page.get('Contents', {}): - yield data['Key'] + yield [data['Key'] for data in page.get('Contents', [])] def list_keys(module, s3, bucket, prefix, marker, max_keys): @@ -336,7 +335,7 @@ def list_keys(module, s3, bucket, prefix, marker, max_keys): for param_name, param_value in (('Prefix', prefix), ('StartAfter', marker), ('MaxKeys', max_keys)): pagination_params[param_name] = param_value try: - keys = [key for key in paginated_list(s3, **pagination_params)] + keys = sum(paginated_list(s3, **pagination_params), []) module.exit_json(msg="LIST operation complete", s3_keys=keys) except botocore.exceptions.ClientError as e: module.fail_json(msg="Failed while listing the keys in the bucket {0}".format(bucket), @@ -352,9 +351,9 @@ def delete_bucket(module, s3, bucket): if exists is False: return False # if there are contents then we need to delete them before we can delete the bucket - keys = [{'Key': key} for key in paginated_list(s3, Bucket=bucket)] - if keys: - s3.delete_objects(Bucket=bucket, Delete={'Objects': keys}) + for keys in paginated_list(s3, Bucket=bucket): + formatted_keys = [{'Key': key} for key in keys] + s3.delete_objects(Bucket=bucket, Delete={'Objects': formatted_keys}) s3.delete_bucket(Bucket=bucket) return True except botocore.exceptions.ClientError as e: