From 195ded65279672da8f77b194d1bbaf1f9877ea74 Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Fri, 21 Feb 2020 08:17:24 -0500 Subject: [PATCH] Update AWS modules that use to implicitly retry on NotFound errors (#67369) * Update AWS modules that expect to retry on exception codes that match the regex '^\w+.NotFound' Modules should intentionally define any extra error codes Use a waiter for ec2_vpc_igw after creating an internet gateway instead of retrying on InvalidInternetGatewayID.NotFound --- lib/ansible/module_utils/aws/acm.py | 6 ++--- lib/ansible/module_utils/aws/waiters.py | 24 +++++++++++++++++++ .../cloud/amazon/cloudformation_stack_set.py | 11 +++++---- lib/ansible/modules/cloud/amazon/ec2_group.py | 2 +- .../cloud/amazon/ec2_launch_template.py | 2 +- .../cloud/amazon/ec2_transit_gateway_info.py | 10 ++++---- .../cloud/amazon/ec2_vpc_dhcp_option.py | 2 +- .../modules/cloud/amazon/ec2_vpc_igw.py | 6 +++++ .../modules/cloud/amazon/ec2_vpc_net.py | 10 ++------ .../modules/cloud/amazon/elb_target.py | 4 ++-- .../modules/cloud/amazon/rds_snapshot.py | 2 +- 11 files changed, 53 insertions(+), 26 deletions(-) diff --git a/lib/ansible/module_utils/aws/acm.py b/lib/ansible/module_utils/aws/acm.py index ed38027998b..58363bee617 100644 --- a/lib/ansible/module_utils/aws/acm.py +++ b/lib/ansible/module_utils/aws/acm.py @@ -71,18 +71,18 @@ class ACMServiceManager(object): kwargs['CertificateStatuses'] = statuses return paginator.paginate(**kwargs).build_full_result()['CertificateSummaryList'] - @AWSRetry.backoff(tries=5, delay=5, backoff=2.0) + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException']) def get_certificate_with_backoff(self, client, certificate_arn): response = client.get_certificate(CertificateArn=certificate_arn) # strip out response metadata return {'Certificate': response['Certificate'], 'CertificateChain': response['CertificateChain']} - @AWSRetry.backoff(tries=5, delay=5, backoff=2.0) + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException']) def describe_certificate_with_backoff(self, client, certificate_arn): return client.describe_certificate(CertificateArn=certificate_arn)['Certificate'] - @AWSRetry.backoff(tries=5, delay=5, backoff=2.0) + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException']) def list_certificate_tags_with_backoff(self, client, certificate_arn): return client.list_tags_for_certificate(CertificateArn=certificate_arn)['Tags'] diff --git a/lib/ansible/module_utils/aws/waiters.py b/lib/ansible/module_utils/aws/waiters.py index 79f4bc2ece2..25db598bcb3 100644 --- a/lib/ansible/module_utils/aws/waiters.py +++ b/lib/ansible/module_utils/aws/waiters.py @@ -13,6 +13,24 @@ except ImportError: ec2_data = { "version": 2, "waiters": { + "InternetGatewayExists": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeInternetGateways", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "length(InternetGateways) > `0`", + "state": "success" + }, + { + "matcher": "error", + "expected": "InvalidInternetGatewayID.NotFound", + "state": "retry" + }, + ] + }, "RouteTableExists": { "delay": 5, "maxAttempts": 40, @@ -280,6 +298,12 @@ def rds_model(name): waiters_by_name = { + ('EC2', 'internet_gateway_exists'): lambda ec2: core_waiter.Waiter( + 'internet_gateway_exists', + ec2_model('InternetGatewayExists'), + core_waiter.NormalizedOperationMethod( + ec2.describe_internet_gateways + )), ('EC2', 'route_table_exists'): lambda ec2: core_waiter.Waiter( 'route_table_exists', ec2_model('RouteTableExists'), diff --git a/lib/ansible/modules/cloud/amazon/cloudformation_stack_set.py b/lib/ansible/modules/cloud/amazon/cloudformation_stack_set.py index 61fe5f55844..ba6c2576f4b 100644 --- a/lib/ansible/modules/cloud/amazon/cloudformation_stack_set.py +++ b/lib/ansible/modules/cloud/amazon/cloudformation_stack_set.py @@ -370,8 +370,7 @@ def stack_set_facts(cfn, stack_set_name): ss['Tags'] = boto3_tag_list_to_ansible_dict(ss['Tags']) return ss except cfn.exceptions.from_code('StackSetNotFound'): - # catch NotFound error before the retry kicks in to avoid waiting - # if the stack does not exist + # Return None if the stack doesn't exist return @@ -429,14 +428,15 @@ def await_stack_instance_completion(module, cfn, stack_set_name, max_wait): def await_stack_set_exists(cfn, stack_set_name): - # AWSRetry will retry on `NotFound` errors for us + # AWSRetry will retry on `StackSetNotFound` errors for us ss = cfn.describe_stack_set(StackSetName=stack_set_name, aws_retry=True)['StackSet'] ss['Tags'] = boto3_tag_list_to_ansible_dict(ss['Tags']) return camel_dict_to_snake_dict(ss, ignore_list=('Tags',)) def describe_stack_tree(module, stack_set_name, operation_ids=None): - cfn = module.client('cloudformation', retry_decorator=AWSRetry.jittered_backoff(retries=5, delay=3, max_delay=5)) + jittered_backoff_decorator = AWSRetry.jittered_backoff(retries=5, delay=3, max_delay=5, catch_extra_error_codes=['StackSetNotFound']) + cfn = module.client('cloudformation', retry_decorator=jittered_backoff_decorator) result = dict() result['stack_set'] = camel_dict_to_snake_dict( cfn.describe_stack_set( @@ -536,7 +536,8 @@ def main(): # Wrap the cloudformation client methods that this module uses with # automatic backoff / retry for throttling error codes - cfn = module.client('cloudformation', retry_decorator=AWSRetry.jittered_backoff(retries=10, delay=3, max_delay=30)) + jittered_backoff_decorator = AWSRetry.jittered_backoff(retries=10, delay=3, max_delay=30, catch_extra_error_codes=['StackSetNotFound']) + cfn = module.client('cloudformation', retry_decorator=jittered_backoff_decorator) existing_stack_set = stack_set_facts(cfn, module.params['name']) operation_uuid = to_native(uuid.uuid4()) diff --git a/lib/ansible/modules/cloud/amazon/ec2_group.py b/lib/ansible/modules/cloud/amazon/ec2_group.py index d420790ad7e..bc416f66b57 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_group.py +++ b/lib/ansible/modules/cloud/amazon/ec2_group.py @@ -545,7 +545,7 @@ def rule_from_group_permission(perm): ) -@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['InvalidGroup.NotFound']) def get_security_groups_with_backoff(connection, **kwargs): return connection.describe_security_groups(**kwargs) diff --git a/lib/ansible/modules/cloud/amazon/ec2_launch_template.py b/lib/ansible/modules/cloud/amazon/ec2_launch_template.py index f64dfb53fc6..d81f6fec250 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_launch_template.py +++ b/lib/ansible/modules/cloud/amazon/ec2_launch_template.py @@ -479,7 +479,7 @@ def delete_template(module): def create_or_update(module, template_options): - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff(catch_extra_error_codes=['InvalidLaunchTemplateId.NotFound'])) template, template_versions = existing_templates(module) out = {} lt_data = params_to_launch_data(module, dict((k, v) for k, v in module.params.items() if k in template_options)) diff --git a/lib/ansible/modules/cloud/amazon/ec2_transit_gateway_info.py b/lib/ansible/modules/cloud/amazon/ec2_transit_gateway_info.py index 1022598b162..94f86ec11e7 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_transit_gateway_info.py +++ b/lib/ansible/modules/cloud/amazon/ec2_transit_gateway_info.py @@ -213,12 +213,11 @@ class AnsibleEc2TgwInfo(object): try: response = self._connection.describe_transit_gateways( TransitGatewayIds=transit_gateway_ids, Filters=filters) - except (BotoCoreError, ClientError) as e: + except ClientError as e: if e.response['Error']['Code'] == 'InvalidTransitGatewayID.NotFound': self._results['transit_gateways'] = [] return - else: - self._module.fail_json_aws(e) + raise for transit_gateway in response['TransitGateways']: transit_gateway_info.append(camel_dict_to_snake_dict(transit_gateway, ignore_list=['Tags'])) @@ -257,7 +256,10 @@ def main(): ) tgwf_manager = AnsibleEc2TgwInfo(module=module, results=results) - tgwf_manager.describe_transit_gateways() + try: + tgwf_manager.describe_transit_gateways() + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) module.exit_json(**results) diff --git a/lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_option.py b/lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_option.py index 2e244344899..d111bed0f08 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_option.py +++ b/lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_option.py @@ -219,7 +219,7 @@ def retry_not_found(to_call, *args, **kwargs): try: return to_call(*args, **kwargs) except EC2ResponseError as e: - if e.error_code == 'InvalidDhcpOptionID.NotFound': + if e.error_code in ['InvalidDhcpOptionID.NotFound', 'InvalidDhcpOptionsID.NotFound']: sleep(3) continue raise e diff --git a/lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py b/lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py index 7d34fb965c9..5198527af7a 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py +++ b/lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py @@ -91,6 +91,7 @@ except ImportError: pass # caught by AnsibleAWSModule from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.waiters import get_waiter from ansible.module_utils.ec2 import ( AWSRetry, camel_dict_to_snake_dict, @@ -237,6 +238,11 @@ class AnsibleEc2Igw(object): try: response = self._connection.create_internet_gateway() + + # Ensure the gateway exists before trying to attach it or add tags + waiter = get_waiter(self._connection, 'internet_gateway_exists') + waiter.wait(InternetGatewayIds=[response['InternetGateway']['InternetGatewayId']]) + igw = camel_dict_to_snake_dict(response['InternetGateway']) self._connection.attach_internet_gateway(InternetGatewayId=igw['internet_gateway_id'], VpcId=vpc_id) self._results['changed'] = True diff --git a/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py b/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py index 5cd49ccb126..30e4b1e94c5 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py +++ b/lib/ansible/modules/cloud/amazon/ec2_vpc_net.py @@ -253,10 +253,7 @@ def get_vpc(module, connection, vpc_id): module.fail_json_aws(e, msg="Unable to wait for VPC {0} to be available.".format(vpc_id)) try: - vpc_obj = AWSRetry.backoff( - delay=3, tries=8, - catch_extra_error_codes=['InvalidVpcID.NotFound'], - )(connection.describe_vpcs)(VpcIds=[vpc_id])['Vpcs'][0] + vpc_obj = connection.describe_vpcs(VpcIds=[vpc_id], aws_retry=True)['Vpcs'][0] except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Failed to describe VPCs") try: @@ -279,10 +276,7 @@ def update_vpc_tags(connection, module, vpc_id, tags, name): if tags_to_update: if not module.check_mode: tags = ansible_dict_to_boto3_tag_list(tags_to_update) - vpc_obj = AWSRetry.backoff( - delay=1, tries=5, - catch_extra_error_codes=['InvalidVpcID.NotFound'], - )(connection.create_tags)(Resources=[vpc_id], Tags=tags) + vpc_obj = connection.create_tags(Resources=[vpc_id], Tags=tags, aws_retry=True) # Wait for tags to be updated expected_tags = boto3_tag_list_to_ansible_dict(tags) diff --git a/lib/ansible/modules/cloud/amazon/elb_target.py b/lib/ansible/modules/cloud/amazon/elb_target.py index 61510909fca..acb7c590dd1 100644 --- a/lib/ansible/modules/cloud/amazon/elb_target.py +++ b/lib/ansible/modules/cloud/amazon/elb_target.py @@ -126,7 +126,7 @@ except ImportError: HAS_BOTO3 = False -@AWSRetry.jittered_backoff(retries=10, delay=10) +@AWSRetry.jittered_backoff(retries=10, delay=10, catch_extra_error_codes=['TargetGroupNotFound']) def describe_target_groups_with_backoff(connection, tg_name): return connection.describe_target_groups(Names=[tg_name]) @@ -147,7 +147,7 @@ def convert_tg_name_to_arn(connection, module, tg_name): return tg_arn -@AWSRetry.jittered_backoff(retries=10, delay=10) +@AWSRetry.jittered_backoff(retries=10, delay=10, catch_extra_error_codes=['TargetGroupNotFound']) def describe_targets_with_backoff(connection, tg_arn, target): if target is None: tg = [] diff --git a/lib/ansible/modules/cloud/amazon/rds_snapshot.py b/lib/ansible/modules/cloud/amazon/rds_snapshot.py index 7fccea22a88..5df2808feeb 100644 --- a/lib/ansible/modules/cloud/amazon/rds_snapshot.py +++ b/lib/ansible/modules/cloud/amazon/rds_snapshot.py @@ -338,7 +338,7 @@ def main(): required_if=[['state', 'present', ['db_instance_identifier']]] ) - client = module.client('rds', retry_decorator=AWSRetry.jittered_backoff(retries=10)) + client = module.client('rds', retry_decorator=AWSRetry.jittered_backoff(retries=10, catch_extra_error_codes=['DBSnapshotNotFound'])) if module.params['state'] == 'absent': ret_dict = ensure_snapshot_absent(client, module)