Update AWS modules that use to implicitly retry on NotFound errors (#67369)

* Update AWS modules that expect to retry on exception codes that match the regex '^\w+.NotFound'

Modules should intentionally define any extra error codes

Use a waiter for ec2_vpc_igw after creating an internet gateway instead of retrying on InvalidInternetGatewayID.NotFound

lib/ansible/module_utils/aws/acm.py

@@ -71,18 +71,18 @@ class ACMServiceManager(object):
             kwargs['CertificateStatuses'] = statuses
         return paginator.paginate(**kwargs).build_full_result()['CertificateSummaryList']
 
-    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException'])
     def get_certificate_with_backoff(self, client, certificate_arn):
         response = client.get_certificate(CertificateArn=certificate_arn)
         # strip out response metadata
         return {'Certificate': response['Certificate'],
                 'CertificateChain': response['CertificateChain']}
 
-    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException'])
     def describe_certificate_with_backoff(self, client, certificate_arn):
         return client.describe_certificate(CertificateArn=certificate_arn)['Certificate']
 
-    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException'])
     def list_certificate_tags_with_backoff(self, client, certificate_arn):
         return client.list_tags_for_certificate(CertificateArn=certificate_arn)['Tags']
 

lib/ansible/module_utils/aws/waiters.py

@@ -13,6 +13,24 @@ except ImportError:
 ec2_data = {
     "version": 2,
     "waiters": {
+        "InternetGatewayExists": {
+            "delay": 5,
+            "maxAttempts": 40,
+            "operation": "DescribeInternetGateways",
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "expected": True,
+                    "argument": "length(InternetGateways) > `0`",
+                    "state": "success"
+                },
+                {
+                    "matcher": "error",
+                    "expected": "InvalidInternetGatewayID.NotFound",
+                    "state": "retry"
+                },
+            ]
+        },
         "RouteTableExists": {
             "delay": 5,
             "maxAttempts": 40,
@@ -280,6 +298,12 @@ def rds_model(name):
 
 
 waiters_by_name = {
+    ('EC2', 'internet_gateway_exists'): lambda ec2: core_waiter.Waiter(
+        'internet_gateway_exists',
+        ec2_model('InternetGatewayExists'),
+        core_waiter.NormalizedOperationMethod(
+            ec2.describe_internet_gateways
+        )),
     ('EC2', 'route_table_exists'): lambda ec2: core_waiter.Waiter(
         'route_table_exists',
         ec2_model('RouteTableExists'),

lib/ansible/modules/cloud/amazon/cloudformation_stack_set.py

@@ -370,8 +370,7 @@ def stack_set_facts(cfn, stack_set_name):
         ss['Tags'] = boto3_tag_list_to_ansible_dict(ss['Tags'])
         return ss
     except cfn.exceptions.from_code('StackSetNotFound'):
-        # catch NotFound error before the retry kicks in to avoid waiting
-        # if the stack does not exist
+        # Return None if the stack doesn't exist
         return
 
 
@@ -429,14 +428,15 @@ def await_stack_instance_completion(module, cfn, stack_set_name, max_wait):
 
 
 def await_stack_set_exists(cfn, stack_set_name):
-    # AWSRetry will retry on `NotFound` errors for us
+    # AWSRetry will retry on `StackSetNotFound` errors for us
     ss = cfn.describe_stack_set(StackSetName=stack_set_name, aws_retry=True)['StackSet']
     ss['Tags'] = boto3_tag_list_to_ansible_dict(ss['Tags'])
     return camel_dict_to_snake_dict(ss, ignore_list=('Tags',))
 
 
 def describe_stack_tree(module, stack_set_name, operation_ids=None):
-    cfn = module.client('cloudformation', retry_decorator=AWSRetry.jittered_backoff(retries=5, delay=3, max_delay=5))
+    jittered_backoff_decorator = AWSRetry.jittered_backoff(retries=5, delay=3, max_delay=5, catch_extra_error_codes=['StackSetNotFound'])
+    cfn = module.client('cloudformation', retry_decorator=jittered_backoff_decorator)
     result = dict()
     result['stack_set'] = camel_dict_to_snake_dict(
         cfn.describe_stack_set(
@@ -536,7 +536,8 @@ def main():
 
     # Wrap the cloudformation client methods that this module uses with
     # automatic backoff / retry for throttling error codes
-    cfn = module.client('cloudformation', retry_decorator=AWSRetry.jittered_backoff(retries=10, delay=3, max_delay=30))
+    jittered_backoff_decorator = AWSRetry.jittered_backoff(retries=10, delay=3, max_delay=30, catch_extra_error_codes=['StackSetNotFound'])
+    cfn = module.client('cloudformation', retry_decorator=jittered_backoff_decorator)
     existing_stack_set = stack_set_facts(cfn, module.params['name'])
 
     operation_uuid = to_native(uuid.uuid4())

lib/ansible/modules/cloud/amazon/ec2_group.py

@@ -545,7 +545,7 @@ def rule_from_group_permission(perm):
             )
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['InvalidGroup.NotFound'])
 def get_security_groups_with_backoff(connection, **kwargs):
     return connection.describe_security_groups(**kwargs)
 

lib/ansible/modules/cloud/amazon/ec2_launch_template.py

@@ -479,7 +479,7 @@ def delete_template(module):
 
 
 def create_or_update(module, template_options):
-    ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff())
+    ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff(catch_extra_error_codes=['InvalidLaunchTemplateId.NotFound']))
     template, template_versions = existing_templates(module)
     out = {}
     lt_data = params_to_launch_data(module, dict((k, v) for k, v in module.params.items() if k in template_options))

lib/ansible/modules/cloud/amazon/ec2_transit_gateway_info.py

@@ -213,12 +213,11 @@ class AnsibleEc2TgwInfo(object):
         try:
             response = self._connection.describe_transit_gateways(
                 TransitGatewayIds=transit_gateway_ids, Filters=filters)
-        except (BotoCoreError, ClientError) as e:
+        except ClientError as e:
             if e.response['Error']['Code'] == 'InvalidTransitGatewayID.NotFound':
                 self._results['transit_gateways'] = []
                 return
-            else:
-                self._module.fail_json_aws(e)
+            raise
 
         for transit_gateway in response['TransitGateways']:
             transit_gateway_info.append(camel_dict_to_snake_dict(transit_gateway, ignore_list=['Tags']))
@@ -257,7 +256,10 @@ def main():
     )
 
     tgwf_manager = AnsibleEc2TgwInfo(module=module, results=results)
+    try:
         tgwf_manager.describe_transit_gateways()
+    except (BotoCoreError, ClientError) as e:
+        module.fail_json_aws(e)
 
     module.exit_json(**results)
 

lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_option.py

@@ -219,7 +219,7 @@ def retry_not_found(to_call, *args, **kwargs):
         try:
             return to_call(*args, **kwargs)
         except EC2ResponseError as e:
-            if e.error_code == 'InvalidDhcpOptionID.NotFound':
+            if e.error_code in ['InvalidDhcpOptionID.NotFound', 'InvalidDhcpOptionsID.NotFound']:
                 sleep(3)
                 continue
             raise e

lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py

@@ -91,6 +91,7 @@ except ImportError:
     pass  # caught by AnsibleAWSModule
 
 from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.waiters import get_waiter
 from ansible.module_utils.ec2 import (
     AWSRetry,
     camel_dict_to_snake_dict,
@@ -237,6 +238,11 @@ class AnsibleEc2Igw(object):
 
             try:
                 response = self._connection.create_internet_gateway()
+
+                # Ensure the gateway exists before trying to attach it or add tags
+                waiter = get_waiter(self._connection, 'internet_gateway_exists')
+                waiter.wait(InternetGatewayIds=[response['InternetGateway']['InternetGatewayId']])
+
                 igw = camel_dict_to_snake_dict(response['InternetGateway'])
                 self._connection.attach_internet_gateway(InternetGatewayId=igw['internet_gateway_id'], VpcId=vpc_id)
                 self._results['changed'] = True

lib/ansible/modules/cloud/amazon/ec2_vpc_net.py

@@ -253,10 +253,7 @@ def get_vpc(module, connection, vpc_id):
         module.fail_json_aws(e, msg="Unable to wait for VPC {0} to be available.".format(vpc_id))
 
     try:
-        vpc_obj = AWSRetry.backoff(
-            delay=3, tries=8,
-            catch_extra_error_codes=['InvalidVpcID.NotFound'],
-        )(connection.describe_vpcs)(VpcIds=[vpc_id])['Vpcs'][0]
+        vpc_obj = connection.describe_vpcs(VpcIds=[vpc_id], aws_retry=True)['Vpcs'][0]
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Failed to describe VPCs")
     try:
@@ -279,10 +276,7 @@ def update_vpc_tags(connection, module, vpc_id, tags, name):
         if tags_to_update:
             if not module.check_mode:
                 tags = ansible_dict_to_boto3_tag_list(tags_to_update)
-                vpc_obj = AWSRetry.backoff(
-                    delay=1, tries=5,
-                    catch_extra_error_codes=['InvalidVpcID.NotFound'],
-                )(connection.create_tags)(Resources=[vpc_id], Tags=tags)
+                vpc_obj = connection.create_tags(Resources=[vpc_id], Tags=tags, aws_retry=True)
 
                 # Wait for tags to be updated
                 expected_tags = boto3_tag_list_to_ansible_dict(tags)

lib/ansible/modules/cloud/amazon/elb_target.py

@@ -126,7 +126,7 @@ except ImportError:
     HAS_BOTO3 = False
 
 
-@AWSRetry.jittered_backoff(retries=10, delay=10)
+@AWSRetry.jittered_backoff(retries=10, delay=10, catch_extra_error_codes=['TargetGroupNotFound'])
 def describe_target_groups_with_backoff(connection, tg_name):
     return connection.describe_target_groups(Names=[tg_name])
 
@@ -147,7 +147,7 @@ def convert_tg_name_to_arn(connection, module, tg_name):
     return tg_arn
 
 
-@AWSRetry.jittered_backoff(retries=10, delay=10)
+@AWSRetry.jittered_backoff(retries=10, delay=10, catch_extra_error_codes=['TargetGroupNotFound'])
 def describe_targets_with_backoff(connection, tg_arn, target):
     if target is None:
         tg = []

lib/ansible/modules/cloud/amazon/rds_snapshot.py

@@ -338,7 +338,7 @@ def main():
         required_if=[['state', 'present', ['db_instance_identifier']]]
     )
 
-    client = module.client('rds', retry_decorator=AWSRetry.jittered_backoff(retries=10))
+    client = module.client('rds', retry_decorator=AWSRetry.jittered_backoff(retries=10, catch_extra_error_codes=['DBSnapshotNotFound']))
 
     if module.params['state'] == 'absent':
         ret_dict = ensure_snapshot_absent(client, module)