[cloud] Add AWSRetry to ec2_vpc_net and ec2_vpc_dhcp_option to stabilize return values (#36264) (#38975)

Fixes #36063, fixes #37323, fixes #36078 (#37354)

* Add AWSRetry when describing VPCs to help stabilize integration tests

* Add retry on create_tags because it is possible to reach this API call before the VPC is finished creating

* Increase delay and tries for ec2_vpc_net backoff

* Wait for DHCP option to be created in ec2_vpc_dhcp_option

* Wait for all modifications to the VPC

* Use the vpc_available waiter because is uses Filters

* Optimize retries to only occur if the functionality is available

Cherry-from:
- 16f8a993a0
- e9c57e732f

lib/ansible/modules/cloud/amazon/ec2_vpc_dhcp_option.py

@@ -204,7 +204,7 @@ EXAMPLES = """
 
 import collections
 import traceback
-
+from time import sleep, time
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.ec2 import HAS_BOTO, connect_to_aws, ec2_argument_spec, get_aws_connection_info
 
@@ -376,6 +376,23 @@ def main():
                 new_options['ntp-servers'],
                 new_options['netbios-name-servers'],
                 new_options['netbios-node-type'])
+
+            # wait for dhcp option to be accessible
+            found_dhcp_opt = False
+            start_time = time()
+            while time() < start_time + 300:
+                try:
+                    found_dhcp_opt = connection.get_all_dhcp_options(dhcp_options_ids=[dhcp_option.id])
+                except EC2ResponseError as e:
+                    if e.error_code == 'InvalidDhcpOptionID.NotFound':
+                        sleep(3)
+                    else:
+                        module.fail_json(msg="Failed to describe DHCP options", exception=traceback.format_exc)
+                else:
+                    break
+            if not found_dhcp_opt:
+                module.fail_json(msg="Failed to wait for {0} to be available.".format(dhcp_option.id))
+
             changed = True
             if params['tags']:
                 ensure_tags(module, connection, dhcp_option.id, params['tags'], False, module.check_mode)

lib/ansible/modules/cloud/amazon/ec2_vpc_net.py

@@ -164,9 +164,10 @@ try:
 except ImportError:
     pass  # Handled by AnsibleAWSModule
 
+from time import sleep, time
 from ansible.module_utils.aws.core import AnsibleAWSModule
 from ansible.module_utils.ec2 import (boto3_conn, get_aws_connection_info, ec2_argument_spec, camel_dict_to_snake_dict,
-                                      ansible_dict_to_boto3_tag_list, boto3_tag_list_to_ansible_dict)
+                                      ansible_dict_to_boto3_tag_list, boto3_tag_list_to_ansible_dict, AWSRetry)
 from ansible.module_utils.six import string_types
 
 
@@ -194,20 +195,34 @@ def vpc_exists(module, vpc, name, cidr_block, multi):
     return None
 
 
+@AWSRetry.backoff(delay=3, tries=8, catch_extra_error_codes=['InvalidVpcID.NotFound'])
+def get_classic_link_with_backoff(connection, vpc_id):
+    try:
+        return connection.describe_vpc_classic_link(VpcIds=[vpc_id])['Vpcs'][0].get('ClassicLinkEnabled')
+    except botocore.exceptions.ClientError as e:
+        if e.response["Error"]["Message"] == "The functionality you requested is not available in this region.":
+            return False
+        else:
+            raise
+
+
 def get_vpc(module, connection, vpc_id):
+    # wait for vpc to be available
     try:
-        vpc_obj = connection.describe_vpcs(VpcIds=[vpc_id])['Vpcs'][0]
+        connection.get_waiter('vpc_available').wait(VpcIds=[vpc_id])
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Unable to wait for VPC {0} to be available.".format(vpc_id))
+
+    try:
+        vpc_obj = AWSRetry.backoff(
+            delay=3, tries=8,
+            catch_extra_error_codes=['InvalidVpcID.NotFound'],
+        )(connection.describe_vpcs)(VpcIds=[vpc_id])['Vpcs'][0]
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Failed to describe VPCs")
     try:
-        classic_link = connection.describe_vpc_classic_link(VpcIds=[vpc_id])['Vpcs'][0].get('ClassicLinkEnabled')
-        vpc_obj['ClassicLinkEnabled'] = classic_link
-    except botocore.exceptions.ClientError as e:
-        if e.response["Error"]["Message"] == "The functionality you requested is not available in this region.":
-            vpc_obj['ClassicLinkEnabled'] = False
-        else:
-            module.fail_json_aws(e, msg="Failed to describe VPCs")
-    except botocore.exceptions.BotoCoreError as e:
+        vpc_obj['ClassicLinkEnabled'] = get_classic_link_with_backoff(connection, vpc_id)
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Failed to describe VPCs")
 
     return vpc_obj
@@ -224,7 +239,16 @@ def update_vpc_tags(connection, module, vpc_id, tags, name):
         if tags != current_tags:
             if not module.check_mode:
                 tags = ansible_dict_to_boto3_tag_list(tags)
-                connection.create_tags(Resources=[vpc_id], Tags=tags)
+                vpc_obj = AWSRetry.backoff(
+                    delay=1, tries=5,
+                    catch_extra_error_codes=['InvalidVpcID.NotFound'],
+                )(connection.create_tags)(Resources=[vpc_id], Tags=tags)
+
+                # Wait for tags to be updated
+                expected_tags = boto3_tag_list_to_ansible_dict(tags)
+                filters = [{'Name': 'tag:{0}'.format(key), 'Values': [value]} for key, value in expected_tags.items()]
+                connection.get_waiter('vpc_available').wait(VpcIds=[vpc_id], Filters=filters)
+
             return True
         else:
             return False
@@ -239,6 +263,14 @@ def update_dhcp_opts(connection, module, vpc_obj, dhcp_id):
                 connection.associate_dhcp_options(DhcpOptionsId=dhcp_id, VpcId=vpc_obj['VpcId'])
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg="Failed to associate DhcpOptionsId {0}".format(dhcp_id))
+
+            try:
+                # Wait for DhcpOptionsId to be updated
+                filters = [{'Name': 'dhcp-options-id', 'Values': [dhcp_id]}]
+                connection.get_waiter('vpc_available').wait(VpcIds=[vpc_obj['VpcId']], Filters=filters)
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                module.fail_json(msg="Failed to wait for DhcpOptionsId to be updated")
+
         return True
     else:
         return False
@@ -252,9 +284,33 @@ def create_vpc(connection, module, cidr_block, tenancy):
             module.exit_json(changed=True)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, "Failed to create the VPC")
+
+    # wait for vpc to exist
+    try:
+        connection.get_waiter('vpc_exists').wait(VpcIds=[vpc_obj['Vpc']['VpcId']])
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg="Unable to wait for VPC {0} to be created.".format(vpc_obj['Vpc']['VpcId']))
+
     return vpc_obj['Vpc']['VpcId']
 
 
+def wait_for_vpc_attribute(connection, module, vpc_id, attribute, expected_value):
+    start_time = time()
+    updated = False
+    while time() < start_time + 300:
+        current_value = connection.describe_vpc_attribute(
+            Attribute=attribute,
+            VpcId=vpc_id
+        )['{0}{1}'.format(attribute[0].upper(), attribute[1:])]['Value']
+        if current_value != expected_value:
+            sleep(3)
+        else:
+            updated = True
+            break
+    if not updated:
+        module.fail_json(msg="Failed to wait for {0} to be updated".format(attribute))
+
+
 def main():
     argument_spec = ec2_argument_spec()
     argument_spec.update(dict(
@@ -310,6 +366,7 @@ def main():
                                 if cidr['CidrBlockState']['State'] != 'disassociated')
         to_add = [cidr for cidr in cidr_block if cidr not in associated_cidrs]
         to_remove = [associated_cidrs[cidr] for cidr in associated_cidrs if cidr not in cidr_block]
+        expected_cidrs = [cidr for cidr in associated_cidrs if associated_cidrs[cidr] not in to_remove] + to_add
 
         if len(cidr_block) > 1:
             for cidr in to_add:
@@ -356,10 +413,19 @@ def main():
                 except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                     module.fail_json_aws(e, "Failed to update enabled dns hostnames attribute")
 
-        try:
-            connection.get_waiter('vpc_available').wait(VpcIds=[vpc_id])
-        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-            module.fail_json_aws(e, msg="Unable to wait for VPC {0} to be available.".format(vpc_id))
+        # wait for associated cidrs to match
+        if to_add or to_remove:
+            try:
+                connection.get_waiter('vpc_available').wait(
+                    VpcIds=[vpc_id],
+                    Filters=[{'Name': 'cidr-block-association.cidr-block', 'Values': expected_cidrs}]
+                )
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                module.fail_json_aws(e, "Failed to wait for CIDRs to update")
+
+        # try to wait for enableDnsSupport and enableDnsHostnames to match
+        wait_for_vpc_attribute(connection, module, vpc_id, 'enableDnsSupport', dns_support)
+        wait_for_vpc_attribute(connection, module, vpc_id, 'enableDnsHostnames', dns_hostnames)
 
         final_state = camel_dict_to_snake_dict(get_vpc(module, connection, vpc_id))
         final_state['tags'] = boto3_tag_list_to_ansible_dict(final_state.get('tags', []))