Fix ec2_vpc_vgw broken tests (#60981)

* Fix ec2_vpc_vgw broken tests

Add waiter function to wait for API to report detached vgw is available.
Also catch extra error code in attach retry as EC2 sometimes reports that
the vgw is available several seconds before permitting the attachment.

Fixes: #53185

* Re-enable ec2_vgc_vgw test target

lib/ansible/module_utils/aws/waiters.py

@@ -151,6 +151,19 @@ ec2_data = {
                 },
             ]
         },
+        "VpnGatewayDetached": {
+            "delay": 5,
+            "maxAttempts": 40,
+            "operation": "DescribeVpnGateways",
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "expected": True,
+                    "argument": "VpnGateways[0].State == 'available'",
+                    "state": "success"
+                },
+            ]
+        },
     }
 }
 
@@ -317,6 +330,12 @@ waiters_by_name = {
         core_waiter.NormalizedOperationMethod(
             ec2.describe_vpn_gateways
         )),
+    ('EC2', 'vpn_gateway_detached'): lambda ec2: core_waiter.Waiter(
+        'vpn_gateway_detached',
+        ec2_model('VpnGatewayDetached'),
+        core_waiter.NormalizedOperationMethod(
+            ec2.describe_vpn_gateways
+        )),
     ('WAF', 'change_token_in_sync'): lambda waf: core_waiter.Waiter(
         'change_token_in_sync',
         waf_model('ChangeTokenInSync'),

lib/ansible/modules/cloud/amazon/ec2_vpc_vgw.py

@@ -171,7 +171,13 @@ def attach_vgw(client, module, vpn_gateway_id):
     params['VpcId'] = module.params.get('vpc_id')
 
     try:
-        response = AWSRetry.jittered_backoff()(client.attach_vpn_gateway)(VpnGatewayId=vpn_gateway_id, VpcId=params['VpcId'])
+        # Immediately after a detachment, the EC2 API sometimes will report the VpnGateways[0].State
+        # as available several seconds before actually permitting a new attachment.
+        # So we catch and retry that error.  See https://github.com/ansible/ansible/issues/53185
+        response = AWSRetry.jittered_backoff(retries=5,
+                                             catch_extra_error_codes=['InvalidParameterValue']
+                                             )(client.attach_vpn_gateway)(VpnGatewayId=vpn_gateway_id,
+                                                                          VpcId=params['VpcId'])
     except botocore.exceptions.ClientError as e:
         module.fail_json(msg=to_native(e), exception=traceback.format_exc())
 
@@ -400,7 +406,7 @@ def ensure_vgw_present(client, module):
                     # detach the existing vpc from the virtual gateway
                     vpc_to_detach = current_vpc_attachments[0]['VpcId']
                     detach_vgw(client, module, vpn_gateway_id, vpc_to_detach)
-                    time.sleep(5)
+                    get_waiter(client, 'vpn_gateway_detached').wait(VpnGatewayIds=[vpn_gateway_id])
                     attached_vgw = attach_vgw(client, module, vpn_gateway_id)
                     changed = True
             else:

test/integration/targets/ec2_vpc_vgw/aliases

@@ -1,3 +1,2 @@
 cloud/aws
 shippable/aws/group2
-disabled