[cloud] Add jittered backoff+retries to CloudFormation module (#27907)

lib/ansible/modules/cloud/amazon/cloudformation.py

@@ -417,13 +417,10 @@ def stack_operation(cfn, stack_name, operation):
             time.sleep(5)
     return {'failed': True, 'output':'Failed for unknown reasons.'}
 
-@AWSRetry.backoff(tries=3, delay=5)
-def describe_stacks(cfn, stack_name):
-    return cfn.describe_stacks(StackName=stack_name)
 
 def get_stack_facts(cfn, stack_name):
     try:
-        stack_response = describe_stacks(cfn, stack_name)
+        stack_response = cfn.describe_stacks(StackName=stack_name)
         stack_info = stack_response['Stacks'][0]
     #except AmazonCloudFormationException as e:
     except (botocore.exceptions.ValidationError,botocore.exceptions.ClientError) as err:
@@ -509,6 +506,18 @@ def main():
     except botocore.exceptions.NoCredentialsError as e:
         module.fail_json(msg=boto_exception(e))
 
+    # Wrap the cloudformation client methods that this module uses with
+    # automatic backoff / retry for throttling error codes
+    backoff_wrapper = AWSRetry.jittered_backoff(retries=10, delay=3, max_delay=30)
+    cfn.describe_stack_events = backoff_wrapper(cfn.describe_stack_events)
+    cfn.create_stack = backoff_wrapper(cfn.create_stack)
+    cfn.list_change_sets = backoff_wrapper(cfn.list_change_sets)
+    cfn.create_change_set = backoff_wrapper(cfn.create_change_set)
+    cfn.update_stack = backoff_wrapper(cfn.update_stack)
+    cfn.describe_stacks = backoff_wrapper(cfn.describe_stacks)
+    cfn.list_stack_resources = backoff_wrapper(cfn.list_stack_resources)
+    cfn.delete_stack = backoff_wrapper(cfn.delete_stack)
+
     stack_info = get_stack_facts(cfn, stack_params['StackName'])
 
     if state == 'present':