From f7d79d4789af827c7fabd21dac825a95a2b39516 Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Wed, 21 Feb 2018 08:14:17 -0500 Subject: [PATCH] [cloud] Retry WAF actions on WAFStaleDataException (#36405) Add a util to run functions with AWSRetry to retry on WAFStaleDataExceptions and update ChangeToken for each attempt --- lib/ansible/module_utils/aws/waf.py | 6 +++ .../modules/cloud/amazon/aws_waf_condition.py | 34 ++++++++--------- .../modules/cloud/amazon/aws_waf_rule.py | 17 +++++---- .../modules/cloud/amazon/aws_waf_web_acl.py | 37 ++++++++++++------- .../targets/aws_waf_web_acl/tasks/main.yml | 10 +++++ 5 files changed, 65 insertions(+), 39 deletions(-) diff --git a/lib/ansible/module_utils/aws/waf.py b/lib/ansible/module_utils/aws/waf.py index dfdb327b427..6758b40ed65 100644 --- a/lib/ansible/module_utils/aws/waf.py +++ b/lib/ansible/module_utils/aws/waf.py @@ -180,3 +180,9 @@ def get_change_token(client, module): return token['ChangeToken'] except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Couldn't obtain change token") + + +@AWSRetry.backoff(tries=10, delay=2, backoff=2.0, catch_extra_error_codes=['WAFStaleDataException']) +def run_func_with_change_token_backoff(client, module, params, func): + params['ChangeToken'] = get_change_token(client, module) + return func(**params) diff --git a/lib/ansible/modules/cloud/amazon/aws_waf_condition.py b/lib/ansible/modules/cloud/amazon/aws_waf_condition.py index 4565a0d7093..4ae88d1d1df 100644 --- a/lib/ansible/modules/cloud/amazon/aws_waf_condition.py +++ b/lib/ansible/modules/cloud/amazon/aws_waf_condition.py @@ -331,7 +331,7 @@ except ImportError: from ansible.module_utils.aws.core import AnsibleAWSModule from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec from ansible.module_utils.ec2 import camel_dict_to_snake_dict, AWSRetry, compare_policies -from ansible.module_utils.aws.waf import get_change_token, MATCH_LOOKUP +from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, MATCH_LOOKUP from ansible.module_utils.aws.waf import get_rule_with_backoff, list_rules_with_backoff @@ -397,12 +397,10 @@ class Condition(object): kwargs['Updates'].append({'Action': 'INSERT', self.conditiontuple: condition_insert}) kwargs[self.conditionsetid] = condition_set_id - kwargs['ChangeToken'] = get_change_token(self.client, self.module) return kwargs def format_for_deletion(self, condition): - return {'ChangeToken': get_change_token(self.client, self.module), - 'Updates': [{'Action': 'DELETE', self.conditiontuple: current_condition_tuple} + return {'Updates': [{'Action': 'DELETE', self.conditiontuple: current_condition_tuple} for current_condition_tuple in condition[self.conditiontuples]], self.conditionsetid: condition[self.conditionsetid]} @@ -443,15 +441,17 @@ class Condition(object): pattern_set = self.get_regex_pattern_by_name(name) if not pattern_set: - pattern_set = self.client.create_regex_pattern_set(Name=name, ChangeToken=get_change_token(self.client, self.module))['RegexPatternSet'] + pattern_set = run_func_with_change_token_backoff(self.client, self.module, {'Name': name}, + self.client.create_regex_pattern_set)['RegexPatternSet'] missing = set(regex_pattern['regex_strings']) - set(pattern_set['RegexPatternStrings']) extra = set(pattern_set['RegexPatternStrings']) - set(regex_pattern['regex_strings']) if not missing and not extra: return pattern_set updates = [{'Action': 'INSERT', 'RegexPatternString': pattern} for pattern in missing] updates.extend([{'Action': 'DELETE', 'RegexPatternString': pattern} for pattern in extra]) - self.client.update_regex_pattern_set(RegexPatternSetId=pattern_set['RegexPatternSetId'], - Updates=updates, ChangeToken=get_change_token(self.client, self.module)) + run_func_with_change_token_backoff(self.client, self.module, + {'RegexPatternSetId': pattern_set['RegexPatternSetId'], 'Updates': updates}, + self.client.update_regex_pattern_set) return self.get_regex_pattern_set_with_backoff(pattern_set['RegexPatternSetId'])['RegexPatternSet'] def delete_unused_regex_pattern(self, regex_pattern_set_id): @@ -460,11 +460,13 @@ class Condition(object): updates = list() for regex_pattern_string in regex_pattern_set['RegexPatternStrings']: updates.append({'Action': 'DELETE', 'RegexPatternString': regex_pattern_string}) - self.client.update_regex_pattern_set(RegexPatternSetId=regex_pattern_set_id, Updates=updates, - ChangeToken=get_change_token(self.client, self.module)) + run_func_with_change_token_backoff(self.client, self.module, + {'RegexPatternSetId': regex_pattern_set_id, 'Updates': updates}, + self.client.update_regex_pattern_set) - self.client.delete_regex_pattern_set(RegexPatternSetId=regex_pattern_set_id, - ChangeToken=get_change_token(self.client, self.module)) + run_func_with_change_token_backoff(self.client, self.module, + {'RegexPatternSetId': regex_pattern_set_id}, + self.client.delete_regex_pattern_set) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self.module.fail_json_aws(e, msg='Could not delete regex pattern') @@ -535,15 +537,14 @@ class Condition(object): func = getattr(self.client, 'update_' + self.method_suffix) params = self.format_for_deletion(current_condition) try: - func(**params) + run_func_with_change_token_backoff(self.client, self.module, params, func) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self.module.fail_json_aws(e, msg='Could not delete filters from condition') func = getattr(self.client, 'delete_' + self.method_suffix) params = dict() params[self.conditionsetid] = condition_set_id - params['ChangeToken'] = get_change_token(self.client, self.module) try: - func(**params) + run_func_with_change_token_backoff(self.client, self.module, params, func) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self.module.fail_json_aws(e, msg='Could not delete condition') # tidy up regex patterns @@ -579,7 +580,7 @@ class Condition(object): update['Updates'] = missing + extra func = getattr(self.client, 'update_' + self.method_suffix) try: - func(**update) + run_func_with_change_token_backoff(self.client, self.module, update, func) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self.module.fail_json_aws(e, msg='Could not update condition') return changed, self.get_condition_by_id(condition_set_id) @@ -592,10 +593,9 @@ class Condition(object): else: params = dict() params['Name'] = name - params['ChangeToken'] = get_change_token(self.client, self.module) func = getattr(self.client, 'create_' + self.method_suffix) try: - condition = func(**params) + condition = run_func_with_change_token_backoff(self.client, self.module, params, func) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: self.module.fail_json_aws(e, msg='Could not create condition') return self.find_and_update_condition(condition[self.conditionset][self.conditionsetid]) diff --git a/lib/ansible/modules/cloud/amazon/aws_waf_rule.py b/lib/ansible/modules/cloud/amazon/aws_waf_rule.py index 1593cb3117b..afc5fab15ee 100644 --- a/lib/ansible/modules/cloud/amazon/aws_waf_rule.py +++ b/lib/ansible/modules/cloud/amazon/aws_waf_rule.py @@ -126,7 +126,7 @@ except ImportError: from ansible.module_utils.aws.core import AnsibleAWSModule from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec from ansible.module_utils.ec2 import camel_dict_to_snake_dict -from ansible.module_utils.aws.waf import get_change_token, list_rules_with_backoff, MATCH_LOOKUP +from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, list_rules_with_backoff, MATCH_LOOKUP from ansible.module_utils.aws.waf import get_web_acl_with_backoff, list_web_acls_with_backoff @@ -201,10 +201,13 @@ def find_and_update_rule(client, module, rule_id): if not all_conditions[condition_type][condition['data_id']]['name'] in desired_conditions[condition_type]]) changed = bool(insertions or deletions) + update = { + 'RuleId': rule_id, + 'Updates': insertions + deletions + } if changed: try: - client.update_rule(RuleId=rule_id, ChangeToken=get_change_token(client, module), - Updates=insertions + deletions) + run_func_with_change_token_backoff(client, module, update, client.update_rule) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not update rule conditions') @@ -229,8 +232,7 @@ def remove_rule_conditions(client, module, rule_id): conditions = get_rule(client, module, rule_id)['Predicates'] updates = [format_for_deletion(camel_dict_to_snake_dict(condition)) for condition in conditions] try: - client.update_rule(RuleId=rule_id, - ChangeToken=get_change_token(client, module), Updates=updates) + run_func_with_change_token_backoff(client, module, {'RuleId': rule_id, 'Updates': updates}, client.update_rule) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not remove rule conditions') @@ -247,9 +249,8 @@ def ensure_rule_present(client, module): if not metric_name: metric_name = re.sub(r'[^a-zA-Z0-9]', '', module.params['name']) params['MetricName'] = metric_name - params['ChangeToken'] = get_change_token(client, module) try: - new_rule = client.create_rule(**params)['Rule'] + new_rule = run_func_with_change_token_backoff(client, module, params, client.create_rule)['Rule'] except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not create rule') return find_and_update_rule(client, module, new_rule['RuleId']) @@ -281,7 +282,7 @@ def ensure_rule_absent(client, module): if rule_id: remove_rule_conditions(client, module, rule_id) try: - return True, client.delete_rule(RuleId=rule_id, ChangeToken=get_change_token(client, module)) + return True, run_func_with_change_token_backoff(client, module, {'RuleId': rule_id}, client.delete_rule) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not delete rule') return False, {} diff --git a/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py b/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py index 0e705d816f7..2b0ea67714c 100644 --- a/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py +++ b/lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py @@ -136,7 +136,7 @@ import re from ansible.module_utils.aws.core import AnsibleAWSModule from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec, camel_dict_to_snake_dict -from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, get_change_token +from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, run_func_with_change_token_backoff def get_web_acl_by_name(client, module, name): @@ -186,16 +186,26 @@ def find_and_update_web_acl(client, module, web_acl_id): insertions = [format_for_update(rule, 'INSERT') for rule in missing] deletions = [format_for_update(rule, 'DELETE') for rule in extras] changed = bool(insertions + deletions) - if changed: + + # Purge rules before adding new ones in case a deletion shares the same + # priority as an insertion. + params = { + 'WebACLId': acl['WebACLId'], + 'DefaultAction': acl['DefaultAction'] + } + if deletions: try: - client.update_web_acl( - WebACLId=acl['WebACLId'], - ChangeToken=get_change_token(client, module), - Updates=insertions + deletions, - DefaultAction=acl['DefaultAction'] - ) + params['Updates'] = deletions + run_func_with_change_token_backoff(client, module, params, client.update_web_acl) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not update Web ACL') + if insertions: + try: + params['Updates'] = insertions + run_func_with_change_token_backoff(client, module, params, client.update_web_acl) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg='Could not update Web ACL') + if changed: acl = get_web_acl(client, module, web_acl_id) return changed, acl @@ -217,8 +227,8 @@ def remove_rules_from_web_acl(client, module, web_acl_id): acl = get_web_acl(client, module, web_acl_id) deletions = [format_for_update(rule, 'DELETE') for rule in acl['Rules']] try: - client.update_web_acl(WebACLId=acl['WebACLId'], ChangeToken=get_change_token(client, module), - Updates=deletions, DefaultAction=acl['DefaultAction']) + params = {'WebACLId': acl['WebACLId'], 'DefaultAction': acl['DefaultAction'], 'Updates': deletions} + run_func_with_change_token_backoff(client, module, params, client.update_web_acl) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not remove rule') @@ -236,9 +246,8 @@ def ensure_web_acl_present(client, module): metric_name = re.sub(r'[^A-Za-z0-9]', '', module.params['name']) default_action = module.params['default_action'].upper() try: - new_web_acl = client.create_web_acl(Name=name, MetricName=metric_name, - DefaultAction={'Type': default_action}, - ChangeToken=get_change_token(client, module)) + params = {'Name': name, 'MetricName': metric_name, 'DefaultAction': {'Type': default_action}} + new_web_acl = run_func_with_change_token_backoff(client, module, params, client.create_web_acl) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not create Web ACL') (changed, result) = find_and_update_web_acl(client, module, new_web_acl['WebACL']['WebACLId']) @@ -252,7 +261,7 @@ def ensure_web_acl_absent(client, module): if web_acl['Rules']: remove_rules_from_web_acl(client, module, web_acl_id) try: - client.delete_web_acl(WebACLId=web_acl_id, ChangeToken=get_change_token(client, module)) + run_func_with_change_token_backoff(client, module, {'WebACLId': web_acl_id}, client.delete_web_acl) return True, {} except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Could not delete Web ACL') diff --git a/test/integration/targets/aws_waf_web_acl/tasks/main.yml b/test/integration/targets/aws_waf_web_acl/tasks/main.yml index faf5cc88f10..e1dac5fca75 100644 --- a/test/integration/targets/aws_waf_web_acl/tasks/main.yml +++ b/test/integration/targets/aws_waf_web_acl/tasks/main.yml @@ -481,10 +481,19 @@ - debug: msg: "****** TEARDOWN STARTS HERE ******" + - name: delete the web acl + aws_waf_web_acl: + name: "{{ resource_prefix }}_web_acl" + state: absent + purge_rules: yes + <<: *aws_connection_info + ignore_errors: yes + - name: remove second WAF rule aws_waf_rule: name: "{{ resource_prefix }}_rule_2" state: absent + purge_conditions: yes <<: *aws_connection_info ignore_errors: yes @@ -492,6 +501,7 @@ aws_waf_rule: name: "{{ resource_prefix }}_rule" state: absent + purge_conditions: yes <<: *aws_connection_info ignore_errors: yes