From 40d2df0ef3b33a2661c5c4abdd92215be0efd158 Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Tue, 12 Jun 2018 12:15:16 -0400 Subject: [PATCH] Add AWS boto3 error code exception function is_boto3_error_code (#41202) * Add aws/core.py function to check for specific AWS error codes * Use sys.exc_info to get exception object if it isn't passed in * Allow catching exceptions with is_boto3_error_code * Replace from_code with is_boto3_error_code * Return a type that will never be raised to support stricter type comparisons in Python 3+ * Use is_boto3_error_code in aws_eks_cluster * Add duplicate-except to ignores when using is_boto3_error_code * Add is_boto3_error_code to module development guideline docs --- .../aws_core_is_boto3_error_code.yml | 4 +++ lib/ansible/module_utils/aws/core.py | 22 ++++++++++++++++ .../modules/cloud/amazon/GUIDELINES.md | 26 +++++++++++++++---- .../cloud/amazon/aws_config_aggregator.py | 6 ++--- .../amazon/aws_config_delivery_channel.py | 18 ++++++------- .../cloud/amazon/aws_config_recorder.py | 6 ++--- .../modules/cloud/amazon/aws_config_rule.py | 6 ++--- .../modules/cloud/amazon/aws_eks_cluster.py | 8 +++--- lib/ansible/modules/cloud/amazon/ec2_group.py | 12 ++++----- .../modules/cloud/amazon/ec2_vpc_vgw.py | 5 ++-- .../cloud/amazon/rds_instance_facts.py | 6 ++--- .../cloud/amazon/rds_snapshot_facts.py | 6 ++--- 12 files changed, 84 insertions(+), 41 deletions(-) create mode 100644 changelogs/fragments/aws_core_is_boto3_error_code.yml diff --git a/changelogs/fragments/aws_core_is_boto3_error_code.yml b/changelogs/fragments/aws_core_is_boto3_error_code.yml new file mode 100644 index 00000000000..d06a227773c --- /dev/null +++ b/changelogs/fragments/aws_core_is_boto3_error_code.yml @@ -0,0 +1,4 @@ +--- +minor_changes: + - Add `is_boto3_error_code` function to `module_utils/aws/core.py` to make it + easier for modules to handle special AWS error codes. diff --git a/lib/ansible/module_utils/aws/core.py b/lib/ansible/module_utils/aws/core.py index 99a87dd4403..466e72b0317 100644 --- a/lib/ansible/module_utils/aws/core.py +++ b/lib/ansible/module_utils/aws/core.py @@ -254,3 +254,25 @@ class _RetryingBotoClientWrapper(object): return wrapped else: return unwrapped + + +def is_boto3_error_code(code, e=None): + """Check if the botocore exception is raised by a specific error code. + + Returns ClientError if the error code matches, a dummy exception if it does not have an error code or does not match + + Example: + try: + ec2.describe_instances(InstanceIds=['potato']) + except is_boto3_error_code('InvalidInstanceID.Malformed'): + # handle the error for that code case + except botocore.exceptions.ClientError as e: + # handle the generic error case for all other codes + """ + from botocore.exceptions import ClientError + if e is None: + import sys + dummy, e, dummy = sys.exc_info() + if isinstance(e, ClientError) and e.response['Error']['Code'] == code: + return ClientError + return type('NeverEverRaisedException', (Exception,), {}) diff --git a/lib/ansible/modules/cloud/amazon/GUIDELINES.md b/lib/ansible/modules/cloud/amazon/GUIDELINES.md index 0b3ffa41f78..ca10c013eaf 100644 --- a/lib/ansible/modules/cloud/amazon/GUIDELINES.md +++ b/lib/ansible/modules/cloud/amazon/GUIDELINES.md @@ -207,14 +207,30 @@ extends_documentation_fragment: You should wrap any boto3 or botocore call in a try block. If an exception is thrown, then there are a number of possibilities for handling it. -* use aws_module.fail_json_aws() to report the module failure in a standard way -* retry using AWSRetry -* use fail_json() to report the failure without using `ansible.module_utils.aws.core` -* do something custom in the case where you know how to handle the exception +* Catch the general `ClientError` or look for a specific error code with + `is_boto3_error_code`. +* Use aws_module.fail_json_aws() to report the module failure in a standard way +* Retry using AWSRetry +* Use fail_json() to report the failure without using `ansible.module_utils.aws.core` +* Do something custom in the case where you know how to handle the exception For more information on botocore exception handling see [the botocore error documentation](http://botocore.readthedocs.org/en/latest/client_upgrades.html#error-handling). -#### using fail_json_aws() +### Using is_boto3_error_code + +To use `ansible.module_utils.aws.core.is_boto3_error_code` to catch a single +AWS error code, call it in place of `ClientError` in your except clauses. In +this case, *only* the `InvalidGroup.NotFound` error code will be caught here, +and any other error will be raised for handling elsewhere in the program. + +```python +try: + return connection.describe_security_groups(**kwargs) +except is_boto3_error_code('InvalidGroup.NotFound'): + return {'SecurityGroups': []} +``` + +#### Using fail_json_aws() In the AnsibleAWSModule there is a special method, `module.fail_json_aws()` for nice reporting of exceptions. Call this on your exception and it will report the error together with a traceback for diff --git a/lib/ansible/modules/cloud/amazon/aws_config_aggregator.py b/lib/ansible/modules/cloud/amazon/aws_config_aggregator.py index e8091221c94..311a5b71de2 100644 --- a/lib/ansible/modules/cloud/amazon/aws_config_aggregator.py +++ b/lib/ansible/modules/cloud/amazon/aws_config_aggregator.py @@ -85,7 +85,7 @@ try: except ImportError: pass # handled by AnsibleAWSModule -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, AWSRetry from ansible.module_utils.ec2 import camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict @@ -96,9 +96,9 @@ def resource_exists(client, module, resource_type, params): ConfigurationAggregatorNames=[params['name']] ) return aggregator['ConfigurationAggregators'][0] - except client.exceptions.from_code('NoSuchConfigurationAggregatorException'): + except is_boto3_error_code('NoSuchConfigurationAggregatorException'): return - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e) diff --git a/lib/ansible/modules/cloud/amazon/aws_config_delivery_channel.py b/lib/ansible/modules/cloud/amazon/aws_config_delivery_channel.py index d5eaf974ff6..f315720e876 100644 --- a/lib/ansible/modules/cloud/amazon/aws_config_delivery_channel.py +++ b/lib/ansible/modules/cloud/amazon/aws_config_delivery_channel.py @@ -69,7 +69,7 @@ try: except ImportError: pass # handled by AnsibleAWSModule -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, AWSRetry from ansible.module_utils.ec2 import camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict @@ -88,9 +88,9 @@ def resource_exists(client, module, params): aws_retry=True, ) return channel['DeliveryChannels'][0] - except client.exceptions.from_code('NoSuchDeliveryChannelException'): + except is_boto3_error_code('NoSuchDeliveryChannelException'): return - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e) @@ -104,12 +104,12 @@ def create_resource(client, module, params, result): result['changed'] = True result['channel'] = camel_dict_to_snake_dict(resource_exists(client, module, params)) return result - except client.exceptions.from_code('InvalidS3KeyPrefixException') as e: + except is_boto3_error_code('InvalidS3KeyPrefixException') as e: module.fail_json_aws(e, msg="The `s3_prefix` parameter was invalid. Try '/' for no prefix") - except client.exceptions.from_code('InsufficientDeliveryPolicyException') as e: + except is_boto3_error_code('InsufficientDeliveryPolicyException') as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="The `s3_prefix` or `s3_bucket` parameter is invalid. " "Make sure the bucket exists and is available") - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Couldn't create AWS Config delivery channel") @@ -129,12 +129,12 @@ def update_resource(client, module, params, result): result['changed'] = True result['channel'] = camel_dict_to_snake_dict(resource_exists(client, module, params)) return result - except client.exceptions.from_code('InvalidS3KeyPrefixException') as e: + except is_boto3_error_code('InvalidS3KeyPrefixException') as e: module.fail_json_aws(e, msg="The `s3_prefix` parameter was invalid. Try '/' for no prefix") - except client.exceptions.from_code('InsufficientDeliveryPolicyException') as e: + except is_boto3_error_code('InsufficientDeliveryPolicyException') as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="The `s3_prefix` or `s3_bucket` parameter is invalid. " "Make sure the bucket exists and is available") - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Couldn't create AWS Config delivery channel") diff --git a/lib/ansible/modules/cloud/amazon/aws_config_recorder.py b/lib/ansible/modules/cloud/amazon/aws_config_recorder.py index db2ef749014..dbf7252bc7f 100644 --- a/lib/ansible/modules/cloud/amazon/aws_config_recorder.py +++ b/lib/ansible/modules/cloud/amazon/aws_config_recorder.py @@ -86,7 +86,7 @@ try: except ImportError: pass # handled by AnsibleAWSModule -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, AWSRetry from ansible.module_utils.ec2 import camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict @@ -97,9 +97,9 @@ def resource_exists(client, module, params): ConfigurationRecorderNames=[params['name']] ) return recorder['ConfigurationRecorders'][0] - except client.exceptions.from_code('NoSuchConfigurationRecorderException'): + except is_boto3_error_code('NoSuchConfigurationRecorderException'): return - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e) diff --git a/lib/ansible/modules/cloud/amazon/aws_config_rule.py b/lib/ansible/modules/cloud/amazon/aws_config_rule.py index 51909de754d..c8a0ba5eb97 100644 --- a/lib/ansible/modules/cloud/amazon/aws_config_rule.py +++ b/lib/ansible/modules/cloud/amazon/aws_config_rule.py @@ -110,7 +110,7 @@ try: except ImportError: pass # handled by AnsibleAWSModule -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.ec2 import AWSRetry, camel_dict_to_snake_dict @@ -121,9 +121,9 @@ def rule_exists(client, module, params): aws_retry=True, ) return rule['ConfigRules'][0] - except client.exceptions.from_code('NoSuchConfigRuleException'): + except is_boto3_error_code('NoSuchConfigRuleException'): return - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e) diff --git a/lib/ansible/modules/cloud/amazon/aws_eks_cluster.py b/lib/ansible/modules/cloud/amazon/aws_eks_cluster.py index c7795484ef4..968dbb717b0 100644 --- a/lib/ansible/modules/cloud/amazon/aws_eks_cluster.py +++ b/lib/ansible/modules/cloud/amazon/aws_eks_cluster.py @@ -129,7 +129,7 @@ version: ''' -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.ec2 import camel_dict_to_snake_dict, get_ec2_security_group_ids_from_names try: @@ -197,11 +197,11 @@ def get_cluster(client, module): name = module.params.get('name') try: return client.describe_cluster(name=name)['cluster'] - except client.exceptions.from_code('ResourceNotFoundException'): + except is_boto3_error_code('ResourceNotFoundException'): return None - except botocore.exceptions.EndpointConnectionError as e: + except botocore.exceptions.EndpointConnectionError as e: # pylint: disable=duplicate-except module.fail_json(msg="Region %s is not supported by EKS" % client.meta.region_name) - except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json(e, msg="Couldn't get cluster %s" % name) diff --git a/lib/ansible/modules/cloud/amazon/ec2_group.py b/lib/ansible/modules/cloud/amazon/ec2_group.py index e3ace317dea..7e007582a35 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_group.py +++ b/lib/ansible/modules/cloud/amazon/ec2_group.py @@ -292,7 +292,7 @@ import json import re from time import sleep from collections import namedtuple -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.aws.iam import get_aws_account_id from ansible.module_utils.aws.waiters import get_waiter from ansible.module_utils.ec2 import AWSRetry, camel_dict_to_snake_dict, compare_aws_tags @@ -430,7 +430,7 @@ def get_security_groups_with_backoff(connection, **kwargs): def sg_exists_with_backoff(connection, **kwargs): try: return connection.describe_security_groups(**kwargs) - except connection.exceptions.from_code('InvalidGroup.NotFound') as e: + except is_boto3_error_code('InvalidGroup.NotFound'): return {'SecurityGroups': []} @@ -519,10 +519,10 @@ def get_target_from_rule(module, client, rule, name, group, groups, vpc_id): # retry describing the group once try: auto_group = get_security_groups_with_backoff(client, Filters=ansible_dict_to_boto3_filter_list(filters)).get('SecurityGroups', [])[0] - except (client.exceptions.from_code('InvalidGroup.NotFound'), IndexError) as e: + except (is_boto3_error_code('InvalidGroup.NotFound'), IndexError): module.fail_json(msg="group %s will be automatically created by rule %s but " "no description was provided" % (group_name, rule)) - except ClientError as e: + except ClientError as e: # pylint: disable=duplicate-except module.fail_json_aws(e) elif not module.check_mode: params = dict(GroupName=group_name, Description=rule['group_desc']) @@ -535,7 +535,7 @@ def get_target_from_rule(module, client, rule, name, group, groups, vpc_id): ).wait( GroupIds=[auto_group['GroupId']], ) - except client.exceptions.from_code('InvalidGroup.Duplicate') as e: + except is_boto3_error_code('InvalidGroup.Duplicate'): # The group exists, but didn't show up in any of our describe-security-groups calls # Try searching on a filter for the name, and allow a retry window for AWS to update # the model on their end. @@ -829,7 +829,7 @@ def group_exists(client, module, vpc_id, group_id, name): try: security_groups = sg_exists_with_backoff(client, **params).get('SecurityGroups', []) all_groups = get_security_groups_with_backoff(client).get('SecurityGroups', []) - except (BotoCoreError, ClientError) as e: + except (BotoCoreError, ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Error in describe_security_groups") if security_groups: diff --git a/lib/ansible/modules/cloud/amazon/ec2_vpc_vgw.py b/lib/ansible/modules/cloud/amazon/ec2_vpc_vgw.py index 7be706269e6..4a92242ae92 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_vpc_vgw.py +++ b/lib/ansible/modules/cloud/amazon/ec2_vpc_vgw.py @@ -116,6 +116,7 @@ try: except ImportError: HAS_BOTO3 = False +from ansible.module_utils.aws.core import is_boto3_error_code from ansible.module_utils.aws.waiters import get_waiter from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.ec2 import HAS_BOTO3, boto3_conn, ec2_argument_spec, get_aws_connection_info, AWSRetry @@ -220,9 +221,9 @@ def create_vgw(client, module): except botocore.exceptions.WaiterError as e: module.fail_json(msg="Failed to wait for Vpn Gateway {0} to be available".format(response['VpnGateway']['VpnGatewayId']), exception=traceback.format_exc()) - except client.exceptions.from_code('VpnGatewayLimitExceeded') as e: + except is_boto3_error_code('VpnGatewayLimitExceeded'): module.fail_json(msg="Too many VPN gateways exist in this account.", exception=traceback.format_exc()) - except botocore.exceptions.ClientError as e: + except botocore.exceptions.ClientError as e: # pylint: disable=duplicate-except module.fail_json(msg=to_native(e), exception=traceback.format_exc()) result = response diff --git a/lib/ansible/modules/cloud/amazon/rds_instance_facts.py b/lib/ansible/modules/cloud/amazon/rds_instance_facts.py index aee3e8380e9..d7e3ddac35a 100644 --- a/lib/ansible/modules/cloud/amazon/rds_instance_facts.py +++ b/lib/ansible/modules/cloud/amazon/rds_instance_facts.py @@ -340,7 +340,7 @@ instances: sample: sg-abcd1234 ''' -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.ec2 import ansible_dict_to_boto3_filter_list, boto3_tag_list_to_ansible_dict, AWSRetry, camel_dict_to_snake_dict @@ -363,9 +363,9 @@ def instance_facts(module, conn): paginator = conn.get_paginator('describe_db_instances') try: results = paginator.paginate(**params).build_full_result()['DBInstances'] - except conn.exceptions.from_code('DBInstanceNotFound'): + except is_boto3_error_code('DBInstanceNotFound'): results = [] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, "Couldn't get instance information") for instance in results: diff --git a/lib/ansible/modules/cloud/amazon/rds_snapshot_facts.py b/lib/ansible/modules/cloud/amazon/rds_snapshot_facts.py index 4ceb3131337..bfad0502a66 100644 --- a/lib/ansible/modules/cloud/amazon/rds_snapshot_facts.py +++ b/lib/ansible/modules/cloud/amazon/rds_snapshot_facts.py @@ -284,7 +284,7 @@ cluster_snapshots: sample: vpc-abcd1234 ''' -from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code from ansible.module_utils.ec2 import AWSRetry, boto3_tag_list_to_ansible_dict, camel_dict_to_snake_dict try: @@ -297,9 +297,9 @@ def common_snapshot_facts(module, conn, method, prefix, params): paginator = conn.get_paginator(method) try: results = paginator.paginate(**params).build_full_result()['%ss' % prefix] - except conn.exceptions.from_code('%sNotFound' % prefix): + except is_boto3_error_code('%sNotFound' % prefix): results = [] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, "trying to get snapshot information") for snapshot in results: