Migrate ec2_eip module to boto3 (#61575)

* Migrate ec2_eip module to boto3 This patch is a step towards the integration of several PRs that have attempted to migrate this code closes #55190 closes #45478 Follow-up PRs will address the outstanding changes made in #55190
5 years ago · 2ebeadfc14
parent 00add5b668
commit 2ebeadfc14
2 changed files with 332 additions and 265 deletions
--- a/lib/ansible/modules/cloud/amazon/ec2_eip.py
+++ b/lib/ansible/modules/cloud/amazon/ec2_eip.py
@ -223,163 +223,203 @@ public_ip:
 '''

 try:
-    import boto.exception
-    from boto.ec2.address import Address
+    import botocore.exceptions
 except ImportError:
-    pass  # Taken care of by ec2.HAS_BOTO
+    pass  # Taken care of by ec2.HAS_BOTO3

-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.ec2 import HAS_BOTO, ec2_argument_spec, ec2_connect
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
+from ansible.module_utils.ec2 import AWSRetry, ansible_dict_to_boto3_filter_list, ec2_argument_spec


-class EIPException(Exception):
-    pass
-
-
-def associate_ip_and_device(ec2, address, private_ip_address, device_id, allow_reassociation, check_mode, is_instance=True):
-    if address_is_associated_with_device(ec2, address, device_id, is_instance):
+def associate_ip_and_device(ec2, module, address, private_ip_address, device_id, allow_reassociation, check_mode, is_instance=True):
+    if address_is_associated_with_device(ec2, module, address, device_id, is_instance):
        return {'changed': False}

    # If we're in check mode, nothing else to do
    if not check_mode:
        if is_instance:
+            try:
+                params = dict(
+                    InstanceId=device_id,
+                    PrivateIpAddress=private_ip_address,
+                    AllowReassociation=allow_reassociation,
+                )
                if address.domain == "vpc":
-                res = ec2.associate_address(device_id,
-                                            allocation_id=address.allocation_id,
-                                            private_ip_address=private_ip_address,
-                                            allow_reassociation=allow_reassociation)
+                    params['AllocationId'] = address['AllocationId']
                else:
-                res = ec2.associate_address(device_id,
-                                            public_ip=address.public_ip,
-                                            private_ip_address=private_ip_address,
-                                            allow_reassociation=allow_reassociation)
+                    params['PublicIp'] = address['PublicIp']
+                res = ec2.associate_address(**params)
+            except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+                msg = "Couldn't associate Elastic IP address with instance '{0}'".format(device_id)
+                module.fail_json_aws(e, msg=msg)
        else:
-            res = ec2.associate_address(network_interface_id=device_id,
-                                        allocation_id=address.allocation_id,
-                                        private_ip_address=private_ip_address,
-                                        allow_reassociation=allow_reassociation)
+            params = dict(
+                NetworkInterfaceId=device_id,
+                AllocationId=address['AllocationId'],
+                AllowReassociation=allow_reassociation,
+            )
+
+            if private_ip_address:
+                params['PrivateIpAddress'] = private_ip_address
+
+            try:
+                res = ec2.associate_address(aws_retry=True, **params)
+            except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+                msg = "Couldn't associate Elastic IP address with network interface '{0}'".format(device_id)
+                module.fail_json_aws(e, msg=msg)
        if not res:
-            raise EIPException('association failed')
+            module.fail_json_aws(e, msg='Association failed.')

    return {'changed': True}


-def disassociate_ip_and_device(ec2, address, device_id, check_mode, is_instance=True):
-    if not address_is_associated_with_device(ec2, address, device_id, is_instance):
+def disassociate_ip_and_device(ec2, module, address, device_id, check_mode, is_instance=True):
+    if not address_is_associated_with_device(ec2, module, address, device_id, is_instance):
        return {'changed': False}

    # If we're in check mode, nothing else to do
    if not check_mode:
-        if address.domain == 'vpc':
+        try:
+            if address['Domain'] == 'vpc':
                res = ec2.disassociate_address(
-                association_id=address.association_id)
+                    AssociationId=address['AssociationId'], aws_retry=True
+                )
            else:
-            res = ec2.disassociate_address(public_ip=address.public_ip)
-
-        if not res:
-            raise EIPException('disassociation failed')
+                res = ec2.disassociate_address(
+                    PublicIp=address['PublicIp'], aws_retry=True
+                )
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+            module.fail_json_aws(e, msg="Dissassociation of Elastic IP failed")

    return {'changed': True}


-def _find_address_by_ip(ec2, public_ip):
-    try:
-        return ec2.get_all_addresses([public_ip])[0]
-    except boto.exception.EC2ResponseError as e:
-        if "Address '{0}' not found.".format(public_ip) not in e.message:
-            raise
-
+@AWSRetry.jittered_backoff()
+def find_address(ec2, module, public_ip, device_id, is_instance=True):
+    """ Find an existing Elastic IP address """
+    filters = []
+    kwargs = {}

-def _find_address_by_device_id(ec2, device_id, is_instance=True):
+    if public_ip:
+        kwargs["PublicIps"] = [public_ip]
+    elif device_id:
        if is_instance:
-        addresses = ec2.get_all_addresses(None, {'instance-id': device_id})
+            filters.append({"Name": 'instance-id', "Values": [device_id]})
        else:
-        addresses = ec2.get_all_addresses(None, {'network-interface-id': device_id})
-    if addresses:
-        return addresses[0]
+            filters.append({'Name': 'network-interface-id', "Values": [device_id]})

+    if len(filters) > 0:
+        kwargs["Filters"] = filters
+    elif len(filters) == 0 and public_ip is None:
+        return None

-def find_address(ec2, public_ip, device_id, is_instance=True):
-    """ Find an existing Elastic IP address """
-    if public_ip:
-        return _find_address_by_ip(ec2, public_ip)
-    elif device_id and is_instance:
-        return _find_address_by_device_id(ec2, device_id)
-    elif device_id:
-        return _find_address_by_device_id(ec2, device_id, is_instance=False)
+    try:
+        addresses = ec2.describe_addresses(**kwargs)
+    except is_boto3_error_code('InvalidAddress.NotFound') as e:
+        module.fail_json_aws(e, msg="Couldn't obtain list of existing Elastic IP addresses")
+
+    addresses = addresses["Addresses"]
+    if len(addresses) == 1:
+        return addresses[0]
+    elif len(addresses) > 1:
+        msg = "Found more than one address using args {0}".format(kwargs)
+        msg += "Addresses found: {0}".format(addresses)
+        module.fail_json_aws(botocore.exceptions.ClientError, msg=msg)


-def address_is_associated_with_device(ec2, address, device_id, is_instance=True):
+def address_is_associated_with_device(ec2, module, address, device_id, is_instance=True):
    """ Check if the elastic IP is currently associated with the device """
-    address = ec2.get_all_addresses(address.public_ip)
+    address = find_address(ec2, module, address["PublicIp"], device_id, is_instance)
    if address:
        if is_instance:
-            return address and address[0].instance_id == device_id
+            if "InstanceId" in address and address["InstanceId"] == device_id:
+                return address
        else:
-            return address and address[0].network_interface_id == device_id
+            if "NetworkInterfaceId" in address and address["NetworkInterfaceId"] == device_id:
+                return address
    return False


-def allocate_address(ec2, domain, reuse_existing_ip_allowed, check_mode, tag_dict=None, public_ipv4_pool=None):
+def allocate_address(ec2, module, domain, reuse_existing_ip_allowed, check_mode, tag_dict=None, public_ipv4_pool=None):
    """ Allocate a new elastic IP address (when needed) and return it """
    if reuse_existing_ip_allowed:
-        domain_filter = {'domain': domain or 'standard'}
+        filters = []
+        if not domain:
+            domain = 'standard'
+        filters.append({'Name': 'domain', "Values": [domain]})

        if tag_dict is not None:
-            domain_filter.update(tag_dict)
+            filters += ansible_dict_to_boto3_filter_list(tag_dict)
+
+        try:
+            all_addresses = ec2.describe_addresses(Filters=filters, aws_retry=True)
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't obtain list of existing Elastic IP addresses")

-        all_addresses = ec2.get_all_addresses(filters=domain_filter)
+        all_addresses = all_addresses["Addresses"]

        if domain == 'vpc':
            unassociated_addresses = [a for a in all_addresses
-                                      if not a.association_id]
+                                      if not a.get('AssociationId', None)]
        else:
            unassociated_addresses = [a for a in all_addresses
-                                      if not a.instance_id]
+                                      if not a['InstanceId']]
        if unassociated_addresses:
            return unassociated_addresses[0], False

    if public_ipv4_pool:
-        return allocate_address_from_pool(ec2, domain, check_mode, public_ipv4_pool), True
+        return allocate_address_from_pool(ec2, module, domain, check_mode, public_ipv4_pool), True

-    return ec2.allocate_address(domain=domain), True
+    try:
+        result = ec2.allocate_address(Domain=domain, aws_retry=True), True
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+        module.fail_json_aws(e, msg="Couldn't allocate Elastic IP address")
+    return result


-def release_address(ec2, address, check_mode):
+def release_address(ec2, module, address, check_mode):
    """ Release a previously allocated elastic IP address """

    # If we're in check mode, nothing else to do
    if not check_mode:
-        if not address.release():
-            raise EIPException('release failed')
+        try:
+            result = ec2.release_address(AllocationId=address['AllocationId'], aws_retry=True)
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't release Elastic IP address")

    return {'changed': True}


+@AWSRetry.jittered_backoff()
+def describe_eni_with_backoff(ec2, module, device_id):
+    try:
+        return ec2.describe_network_interfaces(NetworkInterfaceIds=[device_id])
+    except is_boto3_error_code('InvalidNetworkInterfaceID.NotFound') as e:
+        module.fail_json_aws(e, msg="Couldn't get list of network interfaces.")
+
+
 def find_device(ec2, module, device_id, is_instance=True):
    """ Attempt to find the EC2 instance and return it """

    if is_instance:
        try:
-            reservations = ec2.get_all_reservations(instance_ids=[device_id])
-        except boto.exception.EC2ResponseError as e:
-            module.fail_json(msg=str(e))
+            paginator = ec2.get_paginator('describe_instances')
+            reservations = list(paginator.paginate(InstanceIds=[device_id]).search('Reservations[]'))
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't get list of instances")

        if len(reservations) == 1:
-            instances = reservations[0].instances
+            instances = reservations[0]['Instances']
            if len(instances) == 1:
                return instances[0]
    else:
        try:
-            interfaces = ec2.get_all_network_interfaces(network_interface_ids=[device_id])
-        except boto.exception.EC2ResponseError as e:
-            module.fail_json(msg=str(e))
-
+            interfaces = describe_eni_with_backoff(ec2, module, device_id)
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't get list of network interfaces.")
        if len(interfaces) == 1:
            return interfaces[0]

-    raise EIPException("could not find instance" + device_id)
-

 def ensure_present(ec2, module, domain, address, private_ip_address, device_id,
                   reuse_existing_ip_allowed, allow_reassociation, check_mode, is_instance=True):
@ -390,7 +430,7 @@ def ensure_present(ec2, module, domain, address, private_ip_address, device_id,
        if check_mode:
            return {'changed': True}

-        address, changed = allocate_address(ec2, domain, reuse_existing_ip_allowed, check_mode)
+        address, changed = allocate_address(ec2, module, domain, reuse_existing_ip_allowed, check_mode)

    if device_id:
        # Allocate an IP for instance since no public_ip was provided
@ -398,42 +438,47 @@ def ensure_present(ec2, module, domain, address, private_ip_address, device_id,
            instance = find_device(ec2, module, device_id)
            if reuse_existing_ip_allowed:
                if instance.vpc_id and len(instance.vpc_id) > 0 and domain is None:
-                    raise EIPException("You must set 'in_vpc' to true to associate an instance with an existing ip in a vpc")
+                    msg = "You must set 'in_vpc' to true to associate an instance with an existing ip in a vpc"
+                    module.fail_json_aws(botocore.exceptions.ClientError, msg=msg)
+
            # Associate address object (provided or allocated) with instance
-            assoc_result = associate_ip_and_device(ec2, address, private_ip_address, device_id, allow_reassociation,
-                                                   check_mode)
+            assoc_result = associate_ip_and_device(
+                ec2, module, address, private_ip_address, device_id, allow_reassociation,
+                check_mode
+            )
        else:
            instance = find_device(ec2, module, device_id, is_instance=False)
            # Associate address object (provided or allocated) with instance
-            assoc_result = associate_ip_and_device(ec2, address, private_ip_address, device_id, allow_reassociation,
-                                                   check_mode, is_instance=False)
-
-        if instance.vpc_id:
-            domain = 'vpc'
+            assoc_result = associate_ip_and_device(
+                ec2, module, address, private_ip_address, device_id, allow_reassociation,
+                check_mode, is_instance=False
+            )

        changed = changed or assoc_result['changed']

-    return {'changed': changed, 'public_ip': address.public_ip, 'allocation_id': address.allocation_id}
+    return {'changed': changed, 'public_ip': address['PublicIp'], 'allocation_id': address['AllocationId']}


-def ensure_absent(ec2, address, device_id, check_mode, is_instance=True):
+def ensure_absent(ec2, module, address, device_id, check_mode, is_instance=True):
    if not address:
        return {'changed': False}

    # disassociating address from instance
    if device_id:
        if is_instance:
-            return disassociate_ip_and_device(ec2, address, device_id,
-                                              check_mode)
+            return disassociate_ip_and_device(
+                ec2, module, address, device_id, check_mode
+            )
        else:
-            return disassociate_ip_and_device(ec2, address, device_id,
-                                              check_mode, is_instance=False)
+            return disassociate_ip_and_device(
+                ec2, module, address, device_id, check_mode, is_instance=False
+            )
    # releasing address
    else:
-        return release_address(ec2, address, check_mode)
+        return release_address(ec2, module, address, check_mode)


-def allocate_address_from_pool(ec2, domain, check_mode, public_ipv4_pool):
+def allocate_address_from_pool(ec2, module, domain, check_mode, public_ipv4_pool):
    # type: (EC2Connection, str, bool, str) -> Address
    """ Overrides boto's allocate_address function to support BYOIP """
    params = {}
@ -442,13 +487,16 @@ def allocate_address_from_pool(ec2, domain, check_mode, public_ipv4_pool):
        params['Domain'] = domain

    if public_ipv4_pool is not None:
-        ec2.APIVersion = "2016-11-15"  # Workaround to force amazon to accept this attribute
        params['PublicIpv4Pool'] = public_ipv4_pool

    if check_mode:
        params['DryRun'] = 'true'

-    return ec2.get_object('AllocateAddress', params, Address, verb='POST')
+    try:
+        result = ec2.allocate_address(aws_retry=True, **params)
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+        module.fail_json_aws(e, msg="Couldn't allocate Elastic IP address")
+    return result


 def generate_tag_dict(module, tag_name, tag_value):
@ -487,7 +535,7 @@ def main():
        public_ipv4_pool=dict()
    ))

-    module = AnsibleModule(
+    module = AnsibleAWSModule(
        argument_spec=argument_spec,
        supports_check_mode=True,
        required_by={
@ -495,10 +543,7 @@ def main():
        },
    )

-    if not HAS_BOTO:
-        module.fail_json(msg='boto required for this module')
-
-    ec2 = ec2_connect(module)
+    ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff())

    device_id = module.params.get('device_id')
    instance_id = module.params.get('instance_id')
@ -530,36 +575,59 @@ def main():

    try:
        if device_id:
-            address = find_address(ec2, public_ip, device_id, is_instance=is_instance)
+            address = find_address(ec2, module, public_ip, device_id, is_instance=is_instance)
        else:
-            address = find_address(ec2, public_ip, None)
+            address = find_address(ec2, module, public_ip, None)

        if state == 'present':
            if device_id:
-                result = ensure_present(ec2, module, domain, address, private_ip_address, device_id,
+                result = ensure_present(
+                    ec2, module, domain, address, private_ip_address, device_id,
                    reuse_existing_ip_allowed, allow_reassociation,
-                                        module.check_mode, is_instance=is_instance)
+                    module.check_mode, is_instance=is_instance
+                )
            else:
                if address:
                    changed = False
                else:
-                    address, changed = allocate_address(ec2, domain, reuse_existing_ip_allowed, module.check_mode, tag_dict, public_ipv4_pool)
-                result = {'changed': changed, 'public_ip': address.public_ip, 'allocation_id': address.allocation_id}
+                    address, changed = allocate_address(
+                        ec2, module, domain, reuse_existing_ip_allowed,
+                        module.check_mode, tag_dict, public_ipv4_pool
+                    )
+                result = {
+                    'changed': changed,
+                    'public_ip': address['PublicIp'],
+                    'allocation_id': address['AllocationId']
+                }
        else:
            if device_id:
-                disassociated = ensure_absent(ec2, address, device_id, module.check_mode, is_instance=is_instance)
+                disassociated = ensure_absent(
+                    ec2, module, address, device_id, module.check_mode, is_instance=is_instance
+                )

                if release_on_disassociation and disassociated['changed']:
-                    released = release_address(ec2, address, module.check_mode)
-                    result = {'changed': True, 'disassociated': disassociated, 'released': released}
+                    released = release_address(ec2, module, address, module.check_mode)
+                    result = {
+                        'changed': True,
+                        'disassociated': disassociated,
+                        'released': released
+                    }
                else:
-                    result = {'changed': disassociated['changed'], 'disassociated': disassociated, 'released': {'changed': False}}
+                    result = {
+                        'changed': disassociated['changed'],
+                        'disassociated': disassociated,
+                        'released': {'changed': False}
+                    }
            else:
-                released = release_address(ec2, address, module.check_mode)
-                result = {'changed': released['changed'], 'disassociated': {'changed': False}, 'released': released}
-
-    except (boto.exception.EC2ResponseError, EIPException) as e:
-        module.fail_json(msg=str(e))
+                released = release_address(ec2, module, address, module.check_mode)
+                result = {
+                    'changed': released['changed'],
+                    'disassociated': {'changed': False},
+                    'released': released
+                }
+
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+        module.fail_json_aws(str(e))

    if instance_id:
        result['warnings'] = warnings
--- a/test/integration/targets/ec2_eip/tasks/main.yml
+++ b/test/integration/targets/ec2_eip/tasks/main.yml
@ -1,7 +1,6 @@
 ---
-    - name: Integration testing for ec2_eip
+- name: Integration testing for ec2_eip
  block:
-
    - name: set up aws connection info
      set_fact:
        aws_connection_info: &aws_connection_info
@ -117,9 +116,9 @@
          - pool_eip is changed
          - pool_eip.public_ip is defined and pool_eip.public_ip != ""
          - pool_eip.allocation_id is defined and pool_eip.allocation_id != ""
-
  always:
-        - debug: msg="{{ item }}"
+    - debug:
+        msg: "{{ item }}"
      when: item is defined and item.public_ip is defined and item.allocation_id is defined
      loop:
        - eip