From 49f569d915d196afd36072ada55e29387413662c Mon Sep 17 00:00:00 2001 From: Rob Date: Fri, 25 May 2018 01:35:24 +1000 Subject: [PATCH] New module: AWS Glue connection (#39492) * New module = AWS Glue connection * Add a few initial integration tests * Add alias for CI * module rename * finish module rename * add loop when getting glue connection again so we dont get None * Limit number of retries to get new glue connection info --- .../cloud/amazon/aws_glue_connection.py | 329 ++++++++++++++++++ .../targets/aws_glue_connection/aliases | 2 + .../aws_glue_connection/tasks/main.yml | 87 +++++ 3 files changed, 418 insertions(+) create mode 100644 lib/ansible/modules/cloud/amazon/aws_glue_connection.py create mode 100644 test/integration/targets/aws_glue_connection/aliases create mode 100644 test/integration/targets/aws_glue_connection/tasks/main.yml diff --git a/lib/ansible/modules/cloud/amazon/aws_glue_connection.py b/lib/ansible/modules/cloud/amazon/aws_glue_connection.py new file mode 100644 index 00000000000..895d1928285 --- /dev/null +++ b/lib/ansible/modules/cloud/amazon/aws_glue_connection.py @@ -0,0 +1,329 @@ +#!/usr/bin/python + +# Copyright: (c) 2018, Rob White (@wimnat) +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = ''' +--- +module: aws_glue_connection +short_description: Manage an AWS Glue connection +description: + - Manage an AWS Glue connection. See U(https://aws.amazon.com/glue/) for details. +version_added: "2.6" +requirements: [ boto3 ] +author: "Rob White (@wimnat)" +options: + catalog_id: + description: + - The ID of the Data Catalog in which to create the connection. If none is supplied, + the AWS account ID is used by default. + required: false + connection_properties: + description: + - A dict of key-value pairs used as parameters for this connection. + required: true + connection_type: + description: + - The type of the connection. Currently, only JDBC is supported; SFTP is not supported. + required: false + default: JDBC + choices: [ 'JDBC', 'SFTP' ] + description: + description: + - The description of the connection. + required: false + match_criteria: + description: + - A list of UTF-8 strings that specify the criteria that you can use in selecting this connection. + required: false + name: + description: + - The name of the connection. + required: true + security_groups: + description: + - A list of security groups to be used by the connection. Use either security group name or ID. + required: false + state: + description: + - Create or delete the AWS Glue connection. + required: true + choices: [ 'present', 'absent' ] + subnet_id: + description: + - The subnet ID used by the connection. + required: false +extends_documentation_fragment: + - aws + - ec2 +''' + +EXAMPLES = ''' +# Note: These examples do not set authentication details, see the AWS Guide for details. + +# Create an AWS Glue connection +- aws_glue_connection: + name: my-glue-connection + connection_properties: + JDBC_CONNECTION_URL: jdbc:mysql://mydb:3306/databasename + USERNAME: my-username + PASSWORD: my-password + state: present + +# Delete an AWS Glue connection +- aws_glue_connection: + name: my-glue-connection + state: absent + +''' + +RETURN = ''' +connection_properties: + description: A dict of key-value pairs used as parameters for this connection. + returned: when state is present + type: dict + sample: {'JDBC_CONNECTION_URL':'jdbc:mysql://mydb:3306/databasename','USERNAME':'x','PASSWORD':'y'} +connection_type: + description: The type of the connection. + returned: when state is present + type: string + sample: JDBC +creation_time: + description: The time this connection definition was created. + returned: when state is present + type: string + sample: "2018-04-21T05:19:58.326000+00:00" +description: + description: Description of the job being defined. + returned: when state is present + type: string + sample: My first Glue job +last_updated_time: + description: The last time this connection definition was updated. + returned: when state is present + type: string + sample: "2018-04-21T05:19:58.326000+00:00" +match_criteria: + description: A list of criteria that can be used in selecting this connection. + returned: when state is present + type: list + sample: [] +name: + description: The name of the connection definition. + returned: when state is present + type: string + sample: my-glue-connection +physical_connection_requirements: + description: A dict of physical connection requirements, such as VPC and SecurityGroup, + needed for making this connection successfully. + returned: when state is present + type: dict + sample: {'subnet-id':'subnet-aabbccddee'} +''' + +from ansible.module_utils.aws.core import AnsibleAWSModule +from ansible.module_utils.ec2 import camel_dict_to_snake_dict, get_ec2_security_group_ids_from_names + +# Non-ansible imports +import copy +import time +try: + from botocore.exceptions import BotoCoreError, ClientError +except ImportError: + pass + + +def _get_glue_connection(connection, module): + """ + Get an AWS Glue connection based on name. If not found, return None. + + :param connection: AWS boto3 glue connection + :param module: Ansible module + :return: boto3 Glue connection dict or None if not found + """ + + connection_name = module.params.get("name") + connection_catalog_id = module.params.get("catalog_id") + + params = {'Name': connection_name} + if connection_catalog_id is not None: + params['CatalogId'] = connection_catalog_id + + try: + return connection.get_connection(**params)['Connection'] + except (BotoCoreError, ClientError) as e: + if e.response['Error']['Code'] == 'EntityNotFoundException': + return None + else: + raise e + + +def _compare_glue_connection_params(user_params, current_params): + """ + Compare Glue connection params. If there is a difference, return True immediately else return False + + :param user_params: the Glue connection parameters passed by the user + :param current_params: the Glue connection parameters currently configured + :return: True if any parameter is mismatched else False + """ + + # Weirdly, boto3 doesn't return some keys if the value is empty e.g. Description + # To counter this, add the key if it's missing with a blank value + + if 'Description' not in current_params: + current_params['Description'] = "" + if 'MatchCriteria' not in current_params: + current_params['MatchCriteria'] = list() + if 'PhysicalConnectionRequirements' not in current_params: + current_params['PhysicalConnectionRequirements'] = dict() + current_params['PhysicalConnectionRequirements']['SecurityGroupIdList'] = [] + current_params['PhysicalConnectionRequirements']['SubnetId'] = "" + + if 'ConnectionProperties' in user_params['ConnectionInput'] and user_params['ConnectionInput']['ConnectionProperties'] \ + != current_params['ConnectionProperties']: + return True + if 'ConnectionType' in user_params['ConnectionInput'] and user_params['ConnectionInput']['ConnectionType'] \ + != current_params['ConnectionType']: + return True + if 'Description' in user_params['ConnectionInput'] and user_params['ConnectionInput']['Description'] != current_params['Description']: + return True + if 'MatchCriteria' in user_params['ConnectionInput'] and set(user_params['ConnectionInput']['MatchCriteria']) != set(current_params['MatchCriteria']): + return True + if 'PhysicalConnectionRequirements' in user_params['ConnectionInput']: + if 'SecurityGroupIdList' in user_params['ConnectionInput']['PhysicalConnectionRequirements'] and \ + set(user_params['ConnectionInput']['PhysicalConnectionRequirements']['SecurityGroupIdList']) \ + != set(current_params['PhysicalConnectionRequirements']['SecurityGroupIdList']): + return True + if 'SubnetId' in user_params['ConnectionInput']['PhysicalConnectionRequirements'] and \ + user_params['ConnectionInput']['PhysicalConnectionRequirements']['SubnetId'] \ + != current_params['PhysicalConnectionRequirements']['SubnetId']: + return True + + return False + + +def create_or_update_glue_connection(connection, connection_ec2, module, glue_connection): + """ + Create or update an AWS Glue connection + + :param connection: AWS boto3 glue connection + :param module: Ansible module + :param glue_connection: a dict of AWS Glue connection parameters or None + :return: + """ + + changed = False + params = dict() + params['ConnectionInput'] = dict() + params['ConnectionInput']['Name'] = module.params.get("name") + params['ConnectionInput']['ConnectionType'] = module.params.get("connection_type") + params['ConnectionInput']['ConnectionProperties'] = module.params.get("connection_properties") + if module.params.get("catalog_id") is not None: + params['CatalogId'] = module.params.get("catalog_id") + if module.params.get("description") is not None: + params['ConnectionInput']['Description'] = module.params.get("description") + if module.params.get("match_criteria") is not None: + params['ConnectionInput']['MatchCriteria'] = module.params.get("match_criteria") + if module.params.get("security_groups") is not None or module.params.get("subnet_id") is not None: + params['ConnectionInput']['PhysicalConnectionRequirements'] = dict() + if module.params.get("security_groups") is not None: + # Get security group IDs from names + security_group_ids = get_ec2_security_group_ids_from_names(module.params.get('security_groups'), connection_ec2, boto3=True) + params['ConnectionInput']['PhysicalConnectionRequirements']['SecurityGroupIdList'] = security_group_ids + if module.params.get("subnet_id") is not None: + params['ConnectionInput']['PhysicalConnectionRequirements']['SubnetId'] = module.params.get("subnet_id") + + # If glue_connection is not None then check if it needs to be modified, else create it + if glue_connection: + if _compare_glue_connection_params(params, glue_connection): + try: + # We need to slightly modify the params for an update + update_params = copy.deepcopy(params) + update_params['Name'] = update_params['ConnectionInput']['Name'] + connection.update_connection(**update_params) + changed = True + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) + else: + try: + connection.create_connection(**params) + changed = True + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) + + # If changed, get the Glue connection again + if changed: + glue_connection = None + for i in range(10): + glue_connection = _get_glue_connection(connection, module) + if glue_connection is not None: + break + time.sleep(10) + + module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_connection)) + + +def delete_glue_connection(connection, module, glue_connection): + """ + Delete an AWS Glue connection + + :param connection: AWS boto3 glue connection + :param module: Ansible module + :param glue_connection: a dict of AWS Glue connection parameters or None + :return: + """ + + changed = False + + params = {'ConnectionName': module.params.get("name")} + if module.params.get("catalog_id") is not None: + params['CatalogId'] = module.params.get("catalog_id") + + if glue_connection: + try: + connection.delete_connection(**params) + changed = True + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) + + module.exit_json(changed=changed) + + +def main(): + + argument_spec = ( + dict( + catalog_id=dict(type='str'), + connection_properties=dict(type='dict'), + connection_type=dict(type='str', default='JDBC', choices=['JDBC', 'SFTP']), + description=dict(type='str'), + match_criteria=dict(type='list'), + name=dict(required=True, type='str'), + security_groups=dict(type='list'), + state=dict(required=True, choices=['present', 'absent'], type='str'), + subnet_id=dict(type='str') + ) + ) + + module = AnsibleAWSModule(argument_spec=argument_spec, + required_if=[ + ('state', 'present', ['connection_properties']) + ] + ) + + connection_glue = module.client('glue') + connection_ec2 = module.client('ec2') + + glue_connection = _get_glue_connection(connection_glue, module) + + if module.params.get("state") == 'present': + create_or_update_glue_connection(connection_glue, connection_ec2, module, glue_connection) + else: + delete_glue_connection(connection_glue, module, glue_connection) + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/aws_glue_connection/aliases b/test/integration/targets/aws_glue_connection/aliases new file mode 100644 index 00000000000..d6ae2f116bc --- /dev/null +++ b/test/integration/targets/aws_glue_connection/aliases @@ -0,0 +1,2 @@ +cloud/aws +posix/ci/cloud/group4/aws diff --git a/test/integration/targets/aws_glue_connection/tasks/main.yml b/test/integration/targets/aws_glue_connection/tasks/main.yml new file mode 100644 index 00000000000..84ca6e5eff0 --- /dev/null +++ b/test/integration/targets/aws_glue_connection/tasks/main.yml @@ -0,0 +1,87 @@ +- block: + + # TODO: description, match_criteria, security_groups, and subnet_id are unused module options + + - name: set up aws connection info + set_fact: + aws_connection_info: &aws_connection_info + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token }}" + region: "{{ aws_region }}" + no_log: yes + + - name: create glue connection + aws_glue_connection: + name: "{{ resource_prefix }}" + connection_properties: + JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}" + USERNAME: my-username + PASSWORD: my-password + state: present + <<: *aws_connection_info + register: result + + - assert: + that: + - result.changed + + - name: test idempotence creating glue connection + aws_glue_connection: + name: "{{ resource_prefix }}" + connection_properties: + JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}" + USERNAME: my-username + PASSWORD: my-password + state: present + <<: *aws_connection_info + register: result + + - assert: + that: + - not result.changed + + - name: test updating JDBC connection url + aws_glue_connection: + name: "{{ resource_prefix }}" + connection_properties: + JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}-updated" + USERNAME: my-username + PASSWORD: my-password + state: present + <<: *aws_connection_info + register: result + + - assert: + that: + - result.changed + + - name: delete glue connection + aws_glue_connection: + name: "{{ resource_prefix }}" + state: absent + <<: *aws_connection_info + register: result + + - assert: + that: + - result.changed + + - name: test idempotence removing glue connection + aws_glue_connection: + name: "{{ resource_prefix }}" + state: absent + <<: *aws_connection_info + register: result + + - assert: + that: + - not result.changed + + always: + + - name: delete glue connection + aws_glue_connection: + name: "{{ resource_prefix }}" + state: absent + <<: *aws_connection_info