mirror of https://github.com/ansible/ansible.git
New module: AWS Glue connection (#39492)
* New module = AWS Glue connection * Add a few initial integration tests * Add alias for CI * module rename * finish module rename * add loop when getting glue connection again so we dont get None * Limit number of retries to get new glue connection infopull/40703/head
Rob
6 years ago
committed by
Ryan Brown
parent
ba4680c141
commit
49f569d915
@ -0,0 +1,329 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
# Copyright: (c) 2018, Rob White (@wimnat)
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
ANSIBLE_METADATA = {'metadata_version': '1.1',
|
||||
'status': ['preview'],
|
||||
'supported_by': 'community'}
|
||||
|
||||
DOCUMENTATION = '''
|
||||
---
|
||||
module: aws_glue_connection
|
||||
short_description: Manage an AWS Glue connection
|
||||
description:
|
||||
- Manage an AWS Glue connection. See U(https://aws.amazon.com/glue/) for details.
|
||||
version_added: "2.6"
|
||||
requirements: [ boto3 ]
|
||||
author: "Rob White (@wimnat)"
|
||||
options:
|
||||
catalog_id:
|
||||
description:
|
||||
- The ID of the Data Catalog in which to create the connection. If none is supplied,
|
||||
the AWS account ID is used by default.
|
||||
required: false
|
||||
connection_properties:
|
||||
description:
|
||||
- A dict of key-value pairs used as parameters for this connection.
|
||||
required: true
|
||||
connection_type:
|
||||
description:
|
||||
- The type of the connection. Currently, only JDBC is supported; SFTP is not supported.
|
||||
required: false
|
||||
default: JDBC
|
||||
choices: [ 'JDBC', 'SFTP' ]
|
||||
description:
|
||||
description:
|
||||
- The description of the connection.
|
||||
required: false
|
||||
match_criteria:
|
||||
description:
|
||||
- A list of UTF-8 strings that specify the criteria that you can use in selecting this connection.
|
||||
required: false
|
||||
name:
|
||||
description:
|
||||
- The name of the connection.
|
||||
required: true
|
||||
security_groups:
|
||||
description:
|
||||
- A list of security groups to be used by the connection. Use either security group name or ID.
|
||||
required: false
|
||||
state:
|
||||
description:
|
||||
- Create or delete the AWS Glue connection.
|
||||
required: true
|
||||
choices: [ 'present', 'absent' ]
|
||||
subnet_id:
|
||||
description:
|
||||
- The subnet ID used by the connection.
|
||||
required: false
|
||||
extends_documentation_fragment:
|
||||
- aws
|
||||
- ec2
|
||||
'''
|
||||
|
||||
EXAMPLES = '''
|
||||
# Note: These examples do not set authentication details, see the AWS Guide for details.
|
||||
|
||||
# Create an AWS Glue connection
|
||||
- aws_glue_connection:
|
||||
name: my-glue-connection
|
||||
connection_properties:
|
||||
JDBC_CONNECTION_URL: jdbc:mysql://mydb:3306/databasename
|
||||
USERNAME: my-username
|
||||
PASSWORD: my-password
|
||||
state: present
|
||||
|
||||
# Delete an AWS Glue connection
|
||||
- aws_glue_connection:
|
||||
name: my-glue-connection
|
||||
state: absent
|
||||
|
||||
'''
|
||||
|
||||
RETURN = '''
|
||||
connection_properties:
|
||||
description: A dict of key-value pairs used as parameters for this connection.
|
||||
returned: when state is present
|
||||
type: dict
|
||||
sample: {'JDBC_CONNECTION_URL':'jdbc:mysql://mydb:3306/databasename','USERNAME':'x','PASSWORD':'y'}
|
||||
connection_type:
|
||||
description: The type of the connection.
|
||||
returned: when state is present
|
||||
type: string
|
||||
sample: JDBC
|
||||
creation_time:
|
||||
description: The time this connection definition was created.
|
||||
returned: when state is present
|
||||
type: string
|
||||
sample: "2018-04-21T05:19:58.326000+00:00"
|
||||
description:
|
||||
description: Description of the job being defined.
|
||||
returned: when state is present
|
||||
type: string
|
||||
sample: My first Glue job
|
||||
last_updated_time:
|
||||
description: The last time this connection definition was updated.
|
||||
returned: when state is present
|
||||
type: string
|
||||
sample: "2018-04-21T05:19:58.326000+00:00"
|
||||
match_criteria:
|
||||
description: A list of criteria that can be used in selecting this connection.
|
||||
returned: when state is present
|
||||
type: list
|
||||
sample: []
|
||||
name:
|
||||
description: The name of the connection definition.
|
||||
returned: when state is present
|
||||
type: string
|
||||
sample: my-glue-connection
|
||||
physical_connection_requirements:
|
||||
description: A dict of physical connection requirements, such as VPC and SecurityGroup,
|
||||
needed for making this connection successfully.
|
||||
returned: when state is present
|
||||
type: dict
|
||||
sample: {'subnet-id':'subnet-aabbccddee'}
|
||||
'''
|
||||
|
||||
from ansible.module_utils.aws.core import AnsibleAWSModule
|
||||
from ansible.module_utils.ec2 import camel_dict_to_snake_dict, get_ec2_security_group_ids_from_names
|
||||
|
||||
# Non-ansible imports
|
||||
import copy
|
||||
import time
|
||||
try:
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def _get_glue_connection(connection, module):
|
||||
"""
|
||||
Get an AWS Glue connection based on name. If not found, return None.
|
||||
|
||||
:param connection: AWS boto3 glue connection
|
||||
:param module: Ansible module
|
||||
:return: boto3 Glue connection dict or None if not found
|
||||
"""
|
||||
|
||||
connection_name = module.params.get("name")
|
||||
connection_catalog_id = module.params.get("catalog_id")
|
||||
|
||||
params = {'Name': connection_name}
|
||||
if connection_catalog_id is not None:
|
||||
params['CatalogId'] = connection_catalog_id
|
||||
|
||||
try:
|
||||
return connection.get_connection(**params)['Connection']
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
if e.response['Error']['Code'] == 'EntityNotFoundException':
|
||||
return None
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
||||
def _compare_glue_connection_params(user_params, current_params):
|
||||
"""
|
||||
Compare Glue connection params. If there is a difference, return True immediately else return False
|
||||
|
||||
:param user_params: the Glue connection parameters passed by the user
|
||||
:param current_params: the Glue connection parameters currently configured
|
||||
:return: True if any parameter is mismatched else False
|
||||
"""
|
||||
|
||||
# Weirdly, boto3 doesn't return some keys if the value is empty e.g. Description
|
||||
# To counter this, add the key if it's missing with a blank value
|
||||
|
||||
if 'Description' not in current_params:
|
||||
current_params['Description'] = ""
|
||||
if 'MatchCriteria' not in current_params:
|
||||
current_params['MatchCriteria'] = list()
|
||||
if 'PhysicalConnectionRequirements' not in current_params:
|
||||
current_params['PhysicalConnectionRequirements'] = dict()
|
||||
current_params['PhysicalConnectionRequirements']['SecurityGroupIdList'] = []
|
||||
current_params['PhysicalConnectionRequirements']['SubnetId'] = ""
|
||||
|
||||
if 'ConnectionProperties' in user_params['ConnectionInput'] and user_params['ConnectionInput']['ConnectionProperties'] \
|
||||
!= current_params['ConnectionProperties']:
|
||||
return True
|
||||
if 'ConnectionType' in user_params['ConnectionInput'] and user_params['ConnectionInput']['ConnectionType'] \
|
||||
!= current_params['ConnectionType']:
|
||||
return True
|
||||
if 'Description' in user_params['ConnectionInput'] and user_params['ConnectionInput']['Description'] != current_params['Description']:
|
||||
return True
|
||||
if 'MatchCriteria' in user_params['ConnectionInput'] and set(user_params['ConnectionInput']['MatchCriteria']) != set(current_params['MatchCriteria']):
|
||||
return True
|
||||
if 'PhysicalConnectionRequirements' in user_params['ConnectionInput']:
|
||||
if 'SecurityGroupIdList' in user_params['ConnectionInput']['PhysicalConnectionRequirements'] and \
|
||||
set(user_params['ConnectionInput']['PhysicalConnectionRequirements']['SecurityGroupIdList']) \
|
||||
!= set(current_params['PhysicalConnectionRequirements']['SecurityGroupIdList']):
|
||||
return True
|
||||
if 'SubnetId' in user_params['ConnectionInput']['PhysicalConnectionRequirements'] and \
|
||||
user_params['ConnectionInput']['PhysicalConnectionRequirements']['SubnetId'] \
|
||||
!= current_params['PhysicalConnectionRequirements']['SubnetId']:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def create_or_update_glue_connection(connection, connection_ec2, module, glue_connection):
|
||||
"""
|
||||
Create or update an AWS Glue connection
|
||||
|
||||
:param connection: AWS boto3 glue connection
|
||||
:param module: Ansible module
|
||||
:param glue_connection: a dict of AWS Glue connection parameters or None
|
||||
:return:
|
||||
"""
|
||||
|
||||
changed = False
|
||||
params = dict()
|
||||
params['ConnectionInput'] = dict()
|
||||
params['ConnectionInput']['Name'] = module.params.get("name")
|
||||
params['ConnectionInput']['ConnectionType'] = module.params.get("connection_type")
|
||||
params['ConnectionInput']['ConnectionProperties'] = module.params.get("connection_properties")
|
||||
if module.params.get("catalog_id") is not None:
|
||||
params['CatalogId'] = module.params.get("catalog_id")
|
||||
if module.params.get("description") is not None:
|
||||
params['ConnectionInput']['Description'] = module.params.get("description")
|
||||
if module.params.get("match_criteria") is not None:
|
||||
params['ConnectionInput']['MatchCriteria'] = module.params.get("match_criteria")
|
||||
if module.params.get("security_groups") is not None or module.params.get("subnet_id") is not None:
|
||||
params['ConnectionInput']['PhysicalConnectionRequirements'] = dict()
|
||||
if module.params.get("security_groups") is not None:
|
||||
# Get security group IDs from names
|
||||
security_group_ids = get_ec2_security_group_ids_from_names(module.params.get('security_groups'), connection_ec2, boto3=True)
|
||||
params['ConnectionInput']['PhysicalConnectionRequirements']['SecurityGroupIdList'] = security_group_ids
|
||||
if module.params.get("subnet_id") is not None:
|
||||
params['ConnectionInput']['PhysicalConnectionRequirements']['SubnetId'] = module.params.get("subnet_id")
|
||||
|
||||
# If glue_connection is not None then check if it needs to be modified, else create it
|
||||
if glue_connection:
|
||||
if _compare_glue_connection_params(params, glue_connection):
|
||||
try:
|
||||
# We need to slightly modify the params for an update
|
||||
update_params = copy.deepcopy(params)
|
||||
update_params['Name'] = update_params['ConnectionInput']['Name']
|
||||
connection.update_connection(**update_params)
|
||||
changed = True
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
module.fail_json_aws(e)
|
||||
else:
|
||||
try:
|
||||
connection.create_connection(**params)
|
||||
changed = True
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
module.fail_json_aws(e)
|
||||
|
||||
# If changed, get the Glue connection again
|
||||
if changed:
|
||||
glue_connection = None
|
||||
for i in range(10):
|
||||
glue_connection = _get_glue_connection(connection, module)
|
||||
if glue_connection is not None:
|
||||
break
|
||||
time.sleep(10)
|
||||
|
||||
module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_connection))
|
||||
|
||||
|
||||
def delete_glue_connection(connection, module, glue_connection):
|
||||
"""
|
||||
Delete an AWS Glue connection
|
||||
|
||||
:param connection: AWS boto3 glue connection
|
||||
:param module: Ansible module
|
||||
:param glue_connection: a dict of AWS Glue connection parameters or None
|
||||
:return:
|
||||
"""
|
||||
|
||||
changed = False
|
||||
|
||||
params = {'ConnectionName': module.params.get("name")}
|
||||
if module.params.get("catalog_id") is not None:
|
||||
params['CatalogId'] = module.params.get("catalog_id")
|
||||
|
||||
if glue_connection:
|
||||
try:
|
||||
connection.delete_connection(**params)
|
||||
changed = True
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
module.fail_json_aws(e)
|
||||
|
||||
module.exit_json(changed=changed)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
argument_spec = (
|
||||
dict(
|
||||
catalog_id=dict(type='str'),
|
||||
connection_properties=dict(type='dict'),
|
||||
connection_type=dict(type='str', default='JDBC', choices=['JDBC', 'SFTP']),
|
||||
description=dict(type='str'),
|
||||
match_criteria=dict(type='list'),
|
||||
name=dict(required=True, type='str'),
|
||||
security_groups=dict(type='list'),
|
||||
state=dict(required=True, choices=['present', 'absent'], type='str'),
|
||||
subnet_id=dict(type='str')
|
||||
)
|
||||
)
|
||||
|
||||
module = AnsibleAWSModule(argument_spec=argument_spec,
|
||||
required_if=[
|
||||
('state', 'present', ['connection_properties'])
|
||||
]
|
||||
)
|
||||
|
||||
connection_glue = module.client('glue')
|
||||
connection_ec2 = module.client('ec2')
|
||||
|
||||
glue_connection = _get_glue_connection(connection_glue, module)
|
||||
|
||||
if module.params.get("state") == 'present':
|
||||
create_or_update_glue_connection(connection_glue, connection_ec2, module, glue_connection)
|
||||
else:
|
||||
delete_glue_connection(connection_glue, module, glue_connection)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,2 @@
|
||||
cloud/aws
|
||||
posix/ci/cloud/group4/aws
|
||||
@ -0,0 +1,87 @@
|
||||
- block:
|
||||
|
||||
# TODO: description, match_criteria, security_groups, and subnet_id are unused module options
|
||||
|
||||
- name: set up aws connection info
|
||||
set_fact:
|
||||
aws_connection_info: &aws_connection_info
|
||||
aws_access_key: "{{ aws_access_key }}"
|
||||
aws_secret_key: "{{ aws_secret_key }}"
|
||||
security_token: "{{ security_token }}"
|
||||
region: "{{ aws_region }}"
|
||||
no_log: yes
|
||||
|
||||
- name: create glue connection
|
||||
aws_glue_connection:
|
||||
name: "{{ resource_prefix }}"
|
||||
connection_properties:
|
||||
JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}"
|
||||
USERNAME: my-username
|
||||
PASSWORD: my-password
|
||||
state: present
|
||||
<<: *aws_connection_info
|
||||
register: result
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- result.changed
|
||||
|
||||
- name: test idempotence creating glue connection
|
||||
aws_glue_connection:
|
||||
name: "{{ resource_prefix }}"
|
||||
connection_properties:
|
||||
JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}"
|
||||
USERNAME: my-username
|
||||
PASSWORD: my-password
|
||||
state: present
|
||||
<<: *aws_connection_info
|
||||
register: result
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- not result.changed
|
||||
|
||||
- name: test updating JDBC connection url
|
||||
aws_glue_connection:
|
||||
name: "{{ resource_prefix }}"
|
||||
connection_properties:
|
||||
JDBC_CONNECTION_URL: "jdbc:mysql://mydb:3306/{{ resource_prefix }}-updated"
|
||||
USERNAME: my-username
|
||||
PASSWORD: my-password
|
||||
state: present
|
||||
<<: *aws_connection_info
|
||||
register: result
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- result.changed
|
||||
|
||||
- name: delete glue connection
|
||||
aws_glue_connection:
|
||||
name: "{{ resource_prefix }}"
|
||||
state: absent
|
||||
<<: *aws_connection_info
|
||||
register: result
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- result.changed
|
||||
|
||||
- name: test idempotence removing glue connection
|
||||
aws_glue_connection:
|
||||
name: "{{ resource_prefix }}"
|
||||
state: absent
|
||||
<<: *aws_connection_info
|
||||
register: result
|
||||
|
||||
- assert:
|
||||
that:
|
||||
- not result.changed
|
||||
|
||||
always:
|
||||
|
||||
- name: delete glue connection
|
||||
aws_glue_connection:
|
||||
name: "{{ resource_prefix }}"
|
||||
state: absent
|
||||
<<: *aws_connection_info
|
||||
Loading…
Reference in New Issue