have elb_application_lb use modify_listeners to avoid removing/recreating them (#25650)

* Rework how listeners and rules and handled. Fixes #25270

* Tidy up, documentation and add rules to returned output

* Remove required=False from argument_spec

* Remove unused functions. Add or [] in case of no elb

* Handle when listners is None in ensure_listeners_default_action_has_arn
pull/25330/head
Rob 7 years ago committed by Will Thames
parent b980a5c02a
commit d0d2beafba

@ -62,10 +62,16 @@ options:
- The name of the load balancer. This name must be unique within your AWS account, can have a maximum of 32 characters, must contain only alphanumeric - The name of the load balancer. This name must be unique within your AWS account, can have a maximum of 32 characters, must contain only alphanumeric
characters or hyphens, and must not begin or end with a hyphen. characters or hyphens, and must not begin or end with a hyphen.
required: true required: true
purge_listeners:
description:
- If yes, existing listeners will be purged from the ELB to match exactly what is defined by I(listeners) parameter. If the I(listeners) parameter is
not set then listeners will not be modified
default: yes
choices: [ 'yes', 'no' ]
purge_tags: purge_tags:
description: description:
- If yes, existing tags will be purged from the resource to match exactly what is defined by tags parameter. If the tag parameter is not set then tags - If yes, existing tags will be purged from the resource to match exactly what is defined by I(tags) parameter. If the I(tags) parameter is not set then
will not be modified. tags will not be modified.
required: false required: false
default: yes default: yes
choices: [ 'yes', 'no' ] choices: [ 'yes', 'no' ]
@ -97,6 +103,9 @@ options:
extends_documentation_fragment: extends_documentation_fragment:
- aws - aws
- ec2 - ec2
notes:
- Listeners are matched based on port. If a listener's port is changed then a new listener will be created.
- Listener rules are matched based on priority. If a rule's priority is changed then a new rule will be created.
''' '''
EXAMPLES = ''' EXAMPLES = '''
@ -154,7 +163,7 @@ EXAMPLES = '''
- subnet-12345678 - subnet-12345678
- subnet-87654321 - subnet-87654321
security_groups: security_groups:
- sg-12345678 - sg-12345678
scheme: internal scheme: internal
listeners: listeners:
- Protocol: HTTPS - Protocol: HTTPS
@ -167,9 +176,9 @@ EXAMPLES = '''
SslPolicy: ELBSecurityPolicy-2015-05 SslPolicy: ELBSecurityPolicy-2015-05
Rules: Rules:
- Conditions: - Conditions:
- Field: path-pattern - Field: path-pattern
Values: Values:
- '/test' - '/test'
Priority: '1' Priority: '1'
Actions: Actions:
- TargetGroupName: test-target-group - TargetGroupName: test-target-group
@ -348,18 +357,7 @@ except ImportError:
HAS_BOTO3 = False HAS_BOTO3 = False
def convert(data): def convert_tg_name_to_arn(connection, module, tg_name):
if isinstance(data, string_types):
return str(data)
elif isinstance(data, collections.Mapping):
return dict(map(convert, data.items()))
elif isinstance(data, collections.Iterable):
return type(data)(map(convert, data))
else:
return data
def convert_tg_name_arn(connection, module, tg_name):
try: try:
response = connection.describe_target_groups(Names=[tg_name]) response = connection.describe_target_groups(Names=[tg_name])
@ -371,21 +369,9 @@ def convert_tg_name_arn(connection, module, tg_name):
return tg_arn return tg_arn
def convert_tg_arn_name(connection, module, tg_arn):
try:
response = connection.describe_target_groups(TargetGroupArns=[tg_arn])
except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
tg_name = response['TargetGroups'][0]['TargetGroupName']
return tg_name
def wait_for_status(connection, module, elb_arn, status): def wait_for_status(connection, module, elb_arn, status):
polling_increment_secs = 15 polling_increment_secs = 15
max_retries = (module.params.get('wait_timeout') / polling_increment_secs) max_retries = module.params.get('wait_timeout') / polling_increment_secs
status_achieved = False status_achieved = False
for x in range(0, max_retries): for x in range(0, max_retries):
@ -415,7 +401,8 @@ def _get_subnet_ids_from_subnet_list(subnet_list):
def get_elb_listeners(connection, module, elb_arn): def get_elb_listeners(connection, module, elb_arn):
try: try:
return connection.describe_listeners(LoadBalancerArn=elb_arn)['Listeners'] listener_paginator = connection.get_paginator('describe_listeners')
return (listener_paginator.paginate(LoadBalancerArn=elb_arn).build_full_result())['Listeners']
except ClientError as e: except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
@ -435,9 +422,36 @@ def get_elb_attributes(connection, module, elb_arn):
return elb_attributes return elb_attributes
def get_listener(connection, module, elb_arn, listener_port):
"""
Get a listener based on the port provided.
:param connection: ELBv2 boto3 connection
:param module: Ansible module object
:param listener_port:
:return:
"""
try:
listener_paginator = connection.get_paginator('describe_listeners')
listeners = (listener_paginator.paginate(LoadBalancerArn=elb_arn).build_full_result())['Listeners']
except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
l = None
for listener in listeners:
if listener['Port'] == listener_port:
l = listener
break
return l
def get_elb(connection, module): def get_elb(connection, module):
""" """
Get an application load balancer based on name. If not found, return None Get an application load balancer based on name. If not found, return None
:param connection: ELBv2 boto3 connection :param connection: ELBv2 boto3 connection
:param module: Ansible module object :param module: Ansible module object
:return: Dict of load balancer attributes or None if not found :return: Dict of load balancer attributes or None if not found
@ -453,173 +467,314 @@ def get_elb(connection, module):
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
def get_listener_rules(connection, module, listener_arn):
try:
return connection.describe_rules(ListenerArn=listener_arn)['Rules']
except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
def ensure_listeners_default_action_has_arn(connection, module, listeners):
"""
If a listener DefaultAction has been passed with a Target Group Name instead of ARN, lookup the ARN and
replace the name.
:param connection: ELBv2 boto3 connection
:param module: Ansible module object
:param listeners: a list of listener dicts
:return: the same list of dicts ensuring that each listener DefaultActions dict has TargetGroupArn key. If a TargetGroupName key exists, it is removed.
"""
if not listeners:
listeners = []
for listener in listeners:
if 'TargetGroupName' in listener['DefaultActions'][0]:
listener['DefaultActions'][0]['TargetGroupArn'] = convert_tg_name_to_arn(connection, module, listener['DefaultActions'][0]['TargetGroupName'])
del listener['DefaultActions'][0]['TargetGroupName']
return listeners
def ensure_rules_action_has_arn(connection, module, rules):
"""
If a rule Action has been passed with a Target Group Name instead of ARN, lookup the ARN and
replace the name.
:param connection: ELBv2 boto3 connection
:param module: Ansible module object
:param rules: a list of rule dicts
:return: the same list of dicts ensuring that each rule Actions dict has TargetGroupArn key. If a TargetGroupName key exists, it is removed.
"""
for rule in rules:
if 'TargetGroupName' in rule['Actions'][0]:
rule['Actions'][0]['TargetGroupArn'] = convert_tg_name_to_arn(connection, module, rule['Actions'][0]['TargetGroupName'])
del rule['Actions'][0]['TargetGroupName']
return rules
def compare_listener(current_listener, new_listener):
"""
Compare two listeners.
:param current_listener:
:param new_listener:
:return:
"""
modified_listener = {}
# Port
if current_listener['Port'] != new_listener['Port']:
modified_listener['Port'] = new_listener['Port']
# Protocol
if current_listener['Protocol'] != new_listener['Protocol']:
modified_listener['Protocol'] = new_listener['Protocol']
# If Protocol is HTTPS, check additional attributes
if current_listener['Protocol'] == 'HTTPS' and new_listener['Protocol'] == 'HTTPS':
# Cert
if current_listener['SslPolicy'] != new_listener['SslPolicy']:
modified_listener['SslPolicy'] = new_listener['SslPolicy']
if current_listener['Certificates'][0]['CertificateArn'] != new_listener['Certificates'][0]['CertificateArn']:
modified_listener['Certificates'][0]['CertificateArn'] = new_listener['Certificates'][0]['CertificateArn']
elif current_listener['Protocol'] != 'HTTPS' and new_listener['Protocol'] == 'HTTPS':
modified_listener['SslPolicy'] = new_listener['SslPolicy']
modified_listener['Certificates'][0]['CertificateArn'] = new_listener['Certificates'][0]['CertificateArn']
# Default action
# We wont worry about the Action Type because it is always 'forward'
if current_listener['DefaultActions'][0]['TargetGroupArn'] != new_listener['DefaultActions'][0]['TargetGroupArn']:
modified_listener['DefaultActions'] = []
modified_listener['DefaultActions'].append({})
modified_listener['DefaultActions'][0]['TargetGroupArn'] = new_listener['DefaultActions'][0]['TargetGroupArn']
modified_listener['DefaultActions'][0]['Type'] = 'forward'
if modified_listener:
return modified_listener
else:
return None
def compare_condition(current_conditions, condition):
"""
:param current_conditions:
:param condition:
:return:
"""
condition_found = False
for current_condition in current_conditions:
if current_condition['Field'] == condition['Field'] and current_condition['Values'][0] == condition['Values'][0]:
condition_found = True
break
return condition_found
def compare_rule(current_rule, new_rule):
"""
Compare two rules.
:param current_rule:
:param new_rule:
:return:
"""
modified_rule = {}
# Priority
if current_rule['Priority'] != new_rule['Priority']:
modified_rule['Priority'] = new_rule['Priority']
# Actions
# We wont worry about the Action Type because it is always 'forward'
if current_rule['Actions'][0]['TargetGroupArn'] != new_rule['Actions'][0]['TargetGroupArn']:
modified_rule['Actions'] = []
modified_rule['Actions'].append({})
modified_rule['Actions'][0]['TargetGroupArn'] = new_rule['Actions'][0]['TargetGroupArn']
modified_rule['Actions'][0]['Type'] = 'forward'
# Conditions
modified_conditions = []
for condition in new_rule['Conditions']:
if not compare_condition(current_rule['Conditions'], condition):
modified_conditions.append(condition)
if modified_conditions:
modified_rule['Conditions'] = modified_conditions
return modified_rule
def compare_listeners(connection, module, current_listeners, new_listeners, purge_listeners):
"""
Compare listeners and return listeners to add, listeners to modify and listeners to remove
Listeners are compared based on port
:param current_listeners:
:param new_listeners:
:param purge_listeners:
:return:
"""
listeners_to_modify = []
listeners_to_delete = []
# Check each current listener port to see if it's been passed to the module
for current_listener in current_listeners:
current_listener_passed_to_module = False
for new_listener in new_listeners[:]:
if current_listener['Port'] == new_listener['Port']:
current_listener_passed_to_module = True
# Remove what we match so that what is left can be marked as 'to be added'
new_listeners.remove(new_listener)
modified_listener = compare_listener(current_listener, new_listener)
if modified_listener:
modified_listener['Port'] = current_listener['Port']
modified_listener['ListenerArn'] = current_listener['ListenerArn']
listeners_to_modify.append(modified_listener)
break
# If the current listener was not matched against passed listeners and purge is True, mark for removal
if not current_listener_passed_to_module and purge_listeners:
listeners_to_delete.append(current_listener['ListenerArn'])
listeners_to_add = new_listeners
return listeners_to_add, listeners_to_modify, listeners_to_delete
def compare_rules(connection, module, current_listeners, listener):
"""
Compare rules and return rules to add, rules to modify and rules to remove
Rules are compared based on priority
:param connection:
:param module:
:param current_listeners:
:param listener:
:return:
"""
# Run through listeners looking for a match (by port) to get the ARN
for current_listener in current_listeners:
if current_listener['Port'] == listener['Port']:
listener['ListenerArn'] = current_listener['ListenerArn']
break
# Get rules for the listener
current_rules = get_listener_rules(connection, module, listener['ListenerArn'])
rules_to_modify = []
rules_to_delete = []
for current_rule in current_rules:
current_rule_passed_to_module = False
for new_rule in listener['Rules'][:]:
if current_rule['Priority'] == new_rule['Priority']:
current_rule_passed_to_module = True
# Remove what we match so that what is left can be marked as 'to be added'
listener['Rules'].remove(new_rule)
modified_rule = compare_rule(current_rule, new_rule)
if modified_rule:
modified_rule['Priority'] = int(current_rule['Priority'])
modified_rule['RuleArn'] = current_rule['RuleArn']
modified_rule['Actions'] = current_rule['Actions']
modified_rule['Conditions'] = current_rule['Conditions']
rules_to_modify.append(modified_rule)
break
# If the current rule was not matched against passed rules, mark for removal
if not current_rule_passed_to_module and not current_rule['IsDefault']:
rules_to_delete.append(current_rule['RuleArn'])
rules_to_add = listener['Rules']
return rules_to_add, rules_to_modify, rules_to_delete
def create_or_update_elb_listeners(connection, module, elb): def create_or_update_elb_listeners(connection, module, elb):
"""Create or update ELB listeners. Return true if changed, else false""" """Create or update ELB listeners. Return true if changed, else false"""
listener_changed = False listener_changed = False
listeners = module.params.get("listeners") # Ensure listeners are using Target Group ARN not name
listeners = ensure_listeners_default_action_has_arn(connection, module, module.params.get("listeners"))
purge_listeners = module.params.get("purge_listeners")
# create a copy of original list as we remove list elements for initial comparisons # Does the ELB have any listeners exist?
listeners_rules = deepcopy(listeners) current_listeners = get_elb_listeners(connection, module, elb['LoadBalancerArn'])
listener_matches = False
if listeners is not None: listeners_to_add, listeners_to_modify, listeners_to_delete = compare_listeners(connection, module, current_listeners, deepcopy(listeners), purge_listeners)
current_listeners = get_elb_listeners(connection, module, elb['LoadBalancerArn'])
# If there are no current listeners we can just create the new ones
current_listeners_array = []
if current_listeners:
# the describe_listeners action returns keys as unicode so I've converted them to string for easier comparison
for current_listener in current_listeners:
del current_listener['ListenerArn']
del current_listener['LoadBalancerArn']
current_listeners_s = convert(current_listener)
current_listeners_array.append(current_listeners_s)
for curr_listener in current_listeners_array:
for default_action in curr_listener['DefaultActions']:
default_action['TargetGroupName'] = convert_tg_arn_name(connection, module, default_action['TargetGroupArn'])
del default_action['TargetGroupArn']
listeners_to_add = []
# remove rules from the comparison. We will handle them separately.
for listener in listeners:
if 'Rules' in listener.keys():
del listener['Rules']
for listener in listeners:
if listener not in current_listeners_array:
listeners_to_add.append(listener)
listeners_to_remove = []
for current_listener in current_listeners_array:
if current_listener not in listeners:
listeners_to_remove.append(current_listener)
# for listeners to remove, we need to lookup the arns using unique listener attributes. Port must be unique for
# all listeners so I've retrieved the ARN based on Port.
if listeners_to_remove:
arns_to_remove = []
current_listeners = connection.describe_listeners(LoadBalancerArn=elb['LoadBalancerArn'])['Listeners']
listener_changed = True
for listener in listeners_to_remove:
for current_listener in current_listeners:
if current_listener['Port'] == listener['Port']:
arns_to_remove.append(current_listener['ListenerArn'])
for arn in arns_to_remove:
connection.delete_listener(ListenerArn=arn)
if listeners_to_add:
listener_changed = True
for listener in listeners_to_add:
listener['LoadBalancerArn'] = elb['LoadBalancerArn']
for default_action in listener['DefaultActions']:
default_action['TargetGroupArn'] = convert_tg_name_arn(connection, module, default_action['TargetGroupName'])
del default_action['TargetGroupName']
connection.create_listener(**listener)
# Lookup the listeners again and this time we will retain the rules so we can comapre for changes:
current_listeners = connection.describe_listeners(LoadBalancerArn=elb['LoadBalancerArn'])['Listeners']
# lookup the arns of the current listeners
for listener in listeners_rules:
# we only want listeners which have rules defined
if 'Rules' in listener.keys():
for current_listener in current_listeners:
if current_listener['Port'] == listener['Port']:
# look up current rules for the current listener
current_rules = connection.describe_rules(ListenerArn=current_listener['ListenerArn'])['Rules']
current_rules_array = []
for rules in current_rules:
del rules['RuleArn']
del rules['IsDefault']
if rules['Priority'] != 'default':
current_rules_s = convert(rules)
current_rules_array.append(current_rules_s)
for curr_rule in current_rules_array:
for action in curr_rule['Actions']:
action['TargetGroupName'] = convert_tg_arn_name(connection, module, action['TargetGroupArn'])
del action['TargetGroupArn']
rules_to_remove = []
for current_rule in current_rules_array:
if listener['Rules']:
if current_rule not in listener['Rules']:
rules_to_remove.append(current_rule)
else:
rules_to_remove.append(current_rule)
# for rules to remove we need to lookup the rule arn using unique attributes.
# I have used path and priority
if rules_to_remove:
rule_arns_to_remove = []
current_rules = connection.describe_rules(ListenerArn=current_listener['ListenerArn'])['Rules']
# listener_changed = True
for rules in rules_to_remove:
for current_rule in current_rules:
# if current_rule['Priority'] != 'default':
if current_rule['Conditions'] == rules['Conditions'] and current_rule['Priority'] == rules['Priority']:
rule_arns_to_remove.append(current_rule['RuleArn'])
listener_changed = True
for arn in rule_arns_to_remove:
connection.delete_rule(RuleArn=arn)
rules_to_add = []
if listener['Rules']:
for rules in listener['Rules']:
if rules not in current_rules_array:
rules_to_add.append(rules)
if rules_to_add:
listener_changed = True
for rule in rules_to_add:
rule['ListenerArn'] = current_listener['ListenerArn']
rule['Priority'] = int(rule['Priority'])
for action in rule['Actions']:
action['TargetGroupArn'] = convert_tg_name_arn(connection, module, action['TargetGroupName'])
del action['TargetGroupName']
connection.create_rule(**rule)
else: # Add listeners
for listener in listeners: for listener_to_add in listeners_to_add:
listener['LoadBalancerArn'] = elb['LoadBalancerArn'] try:
if 'Rules' in listener.keys(): listener_to_add['LoadBalancerArn'] = elb['LoadBalancerArn']
del listener['Rules'] connection.create_listener(**listener_to_add)
listener_changed = True
# handle default except ClientError as e:
for default_action in listener['DefaultActions']: module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
default_action['TargetGroupArn'] = convert_tg_name_arn(connection, module, default_action['TargetGroupName'])
del default_action['TargetGroupName'] # Modify listeners
for listener_to_modify in listeners_to_modify:
connection.create_listener(**listener) try:
listener_changed = True connection.modify_listener(**listener_to_modify)
listener_changed = True
# lookup the new listeners except ClientError as e:
current_listeners = connection.describe_listeners(LoadBalancerArn=elb['LoadBalancerArn'])['Listeners'] module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
for current_listener in current_listeners: # Delete listeners
for listener in listeners_rules: for listener_to_delete in listeners_to_delete:
if current_listener['Port'] == listener['Port']: try:
if 'Rules' in listener.keys(): connection.delete_listener(ListenerArn=listener_to_delete)
for rules in listener['Rules']: listener_changed = True
rules['ListenerArn'] = current_listener['ListenerArn'] except ClientError as e:
rules['Priority'] = int(rules['Priority']) module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
for action in rules['Actions']:
action['TargetGroupArn'] = convert_tg_name_arn(connection, module, action['TargetGroupName']) # For each listener, check rules
del action['TargetGroupName'] for listener in listeners:
connection.create_rule(**rules) if 'Rules' in listener:
# Ensure rules are using Target Group ARN not name
# listeners is none. If we have any current listeners we need to remove them listener['Rules'] = ensure_rules_action_has_arn(connection, module, listener['Rules'])
else: rules_to_add, rules_to_modify, rules_to_delete = compare_rules(connection, module, current_listeners, listener)
current_listeners = connection.describe_listeners(LoadBalancerArn=elb['LoadBalancerArn'])['Listeners']
if current_listeners: # Get listener based on port so we can use ARN
for listener in current_listeners: looked_up_listener = get_listener(connection, module, elb['LoadBalancerArn'], listener['Port'])
listener_changed = True
connection.delete_listener(ListenerArn=listener['ListenerArn']) # Add rules
for rule in rules_to_add:
try:
rule['ListenerArn'] = looked_up_listener['ListenerArn']
rule['Priority'] = int(rule['Priority'])
connection.create_rule(**rule)
listener_changed = True
except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
# Modify rules
for rule in rules_to_modify:
try:
del rule['Priority']
connection.modify_rule(**rule)
listener_changed = True
except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
# Delete rules
for rule in rules_to_delete:
try:
connection.delete_rule(RuleArn=rule)
listener_changed = True
except ClientError as e:
module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
return listener_changed return listener_changed
@ -757,6 +912,10 @@ def create_or_update_elb(connection, connection_ec2, module):
# Get the ELB listeners again # Get the ELB listeners again
elb['listeners'] = get_elb_listeners(connection, module, elb['LoadBalancerArn']) elb['listeners'] = get_elb_listeners(connection, module, elb['LoadBalancerArn'])
# For each listener, get listener rules
for listener in elb['listeners']:
listener['rules'] = get_listener_rules(connection, module, listener['ListenerArn'])
# Get the ELB attributes again # Get the ELB attributes again
elb.update(get_elb_attributes(connection, module, elb['LoadBalancerArn'])) elb.update(get_elb_attributes(connection, module, elb['LoadBalancerArn']))
@ -792,21 +951,22 @@ def main():
argument_spec = ec2_argument_spec() argument_spec = ec2_argument_spec()
argument_spec.update( argument_spec.update(
dict( dict(
access_logs_enabled=dict(required=False, type='bool'), access_logs_enabled=dict(type='bool'),
access_logs_s3_bucket=dict(required=False, type='str'), access_logs_s3_bucket=dict(type='str'),
access_logs_s3_prefix=dict(required=False, type='str'), access_logs_s3_prefix=dict(type='str'),
deletion_protection=dict(required=False, default=False, type='bool'), deletion_protection=dict(default=False, type='bool'),
idle_timeout=dict(required=False, type='int'), idle_timeout=dict(type='int'),
listeners=dict(required=False, type='list'), listeners=dict(type='list'),
name=dict(required=True, type='str'), name=dict(required=True, type='str'),
purge_tags=dict(required=False, default=True, type='bool'), purge_listeners=dict(default=True, type='bool'),
subnets=dict(required=False, type='list'), purge_tags=dict(default=True, type='bool'),
security_groups=dict(required=False, type='list'), subnets=dict(type='list'),
scheme=dict(required=False, default='internet-facing', choices=['internet-facing', 'internal']), security_groups=dict(type='list'),
state=dict(required=True, choices=['present', 'absent'], type='str'), scheme=dict(default='internet-facing', choices=['internet-facing', 'internal']),
tags=dict(required=False, default={}, type='dict'), state=dict(choices=['present', 'absent'], type='str'),
wait_timeout=dict(required=False, type='int'), tags=dict(default={}, type='dict'),
wait=dict(required=False, type='bool') wait_timeout=dict(type='int'),
wait=dict(type='bool')
) )
) )

Loading…
Cancel
Save