From 16b877e2b37852ac7a82aeed1bdf9d572887014c Mon Sep 17 00:00:00 2001 From: Will Thames Date: Mon, 12 Jun 2017 21:15:04 +1000 Subject: [PATCH] ec2_asg and ec2_asg_facts module improvements (#25166) * ec2_asg and ec2_asg_facts module improvements Return target group information for both ec2_asg and ec2_asg_facts modules Provide RETURN documentation for ec2_asg module PEP8 fixes for ec2_asg_facts * ec2_asg: use pagination when describing target groups In case an ASG has 100s of target groups, ensure that we get the full result using build_full_result --- lib/ansible/modules/cloud/amazon/ec2_asg.py | 179 ++++++++++++++++-- .../modules/cloud/amazon/ec2_asg_facts.py | 62 +++++- test/sanity/pep8/legacy-files.txt | 1 - 3 files changed, 215 insertions(+), 27 deletions(-) diff --git a/lib/ansible/modules/cloud/amazon/ec2_asg.py b/lib/ansible/modules/cloud/amazon/ec2_asg.py index ff98b2638ad..ba1fef213d8 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_asg.py +++ b/lib/ansible/modules/cloud/amazon/ec2_asg.py @@ -243,6 +243,136 @@ EXAMPLES = ''' region: us-east-1 ''' +RETURN = ''' +--- +default_cooldown: + description: The default cooldown time in seconds. + returned: success + type: int + sample: 300 +desired_capacity: + description: The number of EC2 instances that should be running in this group. + returned: success + type: int + sample: 3 +healthcheck_period: + description: Length of time in seconds after a new EC2 instance comes into service that Auto Scaling starts checking its health. + returned: success + type: int + sample: 30 +healthcheck_type: + description: The service you want the health status from, one of "EC2" or "ELB". + returned: success + type: str + sample: "ELB" +healthy_instances: + description: Number of instances in a healthy state + returned: success + type: int + sample: 5 +in_service_instances: + description: Number of instances in service + returned: success + type: int + sample: 3 +instance_facts: + description: Dictionary of EC2 instances and their status as it relates to the ASG. + returned: success + type: dict + sample: { + "i-0123456789012": { + "health_status": "Healthy", + "launch_config_name": "public-webapp-production-1", + "lifecycle_state": "InService" + } + } +instances: + description: list of instance IDs in the ASG + returned: success + type: list + sample: [ + "i-0123456789012" + ] +launch_config_name: + description: > + Name of launch configuration associated with the ASG. Same as launch_configuration_name, + provided for compatibility with ec2_asg module. + returned: success + type: str + sample: "public-webapp-production-1" +load_balancers: + description: List of load balancers names attached to the ASG. + returned: success + type: list + sample: ["elb-webapp-prod"] +max_size: + description: Maximum size of group + returned: success + type: int + sample: 3 +min_size: + description: Minimum size of group + returned: success + type: int + sample: 1 +pending_instances: + description: Number of instances in pending state + returned: success + type: int + sample: 1 +tags: + description: List of tags for the ASG, and whether or not each tag propagates to instances at launch. + returned: success + type: list + sample: [ + { + "key": "Name", + "value": "public-webapp-production-1", + "resource_id": "public-webapp-production-1", + "resource_type": "auto-scaling-group", + "propagate_at_launch": "true" + }, + { + "key": "env", + "value": "production", + "resource_id": "public-webapp-production-1", + "resource_type": "auto-scaling-group", + "propagate_at_launch": "true" + } + ] +target_group_arns: + description: List of ARNs of the target groups that the ASG populates + returned: success + type: list + sample: [ + "arn:aws:elasticloadbalancing:ap-southeast-2:123456789012:targetgroup/target-group-host-hello/1a2b3c4d5e6f1a2b", + "arn:aws:elasticloadbalancing:ap-southeast-2:123456789012:targetgroup/target-group-path-world/abcd1234abcd1234" + ] +target_group_names: + description: List of names of the target groups that the ASG populates + returned: success + type: list + sample: [ + "target-group-host-hello", + "target-group-path-world" + ] +termination_policies: + description: A list of termination policies for the group. + returned: success + type: str + sample: ["Default"] +unhealthy_instances: + description: Number of instances in an unhealthy state + returned: success + type: int + sample: 0 +viable_instances: + description: Number of instances in a viable state + returned: success + type: int + sample: 1 +''' + import time import logging as log import traceback @@ -277,7 +407,7 @@ def enforce_required_arguments(module): module.fail_json(msg="Missing required arguments for autoscaling group create/update: %s" % ",".join(missing_args)) -def get_properties(autoscaling_group): +def get_properties(autoscaling_group, module): properties = dict() properties['healthy_instances'] = 0 properties['in_service_instances'] = 0 @@ -320,6 +450,21 @@ def get_properties(autoscaling_group): properties['healthcheck_type'] = autoscaling_group.get('HealthCheckType') properties['default_cooldown'] = autoscaling_group.get('DefaultCooldown') properties['termination_policies'] = autoscaling_group.get('TerminationPolicies') + properties['target_group_arns'] = autoscaling_group.get('TargetGroupARNs') + if properties['target_group_arns']: + region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True) + elbv2_connection = boto3_conn(module, + conn_type='client', + resource='elbv2', + region=region, + endpoint=ec2_url, + **aws_connect_params) + tg_paginator = elbv2_connection.get_paginator('describe_target_groups') + tg_result = tg_paginator.paginate(TargetGroupArns=properties['target_group_arns']).build_full_result() + target_groups = tg_result['TargetGroups'] + else: + target_groups = [] + properties['target_group_names'] = [tg['TargetGroupName'] for tg in target_groups] return properties @@ -363,7 +508,7 @@ def elb_dreg(asg_connection, module, group_name, instance_id): def elb_healthy(asg_connection, elb_connection, module, group_name): healthy_instances = set() as_group = asg_connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) # get healthy, inservice instances from ASG instances = [] for instance, settings in props['instance_facts'].items(): @@ -397,7 +542,7 @@ def elb_healthy(asg_connection, elb_connection, module, group_name): def tg_healthy(asg_connection, elbv2_connection, module, group_name): healthy_instances = set() as_group = asg_connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) # get healthy, inservice instances from ASG instances = [] for instance, settings in props['instance_facts'].items(): @@ -605,7 +750,7 @@ def create_autoscaling_group(connection, module): NotificationTypes=notification_types ) as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - asg_properties = get_properties(as_group) + asg_properties = get_properties(as_group, module) changed = True return changed, asg_properties except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: @@ -613,7 +758,7 @@ def create_autoscaling_group(connection, module): exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) else: as_group = as_groups['AutoScalingGroups'][0] - initial_asg_properties = get_properties(as_group) + initial_asg_properties = get_properties(as_group, module) changed = False if suspend_processes(connection, as_group, module): @@ -768,7 +913,7 @@ def create_autoscaling_group(connection, module): try: as_group = connection.describe_auto_scaling_groups( AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - asg_properties = get_properties(as_group) + asg_properties = get_properties(as_group, module) if asg_properties != initial_asg_properties: changed = True except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: @@ -851,7 +996,7 @@ def replace(connection, module): as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] wait_for_new_inst(module, connection, group_name, wait_timeout, as_group['MinSize'], 'viable_instances') - props = get_properties(as_group) + props = get_properties(as_group, module) instances = props['instances'] if replace_instances: instances = replace_instances @@ -865,7 +1010,7 @@ def replace(connection, module): log.debug("No new instances needed, but old instances are present. Removing old instances") terminate_batch(connection, module, old_instances, instances, True) as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) changed = True return(changed, props) @@ -894,7 +1039,7 @@ def replace(connection, module): wait_for_elb(connection, module, group_name) wait_for_target_group(connection, module, group_name) as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) instances = props['instances'] if replace_instances: instances = replace_instances @@ -912,7 +1057,7 @@ def replace(connection, module): break update_size(connection, as_group, max_size, min_size, desired_capacity) as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - asg_properties = get_properties(as_group) + asg_properties = get_properties(as_group, module) log.debug("Rolling update complete.") changed = True return(changed, asg_properties) @@ -965,13 +1110,12 @@ def terminate_batch(connection, module, replace_instances, initial_instances, le min_size = module.params.get('min_size') desired_capacity = module.params.get('desired_capacity') group_name = module.params.get('name') - wait_timeout = int(module.params.get('wait_timeout')) lc_check = module.params.get('lc_check') decrement_capacity = False break_loop = False as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) desired_size = as_group['MinSize'] new_instances, old_instances = get_instances_by_lc(props, lc_check, initial_instances) @@ -1022,20 +1166,17 @@ def terminate_batch(connection, module, replace_instances, initial_instances, le def wait_for_term_inst(connection, module, term_instances): - - batch_size = module.params.get('replace_batch_size') wait_timeout = module.params.get('wait_timeout') group_name = module.params.get('name') - lc_check = module.params.get('lc_check') as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) count = 1 wait_timeout = time.time() + wait_timeout while wait_timeout > time.time() and count > 0: log.debug("waiting for instances to terminate") count = 0 as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) instance_facts = props['instance_facts'] instances = (i for i in instance_facts if i in term_instances) for i in instances: @@ -1055,7 +1196,7 @@ def wait_for_new_inst(module, connection, group_name, wait_timeout, desired_size # make sure we have the latest stats after that last loop. as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) log.debug("Waiting for {0} = {1}, currently {2}".format(prop, desired_size, props[prop])) # now we make sure that we have enough instances in a viable state wait_timeout = time.time() + wait_timeout @@ -1063,7 +1204,7 @@ def wait_for_new_inst(module, connection, group_name, wait_timeout, desired_size log.debug("Waiting for {0} = {1}, currently {2}".format(prop, desired_size, props[prop])) time.sleep(10) as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] - props = get_properties(as_group) + props = get_properties(as_group, module) if wait_timeout <= time.time(): # waiting took too long module.fail_json(msg="Waited too long for new instances to become viable. %s" % time.asctime()) diff --git a/lib/ansible/modules/cloud/amazon/ec2_asg_facts.py b/lib/ansible/modules/cloud/amazon/ec2_asg_facts.py index 583a693ceea..53cb6c75bf1 100644 --- a/lib/ansible/modules/cloud/amazon/ec2_asg_facts.py +++ b/lib/ansible/modules/cloud/amazon/ec2_asg_facts.py @@ -139,6 +139,13 @@ instances: "protected_from_scale_in": "false" } ] +launch_config_name: + description: > + Name of launch configuration associated with the ASG. Same as launch_configuration_name, + provided for compatibility with ec2_asg module. + returned: success + type: str + sample: "public-webapp-production-1" launch_configuration_name: description: Name of launch configuration associated with the ASG. returned: success @@ -194,6 +201,22 @@ tags: "propagate_at_launch": "true" } ] +target_group_arns: + description: List of ARNs of the target groups that the ASG populates + returned: success + type: list + sample: [ + "arn:aws:elasticloadbalancing:ap-southeast-2:123456789012:targetgroup/target-group-host-hello/1a2b3c4d5e6f1a2b", + "arn:aws:elasticloadbalancing:ap-southeast-2:123456789012:targetgroup/target-group-path-world/abcd1234abcd1234" + ] +target_group_names: + description: List of names of the target groups that the ASG populates + returned: success + type: list + sample: [ + "target-group-host-hello", + "target-group-path-world" + ] termination_policies: description: A list of termination policies for the group. returned: success @@ -201,12 +224,17 @@ termination_policies: sample: ["Default"] ''' +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.ec2 import get_aws_connection_info, boto3_conn, ec2_argument_spec +from ansible.module_utils.ec2 import camel_dict_to_snake_dict, HAS_BOTO3 + +import re + try: - import boto3 from botocore.exceptions import ClientError - HAS_BOTO3 = True except ImportError: - HAS_BOTO3 = False + pass # caught by imported HAS_BOTO3 + def match_asg_tags(tags_to_match, asg): for key, value in tags_to_match.items(): @@ -217,6 +245,7 @@ def match_asg_tags(tags_to_match, asg): return False return True + def find_asgs(conn, module, name=None, tags=None): """ Args: @@ -265,6 +294,7 @@ def find_asgs(conn, module, name=None, tags=None): "protected_from_scale_in": false } ], + "launch_config_name": "public-webapp-production-1", "launch_configuration_name": "public-webapp-production-1", "load_balancer_names": ["public-webapp-production-lb"], "max_size": 4, @@ -290,6 +320,8 @@ def find_asgs(conn, module, name=None, tags=None): "value": "production" } ], + "target_group_names": [], + "target_group_arns": [], "termination_policies": [ "Default" @@ -310,6 +342,14 @@ def find_asgs(conn, module, name=None, tags=None): except ClientError as e: module.fail_json(msg=e.message, **camel_dict_to_snake_dict(e.response)) + if not asgs: + return asgs + try: + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) + elbv2 = boto3_conn(module, conn_type='client', resource='elbv2', region=region, endpoint=ec2_url, **aws_connect_kwargs) + except ClientError as e: + # This is nice to have, not essential + elbv2 = None matched_asgs = [] if name is not None: @@ -328,7 +368,18 @@ def find_asgs(conn, module, name=None, tags=None): matched_tags = True if matched_name and matched_tags: - matched_asgs.append(camel_dict_to_snake_dict(asg)) + asg = camel_dict_to_snake_dict(asg) + # compatibility with ec2_asg module + asg['launch_config_name'] = asg['launch_configuration_name'] + # workaround for https://github.com/ansible/ansible/pull/25015 + if 'target_group_ar_ns' in asg: + asg['target_group_arns'] = asg['target_group_ar_ns'] + del(asg['target_group_ar_ns']) + if elbv2 and asg.get('target_group_arns'): + tg_paginator = elbv2.get_paginator('describe_target_groups') + tg_result = tg_paginator.paginate(TargetGroupArns=asg['target_group_arns']).build_full_result() + asg['target_group_names'] = [tg['TargetGroupName'] for tg in tg_result['TargetGroups']] + matched_asgs.append(asg) return matched_asgs @@ -359,9 +410,6 @@ def main(): results = find_asgs(autoscaling, module, name=asg_name, tags=asg_tags) module.exit_json(results=results) -# import module snippets -from ansible.module_utils.basic import * -from ansible.module_utils.ec2 import * if __name__ == '__main__': main() diff --git a/test/sanity/pep8/legacy-files.txt b/test/sanity/pep8/legacy-files.txt index ea95ed7160e..6e68574c318 100644 --- a/test/sanity/pep8/legacy-files.txt +++ b/test/sanity/pep8/legacy-files.txt @@ -11,7 +11,6 @@ lib/ansible/modules/cloud/amazon/ec2.py lib/ansible/modules/cloud/amazon/ec2_ami.py lib/ansible/modules/cloud/amazon/ec2_ami_copy.py lib/ansible/modules/cloud/amazon/ec2_ami_find.py -lib/ansible/modules/cloud/amazon/ec2_asg_facts.py lib/ansible/modules/cloud/amazon/ec2_customer_gateway.py lib/ansible/modules/cloud/amazon/ec2_eip.py lib/ansible/modules/cloud/amazon/ec2_elb.py