From b538e34a3277ea319607621028d0f6ab3a644f23 Mon Sep 17 00:00:00 2001 From: Tad Merchant Date: Tue, 26 Feb 2019 22:20:19 -0500 Subject: [PATCH] Ecs service add features (#50059) * Support UpdateService forceNewDeployment in ecs_service module * Fixes for review * Add force_new_deployment option to ecs_service.py cherrypicks changes from via/ansible Adds tests for pull request #42518 fixes backwards compatability with boto<1.8.4 * WIP commit so I don't have to stash * WIP commit for healthcheck grace period * WIP commit; ecs_module handles service registries * Fix bad check for desired_count * Add scheduling strategy test, comment out service registry test * Fix names in ecs_cluster role main task. * move full test run back to the end * Change botocore version for full test to support scheduling strategy * fix bug with desired_count==0 in amazon/ecs_service * Fix changed checking for scheduling strategy DAEMON in ecs_service * Pass testS * Fix some unhelpful comments * Add changelog for ecs_service --- .../50059-ecs-service-add-features.yml | 2 + .../modules/cloud/amazon/ecs_service.py | 87 +++++++-- .../roles/ecs_cluster/tasks/main.yml | 167 +++++++++++++++++- test/integration/targets/ecs_cluster/runme.sh | 4 +- 4 files changed, 246 insertions(+), 14 deletions(-) create mode 100644 changelogs/fragments/50059-ecs-service-add-features.yml diff --git a/changelogs/fragments/50059-ecs-service-add-features.yml b/changelogs/fragments/50059-ecs-service-add-features.yml new file mode 100644 index 00000000000..5f7ebe92ec5 --- /dev/null +++ b/changelogs/fragments/50059-ecs-service-add-features.yml @@ -0,0 +1,2 @@ +minor_changes: + - ecs_service - adds support for service_registries and scheduling_strategies. desired_count may now be none to support scheduling_strategies diff --git a/lib/ansible/modules/cloud/amazon/ecs_service.py b/lib/ansible/modules/cloud/amazon/ecs_service.py index c9116bab32f..630d2b41e70 100644 --- a/lib/ansible/modules/cloud/amazon/ecs_service.py +++ b/lib/ansible/modules/cloud/amazon/ecs_service.py @@ -132,6 +132,27 @@ options: - Seconds to wait before health checking the freshly added/updated services. This option requires botocore >= 1.8.20. required: false version_added: 2.8 + service_registries: + description: + - describes service disovery registries this service will register with. + required: false + version_added: 2.8 + suboptions: + container_name: + description: + - container name for service disovery registration + container_port: + description: + - container port for service disovery registration + arn: + description: + - Service discovery registry ARN + scheduling_strategy: + description: + - The scheduling strategy, defaults to "REPLICA" if not given to preserve previous behavior + required: false + version_added: 2.8 + choices: ["DAEMON", "REPLICA"] extends_documentation_fragment: - aws - ec2 @@ -387,21 +408,24 @@ class EcsServiceManager: if (expected['load_balancers'] or []) != existing['loadBalancers']: return False - if (expected['desired_count'] or 0) != existing['desiredCount']: - return False + # expected is params. DAEMON scheduling strategy returns desired count equal to + # number of instances running; don't check desired count if scheduling strat is daemon + if (expected['scheduling_strategy'] != 'DAEMON'): + if (expected['desired_count'] or 0) != existing['desiredCount']: + return False return True def create_service(self, service_name, cluster_name, task_definition, load_balancers, desired_count, client_token, role, deployment_configuration, - placement_constraints, placement_strategy, network_configuration, - launch_type, health_check_grace_period_seconds): + placement_constraints, placement_strategy, health_check_grace_period_seconds, + network_configuration, service_registries, launch_type, scheduling_strategy): + params = dict( cluster=cluster_name, serviceName=service_name, taskDefinition=task_definition, loadBalancers=load_balancers, - desiredCount=desired_count, clientToken=client_token, role=role, deploymentConfiguration=deployment_configuration, @@ -414,6 +438,14 @@ class EcsServiceManager: params['launchType'] = launch_type if self.health_check_setable(params) and health_check_grace_period_seconds is not None: params['healthCheckGracePeriodSeconds'] = health_check_grace_period_seconds + if service_registries: + params['serviceRegistries'] = service_registries + # desired count is not required if scheduling strategy is daemon + if desired_count is not None: + params['desiredCount'] = desired_count + + if scheduling_strategy: + params['schedulingStrategy'] = scheduling_strategy response = self.ecs.create_service(**params) return self.jsonize(response['service']) @@ -424,14 +456,17 @@ class EcsServiceManager: cluster=cluster_name, service=service_name, taskDefinition=task_definition, - desiredCount=desired_count, deploymentConfiguration=deployment_configuration) if network_configuration: params['networkConfiguration'] = network_configuration - if self.health_check_setable(params): - params['healthCheckGracePeriodSeconds'] = health_check_grace_period_seconds if force_new_deployment: params['forceNewDeployment'] = force_new_deployment + if health_check_grace_period_seconds is not None: + params['healthCheckGracePeriodSeconds'] = health_check_grace_period_seconds + # desired count is not required if scheduling strategy is daemon + if desired_count is not None: + params['desiredCount'] = desired_count + response = self.ecs.update_service(**params) return self.jsonize(response['service']) @@ -484,21 +519,27 @@ def main(): deployment_configuration=dict(required=False, default={}, type='dict'), placement_constraints=dict(required=False, default=[], type='list'), placement_strategy=dict(required=False, default=[], type='list'), + health_check_grace_period_seconds=dict(required=False, type='int'), network_configuration=dict(required=False, type='dict', options=dict( subnets=dict(type='list'), security_groups=dict(type='list'), - assign_public_ip=dict(type='bool'), + assign_public_ip=dict(type='bool') )), launch_type=dict(required=False, choices=['EC2', 'FARGATE']), - health_check_grace_period_seconds=dict(required=False, type='int') + service_registries=dict(required=False, type='list', default=[]), + scheduling_strategy=dict(required=False, choices=['DAEMON', 'REPLICA']) )) module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True, - required_if=[('state', 'present', ['task_definition', 'desired_count']), + required_if=[('state', 'present', ['task_definition']), ('launch_type', 'FARGATE', ['network_configuration'])], required_together=[['load_balancers', 'role']]) + if module.params['state'] == 'present' and module.params['scheduling_strategy'] == 'REPLICA': + if module.params['desired_count'] is None: + module.fail_json(msg='state is present, scheduling_strategy is REPLICA; missing desired_count') + service_mgr = EcsServiceManager(module) if module.params['network_configuration']: if not service_mgr.ecs_api_handles_network_configuration(): @@ -511,6 +552,7 @@ def main(): DEPLOYMENT_CONFIGURATION_TYPE_MAP) deploymentConfiguration = snake_dict_to_camel_dict(deployment_configuration) + serviceRegistries = list(map(snake_dict_to_camel_dict, module.params['service_registries'])) try: existing = service_mgr.describe_service(module.params['cluster'], module.params['name']) @@ -525,6 +567,9 @@ def main(): if module.params['force_new_deployment']: if not module.botocore_at_least('1.8.4'): module.fail_json(msg='botocore needs to be version 1.8.4 or higher to use force_new_deployment') + if module.params['health_check_grace_period_seconds']: + if not module.botocore_at_least('1.8.20'): + module.fail_json(msg='botocore needs to be version 1.8.20 or higher to use health_check_grace_period_seconds') if module.params['state'] == 'present': @@ -557,8 +602,23 @@ def main(): loadBalancer['containerPort'] = int(loadBalancer['containerPort']) if update: + # check various parameters and boto versions and give a helpful erro in boto is not new enough for feature + + if module.params['scheduling_strategy']: + if not module.botocore_at_least('1.10.37'): + module.fail_json(msg='botocore needs to be version 1.10.37 or higher to use scheduling_strategy') + elif (existing['schedulingStrategy']) != module.params['scheduling_strategy']: + module.fail_json(msg="It is not possible to update the scheduling strategy of an existing service") + + if module.params['service_registries']: + if not module.botocore_at_least('1.9.15'): + module.fail_json(msg='botocore needs to be version 1.9.15 or higher to use service_registries') + elif (existing['serviceRegistries'] or []) != serviceRegistries: + module.fail_json(msg="It is not possible to update the service registries of an existing service") + if (existing['loadBalancers'] or []) != loadBalancers: module.fail_json(msg="It is not possible to update the load balancers of an existing service") + # update required response = service_mgr.update_service(module.params['name'], module.params['cluster'], @@ -568,6 +628,7 @@ def main(): network_configuration, module.params['health_check_grace_period_seconds'], module.params['force_new_deployment']) + else: try: response = service_mgr.create_service(module.params['name'], @@ -580,9 +641,11 @@ def main(): deploymentConfiguration, module.params['placement_constraints'], module.params['placement_strategy'], + module.params['health_check_grace_period_seconds'], network_configuration, + serviceRegistries, module.params['launch_type'], - module.params['health_check_grace_period_seconds'] + module.params['scheduling_strategy'] ) except botocore.exceptions.ClientError as e: module.fail_json_aws(e, msg="Couldn't create service") diff --git a/test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml b/test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml index 78358b88d77..2a0c783bda8 100644 --- a/test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml +++ b/test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml @@ -479,6 +479,95 @@ - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.subnets|length == 2" - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.securityGroups|length == 1" + - name: create ecs_service using health_check_grace_period_seconds + ecs_service: + name: "{{ ecs_service_name }}-mft" + cluster: "{{ ecs_cluster_name }}" + load_balancers: + - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}" + containerName: "{{ ecs_task_name }}" + containerPort: "{{ ecs_task_container_port }}" + task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}" + scheduling_strategy: "REPLICA" + health_check_grace_period_seconds: 10 + desired_count: 1 + state: present + <<: *aws_connection_info + register: ecs_service_creation_hcgp + + + - name: health_check_grace_period_seconds sets HealthChecGracePeriodSeconds + assert: + that: + - ecs_service_creation_hcgp.changed + - "{{ecs_service_creation_hcgp.service.healthCheckGracePeriodSeconds}} == 10" + + - name: update ecs_service using health_check_grace_period_seconds + ecs_service: + name: "{{ ecs_service_name }}-mft" + cluster: "{{ ecs_cluster_name }}" + load_balancers: + - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}" + containerName: "{{ ecs_task_name }}" + containerPort: "{{ ecs_task_container_port }}" + task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}" + desired_count: 1 + health_check_grace_period_seconds: 30 + state: present + <<: *aws_connection_info + register: ecs_service_creation_hcgp2 + ignore_errors: no + + - name: check that module returns success + assert: + that: + - ecs_service_creation_hcgp2.changed + - "{{ecs_service_creation_hcgp2.service.healthCheckGracePeriodSeconds}} == 30" + +# until ansible supports service registries, this test can't run. +# - name: update ecs_service using service_registries +# ecs_service: +# name: "{{ ecs_service_name }}-service-registries" +# cluster: "{{ ecs_cluster_name }}" +# load_balancers: +# - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}" +# containerName: "{{ ecs_task_name }}" +# containerPort: "{{ ecs_task_container_port }}" +# service_registries: +# - containerName: "{{ ecs_task_name }}" +# containerPort: "{{ ecs_task_container_port }}" +# ### TODO: Figure out how to get a service registry ARN without a service registry module. +# registryArn: "{{ ecs_task_service_registry_arn }}" +# task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}" +# desired_count: 1 +# state: present +# <<: *aws_connection_info +# register: ecs_service_creation_sr +# ignore_errors: yes + +# - name: dump sr output +# debug: var=ecs_service_creation_sr + +# - name: check that module returns success +# assert: +# that: +# - ecs_service_creation_sr.changed + + - name: update ecs_service using REPLICA scheduling_strategy + ecs_service: + name: "{{ ecs_service_name }}-replica" + cluster: "{{ ecs_cluster_name }}" + load_balancers: + - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}" + containerName: "{{ ecs_task_name }}" + containerPort: "{{ ecs_task_container_port }}" + scheduling_strategy: "REPLICA" + task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}" + desired_count: 1 + state: present + <<: *aws_connection_info + register: ecs_service_creation_replica + - name: obtain facts for all ECS services in the cluster ecs_service_facts: cluster: "{{ ecs_cluster_name }}" @@ -728,6 +817,56 @@ ignore_errors: yes register: ecs_service_scale_down + - name: scale down multifunction-test service + ecs_service: + name: "{{ ecs_service_name }}-mft" + cluster: "{{ ecs_cluster_name }}" + state: present + load_balancers: + - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}" + containerName: "{{ ecs_task_name }}" + containerPort: "{{ ecs_task_container_port }}" + task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}" + desired_count: 0 + <<: *aws_connection_info + ignore_errors: yes + register: ecs_service_scale_down + + + + - name: scale down scheduling_strategy service + ecs_service: + name: "{{ ecs_service_name }}-replica" + cluster: "{{ ecs_cluster_name }}" + state: present + load_balancers: + - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}" + containerName: "{{ ecs_task_name }}" + containerPort: "{{ ecs_task_container_port }}" + task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}" + desired_count: 0 + <<: *aws_connection_info + ignore_errors: yes + register: ecs_service_scale_down + + +# until ansible supports service registries, the test for it can't run and this +# scale down is not needed +# - name: scale down service_registries service +# ecs_service: +# name: "{{ ecs_service_name }}-service-registries" +# cluster: "{{ ecs_cluster_name }}" +# state: present +# load_balancers: +# - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}" +# containerName: "{{ ecs_task_name }}" +# containerPort: "{{ ecs_task_container_port }}" +# task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}" +# desired_count: 0 +# <<: *aws_connection_info +# ignore_errors: yes +# register: ecs_service_scale_down + - name: scale down Fargate ECS service ecs_service: state: present @@ -761,6 +900,32 @@ <<: *aws_connection_info ignore_errors: yes + - name: remove mft ecs service + ecs_service: + state: absent + cluster: "{{ ecs_cluster_name }}" + name: "{{ ecs_service_name }}-mft" + <<: *aws_connection_info + ignore_errors: yes + + - name: remove scheduling_strategy ecs service + ecs_service: + state: absent + cluster: "{{ ecs_cluster_name }}" + name: "{{ ecs_service_name }}-replica" + <<: *aws_connection_info + ignore_errors: yes + +# until ansible supports service registries, the test for it can't run and this +# removal is not needed +# - name: remove service_registries ecs service +# ecs_service: +# state: absent +# cluster: "{{ ecs_cluster_name }}" +# name: "{{ ecs_service_name }}-service-registries" +# <<: *aws_connection_info +# ignore_errors: yes + - name: remove fargate ECS service ecs_service: state: absent @@ -769,7 +934,7 @@ <<: *aws_connection_info ignore_errors: yes register: ecs_fargate_service_network_with_awsvpc - + - name: remove ecs task definition ecs_taskdefinition: containers: "{{ ecs_task_containers }}" diff --git a/test/integration/targets/ecs_cluster/runme.sh b/test/integration/targets/ecs_cluster/runme.sh index 790a06c1fb1..f70e8f01362 100755 --- a/test/integration/targets/ecs_cluster/runme.sh +++ b/test/integration/targets/ecs_cluster/runme.sh @@ -12,6 +12,8 @@ trap 'rm -rf "${MYTMPDIR}"' EXIT # but for the python3 tests we need virtualenv to use python3 PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python} + + # Test graceful failure for older versions of botocore virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-1.7.40" source "${MYTMPDIR}/botocore-1.7.40/bin/activate" @@ -42,5 +44,5 @@ ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../c # Run full test suite virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-recent" source "${MYTMPDIR}/botocore-recent/bin/activate" -$PYTHON -m pip install 'botocore>=1.8.4' boto3 +$PYTHON -m pip install 'botocore>=1.10.37' boto3 # version 1.10.37 for scheduling strategy ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"