Ecs service add features (#50059)

* Support UpdateService forceNewDeployment in ecs_service module

* Fixes for review

* Add force_new_deployment option to ecs_service.py

cherrypicks changes from via/ansible
Adds tests for pull request #42518
fixes backwards compatability with boto<1.8.4

* WIP commit so I don't have to stash

* WIP commit for healthcheck grace period

* WIP commit; ecs_module handles service registries

* Fix bad check for desired_count

* Add scheduling strategy test, comment out service registry test

* Fix names in ecs_cluster role main task.

* move full test run back to the end

* Change botocore version for full test to support scheduling strategy

* fix bug with desired_count==0 in amazon/ecs_service

* Fix changed checking for scheduling strategy DAEMON in ecs_service

* Pass testS

* Fix some unhelpful comments

* Add changelog for ecs_service

changelogs/fragments/50059-ecs-service-add-features.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ecs_service - adds support for service_registries and scheduling_strategies. desired_count may now be none to support scheduling_strategies

lib/ansible/modules/cloud/amazon/ecs_service.py

@@ -132,6 +132,27 @@ options:
           - Seconds to wait before health checking the freshly added/updated services. This option requires botocore >= 1.8.20.
         required: false
         version_added: 2.8
+    service_registries:
+        description:
+          - describes service disovery registries this service will register with.
+        required: false
+        version_added: 2.8
+        suboptions:
+            container_name:
+                description:
+                  - container name for service disovery registration
+            container_port:
+                description:
+                  - container port for service disovery registration
+            arn:
+                description:
+                  - Service discovery registry ARN
+    scheduling_strategy:
+        description:
+          - The scheduling strategy, defaults to "REPLICA" if not given to preserve previous behavior
+        required: false
+        version_added: 2.8
+        choices: ["DAEMON", "REPLICA"]
 extends_documentation_fragment:
     - aws
     - ec2
@@ -387,21 +408,24 @@ class EcsServiceManager:
         if (expected['load_balancers'] or []) != existing['loadBalancers']:
             return False
 
-        if (expected['desired_count'] or 0) != existing['desiredCount']:
-            return False
+        # expected is params. DAEMON scheduling strategy returns desired count equal to
+        # number of instances running; don't check desired count if scheduling strat is daemon
+        if (expected['scheduling_strategy'] != 'DAEMON'):
+            if (expected['desired_count'] or 0) != existing['desiredCount']:
+                return False
 
         return True
 
     def create_service(self, service_name, cluster_name, task_definition, load_balancers,
                        desired_count, client_token, role, deployment_configuration,
-                       placement_constraints, placement_strategy, network_configuration,
-                       launch_type, health_check_grace_period_seconds):
+                       placement_constraints, placement_strategy, health_check_grace_period_seconds,
+                       network_configuration, service_registries, launch_type, scheduling_strategy):
+
         params = dict(
             cluster=cluster_name,
             serviceName=service_name,
             taskDefinition=task_definition,
             loadBalancers=load_balancers,
-            desiredCount=desired_count,
             clientToken=client_token,
             role=role,
             deploymentConfiguration=deployment_configuration,
@@ -414,6 +438,14 @@ class EcsServiceManager:
             params['launchType'] = launch_type
         if self.health_check_setable(params) and health_check_grace_period_seconds is not None:
             params['healthCheckGracePeriodSeconds'] = health_check_grace_period_seconds
+        if service_registries:
+            params['serviceRegistries'] = service_registries
+        # desired count is not required if scheduling strategy is daemon
+        if desired_count is not None:
+            params['desiredCount'] = desired_count
+
+        if scheduling_strategy:
+            params['schedulingStrategy'] = scheduling_strategy
         response = self.ecs.create_service(**params)
         return self.jsonize(response['service'])
 
@@ -424,14 +456,17 @@ class EcsServiceManager:
             cluster=cluster_name,
             service=service_name,
             taskDefinition=task_definition,
-            desiredCount=desired_count,
             deploymentConfiguration=deployment_configuration)
         if network_configuration:
             params['networkConfiguration'] = network_configuration
-        if self.health_check_setable(params):
-            params['healthCheckGracePeriodSeconds'] = health_check_grace_period_seconds
         if force_new_deployment:
             params['forceNewDeployment'] = force_new_deployment
+        if health_check_grace_period_seconds is not None:
+            params['healthCheckGracePeriodSeconds'] = health_check_grace_period_seconds
+        # desired count is not required if scheduling strategy is daemon
+        if desired_count is not None:
+            params['desiredCount'] = desired_count
+
         response = self.ecs.update_service(**params)
         return self.jsonize(response['service'])
 
@@ -484,21 +519,27 @@ def main():
         deployment_configuration=dict(required=False, default={}, type='dict'),
         placement_constraints=dict(required=False, default=[], type='list'),
         placement_strategy=dict(required=False, default=[], type='list'),
+        health_check_grace_period_seconds=dict(required=False, type='int'),
         network_configuration=dict(required=False, type='dict', options=dict(
             subnets=dict(type='list'),
             security_groups=dict(type='list'),
-            assign_public_ip=dict(type='bool'),
+            assign_public_ip=dict(type='bool')
         )),
         launch_type=dict(required=False, choices=['EC2', 'FARGATE']),
-        health_check_grace_period_seconds=dict(required=False, type='int')
+        service_registries=dict(required=False, type='list', default=[]),
+        scheduling_strategy=dict(required=False, choices=['DAEMON', 'REPLICA'])
     ))
 
     module = AnsibleAWSModule(argument_spec=argument_spec,
                               supports_check_mode=True,
-                              required_if=[('state', 'present', ['task_definition', 'desired_count']),
+                              required_if=[('state', 'present', ['task_definition']),
                                            ('launch_type', 'FARGATE', ['network_configuration'])],
                               required_together=[['load_balancers', 'role']])
 
+    if module.params['state'] == 'present' and module.params['scheduling_strategy'] == 'REPLICA':
+        if module.params['desired_count'] is None:
+            module.fail_json(msg='state is present, scheduling_strategy is REPLICA; missing desired_count')
+
     service_mgr = EcsServiceManager(module)
     if module.params['network_configuration']:
         if not service_mgr.ecs_api_handles_network_configuration():
@@ -511,6 +552,7 @@ def main():
                                                 DEPLOYMENT_CONFIGURATION_TYPE_MAP)
 
     deploymentConfiguration = snake_dict_to_camel_dict(deployment_configuration)
+    serviceRegistries = list(map(snake_dict_to_camel_dict, module.params['service_registries']))
 
     try:
         existing = service_mgr.describe_service(module.params['cluster'], module.params['name'])
@@ -525,6 +567,9 @@ def main():
     if module.params['force_new_deployment']:
         if not module.botocore_at_least('1.8.4'):
             module.fail_json(msg='botocore needs to be version 1.8.4 or higher to use force_new_deployment')
+    if module.params['health_check_grace_period_seconds']:
+        if not module.botocore_at_least('1.8.20'):
+            module.fail_json(msg='botocore needs to be version 1.8.20 or higher to use health_check_grace_period_seconds')
 
     if module.params['state'] == 'present':
 
@@ -557,8 +602,23 @@ def main():
                         loadBalancer['containerPort'] = int(loadBalancer['containerPort'])
 
                 if update:
+                    # check various parameters and boto versions and give a helpful erro in boto is not new enough for feature
+
+                    if module.params['scheduling_strategy']:
+                        if not module.botocore_at_least('1.10.37'):
+                            module.fail_json(msg='botocore needs to be version 1.10.37 or higher to use scheduling_strategy')
+                        elif (existing['schedulingStrategy']) != module.params['scheduling_strategy']:
+                            module.fail_json(msg="It is not possible to update the scheduling strategy of an existing service")
+
+                    if module.params['service_registries']:
+                        if not module.botocore_at_least('1.9.15'):
+                            module.fail_json(msg='botocore needs to be version 1.9.15 or higher to use service_registries')
+                        elif (existing['serviceRegistries'] or []) != serviceRegistries:
+                            module.fail_json(msg="It is not possible to update the service registries of an existing service")
+
                     if (existing['loadBalancers'] or []) != loadBalancers:
                         module.fail_json(msg="It is not possible to update the load balancers of an existing service")
+
                     # update required
                     response = service_mgr.update_service(module.params['name'],
                                                           module.params['cluster'],
@@ -568,6 +628,7 @@ def main():
                                                           network_configuration,
                                                           module.params['health_check_grace_period_seconds'],
                                                           module.params['force_new_deployment'])
+
                 else:
                     try:
                         response = service_mgr.create_service(module.params['name'],
@@ -580,9 +641,11 @@ def main():
                                                               deploymentConfiguration,
                                                               module.params['placement_constraints'],
                                                               module.params['placement_strategy'],
+                                                              module.params['health_check_grace_period_seconds'],
                                                               network_configuration,
+                                                              serviceRegistries,
                                                               module.params['launch_type'],
-                                                              module.params['health_check_grace_period_seconds']
+                                                              module.params['scheduling_strategy']
                                                               )
                     except botocore.exceptions.ClientError as e:
                         module.fail_json_aws(e, msg="Couldn't create service")

test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml

@@ -479,6 +479,95 @@
           - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.subnets|length == 2"
           - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.securityGroups|length == 1"
 
+    - name: create ecs_service using health_check_grace_period_seconds
+      ecs_service:
+        name: "{{ ecs_service_name }}-mft"
+        cluster: "{{ ecs_cluster_name }}"
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        scheduling_strategy: "REPLICA"
+        health_check_grace_period_seconds: 10
+        desired_count: 1
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_hcgp
+
+
+    - name: health_check_grace_period_seconds sets HealthChecGracePeriodSeconds
+      assert:
+        that:
+          - ecs_service_creation_hcgp.changed
+          - "{{ecs_service_creation_hcgp.service.healthCheckGracePeriodSeconds}} == 10"
+
+    - name: update ecs_service using health_check_grace_period_seconds
+      ecs_service:
+        name: "{{ ecs_service_name }}-mft"
+        cluster: "{{ ecs_cluster_name }}"
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        desired_count: 1
+        health_check_grace_period_seconds: 30
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_hcgp2
+      ignore_errors: no
+
+    - name: check that module returns success
+      assert:
+        that:
+          - ecs_service_creation_hcgp2.changed
+          - "{{ecs_service_creation_hcgp2.service.healthCheckGracePeriodSeconds}} == 30"
+
+# until ansible supports service registries, this test can't run.
+#   - name: update ecs_service using service_registries
+#     ecs_service:
+#       name: "{{ ecs_service_name }}-service-registries"
+#       cluster: "{{ ecs_cluster_name }}"
+#       load_balancers:
+#         - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+#           containerName: "{{ ecs_task_name }}"
+#           containerPort: "{{ ecs_task_container_port }}"
+#       service_registries:
+#         - containerName: "{{ ecs_task_name }}"
+#           containerPort: "{{ ecs_task_container_port }}"
+#           ### TODO: Figure out how to get a service registry ARN without a service registry module.
+#           registryArn: "{{ ecs_task_service_registry_arn }}"
+#       task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+#       desired_count: 1
+#       state: present
+#       <<: *aws_connection_info
+#     register: ecs_service_creation_sr
+#     ignore_errors: yes
+
+#   - name: dump sr output
+#     debug: var=ecs_service_creation_sr
+
+#   - name: check that module returns success
+#     assert:
+#       that:
+#         - ecs_service_creation_sr.changed
+
+    - name: update ecs_service using REPLICA scheduling_strategy
+      ecs_service:
+        name: "{{ ecs_service_name }}-replica"
+        cluster: "{{ ecs_cluster_name }}"
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        scheduling_strategy: "REPLICA"
+        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        desired_count: 1
+        state: present
+        <<: *aws_connection_info
+      register: ecs_service_creation_replica
+
     - name: obtain facts for all ECS services in the cluster
       ecs_service_facts:
         cluster: "{{ ecs_cluster_name }}"
@@ -728,6 +817,56 @@
       ignore_errors: yes
       register: ecs_service_scale_down
 
+    - name: scale down multifunction-test service
+      ecs_service:
+        name: "{{ ecs_service_name }}-mft"
+        cluster: "{{ ecs_cluster_name }}"
+        state: present
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        desired_count: 0
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_service_scale_down
+
+
+
+    - name: scale down scheduling_strategy service
+      ecs_service:
+        name: "{{ ecs_service_name }}-replica"
+        cluster: "{{ ecs_cluster_name }}"
+        state: present
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        desired_count: 0
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_service_scale_down
+
+
+# until ansible supports service registries, the test for it can't run and this
+# scale down is not needed
+#   - name: scale down service_registries service
+#     ecs_service:
+#       name: "{{ ecs_service_name }}-service-registries"
+#       cluster: "{{ ecs_cluster_name }}"
+#       state: present
+#       load_balancers:
+#         - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+#           containerName: "{{ ecs_task_name }}"
+#           containerPort: "{{ ecs_task_container_port }}"
+#       task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+#       desired_count: 0
+#       <<: *aws_connection_info
+#     ignore_errors: yes
+#     register: ecs_service_scale_down
+
     - name: scale down Fargate ECS service
       ecs_service:
         state: present
@@ -761,6 +900,32 @@
         <<: *aws_connection_info
       ignore_errors: yes
 
+    - name: remove mft ecs service
+      ecs_service:
+        state: absent
+        cluster: "{{ ecs_cluster_name }}"
+        name: "{{ ecs_service_name }}-mft"
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: remove scheduling_strategy ecs service
+      ecs_service:
+        state: absent
+        cluster: "{{ ecs_cluster_name }}"
+        name: "{{ ecs_service_name }}-replica"
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+# until ansible supports service registries, the test for it can't run and this
+# removal is not needed
+#   - name: remove service_registries ecs service
+#     ecs_service:
+#       state: absent
+#       cluster: "{{ ecs_cluster_name }}"
+#       name: "{{ ecs_service_name }}-service-registries"
+#       <<: *aws_connection_info
+#     ignore_errors: yes
+
     - name: remove fargate ECS service 
       ecs_service:
         state: absent
@@ -769,7 +934,7 @@
         <<: *aws_connection_info
       ignore_errors: yes
       register: ecs_fargate_service_network_with_awsvpc
- 
+
     - name: remove ecs task definition
       ecs_taskdefinition:
         containers: "{{ ecs_task_containers }}"

test/integration/targets/ecs_cluster/runme.sh

@@ -12,6 +12,8 @@ trap 'rm -rf "${MYTMPDIR}"' EXIT
 # but for the python3 tests we need virtualenv to use python3
 PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python}
 
+
+
 # Test graceful failure for older versions of botocore
 virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-1.7.40"
 source "${MYTMPDIR}/botocore-1.7.40/bin/activate"
@@ -42,5 +44,5 @@ ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../c
 # Run full test suite
 virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-recent"
 source "${MYTMPDIR}/botocore-recent/bin/activate"
-$PYTHON -m pip install 'botocore>=1.8.4' boto3
+$PYTHON -m pip install 'botocore>=1.10.37' boto3 # version 1.10.37 for scheduling strategy
 ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"