ec2_instance.py: Add CpuOptions to EC2 (#43799)

* ec2_instance.py: Add CpuOptions to EC2

* ec2_instance_facts.py: Add CpuOptions to EC2 facts

* ec2_instance.py: LooseVersion and pylint indentation

* Implement nested cpu_options dict

* Fix botocore required version

* ec2_instance: Split integration tests to add executions with old botocore versions

* Ensure runme.sh is executable

lib/ansible/modules/cloud/amazon/ec2_instance.py

@@ -141,6 +141,23 @@ options:
       - For T2 series instances, choose whether to allow increased charges to buy CPU credits if the default pool is depleted.
       - Choose I(unlimited) to enable buying additional CPU credits.
     choices: [unlimited, standard]
+  cpu_options:
+    description:
+      - Reduce the number of vCPU exposed to the instance.
+      - Those parameters can only be set at instance launch. The two suboptions threads_per_core and core_count are mandatory.
+      - See U(https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-optimize-cpu.html) for combinations available.
+      - Requires botocore >= 1.10.16
+    version_added: 2.7
+    suboptions:
+      threads_per_core:
+        description:
+        - Select the number of threads per core to enable. Disable or Enable Intel HT
+        choices: [1, 2]
+        required: true
+      core_count:
+        description:
+        - Set the number of core to enable.
+        required: true
   detailed_monitoring:
     description:
       - Whether to allow detailed cloudwatch metrics to be collected, enabling more detailed alerting.
@@ -901,6 +918,30 @@ def warn_if_public_ip_assignment_changed(instance):
                 assign_public_ip, instance['InstanceId']))
 
 
+def warn_if_cpu_options_changed(instance):
+    # This is a non-modifiable attribute.
+    cpu_options = module.params.get('cpu_options')
+    if cpu_options is None:
+        return
+
+    # Check that the CpuOptions set are the same and warn if not
+    core_count_curr = instance['CpuOptions'].get('CoreCount')
+    core_count = cpu_options.get('core_count')
+    threads_per_core_curr = instance['CpuOptions'].get('ThreadsPerCore')
+    threads_per_core = cpu_options.get('threads_per_core')
+    if core_count_curr != core_count:
+        module.warn(
+            "Unable to modify core_count from {0} to {1}. "
+            "Assigning a number of core is determinted during instance creation".format(
+                core_count_curr, core_count))
+
+    if threads_per_core_curr != threads_per_core:
+        module.warn(
+            "Unable to modify threads_per_core from {0} to {1}. "
+            "Assigning a number of threads per core is determined during instance creation.".format(
+                threads_per_core_curr, threads_per_core))
+
+
 def discover_security_groups(group, groups, parent_vpc_id=None, subnet_id=None, ec2=None):
     if ec2 is None:
         ec2 = module.client('ec2')
@@ -1019,6 +1060,10 @@ def build_top_level_options(params):
         spec['InstanceInitiatedShutdownBehavior'] = params.get('instance_initiated_shutdown_behavior')
     if params.get('termination_protection') is not None:
         spec['DisableApiTermination'] = params.get('termination_protection')
+    if params.get('cpu_options') is not None:
+        spec['CpuOptions'] = {}
+        spec['CpuOptions']['ThreadsPerCore'] = params.get('cpu_options').get('threads_per_core')
+        spec['CpuOptions']['CoreCount'] = params.get('cpu_options').get('core_count')
     return spec
 
 
@@ -1445,6 +1490,10 @@ def main():
         launch_template=dict(type='dict'),
         key_name=dict(type='str'),
         cpu_credit_specification=dict(type='str', choices=['standard', 'unlimited']),
+        cpu_options=dict(type='dict', options=dict(
+            core_count=dict(type='int', required=True),
+            threads_per_core=dict(type='int', choices=[1, 2], required=True)
+        )),
         tenancy=dict(type='str', choices=['dedicated', 'default']),
         instance_initiated_shutdown_behavior=dict(type='str', choices=['stop', 'terminate']),
         termination_protection=dict(type='bool'),
@@ -1515,12 +1564,16 @@ def main():
 
         module.params['filters'] = filters
 
+    if module.params.get('cpu_options') and not module.botocore_at_least('1.10.16'):
+            module.fail_json(msg="cpu_options is only supported with botocore >= 1.10.16")
+
     existing_matches = find_instances(ec2, filters=module.params.get('filters'))
     changed = False
 
     if state not in ('terminated', 'absent') and existing_matches:
         for match in existing_matches:
             warn_if_public_ip_assignment_changed(match)
+            warn_if_cpu_options_changed(match)
             changed |= manage_tags(match, (module.params.get('tags') or {}), module.params.get('purge_tags', False), ec2)
 
     if state in ('present', 'running', 'started'):

lib/ansible/modules/cloud/amazon/ec2_instance_facts.py

@@ -113,6 +113,21 @@ instances:
                             returned: always
                             type: string
                             sample: vol-12345678
+        cpu_options:
+            description: The CPU options set for the instance.
+            returned: always if botocore version >= 1.10.16
+            type: complex
+            contains:
+                core_count:
+                     description: The number of CPU cores for the instance.
+                     returned: always
+                     type: int
+                     sample: 1
+                threads_per_core:
+                     description: The number of threads per CPU core. On supported instance, a value of 1 means Intel Hyper-Threading Technology is disabled.
+                     returned: always
+                     type: int
+                     sample: 1
         client_token:
             description: The idempotency token you provided when you launched the instance, if applicable.
             returned: always

test/integration/targets/ec2_instance/playbooks/full_test.yml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  connection: local
+
+  roles:
+    - ec2_instance

test/integration/targets/ec2_instance/playbooks/roles/ec2_instance/tasks/cpu_options.yml

@@ -0,0 +1,52 @@
+- name: set connection information for all tasks
+  set_fact:
+    aws_connection_info: &aws_connection_info
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token }}"
+      region: "{{ aws_region }}"
+  no_log: true
+
+- name: create c4.large instance with cpu_options
+  ec2_instance:
+    name: "{{ resource_prefix }}-test-c4large-1-threads-per-core"
+    image_id: "{{ ec2_ami_image[aws_region] }}"
+    tags:
+      TestId: "{{ resource_prefix }}"
+    vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}"
+    instance_type: c4.large
+    cpu_options:
+        core_count: 1
+        threads_per_core: 1
+    <<: *aws_connection_info
+  register: instance_creation
+
+- name: instance with cpu_options created with the right options
+  assert:
+    that:
+      - instance_creation is success
+      - instance_creation is changed
+      - "instance_creation.instances[0].cpu_options.core_count == 1"
+      - "instance_creation.instances[0].cpu_options.threads_per_core == 1"
+
+- name: modify cpu_options on existing instance (warning displayed)
+  ec2_instance:
+    state: present
+    name: "{{ resource_prefix }}-test-c4large-1-threads-per-core"
+    image_id: "{{ ec2_ami_image[aws_region] }}"
+    tags:
+      TestId: "{{ resource_prefix }}"
+    vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}"
+    instance_type: c4.large
+    cpu_options:
+        core_count: 1
+        threads_per_core: 2
+    <<: *aws_connection_info
+  register: cpu_options_update
+  ignore_errors: yes
+
+- name: modify cpu_options has no effect on existing instance
+  assert:
+    that:
+      - cpu_options_update is success
+      - cpu_options_update is not changed

test/integration/targets/ec2_instance/playbooks/roles/ec2_instance/tasks/main.yml

@@ -6,7 +6,7 @@
 #  - EC2_REGION -> AWS_REGION
 #
 
-# - include: ../../setup_ec2/tasks/common.yml module_name: ec2_instance
+# - include: ../../../../../setup_ec2/tasks/common.yml module_name: ec2_instance
 
 - block:
 
@@ -89,12 +89,13 @@
         <<: *aws_connection_info
       register: sg
 
-    - include_tasks: tasks/termination_protection.yml
-    - include_tasks: tasks/tags_and_vpc_settings.yml
-    - include_tasks: tasks/external_resource_attach.yml
-    - include_tasks: tasks/block_devices.yml
-    - include_tasks: tasks/default_vpc_tests.yml
-    - include_tasks: tasks/iam_instance_role.yml
+    - include_tasks: cpu_options.yml
+    - include_tasks: termination_protection.yml
+    - include_tasks: tags_and_vpc_settings.yml
+    - include_tasks: external_resource_attach.yml
+    - include_tasks: block_devices.yml
+    - include_tasks: default_vpc_tests.yml
+    - include_tasks: iam_instance_role.yml
 
 
     # ============================================================

test/integration/targets/ec2_instance/playbooks/version_fail.yml

@@ -0,0 +1,38 @@
+- hosts: localhost
+  connection: local
+  vars:
+     resource_prefix: 'ansible-testing'
+
+  tasks:
+    - block:
+        - name: set up aws connection info
+          set_fact:
+            aws_connection_info: &aws_connection_info
+              aws_access_key: "{{ aws_access_key }}"
+              aws_secret_key: "{{ aws_secret_key }}"
+              security_token: "{{ security_token }}"
+              region: "{{ aws_region }}"
+          no_log: True
+          
+        - name: Include vars file in roles/ec2_instance/defaults/main.yml
+          include_vars:
+            file: 'roles/ec2_instance/defaults/main.yml'
+
+        - name: create c4.large with cpu options (fails gracefully)
+          ec2_instance:
+            state: present
+            name: "ansible-test-{{ resource_prefix | regex_search('([0-9]+)$') }}-ec2"
+            image_id: "{{ ec2_ami_image[aws_region] }}"
+            instance_type: c4.large
+            cpu_options:
+                core_count: 1
+                threads_per_core: 1
+            <<: *aws_connection_info
+          register: ec2_instance_cpu_options_creation
+          ignore_errors: yes
+
+        - name: check that graceful error message is returned when creation with cpu_options and old botocore 
+          assert:
+            that:
+              - ec2_instance_cpu_options_creation.failed
+              - 'ec2_instance_cpu_options_creation.msg == "cpu_options is only supported with botocore >= 1.10.16"' 

test/integration/targets/ec2_instance/runme.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# We don't set -u here, due to pypa/virtualenv#150
+set -ex
+
+MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+
+# This is needed for the ubuntu1604py3 tests
+# Ubuntu patches virtualenv to make the default python2
+# but for the python3 tests we need virtualenv to use python3
+PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python}
+
+# Test graceful failure for older versions of botocore
+export ANSIBLE_ROLES_PATH=../
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-less-than-1.10.16"
+source "${MYTMPDIR}/botocore-less-than-1.10.16/bin/activate"
+"${PYTHON}" -m pip install 'botocore<1.10.16' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/version_fail.yml "$@"
+
+# Run full test suite
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-recent"
+source "${MYTMPDIR}/botocore-recent/bin/activate"
+$PYTHON -m pip install 'botocore>=1.10.16' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"