[cloud] Improve aws testing and use `{{ region }}` var (#26892)

* Protect AWS credentials from accidental commits

* Improve documentation for IAM policies

* Update ELB tests to be multi region

Allow AZs to reflect the region in which the tests are run.

This will not work for regions with fewer than three AZs,
but those are relatively rare

.gitignore

@@ -94,6 +94,7 @@ packaging/release/ansible_release
 /test/integration/inventory.networking
 /test/integration/inventory.winrm
 /test/integration/cloud-config-aws.yml
+/test/integration/cloud-config-cs.ini
 # python 'rope' stuff
 .ropeproject
 # local 'ack' config files

docs/docsite/rst/dev_guide/testing_integration.rst

@@ -161,11 +161,17 @@ IAM policies for AWS
 
 Ansible needs fairly wide ranging powers to run the tests in an AWS account.  This rights can be provided to a dedicated user. These need to be configured before running the test.
 
-testing-iam-policy.json.j2
---------------------------
+testing-policies
+----------------
 
-The testing-iam-policy.json.j2 file contains a policy which can be given to the user
-running the tests to minimize the rights of that user.  Please note that while this policy does limit the user to one region, this does not fully restrict the user (primarily due to the limitations of the Amazon ARN notation). The user will still have wide privileges for viewing account definitions, and will also able to manage some resources that are not related to testing (for example, AWS lambdas with different names).  Tests should not be run in a primary production account in any case.
+``hacking/aws_config/testing_policies`` contains a set of policies that are required for all existing AWS module tests.
+The ``hacking/aws_config/setup_iam.yml`` playbook can be used to add all of those policies to an IAM group (using
+``-e iam_group=GROUP_NAME``. Once the group is created, you'll need to create a user and make the user a member of the
+group. The policies are designed to minimize the rights of that user.  Please note that while this policy does limit
+the user to one region, this does not fully restrict the user (primarily due to the limitations of the Amazon ARN
+notation). The user will still have wide privileges for viewing account definitions, and will also able to manage
+some resources that are not related to testing (for example, AWS lambdas with different names).  Tests should not
+be run in a primary production account in any case.
 
 Other Definitions required
 --------------------------

test/integration/targets/ec2_elb_lb/tasks/main.yml

@@ -38,8 +38,8 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -61,8 +61,8 @@
         that:
           - 'info.changed'
           - 'info.elb.status == "created"'
-          - '"us-east-1c" in info.elb.zones'
-          - '"us-east-1d" in info.elb.zones'
+          - '"{{ ec2_region }}a" in info.elb.zones'
+          - '"{{ ec2_region }}b" in info.elb.zones'
           - 'info.elb.health_check.healthy_threshold == 10'
           - 'info.elb.health_check.interval == 30'
           - 'info.elb.health_check.target == "HTTP:80/index.html"'
@@ -103,7 +103,7 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -125,7 +125,7 @@
         that:
           - 'info.elb.status == "ok"'
           - 'info.changed'
-          - 'info.elb.zones[0] == "us-east-1b"'
+          - 'info.elb.zones[0] == "{{ ec2_region }}c"'
 
     # ============================================================
 
@@ -140,9 +140,9 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -154,9 +154,9 @@
         that:
           - 'info.changed'
           - 'info.elb.status == "ok"'
-          - '"us-east-1b" in info.elb.zones'
-          - '"us-east-1c" in info.elb.zones'
-          - '"us-east-1d" in info.elb.zones'
+          - '"{{ ec2_region }}a" in info.elb.zones'
+          - '"{{ ec2_region }}b" in info.elb.zones'
+          - '"{{ ec2_region }}c" in info.elb.zones'
 
 
     # ============================================================
@@ -172,9 +172,9 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -204,9 +204,9 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 8081
@@ -263,8 +263,9 @@
             load_balancer_port: 80
             instance_port: 80
         zones:
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
       register: result
       ignore_errors: true
 
@@ -282,8 +283,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -307,8 +309,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -332,8 +335,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -357,8 +361,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -383,8 +388,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80

test/integration/targets/elb_classic_lb/tasks/main.yml

@@ -38,8 +38,8 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -61,8 +61,8 @@
         that:
           - 'info.changed'
           - 'info.elb.status == "created"'
-          - '"us-east-1c" in info.elb.zones'
-          - '"us-east-1d" in info.elb.zones'
+          - '"{{ ec2_region }}a" in info.elb.zones'
+          - '"{{ ec2_region }}b" in info.elb.zones'
           - 'info.elb.health_check.healthy_threshold == 10'
           - 'info.elb.health_check.interval == 30'
           - 'info.elb.health_check.target == "HTTP:80/index.html"'
@@ -103,7 +103,7 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -125,7 +125,7 @@
         that:
           - 'info.elb.status == "ok"'
           - 'info.changed'
-          - 'info.elb.zones[0] == "us-east-1b"'
+          - 'info.elb.zones[0] == "{{ ec2_region }}c"'
 
     # ============================================================
 
@@ -140,9 +140,9 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -154,9 +154,9 @@
         that:
           - 'info.changed'
           - 'info.elb.status == "ok"'
-          - '"us-east-1b" in info.elb.zones'
-          - '"us-east-1c" in info.elb.zones'
-          - '"us-east-1d" in info.elb.zones'
+          - '"{{ ec2_region }}a" in info.elb.zones'
+          - '"{{ ec2_region }}b" in info.elb.zones'
+          - '"{{ ec2_region }}c" in info.elb.zones'
 
 
     # ============================================================
@@ -172,9 +172,9 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -204,9 +204,9 @@
         security_token: "{{ security_token }}"
         state: present
         zones:
-          - us-east-1b
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 8081
@@ -263,8 +263,9 @@
             load_balancer_port: 80
             instance_port: 80
         zones:
-          - us-east-1c
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
       register: result
       ignore_errors: true
 
@@ -282,8 +283,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -307,8 +309,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -332,8 +335,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -357,8 +361,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80
@@ -383,8 +388,9 @@
         region: "{{ ec2_region }}"
         state: present
         zones:
-          - us-east-1a
-          - us-east-1d
+          - "{{ ec2_region }}a"
+          - "{{ ec2_region }}b"
+          - "{{ ec2_region }}c"
         listeners:
           - protocol: http
             load_balancer_port: 80