AWS dynamic inventory: fixup RDS integration tests (#61259)

* ec2 inventory/rds: (integration tests) migrate to module_defaults and cope with missing security_token

* ec2 inventory/rds: (integration tests) Avoid hard coding the parameter group name

Generated it based on the information returned when creating the RDS instance.

* ec2 inventory/rds: (integration tests) rename templates to .j2 rather than .yml
- avoids throwing yamllint errors
- makes it clear they're templates not the final yaml files.

test/integration/targets/inventory_aws_rds/playbooks/create_inventory_config.yml

@@ -3,7 +3,7 @@
   connection: local
   gather_facts: no
   vars:
-    template_name: "../templates/{{ template | default('inventory.yml') }}"
+    template_name: "../templates/{{ template | default('inventory.j2') }}"
   tasks:
     - name: write inventory config file
       copy:

test/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml

@@ -5,7 +5,13 @@
   environment: "{{ ansible_test.environment }}"
   tasks:
 
-    - block:
+    - module_defaults:
+        group/aws:
+          aws_access_key: '{{ aws_access_key }}'
+          aws_secret_key: '{{ aws_secret_key }}'
+          security_token: '{{ security_token | default(omit) }}'
+          region: '{{ aws_region }}'
+      block:
         - set_fact:
             instance_id: '{{ resource_prefix }}-mariadb'
 
@@ -17,15 +23,6 @@
 
         # Create new host, add it to inventory and then terminate it without updating the cache
 
-        - name: set connection information for all tasks
-          set_fact:
-            aws_connection_info: &aws_connection_info
-              aws_access_key: '{{ aws_access_key }}'
-              aws_secret_key: '{{ aws_secret_key }}'
-              security_token: '{{ security_token }}'
-              region: '{{ aws_region }}'
-          no_log: yes
-
         - name: create minimal mariadb instance in default VPC and default subnet group
           rds_instance:
             state: present
@@ -37,7 +34,6 @@
             master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
             tags:
               workload_type: other
-            <<: *aws_connection_info
           register: setup_instance
 
         - meta: refresh_inventory
@@ -54,6 +50,5 @@
             engine: mariadb
             skip_final_snapshot: yes
             instance_id: '{{ instance_id }}'
-            <<: *aws_connection_info
           ignore_errors: yes
           when: setup_instance is defined

test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml

@@ -5,7 +5,13 @@
   environment: "{{ ansible_test.environment }}"
   tasks:
 
-    - block:
+    - module_defaults:
+        group/aws:
+          aws_access_key: '{{ aws_access_key }}'
+          aws_secret_key: '{{ aws_secret_key }}'
+          security_token: '{{ security_token | default(omit) }}'
+          region: '{{ aws_region }}'
+      block:
 
         - set_fact:
             instance_id: "{{ resource_prefix }}-mariadb"
@@ -19,15 +25,6 @@
 
         # Create new host, refresh inventory, remove host, refresh inventory
 
-        - name: set connection information for all tasks
-          set_fact:
-            aws_connection_info: &aws_connection_info
-              aws_access_key: '{{ aws_access_key }}'
-              aws_secret_key: '{{ aws_secret_key }}'
-              security_token: '{{ security_token }}'
-              region: '{{ aws_region }}'
-          no_log: yes
-
         - name: create minimal mariadb instance in default VPC and default subnet group
           rds_instance:
             state: present
@@ -39,7 +36,6 @@
             master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
             tags:
               workload_type: other
-            <<: *aws_connection_info
           register: setup_instance
 
         - meta: refresh_inventory
@@ -57,7 +53,6 @@
             engine: mariadb
             skip_final_snapshot: yes
             instance_id: '{{ instance_id }}'
-            <<: *aws_connection_info
 
         - meta: refresh_inventory
 
@@ -75,6 +70,5 @@
             engine: mariadb
             skip_final_snapshot: yes
             instance_id: '{{ instance_id }}'
-            <<: *aws_connection_info
           ignore_errors: yes
           when: setup_instance is defined

test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml

@@ -5,21 +5,16 @@
   environment: "{{ ansible_test.environment }}"
   tasks:
 
-    - block:
+    - module_defaults:
-
+        group/aws:
-        - set_fact:
-            instance_id: "{{ resource_prefix }}-mariadb"
-
-        # Create new host, refresh inventory
-
-        - name: set connection information for all tasks
-          set_fact:
-            aws_connection_info: &aws_connection_info
           aws_access_key: '{{ aws_access_key }}'
           aws_secret_key: '{{ aws_secret_key }}'
-              security_token: '{{ security_token }}'
+          security_token: '{{ security_token | default(omit) }}'
           region: '{{ aws_region }}'
-          no_log: yes
+      block:
+
+        - set_fact:
+            instance_id: "{{ resource_prefix }}-mariadb"
 
         - name: create minimal mariadb instance in default VPC and default subnet group
           rds_instance:
@@ -32,20 +27,28 @@
             master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
             tags:
               workload_type: other
-            <<: *aws_connection_info
           register: setup_instance
 
         - meta: refresh_inventory
         - debug: var=groups
 
+        - name: 'generate expected group name based off the db parameter groups'
+          vars:
+            parameter_group_name: '{{ setup_instance.db_parameter_groups[0].db_parameter_group_name }}'
+          set_fact:
+            parameter_group_key: 'rds_parameter_group_{{ parameter_group_name | replace(".", "_") }}'
+
         - name: assert the keyed groups from constructed config were added to inventory
           assert:
             that:
               # There are 6 groups: all, ungrouped, aws_rds, tag keyed group, engine keyed group, parameter group keyed group
               - "groups | length == 6"
-              - groups.tag_workload_type_other
+              - '"all" in groups'
-              - groups.rds_mariadb
+              - '"ungrouped" in groups'
-              - groups.rds_parameter_group_default_mariadb10_3
+              - '"aws_rds" in groups'
+              - '"tag_workload_type_other" in groups'
+              - '"rds_mariadb" in groups'
+              - 'parameter_group_key in groups'
 
       always:
 
@@ -55,6 +58,5 @@
             engine: mariadb
             skip_final_snapshot: yes
             instance_id: '{{ instance_id }}'
-            <<: *aws_connection_info
           ignore_errors: yes
           when: setup_instance is defined

test/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml

@@ -1,4 +1,10 @@
 - name: test updating inventory
+  module_defaults:
+    group/aws:
+      aws_access_key: '{{ aws_access_key }}'
+      aws_secret_key: '{{ aws_secret_key }}'
+      security_token: '{{ security_token | default(omit) }}'
+      region: '{{ aws_region }}'
   block:
     - set_fact:
         instance_id: "{{ resource_prefix }}update"
@@ -9,15 +15,6 @@
           - "'aws_rds' in groups"
           - "not groups.aws_rds"
 
-    - name: set connection information for all tasks
-      set_fact:
-        aws_connection_info: &aws_connection_info
-          aws_access_key: "{{ aws_access_key }}"
-          aws_secret_key: "{{ aws_secret_key }}"
-          security_token: "{{ security_token }}"
-          region: "{{ aws_region }}"
-      no_log: yes
-
     - name: create minimal mariadb instance in default VPC and default subnet group
       rds_instance:
         state: present
@@ -29,7 +26,6 @@
         master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}'
         tags:
           workload_type: other
-        <<: *aws_connection_info
       register: setup_instance
 
     - meta: refresh_inventory
@@ -47,7 +43,6 @@
         engine: mariadb
         skip_final_snapshot: yes
         instance_id: ansible-rds-mariadb-example
-        <<: *aws_connection_info
 
     - meta: refresh_inventory
 
@@ -65,6 +60,5 @@
         engine: mariadb
         skip_final_snapshot: yes
         instance_id: ansible-rds-mariadb-example
-        <<: *aws_connection_info
       ignore_errors: yes
       when: setup_instance is defined

test/integration/targets/inventory_aws_rds/runme.sh

@@ -20,7 +20,7 @@ ansible-playbook playbooks/create_inventory_config.yml "$@"
 ansible-playbook playbooks/test_populating_inventory.yml "$@"
 
 # generate inventory config with caching and test using it
-ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_cache.yml'" "$@"
+ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_cache.j2'" "$@"
 ansible-playbook playbooks/populate_cache.yml "$@"
 ansible-playbook playbooks/test_inventory_cache.yml "$@"
 
@@ -28,7 +28,7 @@ ansible-playbook playbooks/test_inventory_cache.yml "$@"
 rm -r aws_rds_cache_dir/
 
 # generate inventory config with constructed features and test using it
-ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_constructed.yml'" "$@"
+ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_constructed.j2'" "$@"
 ansible-playbook playbooks/test_populating_inventory_with_constructed.yml "$@"
 
 # cleanup inventory config

test/integration/targets/inventory_aws_rds/templates/inventory.j2

@@ -1,7 +1,9 @@
 plugin: aws_rds
 aws_access_key_id: '{{ aws_access_key }}'
 aws_secret_access_key: '{{ aws_secret_key }}'
+{% if security_token | default(false) %}
 aws_security_token: '{{ security_token }}'
+{% endif %}
 regions:
   - '{{ aws_region }}'
 filters:

test/integration/targets/inventory_aws_rds/templates/inventory_with_cache.j2

@@ -4,7 +4,9 @@ cache_plugin: jsonfile
 cache_connection: aws_rds_cache_dir
 aws_access_key_id: '{{ aws_access_key }}'
 aws_secret_access_key: '{{ aws_secret_key }}'
+{% if security_token | default(false) %}
 aws_security_token: '{{ security_token }}'
+{% endif %}
 regions:
   - '{{ aws_region }}'
 filters:

test/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2

@@ -1,7 +1,9 @@
 plugin: aws_rds
 aws_access_key_id: '{{ aws_access_key }}'
 aws_secret_access_key: '{{ aws_secret_key }}'
+{% if security_token | default(false) %}
 aws_security_token: '{{ security_token }}'
+{% endif %}
 regions:
   - '{{ aws_region }}'
 keyed_groups: