Extend `module_defaults` by adding default groups for cloud modules (#44127)

Extends `module_defaults` by adding a prefix to defaults `group/` which denotes a builtin list of modules. Initial groups are: `group/aws`, `group/azure`, and `group/gcp`

changelogs/fragments/module_defaults_groups.yml

@@ -0,0 +1,2 @@
+major_changes:
+  - Extends `module_defaults` by adding a prefix to defaults `group/` which denotes a builtin or user-specified list of modules, such as `group/aws` or `group/gcp`

docs/docsite/rst/user_guide/module_defaults_config.rst

@@ -0,0 +1,24 @@
+.. _module_defaults_config:
+
+Module Defaults Configuration
+=============================
+
+Ansible 2.7 adds a preview-status feature to group together modules that share common sets of parameters. This makes
+it easier to author playbooks making heavy use of API-based modules such as cloud modules. By default Ansible ships
+with groups for AWS and GCP modules that share parameters.
+
+In a playbook, you can set module defaults for whole groups of modules, such as setting a common AWS region.
+
+.. code-block:: YAML
+
+    # example_play.yml
+    - hosts: localhost
+      module_defaults:
+        group/aws:
+          region: us-west-2
+      tasks:
+      - aws_s3_bucket_facts:
+      # now the region is shared between both facts modules
+      - ec2_ami_facts:
+          filters:
+            name: 'RHEL*7.5*'

lib/ansible/config/manager.py

@@ -223,18 +223,7 @@ class ConfigManager(object):
         self._config_file = conf_file
         self.data = ConfigData()
 
-        if defs_file is None:
+        self._base_defs = self._read_config_yaml_file(defs_file or ('%s/base.yml' % os.path.dirname(__file__)))
-            # Create configuration definitions from source
-            b_defs_file = to_bytes('%s/base.yml' % os.path.dirname(__file__))
-        else:
-            b_defs_file = to_bytes(defs_file)
-
-        # consume definitions
-        if os.path.exists(b_defs_file):
-            with open(b_defs_file, 'rb') as config_def:
-                self._base_defs = yaml_load(config_def, Loader=SafeLoader)
-        else:
-            raise AnsibleError("Missing base configuration definition file (bad install?): %s" % to_native(b_defs_file))
 
         if self._config_file is None:
             # set config using ini
@@ -248,6 +237,22 @@ class ConfigManager(object):
 
         # update constants
         self.update_config_data()
+        try:
+            self.update_module_defaults_groups()
+        except Exception as e:
+            # Since this is a 2.7 preview feature, we want to have it fail as gracefully as possible when there are issues.
+            sys.stderr.write('Could not load module_defaults_groups: %s: %s\n\n' % (type(e).__name__, e))
+            self.module_defaults_groups = {}
+
+    def _read_config_yaml_file(self, yml_file):
+        # TODO: handle relative paths as relative to the directory containing the current playbook instead of CWD
+        # Currently this is only used with absolute paths to the `ansible/config` directory
+        yml_file = to_bytes(yml_file)
+        if os.path.exists(yml_file):
+            with open(yml_file, 'rb') as config_def:
+                return yaml_load(config_def, Loader=SafeLoader) or {}
+        raise AnsibleError(
+            "Missing base YAML definition file (bad install?): %s" % to_native(yml_file))
 
     def _parse_config_file(self, cfile=None):
         ''' return flat configuration settings from file(s) '''
@@ -444,6 +449,14 @@ class ConfigManager(object):
 
         self._plugins[plugin_type][name] = defs
 
+    def update_module_defaults_groups(self):
+        defaults_config = self._read_config_yaml_file(
+            '%s/module_defaults.yml' % os.path.join(os.path.dirname(__file__))
+        )
+        if defaults_config.get('version') not in ('1', '1.0', 1, 1.0):
+            raise AnsibleError('module_defaults.yml has an invalid version "%s" for configuration. Could be a bad install.' % defaults_config.get('version'))
+        self.module_defaults_groups = defaults_config.get('groupings', {})
+
     def update_config_data(self, defs=None, configfile=None):
         ''' really: update constants '''
 

lib/ansible/config/module_defaults.yml

@@ -0,0 +1,592 @@
+version: '1.0'
+groupings:
+  aws_acm_facts:
+  - aws
+  aws_api_gateway:
+  - aws
+  aws_application_scaling_policy:
+  - aws
+  aws_az_facts:
+  - aws
+  aws_batch_compute_environment:
+  - aws
+  aws_batch_job_definition:
+  - aws
+  aws_batch_job_queue:
+  - aws
+  aws_caller_facts:
+  - aws
+  aws_config_aggregation_authorization:
+  - aws
+  aws_config_aggregator:
+  - aws
+  aws_config_delivery_channel:
+  - aws
+  aws_config_recorder:
+  - aws
+  aws_config_rule:
+  - aws
+  aws_direct_connect_connection:
+  - aws
+  aws_direct_connect_gateway:
+  - aws
+  aws_direct_connect_link_aggregation_group:
+  - aws
+  aws_direct_connect_virtual_interface:
+  - aws
+  aws_eks_cluster:
+  - aws
+  aws_elasticbeanstalk_app:
+  - aws
+  aws_glue_connection:
+  - aws
+  aws_glue_job:
+  - aws
+  aws_inspector_target:
+  - aws
+  aws_kms:
+  - aws
+  aws_kms_facts:
+  - aws
+  aws_region_facts:
+  - aws
+  aws_s3:
+  - aws
+  aws_s3_bucket_facts:
+  - aws
+  aws_s3_cors:
+  - aws
+  aws_ses_identity:
+  - aws
+  aws_ses_identity_policy:
+  - aws
+  aws_sgw_facts:
+  - aws
+  aws_ssm_parameter_store:
+  - aws
+  aws_waf_condition:
+  - aws
+  aws_waf_facts:
+  - aws
+  aws_waf_rule:
+  - aws
+  aws_waf_web_acl:
+  - aws
+  cloudformation:
+  - aws
+  cloudformation_facts:
+  - aws
+  cloudfront_distribution:
+  - aws
+  cloudfront_facts:
+  - aws
+  cloudfront_invalidation:
+  - aws
+  cloudfront_origin_access_identity:
+  - aws
+  cloudtrail:
+  - aws
+  cloudwatchevent_rule:
+  - aws
+  cloudwatchlogs_log_group:
+  - aws
+  cloudwatchlogs_log_group_facts:
+  - aws
+  data_pipeline:
+  - aws
+  dynamodb_table:
+  - aws
+  dynamodb_ttl:
+  - aws
+  ec2:
+  - aws
+  ec2_ami:
+  - aws
+  ec2_ami_copy:
+  - aws
+  ec2_ami_facts:
+  - aws
+  ec2_asg:
+  - aws
+  ec2_asg_facts:
+  - aws
+  ec2_asg_lifecycle_hook:
+  - aws
+  ec2_customer_gateway:
+  - aws
+  ec2_customer_gateway_facts:
+  - aws
+  ec2_eip:
+  - aws
+  ec2_eip_facts:
+  - aws
+  ec2_elb:
+  - aws
+  ec2_elb_facts:
+  - aws
+  ec2_elb_lb:
+  - aws
+  ec2_eni:
+  - aws
+  ec2_eni_facts:
+  - aws
+  ec2_group:
+  - aws
+  ec2_group_facts:
+  - aws
+  ec2_instance:
+  - aws
+  ec2_instance_facts:
+  - aws
+  ec2_key:
+  - aws
+  ec2_lc:
+  - aws
+  ec2_lc_facts:
+  - aws
+  ec2_lc_find:
+  - aws
+  ec2_metric_alarm:
+  - aws
+  ec2_placement_group:
+  - aws
+  ec2_placement_group_facts:
+  - aws
+  ec2_scaling_policy:
+  - aws
+  ec2_snapshot:
+  - aws
+  ec2_snapshot_copy:
+  - aws
+  ec2_snapshot_facts:
+  - aws
+  ec2_tag:
+  - aws
+  ec2_vol:
+  - aws
+  ec2_vol_facts:
+  - aws
+  ec2_vpc_dhcp_option:
+  - aws
+  ec2_vpc_dhcp_option_facts:
+  - aws
+  ec2_vpc_egress_igw:
+  - aws
+  ec2_vpc_endpoint:
+  - aws
+  ec2_vpc_endpoint_facts:
+  - aws
+  ec2_vpc_igw:
+  - aws
+  ec2_vpc_igw_facts:
+  - aws
+  ec2_vpc_nacl:
+  - aws
+  ec2_vpc_nacl_facts:
+  - aws
+  ec2_vpc_nat_gateway:
+  - aws
+  ec2_vpc_nat_gateway_facts:
+  - aws
+  ec2_vpc_net:
+  - aws
+  ec2_vpc_net_facts:
+  - aws
+  ec2_vpc_peer:
+  - aws
+  ec2_vpc_peering_facts:
+  - aws
+  ec2_vpc_route_table:
+  - aws
+  ec2_vpc_route_table_facts:
+  - aws
+  ec2_vpc_subnet:
+  - aws
+  ec2_vpc_subnet_facts:
+  - aws
+  ec2_vpc_vgw:
+  - aws
+  ec2_vpc_vgw_facts:
+  - aws
+  ec2_vpc_vpn:
+  - aws
+  ec2_vpc_vpn_facts:
+  - aws
+  ec2_win_password:
+  - aws
+  ecs_attribute:
+  - aws
+  ecs_cluster:
+  - aws
+  ecs_ecr:
+  - aws
+  ecs_service:
+  - aws
+  ecs_service_facts:
+  - aws
+  ecs_task:
+  - aws
+  ecs_taskdefinition:
+  - aws
+  ecs_taskdefinition_facts:
+  - aws
+  efs:
+  - aws
+  efs_facts:
+  - aws
+  elasticache:
+  - aws
+  elasticache_facts:
+  - aws
+  elasticache_parameter_group:
+  - aws
+  elasticache_snapshot:
+  - aws
+  elasticache_subnet_group:
+  - aws
+  elb_application_lb:
+  - aws
+  elb_application_lb_facts:
+  - aws
+  elb_classic_lb:
+  - aws
+  elb_classic_lb_facts:
+  - aws
+  elb_instance:
+  - aws
+  elb_network_lb:
+  - aws
+  elb_target:
+  - aws
+  elb_target_group:
+  - aws
+  elb_target_group_facts:
+  - aws
+  execute_lambda:
+  - aws
+  iam:
+  - aws
+  iam_cert:
+  - aws
+  iam_group:
+  - aws
+  iam_managed_policy:
+  - aws
+  iam_mfa_device_facts:
+  - aws
+  iam_policy:
+  - aws
+  iam_role:
+  - aws
+  iam_role_facts:
+  - aws
+  iam_server_certificate_facts:
+  - aws
+  iam_user:
+  - aws
+  kinesis_stream:
+  - aws
+  lambda:
+  - aws
+  lambda_alias:
+  - aws
+  lambda_event:
+  - aws
+  lambda_facts:
+  - aws
+  lambda_policy:
+  - aws
+  lightsail:
+  - aws
+  rds:
+  - aws
+  rds_instance_facts:
+  - aws
+  rds_param_group:
+  - aws
+  rds_snapshot_facts:
+  - aws
+  rds_subnet_group:
+  - aws
+  redshift:
+  - aws
+  redshift_facts:
+  - aws
+  redshift_subnet_group:
+  - aws
+  route53:
+  - aws
+  route53_facts:
+  - aws
+  route53_health_check:
+  - aws
+  route53_zone:
+  - aws
+  s3_bucket:
+  - aws
+  s3_lifecycle:
+  - aws
+  s3_logging:
+  - aws
+  s3_sync:
+  - aws
+  s3_website:
+  - aws
+  sns:
+  - aws
+  sns_topic:
+  - aws
+  sqs_queue:
+  - aws
+  sts_assume_role:
+  - aws
+  sts_session_token:
+  - aws
+  gcp_compute_address:
+  - gcp
+  gcp_compute_address_facts:
+  - gcp
+  gcp_compute_backend_bucket:
+  - gcp
+  gcp_compute_backend_bucket_facts:
+  - gcp
+  gcp_compute_backend_service:
+  - gcp
+  gcp_compute_backend_service_facts:
+  - gcp
+  gcp_compute_disk:
+  - gcp
+  gcp_compute_disk_facts:
+  - gcp
+  gcp_compute_firewall:
+  - gcp
+  gcp_compute_firewall_facts:
+  - gcp
+  gcp_compute_forwarding_rule:
+  - gcp
+  gcp_compute_forwarding_rule_facts:
+  - gcp
+  gcp_compute_global_address:
+  - gcp
+  gcp_compute_global_address_facts:
+  - gcp
+  gcp_compute_global_forwarding_rule:
+  - gcp
+  gcp_compute_global_forwarding_rule_facts:
+  - gcp
+  gcp_compute_health_check:
+  - gcp
+  gcp_compute_health_check_facts:
+  - gcp
+  gcp_compute_http_health_check:
+  - gcp
+  gcp_compute_http_health_check_facts:
+  - gcp
+  gcp_compute_https_health_check:
+  - gcp
+  gcp_compute_https_health_check_facts:
+  - gcp
+  gcp_compute_image:
+  - gcp
+  gcp_compute_image_facts:
+  - gcp
+  gcp_compute_instance:
+  - gcp
+  gcp_compute_instance_facts:
+  - gcp
+  gcp_compute_instance_group:
+  - gcp
+  gcp_compute_instance_group_facts:
+  - gcp
+  gcp_compute_instance_group_manager:
+  - gcp
+  gcp_compute_instance_group_manager_facts:
+  - gcp
+  gcp_compute_instance_template:
+  - gcp
+  gcp_compute_instance_template_facts:
+  - gcp
+  gcp_compute_network:
+  - gcp
+  gcp_compute_network_facts:
+  - gcp
+  gcp_compute_route:
+  - gcp
+  gcp_compute_route_facts:
+  - gcp
+  gcp_compute_router_facts:
+  - gcp
+  gcp_compute_ssl_certificate:
+  - gcp
+  gcp_compute_ssl_certificate_facts:
+  - gcp
+  gcp_compute_ssl_policy:
+  - gcp
+  gcp_compute_ssl_policy_facts:
+  - gcp
+  gcp_compute_subnetwork:
+  - gcp
+  gcp_compute_subnetwork_facts:
+  - gcp
+  gcp_compute_target_http_proxy:
+  - gcp
+  gcp_compute_target_http_proxy_facts:
+  - gcp
+  gcp_compute_target_https_proxy:
+  - gcp
+  gcp_compute_target_https_proxy_facts:
+  - gcp
+  gcp_compute_target_pool:
+  - gcp
+  gcp_compute_target_pool_facts:
+  - gcp
+  gcp_compute_target_ssl_proxy:
+  - gcp
+  gcp_compute_target_ssl_proxy_facts:
+  - gcp
+  gcp_compute_target_tcp_proxy:
+  - gcp
+  gcp_compute_target_tcp_proxy_facts:
+  - gcp
+  gcp_compute_target_vpn_gateway:
+  - gcp
+  gcp_compute_target_vpn_gateway_facts:
+  - gcp
+  gcp_compute_url_map:
+  - gcp
+  gcp_compute_url_map_facts:
+  - gcp
+  gcp_compute_vpn_tunnel:
+  - gcp
+  gcp_compute_vpn_tunnel_facts:
+  - gcp
+  gcp_container_cluster:
+  - gcp
+  gcp_container_node_pool:
+  - gcp
+  gcp_dns_managed_zone:
+  - gcp
+  gcp_dns_resource_record_set:
+  - gcp
+  gcp_pubsub_subscription:
+  - gcp
+  gcp_pubsub_topic:
+  - gcp
+  gcp_storage_bucket:
+  - gcp
+  gcp_storage_bucket_access_control:
+  - gcp
+  azure_rm_acs:
+  - azure
+  azure_rm_aks:
+  - azure
+  azure_rm_aks_facts:
+  - azure
+  azure_rm_appserviceplan:
+  - azure
+  azure_rm_appserviceplan_facts:
+  - azure
+  azure_rm_availabilityset:
+  - azure
+  azure_rm_availabilityset_facts:
+  - azure
+  azure_rm_containerinstance:
+  - azure
+  azure_rm_containerregistry:
+  - azure
+  azure_rm_deployment:
+  - azure
+  azure_rm_dnsrecordset:
+  - azure
+  azure_rm_dnsrecordset_facts:
+  - azure
+  azure_rm_dnszone:
+  - azure
+  azure_rm_dnszone_facts:
+  - azure
+  azure_rm_functionapp:
+  - azure
+  azure_rm_functionapp_facts:
+  - azure
+  azure_rm_image:
+  - azure
+  azure_rm_keyvault:
+  - azure
+  azure_rm_keyvaultkey:
+  - azure
+  azure_rm_keyvaultsecret:
+  - azure
+  azure_rm_loadbalancer:
+  - azure
+  azure_rm_loadbalancer_facts:
+  - azure
+  azure_rm_managed_disk:
+  - azure
+  azure_rm_managed_disk_facts:
+  - azure
+  azure_rm_mysqldatabase:
+  - azure
+  azure_rm_mysqldatabase_facts:
+  - azure
+  azure_rm_mysqlserver:
+  - azure
+  azure_rm_mysqlserver_facts:
+  - azure
+  azure_rm_networkinterface:
+  - azure
+  azure_rm_networkinterface_facts:
+  - azure
+  azure_rm_postgresqldatabase:
+  - azure
+  azure_rm_postgresqldatabase_facts:
+  - azure
+  azure_rm_postgresqlserver:
+  - azure
+  azure_rm_publicipaddress:
+  - azure
+  azure_rm_publicipaddress_facts:
+  - azure
+  azure_rm_resource:
+  - azure
+  azure_rm_resource_facts:
+  - azure
+  azure_rm_resourcegroup:
+  - azure
+  azure_rm_resourcegroup_facts:
+  - azure
+  azure_rm_securitygroup:
+  - azure
+  azure_rm_securitygroup_facts:
+  - azure
+  azure_rm_sqldatabase:
+  - azure
+  azure_rm_sqlserver:
+  - azure
+  azure_rm_sqlserver_facts:
+  - azure
+  azure_rm_storageaccount:
+  - azure
+  azure_rm_storageaccount_facts:
+  - azure
+  azure_rm_storageblob:
+  - azure
+  azure_rm_subnet:
+  - azure
+  azure_rm_virtualmachine:
+  - azure
+  azure_rm_virtualmachine_extension:
+  - azure
+  azure_rm_virtualmachine_facts:
+  - azure
+  azure_rm_virtualmachineimage_facts:
+  - azure
+  azure_rm_virtualmachine_scaleset:
+  - azure
+  azure_rm_virtualmachine_scaleset_facts:
+  - azure
+  azure_rm_virtualnetwork:
+  - azure
+  azure_rm_virtualnetwork_facts:
+  - azure
+  azure_rm_webapp:
+  - azure

lib/ansible/executor/task_executor.py

@@ -559,6 +559,11 @@ class TaskExecutor:
             tmp_args = module_defaults[self._task.action].copy()
             tmp_args.update(self._task.args)
             self._task.args = tmp_args
+        if self._task.action in C.config.module_defaults_groups:
+            for group in C.config.module_defaults_groups.get(self._task.action, []):
+                tmp_args = (module_defaults.get('group/{0}'.format(group)) or {}).copy()
+                tmp_args.update(self._task.args)
+                self._task.args = tmp_args
 
         # And filter out any fields which were set to default(omit), and got the omit token value
         omit_token = variables.get('omit')

lib/ansible/modules/cloud/amazon/aws_s3_bucket_facts.py

@@ -53,6 +53,7 @@ except ImportError:
     pass  # will be detected by imported HAS_BOTO3
 
 from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils._text import to_native
 from ansible.module_utils.ec2 import (boto3_conn, ec2_argument_spec, HAS_BOTO3, camel_dict_to_snake_dict,
                                       get_aws_connection_info)
 
@@ -67,7 +68,7 @@ def get_bucket_list(module, connection):
     try:
         buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets']
     except botocore.exceptions.ClientError as e:
-        module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
+        module.fail_json(msg=to_native(e), exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
 
     return buckets
 

test/integration/targets/module_defaults/tasks/main.yml

@@ -87,3 +87,28 @@
       - assert:
           that:
             foo.msg == "Hello world!"
+- name: Module group defaults block
+  module_defaults:
+    group/aws:
+      region: us-east-1
+      aws_secret_key: foobar
+  block:
+    - aws_s3_bucket_facts:
+      ignore_errors: true
+      register: s3
+    - assert:
+        that:
+          - "'Partial credentials' in s3.msg or 'boto3 required' in s3.msg"
+- name: Module group defaults block
+  module_defaults:
+    group/aws:
+      region: us-east-1
+      aws_secret_key: foobar
+      aws_access_key: foobar
+  block:
+    - aws_s3_bucket_facts:
+      ignore_errors: true
+      register: s3
+    - assert:
+        that:
+          - "'InvalidAccessKeyId' in s3.msg or 'boto3 required' in s3.msg"