Merge c6b4e1a232 into 6aa8192131

lib/ansible/cli/config.py

@@ -149,6 +149,10 @@ class ConfigCLI(CLI):
 
         super(ConfigCLI, self).run()
 
+        # initialize each galaxy server's options from known listed servers
+        self._galaxy_servers = [s for s in C.GALAXY_SERVER_LIST or [] if s]  # clean list, reused later here
+        C.config.load_galaxy_server_defs(self._galaxy_servers)
+
         if context.CLIARGS['config_file']:
             self.config_file = unfrackpath(context.CLIARGS['config_file'], follow=False)
             b_config = to_bytes(self.config_file)
@@ -262,11 +266,17 @@ class ConfigCLI(CLI):
         '''
         build a dict with the list requested configs
         '''
+
         config_entries = {}
         if context.CLIARGS['type'] in ('base', 'all'):
             # this dumps main/common configs
             config_entries = self.config.get_configuration_definitions(ignore_private=True)
 
+            # for base and all, we include galaxy servers
+            config_entries['GALAXY_SERVERS'] = {}
+            for server in self._galaxy_servers:
+                config_entries['GALAXY_SERVERS'][server] = self.config.get_configuration_definitions('galaxy_server', server)
+
         if context.CLIARGS['type'] != 'base':
             config_entries['PLUGINS'] = {}
 
@@ -468,6 +478,8 @@ class ConfigCLI(CLI):
             else:
                 entry = {}
                 for key in config[setting]._fields:
+                    if key == 'type':
+                        continue
                     entry[key] = getattr(config[setting], key)
 
             if not context.CLIARGS['only_changed'] or changed:
@@ -476,7 +488,10 @@ class ConfigCLI(CLI):
         return entries
 
     def _get_global_configs(self):
-        config = self.config.get_configuration_definitions(ignore_private=True).copy()
+
+        # Add base
+        config = self.config.get_configuration_definitions(ignore_private=True)
+        # convert to settings
         for setting in config.keys():
             v, o = C.config.get_config_value_and_origin(setting, cfile=self.config_file, variables=get_constants())
             config[setting] = Setting(setting, v, o, None)
@@ -553,17 +568,54 @@ class ConfigCLI(CLI):
 
         return output
 
+    def _get_galaxy_server_configs(self):
+
+        output = []
+        # add galaxy servers
+        for server in self._galaxy_servers:
+            server_config = {}
+            s_config = self.config.get_configuration_definitions('galaxy_server', server)
+            for setting in s_config.keys():
+                try:
+                    v, o = C.config.get_config_value_and_origin(setting, plugin_type='galaxy_server', plugin_name=server, cfile=self.config_file)
+                except AnsibleError as e:
+                    if to_text(e).startswith('No setting was provided for required configuration'):
+                        v = None
+                        o = 'REQUIRED'
+                    else:
+                        raise e
+                if v is None and o is None:
+                    # not all cases will be error
+                    o = 'REQUIRED'
+                server_config[setting] = Setting(setting, v, o, None)
+            if context.CLIARGS['format'] == 'display':
+                if not context.CLIARGS['only_changed'] or server_config:
+                    equals = '=' * len(server)
+                    output.append(f'\n{server}\n{equals}')
+                    output.extend(self._render_settings(server_config))
+            else:
+                output.append({server: server_config})
+
+        return output
+
     def execute_dump(self):
         '''
         Shows the current settings, merges ansible.cfg if specified
         '''
-        if context.CLIARGS['type'] == 'base':
-            # deal with base
-            output = self._get_global_configs()
-        elif context.CLIARGS['type'] == 'all':
+        output = []
+        if context.CLIARGS['type'] in ('base', 'all'):
             # deal with base
             output = self._get_global_configs()
-            # deal with plugins
+
+            # add galaxy servers
+            server_config_list = self._get_galaxy_server_configs()
+            if context.CLIARGS['format'] == 'display':
+                output.append('\nGALAXY_SERVERS:\n')
+            else:
+                output.append({'GALAXY_SERVERS': server_config_list})
+
+        if context.CLIARGS['type'] == 'all':
+            # add all plugins
             for ptype in C.CONFIGURABLE_PLUGINS:
                 plugin_list = self._get_plugin_configs(ptype, context.CLIARGS['args'])
                 if context.CLIARGS['format'] == 'display':
@@ -576,8 +628,9 @@ class ConfigCLI(CLI):
                     else:
                         pname = '%s_PLUGINS' % ptype.upper()
                     output.append({pname: plugin_list})
-        else:
-            # deal with plugins
+
+        elif context.CLIARGS['type'] != 'base':
+            # deal with specific plugin
             output = self._get_plugin_configs(context.CLIARGS['type'], context.CLIARGS['args'])
 
         if context.CLIARGS['format'] == 'display':
@@ -594,6 +647,7 @@ class ConfigCLI(CLI):
         found = False
         config_entries = self._list_entries_from_args()
         plugin_types = config_entries.pop('PLUGINS', None)
+        galaxy_servers = config_entries.pop('GALAXY_SERVERS', None)
 
         if context.CLIARGS['format'] == 'ini':
             if C.CONFIG_FILE is not None:
@@ -610,6 +664,14 @@ class ConfigCLI(CLI):
                                     sections[s].update(plugin_sections[s])
                                 else:
                                     sections[s] = plugin_sections[s]
+                if galaxy_servers:
+                    for server in galaxy_servers:
+                        server_sections = _get_ini_entries(galaxy_servers[server])
+                        for s in server_sections:
+                            if s in sections:
+                                sections[s].update(server_sections[s])
+                            else:
+                                sections[s] = server_sections[s]
                 if sections:
                     p = C.config._parsers[C.CONFIG_FILE]
                     for s in p.sections():

lib/ansible/cli/galaxy.py

@@ -55,7 +55,6 @@ from ansible.module_utils.common.yaml import yaml_dump, yaml_load
 from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
 from ansible.module_utils import six
 from ansible.parsing.dataloader import DataLoader
-from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.playbook.role.requirement import RoleRequirement
 from ansible.template import Templar
 from ansible.utils.collection_loader import AnsibleCollectionConfig
@@ -66,27 +65,6 @@ from ansible.utils.vars import load_extra_vars
 display = Display()
 urlparse = six.moves.urllib.parse.urlparse
 
-# config definition by position: name, required, type
-SERVER_DEF = [
-    ('url', True, 'str'),
-    ('username', False, 'str'),
-    ('password', False, 'str'),
-    ('token', False, 'str'),
-    ('auth_url', False, 'str'),
-    ('api_version', False, 'int'),
-    ('validate_certs', False, 'bool'),
-    ('client_id', False, 'str'),
-    ('timeout', False, 'int'),
-]
-
-# config definition fields
-SERVER_ADDITIONAL = {
-    'api_version': {'default': None, 'choices': [2, 3]},
-    'validate_certs': {'cli': [{'name': 'validate_certs'}]},
-    'timeout': {'default': C.GALAXY_SERVER_TIMEOUT, 'cli': [{'name': 'timeout'}]},
-    'token': {'default': None},
-}
-
 
 def with_collection_artifacts_manager(wrapped_method):
     """Inject an artifacts manager if not passed explicitly.
@@ -618,25 +596,8 @@ class GalaxyCLI(CLI):
 
         self.galaxy = Galaxy()
 
-        def server_config_def(section, key, required, option_type):
-            config_def = {
-                'description': 'The %s of the %s Galaxy server' % (key, section),
-                'ini': [
-                    {
-                        'section': 'galaxy_server.%s' % section,
-                        'key': key,
-                    }
-                ],
-                'env': [
-                    {'name': 'ANSIBLE_GALAXY_SERVER_%s_%s' % (section.upper(), key.upper())},
-                ],
-                'required': required,
-                'type': option_type,
-            }
-            if key in SERVER_ADDITIONAL:
-                config_def.update(SERVER_ADDITIONAL[key])
-
-            return config_def
+        # dynamically add per server config depending on declared servers
+        C.config.load_galaxy_server_defs(C.GALAXY_SERVER_LIST)
 
         galaxy_options = {}
         for optional_key in ['clear_response_cache', 'no_cache']:
@@ -644,19 +605,12 @@ class GalaxyCLI(CLI):
                 galaxy_options[optional_key] = context.CLIARGS[optional_key]
 
         config_servers = []
-
         # Need to filter out empty strings or non truthy values as an empty server list env var is equal to [''].
         server_list = [s for s in C.GALAXY_SERVER_LIST or [] if s]
         for server_priority, server_key in enumerate(server_list, start=1):
-            # Abuse the 'plugin config' by making 'galaxy_server' a type of plugin
-            # Config definitions are looked up dynamically based on the C.GALAXY_SERVER_LIST entry. We look up the
-            # section [galaxy_server.<server>] for the values url, username, password, and token.
-            config_dict = dict((k, server_config_def(server_key, k, req, ensure_type)) for k, req, ensure_type in SERVER_DEF)
-            defs = AnsibleLoader(yaml_dump(config_dict)).get_single_data()
-            C.config.initialize_plugin_configuration_definitions('galaxy_server', server_key, defs)
 
             # resolve the config created options above with existing config and user options
-            server_options = C.config.get_plugin_options('galaxy_server', server_key)
+            server_options = C.config.get_plugin_options(plugin_type='galaxy_server', name=server_key)
 
             # auth_url is used to create the token, but not directly by GalaxyAPI, so
             # it doesn't need to be passed as kwarg to GalaxyApi, same for others we pop here

lib/ansible/config/manager.py

@@ -29,6 +29,26 @@ Setting = namedtuple('Setting', 'name value origin type')
 
 INTERNAL_DEFS = {'lookup': ('_terms',)}
 
+GALAXY_SERVER_DEF = [
+    ('url', True, 'str'),
+    ('username', False, 'str'),
+    ('password', False, 'str'),
+    ('token', False, 'str'),
+    ('auth_url', False, 'str'),
+    ('api_version', False, 'int'),
+    ('validate_certs', False, 'bool'),
+    ('client_id', False, 'str'),
+    ('timeout', False, 'int'),
+]
+
+# config definition fields
+GALAXY_SERVER_ADDITIONAL = {
+    'api_version': {'default': None, 'choices': [2, 3]},
+    'validate_certs': {'cli': [{'name': 'validate_certs'}]},
+    'timeout': {'cli': [{'name': 'timeout'}]},
+    'token': {'default': None},
+}
+
 
 def _get_entry(plugin_type, plugin_name, config):
     ''' construct entry for requested config '''
@@ -302,6 +322,42 @@ class ConfigManager(object):
         # ensure we always have config def entry
         self._base_defs['CONFIG_FILE'] = {'default': None, 'type': 'path'}
 
+    def load_galaxy_server_defs(self, server_list):
+
+        def server_config_def(section, key, required, option_type):
+            config_def = {
+                'description': 'The %s of the %s Galaxy server' % (key, section),
+                'ini': [
+                    {
+                        'section': 'galaxy_server.%s' % section,
+                        'key': key,
+                    }
+                ],
+                'env': [
+                    {'name': 'ANSIBLE_GALAXY_SERVER_%s_%s' % (section.upper(), key.upper())},
+                ],
+                'required': required,
+                'type': option_type,
+            }
+            if key in GALAXY_SERVER_ADDITIONAL:
+                config_def.update(GALAXY_SERVER_ADDITIONAL[key])
+                # ensure we always have a default timeout
+                if key == 'timeout' and 'default' not in config_def:
+                    config_def['default'] = self.get_config_value('GALAXY_SERVER_TIMEOUT')
+
+            return config_def
+
+        if server_list:
+            for server_key in server_list:
+                if not server_key:
+                    # To filter out empty strings or non truthy values as an empty server list env var is equal to [''].
+                    continue
+
+                # Config definitions are looked up dynamically based on the C.GALAXY_SERVER_LIST entry. We look up the
+                # section [galaxy_server.<server>] for the values url, username, password, and token.
+                defs = dict((k, server_config_def(server_key, k, req, value_type)) for k, req, value_type in GALAXY_SERVER_DEF)
+                self.initialize_plugin_configuration_definitions('galaxy_server', server_key, defs)
+
     def template_default(self, value, variables):
         if isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')) and variables is not None:
             # template default values if possible
@@ -357,7 +413,7 @@ class ConfigManager(object):
     def get_plugin_options(self, plugin_type, name, keys=None, variables=None, direct=None):
 
         options = {}
-        defs = self.get_configuration_definitions(plugin_type, name)
+        defs = self.get_configuration_definitions(plugin_type=plugin_type, name=name)
         for option in defs:
             options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables, direct=direct)
 
@@ -366,7 +422,7 @@ class ConfigManager(object):
     def get_plugin_vars(self, plugin_type, name):
 
         pvars = []
-        for pdef in self.get_configuration_definitions(plugin_type, name).values():
+        for pdef in self.get_configuration_definitions(plugin_type=plugin_type, name=name).values():
             if 'vars' in pdef and pdef['vars']:
                 for var_entry in pdef['vars']:
                     pvars.append(var_entry['name'])
@@ -375,7 +431,7 @@ class ConfigManager(object):
     def get_plugin_options_from_var(self, plugin_type, name, variable):
 
         options = []
-        for option_name, pdef in self.get_configuration_definitions(plugin_type, name).items():
+        for option_name, pdef in self.get_configuration_definitions(plugin_type=plugin_type, name=name).items():
             if 'vars' in pdef and pdef['vars']:
                 for var_entry in pdef['vars']:
                     if variable == var_entry['name']:
@@ -417,7 +473,6 @@ class ConfigManager(object):
             for cdef in list(ret.keys()):
                 if cdef.startswith('_'):
                     del ret[cdef]
-
         return ret
 
     def _loop_entries(self, container, entry_list):
@@ -472,7 +527,7 @@ class ConfigManager(object):
         origin = None
         origin_ftype = None
 
-        defs = self.get_configuration_definitions(plugin_type, plugin_name)
+        defs = self.get_configuration_definitions(plugin_type=plugin_type, name=plugin_name)
         if config in defs:
 
             aliases = defs[config].get('aliases', [])

test/integration/targets/ansible-config/files/galaxy_server.ini

@@ -0,0 +1,21 @@
+[galaxy]
+server_list = my_org_hub, release_galaxy, test_galaxy, my_galaxy_ng
+
+[galaxy_server.my_org_hub]
+url=https://automation.my_org/
+username=my_user
+password=my_pass
+
+[galaxy_server.release_galaxy]
+url=https://galaxy.ansible.com/
+token=my_token
+
+[galaxy_server.test_galaxy]
+url=https://galaxy-dev.ansible.com/
+token=my_test_token
+
+[galaxy_server.my_galaxy_ng]
+url=http://my_galaxy_ng:8000/api/automation-hub/
+auth_url=http://my_keycloak:8080/auth/realms/myco/protocol/openid-connect/token
+client_id=galaxy-ng
+token=my_keycloak_access_token

test/integration/targets/ansible-config/tasks/main.yml

@@ -56,3 +56,79 @@
       that:
           - valid_env is success
           - invalid_env is failed
+
+
+- name: dump galaxy_server config
+  environment:
+    ANSIBLE_CONFIG: '{{ role_path }}/files/galaxy_server.ini'
+  vars:
+    expected:
+      my_org_hub:
+        url:
+          value: "https://automation.my_org/"
+          origin: role_path ~ "/files/galaxy_server.ini"
+        username:
+          value: my_user
+          origin: role_path ~ "/files/galaxy_server.ini"
+        password:
+          value: my_pass
+          origin: role_path ~ "/files/galaxy_server.ini"
+        api_version:
+          value: None
+          origin: default
+      release_galaxy:
+      test_galaxy:
+      my_galaxy_ng:
+  block:
+  - ansible.builtin.command: ansible-config dump --type {{ item }} --format json
+    loop:
+    - base
+    - all
+    register: galaxy_server_dump
+
+  - debug: msg='{{ (galaxy_server_dump.results[0].stdout | from_json) }}'
+
+  - set_fact:
+      galaxy_server_dump_base: '{{ (galaxy_server_dump.results[0].stdout | from_json).get("GALAXY_SERVERS") }}'
+      #galaxy_server_dump_all: '{{ (galaxy_server_dump.results[1].stdout | from_json | rejectattr("name", "defined")) }}'
+
+  - name: Check galaxy server values are present and match expectations
+    vars:
+      gs:
+        my_org_hub:
+          url: "https://automation.my_org/"
+          username: "my_user"
+          password: "my_pass"
+        release_galaxy:
+          url: "https://galaxy.ansible.com/"
+          token: "my_token"
+        test_galaxy:
+          url: "https://galaxy-dev.ansible.com/"
+          token: "my_test_token"
+        my_galaxy_ng:
+          url: "http://my_galaxy_ng:8000/api/automation-hub/"
+          token: "my_keycloak_access_token"
+          auth_url: "http://my_keycloak:8080/auth/realms/myco/protocol/openid-connect/token"
+          client_id: "galaxy-ng"
+      gs_all:
+          url:
+          token:
+          auth_url:
+          username:
+          password:
+          api_version:
+          timeout:
+      origin: '{{ role_path ~ "/files/galaxy_server.ini" }}'
+    block:
+    - debug: var=galaxy_server_dump_base[2]
+    - name: Check galaxy server config reflects what we expect
+      assert:
+        that:
+        - (galaxy_server_dump_base | count) == 4
+
+
+        - galaxy_server_dump_base[2].keys() | list == ["GALAXY_SERVER.TEST_GALAXY"]
+        #- (galaxy_server_dump_base[2].test_galaxy | selectattr("name", "equalto", "url"))[0].value == "https://galaxy-dev.ansible.com/"
+
+
+        - galaxy_server_dump_base == galaxy_server_dump_all

test/units/galaxy/test_collection.py

@@ -18,8 +18,8 @@ from unittest.mock import MagicMock, mock_open, patch
 
 import ansible.constants as C
 from ansible import context
-from ansible.cli import galaxy
 from ansible.cli.galaxy import GalaxyCLI
+from ansible.config import manager
 from ansible.errors import AnsibleError
 from ansible.galaxy import api, collection, token
 from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
@@ -414,9 +414,9 @@ def test_timeout_server_config(timeout_cli, timeout_cfg, timeout_fallback, expec
         cfg_lines.append(f"server_timeout={timeout_fallback}")
 
         # fix default in server config since C.GALAXY_SERVER_TIMEOUT was already evaluated
-        server_additional = galaxy.SERVER_ADDITIONAL.copy()
+        server_additional = manager.GALAXY_SERVER_ADDITIONAL.copy()
         server_additional['timeout']['default'] = timeout_fallback
-        monkeypatch.setattr(galaxy, 'SERVER_ADDITIONAL', server_additional)
+        monkeypatch.setattr(manager, 'GALAXY_SERVER_ADDITIONAL', server_additional)
 
     cfg_lines.extend(["[galaxy_server.server1]", "url=https://galaxy.ansible.com/api/"])
     if timeout_cfg is not None:

test/units/galaxy/test_token.py

@@ -9,7 +9,8 @@ import pytest
 from unittest.mock import MagicMock
 
 import ansible.constants as C
-from ansible.cli.galaxy import GalaxyCLI, SERVER_DEF
+from ansible.cli.galaxy import GalaxyCLI
+from ansible.config import manager
 from ansible.galaxy.token import GalaxyToken, NoTokenSentinel
 from ansible.module_utils.common.text.converters import to_bytes, to_text
 
@@ -35,7 +36,7 @@ def b_token_file(request, tmp_path_factory):
 def test_client_id(monkeypatch):
     monkeypatch.setattr(C, 'GALAXY_SERVER_LIST', ['server1', 'server2'])
 
-    test_server_config = {option[0]: None for option in SERVER_DEF}
+    test_server_config = {option[0]: None for option in manager.GALAXY_SERVER_DEF}
     test_server_config.update(
         {
             'url': 'http://my_galaxy_ng:8000/api/automation-hub/',
@@ -45,7 +46,7 @@ def test_client_id(monkeypatch):
         }
     )
 
-    test_server_default = {option[0]: None for option in SERVER_DEF}
+    test_server_default = {option[0]: None for option in manager.GALAXY_SERVER_DEF}
     test_server_default.update(
         {
             'url': 'https://cloud.redhat.com/api/automation-hub/',