config, integrate dynamic galaxy servers (#83129)

Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>
pull/34228/head
Brian Coca 6 months ago committed by GitHub
parent 375d3889de
commit 949c503f2e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -22,7 +22,7 @@ import ansible.plugins.loader as plugin_loader
from ansible import constants as C
from ansible.cli.arguments import option_helpers as opt_help
from ansible.config.manager import ConfigManager, Setting
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleRequiredOptionError
from ansible.module_utils.common.text.converters import to_native, to_text, to_bytes
from ansible.module_utils.common.json import json_dump
from ansible.module_utils.six import string_types
@ -35,6 +35,9 @@ from ansible.utils.path import unfrackpath
display = Display()
_IGNORE_CHANGED = frozenset({'_terms', '_input'})
def yaml_dump(data, default_flow_style=False, default_style=None):
return yaml.dump(data, Dumper=AnsibleDumper, default_flow_style=default_flow_style, default_style=default_style)
@ -149,6 +152,10 @@ class ConfigCLI(CLI):
super(ConfigCLI, self).run()
# initialize each galaxy server's options from known listed servers
self._galaxy_servers = [s for s in C.GALAXY_SERVER_LIST or [] if s] # clean list, reused later here
C.config.load_galaxy_server_defs(self._galaxy_servers)
if context.CLIARGS['config_file']:
self.config_file = unfrackpath(context.CLIARGS['config_file'], follow=False)
b_config = to_bytes(self.config_file)
@ -262,11 +269,17 @@ class ConfigCLI(CLI):
'''
build a dict with the list requested configs
'''
config_entries = {}
if context.CLIARGS['type'] in ('base', 'all'):
# this dumps main/common configs
config_entries = self.config.get_configuration_definitions(ignore_private=True)
# for base and all, we include galaxy servers
config_entries['GALAXY_SERVERS'] = {}
for server in self._galaxy_servers:
config_entries['GALAXY_SERVERS'][server] = self.config.get_configuration_definitions('galaxy_server', server)
if context.CLIARGS['type'] != 'base':
config_entries['PLUGINS'] = {}
@ -445,13 +458,13 @@ class ConfigCLI(CLI):
entries = []
for setting in sorted(config):
changed = (config[setting].origin not in ('default', 'REQUIRED'))
changed = (config[setting].origin not in ('default', 'REQUIRED') and setting not in _IGNORE_CHANGED)
if context.CLIARGS['format'] == 'display':
if isinstance(config[setting], Setting):
# proceed normally
value = config[setting].value
if config[setting].origin == 'default':
if config[setting].origin == 'default' or setting in _IGNORE_CHANGED:
color = 'green'
value = self.config.template_default(value, get_constants())
elif config[setting].origin == 'REQUIRED':
@ -468,6 +481,8 @@ class ConfigCLI(CLI):
else:
entry = {}
for key in config[setting]._fields:
if key == 'type':
continue
entry[key] = getattr(config[setting], key)
if not context.CLIARGS['only_changed'] or changed:
@ -476,7 +491,10 @@ class ConfigCLI(CLI):
return entries
def _get_global_configs(self):
config = self.config.get_configuration_definitions(ignore_private=True).copy()
# Add base
config = self.config.get_configuration_definitions(ignore_private=True)
# convert to settings
for setting in config.keys():
v, o = C.config.get_config_value_and_origin(setting, cfile=self.config_file, variables=get_constants())
config[setting] = Setting(setting, v, o, None)
@ -528,12 +546,9 @@ class ConfigCLI(CLI):
for setting in config_entries[finalname].keys():
try:
v, o = C.config.get_config_value_and_origin(setting, cfile=self.config_file, plugin_type=ptype, plugin_name=name, variables=get_constants())
except AnsibleError as e:
if to_text(e).startswith('No setting was provided for required configuration'):
except AnsibleRequiredOptionError:
v = None
o = 'REQUIRED'
else:
raise e
if v is None and o is None:
# not all cases will be error
@ -553,17 +568,60 @@ class ConfigCLI(CLI):
return output
def _get_galaxy_server_configs(self):
output = []
# add galaxy servers
for server in self._galaxy_servers:
server_config = {}
s_config = self.config.get_configuration_definitions('galaxy_server', server)
for setting in s_config.keys():
try:
v, o = C.config.get_config_value_and_origin(setting, plugin_type='galaxy_server', plugin_name=server, cfile=self.config_file)
except AnsibleError as e:
if s_config[setting].get('required', False):
v = None
o = 'REQUIRED'
else:
raise e
if v is None and o is None:
# not all cases will be error
o = 'REQUIRED'
server_config[setting] = Setting(setting, v, o, None)
if context.CLIARGS['format'] == 'display':
if not context.CLIARGS['only_changed'] or server_config:
equals = '=' * len(server)
output.append(f'\n{server}\n{equals}')
output.extend(self._render_settings(server_config))
else:
output.append({server: server_config})
return output
def execute_dump(self):
'''
Shows the current settings, merges ansible.cfg if specified
'''
if context.CLIARGS['type'] == 'base':
# deal with base
output = self._get_global_configs()
elif context.CLIARGS['type'] == 'all':
output = []
if context.CLIARGS['type'] in ('base', 'all'):
# deal with base
output = self._get_global_configs()
# deal with plugins
# add galaxy servers
server_config_list = self._get_galaxy_server_configs()
if context.CLIARGS['format'] == 'display':
output.append('\nGALAXY_SERVERS:\n')
output.extend(server_config_list)
else:
configs = {}
for server_config in server_config_list:
server = list(server_config.keys())[0]
server_reduced_config = server_config.pop(server)
configs[server] = server_reduced_config
output.append({'GALAXY_SERVERS': configs})
if context.CLIARGS['type'] == 'all':
# add all plugins
for ptype in C.CONFIGURABLE_PLUGINS:
plugin_list = self._get_plugin_configs(ptype, context.CLIARGS['args'])
if context.CLIARGS['format'] == 'display':
@ -576,8 +634,9 @@ class ConfigCLI(CLI):
else:
pname = '%s_PLUGINS' % ptype.upper()
output.append({pname: plugin_list})
else:
# deal with plugins
elif context.CLIARGS['type'] != 'base':
# deal with specific plugin
output = self._get_plugin_configs(context.CLIARGS['type'], context.CLIARGS['args'])
if context.CLIARGS['format'] == 'display':
@ -594,6 +653,7 @@ class ConfigCLI(CLI):
found = False
config_entries = self._list_entries_from_args()
plugin_types = config_entries.pop('PLUGINS', None)
galaxy_servers = config_entries.pop('GALAXY_SERVERS', None)
if context.CLIARGS['format'] == 'ini':
if C.CONFIG_FILE is not None:
@ -610,6 +670,14 @@ class ConfigCLI(CLI):
sections[s].update(plugin_sections[s])
else:
sections[s] = plugin_sections[s]
if galaxy_servers:
for server in galaxy_servers:
server_sections = _get_ini_entries(galaxy_servers[server])
for s in server_sections:
if s in sections:
sections[s].update(server_sections[s])
else:
sections[s] = server_sections[s]
if sections:
p = C.config._parsers[C.CONFIG_FILE]
for s in p.sections():

@ -55,7 +55,6 @@ from ansible.module_utils.common.yaml import yaml_dump, yaml_load
from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
from ansible.module_utils import six
from ansible.parsing.dataloader import DataLoader
from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.playbook.role.requirement import RoleRequirement
from ansible.template import Templar
from ansible.utils.collection_loader import AnsibleCollectionConfig
@ -66,27 +65,6 @@ from ansible.utils.vars import load_extra_vars
display = Display()
urlparse = six.moves.urllib.parse.urlparse
# config definition by position: name, required, type
SERVER_DEF = [
('url', True, 'str'),
('username', False, 'str'),
('password', False, 'str'),
('token', False, 'str'),
('auth_url', False, 'str'),
('api_version', False, 'int'),
('validate_certs', False, 'bool'),
('client_id', False, 'str'),
('timeout', False, 'int'),
]
# config definition fields
SERVER_ADDITIONAL = {
'api_version': {'default': None, 'choices': [2, 3]},
'validate_certs': {'cli': [{'name': 'validate_certs'}]},
'timeout': {'default': C.GALAXY_SERVER_TIMEOUT, 'cli': [{'name': 'timeout'}]},
'token': {'default': None},
}
def with_collection_artifacts_manager(wrapped_method):
"""Inject an artifacts manager if not passed explicitly.
@ -618,25 +596,8 @@ class GalaxyCLI(CLI):
self.galaxy = Galaxy()
def server_config_def(section, key, required, option_type):
config_def = {
'description': 'The %s of the %s Galaxy server' % (key, section),
'ini': [
{
'section': 'galaxy_server.%s' % section,
'key': key,
}
],
'env': [
{'name': 'ANSIBLE_GALAXY_SERVER_%s_%s' % (section.upper(), key.upper())},
],
'required': required,
'type': option_type,
}
if key in SERVER_ADDITIONAL:
config_def.update(SERVER_ADDITIONAL[key])
return config_def
# dynamically add per server config depending on declared servers
C.config.load_galaxy_server_defs(C.GALAXY_SERVER_LIST)
galaxy_options = {}
for optional_key in ['clear_response_cache', 'no_cache']:
@ -644,19 +605,12 @@ class GalaxyCLI(CLI):
galaxy_options[optional_key] = context.CLIARGS[optional_key]
config_servers = []
# Need to filter out empty strings or non truthy values as an empty server list env var is equal to [''].
server_list = [s for s in C.GALAXY_SERVER_LIST or [] if s]
for server_priority, server_key in enumerate(server_list, start=1):
# Abuse the 'plugin config' by making 'galaxy_server' a type of plugin
# Config definitions are looked up dynamically based on the C.GALAXY_SERVER_LIST entry. We look up the
# section [galaxy_server.<server>] for the values url, username, password, and token.
config_dict = dict((k, server_config_def(server_key, k, req, ensure_type)) for k, req, ensure_type in SERVER_DEF)
defs = AnsibleLoader(yaml_dump(config_dict)).get_single_data()
C.config.initialize_plugin_configuration_definitions('galaxy_server', server_key, defs)
# resolve the config created options above with existing config and user options
server_options = C.config.get_plugin_options('galaxy_server', server_key)
server_options = C.config.get_plugin_options(plugin_type='galaxy_server', name=server_key)
# auth_url is used to create the token, but not directly by GalaxyAPI, so
# it doesn't need to be passed as kwarg to GalaxyApi, same for others we pop here

@ -15,7 +15,7 @@ from collections import namedtuple
from collections.abc import Mapping, Sequence
from jinja2.nativetypes import NativeEnvironment
from ansible.errors import AnsibleOptionsError, AnsibleError
from ansible.errors import AnsibleOptionsError, AnsibleError, AnsibleRequiredOptionError
from ansible.module_utils.common.text.converters import to_text, to_bytes, to_native
from ansible.module_utils.common.yaml import yaml_load
from ansible.module_utils.six import string_types
@ -29,6 +29,26 @@ Setting = namedtuple('Setting', 'name value origin type')
INTERNAL_DEFS = {'lookup': ('_terms',)}
GALAXY_SERVER_DEF = [
('url', True, 'str'),
('username', False, 'str'),
('password', False, 'str'),
('token', False, 'str'),
('auth_url', False, 'str'),
('api_version', False, 'int'),
('validate_certs', False, 'bool'),
('client_id', False, 'str'),
('timeout', False, 'int'),
]
# config definition fields
GALAXY_SERVER_ADDITIONAL = {
'api_version': {'default': None, 'choices': [2, 3]},
'validate_certs': {'cli': [{'name': 'validate_certs'}]},
'timeout': {'cli': [{'name': 'timeout'}]},
'token': {'default': None},
}
def _get_entry(plugin_type, plugin_name, config):
''' construct entry for requested config '''
@ -302,6 +322,42 @@ class ConfigManager(object):
# ensure we always have config def entry
self._base_defs['CONFIG_FILE'] = {'default': None, 'type': 'path'}
def load_galaxy_server_defs(self, server_list):
def server_config_def(section, key, required, option_type):
config_def = {
'description': 'The %s of the %s Galaxy server' % (key, section),
'ini': [
{
'section': 'galaxy_server.%s' % section,
'key': key,
}
],
'env': [
{'name': 'ANSIBLE_GALAXY_SERVER_%s_%s' % (section.upper(), key.upper())},
],
'required': required,
'type': option_type,
}
if key in GALAXY_SERVER_ADDITIONAL:
config_def.update(GALAXY_SERVER_ADDITIONAL[key])
# ensure we always have a default timeout
if key == 'timeout' and 'default' not in config_def:
config_def['default'] = self.get_config_value('GALAXY_SERVER_TIMEOUT')
return config_def
if server_list:
for server_key in server_list:
if not server_key:
# To filter out empty strings or non truthy values as an empty server list env var is equal to [''].
continue
# Config definitions are looked up dynamically based on the C.GALAXY_SERVER_LIST entry. We look up the
# section [galaxy_server.<server>] for the values url, username, password, and token.
defs = dict((k, server_config_def(server_key, k, req, value_type)) for k, req, value_type in GALAXY_SERVER_DEF)
self.initialize_plugin_configuration_definitions('galaxy_server', server_key, defs)
def template_default(self, value, variables):
if isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')) and variables is not None:
# template default values if possible
@ -357,7 +413,7 @@ class ConfigManager(object):
def get_plugin_options(self, plugin_type, name, keys=None, variables=None, direct=None):
options = {}
defs = self.get_configuration_definitions(plugin_type, name)
defs = self.get_configuration_definitions(plugin_type=plugin_type, name=name)
for option in defs:
options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables, direct=direct)
@ -366,7 +422,7 @@ class ConfigManager(object):
def get_plugin_vars(self, plugin_type, name):
pvars = []
for pdef in self.get_configuration_definitions(plugin_type, name).values():
for pdef in self.get_configuration_definitions(plugin_type=plugin_type, name=name).values():
if 'vars' in pdef and pdef['vars']:
for var_entry in pdef['vars']:
pvars.append(var_entry['name'])
@ -375,7 +431,7 @@ class ConfigManager(object):
def get_plugin_options_from_var(self, plugin_type, name, variable):
options = []
for option_name, pdef in self.get_configuration_definitions(plugin_type, name).items():
for option_name, pdef in self.get_configuration_definitions(plugin_type=plugin_type, name=name).items():
if 'vars' in pdef and pdef['vars']:
for var_entry in pdef['vars']:
if variable == var_entry['name']:
@ -417,7 +473,6 @@ class ConfigManager(object):
for cdef in list(ret.keys()):
if cdef.startswith('_'):
del ret[cdef]
return ret
def _loop_entries(self, container, entry_list):
@ -472,7 +527,7 @@ class ConfigManager(object):
origin = None
origin_ftype = None
defs = self.get_configuration_definitions(plugin_type, plugin_name)
defs = self.get_configuration_definitions(plugin_type=plugin_type, name=plugin_name)
if config in defs:
aliases = defs[config].get('aliases', [])
@ -562,7 +617,7 @@ class ConfigManager(object):
if value is None:
if defs[config].get('required', False):
if not plugin_type or config not in INTERNAL_DEFS.get(plugin_type, {}):
raise AnsibleError("No setting was provided for required configuration %s" %
raise AnsibleRequiredOptionError("No setting was provided for required configuration %s" %
to_native(_get_entry(plugin_type, plugin_name, config)))
else:
origin = 'default'

@ -227,6 +227,11 @@ class AnsibleOptionsError(AnsibleError):
pass
class AnsibleRequiredOptionError(AnsibleOptionsError):
''' bad or incomplete options passed '''
pass
class AnsibleParserError(AnsibleError):
''' something was detected early that is wrong about a playbook or data file '''
pass

@ -0,0 +1,21 @@
[galaxy]
server_list = my_org_hub, release_galaxy, test_galaxy, my_galaxy_ng
[galaxy_server.my_org_hub]
url=https://automation.my_org/
username=my_user
password=my_pass
[galaxy_server.release_galaxy]
url=https://galaxy.ansible.com/
token=my_token
[galaxy_server.test_galaxy]
url=https://galaxy-dev.ansible.com/
token=my_test_token
[galaxy_server.my_galaxy_ng]
url=http://my_galaxy_ng:8000/api/automation-hub/
auth_url=http://my_keycloak:8080/auth/realms/myco/protocol/openid-connect/token
client_id=galaxy-ng
token=my_keycloak_access_token

@ -56,3 +56,91 @@
that:
- valid_env is success
- invalid_env is failed
- name: dump galaxy_server config
environment:
ANSIBLE_CONFIG: '{{ role_path }}/files/galaxy_server.ini'
vars:
expected:
my_org_hub:
url:
value: "https://automation.my_org/"
origin: role_path ~ "/files/galaxy_server.ini"
username:
value: my_user
origin: role_path ~ "/files/galaxy_server.ini"
password:
value: my_pass
origin: role_path ~ "/files/galaxy_server.ini"
api_version:
value: None
origin: default
release_galaxy:
test_galaxy:
my_galaxy_ng:
block:
- ansible.builtin.command: ansible-config dump --type {{ item }} --format json
loop:
- base
- all
register: galaxy_server_dump
#- debug: msg='{{ (galaxy_server_dump.results[0].stdout | from_json) }}'
#- debug: msg='{{ (galaxy_server_dump.results[1].stdout | from_json) }}'
- name: extract galaxy servers from config dump
set_fact:
galaxy_server_dump_base: '{{ (galaxy_server_dump.results[0].stdout | from_json | select("contains", "GALAXY_SERVERS"))[0].get("GALAXY_SERVERS") }}'
galaxy_server_dump_all: '{{ (galaxy_server_dump.results[1].stdout | from_json | select("contains", "GALAXY_SERVERS"))[0].get("GALAXY_SERVERS") }}'
- name: set keys vars as we reuse a few times
set_fact:
galaxy_server_dump_base_keys: '{{ galaxy_server_dump_base.keys()|list|sort }}'
galaxy_server_dump_all_keys: '{{ galaxy_server_dump_all.keys()|list|sort }}'
- name: Check galaxy server values are present and match expectations
vars:
gs:
my_org_hub:
url: "https://automation.my_org/"
username: "my_user"
password: "my_pass"
release_galaxy:
url: "https://galaxy.ansible.com/"
token: "my_token"
test_galaxy:
url: "https://galaxy-dev.ansible.com/"
token: "my_test_token"
my_galaxy_ng:
url: "http://my_galaxy_ng:8000/api/automation-hub/"
token: "my_keycloak_access_token"
auth_url: "http://my_keycloak:8080/auth/realms/myco/protocol/openid-connect/token"
client_id: "galaxy-ng"
gs_all:
url:
token:
auth_url:
username:
password:
api_version:
timeout:
origin: '{{ role_path ~ "/files/galaxy_server.ini" }}'
gs_keys: '{{ gs.keys()|list|sort }}'
block:
- name: Check galaxy server config reflects what we expect
assert:
that:
- (galaxy_server_dump_base_keys | count) == 4
- galaxy_server_dump_base_keys == gs_keys
- (galaxy_server_dump_all_keys | count) == 4
- galaxy_server_dump_all_keys == gs_keys
- name: Check individual settings
assert:
that:
- gs[item[0]][item[1]] == galaxy_server_dump_base[item[0]][item[1]][1]
- gs[item[0]][item[1]] == galaxy_server_dump_all[item[0]][item[1]][1]
when:
- item[1] in gs[item[0]]
loop: '{{gs_keys | product(gs_all) }}'

@ -18,8 +18,8 @@ from unittest.mock import MagicMock, mock_open, patch
import ansible.constants as C
from ansible import context
from ansible.cli import galaxy
from ansible.cli.galaxy import GalaxyCLI
from ansible.config import manager
from ansible.errors import AnsibleError
from ansible.galaxy import api, collection, token
from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
@ -414,9 +414,9 @@ def test_timeout_server_config(timeout_cli, timeout_cfg, timeout_fallback, expec
cfg_lines.append(f"server_timeout={timeout_fallback}")
# fix default in server config since C.GALAXY_SERVER_TIMEOUT was already evaluated
server_additional = galaxy.SERVER_ADDITIONAL.copy()
server_additional = manager.GALAXY_SERVER_ADDITIONAL.copy()
server_additional['timeout']['default'] = timeout_fallback
monkeypatch.setattr(galaxy, 'SERVER_ADDITIONAL', server_additional)
monkeypatch.setattr(manager, 'GALAXY_SERVER_ADDITIONAL', server_additional)
cfg_lines.extend(["[galaxy_server.server1]", "url=https://galaxy.ansible.com/api/"])
if timeout_cfg is not None:

@ -9,7 +9,8 @@ import pytest
from unittest.mock import MagicMock
import ansible.constants as C
from ansible.cli.galaxy import GalaxyCLI, SERVER_DEF
from ansible.cli.galaxy import GalaxyCLI
from ansible.config import manager
from ansible.galaxy.token import GalaxyToken, NoTokenSentinel
from ansible.module_utils.common.text.converters import to_bytes, to_text
@ -35,7 +36,7 @@ def b_token_file(request, tmp_path_factory):
def test_client_id(monkeypatch):
monkeypatch.setattr(C, 'GALAXY_SERVER_LIST', ['server1', 'server2'])
test_server_config = {option[0]: None for option in SERVER_DEF}
test_server_config = {option[0]: None for option in manager.GALAXY_SERVER_DEF}
test_server_config.update(
{
'url': 'http://my_galaxy_ng:8000/api/automation-hub/',
@ -45,7 +46,7 @@ def test_client_id(monkeypatch):
}
)
test_server_default = {option[0]: None for option in SERVER_DEF}
test_server_default = {option[0]: None for option in manager.GALAXY_SERVER_DEF}
test_server_default.update(
{
'url': 'https://cloud.redhat.com/api/automation-hub/',

Loading…
Cancel
Save