Config, ensure templating happens at functions (#77483)

move away from special templating in constants.py
 simplified main get, unified paths to get info

* Let native jinja deal with eval for type
* make constants available for templating defaults
* ensure we use config file

changelogs/fragments/ensure_config_always_templated.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - config, ensure that pulling values from configmanager are templated if possible.

lib/ansible/cli/config.py

@@ -35,6 +35,13 @@ from ansible.utils.path import unfrackpath
 display = Display()
 
 
+def get_constants():
+    ''' helper method to ensure we can template based on existing constants '''
+    if not hasattr(get_constants, 'cvars'):
+        get_constants.cvars = {k: getattr(C, k) for k in dir(C) if not k.startswith('__')}
+    return get_constants.cvars
+
+
 class ConfigCLI(CLI):
     """ Config command line class """
 
@@ -395,9 +402,9 @@ class ConfigCLI(CLI):
 
     def _get_global_configs(self):
         config = self.config.get_configuration_definitions(ignore_private=True).copy()
-        for setting in self.config.data.get_settings():
-            if setting.name in config:
-                config[setting.name] = setting
+        for setting in config.keys():
+            v, o = C.config.get_config_value_and_origin(setting, cfile=self.config_file, variables=get_constants())
+            config[setting] = Setting(setting, v, o, None)
 
         return self._render_settings(config)
 
@@ -445,7 +452,7 @@ class ConfigCLI(CLI):
             # actually get the values
             for setting in config_entries[finalname].keys():
                 try:
-                    v, o = C.config.get_config_value_and_origin(setting, plugin_type=ptype, plugin_name=name)
+                    v, o = C.config.get_config_value_and_origin(setting, cfile=self.config_file, plugin_type=ptype, plugin_name=name, variables=get_constants())
                 except AnsibleError as e:
                     if to_text(e).startswith('No setting was provided for required configuration'):
                         v = None

lib/ansible/config/manager.py

@@ -13,9 +13,9 @@ import stat
 import tempfile
 import traceback
 
-from collections.abc import Mapping, Sequence
-
 from collections import namedtuple
+from collections.abc import Mapping, Sequence
+from jinja2.nativetypes import NativeEnvironment
 
 from ansible.config.data import ConfigData
 from ansible.errors import AnsibleOptionsError, AnsibleError
@@ -302,7 +302,8 @@ class ConfigManager(object):
             self._parse_config_file()
 
         # update constants
-        self.update_config_data()
+        self.update_config_data(self._base_defs, self._config_file)
+        self._base_defs['CONFIG_FILE'] = {'default': None, 'type': 'path'}
 
     def _read_config_yaml_file(self, yml_file):
         # TODO: handle relative paths as relative to the directory containing the current playbook instead of CWD
@@ -447,6 +448,9 @@ class ConfigManager(object):
             # use default config
             cfile = self._config_file
 
+        if config == 'CONFIG_FILE':
+            return cfile, ''
+
         # Note: sources that are lists listed in low to high precedence (last one wins)
         value = None
         origin = None
@@ -457,19 +461,18 @@ class ConfigManager(object):
             aliases = defs[config].get('aliases', [])
 
             # direct setting via plugin arguments, can set to None so we bypass rest of processing/defaults
-            direct_aliases = []
             if direct:
-                direct_aliases = [direct[alias] for alias in aliases if alias in direct]
-            if direct and config in direct:
+                if config in direct:
                     value = direct[config]
                     origin = 'Direct'
-            elif direct and direct_aliases:
+                else:
+                    direct_aliases = [direct[alias] for alias in aliases if alias in direct]
+                    if direct_aliases:
                         value = direct_aliases[0]
                         origin = 'Direct'
 
-            else:
+            if value is None and variables and defs[config].get('vars'):
                 # Use 'variable overrides' if present, highest precedence, but only present when querying running play
-                if variables and defs[config].get('vars'):
                 value, origin = self._loop_entries(variables, defs[config]['vars'])
                 origin = 'var: %s' % origin
 
@@ -536,11 +539,16 @@ class ConfigManager(object):
                         raise AnsibleError("No setting was provided for required configuration %s" %
                                            to_native(_get_entry(plugin_type, plugin_name, config)))
                 else:
-                        value = defs[config].get('default')
                     origin = 'default'
-                        # skip typing as this is a templated default that will be resolved later in constants, which has needed vars
-                        if plugin_type is None and isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')):
-                            return value, origin
+                    value = defs[config].get('default')
+                    if isinstance(value, string_types) and (value.startswith('{{') and value.endswith('}}')) and variables is not None:
+                        # template default values if possible
+                        # NOTE: cannot use is_template due to circular dep
+                        try:
+                            t = NativeEnvironment().from_string(value)
+                            value = t.render(variables)
+                        except Exception:
+                            pass  # not templatable
 
             # ensure correct type, can raise exceptions on mismatched types
             try:
@@ -598,19 +606,16 @@ class ConfigManager(object):
 
         if defs is None:
             defs = self._base_defs
-
         if configfile is None:
             configfile = self._config_file
 
         if not isinstance(defs, dict):
             raise AnsibleOptionsError("Invalid configuration definition type: %s for %s" % (type(defs), defs))
 
-        # update the constant for config file
-        self.data.update_setting(Setting('CONFIG_FILE', configfile, '', 'string'))
-
         origin = None
         # env and config defs can have several entries, ordered in list from lowest to highest precedence
         for config in defs:
+
             if not isinstance(defs[config], dict):
                 raise AnsibleOptionsError("Invalid configuration definition '%s': type is %s" % (to_native(config), type(defs[config])))
 
@@ -632,3 +637,6 @@ class ConfigManager(object):
 
             # set the constant
             self.data.update_setting(Setting(config, value, origin, defs[config].get('type', 'string')))
+
+        # update the constant for config file
+        self.data.update_setting(Setting('CONFIG_FILE', configfile, '', 'string'))

lib/ansible/constants.py

@@ -7,11 +7,9 @@ __metaclass__ = type
 
 import re
 
-from ast import literal_eval
-from jinja2 import Template
 from string import ascii_letters, digits
 
-from ansible.config.manager import ConfigManager, ensure_type
+from ansible.config.manager import ConfigManager
 from ansible.module_utils._text import to_text
 from ansible.module_utils.common.collections import Sequence
 from ansible.module_utils.parsing.convert_bool import BOOLEANS_TRUE
@@ -181,25 +179,8 @@ MAGIC_VARIABLE_MAPPING = dict(
 config = ConfigManager()
 
 # Generate constants from config
-for setting in config.data.get_settings():
-
-    value = setting.value
-    if setting.origin == 'default' and \
-       isinstance(setting.value, string_types) and \
-       (setting.value.startswith('{{') and setting.value.endswith('}}')):
-        try:
-            t = Template(setting.value)
-            value = t.render(vars())
-            try:
-                value = literal_eval(value)
-            except ValueError:
-                pass  # not a python data structure
-        except Exception:
-            pass  # not templatable
-
-        value = ensure_type(value, setting.type)
-
-    set_constant(setting.name, value)
+for setting in config.get_configuration_definitions():
+    set_constant(setting, config.get_config_value(setting, variables=vars()))
 
 for warn in config.WARNINGS:
     _warning(warn)