exclude lookup_terms from config errors (#41740)

* exclude lookup_terms from config errors
* moved direct

(cherry picked from commit 0102e42272)

changelogs/fragments/fix_config_required.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - fixed config required handling, specifically for _terms in lookups https://github.com/ansible/ansible/pull/41740

lib/ansible/config/manager.py

@@ -30,6 +30,8 @@ from ansible.utils.path import makedirs_safe
 Plugin = namedtuple('Plugin', 'name type')
 Setting = namedtuple('Setting', 'name value origin type')
 
+INTERNAL_DEFS = {'lookup': ('_terms',)}
+
 
 # FIXME: see if we can unify in module_utils with similar function used by argspec
 def ensure_type(value, value_type, origin=None):
@@ -215,6 +217,11 @@ class ConfigManager(object):
         if cfile is not None:
             if ftype == 'ini':
                 self._parsers[cfile] = configparser.ConfigParser()
+                with open(cfile, 'rb') as f:
+                    try:
+                        cfg_text = to_text(f.read(), errors='surrogate_or_strict')
+                    except UnicodeError as e:
+                        raise AnsibleOptionsError("Error reading config file(%s) because the config file was not utf8 encoded: %s" % (cfile, to_native(e)))
                 try:
                     self._parsers[cfile].read(cfile)
                 except configparser.Error as e:
@@ -230,12 +237,12 @@ class ConfigManager(object):
         ''' Load YAML Config Files in order, check merge flags, keep origin of settings'''
         pass
 
-    def get_plugin_options(self, plugin_type, name, keys=None, variables=None):
+    def get_plugin_options(self, plugin_type, name, keys=None, variables=None, direct=None):
 
         options = {}
         defs = self.get_configuration_definitions(plugin_type, name)
         for option in defs:
-            options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables)
+            options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables, direct=direct)
 
         return options
 
@@ -279,17 +286,17 @@ class ConfigManager(object):
 
         return value, origin
 
-    def get_config_value(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None):
+    def get_config_value(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None, direct=None):
         ''' wrapper '''
 
         try:
             value, _drop = self.get_config_value_and_origin(config, cfile=cfile, plugin_type=plugin_type, plugin_name=plugin_name,
-                                                            keys=keys, variables=variables)
+                                                            keys=keys, variables=variables, direct=direct)
         except Exception as e:
             raise AnsibleError("Invalid settings supplied for %s: %s" % (config, to_native(e)))
         return value
 
-    def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None):
+    def get_config_value_and_origin(self, config, cfile=None, plugin_type=None, plugin_name=None, keys=None, variables=None, direct=None):
         ''' Given a config key figure out the actual value and report on the origin of the settings '''
 
         if cfile is None:
@@ -308,13 +315,20 @@ class ConfigManager(object):
             defs = self._plugins[plugin_type][plugin_name]
 
         if config in defs:
+
+            # direct setting via plugin arguments, can set to None so we bypass rest of processing/defaults
+            if direct and config in direct:
+                value = direct[config]
+                origin = 'Direct'
+
+            else:
                 # Use 'variable overrides' if present, highest precedence, but only present when querying running play
                 if variables and defs[config].get('vars'):
                     value, origin = self._loop_entries(variables, defs[config]['vars'])
                     origin = 'var: %s' % origin
 
                 # use playbook keywords if you have em
-            if value is None and keys:
+                if value is None and keys and defs[config].get('keywords'):
                     value, origin = self._loop_entries(keys, defs[config]['keywords'])
                     origin = 'keyword: %s' % origin
 
@@ -355,6 +369,7 @@ class ConfigManager(object):
                             if plugin_name:
                                 entry += 'plugin: %s ' % plugin_name
                         entry += 'setting: %s ' % config
+                        if not plugin_type or config not in INTERNAL_DEFS.get(plugin_type, {}):
                             raise AnsibleError("No setting was provided for required configuration %s" % (entry))
                     else:
                         value = defs[config].get('default')

lib/ansible/plugins/__init__.py

@@ -73,14 +73,7 @@ class AnsiblePlugin(with_metaclass(ABCMeta, object)):
 
         if not self._options:
             # load config options if we have not done so already, if vars provided we should be mostly done
-            self._options = C.config.get_plugin_options(get_plugin_class(self), self._load_name, keys=task_keys, variables=var_options)
+            self._options = C.config.get_plugin_options(get_plugin_class(self), self._load_name, keys=task_keys, variables=var_options, direct=direct)
-
-        # they can be direct options overriding config
-        if direct:
-            for k in self._options:
-                if k in direct:
-                    self.set_option(k, direct[k])
-
         # allow extras/wildcards from vars that are not directly consumed in configuration
         if self.allow_extras and var_options and '_extras' in var_options:
             self.set_option('_extras', var_options['_extras'])

lib/ansible/plugins/callback/__init__.py

@@ -98,13 +98,7 @@ class CallbackBase(AnsiblePlugin):
         '''
 
         # load from config
-        self._plugin_options = C.config.get_plugin_options(get_plugin_class(self), self._load_name, keys=task_keys, variables=var_options)
+        self._plugin_options = C.config.get_plugin_options(get_plugin_class(self), self._load_name, keys=task_keys, variables=var_options, direct=direct)
-
-        # or parse specific options
-        if direct:
-            for k in direct:
-                if k in self._plugin_options:
-                    self.set_option(k, direct[k])
 
     def _dump_results(self, result, indent=None, sort_keys=True, keep_invocation=False):
 

lib/ansible/plugins/lookup/vars.py

@@ -69,7 +69,6 @@ class LookupModule(LookupBase):
             self._templar.set_available_variables(variables)
         myvars = getattr(self._templar, '_available_variables', {})
 
-        self.set_option('_terms', terms)
         self.set_options(direct=kwargs)
         default = self.get_option('default')