minimize loading files/plugins

avoids some repetitive loading
 - read config file only once
 - now cache the ini parser per file
 - optimize shell plugin loading

tried to 'optimize' vars_plugins loading but it creates issues with precedence,
probalby due to iterator not being reset, will look into it in subsequent fix/PR

(cherry picked from commit 42912e1ac8)

changelogs/fragments/load_files_once.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - avoid uneeded reloading of plugin files https://github.com/ansible/ansible/pull/37648

lib/ansible/config/manager.py

@@ -173,7 +173,7 @@ class ConfigManager(object):
 
         self._base_defs = {}
         self._plugins = {}
-        self._parser = None
+        self._parsers = {}
 
         self._config_file = conf_file
         self.data = ConfigData()
@@ -214,15 +214,15 @@ class ConfigManager(object):
         ftype = get_config_type(cfile)
         if cfile is not None:
             if ftype == 'ini':
-                self._parser = configparser.ConfigParser()
+                self._parsers[cfile] = configparser.ConfigParser()
                 try:
-                    self._parser.read(cfile)
+                    self._parsers[cfile].read(cfile)
                 except configparser.Error as e:
                     raise AnsibleOptionsError("Error reading config file (%s): %s" % (cfile, to_native(e)))
             # FIXME: this should eventually handle yaml config files
             # elif ftype == 'yaml':
             #     with open(cfile, 'rb') as config_stream:
-            #         self._parser = yaml.safe_load(config_stream)
+            #         self._parsers[cfile] = yaml.safe_load(config_stream)
             else:
                 raise AnsibleOptionsError("Unsupported configuration file type: %s" % to_native(ftype))
 
@@ -288,9 +288,8 @@ class ConfigManager(object):
         ''' Given a config key figure out the actual value and report on the origin of the settings '''
 
         if cfile is None:
+            # use default config
             cfile = self._config_file
-        else:
-            self._parse_config_file(cfile)
 
         # Note: sources that are lists listed in low to high precedence (last one wins)
         value = None
@@ -320,6 +319,9 @@ class ConfigManager(object):
                 origin = 'env: %s' % origin
 
             # try config file entries next, if we have one
+            if self._parsers.get(cfile, None) is None:
+                self._parse_config_file(cfile)
+
             if value is None and cfile is not None:
                 ftype = get_config_type(cfile)
                 if ftype and defs[config].get(ftype):
@@ -327,7 +329,7 @@ class ConfigManager(object):
                         # load from ini config
                         try:  # FIXME: generalize _loop_entries to allow for files also, most of this code is dupe
                             for ini_entry in defs[config]['ini']:
-                                temp_value = get_ini_config_value(self._parser, ini_entry)
+                                temp_value = get_ini_config_value(self._parsers[cfile], ini_entry)
                                 if temp_value is not None:
                                     value = temp_value
                                     origin = cfile

lib/ansible/plugins/connection/__init__.py

@@ -91,10 +91,15 @@ class ConnectionBase(AnsiblePlugin):
         else:
             shell_type = 'sh'
             shell_filename = os.path.basename(self._play_context.executable)
-            for shell in shell_loader.all():
+            try:
-                if shell_filename in shell.COMPATIBLE_SHELLS:
+                shell = shell_loader.get(shell_filename)
-                    shell_type = shell.SHELL_FAMILY
+            except Exception:
-                    break
+                shell = None
+            if shell is None:
+                for shell in shell_loader.all():
+                    if shell_filename in shell.COMPATIBLE_SHELLS:
+                        break
+            shell_type = shell.SHELL_FAMILY
 
         self._shell = shell_loader.get(shell_type)
         if not self._shell:

lib/ansible/vars/manager.py

@@ -241,6 +241,7 @@ class VariableManager:
                         inventory_dir = os.path.dirname(inventory_dir)
 
                     for plugin in vars_loader.all():
+
                         data = combine_vars(data, _get_plugin_vars(plugin, inventory_dir, entities))
                 return data
 
@@ -248,6 +249,7 @@ class VariableManager:
                 ''' merges all entities adjacent to play '''
                 data = {}
                 for plugin in vars_loader.all():
+
                     for path in basedirs:
                         data = combine_vars(data, _get_plugin_vars(plugin, path, entities))
                 return data