From f2c5c788f3692dfb575944c6f0bc907706ddab27 Mon Sep 17 00:00:00 2001 From: stanley karunditu Date: Wed, 20 Dec 2023 14:33:15 -0500 Subject: [PATCH] Add a cache for .py files that stays in sync with the self._paths cache this is based on https://github.com/ansible/ansible/pull/32609 * avoid calling glob.glob() when looking for py files in variable folders for each host listed in the inventory. In addition, I added a global cache so all plugin loaders stay in sync (not sure if this is wanted), and reset the cache when new directories are added to the loader, like the other caches. https://github.com/ansible/ansible/pull/79687 also identifies this as a hot code path. Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> --- lib/ansible/plugins/__init__.py | 1 + lib/ansible/plugins/loader.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index c083dee93e8..5a4fd08087c 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -39,6 +39,7 @@ if t.TYPE_CHECKING: MODULE_CACHE = {} # type: dict[str, dict[str, types.ModuleType]] PATH_CACHE = {} # type: dict[str, list[PluginPathContext] | None] PLUGIN_PATH_CACHE = {} # type: dict[str, dict[str, dict[str, PluginPathContext]]] +PY_FILES = {} # type: dict[str, list[str]] def get_plugin_class(obj): diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index caab2f689cc..46f6ab73001 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -29,7 +29,7 @@ from ansible.module_utils.common.text.converters import to_bytes, to_text, to_na from ansible.module_utils.six import string_types from ansible.parsing.utils.yaml import from_yaml from ansible.parsing.yaml.loader import AnsibleLoader -from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE +from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE, PY_FILES from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata from ansible.utils.display import Display @@ -229,6 +229,8 @@ class PluginLoader: PATH_CACHE[class_name] = None if class_name not in PLUGIN_PATH_CACHE: PLUGIN_PATH_CACHE[class_name] = defaultdict(dict) + if class_name not in PY_FILES: + PY_FILES[class_name] = {} # hold dirs added at runtime outside of config self._extra_dirs = [] @@ -236,6 +238,7 @@ class PluginLoader: # caches self._module_cache = MODULE_CACHE[class_name] self._paths = PATH_CACHE[class_name] + self._py_files = PY_FILES[class_name] self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name] self._plugin_instance_cache = {} if self.subdir == 'vars_plugins' else None @@ -257,12 +260,14 @@ class PluginLoader: MODULE_CACHE[self.class_name] = {} PATH_CACHE[self.class_name] = None PLUGIN_PATH_CACHE[self.class_name] = defaultdict(dict) + PY_FILES[self.class_name] = {} # reset internal caches self._module_cache = MODULE_CACHE[self.class_name] self._paths = PATH_CACHE[self.class_name] self._plugin_path_cache = PLUGIN_PATH_CACHE[self.class_name] self._plugin_instance_cache = {} if self.subdir == 'vars_plugins' else None + self._py_files = PY_FILES[self.class_name] self._searched_paths = set() def __setstate__(self, data): @@ -279,6 +284,7 @@ class PluginLoader: PATH_CACHE[class_name] = data.get('PATH_CACHE') PLUGIN_PATH_CACHE[class_name] = data.get('PLUGIN_PATH_CACHE') + PY_FILES[class_name] = data.get('PY_FILES') self.__init__(class_name, package, config, subdir, aliases, base_class) self._extra_dirs = data.get('_extra_dirs', []) @@ -300,6 +306,7 @@ class PluginLoader: _searched_paths=self._searched_paths, PATH_CACHE=PATH_CACHE[self.class_name], PLUGIN_PATH_CACHE=PLUGIN_PATH_CACHE[self.class_name], + PY_FILES=PY_FILES[self.class_name], ) def format_paths(self, paths): @@ -950,6 +957,11 @@ class PluginLoader: display.debug(msg) + def _get_py_files(self, path): + if path not in self._py_files: + self._py_files[path] = glob.glob(to_native(os.path.join(path, "*.py"))) + return self._py_files[path] + def all(self, *args, **kwargs): ''' Iterate through all plugins of this type, in configured paths (no collections) @@ -996,7 +1008,7 @@ class PluginLoader: legacy_excluding_builtin = set() for path_with_context in self._get_paths_with_context(): - matches = glob.glob(to_native(os.path.join(path_with_context.path, "*.py"))) + matches = self._get_py_files(path_with_context.path) if not path_with_context.internal: legacy_excluding_builtin.update(matches) # we sort within each path, but keep path precedence from config