From 68e86de26fd5eac8a0a177528d7ffec0ed78ce4f Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 17 Feb 2015 13:24:46 -0800 Subject: [PATCH 1/2] Optimize the plugin loader. We have been caching the paths to the plugins but before we would only cache the specific plugin that we were looking for. This meant that we might search through all of the plugin directories before finding the specific module we were interested in. The next plugin we needed we might again search through all the plugin directories before finding the plugin we wanted. This new code will cache all the potential plugins in each directory searched. For a play that only uses one plugin we'll only search through directories until we encounter the directory that has the proper plugin. For a large playbook with many plugins used we'll search through each directory at most once each. This should alleviate #10165 --- lib/ansible/utils/plugins.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/ansible/utils/plugins.py b/lib/ansible/utils/plugins.py index 29771d0ed97..fbee4bde684 100644 --- a/lib/ansible/utils/plugins.py +++ b/lib/ansible/utils/plugins.py @@ -64,6 +64,7 @@ class PluginLoader(object): self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name] self._extra_dirs = [] + self._searched_paths = set() def print_paths(self): ''' Returns a string suitable for printing of the search path ''' @@ -167,22 +168,36 @@ class PluginLoader(object): else: suffixes = ['.py', ''] - # loop over paths and then loop over suffixes to find plugin - for i in self._get_paths(): - for suffix in suffixes: - full_name = '%s%s' % (name, suffix) + potential_names = frozenset('%s%s' % (name, s) for s in suffixes) + for full_name in potential_names: + if full_name in self._plugin_path_cache: + return self._plugin_path_cache[full_name] + + found = None + for path in [p for p in self._get_paths() if p not in self._searched_paths]: + for potential_file in os.listdir(path): + for suffix in suffixes: + if potential_file.endswith(suffix): + full_path = os.path.join(path, potential_file) + full_name = os.path.basename(full_path) + break + else: # Yes, this is a for-else: http://bit.ly/1ElPkyg + continue + if full_name not in self._plugin_path_cache: + self._plugin_path_cache[full_name] = full_path + + self._searched_paths.add(path) + for full_name in potential_names: if full_name in self._plugin_path_cache: return self._plugin_path_cache[full_name] - path = os.path.join(i, full_name) - if os.path.isfile(path): - self._plugin_path_cache[full_name] = path - return path - # if nothing is found, try finding alias/deprecated if not name.startswith('_'): - return self.find_plugin('_' + name, suffixes, transport) + for alias_name in ('_%s' % n for n in potential_names): + # We've already cached all the paths at this point + if alias_name in self._plugin_path_cache: + return self._plugin_path_cache[alias_name] return None From 730c2ba40352ef4248461a10f560d10b08b9db35 Mon Sep 17 00:00:00 2001 From: jensdepuydt Date: Wed, 18 Feb 2015 12:07:35 +0100 Subject: [PATCH 2/2] minor change to not search in non-existing paths --- lib/ansible/utils/plugins.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/ansible/utils/plugins.py b/lib/ansible/utils/plugins.py index fbee4bde684..8d356fa0a71 100644 --- a/lib/ansible/utils/plugins.py +++ b/lib/ansible/utils/plugins.py @@ -175,18 +175,19 @@ class PluginLoader(object): found = None for path in [p for p in self._get_paths() if p not in self._searched_paths]: - for potential_file in os.listdir(path): - for suffix in suffixes: - if potential_file.endswith(suffix): - full_path = os.path.join(path, potential_file) - full_name = os.path.basename(full_path) - break - else: # Yes, this is a for-else: http://bit.ly/1ElPkyg - continue - - if full_name not in self._plugin_path_cache: - self._plugin_path_cache[full_name] = full_path - + if os.path.isdir(path): + for potential_file in os.listdir(path): + for suffix in suffixes: + if potential_file.endswith(suffix): + full_path = os.path.join(path, potential_file) + full_name = os.path.basename(full_path) + break + else: # Yes, this is a for-else: http://bit.ly/1ElPkyg + continue + + if full_name not in self._plugin_path_cache: + self._plugin_path_cache[full_name] = full_path + self._searched_paths.add(path) for full_name in potential_names: if full_name in self._plugin_path_cache: