Optimize the plugin loader.

We have been caching the paths to the plugins but before we would only
cache the specific plugin that we were looking for.  This meant that we
might search through all of the plugin directories before finding the
specific module we were interested in.  The next plugin we needed we
might again search through all the plugin directories before finding the
plugin we wanted.

This new code will cache all the potential plugins in each directory
searched.  For a play that only uses one plugin we'll only search
through directories until we encounter the directory that has the proper
plugin.  For a large playbook with many plugins used we'll search
through each directory at most once each.

This should alleviate #10165

lib/ansible/utils/plugins.py

@@ -64,6 +64,7 @@ class PluginLoader(object):
         self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name]
 
         self._extra_dirs = []
+        self._searched_paths = set()
 
     def print_paths(self):
         ''' Returns a string suitable for printing of the search path '''
@@ -167,22 +168,36 @@ class PluginLoader(object):
                 else:
                     suffixes = ['.py', '']
 
-        # loop over paths and then loop over suffixes to find plugin
-        for i in self._get_paths():
+        potential_names = frozenset('%s%s' % (name, s) for s in suffixes)
+        for full_name in potential_names:
+            if full_name in self._plugin_path_cache:
+                return self._plugin_path_cache[full_name]
+
+        found = None
+        for path in [p for p in self._get_paths() if p not in self._searched_paths]:
+            for potential_file in os.listdir(path):
                 for suffix in suffixes:
-                full_name = '%s%s' % (name, suffix)
+                    if potential_file.endswith(suffix):
+                        full_path = os.path.join(path, potential_file)
+                        full_name = os.path.basename(full_path)
+                        break
+                else: # Yes, this is a for-else: http://bit.ly/1ElPkyg
+                    continue
+
+                if full_name not in self._plugin_path_cache:
+                    self._plugin_path_cache[full_name] = full_path
 
+            self._searched_paths.add(path)
+            for full_name in potential_names:
                 if full_name in self._plugin_path_cache:
                     return self._plugin_path_cache[full_name]
 
-                path = os.path.join(i, full_name)
-                if os.path.isfile(path):
-                    self._plugin_path_cache[full_name] = path
-                    return path
-
         # if nothing is found, try finding alias/deprecated
         if not name.startswith('_'):
-            return self.find_plugin('_' + name, suffixes, transport)
+            for alias_name in ('_%s' % n for n in potential_names):
+                # We've already cached all the paths at this point
+                if alias_name in self._plugin_path_cache:
+                    return self._plugin_path_cache[alias_name]
 
         return None