From e05e514861e1dbea342dd5bbc3e96be45e173cff Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Fri, 2 Nov 2012 00:41:50 +0100 Subject: [PATCH] Create a plugin loader system --- lib/ansible/callbacks.py | 10 +--- lib/ansible/inventory/__init__.py | 17 ++---- lib/ansible/playbook/__init__.py | 7 +-- lib/ansible/playbook/play.py | 4 +- lib/ansible/playbook/task.py | 2 +- lib/ansible/runner/__init__.py | 35 +++-------- lib/ansible/runner/connection.py | 17 +----- lib/ansible/utils/__init__.py | 14 +---- lib/ansible/utils/plugins.py | 97 +++++++++++++++++++++++++++++++ 9 files changed, 118 insertions(+), 85 deletions(-) create mode 100644 lib/ansible/utils/plugins.py diff --git a/lib/ansible/callbacks.py b/lib/ansible/callbacks.py index c2bf2911c19..443935ff789 100644 --- a/lib/ansible/callbacks.py +++ b/lib/ansible/callbacks.py @@ -24,14 +24,6 @@ import os.path from ansible.color import stringc import ansible.constants as C -dirname = os.path.dirname(__file__) -callbacks = utils.import_plugins(os.path.join(dirname, 'callback_plugins')) -callbacks = [ c.CallbackModule() for c in callbacks.values() ] -def load_more_callbacks(dirname): - callbacks.extend([c.CallbackModule() for c in utils.import_plugins(dirname).values()]) -for i in C.DEFAULT_CALLBACK_PLUGIN_PATH.split(os.pathsep): - load_more_callbacks(i) - cowsay = None if os.getenv("ANSIBLE_NOCOWS") is not None: cowsay = None @@ -45,7 +37,7 @@ elif os.path.exists("/usr/local/bin/cowsay"): def call_callback_module(method_name, *args, **kwargs): - for callback_plugin in callbacks: + for callback_plugin in utils.plugins.callback_loader.all(): methods = [ getattr(callback_plugin, method_name, None), getattr(callback_plugin, 'on_any', None) diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index e7788a37125..865a6acf8d0 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -29,20 +29,13 @@ from ansible.inventory.host import Host from ansible import errors from ansible import utils -# FIXME, adapt. -dirname = os.path.dirname(__file__) -vars_plugin_list = utils.import_plugins(os.path.join(dirname, 'vars_plugins')) -for i in reversed(C.DEFAULT_VARS_PLUGIN_PATH.split(os.pathsep)): - vars_plugin_list.update(utils.import_plugins(i)) - class Inventory(object): """ Host inventory for ansible. """ __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', '_is_script', - 'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list', - '_vars_plugins' ] + 'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list'] def __init__(self, host_list=C.DEFAULT_HOST_LIST): @@ -74,6 +67,9 @@ class Inventory(object): host_list = host_list.split(",") host_list = [ h for h in host_list if h and h.strip() ] + else: + utils.plugins.push_basedir(self.basedir()) + if type(host_list) == list: all = Group('all') self.groups = [ all ] @@ -95,8 +91,6 @@ class Inventory(object): else: raise errors.AnsibleError("YAML inventory support is deprecated in 0.6 and removed in 0.7, see the migration script in examples/scripts in the git checkout") - self._vars_plugins = [ i.VarsModule(self) for i in vars_plugin_list.values() ] - def _match(self, str, pattern_str): return fnmatch.fnmatch(str, pattern_str) @@ -280,8 +274,7 @@ class Inventory(object): raise errors.AnsibleError("host not found: %s" % hostname) vars = {} - for ip in self._vars_plugins: - updated = ip.run(host) + for updated in map(lambda x: x.run(host), utils.plugins.vars_loader.all(self)): if updated is not None: vars.update(updated) diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py index 69116d1e8bf..0368205d885 100644 --- a/lib/ansible/playbook/__init__.py +++ b/lib/ansible/playbook/__init__.py @@ -27,8 +27,6 @@ from play import Play SETUP_CACHE = collections.defaultdict(dict) -plugins_dir = os.path.join(os.path.dirname(__file__), '..', 'runner') - class PlayBook(object): ''' runs an ansible playbook, given as a datastructure or YAML filename. @@ -112,7 +110,6 @@ class PlayBook(object): self.inventory.subset(subset) self.modules_list = utils.get_available_modules(self.module_path) - self.lookup_plugins_list = ansible.runner.lookup_plugin_list if not self.inventory._is_script: self.global_vars.update(self.inventory.get_group_variables('all')) @@ -120,9 +117,9 @@ class PlayBook(object): self.basedir = os.path.dirname(playbook) (self.playbook, self.play_basedirs) = self._load_playbook_from_file(playbook) self.module_path = self.module_path + os.pathsep + os.path.join(self.basedir, "library") - ansible.callbacks.load_more_callbacks(os.path.join(self.basedir, "callback_plugins")) - self.lookup_plugins_list.update(utils.import_plugins(os.path.join(self.basedir, 'lookup_plugins'))) + for i in self.play_basedirs: + utils.plugins.push_basedir(i) # ***************************************************** diff --git a/lib/ansible/playbook/play.py b/lib/ansible/playbook/play.py index 938901d4a9c..2aa7ff661ac 100644 --- a/lib/ansible/playbook/play.py +++ b/lib/ansible/playbook/play.py @@ -109,10 +109,10 @@ class Play(object): if not k.startswith("with_"): continue plugin_name = k[5:] - if plugin_name not in self.playbook.lookup_plugins_list: + if plugin_name not in utils.plugins.lookup_loader: raise errors.AnsibleError("cannot find lookup plugin named %s for usage in with_%s" % (plugin_name, plugin_name)) terms = utils.varReplaceWithItems(self.basedir, x[k], task_vars) - items = self.playbook.lookup_plugins_list[plugin_name].LookupModule(basedir=self.basedir, runner=None).run(terms, inject=task_vars) + items = utils.plugins.lookup_loader.get(plugin_name, basedir=self.basedir, runner=None).run(terms, inject=task_vars) for item in items: mv = task_vars.copy() diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index a9ceebce08c..ffa17c29720 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -51,7 +51,7 @@ class Task(object): # code to allow "with_glob" and to reference a lookup plugin named glob elif x.startswith("with_"): plugin_name = x.replace("with_","") - if plugin_name in play.playbook.lookup_plugins_list: + if plugin_name in utils.plugins.lookup_loader: ds['items_lookup_plugin'] = plugin_name ds['items_lookup_terms'] = ds[x] ds.pop(x) diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index d337bf3370f..f7e7d6e1c5b 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -45,14 +45,6 @@ try: except ImportError: HAS_ATFORK=False -dirname = os.path.dirname(__file__) -action_plugin_list = utils.import_plugins(os.path.join(dirname, 'action_plugins')) -for i in reversed(C.DEFAULT_ACTION_PLUGIN_PATH.split(os.pathsep)): - action_plugin_list.update(utils.import_plugins(i)) -lookup_plugin_list = utils.import_plugins(os.path.join(dirname, 'lookup_plugins')) -for i in reversed(C.DEFAULT_LOOKUP_PLUGIN_PATH.split(os.pathsep)): - lookup_plugin_list.update(utils.import_plugins(i)) - multiprocessing_runner = None ################################################ @@ -164,19 +156,6 @@ class Runner(object): # ensure we are using unique tmp paths random.seed() - # instantiate plugin classes - self.action_plugins = {} - self.lookup_plugins = {} - for (k,v) in action_plugin_list.iteritems(): - self.action_plugins[k] = v.ActionModule(self) - for (k,v) in lookup_plugin_list.iteritems(): - self.lookup_plugins[k] = v.LookupModule(runner=self, basedir=self.basedir) - - for (k,v) in utils.import_plugins(os.path.join(self.basedir, 'action_plugins')).iteritems(): - self.action_plugins[k] = v.ActionModule(self) - for (k,v) in utils.import_plugins(os.path.join(self.basedir, 'lookup_plugins')).iteritems(): - self.lookup_plugins[k] = v.LookupModule(runner=self, basedir=self.basedir) - # ***************************************************** def _transfer_str(self, conn, tmp, name, data): @@ -292,10 +271,10 @@ class Runner(object): # allow with_foo to work in playbooks... items = None items_plugin = self.module_vars.get('items_lookup_plugin', None) - if items_plugin is not None and items_plugin in self.lookup_plugins: + if items_plugin is not None and items_plugin in utils.plugins.lookup_loader: items_terms = self.module_vars.get('items_lookup_terms', '') items_terms = utils.varReplaceWithItems(self.basedir, items_terms, inject) - items = self.lookup_plugins[items_plugin].run(items_terms, inject=inject) + items = utils.plugins.lookup_loader.get(items_plugin, runner=self, basedir=self.basedir).run(items_terms, inject=inject) if type(items) != list: raise errors.AnsibleError("lookup plugins have to return a list: %r" % items) @@ -417,16 +396,16 @@ class Runner(object): tmp = self._make_tmp_path(conn) result = None - handler = self.action_plugins.get(module_name, None) - if handler: + if module_name in utils.plugins.action_loader: if self.background != 0: raise errors.AnsibleError("async mode is not supported with the %s module" % module_name) + handler = utils.plugins.action_loader.get(module_name, self) result = handler.run(conn, tmp, module_name, module_args, inject) else: if self.background == 0: - result = self.action_plugins['normal'].run(conn, tmp, module_name, module_args, inject) + result = utils.plugins.action_loader.get('normal', self).run(conn, tmp, module_name, module_args, inject) else: - result = self.action_plugins['async'].run(conn, tmp, module_name, module_args, inject) + result = utils.plugins.action_loader.get('async', self).run(conn, tmp, module_name, module_args, inject) conn.close() @@ -648,7 +627,7 @@ class Runner(object): # to be ran once per group of hosts. Example module: pause, # run once per hostgroup, rather than pausing once per each # host. - p = self.action_plugins.get(self.module_name, None) + p = utils.plugins.action_loader.get(self.module_name, self) if p and getattr(p, 'BYPASS_HOST_LOOP', None): # Expose the current hostgroup to the bypassing plugins self.host_set = hosts diff --git a/lib/ansible/runner/connection.py b/lib/ansible/runner/connection.py index 3f5bce3cfd0..46d61a336cb 100644 --- a/lib/ansible/runner/connection.py +++ b/lib/ansible/runner/connection.py @@ -24,32 +24,19 @@ import ansible.constants as C import os import os.path -dirname = os.path.dirname(__file__) -modules = utils.import_plugins(os.path.join(dirname, 'connection_plugins')) -for i in reversed(C.DEFAULT_CONNECTION_PLUGIN_PATH.split(os.pathsep)): - modules.update(utils.import_plugins(i)) - -# rename this module -modules['paramiko'] = modules['paramiko_ssh'] -del modules['paramiko_ssh'] class Connection(object): ''' Handles abstract connections to remote hosts ''' def __init__(self, runner): self.runner = runner - self.modules = None def connect(self, host, port): - if self.modules is None: - self.modules = modules.copy() - self.modules.update(utils.import_plugins(os.path.join(self.runner.basedir, 'connection_plugins'))) conn = None transport = self.runner.transport - module = self.modules.get(transport, None) - if module is None: + conn = utils.plugins.connection_loader.get(transport, self.runner, host, port) + if conn is None: raise AnsibleError("unsupported connection type: %s" % transport) - conn = module.Connection(self.runner, host, port) self.active = conn.connect() return self.active diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index d3a32d0ada1..40fd3c9e103 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -24,11 +24,10 @@ import operator from ansible import errors from ansible import __version__ from ansible.utils.template import * +from ansible.utils.plugins import * import ansible.constants as C import time import StringIO -import imp -import glob import stat import termios import tty @@ -457,17 +456,6 @@ def filter_leading_non_json_lines(buf): filtered_lines.write(line + '\n') return filtered_lines.getvalue() -def import_plugins(directory): - modules = {} - python_files = os.path.join(directory, '*.py') - for path in glob.glob(python_files): - if path.startswith("_"): - continue - name, ext = os.path.splitext(os.path.basename(path)) - if not name.startswith("_"): - modules[name] = imp.load_source(name, path) - return modules - def get_available_modules(dirname=None): """ returns a list of modules available based on current directory diff --git a/lib/ansible/utils/plugins.py b/lib/ansible/utils/plugins.py new file mode 100644 index 00000000000..6ff7241b2ab --- /dev/null +++ b/lib/ansible/utils/plugins.py @@ -0,0 +1,97 @@ +# (c) 2012, Daniel Hokka Zakrisson +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import sys +import glob +import imp +import ansible.constants as C +from ansible import errors + +_basedirs = [] + +def push_basedir(basedir): + _basedirs.insert(0, basedir) + +class PluginLoader(object): + """PluginLoader loads plugins from the best source + + It searches for plugins by iterating through the combined list of + play basedirs, configured paths, and the installed package directory. + The first match is used. + """ + def __init__(self, class_name, package, config, subdir, aliases={}): + """Create a new PluginLoader""" + self.class_name = class_name + self.package = package + self.config = config + self.subdir = subdir + self.aliases = aliases + self._module_cache = {} + + def _get_package_path(self): + """Gets the path of a Python package""" + if not hasattr(self, 'package_path'): + m = __import__(self.package) + parts = self.package.split('.')[1:] + self.package_path = os.path.join(os.path.dirname(m.__file__), *parts) + return self.package_path + + def _get_paths(self): + """Return a list of paths to search for plugins in + + The list is searched in order.""" + return [os.path.join(basedir, self.subdir) for basedir in _basedirs] + self.config.split(os.pathsep) + [self._get_package_path()] + + def find_plugin(self, name): + """Find a plugin named name""" + for i in self._get_paths(): + path = os.path.join(i, "%s.py" % name) + if os.path.exists(path): + return path + return None + + def has_plugin(self, name): + """Checks if a plugin named name exists""" + return self.find_plugin(name) is not None + __contains__ = has_plugin + + def get(self, name, *args, **kwargs): + if name in self.aliases: + name = self.aliases[name] + path = self.find_plugin(name) + if path is None: + return None + if path not in self._module_cache: + self._module_cache[path] = imp.load_source('.'.join([self.package, name]), path) + return getattr(self._module_cache[path], self.class_name)(*args, **kwargs) + + def all(self, *args, **kwargs): + for i in self._get_paths(): + for path in glob.glob(os.path.join(i, "*.py")): + name, ext = os.path.splitext(os.path.basename(path)) + if name.startswith("_"): + continue + if path not in self._module_cache: + self._module_cache[path] = imp.load_source('.'.join([self.package, name]), path) + yield getattr(self._module_cache[path], self.class_name)(*args, **kwargs) + +action_loader = PluginLoader('ActionModule', 'ansible.runner.action_plugins', C.DEFAULT_ACTION_PLUGIN_PATH, 'action_plugins') +callback_loader = PluginLoader('CallbackModule', 'ansible.callback_plugins', C.DEFAULT_CALLBACK_PLUGIN_PATH, 'callback_plugins') +connection_loader = PluginLoader('Connection', 'ansible.runner.connection_plugins', C.DEFAULT_CONNECTION_PLUGIN_PATH, 'connection_plugins', aliases={'paramiko': 'paramiko_ssh'}) +lookup_loader = PluginLoader('LookupModule', 'ansible.runner.lookup_plugins', C.DEFAULT_LOOKUP_PLUGIN_PATH, 'lookup_plugins') +vars_loader = PluginLoader('VarsModule', 'ansible.inventory.vars_plugins', C.DEFAULT_VARS_PLUGIN_PATH, 'vars_plugins')