From c891ab078abeab844011066d49fa1ceb535ce2b9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 31 Mar 2018 10:22:11 +0545 Subject: [PATCH] issue #106: working old-style native module execution Still abusing Python import mechanism, but one big step closer to eliminating that. --- ansible_mitogen/helpers.py | 152 +++++++++--------------------------- ansible_mitogen/mixins.py | 52 ++++++------- ansible_mitogen/planner.py | 156 ++++++++++++++++++++++++++++--------- ansible_mitogen/runner.py | 155 +++++++++++++++++++----------------- 4 files changed, 262 insertions(+), 253 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 9e8693d9..9f2e3c2b 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import json import operator import os @@ -38,10 +39,7 @@ import tempfile import threading import mitogen.core - -# Prevent accidental import of an Ansible module from hanging on stdin read. -import ansible.module_utils.basic -ansible.module_utils.basic._ANSIBLE_ARGS = '{}' +import ansible_mitogen.runner #: Mapping of job_id<->result dict _result_by_job_id = {} @@ -50,124 +48,26 @@ _result_by_job_id = {} _thread_by_job_id = {} -class Exit(Exception): - """ - Raised when a module exits with success. - """ - def __init__(self, dct): - self.dct = dct - - -class ModuleError(Exception): - """ - Raised when a module voluntarily indicates failure via .fail_json(). - """ - def __init__(self, msg, dct): - Exception.__init__(self, msg) - self.dct = dct - - -def monkey_exit_json(self, **kwargs): - """ - Replace AnsibleModule.exit_json() with something that doesn't try to kill - the process or JSON-encode the result dictionary. Instead, cause Exit to be - raised, with a `dct` attribute containing the successful result dictionary. - """ - self.add_path_info(kwargs) - kwargs.setdefault('changed', False) - kwargs.setdefault('invocation', { - 'module_args': self.params - }) - kwargs = ansible.module_utils.basic.remove_values( - kwargs, - self.no_log_values - ) - self.do_cleanup_files() - raise Exit(kwargs) - - -def monkey_fail_json(self, **kwargs): - """ - Replace AnsibleModule.fail_json() with something that raises ModuleError, - which includes a `dct` attribute. - """ - self.add_path_info(kwargs) - kwargs.setdefault('failed', True) - kwargs.setdefault('invocation', { - 'module_args': self.params - }) - kwargs = ansible.module_utils.basic.remove_values( - kwargs, - self.no_log_values - ) - self.do_cleanup_files() - raise ModuleError(kwargs.get('msg'), kwargs) - - -def module_fixups(mod): - """ - Apply fixups for known problems with mainline Ansible modules. - """ - if mod.__name__ == 'ansible.modules.packaging.os.yum_repository': - # https://github.com/dw/mitogen/issues/154 - mod.YumRepo.repofile = mod.configparser.RawConfigParser() - - -class TemporaryEnvironment(object): - def __init__(self, env=None): - self.original = os.environ.copy() - self.env = env or {} - os.environ.update((k, str(v)) for k, v in self.env.iteritems()) - - def revert(self): - os.environ.clear() - os.environ.update(self.original) - - -def run_module(module, raw_params=None, args=None, env=None): +def run_module(kwargs): """ Set up the process environment in preparation for running an Ansible module. This monkey-patches the Ansible libraries in various places to prevent it from trying to kill the process on completion, and to prevent it from reading sys.stdin. """ - if args is None: - args = {} - if raw_params is not None: - args['_raw_params'] = raw_params - - ansible.module_utils.basic.AnsibleModule.exit_json = monkey_exit_json - ansible.module_utils.basic.AnsibleModule.fail_json = monkey_fail_json - ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({ - 'ANSIBLE_MODULE_ARGS': args - }) + runner_name = kwargs.pop('runner_name') + klass = getattr(ansible_mitogen.runner, runner_name) + impl = klass(**kwargs) + return json.dumps(impl.run()) - temp_env = TemporaryEnvironment(env) - try: - try: - mod = __import__(module, {}, {}, ['']) - module_fixups(mod) - # Ansible modules begin execution on import. Thus the above __import__ - # will cause either Exit or ModuleError to be raised. If we reach the - # line below, the module did not execute and must already have been - # imported for a previous invocation, so we need to invoke main - # explicitly. - mod.main() - except (Exit, ModuleError), e: - result = json.dumps(e.dct) - finally: - temp_env.revert() - - return result - - -def _async_main(job_id, module, raw_params, args, env): + +def _async_main(job_id, runner_name, kwargs): """ Implementation for the thread that implements asynchronous module execution. """ try: - rc = run_module(module, raw_params, args, env) + rc = run_module(runner_name, kwargs) except Exception, e: rc = mitogen.core.CallError(e) @@ -189,7 +89,7 @@ def make_temp_directory(base_dir): prefix='ansible-mitogen-tmp-', ) -def run_module_async(module, raw_params=None, args=None): +def run_module_async(runner_name, kwargs): """ Arrange for an Ansible module to be executed in a thread of the current process, with results available via :py:func:`get_async_result`. @@ -200,9 +100,8 @@ def run_module_async(module, raw_params=None, args=None): target=_async_main, kwargs={ 'job_id': job_id, - 'module': module, - 'raw_params': raw_params, - 'args': args, + 'runner_name': runner_name, + 'kwargs': kwargs, } ) _thread_by_job_id[job_id].start() @@ -241,7 +140,7 @@ def get_user_shell(): return pw_shell or '/bin/sh' -def exec_command(cmd, in_data='', chdir=None, shell=None): +def exec_args(args, in_data='', chdir=None, shell=None): """ Run a command in a subprocess, emulating the argument handling behaviour of SSH. @@ -256,7 +155,7 @@ def exec_command(cmd, in_data='', chdir=None, shell=None): assert isinstance(cmd, basestring) proc = subprocess.Popen( - args=[get_user_shell(), '-c', cmd], + args=args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, @@ -266,6 +165,27 @@ def exec_command(cmd, in_data='', chdir=None, shell=None): return proc.returncode, stdout, stderr +def exec_command(cmd, in_data='', chdir=None, shell=None): + """ + Run a command in a subprocess, emulating the argument handling behaviour of + SSH. + + :param bytes cmd: + String command line, passed to user's shell. + :param bytes in_data: + Optional standard input for the command. + :return: + (return code, stdout bytes, stderr bytes) + """ + assert isinstance(cmd, basestring) + return _exec_command( + args=[get_user_shell(), '-c', cmd], + in_data=in_Data, + chdir=chdir, + shell=shell, + ) + + def read_path(path): """ Fetch the contents of a filesystem `path` as bytes. diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 9cef0c3e..d003ce2e 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -52,6 +52,7 @@ import mitogen.master from mitogen.utils import cast import ansible_mitogen.connection +import ansible_mitogen.planner import ansible_mitogen.helpers from ansible.module_utils._text import to_text @@ -59,22 +60,6 @@ from ansible.module_utils._text import to_text LOG = logging.getLogger(__name__) -def get_command_module_name(module_name): - """ - Given the name of an Ansible command module, return its canonical module - path within the ansible. - - :param module_name: - "shell" - :return: - "ansible.modules.commands.shell" - """ - path = module_loader.find_plugin(module_name, '') - relpath = os.path.relpath(path, os.path.dirname(ansible.__file__)) - root, _ = os.path.splitext(relpath) - return 'ansible.' + root.replace('/', '.') - - class ActionModuleMixin(ansible.plugins.action.ActionBase): """ The Mitogen-patched PluginLoader dynamically mixes this into every action @@ -308,29 +293,36 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): helpers.run_module() or helpers.run_module_async() in the target context. """ - if task_vars is None: - task_vars = {} if module_name is None: module_name = self._task.action if module_args is None: module_args = self._task.args + if task_vars is None: + task_vars = {} self._update_module_args(module_name, module_args, task_vars) - if wrap_async: - helper = ansible_mitogen.helpers.run_module_async - else: - helper = ansible_mitogen.helpers.run_module - env = {} self._compute_environment_string(env) - js = self.call( - helper, - get_command_module_name(module_name), - args=cast(module_args), - env=cast(env), + return ansible_mitogen.planner.invoke( + ansible_mitogen.planner.Invocation( + action=self, + connection=self._connection, + module_name=mitogen.utils.cast(module_name), + module_args=mitogen.utils.cast(module_args), + task_vars=task_vars, + tmp=tmp, + env=mitogen.utils.cast(env), + wrap_async=wrap_async, + ) ) + def _postprocess_response(self, js): + """ + Apply fixups mimicking ActionBase._execute_module(); this is copied + verbatim from action/__init__.py, the guts of _parse_returned_data are + garbage and should be removed or reimplemented once tests exist. + """ data = self._parse_returned_data({ 'rc': 0, 'stdout': js, @@ -351,8 +343,8 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): encoding_errors='surrogate_then_replace', chdir=None): """ - Replace the mad rat's nest of logic in the base implementation by - simply calling helpers.exec_command() in the target context. + Override the base implementation by simply calling + helpers.exec_command() in the target context. """ LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)', cmd, type(in_data), executable, chdir) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 6c9f8441..0fb99569 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -27,32 +27,69 @@ # POSSIBILITY OF SUCH DAMAGE. """ -This exists to detect every case defined in [0] and prepare arguments necessary -for the executor implementation running within the target, including preloading -any requisite files/Python modules known to be missing. +Classes to detect each case from [0] and prepare arguments necessary for the +corresponding Runner class within the target, including preloading requisite +files/modules known missing. [0] "Ansible Module Architecture", developing_program_flow_modules.html """ from __future__ import absolute_import +import logging +import os + from ansible.executor import module_common +import ansible.errors + +try: + from ansible.plugins.loader import module_loader +except ImportError: # Ansible <2.4 + from ansible.plugins import module_loader import mitogen import mitogen.service import ansible_mitogen.helpers +LOG = logging.getLogger(__name__) + + +class Invocation(object): + """ + Collect up a module's execution environment then use it to invoke + helpers.run_module() or helpers.run_module_async() in the target context. + """ + def __init__(self, action, connection, module_name, module_args, + task_vars, tmp, env, wrap_async): + #: Instance of the ActionBase subclass invoking the module. Required to + #: access some output postprocessing methods that don't belong in + #: ActionBase at all. + self.action = action + self.connection = connection + self.module_name = module_name + self.module_args = module_args + self.module_path = None + self.module_source = None + self.task_vars = task_vars + self.tmp = tmp + self.env = env + self.wrap_async = wrap_async + + def __repr__(self): + return 'Invocation(module_name=%s)' % (self.module_name,) + + class Planner(object): """ A Planner receives a module name and the contents of its implementation file, indicates whether or not it understands how to run the module, and exports a method to run the module. """ - def detect(self, name, source): - assert 0 + def detect(self, invocation): + raise NotImplementedError() - def run(self, connection, name, source, args, env): - assert 0 + def plan(self, invocation): + raise NotImplementedError() class JsonArgsPlanner(Planner): @@ -60,10 +97,10 @@ class JsonArgsPlanner(Planner): Script that has its interpreter directive and the task arguments substituted into its source as a JSON string. """ - def detect(self, name, source): - return module_common.REPLACER_JSONARGS in source + def detect(self, invocation): + return module_common.REPLACER_JSONARGS in invocation.module_source - def run(self, name, source, args, env): + def plan(self, invocation): path = None # TODO mitogen.service.call(501, ('register', path)) return { @@ -79,17 +116,17 @@ class WantJsonPlanner(Planner): If a module has the string WANT_JSON in it anywhere, Ansible treats it as a non-native module that accepts a filename as its only command line parameter. The filename is for a temporary file containing a JSON string - containing the module’s parameters. The module needs to open the file, read + containing the module's parameters. The module needs to open the file, read and parse the parameters, operate on the data, and print its return data as a JSON encoded dictionary to stdout before exiting. These types of modules are self-contained entities. As of Ansible 2.1, Ansible only modifies them to change a shebang line if present. """ - def detect(self, name, source): - return 'WANT_JSON' in source + def detect(self, invocation): + return 'WANT_JSON' in invocation.module_source - def run(self, name, source, args, env): + def plan(self, name, source, args, env): return { 'func': 'run_want_json_module', 'binary': source, @@ -122,10 +159,10 @@ class ReplacerPlanner(Planner): "ansible/module_utils/powershell.ps1". It should only be used with new-style Powershell modules. """ - def detect(self, name, source): - return module_common.REPLACER in source + def detect(self, invocation): + return module_common.REPLACER in invocation.module_source - def run(self, name, source, args, env): + def plan(self, name, source, args, env): return { 'func': 'run_replacer_module', 'binary': source, @@ -139,37 +176,49 @@ class BinaryPlanner(Planner): Binary modules take their arguments and will return data to Ansible in the same way as want JSON modules. """ - helper = staticmethod(ansible_mitogen.helpers.run_binary) - - def detect(self, name, source): - return module_common._is_binary(source) + def detect(self, invocation): + return module_common._is_binary(invocation.module_source) - def run(self, name, source, args, env): + def plan(self, name, source, args, env): return { - 'func': 'run_binary_module', + 'runner_name': 'BinaryRunner', 'binary': source, 'args': args, 'env': env, } -class PythonPlanner(Planner): +class NativePlanner(Planner): """ The Ansiballz framework differs from module replacer in that it uses real Python imports of things in ansible/module_utils instead of merely preprocessing the module. """ - helper = staticmethod(ansible_mitogen.helpers.run_module) - - def detect(self, name, source): + def detect(self, invocation): return True - def run(self, name, source, args, env): + def get_command_module_name(self, module_name): + """ + Given the name of an Ansible command module, return its canonical + module path within the ansible. + + :param module_name: + "shell" + :return: + "ansible.modules.commands.shell" + """ + path = module_loader.find_plugin(module_name, '') + relpath = os.path.relpath(path, os.path.dirname(ansible.__file__)) + root, _ = os.path.splitext(relpath) + return 'ansible.' + root.replace('/', '.') + + def plan(self, invocation): return { - 'func': 'run_python_module', - 'module': name, - 'args': args, - 'env': env + 'runner_name': 'NativeRunner', + 'module': invocation.module_name, + 'mod_name': self.get_command_module_name(invocation.module_name), + 'args': invocation.module_args, + 'env': invocation.env, } @@ -178,9 +227,46 @@ _planners = [ # WantJsonPlanner, # ReplacerPlanner, BinaryPlanner, - PythonPlanner, + NativePlanner, ] -def plan(): - pass +NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' +CRASHED_MSG = 'Mitogen: internal error: ' + + +def get_module_data(name): + path = module_loader.find_plugin(name, '') + with open(path, 'rb') as fp: + source = fp.read() + return path, source + + +def invoke(invocation): + """ + Find a suitable Planner that knows how to run `invocation`. + """ + (invocation.module_path, + invocation.module_source) = get_module_data(invocation.module_name) + + for klass in _planners: + planner = klass() + if planner.detect(invocation): + break + else: + raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation)) + + kwargs = planner.plan(invocation) + if invocation.wrap_async: + helper = ansible_mitogen.helpers.run_module_async + else: + helper = ansible_mitogen.helpers.run_module + + try: + js = invocation.connection.call(helper, kwargs) + except mitogen.core.CallError as e: + LOG.exception('invocation crashed: %r', invocation) + summary = str(e).splitlines()[0] + raise ansible.errors.AnsibleInternalError(CRASHED_MSG + summary) + + return invocation.action._postprocess_response(js) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 559001f0..2d92626e 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -43,6 +43,60 @@ import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' +class Runner(object): + """ + Ansible module runner. After instantiation (with kwargs supplied by the + corresponding Planner), `.run()` is invoked, upon which `setup()`, + `_run()`, and `revert()` are invoked, with the return value of `_run()` + returned by `run()`. + + Subclasses may override `_run`()` and extend `setup()` and `revert()`. + """ + def __init__(self, module, raw_params=None, args=None, env=None): + if args is None: + args = {} + if raw_params is not None: + args['_raw_params'] = raw_params + + self.module = module + self.raw_params = raw_params + self.args = args + self.env = env + + def setup(self): + """ + Prepare the current process for running a module. The base + implementation simply prepares the environment. + """ + self._env = TemporaryEnvironment(self.env) + + def revert(self): + """ + Revert any changes made to the process after running a module. The base + implementation simply restores the original environment. + """ + self._env.revert() + + def _run(self): + raise NotImplementedError() + + def run(self): + """ + Set up the process environment in preparation for running an Ansible + module. This monkey-patches the Ansible libraries in various places to + prevent it from trying to kill the process on completion, and to + prevent it from reading sys.stdin. + + :returns: + Module result dictionary. + """ + self.setup() + try: + return self._run() + finally: + self.revert() + + class TemporaryEnvironment(object): def __init__(self, env=None): self.original = os.environ.copy() @@ -67,7 +121,7 @@ class NativeModuleExit(Exception): ansible_module.do_cleanup_files() self.dct = ansible.module_utils.basic.remove_values( kwargs, - self.no_log_values, + ansible_module.no_log_values, ) @@ -79,7 +133,7 @@ class NativeMethodOverrides(object): :class:`NativeModuleExit` exception`. """ kwargs.setdefault('changed', False) - return NativeModuleExit(self, **kwargs) + raise NativeModuleExit(self, **kwargs) @staticmethod def fail_json(self, **kwargs): @@ -88,7 +142,7 @@ class NativeMethodOverrides(object): :class:`NativeModuleExit` exception`. """ kwargs.setdefault('failed', True) - return NativeModuleExit(self, **kwargs) + raise NativeModuleExit(self, **kwargs) klass = ansible.module_utils.basic.AnsibleModule @@ -120,68 +174,18 @@ class NativeModuleArguments(object): """ Restore prior state. """ - ansible.module_utils.basic._ANSIBLE_ARGS = self._original_args + ansible.module_utils.basic._ANSIBLE_ARGS = self.original -class Runner(object): - """ - Ansible module runner. After instantiation (with kwargs supplied by the - corresponding Planner), `.run()` is invoked, upon which `setup()`, - `_run()`, and `revert()` are invoked, with the return value of `_run()` - returned by `run()`. - - Subclasses may override `_run`()` and extend `setup()` and `revert()`. - """ - def __init__(self, module, raw_params=None, args=None, env=None): - if args is None: - args = {} - if raw_params is not None: - args['_raw_params'] = raw_params - - self.module = module - self.raw_params = raw_params - self.args = args - self.env = env - - def setup(self): - """ - Prepare the current process for running a module. The base - implementation simply prepares the environment. - """ - self._env = TemporaryEnvironment(self.env) - - def revert(self): - """ - Revert any changes made to the process after running a module. The base - implementation simply restores the original environment. - """ - self._env.revert() - - def _run(self): - raise NotImplementedError() - - def run(self): - """ - Set up the process environment in preparation for running an Ansible - module. This monkey-patches the Ansible libraries in various places to - prevent it from trying to kill the process on completion, and to - prevent it from reading sys.stdin. - - :returns: - Module result dictionary. - """ - self.setup() - try: - return self._run() - finally: - self.revert() - - -class NativeRunner(object): +class NativeRunner(Runner): """ Execute a new-style Ansible module, where Module Replacer-related tricks aren't required. """ + def __init__(self, mod_name, **kwargs): + super(NativeRunner, self).__init__(**kwargs) + self.mod_name = mod_name + def setup(self): super(NativeRunner, self).setup() self._overrides = NativeMethodOverrides() @@ -192,18 +196,18 @@ class NativeRunner(object): self._args.revert() self._overrides.revert() - def module_fixups(mod): - """ - Apply fixups for known problems with mainline Ansible modules. - """ - if mod.__name__ == 'ansible.modules.packaging.os.yum_repository': - # https://github.com/dw/mitogen/issues/154 - mod.YumRepo.repofile = mod.configparser.RawConfigParser() + def _fixup__default(self, mod): + pass + + def _fixup__yum_repository(self, mod): + # https://github.com/dw/mitogen/issues/154 + mod.YumRepo.repofile = mod.configparser.RawConfigParser() def _run(self): + fixup = getattr(self, '_fixup__' + self.module, self._fixup__default) try: - mod = __import__(self.module, {}, {}, ['']) - self.module_fixups(mod) + mod = __import__(self.mod_name, {}, {}, ['']) + fixup(mod) # Ansible modules begin execution on import. Thus the above # __import__ will cause either Exit or ModuleError to be raised. If # we reach the line below, the module did not execute and must @@ -219,7 +223,7 @@ class NativeRunner(object): } -class BinaryRunner(object): +class BinaryRunner(Runner): def __init__(self, path, **kwargs): super(BinaryRunner, self).__init__(**kwargs) self.path = path @@ -282,9 +286,16 @@ class BinaryRunner(object): args=[self.bin_fp.name, self.args_fp.name], ) except Exception, e: - return - # ... - assert 0 + return { + 'failed': True, + 'msg': '%s: %s' % (type(e), e), + } + + return { + 'rc': rc, + 'stdout': stdout, + 'stderr': stderr + } class WantJsonRunner(BinaryRunner):