archive
/
mitogen
mirror of https://github.com/mitogen-hq/mitogen.git
1
0
Fork 0
Code Releases Activity

issue #106: working old-style native module execution

Browse Source 
Still abusing Python import mechanism, but one big step closer to
eliminating that.
pull/193/head
David Wilson 7 years ago
parent 34a37a0ba5
commit c891ab078a
4 changed files with 262 additions and 253 deletions
Show Stats Download Patch File Download Diff File

152
ansible_mitogen/helpers.py
Unescape Escape View File

@ -26,6 +26,7 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
import json
import operator
import os
@ -38,10 +39,7 @@ import tempfile
import threading
import mitogen.core
# Prevent accidental import of an Ansible module from hanging on stdin read.
import ansible.module_utils.basic
ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
import ansible_mitogen.runner
#: Mapping of job_id<->result dict
_result_by_job_id = {}
@ -50,124 +48,26 @@ _result_by_job_id = {}
_thread_by_job_id = {}
class Exit(Exception):
"""
Raised when a module exits with success.
"""
def __init__(self, dct):
self.dct = dct
class ModuleError(Exception):
"""
Raised when a module voluntarily indicates failure via .fail_json().
"""
def __init__(self, msg, dct):
Exception.__init__(self, msg)
self.dct = dct
def monkey_exit_json(self, **kwargs):
"""
Replace AnsibleModule.exit_json() with something that doesn't try to kill
the process or JSON-encode the result dictionary. Instead, cause Exit to be
raised, with a `dct` attribute containing the successful result dictionary.
"""
self.add_path_info(kwargs)
kwargs.setdefault('changed', False)
kwargs.setdefault('invocation', {
'module_args': self.params
})
kwargs = ansible.module_utils.basic.remove_values(
kwargs,
self.no_log_values
)
self.do_cleanup_files()
raise Exit(kwargs)
def monkey_fail_json(self, **kwargs):
"""
Replace AnsibleModule.fail_json() with something that raises ModuleError,
which includes a `dct` attribute.
"""
self.add_path_info(kwargs)
kwargs.setdefault('failed', True)
kwargs.setdefault('invocation', {
'module_args': self.params
})
kwargs = ansible.module_utils.basic.remove_values(
kwargs,
self.no_log_values
)
self.do_cleanup_files()
raise ModuleError(kwargs.get('msg'), kwargs)
def module_fixups(mod):
"""
Apply fixups for known problems with mainline Ansible modules.
"""
if mod.__name__ == 'ansible.modules.packaging.os.yum_repository':
# https://github.com/dw/mitogen/issues/154
mod.YumRepo.repofile = mod.configparser.RawConfigParser()
class TemporaryEnvironment(object):
def __init__(self, env=None):
self.original = os.environ.copy()
self.env = env or {}
os.environ.update((k, str(v)) for k, v in self.env.iteritems())
def revert(self):
os.environ.clear()
os.environ.update(self.original)
def run_module(module, raw_params=None, args=None, env=None):
def run_module(kwargs):
"""
Set up the process environment in preparation for running an Ansible
module. This monkey-patches the Ansible libraries in various places to
prevent it from trying to kill the process on completion, and to prevent it
from reading sys.stdin.
"""
if args is None:
args = {}
if raw_params is not None:
args['_raw_params'] = raw_params
ansible.module_utils.basic.AnsibleModule.exit_json = monkey_exit_json
ansible.module_utils.basic.AnsibleModule.fail_json = monkey_fail_json
ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({
'ANSIBLE_MODULE_ARGS': args
})
runner_name = kwargs.pop('runner_name')
klass = getattr(ansible_mitogen.runner, runner_name)
impl = klass(**kwargs)
return json.dumps(impl.run())
temp_env = TemporaryEnvironment(env)
try:
try:
mod = __import__(module, {}, {}, [''])
module_fixups(mod)
# Ansible modules begin execution on import. Thus the above __import__
# will cause either Exit or ModuleError to be raised. If we reach the
# line below, the module did not execute and must already have been
# imported for a previous invocation, so we need to invoke main
# explicitly.
mod.main()
except (Exit, ModuleError), e:
result = json.dumps(e.dct)
finally:
temp_env.revert()
return result
def _async_main(job_id, module, raw_params, args, env):
def _async_main(job_id, runner_name, kwargs):
"""
Implementation for the thread that implements asynchronous module
execution.
"""
try:
rc = run_module(module, raw_params, args, env)
rc = run_module(runner_name, kwargs)
except Exception, e:
rc = mitogen.core.CallError(e)
@ -189,7 +89,7 @@ def make_temp_directory(base_dir):
prefix='ansible-mitogen-tmp-',
)
def run_module_async(module, raw_params=None, args=None):
def run_module_async(runner_name, kwargs):
"""
Arrange for an Ansible module to be executed in a thread of the current
process, with results available via :py:func:`get_async_result`.
@ -200,9 +100,8 @@ def run_module_async(module, raw_params=None, args=None):
target=_async_main,
kwargs={
'job_id': job_id,
'module': module,
'raw_params': raw_params,
'args': args,
'runner_name': runner_name,
'kwargs': kwargs,
}
)
_thread_by_job_id[job_id].start()
@ -241,7 +140,7 @@ def get_user_shell():
return pw_shell or '/bin/sh'
def exec_command(cmd, in_data='', chdir=None, shell=None):
def exec_args(args, in_data='', chdir=None, shell=None):
"""
Run a command in a subprocess, emulating the argument handling behaviour of
SSH.
@ -256,7 +155,7 @@ def exec_command(cmd, in_data='', chdir=None, shell=None):
assert isinstance(cmd, basestring)
proc = subprocess.Popen(
args=[get_user_shell(), '-c', cmd],
args=args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
@ -266,6 +165,27 @@ def exec_command(cmd, in_data='', chdir=None, shell=None):
return proc.returncode, stdout, stderr
def exec_command(cmd, in_data='', chdir=None, shell=None):
"""
Run a command in a subprocess, emulating the argument handling behaviour of
SSH.
:param bytes cmd:
String command line, passed to user's shell.
:param bytes in_data:
Optional standard input for the command.
:return:
(return code, stdout bytes, stderr bytes)
"""
assert isinstance(cmd, basestring)
return _exec_command(
args=[get_user_shell(), '-c', cmd],
in_data=in_Data,
chdir=chdir,
shell=shell,
)
def read_path(path):
"""
Fetch the contents of a filesystem `path` as bytes.

52
ansible_mitogen/mixins.py
Unescape Escape View File

@ -52,6 +52,7 @@ import mitogen.master
from mitogen.utils import cast
import ansible_mitogen.connection
import ansible_mitogen.planner
import ansible_mitogen.helpers
from ansible.module_utils._text import to_text
@ -59,22 +60,6 @@ from ansible.module_utils._text import to_text
LOG = logging.getLogger(__name__)
def get_command_module_name(module_name):
"""
Given the name of an Ansible command module, return its canonical module
path within the ansible.
:param module_name:
"shell"
:return:
"ansible.modules.commands.shell"
"""
path = module_loader.find_plugin(module_name, '')
relpath = os.path.relpath(path, os.path.dirname(ansible.__file__))
root, _ = os.path.splitext(relpath)
return 'ansible.' + root.replace('/', '.')
class ActionModuleMixin(ansible.plugins.action.ActionBase):
"""
The Mitogen-patched PluginLoader dynamically mixes this into every action
@ -308,29 +293,36 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
helpers.run_module() or helpers.run_module_async() in the target
context.
"""
if task_vars is None:
task_vars = {}
if module_name is None:
module_name = self._task.action
if module_args is None:
module_args = self._task.args
if task_vars is None:
task_vars = {}
self._update_module_args(module_name, module_args, task_vars)
if wrap_async:
helper = ansible_mitogen.helpers.run_module_async
else:
helper = ansible_mitogen.helpers.run_module
env = {}
self._compute_environment_string(env)
js = self.call(
helper,
get_command_module_name(module_name),
args=cast(module_args),
env=cast(env),
return ansible_mitogen.planner.invoke(
ansible_mitogen.planner.Invocation(
action=self,
connection=self._connection,
module_name=mitogen.utils.cast(module_name),
module_args=mitogen.utils.cast(module_args),
task_vars=task_vars,
tmp=tmp,
env=mitogen.utils.cast(env),
wrap_async=wrap_async,
)
)
def _postprocess_response(self, js):
"""
Apply fixups mimicking ActionBase._execute_module(); this is copied
verbatim from action/__init__.py, the guts of _parse_returned_data are
garbage and should be removed or reimplemented once tests exist.
"""
data = self._parse_returned_data({
'rc': 0,
'stdout': js,
@ -351,8 +343,8 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
encoding_errors='surrogate_then_replace',
chdir=None):
"""
Replace the mad rat's nest of logic in the base implementation by
simply calling helpers.exec_command() in the target context.
Override the base implementation by simply calling
helpers.exec_command() in the target context.
"""
LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)',
cmd, type(in_data), executable, chdir)

156
ansible_mitogen/planner.py
Unescape Escape View File

@ -27,32 +27,69 @@
# POSSIBILITY OF SUCH DAMAGE.
"""
This exists to detect every case defined in [0] and prepare arguments necessary
for the executor implementation running within the target, including preloading
any requisite files/Python modules known to be missing.
Classes to detect each case from [0] and prepare arguments necessary for the
corresponding Runner class within the target, including preloading requisite
files/modules known missing.
[0] "Ansible Module Architecture", developing_program_flow_modules.html
"""
from __future__ import absolute_import
import logging
import os
from ansible.executor import module_common
import ansible.errors
try:
from ansible.plugins.loader import module_loader
except ImportError: # Ansible <2.4
from ansible.plugins import module_loader
import mitogen
import mitogen.service
import ansible_mitogen.helpers
LOG = logging.getLogger(__name__)
class Invocation(object):
"""
Collect up a module's execution environment then use it to invoke
helpers.run_module() or helpers.run_module_async() in the target context.
"""
def __init__(self, action, connection, module_name, module_args,
task_vars, tmp, env, wrap_async):
#: Instance of the ActionBase subclass invoking the module. Required to
#: access some output postprocessing methods that don't belong in
#: ActionBase at all.
self.action = action
self.connection = connection
self.module_name = module_name
self.module_args = module_args
self.module_path = None
self.module_source = None
self.task_vars = task_vars
self.tmp = tmp
self.env = env
self.wrap_async = wrap_async
def __repr__(self):
return 'Invocation(module_name=%s)' % (self.module_name,)
class Planner(object):
"""
A Planner receives a module name and the contents of its implementation
file, indicates whether or not it understands how to run the module, and
exports a method to run the module.
"""
def detect(self, name, source):
assert 0
def detect(self, invocation):
raise NotImplementedError()
def run(self, connection, name, source, args, env):
assert 0
def plan(self, invocation):
raise NotImplementedError()
class JsonArgsPlanner(Planner):
@ -60,10 +97,10 @@ class JsonArgsPlanner(Planner):
Script that has its interpreter directive and the task arguments
substituted into its source as a JSON string.
"""
def detect(self, name, source):
return module_common.REPLACER_JSONARGS in source
def detect(self, invocation):
return module_common.REPLACER_JSONARGS in invocation.module_source
def run(self, name, source, args, env):
def plan(self, invocation):
path = None # TODO
mitogen.service.call(501, ('register', path))
return {
@ -79,17 +116,17 @@ class WantJsonPlanner(Planner):
If a module has the string WANT_JSON in it anywhere, Ansible treats it as a
non-native module that accepts a filename as its only command line
parameter. The filename is for a temporary file containing a JSON string
containing the module’s parameters. The module needs to open the file, read
containing the module's parameters. The module needs to open the file, read
and parse the parameters, operate on the data, and print its return data as
a JSON encoded dictionary to stdout before exiting.
These types of modules are self-contained entities. As of Ansible 2.1,
Ansible only modifies them to change a shebang line if present.
"""
def detect(self, name, source):
return 'WANT_JSON' in source
def detect(self, invocation):
return 'WANT_JSON' in invocation.module_source
def run(self, name, source, args, env):
def plan(self, name, source, args, env):
return {
'func': 'run_want_json_module',
'binary': source,
@ -122,10 +159,10 @@ class ReplacerPlanner(Planner):
"ansible/module_utils/powershell.ps1". It should only be used with
new-style Powershell modules.
"""
def detect(self, name, source):
return module_common.REPLACER in source
def detect(self, invocation):
return module_common.REPLACER in invocation.module_source
def run(self, name, source, args, env):
def plan(self, name, source, args, env):
return {
'func': 'run_replacer_module',
'binary': source,
@ -139,37 +176,49 @@ class BinaryPlanner(Planner):
Binary modules take their arguments and will return data to Ansible in the
same way as want JSON modules.
"""
helper = staticmethod(ansible_mitogen.helpers.run_binary)
def detect(self, name, source):
return module_common._is_binary(source)
def detect(self, invocation):
return module_common._is_binary(invocation.module_source)
def run(self, name, source, args, env):
def plan(self, name, source, args, env):
return {
'func': 'run_binary_module',
'runner_name': 'BinaryRunner',
'binary': source,
'args': args,
'env': env,
}
class PythonPlanner(Planner):
class NativePlanner(Planner):
"""
The Ansiballz framework differs from module replacer in that it uses real
Python imports of things in ansible/module_utils instead of merely
preprocessing the module.
"""
helper = staticmethod(ansible_mitogen.helpers.run_module)
def detect(self, name, source):
def detect(self, invocation):
return True
def run(self, name, source, args, env):
def get_command_module_name(self, module_name):
"""
Given the name of an Ansible command module, return its canonical
module path within the ansible.
:param module_name:
"shell"
:return:
"ansible.modules.commands.shell"
"""
path = module_loader.find_plugin(module_name, '')
relpath = os.path.relpath(path, os.path.dirname(ansible.__file__))
root, _ = os.path.splitext(relpath)
return 'ansible.' + root.replace('/', '.')
def plan(self, invocation):
return {
'func': 'run_python_module',
'module': name,
'args': args,
'env': env
'runner_name': 'NativeRunner',
'module': invocation.module_name,
'mod_name': self.get_command_module_name(invocation.module_name),
'args': invocation.module_args,
'env': invocation.env,
}
@ -178,9 +227,46 @@ _planners = [
# WantJsonPlanner,
# ReplacerPlanner,
BinaryPlanner,
PythonPlanner,
NativePlanner,
]
def plan():
pass
NO_METHOD_MSG = 'Mitogen: no invocation method found for: '
CRASHED_MSG = 'Mitogen: internal error: '
def get_module_data(name):
path = module_loader.find_plugin(name, '')
with open(path, 'rb') as fp:
source = fp.read()
return path, source
def invoke(invocation):
"""
Find a suitable Planner that knows how to run `invocation`.
"""
(invocation.module_path,
invocation.module_source) = get_module_data(invocation.module_name)
for klass in _planners:
planner = klass()
if planner.detect(invocation):
break
else:
raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
kwargs = planner.plan(invocation)
if invocation.wrap_async:
helper = ansible_mitogen.helpers.run_module_async
else:
helper = ansible_mitogen.helpers.run_module
try:
js = invocation.connection.call(helper, kwargs)
except mitogen.core.CallError as e:
LOG.exception('invocation crashed: %r', invocation)
summary = str(e).splitlines()[0]
raise ansible.errors.AnsibleInternalError(CRASHED_MSG + summary)
return invocation.action._postprocess_response(js)

151
ansible_mitogen/runner.py
Unescape Escape View File

@ -43,6 +43,60 @@ import ansible.module_utils.basic
ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
class Runner(object):
"""
Ansible module runner. After instantiation (with kwargs supplied by the
corresponding Planner), `.run()` is invoked, upon which `setup()`,
`_run()`, and `revert()` are invoked, with the return value of `_run()`
returned by `run()`.
Subclasses may override `_run`()` and extend `setup()` and `revert()`.
"""
def __init__(self, module, raw_params=None, args=None, env=None):
if args is None:
args = {}
if raw_params is not None:
args['_raw_params'] = raw_params
self.module = module
self.raw_params = raw_params
self.args = args
self.env = env
def setup(self):
"""
Prepare the current process for running a module. The base
implementation simply prepares the environment.
"""
self._env = TemporaryEnvironment(self.env)
def revert(self):
"""
Revert any changes made to the process after running a module. The base
implementation simply restores the original environment.
"""
self._env.revert()
def _run(self):
raise NotImplementedError()
def run(self):
"""
Set up the process environment in preparation for running an Ansible
module. This monkey-patches the Ansible libraries in various places to
prevent it from trying to kill the process on completion, and to
prevent it from reading sys.stdin.
:returns:
Module result dictionary.
"""
self.setup()
try:
return self._run()
finally:
self.revert()
class TemporaryEnvironment(object):
def __init__(self, env=None):
self.original = os.environ.copy()
@ -67,7 +121,7 @@ class NativeModuleExit(Exception):
ansible_module.do_cleanup_files()
self.dct = ansible.module_utils.basic.remove_values(
kwargs,
self.no_log_values,
ansible_module.no_log_values,
)
@ -79,7 +133,7 @@ class NativeMethodOverrides(object):
:class:`NativeModuleExit` exception`.
"""
kwargs.setdefault('changed', False)
return NativeModuleExit(self, **kwargs)
raise NativeModuleExit(self, **kwargs)
@staticmethod
def fail_json(self, **kwargs):
@ -88,7 +142,7 @@ class NativeMethodOverrides(object):
:class:`NativeModuleExit` exception`.
"""
kwargs.setdefault('failed', True)
return NativeModuleExit(self, **kwargs)
raise NativeModuleExit(self, **kwargs)
klass = ansible.module_utils.basic.AnsibleModule
@ -120,68 +174,18 @@ class NativeModuleArguments(object):
"""
Restore prior state.
"""
ansible.module_utils.basic._ANSIBLE_ARGS = self._original_args
class Runner(object):
"""
Ansible module runner. After instantiation (with kwargs supplied by the
corresponding Planner), `.run()` is invoked, upon which `setup()`,
`_run()`, and `revert()` are invoked, with the return value of `_run()`
returned by `run()`.
Subclasses may override `_run`()` and extend `setup()` and `revert()`.
"""
def __init__(self, module, raw_params=None, args=None, env=None):
if args is None:
args = {}
if raw_params is not None:
args['_raw_params'] = raw_params
self.module = module
self.raw_params = raw_params
self.args = args
self.env = env
ansible.module_utils.basic._ANSIBLE_ARGS = self.original
def setup(self):
"""
Prepare the current process for running a module. The base
implementation simply prepares the environment.
"""
self._env = TemporaryEnvironment(self.env)
def revert(self):
"""
Revert any changes made to the process after running a module. The base
implementation simply restores the original environment.
"""
self._env.revert()
def _run(self):
raise NotImplementedError()
def run(self):
"""
Set up the process environment in preparation for running an Ansible
module. This monkey-patches the Ansible libraries in various places to
prevent it from trying to kill the process on completion, and to
prevent it from reading sys.stdin.
:returns:
Module result dictionary.
"""
self.setup()
try:
return self._run()
finally:
self.revert()
class NativeRunner(object):
class NativeRunner(Runner):
"""
Execute a new-style Ansible module, where Module Replacer-related tricks
aren't required.
"""
def __init__(self, mod_name, **kwargs):
super(NativeRunner, self).__init__(**kwargs)
self.mod_name = mod_name
def setup(self):
super(NativeRunner, self).setup()
self._overrides = NativeMethodOverrides()
@ -192,18 +196,18 @@ class NativeRunner(object):
self._args.revert()
self._overrides.revert()
def module_fixups(mod):
"""
Apply fixups for known problems with mainline Ansible modules.
"""
if mod.__name__ == 'ansible.modules.packaging.os.yum_repository':
def _fixup__default(self, mod):
pass
def _fixup__yum_repository(self, mod):
# https://github.com/dw/mitogen/issues/154
mod.YumRepo.repofile = mod.configparser.RawConfigParser()
def _run(self):
fixup = getattr(self, '_fixup__' + self.module, self._fixup__default)
try:
mod = __import__(self.module, {}, {}, [''])
self.module_fixups(mod)
mod = __import__(self.mod_name, {}, {}, [''])
fixup(mod)
# Ansible modules begin execution on import. Thus the above
# __import__ will cause either Exit or ModuleError to be raised. If
# we reach the line below, the module did not execute and must
@ -219,7 +223,7 @@ class NativeRunner(object):
}
class BinaryRunner(object):
class BinaryRunner(Runner):
def __init__(self, path, **kwargs):
super(BinaryRunner, self).__init__(**kwargs)
self.path = path
@ -282,9 +286,16 @@ class BinaryRunner(object):
args=[self.bin_fp.name, self.args_fp.name],
)
except Exception, e:
return
# ...
assert 0
return {
'failed': True,
'msg': '%s: %s' % (type(e), e),
}
return {
'rc': rc,
'stdout': stdout,
'stderr': stderr
}
class WantJsonRunner(BinaryRunner):

Write Preview
Loading…
Cancel
Save