Merge pull request #322 from dw/dmw

Dmw
pull/318/head
dw 6 years ago committed by GitHub
commit 22e3335fa7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -81,10 +81,10 @@ echo travis_fold:end:job_setup
echo travis_fold:start:first_run
/usr/bin/time debops common
/usr/bin/time debops common "$@"
echo travis_fold:end:first_run
echo travis_fold:start:second_run
/usr/bin/time debops common
/usr/bin/time debops common "$@"
echo travis_fold:end:second_run

@ -44,9 +44,10 @@ import ansible.utils.shlex
import mitogen.unix
import mitogen.utils
import ansible_mitogen.target
import ansible_mitogen.parsing
import ansible_mitogen.process
import ansible_mitogen.services
import ansible_mitogen.target
LOG = logging.getLogger(__name__)
@ -248,6 +249,17 @@ CONNECTION_METHOD = {
}
def parse_python_path(s):
"""
Given the string set for ansible_python_interpeter, parse it using shell
syntax and return an appropriate argument vector.
"""
if not s:
return None
return ansible.utils.shlex.shlex_split(s)
def config_from_play_context(transport, inventory_name, connection):
"""
Return a dict representing all important connection configuration, allowing
@ -265,7 +277,7 @@ def config_from_play_context(transport, inventory_name, connection):
'become_pass': connection._play_context.become_pass,
'password': connection._play_context.password,
'port': connection._play_context.port,
'python_path': connection.python_path,
'python_path': parse_python_path(connection.python_path),
'private_key_file': connection._play_context.private_key_file,
'ssh_executable': connection._play_context.ssh_executable,
'timeout': connection._play_context.timeout,
@ -314,7 +326,7 @@ def config_from_hostvars(transport, inventory_name, connection,
'password': (hostvars.get('ansible_ssh_pass') or
hostvars.get('ansible_password')),
'port': hostvars.get('ansible_port'),
'python_path': hostvars.get('ansible_python_interpreter'),
'python_path': parse_python_path(hostvars.get('ansible_python_interpreter')),
'private_key_file': (hostvars.get('ansible_ssh_private_key_file') or
hostvars.get('ansible_private_key_file')),
'mitogen_via': hostvars.get('mitogen_via'),
@ -332,15 +344,19 @@ class Connection(ansible.plugins.connection.ConnectionBase):
#: mitogen.master.Router for this worker.
router = None
#: mitogen.master.Context representing the parent Context, which is
#: mitogen.parent.Context representing the parent Context, which is
#: presently always the connection multiplexer process.
parent = None
#: mitogen.master.Context connected to the target user account on the
#: target machine (i.e. via sudo).
#: mitogen.parent.Context for the target account on the target, possibly
#: reached via become.
context = None
#: mitogen.master.Context connected to the fork parent process in the
#: mitogen.parent.Context for the login account on the target. This is
#: always the login account, even when become=True.
login_context = None
#: mitogen.parent.Context connected to the fork parent process in the
#: target user account.
fork_context = None
@ -480,7 +496,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
"""
Establish a connection to the master process's UNIX listener socket,
constructing a mitogen.master.Router to communicate with the master,
and a mitogen.master.Context to represent it.
and a mitogen.parent.Context to represent it.
Depending on the original transport we should emulate, trigger one of
the _connect_*() service calls defined above to cause the master
@ -517,6 +533,11 @@ class Connection(ansible.plugins.connection.ConnectionBase):
raise ansible.errors.AnsibleConnectionFailure(dct['msg'])
self.context = dct['context']
if self._play_context.become:
self.login_context = dct['via']
else:
self.login_context = self.context
self.fork_context = dct['init_child_result']['fork_context']
self.home_dir = dct['init_child_result']['home_dir']
@ -534,6 +555,8 @@ class Connection(ansible.plugins.connection.ConnectionBase):
)
self.context = None
self.fork_context = None
self.login_context = None
if self.broker and not new_task:
self.broker.shutdown()
self.broker.join()
@ -544,11 +567,18 @@ class Connection(ansible.plugins.connection.ConnectionBase):
"""
Start a function call to the target.
:param bool use_login_context:
If present and :data:`True`, send the call to the login account
context rather than the optional become user context.
:returns:
mitogen.core.Receiver that receives the function call result.
"""
self._connect()
return self.context.call_async(func, *args, **kwargs)
if kwargs.pop('use_login_context', None):
call_context = self.login_context
else:
call_context = self.context
return call_context.call_async(func, *args, **kwargs)
def call(self, func, *args, **kwargs):
"""

@ -34,15 +34,20 @@ import sys
import mitogen.core
import mitogen.utils
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
class Handler(logging.Handler):
"""
Use Mitogen's log format, but send the result to a Display method.
"""
def __init__(self, display, normal_method):
def __init__(self, normal_method):
logging.Handler.__init__(self)
self.formatter = mitogen.utils.log_get_formatter()
self.display = display
self.normal_method = normal_method
#: Set of target loggers that produce warnings and errors that spam the
@ -62,41 +67,34 @@ class Handler(logging.Handler):
s = '[pid %d] %s' % (os.getpid(), self.format(record))
if record.levelno >= logging.ERROR:
self.display.error(s, wrap_text=False)
display.error(s, wrap_text=False)
elif record.levelno >= logging.WARNING:
self.display.warning(s, formatted=True)
display.warning(s, formatted=True)
else:
self.normal_method(s)
def find_display():
"""
Find the CLI tool's display variable somewhere up the stack. Why god why,
right? Because it's the the simplest way to get access to the verbosity
configured on the command line.
"""
f = sys._getframe()
while f:
if 'display' in f.f_locals:
return f.f_locals['display']
f = f.f_back
def setup():
"""
Install a handler for Mitogen's logger to redirect it into the Ansible
display framework, and prevent propagation to the root logger.
"""
display = find_display()
logging.getLogger('ansible_mitogen').handlers = [Handler(display, display.vvv)]
mitogen.core.LOG.handlers = [Handler(display, display.vvv)]
mitogen.core.IOLOG.handlers = [Handler(display, display.vvvv)]
logging.getLogger('ansible_mitogen').handlers = [Handler(display.vvv)]
mitogen.core.LOG.handlers = [Handler(display.vvv)]
mitogen.core.IOLOG.handlers = [Handler(display.vvvv)]
mitogen.core.IOLOG.propagate = False
if display.verbosity > 2:
logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG)
mitogen.core.LOG.setLevel(logging.DEBUG)
logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG)
else:
# Mitogen copies the active log level into new children, allowing them
# to filter tiny messages before they hit the network, and therefore
# before they wake the IO loop. Explicitly setting INFO saves ~4%
# running against just the local machine.
mitogen.core.LOG.setLevel(logging.ERROR)
logging.getLogger('ansible_mitogen').setLevel(logging.ERROR)
if display.verbosity > 3:
mitogen.core.IOLOG.setLevel(logging.DEBUG)
logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG)

@ -188,21 +188,23 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
except AttributeError:
s = ansible.constants.DEFAULT_REMOTE_TMP # <=2.4.x
return self._remote_expand_user(s)
return self._remote_expand_user(s, sudoable=False)
def _make_tmp_path(self, remote_user=None):
"""
Replace the base implementation's use of shell to implement mkdtemp()
with an actual call to mkdtemp().
with an actual call to mkdtemp(). Like vanilla, the directory is always
created in the login account context.
"""
LOG.debug('_make_tmp_path(remote_user=%r)', remote_user)
# _make_tmp_path() is basically a global stashed away as Shell.tmpdir.
# The copy action plugin violates layering and grabs this attribute
# directly.
self._connection._shell.tmpdir = self.call(
self._connection._shell.tmpdir = self._connection.call(
ansible_mitogen.target.make_temp_directory,
base_dir=self._get_remote_tmp(),
use_login_context=True,
)
LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir)
self._cleanup_remote_tmp = True
@ -280,20 +282,26 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
"""
Replace the base implementation's attempt to emulate
os.path.expanduser() with an actual call to os.path.expanduser().
:param bool sudoable:
If :data:`True`, indicate unqualified tilde ("~" with no username)
should be evaluated in the context of the login account, not any
become_user.
"""
LOG.debug('_remote_expand_user(%r, sudoable=%r)', path, sudoable)
if not path.startswith('~'):
# /home/foo -> /home/foo
return path
if path == '~':
# ~ -> /home/dmw
return self._connection.homedir
if path.startswith('~/'):
# ~/.ansible -> /home/dmw/.ansible
return os.path.join(self._connection.homedir, path[2:])
if path.startswith('~'):
# ~root/.ansible -> /root/.ansible
return self.call(os.path.expanduser, mitogen.utils.cast(path))
if sudoable or not self._play_context.become:
if path == '~':
# ~ -> /home/dmw
return self._connection.homedir
if path.startswith('~/'):
# ~/.ansible -> /home/dmw/.ansible
return os.path.join(self._connection.homedir, path[2:])
# ~root/.ansible -> /root/.ansible
return self.call(os.path.expanduser, mitogen.utils.cast(path),
use_login_context=not sudoable)
def get_task_timeout_secs(self):
"""

@ -0,0 +1,84 @@
# Copyright 2017, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
"""
Classes to detect each case from [0] and prepare arguments necessary for the
corresponding Runner class within the target, including preloading requisite
files/modules known missing.
[0] "Ansible Module Architecture", developing_program_flow_modules.html
"""
from __future__ import absolute_import
from __future__ import unicode_literals
import mitogen.core
def parse_script_interpreter(source):
"""
Parse the script interpreter portion of a UNIX hashbang using the rules
Linux uses.
:param str source: String like "/usr/bin/env python".
:returns:
Tuple of `(interpreter, arg)`, where `intepreter` is the script
interpreter and `arg` is its sole argument if present, otherwise
:py:data:`None`.
"""
# Find terminating newline. Assume last byte of binprm_buf if absent.
nl = source.find(b'\n', 0, 128)
if nl == -1:
nl = min(128, len(source))
# Split once on the first run of whitespace. If no whitespace exists,
# bits just contains the interpreter filename.
bits = source[0:nl].strip().split(None, 1)
if len(bits) == 1:
return mitogen.core.to_text(bits[0]), None
return mitogen.core.to_text(bits[0]), mitogen.core.to_text(bits[1])
def parse_hashbang(source):
"""
Parse a UNIX "hashbang line" using the syntax supported by Linux.
:param str source: String like "#!/usr/bin/env python".
:returns:
Tuple of `(interpreter, arg)`, where `intepreter` is the script
interpreter and `arg` is its sole argument if present, otherwise
:py:data:`None`.
"""
# Linux requires first 2 bytes with no whitespace, pretty sure it's the
# same everywhere. See binfmt_script.c.
if not source.startswith(b'#!'):
return None, None
return parse_script_interpreter(source[2:])

@ -48,6 +48,7 @@ import ansible.module_utils
import mitogen.core
import ansible_mitogen.loaders
import ansible_mitogen.parsing
import ansible_mitogen.target
@ -56,34 +57,6 @@ NO_METHOD_MSG = 'Mitogen: no invocation method found for: '
NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line'
def parse_script_interpreter(source):
"""
Extract the script interpreter and its sole argument from the module
source code.
:returns:
Tuple of `(interpreter, arg)`, where `intepreter` is the script
interpreter and `arg` is its sole argument if present, otherwise
:py:data:`None`.
"""
# Linux requires first 2 bytes with no whitespace, pretty sure it's the
# same everywhere. See binfmt_script.c.
if not source.startswith(b'#!'):
return None, None
# Find terminating newline. Assume last byte of binprm_buf if absent.
nl = source.find(b'\n', 0, 128)
if nl == -1:
nl = min(128, len(source))
# Split once on the first run of whitespace. If no whitespace exists,
# bits just contains the interpreter filename.
bits = source[2:nl].strip().split(None, 1)
if len(bits) == 1:
return mitogen.core.to_text(bits[0]), None
return mitogen.core.to_text(bits[0]), mitogen.core.to_text(bits[1])
class Invocation(object):
"""
Collect up a module's execution environment then use it to invoke
@ -214,27 +187,51 @@ class ScriptPlanner(BinaryPlanner):
Common functionality for script module planners -- handle interpreter
detection and rewrite.
"""
def _rewrite_interpreter(self, path):
"""
Given the original interpreter binary extracted from the script's
interpreter line, look up the associated `ansible_*_interpreter`
variable, render it and return it.
:param str path:
Absolute UNIX path to original interpreter.
:returns:
Shell fragment prefix used to execute the script via "/bin/sh -c".
While `ansible_*_interpreter` documentation suggests shell isn't
involved here, the vanilla implementation uses it and that use is
exploited in common playbooks.
"""
try:
key = u'ansible_%s_interpreter' % os.path.basename(path).strip()
template = self._inv.task_vars[key]
except KeyError:
return path
return mitogen.utils.cast(
self._inv.templar.template(self._inv.task_vars[key])
)
def _get_interpreter(self):
interpreter, arg = parse_script_interpreter(
path, arg = ansible_mitogen.parsing.parse_hashbang(
self._inv.module_source
)
if interpreter is None:
if path is None:
raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % (
self._inv.module_name,
))
key = u'ansible_%s_interpreter' % os.path.basename(interpreter).strip()
try:
template = self._inv.task_vars[key].strip()
return self._inv.templar.template(template), arg
except KeyError:
return interpreter, arg
fragment = self._rewrite_interpreter(path)
if arg:
fragment += ' ' + arg
return fragment, path.startswith('python')
def get_kwargs(self, **kwargs):
interpreter, arg = self._get_interpreter()
interpreter_fragment, is_python = self._get_interpreter()
return super(ScriptPlanner, self).get_kwargs(
interpreter_arg=arg,
interpreter=interpreter,
interpreter_fragment=interpreter_fragment,
is_python=is_python,
**kwargs
)

@ -381,11 +381,10 @@ class ProgramRunner(Runner):
)
def _get_program_args(self):
return [
self.args['_ansible_shell_executable'],
'-c',
self.program_fp.name
]
"""
Return any arguments to pass to the program.
"""
return []
def revert(self):
"""
@ -395,14 +394,30 @@ class ProgramRunner(Runner):
self.program_fp.close()
super(ProgramRunner, self).revert()
def _get_argv(self):
"""
Return the final argument vector used to execute the program.
"""
return [
self.args['_ansible_shell_executable'],
'-c',
self._get_shell_fragment(),
]
def _get_shell_fragment(self):
return "%s %s" % (
shlex_quote(self.program_fp.name),
' '.join(map(shlex_quote, self._get_program_args())),
)
def _run(self):
try:
rc, stdout, stderr = ansible_mitogen.target.exec_args(
args=self._get_program_args(),
args=self._get_argv(),
emulate_tty=self.emulate_tty,
)
except Exception as e:
LOG.exception('While running %s', self._get_program_args())
LOG.exception('While running %s', self._get_argv())
return {
'rc': 1,
'stdout': '',
@ -442,11 +457,7 @@ class ArgsFileRunner(Runner):
return json.dumps(self.args)
def _get_program_args(self):
return [
self.args['_ansible_shell_executable'],
'-c',
"%s %s" % (self.program_fp.name, self.args_fp.name),
]
return [self.args_fp.name]
def revert(self):
"""
@ -461,10 +472,10 @@ class BinaryRunner(ArgsFileRunner, ProgramRunner):
class ScriptRunner(ProgramRunner):
def __init__(self, interpreter, interpreter_arg, **kwargs):
def __init__(self, interpreter_fragment, is_python, **kwargs):
super(ScriptRunner, self).__init__(**kwargs)
self.interpreter = interpreter
self.interpreter_arg = interpreter_arg
self.interpreter_fragment = interpreter_fragment
self.is_python = is_python
b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
@ -473,21 +484,34 @@ class ScriptRunner(ProgramRunner):
super(ScriptRunner, self)._get_program()
)
def _get_argv(self):
return [
self.args['_ansible_shell_executable'],
'-c',
self._get_shell_fragment(),
]
def _get_shell_fragment(self):
"""
Scripts are eligible for having their hashbang line rewritten, and to
be executed via /bin/sh using the ansible_*_interpreter value used as a
shell fragment prefixing to the invocation.
"""
return "%s %s %s" % (
self.interpreter_fragment,
shlex_quote(self.program_fp.name),
' '.join(map(shlex_quote, self._get_program_args())),
)
def _rewrite_source(self, s):
"""
Mutate the source according to the per-task parameters.
"""
# Couldn't find shebang, so let shell run it, because shell assumes
# executables like this are just shell scripts.
if not self.interpreter:
return s
shebang = b'#!' + utf8(self.interpreter)
if self.interpreter_arg:
shebang += b' ' + utf8(self.interpreter_arg)
new = [shebang]
if os.path.basename(self.interpreter).startswith('python'):
# While Ansible rewrites the #! using ansible_*_interpreter, it is
# never actually used to execute the script, instead it is a shell
# fragment consumed by shell/__init__.py::build_module_command().
new = [b'#!' + utf8(self.interpreter_fragment)]
if self.is_python:
new.append(self.b_ENCODING_STRING)
_, _, rest = s.partition(b'\n')

@ -267,6 +267,7 @@ class ContextService(mitogen.service.Service):
{
'context': mitogen.core.Context or None,
'via': mitogen.core.Context or None,
'init_child_result': {
'fork_context': mitogen.core.Context,
'home_dir': str or None,
@ -297,7 +298,8 @@ class ContextService(mitogen.service.Service):
lambda: self._on_stream_disconnect(stream))
self._send_module_forwards(context)
init_child_result = context.call(ansible_mitogen.target.init_child)
init_child_result = context.call(ansible_mitogen.target.init_child,
log_level=LOG.getEffectiveLevel())
if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'):
from mitogen import debug
@ -307,6 +309,7 @@ class ContextService(mitogen.service.Service):
self._refs_by_context[context] = 0
return {
'context': context,
'via': via,
'init_child_result': init_child_result,
'msg': None,
}
@ -357,7 +360,7 @@ class ContextService(mitogen.service.Service):
Subsequent elements are proxied via the previous.
:returns dict:
* context: mitogen.master.Context or None.
* context: mitogen.parent.Context or None.
* init_child_result: Result of :func:`init_child`.
* msg: StreamError exception text or None.
* method_name: string failing method name.

@ -202,7 +202,7 @@ def reset_temp_dir(econtext):
@mitogen.core.takes_econtext
def init_child(econtext):
def init_child(econtext, log_level):
"""
Called by ContextService immediately after connection; arranges for the
(presently) spotless Python interpreter to be forked, where the newly
@ -213,6 +213,9 @@ def init_child(econtext):
polluting the global interpreter state in a way that effects explicitly
isolated modules.
:param int log_level:
Logging package level active in the master.
:returns:
Dict like::
@ -230,6 +233,12 @@ def init_child(econtext):
_fork_parent = econtext.router.fork()
reset_temp_dir(econtext)
# Copying the master's log level causes log messages to be filtered before
# they reach LogForwarder, thus reducing an influx of tiny messges waking
# the connection multiplexer process in the master.
LOG.setLevel(log_level)
logging.getLogger('ansible_mitogen').setLevel(log_level)
return {
'fork_context': _fork_parent,
'home_dir': mitogen.core.to_text(os.path.expanduser('~')),
@ -407,6 +416,9 @@ def make_temp_directory(base_dir):
:returns:
Newly created temporary directory.
"""
# issue #301: remote_tmp may contain $vars.
base_dir = os.path.expandvars(base_dir)
if not os.path.exists(base_dir):
os.makedirs(base_dir, mode=int('0700', 8))
return tempfile.mkdtemp(

@ -137,7 +137,7 @@ Noteworthy Differences
`lxd <https://docs.ansible.com/ansible/2.6/plugins/connection/lxd.html>`_,
and `ssh <https://docs.ansible.com/ansible/2.6/plugins/connection/ssh.html>`_
built-in connection types are supported, along with Mitogen-specific
:ref:`machinectl <machinectl>`, :ref:`mitogen_doas< mitogen_doas>`,
:ref:`machinectl <machinectl>`, :ref:`mitogen_doas <doas>`,
:ref:`mitogen_su <su>`, :ref:`mitogen_sudo <sudo>`, and :ref:`setns <setns>`
types. File bugs to register interest in others.
@ -158,6 +158,13 @@ Noteworthy Differences
may be established in parallel by default, this can be modified by setting
the ``MITOGEN_POOL_SIZE`` environment variable.
* The ``ansible_python_interpreter`` variable is parsed using a restrictive
:mod:`shell-like <shlex>` syntax, permitting values such as ``/usr/bin/env
FOO=bar python``, which occur in practice. Ansible `documents this
<https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#ansible-python-interpreter>`_
as an absolute path, however the implementation passes it unquoted through
the shell, permitting arbitrary code to be injected.
* Performance does not scale linearly with target count. This will improve over
time.
@ -514,22 +521,6 @@ connection delegation is supported.
* ``ansible_user``: Name of user within the container to execute as.
.. _machinectl:
Machinectl
~~~~~~~~~~
Like the `machinectl third party plugin
<https://github.com/BaxterStockman/ansible-connection-machinectl>`_ except
connection delegation is supported. This is a light wrapper around the
:ref:`setns <setns>` method.
* ``ansible_host``: Name of Docker container (default: inventory hostname).
* ``ansible_user``: Name of user within the container to execute as.
* ``mitogen_machinectl_path``: path to ``machinectl`` command if not available
as ``/bin/machinectl``.
FreeBSD Jail
~~~~~~~~~~~~
@ -551,6 +542,23 @@ connection delegation is supported.
* ``ansible_python_interpreter``
Process Model
^^^^^^^^^^^^^
Ansible usually executes local connection commands as a transient subprocess of
the forked worker executing a task. With the extension, the local connection
exists as a persistent subprocess of the connection multiplexer.
This means that global state mutations made to the top-level Ansible process
that are normally visible to newly forked subprocesses, such as vars plug-ins
that modify the environment, will not be reflected when executing local
commands without additional effort.
During execution the extension presently mimics the working directory and
process environment inheritence of regular Ansible, however it is possible some
additional differences exist that may break existing playbooks.
.. _method-lxc:
LXC
@ -567,6 +575,22 @@ The ``lxc-attach`` command must be available on the host machine.
* ``ansible_host``: Name of LXC container (default: inventory hostname).
.. _machinectl:
Machinectl
~~~~~~~~~~
Like the `machinectl third party plugin
<https://github.com/BaxterStockman/ansible-connection-machinectl>`_ except
connection delegation is supported. This is a light wrapper around the
:ref:`setns <setns>` method.
* ``ansible_host``: Name of Docker container (default: inventory hostname).
* ``ansible_user``: Name of user within the container to execute as.
* ``mitogen_machinectl_path``: path to ``machinectl`` command if not available
as ``/bin/machinectl``.
.. _setns:
Setns

@ -483,9 +483,13 @@ Router Class
determine its installation prefix. This is required to support
virtualenv.
:param str python_path:
Path to the Python interpreter to use for bootstrap. Defaults to
:data:`sys.executable`. For SSH, defaults to ``python``.
:param str|list python_path:
String or list path to the Python interpreter to use for bootstrap.
Defaults to :data:`sys.executable` for local connections, and
``python`` for remote connections.
It is possible to pass a list to invoke Python wrapped using
another tool, such as ``["/usr/bin/env", "python"]``.
:param bool debug:
If :data:`True`, arrange for debug logging (:py:meth:`enable_debug`) to

@ -21,27 +21,71 @@ v0.2.2 (2018-07-??)
Mitogen for Ansible
~~~~~~~~~~~~~~~~~~~
* `#299 <https://github.com/dw/mitogen/pull/299>`_: fix the ``network_cli``
connection type when the Mitogen strategy is active.
* `#291 <https://github.com/dw/mitogen/issues/291>`_: ``ansible_*_interpreter``
variables are parsed using a restrictive shell-like syntax, supporting a
common idiom where ``ansible_python_interpreter`` is set to ``/usr/bin/env
python``.
* `#309 <https://github.com/dw/mitogen/pull/309>`_: fix a regression to process
environment cleanup, caused by the change in v0.2.1 to run local tasks with
the correct environment.
* `#299 <https://github.com/dw/mitogen/issues/299>`_: fix the ``network_cli``
connection type when the Mitogen strategy is active. Mitogen cannot help
network device connections, however it should still be possible to use device
connections while Mitogen is active.
* `#301 <https://github.com/dw/mitogen/pull/301>`_: variables like ``$HOME`` in
the ``remote_tmp`` setting are evaluated correctly.
* `#303 <https://github.com/dw/mitogen/pull/303>`_: the ``doas`` become method
is supported. Contributed by `Mike Walker
<https://github.com/napkindrawing>`_.
* `#309 <https://github.com/dw/mitogen/issues/309>`_: fix a regression to
process environment cleanup, caused by the change in v0.2.1 to run local
tasks with the correct environment.
* `#315 <https://github.com/dw/mitogen/pull/315>`_: Mitogen for Ansible is
supported under Ansible 2.6. Contributed by `Dan Quackenbush
<https://github.com/danquack>`_.
* `#317 <https://github.com/dw/mitogen/issues/317>`_: respect the verbosity
setting when writing to to Ansible's ``log_path`` log file, if it is enabled.
Child log filtering was also incorrect, causing the master to needlessly wake
many times. This nets a 3.5% runtime improvement running against the local
machine.
Core Library
~~~~~~~~~~~~
* `#291 <https://github.com/dw/mitogen/issues/291>`_: the ``python_path``
paramater may specify an argument vector prefix rather than a single string
program path.
* `#303 <https://github.com/dw/mitogen/pull/303>`_: the ``doas`` become method
is now supported. Contributed by Mike Walker.
is now supported. Contributed by `Mike Walker
<https://github.com/napkindrawing>`_.
* `#307 <https://github.com/dw/mitogen/pull/307>`_: SSH login banner output
* `#307 <https://github.com/dw/mitogen/issues/307>`_: SSH login banner output
containing the word 'password' is no longer confused for a password prompt.
* Debug logs containing command lines are printed with the minimal quoting and
escaping required.
Thanks!
~~~~~~~
Mitogen would not be possible without the support of users. A huge thanks for
the bug reports and pull requests in this release contributed by
`Frances Albanese <https://github.com/falbanese>`_,
`Mark Janssen <https://github.com/sigio>`_,
`Ayaz Ahmed Khan <https://github.com/ayaz>`_,
`Colin McCarthy <https://github.com/colin-mccarthy>`_,
`Dan Quackenbush <https://github.com/danquack>`_,
`Alex Russu <https://github.com/alexrussu>`_,
`Josh Smift <https://github.com/jbscare>`_, and
`Mike Walker <https://github.com/napkindrawing>`_.
v0.2.1 (2018-07-10)
-------------------

@ -101,6 +101,7 @@ sponsorship and outstanding future-thinking of its early adopters.
<ul>
<li>Alex Willmer</li>
<li><a href="https://underwhelm.net/">Dan Dorman</a> &mdash; - <em>When I truly understand my enemy … then in that very moment I also love him.</em></li>
<li>Daniel Foerster</li>
<li><a href="https://www.deps.co/">Deps</a> &mdash; <em>Private Maven Repository Hosting for Java, Scala, Groovy, Clojure</em></li>
<li><a href="https://www.edport.co.uk/">Edward Wilson</a> &mdash; <em>To efficiency and beyond! I wish Mitogen and all who sail in her the best of luck.</em></li>
<li><a href="https://www.epartment.nl/">Epartment</a></li>

@ -163,7 +163,7 @@ Logging Records
Messages received from a child context via :class:`mitogen.master.LogForwarder`
receive extra attributes:
* `mitogen_context`: :class:`mitogen.master.Context` referring to the message
* `mitogen_context`: :class:`mitogen.parent.Context` referring to the message
source.
* `mitogen_name`: original logger name in the source context.
* `mitogen_msg`: original message in the source context.

@ -815,10 +815,21 @@ class Importer(object):
def get_filename(self, fullname):
if fullname in self._cache:
path = self._cache[fullname][2]
if path is None:
# If find_loader() returns self but a subsequent master RPC
# reveals the module can't be loaded, and so load_module()
# throws ImportError, on Python 3.x it is still possible for
# the loader to be called to fetch metadata.
raise ImportError('master cannot serve %r' % (fullname,))
return u'master:' + self._cache[fullname][2]
def get_source(self, fullname):
if fullname in self._cache:
compressed = self._cache[fullname][3]
if compressed is None:
raise ImportError('master cannot serve %r' % (fullname,))
source = zlib.decompress(self._cache[fullname][3])
if PY3:
return to_text(source)
@ -851,7 +862,7 @@ class LogHandler(logging.Handler):
class Side(object):
_fork_refs = weakref.WeakValueDictionary()
def __init__(self, stream, fd, cloexec=True, keep_alive=True):
def __init__(self, stream, fd, cloexec=True, keep_alive=True, blocking=False):
self.stream = stream
self.fd = fd
self.closed = False
@ -859,7 +870,8 @@ class Side(object):
self._fork_refs[id(self)] = self
if cloexec:
set_cloexec(fd)
set_nonblock(fd)
if not blocking:
set_nonblock(fd)
def __repr__(self):
return '<Side of %r fd %s>' % (self.stream, self.fd)
@ -1520,7 +1532,7 @@ class IoLogger(BasicStream):
set_cloexec(self._wsock.fileno())
self.receive_side = Side(self, self._rsock.fileno())
self.transmit_side = Side(self, dest_fd, cloexec=False)
self.transmit_side = Side(self, dest_fd, cloexec=False, blocking=True)
self._broker.start_receive(self)
def __repr__(self):

@ -566,7 +566,10 @@ class KqueuePoller(mitogen.core.Poller):
changelist, 32, timeout)
for event in events:
fd = event.ident
if event.filter == select.KQ_FILTER_READ and fd in self._rfds:
if event.flags & select.KQ_EV_ERROR:
LOG.debug('ignoring stale event for fd %r: errno=%d: %s',
fd, event.data, errno.errorcode.get(event.data))
elif event.filter == select.KQ_FILTER_READ and fd in self._rfds:
# Events can still be read for an already-discarded fd.
mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd)
yield self._rfds[fd]
@ -871,6 +874,19 @@ class Stream(mitogen.core.Stream):
fp.close()
os.write(1,'MITO001\n'.encode())
def get_python_argv(self):
"""
Return the initial argument vector elements necessary to invoke Python,
by returning a 1-element list containing :attr:`python_path` if it is a
string, or simply returning it if it is already a list.
This allows emulation of existing tools where the Python invocation may
be set to e.g. `['/usr/bin/env', 'python']`.
"""
if isinstance(self.python_path, list):
return self.python_path
return [self.python_path]
def get_boot_command(self):
source = inspect.getsource(self._first_stage)
source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:]))
@ -886,8 +902,8 @@ class Stream(mitogen.core.Stream):
# codecs.decode() requires a bytes object. Since we must be compatible
# with 2.4 (no bytes literal), an extra .encode() either returns the
# same str (2.x) or an equivalent bytes (3.x).
return [
self.python_path, '-c',
return self.get_python_argv() + [
'-c',
'import codecs,os,sys;_=codecs.decode;'
'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),)
]

@ -17,8 +17,9 @@ timeout = 10
# On Travis, paramiko check fails due to host key checking enabled.
host_key_checking = False
# Required by integration/runner__remote_tmp.yml
remote_tmp = ~/.ansible/mitogen-tests/
# "mitogen-tests" required by integration/runner/remote_tmp.yml
# "$HOME" required by integration/action/make_tmp_path.yml
remote_tmp = $HOME/.ansible/mitogen-tests/
[ssh_connection]
ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s

@ -1,4 +1,5 @@
- import_playbook: remote_file_exists.yml
- import_playbook: remote_expand_user.yml
- import_playbook: low_level_execute_command.yml
- import_playbook: make_tmp_path.yml
- import_playbook: transfer_data.yml

@ -2,16 +2,54 @@
- name: integration/action/make_tmp_path.yml
hosts: test-targets
any_errors_fatal: true
gather_facts: true
tasks:
- name: "Find out root's homedir."
# Runs first because it blats regular Ansible facts with junk, so
# non-become run fixes that up.
setup: gather_subset=min
become: true
register: root_facts
- name: "Find regular homedir"
setup: gather_subset=min
register: user_facts
#
# non-become
#
- action_passthrough:
method: _make_tmp_path
register: out
- assert:
# This string must match ansible.cfg::remote_tmp
that: out.result.startswith("{{user_facts.ansible_facts.ansible_user_dir}}/.ansible/mitogen-tests/")
- stat:
path: "{{out.result}}"
register: st
- assert:
that: st.stat.exists and st.stat.isdir and st.stat.mode == "0700"
- file:
path: "{{out.result}}"
state: absent
#
# become. make_tmp_path() must evaluate HOME in the context of the SSH
# user, not the become user.
#
- action_passthrough:
method: _make_tmp_path
register: out
become: true
- assert:
# This string must match ansible.cfg::remote_tmp
that: out.result.startswith("{{ansible_user_dir}}/.ansible/mitogen-tests/")
that: out.result.startswith("{{user_facts.ansible_facts.ansible_user_dir}}/.ansible/mitogen-tests/")
- stat:
path: "{{out.result}}"

@ -0,0 +1,104 @@
# related to issue 301. Essentially ensure remote_expand_user does not support
# $HOME expansion.
- name: integration/action/remote_expand_user.yml
hosts: test-targets
any_errors_fatal: true
tasks:
- name: "Find out root's homedir."
# Runs first because it blats regular Ansible facts with junk, so
# non-become run fixes that up.
setup: gather_subset=min
become: true
register: root_facts
- name: "Find regular homedir"
setup: gather_subset=min
register: user_facts
# ------------------------
- name: "Expand ~/foo"
action_passthrough:
method: _remote_expand_user
kwargs:
path: '~/foo'
sudoable: false
register: out
- assert:
that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo'
- name: "Expand ~/foo with become active. ~ is become_user's home."
action_passthrough:
method: _remote_expand_user
kwargs:
path: '~/foo'
sudoable: false
register: out
become: true
- assert:
that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo'
- name: "Expand ~user/foo"
action_passthrough:
method: _remote_expand_user
kwargs:
path: '~{{ansible_user_id}}/foo'
sudoable: false
register: out
- assert:
that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo'
- name: "Expanding $HOME/foo has no effect."
action_passthrough:
method: _remote_expand_user
kwargs:
path: '$HOME/foo'
sudoable: false
register: out
- assert:
that: out.result == '$HOME/foo'
# ------------------------
- name: "sudoable; Expand ~/foo"
action_passthrough:
method: _remote_expand_user
kwargs:
path: '~/foo'
sudoable: true
register: out
- assert:
that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo'
- name: "sudoable; Expand ~/foo with become active. ~ is become_user's home."
action_passthrough:
method: _remote_expand_user
kwargs:
path: '~/foo'
sudoable: true
register: out
become: true
- assert:
that: out.result == '{{root_facts.ansible_facts.ansible_user_dir}}/foo'
- name: "sudoable; Expand ~user/foo"
action_passthrough:
method: _remote_expand_user
kwargs:
path: '~{{ansible_user_id}}/foo'
sudoable: true
register: out
- assert:
that: out.result == '{{user_facts.ansible_facts.ansible_user_dir}}/foo'
- name: "sudoable; Expanding $HOME/foo has no effect."
action_passthrough:
method: _remote_expand_user
kwargs:
path: '$HOME/foo'
sudoable: true
register: out
- assert:
that: out.result == '$HOME/foo'

@ -1,6 +1,7 @@
- import_playbook: builtin_command_module.yml
- import_playbook: custom_bash_old_style_module.yml
- import_playbook: custom_bash_want_json_module.yml
- import_playbook: custom_bash_hashbang_argument.yml
- import_playbook: custom_binary_producing_json.yml
- import_playbook: custom_binary_producing_junk.yml
- import_playbook: custom_binary_single_null.yml

@ -0,0 +1,19 @@
# https://github.com/dw/mitogen/issues/291
- name: integration/runner/custom_bash_hashbang_argument.yml
hosts: test-targets
any_errors_fatal: true
tasks:
- custom_bash_old_style_module:
foo: true
with_sequence: start=1 end={{end|default(1)}}
register: out
vars:
ansible_bash_interpreter: "/usr/bin/env RUN_VIA_ENV=yes bash"
- assert:
that: |
(not out.changed) and
(not out.results[0].changed) and
out.results[0].msg == 'Here is my input' and
out.results[0].run_via_env == "yes"

@ -16,5 +16,6 @@ echo "{"
echo " \"changed\": false,"
echo " \"msg\": \"Here is my input\","
echo " \"filename\": \"$INPUT\","
echo " \"run_via_env\": \"$RUN_VIA_ENV\","
echo " \"input\": [\"$(cat $INPUT | tr \" \' )\"]"
echo "}"

@ -0,0 +1,12 @@
#!/bin/bash
# This script exists to test the behavior of Stream.python_path being set to a
# list. It sets an environmnt variable that we can detect, then executes any
# arguments passed to it.
export EXECUTED_VIA_ENV_WRAPPER=1
if [ "${1:0:1}" == "-" ]; then
exec "$PYTHON" "$@"
else
export ENV_WRAPPER_FIRST_ARG="$1"
shift
exec "$@"
fi

@ -1,5 +1,6 @@
import os
import sys
import unittest2
@ -11,6 +12,14 @@ import testlib
import plain_old_module
def get_sys_executable():
return sys.executable
def get_os_environ():
return dict(os.environ)
class LocalTest(testlib.RouterMixin, unittest2.TestCase):
stream_class = mitogen.ssh.Stream
@ -20,5 +29,35 @@ class LocalTest(testlib.RouterMixin, unittest2.TestCase):
self.assertEquals('local.%d' % (pid,), context.name)
class PythonPathTest(testlib.RouterMixin, unittest2.TestCase):
stream_class = mitogen.ssh.Stream
def test_inherited(self):
context = self.router.local()
self.assertEquals(sys.executable, context.call(get_sys_executable))
def test_string(self):
os.environ['PYTHON'] = sys.executable
context = self.router.local(
python_path=testlib.data_path('env_wrapper.sh'),
)
self.assertEquals(sys.executable, context.call(get_sys_executable))
env = context.call(get_os_environ)
self.assertEquals('1', env['EXECUTED_VIA_ENV_WRAPPER'])
def test_list(self):
context = self.router.local(
python_path=[
testlib.data_path('env_wrapper.sh'),
"magic_first_arg",
sys.executable
]
)
self.assertEquals(sys.executable, context.call(get_sys_executable))
env = context.call(get_os_environ)
self.assertEquals('magic_first_arg', env['ENV_WRAPPER_FIRST_ARG'])
self.assertEquals('1', env['EXECUTED_VIA_ENV_WRAPPER'])
if __name__ == '__main__':
unittest2.main()

Loading…
Cancel
Save