Merge remote-tracking branch 'origin/dmw' into stable

* origin/dmw:
  issue #537: disable just the trivial LinuxPolicyTest on Travis.
  docs: update Changelog; closes #537.
  ansible: refactor affinity class and add abstract tests.
  Bump version for release.
  docs: update Changelog.
  core: serialize calls to _service_stub_main().
  docs: update Changelog; closes #532.
  issue #532: PushFileService race.
  docs: more concise Changelog.
  issue #541: changelog typos.
  ansible: quiesce boto logger; closes #541.
  docs: update Changelog.
  tests/ansible: Spec.port() test & mitogen_via= fix.
  Update copyright year everywhere.
  tests/ansible: Spec.become_pass() test.
  docs: remove top "Table of Contents" link
  docs: remove a little more top margin wastage
  tests/ansible: Spec.become_user() test.
  docs: update Changelog; closes #539.
  issue #539: disable logger propagation.
  ansible: capture stderr stream of async tasks. Closes #540.
  docs: update Changelog.
  issue #536: rework how 2.3-compatible simplejson is served
  .github: add some more questions to issue template
  docs: duplicate word
  docs: update Changelog.
  tests/ansible: Spec.become_method() test & mitogen_via= fix.
  setup.py: include LICENSE; closes #538.
  tests/ansible: Spec.become() test
  tests/ansible: Spec.password() test, document interactive pw limitation.
  tests/ansible: Spec.remote_user() test & mitogen_via= fix.
  tests/ansible: Spec.remote_addr() test & mitogen_via= fix.
  tests/ansible: Spec.transport() test.
  docs: lighter pink
  docs: add 'Fixes' heading
  docs: more margin tweaks for changelog
  docs: tighter <p> margins, even less shouting, red headings
  docs: tidy up footer and GitHub link
  docs: enable fixed_sidebar
  docs: sans-serif fonts, reduce shouty headings
  issue #536: add mitogen_via= tests too.
  ansible: fix a crash on 2.3 when mitogen_via= host is missing.
  tests: for 2.3 compatibility, disable gcloud.py for now
  docs: update Changelog; closes #511, closes #536.
  docs: update Changelog release date.
  issue #536: disable transport_config tests on vanilla
  issue #536: restore correct Python interpreter selection behaviour.
  issue #536: connection_delegation/ tests were erroneously broken
  tests: define MITOGEN_INVENTORY_FILE even if -i unspecified.
  issue #536: add tests for each ansible_python_interpreter case.
  issue #536: stop defining explicit localhost in inventory.
  tests: allow running Ansible tests locally without -udmw again.
  stable: fix preamble_size on stable docs.
  issue #481: add test.
pull/862/head v0.2.5
David Wilson 6 years ago
commit ae80d42cb4

@ -4,6 +4,10 @@ Please drag-drop large logs as text file attachments.
Feel free to write an issue in your preferred format, however if in doubt, use
the following checklist as a guide for what to include.
* Which version of Ansible are you running?
* Is your version of Ansible patched in any way?
* Are you running with any custom modules, or `module_utils` loaded?
* Have you tried the latest master version from Git?
* Do you have some idea of what the underlying problem may be?
https://mitogen.rtfd.io/en/stable/ansible.html#common-problems has

@ -1,4 +1,4 @@
Copyright 2017, David Wilson
Copyright 2019, David Wilson
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

@ -0,0 +1 @@
include LICENSE

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -156,11 +156,11 @@ class Policy(object):
Assign the helper subprocess policy to this process.
"""
class LinuxPolicy(Policy):
class FixedPolicy(Policy):
"""
:class:`Policy` for Linux machines. The scheme here was tested on an
otherwise idle 16 thread machine.
:class:`Policy` for machines where the only control method available is
fixed CPU placement. The scheme here was tested on an otherwise idle 16
thread machine.
- The connection multiplexer is pinned to CPU 0.
- The Ansible top-level (strategy) is pinned to CPU 1.
@ -180,26 +180,35 @@ class LinuxPolicy(Policy):
CPU-intensive children like SSH are not forced to share the same core as
the (otherwise potentially very busy) parent.
"""
def __init__(self):
def __init__(self, cpu_count=None):
#: For tests.
self.cpu_count = cpu_count or multiprocessing.cpu_count()
self.mem = mmap.mmap(-1, 4096)
self.state = State.from_buffer(self.mem)
self.state.lock.init()
if self._cpu_count() < 4:
self._reserve_mask = 3
self._reserve_shift = 2
self._reserve_controller = True
else:
if self.cpu_count < 2:
# uniprocessor
self._reserve_mux = False
self._reserve_controller = False
self._reserve_mask = 0
self._reserve_shift = 0
elif self.cpu_count < 4:
# small SMP
self._reserve_mux = True
self._reserve_controller = False
self._reserve_mask = 1
self._reserve_shift = 1
self._reserve_controller = False
else:
# big SMP
self._reserve_mux = True
self._reserve_controller = True
self._reserve_mask = 3
self._reserve_shift = 2
def _set_affinity(self, mask):
mitogen.parent._preexec_hook = self._clear
s = struct.pack('L', mask)
_sched_setaffinity(os.getpid(), len(s), s)
def _cpu_count(self):
return multiprocessing.cpu_count()
self._set_cpu_mask(mask)
def _balance(self):
self.state.lock.acquire()
@ -210,14 +219,15 @@ class LinuxPolicy(Policy):
self.state.lock.release()
self._set_cpu(self._reserve_shift + (
(n % max(1, (self._cpu_count() - self._reserve_shift)))
(n % (self.cpu_count - self._reserve_shift))
))
def _set_cpu(self, cpu):
self._set_affinity(1 << cpu)
def _clear(self):
self._set_affinity(0xffffffff & ~self._reserve_mask)
all_cpus = (1 << self.cpu_count) - 1
self._set_affinity(all_cpus & ~self._reserve_mask)
def assign_controller(self):
if self._reserve_controller:
@ -235,6 +245,12 @@ class LinuxPolicy(Policy):
self._clear()
class LinuxPolicy(FixedPolicy):
def _set_cpu_mask(self, mask):
s = struct.pack('L', mask)
_sched_setaffinity(os.getpid(), len(s), s)
if _sched_setaffinity is not None:
policy = LinuxPolicy()
else:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -547,7 +547,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
def connected(self):
return self.context is not None
def _spec_from_via(self, via_spec):
def _spec_from_via(self, proxied_inventory_name, via_spec):
"""
Produce a dict connection specifiction given a string `via_spec`, of
the form `[[become_method:]become_user@]inventory_hostname`.
@ -555,17 +555,20 @@ class Connection(ansible.plugins.connection.ConnectionBase):
become_user, _, inventory_name = via_spec.rpartition('@')
become_method, _, become_user = become_user.rpartition(':')
via_vars = self.host_vars[inventory_name]
if isinstance(via_vars, jinja2.runtime.Undefined):
# must use __contains__ to avoid a TypeError for a missing host on
# Ansible 2.3.
if self.host_vars is None or inventory_name not in self.host_vars:
raise ansible.errors.AnsibleConnectionFailure(
self.unknown_via_msg % (
via_spec,
inventory_name,
proxied_inventory_name,
)
)
via_vars = self.host_vars[inventory_name]
return ansible_mitogen.transport_config.MitogenViaSpec(
inventory_name=inventory_name,
play_context=self._play_context,
host_vars=dict(via_vars), # TODO: make it lazy
become_method=become_method or None,
become_user=become_user or None,
@ -615,7 +618,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
if spec.mitogen_via():
stack = self._stack_from_spec(
self._spec_from_via(spec.mitogen_via()),
self._spec_from_via(spec.inventory_name(), spec.mitogen_via()),
stack=stack,
seen_names=seen_names + (spec.inventory_name(),),
)

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -54,7 +54,8 @@ class Handler(logging.Handler):
#: may simply be to bury all target logs in DEBUG output, but not by
#: overriding their log level as done here.
NOISY_LOGGERS = frozenset([
'dnf', # issue #272; warns when a package is already installed.
'dnf', # issue #272; warns when a package is already installed.
'boto', # issue #541; normal boto retry logic can cause ERROR logs.
])
def emit(self, record):
@ -75,25 +76,28 @@ class Handler(logging.Handler):
def setup():
"""
Install a handler for Mitogen's logger to redirect it into the Ansible
display framework, and prevent propagation to the root logger.
Install handlers for Mitogen loggers to redirect them into the Ansible
display framework. Ansible installs its own logging framework handlers when
C.DEFAULT_LOG_PATH is set, therefore disable propagation for our handlers.
"""
logging.getLogger('ansible_mitogen').handlers = [Handler(display.vvv)]
mitogen.core.LOG.handlers = [Handler(display.vvv)]
mitogen.core.IOLOG.handlers = [Handler(display.vvvv)]
mitogen.core.IOLOG.propagate = False
l_mitogen = logging.getLogger('mitogen')
l_mitogen_io = logging.getLogger('mitogen.io')
l_ansible_mitogen = logging.getLogger('ansible_mitogen')
for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen:
logger.handlers = [Handler(display.vvv)]
logger.propagate = False
if display.verbosity > 2:
mitogen.core.LOG.setLevel(logging.DEBUG)
logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG)
l_ansible_mitogen.setLevel(logging.DEBUG)
l_mitogen.setLevel(logging.DEBUG)
else:
# Mitogen copies the active log level into new children, allowing them
# to filter tiny messages before they hit the network, and therefore
# before they wake the IO loop. Explicitly setting INFO saves ~4%
# running against just the local machine.
mitogen.core.LOG.setLevel(logging.ERROR)
logging.getLogger('ansible_mitogen').setLevel(logging.ERROR)
l_mitogen.setLevel(logging.ERROR)
l_ansible_mitogen.setLevel(logging.ERROR)
if display.verbosity > 3:
mitogen.core.IOLOG.setLevel(logging.DEBUG)
logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG)
l_mitogen_io.setLevel(logging.DEBUG)

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -236,6 +236,41 @@ class MuxProcess(object):
if secs:
mitogen.debug.dump_to_logger(secs=secs)
def _setup_simplejson(self, responder):
"""
We support serving simplejson for Python 2.4 targets on Ansible 2.3, at
least so the package's own CI Docker scripts can run without external
help, however newer versions of simplejson no longer support Python
2.4. Therefore override any installed/loaded version with a
2.4-compatible version we ship in the compat/ directory.
"""
responder.whitelist_prefix('simplejson')
# issue #536: must be at end of sys.path, in case existing newer
# version is already loaded.
compat_path = os.path.join(os.path.dirname(__file__), 'compat')
sys.path.append(compat_path)
for fullname, is_pkg, suffix in (
(u'simplejson', True, '__init__.py'),
(u'simplejson.decoder', False, 'decoder.py'),
(u'simplejson.encoder', False, 'encoder.py'),
(u'simplejson.scanner', False, 'scanner.py'),
):
path = os.path.join(compat_path, 'simplejson', suffix)
fp = open(path, 'rb')
try:
source = fp.read()
finally:
fp.close()
responder.add_source_override(
fullname=fullname,
path=path,
source=source,
is_pkg=is_pkg,
)
def _setup_responder(self, responder):
"""
Configure :class:`mitogen.master.ModuleResponder` to only permit
@ -243,9 +278,7 @@ class MuxProcess(object):
"""
responder.whitelist_prefix('ansible')
responder.whitelist_prefix('ansible_mitogen')
responder.whitelist_prefix('simplejson')
simplejson_path = os.path.join(os.path.dirname(__file__), 'compat')
sys.path.insert(0, simplejson_path)
self._setup_simplejson(responder)
# Ansible 2.3 is compatible with Python 2.4 targets, however
# ansible/__init__.py is not. Instead, executor/module_common.py writes

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -377,6 +377,11 @@ def init_child(econtext, log_level, candidate_temp_dirs):
LOG.setLevel(log_level)
logging.getLogger('ansible_mitogen').setLevel(log_level)
# issue #536: if the json module is available, remove simplejson from the
# importer whitelist to avoid confusing certain Ansible modules.
if json.__name__ == 'json':
econtext.importer.whitelist.remove('simplejson')
global _fork_parent
if FORK_SUPPORTED:
mitogen.parent.upgrade_router(econtext)
@ -497,7 +502,7 @@ class AsyncRunner(object):
)
result = json.loads(filtered)
result.setdefault('warnings', []).extend(warnings)
result['stderr'] = dct['stderr']
result['stderr'] = dct['stderr'] or result.get('stderr', '')
self._update(result)
def _run(self):

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -329,9 +329,11 @@ class PlayContextSpec(Spec):
return self._play_context.port
def python_path(self):
return parse_python_path(
self._connection.get_task_var('ansible_python_interpreter')
)
s = self._connection.get_task_var('ansible_python_interpreter')
# #511, #536: executor/module_common.py::_get_shebang() hard-wires
# "/usr/bin/python" as the default interpreter path if no other
# interpreter is specified.
return parse_python_path(s or '/usr/bin/python')
def private_key_file(self):
return self._play_context.private_key_file
@ -428,12 +430,33 @@ class MitogenViaSpec(Spec):
having a configruation problem with connection delegation, the answer to
your problem lies in the method implementations below!
"""
def __init__(self, inventory_name, host_vars,
become_method, become_user):
def __init__(self, inventory_name, host_vars, become_method, become_user,
play_context):
"""
:param str inventory_name:
The inventory name of the intermediary machine, i.e. not the target
machine.
:param dict host_vars:
The HostVars magic dictionary provided by Ansible in task_vars.
:param str become_method:
If the mitogen_via= spec included a become method, the method it
specifies.
:param str become_user:
If the mitogen_via= spec included a become user, the user it
specifies.
:param PlayContext play_context:
For some global values **only**, the PlayContext used to describe
the real target machine. Values from this object are **strictly
restricted** to values that are Ansible-global, e.g. the passwords
specified interactively.
"""
self._inventory_name = inventory_name
self._host_vars = host_vars
self._become_method = become_method
self._become_user = become_user
# Dangerous! You may find a variable you want in this object, but it's
# almost certainly for the wrong machine!
self._dangerous_play_context = play_context
def transport(self):
return (
@ -445,15 +468,17 @@ class MitogenViaSpec(Spec):
return self._inventory_name
def remote_addr(self):
# play_context.py::MAGIC_VARIABLE_MAPPING
return (
self._host_vars.get('ansible_ssh_host') or
self._host_vars.get('ansible_host') or
self._inventory_name
)
def remote_user(self):
return (
self._host_vars.get('ansible_user') or
self._host_vars.get('ansible_ssh_user') or
self._host_vars.get('ansible_user') or
C.DEFAULT_REMOTE_USER
)
@ -461,37 +486,40 @@ class MitogenViaSpec(Spec):
return bool(self._become_user)
def become_method(self):
return self._become_method or C.DEFAULT_BECOME_METHOD
return (
self._become_method or
self._host_vars.get('ansible_become_method') or
C.DEFAULT_BECOME_METHOD
)
def become_user(self):
return self._become_user
def become_pass(self):
return optional_secret(
# TODO: Might have to come from PlayContext.
self._host_vars.get('ansible_become_password') or
self._host_vars.get('ansible_become_pass')
)
def password(self):
return optional_secret(
# TODO: Might have to come from PlayContext.
self._host_vars.get('ansible_ssh_pass') or
self._host_vars.get('ansible_password')
)
def port(self):
return (
self._host_vars.get('ansible_ssh_port') or
self._host_vars.get('ansible_port') or
C.DEFAULT_REMOTE_PORT
)
def python_path(self):
return parse_python_path(
self._host_vars.get('ansible_python_interpreter')
# This variable has no default for remote hosts. For local hosts it
# is sys.executable.
)
s = self._host_vars.get('ansible_python_interpreter')
# #511, #536: executor/module_common.py::_get_shebang() hard-wires
# "/usr/bin/python" as the default interpreter path if no other
# interpreter is specified.
return parse_python_path(s or '/usr/bin/python')
def private_key_file(self):
# TODO: must come from PlayContext too.

@ -1,4 +1,70 @@
body {
font-size: 100%;
}
.sphinxsidebarwrapper {
padding-top: 0 !important;
}
.sphinxsidebar {
font-size: 80% !important;
}
.sphinxsidebar h3 {
font-size: 130% !important;
}
img + p,
h1 + p,
h2 + p,
h3 + p,
h4 + p,
h5 + p
{
margin-top: 0;
}
.section > h3:first-child {
margin-top: 15px !important;
}
.body h1 { font-size: 200% !important; }
.body h2 { font-size: 165% !important; }
.body h3 { font-size: 125% !important; }
.body h4 { font-size: 110% !important; font-weight: bold; }
.body h5 { font-size: 100% !important; font-weight: bold; }
.body h1,
.body h2,
.body h3,
.body h4,
.body h5 {
margin-top: 30px !important;
color: #7f0000;
}
.body h1 {
margin-top: 0 !important;
}
body,
.sphinxsidebar,
.sphinxsidebar h1,
.sphinxsidebar h2,
.sphinxsidebar h3,
.sphinxsidebar h4,
.sphinxsidebar h5,
.body h1,
.body h2,
.body h3,
.body h4,
.body h5 {
/*font-family: sans-serif !important;*/
font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol !important;
}
.document {
width: 1000px !important;
}
@ -38,10 +104,10 @@ div.body p, div.body dd, div.body li, div.body blockquote {
width: 150px;
}
.mitogen-right-200 {
.mitogen-right-180 {
float: right;
padding-left: 8px;
width: 200px;
width: 180px;
}
.mitogen-right-225 {

@ -1,8 +1,4 @@
<p>
<br>
<a class="github-button" href="https://github.com/dw/mitogen" data-size="large" data-show-count="true" aria-label="Star dw/mitogen on GitHub">Star</a>
</p>
<p>
<a href="https://github.com/dw/mitogen/">GitHub Repository</a>
<a class="github-button" href="https://github.com/dw/mitogen/" data-size="large" data-show-count="true" aria-label="Star dw/mitogen on GitHub">Star</a>
</p>

@ -0,0 +1 @@
{{ toctree() }}

@ -3,7 +3,7 @@ Mitogen for Ansible
===================
.. image:: images/ansible/ansible_mitogen.svg
:class: mitogen-right-200 mitogen-logo-wrap
:class: mitogen-right-180 mitogen-logo-wrap
An extension to `Ansible`_ is included that implements connections over
Mitogen, replacing embedded shell invocations with pure-Python equivalents
@ -246,6 +246,11 @@ container.
as duplicate connections between hops, due to not perfectly replicating
the configuration Ansible would normally use for the intermediary.
* Intermediary machines cannot use login and become passwords that were
supplied to Ansible interactively. If an intermediary requires a
password, it must be supplied via ``ansible_ssh_pass``,
``ansible_password``, or ``ansible_become_pass`` inventory variables.
* Automatic tunnelling of SSH-dependent actions, such as the
``synchronize`` module, is not yet supported. This will be added in the
0.3 series.

@ -10,7 +10,7 @@ Release Notes
<style>
div#release-notes h2 {
border-bottom: 1px dotted #c0c0c0;
margin-top: 40px;
margin-top: 50px !important;
}
</style>
@ -125,6 +125,67 @@ Core Library
series.
v0.2.5 (2019-02-14)
-------------------
Fixes
~~~~~
* `#511 <https://github.com/dw/mitogen/issues/511>`_,
`#536 <https://github.com/dw/mitogen/issues/536>`_: changes in 0.2.4 to
repair ``delegate_to`` handling broke default ``ansible_python_interpreter``
handling. Test coverage was added.
* `#532 <https://github.com/dw/mitogen/issues/532>`_: fix a race in the service
used to propagate Ansible modules, that could easily manifest when starting
asynchronous tasks in a loop.
* `#536 <https://github.com/dw/mitogen/issues/536>`_: changes in 0.2.4 to
support Python 2.4 interacted poorly with modules that imported
``simplejson`` from a controller that also loaded an incompatible newer
version of ``simplejson``.
* `#537 <https://github.com/dw/mitogen/issues/537>`_: a swapped operator in the
CPU affinity logic meant 2 cores were reserved on 1<n<4 core machines, rather
than 1 core as desired. Test coverage was added.
* `#538 <https://github.com/dw/mitogen/issues/538>`_: the source distribution
includes a ``LICENSE`` file.
* `#539 <https://github.com/dw/mitogen/issues/539>`_: log output is no longer
duplicated when the Ansible ``log_path`` setting is enabled.
* `#540 <https://github.com/dw/mitogen/issues/540>`_: the ``stderr`` stream of
async module invocations was previously discarded.
* `#541 <https://github.com/dw/mitogen/issues/541>`_: Python error logs
originating from the ``boto`` package are quiesced, and only appear in
``-vvv`` output. This is since EC2 modules may trigger errors during normal
operation, when retrying transiently failing requests.
* `748f5f67 <https://github.com/dw/mitogen/commit/748f5f67>`_,
`21ad299d <https://github.com/dw/mitogen/commit/21ad299d>`_,
`8ae6ca1d <https://github.com/dw/mitogen/commit/8ae6ca1d>`_,
`7fd0d349 <https://github.com/dw/mitogen/commit/7fd0d349>`_:
the ``ansible_ssh_host``, ``ansible_ssh_user``, ``ansible_user``,
``ansible_become_method``, and ``ansible_ssh_port`` variables more correctly
match typical behaviour when ``mitogen_via=`` is active.
* `2a8567b4 <https://github.com/dw/mitogen/commit/2a8567b4>`_: fix a race
initializing a child's service thread pool on Python 3.4+, due to a change in
locking scheme used by the Python import mechanism.
Thanks!
~~~~~~~
Mitogen would not be possible without the support of users. A huge thanks for
bug reports, testing, features and fixes in this release contributed by
`Carl George <https://github.com/carlwgeorge>`_,
`Guy Knights <https://github.com/knightsg>`_, and
`Josh Smift <https://github.com/jbscare>`_.
v0.2.4 (2019-02-10)
-------------------

@ -17,6 +17,10 @@ html_theme = 'alabaster'
html_theme_options = {
'font_family': "Georgia, serif",
'head_font_family': "Georgia, serif",
'fixed_sidebar': True,
'show_powered_by': False,
'pink_2': 'fffafaf',
'pink_1': '#fff0f0',
}
htmlhelp_basename = 'mitogendoc'
intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}

@ -11,7 +11,7 @@ Mitogen
</style>
.. image:: images/mitogen.svg
:class: mitogen-right-200 mitogen-logo-wrap
:class: mitogen-right-180 mitogen-logo-wrap
Mitogen is a Python library for writing distributed self-replicating programs.

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
#: Library version as a tuple.
__version__ = (0, 2, 4)
__version__ = (0, 2, 5)
#: This is :data:`False` in slave contexts. Previously it was used to prevent

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -1125,6 +1125,11 @@ class Importer(object):
self.whitelist = list(whitelist) or ['']
self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST
# Preserve copies of the original server-supplied whitelist/blacklist
# for later use by children.
self.master_whitelist = self.whitelist[:]
self.master_blacklist = self.blacklist[:]
# Presence of an entry in this map indicates in-flight GET_MODULE.
self._callbacks = {}
self._cache = {}
@ -3131,10 +3136,21 @@ class ExternalContext(object):
if not self.config['profiling']:
os.kill(os.getpid(), signal.SIGTERM)
#: On Python >3.4, the global importer lock has been sharded into a
#: per-module lock, meaning there is no guarantee the import statement in
#: service_stub_main will be truly complete before a second thread
#: attempting the same import will see a partially initialized module.
#: Sigh. Therefore serialize execution of the stub itself.
service_stub_lock = threading.Lock()
def _service_stub_main(self, msg):
import mitogen.service
pool = mitogen.service.get_or_create_pool(router=self.router)
pool._receiver._on_receive(msg)
self.service_stub_lock.acquire()
try:
import mitogen.service
pool = mitogen.service.get_or_create_pool(router=self.router)
pool._receiver._on_receive(msg)
finally:
self.service_stub_lock.release()
def _on_call_service_msg(self, msg):
"""

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -98,6 +98,7 @@ def on_fork():
fixup_prngs()
mitogen.core.Latch._on_fork()
mitogen.core.Side._on_fork()
mitogen.core.ExternalContext.service_stub_lock = threading.Lock()
mitogen__service = sys.modules.get('mitogen.service')
if mitogen__service:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -485,8 +485,10 @@ class ModuleFinder(object):
return path, source, is_pkg
def _get_module_via_sys_modules(self, fullname):
"""Attempt to fetch source code via sys.modules. This is specifically
to support __main__, but it may catch a few more cases."""
"""
Attempt to fetch source code via sys.modules. This is specifically to
support __main__, but it may catch a few more cases.
"""
module = sys.modules.get(fullname)
LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module)
if not isinstance(module, types.ModuleType):
@ -883,10 +885,13 @@ class ModuleResponder(object):
if msg.is_dead:
return
LOG.debug('%r._on_get_module(%r)', self, msg.data)
self.get_module_count += 1
stream = self._router.stream_by_id(msg.src_id)
if stream is None:
return
fullname = msg.data.decode()
LOG.debug('%s requested module %s', stream.name, fullname)
self.get_module_count += 1
if fullname in stream.sent_modules:
LOG.warning('_on_get_module(): dup request for %r from %r',
fullname, stream)

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -2054,12 +2054,12 @@ class Router(mitogen.core.Router):
def get_module_blacklist(self):
if mitogen.context_id == 0:
return self.responder.blacklist
return self.importer.blacklist
return self.importer.master_blacklist
def get_module_whitelist(self):
if mitogen.context_id == 0:
return self.responder.whitelist
return self.importer.whitelist
return self.importer.master_whitelist
def allocate_id(self):
return self.id_allocator.allocate()

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
@ -603,8 +603,6 @@ class PushFileService(Service):
This service will eventually be merged into FileService.
"""
invoker_class = SerializedInvoker
def __init__(self, **kwargs):
super(PushFileService, self).__init__(**kwargs)
self._lock = threading.Lock()
@ -613,13 +611,16 @@ class PushFileService(Service):
self._sent_by_stream = {}
def get(self, path):
"""
Fetch a file from the cache.
"""
assert isinstance(path, mitogen.core.UnicodeType)
self._lock.acquire()
try:
if path in self._cache:
return self._cache[path]
waiters = self._waiters.setdefault(path, [])
latch = mitogen.core.Latch()
waiters = self._waiters.setdefault(path, [])
waiters.append(lambda: latch.put(None))
finally:
self._lock.release()
@ -633,14 +634,15 @@ class PushFileService(Service):
stream = self.router.stream_by_id(context.context_id)
child = mitogen.core.Context(self.router, stream.remote_id)
sent = self._sent_by_stream.setdefault(stream, set())
if path in sent and child.context_id != context.context_id:
child.call_service_async(
service_name=self.name(),
method_name='forward',
path=path,
context=context
).close()
elif path not in sent:
if path in sent:
if child.context_id != context.context_id:
child.call_service_async(
service_name=self.name(),
method_name='forward',
path=path,
context=context
).close()
else:
child.call_service_async(
service_name=self.name(),
method_name='store_and_forward',
@ -680,14 +682,6 @@ class PushFileService(Service):
fp.close()
self._forward(context, path)
def _store(self, path, data):
self._lock.acquire()
try:
self._cache[path] = data
return self._waiters.pop(path, [])
finally:
self._lock.release()
@expose(policy=AllowParents())
@no_reply()
@arg_spec({
@ -696,9 +690,16 @@ class PushFileService(Service):
'context': mitogen.core.Context,
})
def store_and_forward(self, path, data, context):
LOG.debug('%r.store_and_forward(%r, %r, %r)',
self, path, data, context)
waiters = self._store(path, data)
LOG.debug('%r.store_and_forward(%r, %r, %r) %r',
self, path, data, context,
threading.currentThread().getName())
self._lock.acquire()
try:
self._cache[path] = data
waiters = self._waiters.pop(path, [])
finally:
self._lock.release()
if context.context_id != mitogen.context_id:
self._forward(context, path)
for callback in waiters:
@ -712,10 +713,17 @@ class PushFileService(Service):
})
def forward(self, path, context):
LOG.debug('%r.forward(%r, %r)', self, path, context)
if path not in self._cache:
LOG.error('%r: %r is not in local cache', self, path)
return
self._forward(context, path)
func = lambda: self._forward(context, path)
self._lock.acquire()
try:
if path in self._cache:
func()
else:
LOG.debug('%r: %r not cached yet, queueing', self, path)
self._waiters.setdefault(path, []).append(func)
finally:
self._lock.release()
class FileService(Service):

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,4 +1,4 @@
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,5 +1,5 @@
#!/usr/bin/env python2
# Copyright 2017, David Wilson
# Copyright 2019, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:

@ -1,5 +1,5 @@
[defaults]
inventory = hosts,lib/inventory
inventory = hosts
gathering = explicit
strategy_plugins = ../../ansible_mitogen/plugins/strategy
action_plugins = lib/action

@ -1,8 +1,9 @@
# vim: syntax=dosini
# When running the tests outside CI, make a single 'target' host which is the
# local machine.
target ansible_host=localhost
# local machine. The ansible_user override is necessary since some tests want a
# fixed ansible.cfg remote_user setting to test against.
target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}"
[test-targets]
target

@ -1,9 +1,8 @@
# vim: syntax=dosini
# This must be defined explicitly, otherwise _create_implicit_localhost()
# generates its own copy, which includes an ansible_python_interpreter that
# varies according to host machine.
localhost
# issue #511, #536: we must not define an explicit localhost, as some
# transport_config/python_path.yml needs to test the implicit localhost
# behaviour.
# This is only used for manual testing.
[localhost-x10]

@ -0,0 +1,49 @@
# integration/transport_config
# Hosts with twiddled configs that need to be checked somehow.
# tansport()
tc-transport-unset
tc-transport-local ansible_connection=local
# python_path()
tc-python-path-unset
tc-python-path-hostvar ansible_python_interpreter=/hostvar/path/to/python
tc-python-path-local-unset ansible_connection=local
tc-python-path-local-explicit ansible_connection=local ansible_python_interpreter=/a/b/c
# remote_addr()
tc-remote-addr-unset # defaults to inventory_hostname
tc-remote-addr-explicit-ssh ansible_ssh_host=ansi.ssh.host
tc-remote-addr-explicit-host ansible_host=ansi.host
tc-remote-addr-explicit-both ansible_ssh_host=a.b.c ansible_host=b.c.d
# password()
tc-password-unset
tc-password-explicit-ssh ansible_ssh_pass=ansi-ssh-pass
tc-password-explicit-user ansible_password=ansi-pass
tc-password-explicit-both ansible_password=a.b.c ansible_ssh_pass=c.b.a
# become()
tc-become-unset
tc-become-set
# become_method()
tc-become-method-unset
tc-become-method-su ansible_become_method=su
# become_user()
tc-become-user-unset
tc-become-user-set ansible_become_user=ansi-become-user
# become_pass()
tc-become-pass-unset
tc-become-pass-password ansible_become_password=apassword
tc-become-pass-pass ansible_become_pass=apass
tc-become-pass-both ansible_become_password=a.b.c ansible_become_pass=c.b.a
# port()
tc-port-unset
tc-port-explicit-port ansible_port=1234
tc-port-explicit-ssh ansible_ssh_port=4321
tc-port-both ansible_port=1717 ansible_ssh_port=1532

@ -0,0 +1,4 @@
# Include me for plays that can't run on vanilla.
#
- meta: end_play
when: not is_mitogen

@ -19,3 +19,4 @@
- include: ssh/all.yml
- include: strategy/all.yml
- include: stub_connections/all.yml
- include: transport_config/all.yml

@ -5,7 +5,7 @@
any_errors_fatal: true
tasks:
- shell: echo hi
- shell: echo hi; echo there >&2
async: 100
poll: 0
register: job
@ -21,10 +21,10 @@
- assert:
that:
- async_out.changed == True
- async_out.cmd == "echo hi"
- async_out.cmd == "echo hi; echo there >&2"
- 'async_out.delta.startswith("0:00:")'
- async_out.end.startswith("20")
- async_out.invocation.module_args._raw_params == "echo hi"
- async_out.invocation.module_args._raw_params == "echo hi; echo there >&2"
- async_out.invocation.module_args._uses_shell == True
- async_out.invocation.module_args.chdir == None
- async_out.invocation.module_args.creates == None
@ -33,7 +33,7 @@
- async_out.invocation.module_args.warn == True
- async_out.rc == 0
- async_out.start.startswith("20")
- async_out.stderr == ""
- async_out.stderr == "there"
- async_out.stdout == "hi"
vars:
async_out: "{{result.content|b64decode|from_json}}"

@ -39,7 +39,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
'python_path': ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -66,7 +66,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
'python_path': ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',

@ -15,7 +15,7 @@
right: [
{
'kwargs': {
'python_path': null
'python_path': ["{{ansible_playbook_python}}"],
},
'method': 'local',
},
@ -23,7 +23,7 @@
'enable_lru': true,
'kwargs': {
'connect_timeout': 10,
'python_path': null,
'python_path': ["{{ansible_playbook_python}}"],
'password': null,
'username': 'root',
'sudo_path': null,

@ -24,7 +24,7 @@
'lxc_info_path': null,
'lxc_path': null,
'machinectl_path': null,
'python_path': null,
'python_path': ["/usr/bin/python"],
'username': 'ansible-cfg-remote-user',
},
'method': 'setns',

@ -43,7 +43,7 @@
"connect_timeout": 10,
"doas_path": null,
"password": null,
"python_path": null,
"python_path": ["/usr/bin/python"],
"username": "normal-user",
},
"method": "doas",
@ -72,7 +72,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -112,7 +112,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -147,7 +147,7 @@
'connect_timeout': 10,
'doas_path': null,
'password': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'username': 'normal-user',
},
'method': 'doas',
@ -162,7 +162,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -202,7 +202,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -229,7 +229,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -264,7 +264,7 @@
'connect_timeout': 10,
'doas_path': null,
'password': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'username': 'normal-user',
},
'method': 'doas',
@ -279,7 +279,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -320,7 +320,7 @@
'identity_file': null,
'password': null,
'port': null,
'python_path': null,
"python_path": ["/usr/bin/python"],
'ssh_args': [
'-o',
'UserKnownHostsFile=/dev/null',
@ -352,7 +352,7 @@
right: [
{
'kwargs': {
'python_path': null
"python_path": ["{{ansible_playbook_python}}"],
},
'method': 'local',
},
@ -374,7 +374,7 @@
'connect_timeout': 10,
'doas_path': null,
'password': null,
'python_path': null,
'python_path': ["/usr/bin/python"],
'username': 'normal-user',
},
'method': 'doas',
@ -384,7 +384,7 @@
'connect_timeout': 10,
'doas_path': null,
'password': null,
'python_path': null,
'python_path': ["/usr/bin/python"],
'username': 'newuser-doas-normal-user',
},
'method': 'doas',

@ -0,0 +1,7 @@
# Tests for correct selection of connection variables.
This directory is a placeholder for a work-in-progress test set that tries
every combination of the variables extracted via `transport_config.py`.
In the meantime, it has ad-hoc scripts for bugs already encountered.

@ -0,0 +1,10 @@
- include: become_method.yml
- include: become_pass.yml
- include: become_user.yml
- include: become.yml
- include: password.yml
- include: port.yml
- include: python_path.yml
- include: remote_addr.yml
- include: remote_user.yml
- include: transport.yml

@ -0,0 +1,68 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# No become set.
- name: integration/transport_config/become.yml
hosts: tc-become-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 1
- out.result[0].method == "ssh"
- out.result[0].kwargs.username == "ansible-cfg-remote-user"
- hosts: tc-become-unset
vars: {mitogen_via: becomeuser@tc-become-set}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[0].kwargs.username == "ansible-cfg-remote-user"
- out.result[1].method == "sudo"
- out.result[1].kwargs.username == "becomeuser"
- out.result[2].method == "ssh"
- out.result[2].kwargs.hostname == "tc-become-unset"
# Become set.
- name: integration/transport_config/become.yml
hosts: tc-become-set
become: true
become_user: becomeuser
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[0].kwargs.username == "ansible-cfg-remote-user"
- out.result[1].method == "sudo"
- out.result[1].kwargs.username == "becomeuser"
- hosts: tc-become-set
vars: {mitogen_via: tc-become-unset}
become: true
become_user: becomeuser
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[0].kwargs.hostname == "tc-become-unset"
- out.result[0].kwargs.username == "ansible-cfg-remote-user"
- out.result[1].method == "ssh"
- out.result[1].kwargs.hostname == "tc-become-set"
- out.result[2].method == "sudo"
- out.result[2].kwargs.username == "becomeuser"

@ -0,0 +1,83 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# No become-method set.
- name: integration/transport_config/become-method.yml
hosts: tc-become-method-unset
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- hosts: tc-become-method-unset
vars: {mitogen_via: becomeuser@tc-become-method-su}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[1].method == "su"
- out.result[1].kwargs.username == "becomeuser"
- out.result[2].method == "ssh"
- out.result[2].kwargs.hostname == "tc-become-method-unset"
# ansible_become_method=su
- hosts: tc-become-method-su
become: true
become_user: becomeuser
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "su"
- out.result[1].kwargs.username == "becomeuser"
- hosts: tc-become-method-su
vars: {mitogen_via: tc-become-method-unset}
become: true
become_user: becomeuser
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[0].kwargs.hostname == "tc-become-method-unset"
- out.result[1].method == "ssh"
- out.result[1].kwargs.hostname == "tc-become-method-su"
- out.result[2].method == "su"
- out.result[2].kwargs.username == "becomeuser"
# mitogen_via used to specify explicit become method
- hosts: tc-become-method-unset
vars: {mitogen_via: "doas:doasuser@tc-become-method-su"}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[0].kwargs.hostname == "tc-become-method-su"
- out.result[1].method == "doas"
- out.result[1].kwargs.username == "doasuser"
- out.result[2].method == "ssh"
- out.result[2].kwargs.hostname == "tc-become-method-unset"

@ -0,0 +1,142 @@
# Each case is followed by mitogen_via= case to test hostvars pass.
# No become-pass set, defaults to "root"
- name: integration/transport_config/become-pass.yml
hosts: tc-become-pass-unset
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == None
# Not set, unbecoming mitogen_via=
- hosts: tc-become-pass-unset
become: true
vars: {mitogen_via: tc-become-pass-password}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[1].method == "ssh"
- out.result[2].method == "sudo"
- out.result[2].kwargs.password == None
# Not set, becoming mitogen_via=
- hosts: tc-become-pass-unset
become: true
vars: {mitogen_via: viapass@tc-become-pass-password}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 4
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "apassword"
- out.result[2].method == "ssh"
- out.result[3].method == "sudo"
- out.result[3].kwargs.password == None
# ansible_become_password= set.
- hosts: tc-become-pass-password
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "apassword"
# ansible_become_password=, via=
- hosts: tc-become-pass-password
vars: {mitogen_via: root@tc-become-pass-pass}
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 4
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "apass"
- out.result[2].method == "ssh"
- out.result[3].method == "sudo"
- out.result[3].kwargs.password == "apassword"
# ansible_become_pass=
- hosts: tc-become-pass-pass
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "apass"
# ansible_become_pass=, via=
- hosts: tc-become-pass-pass
vars: {mitogen_via: root@tc-become-pass-password}
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 4
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "apassword"
- out.result[2].method == "ssh"
- out.result[3].method == "sudo"
- out.result[3].kwargs.password == "apass"
# ansible_become_pass & ansible_become_password set, password takes precedence
- hosts: tc-become-pass-both
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "a.b.c"
# both, mitogen_via
- hosts: tc-become-pass-unset
vars: {mitogen_via: root@tc-become-pass-both}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "a.b.c"
- out.result[2].method == "ssh"

@ -0,0 +1,106 @@
# Each case is followed by mitogen_via= case to test hostvars user.
# No become-user set, defaults to "root"
- name: integration/transport_config/become-user.yml
hosts: tc-become-user-unset
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.username == "root"
# Not set, unbecoming mitogen_via=
- hosts: tc-become-user-unset
become: true
vars: {mitogen_via: tc-become-user-set}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[1].method == "ssh"
- out.result[2].method == "sudo"
- out.result[2].kwargs.username == "root"
# Not set, becoming mitogen_via=
- hosts: tc-become-user-unset
become: true
vars: {mitogen_via: viauser@tc-become-user-set}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 4
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.username == "viauser"
- out.result[2].method == "ssh"
- out.result[3].method == "sudo"
- out.result[3].kwargs.username == "root"
# ansible_become_user= set.
- hosts: tc-become-user-set
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.username == "ansi-become-user"
# ansible_become_user=, unbecoming via=
- hosts: tc-become-user-set
vars: {mitogen_via: tc-become-user-unset}
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 3
- out.result[0].method == "ssh"
- out.result[0].kwargs.hostname == "tc-become-user-unset"
- out.result[1].method == "ssh"
- out.result[1].kwargs.hostname == "tc-become-user-set"
- out.result[2].method == "sudo"
- out.result[2].kwargs.username == "ansi-become-user"
# ansible_become_user=, becoming via=
- hosts: tc-become-user-set
vars: {mitogen_via: "doas:doasuser@tc-become-user-unset"}
become: true
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 4
- out.result[0].method == "ssh"
- out.result[0].kwargs.hostname == "tc-become-user-unset"
- out.result[1].method == "doas"
- out.result[1].kwargs.username == "doasuser"
- out.result[2].method == "ssh"
- out.result[2].kwargs.hostname == "tc-become-user-set"
- out.result[3].method == "sudo"
- out.result[3].kwargs.username == "ansi-become-user"

@ -0,0 +1,94 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# When no ansible_ssh_pass/ansible_password= is set, password comes via
# interactive input.
- name: integration/transport_config/password.yml
hosts: tc-password-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: "" # actually null, but assert_equal limitation
- hosts: tc-password-unset
vars: {mitogen_via: tc-password-explicit-ssh}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: "ansi-ssh-pass"
- assert_equal:
left: out.result[1].kwargs.password
right: ""
# ansible_ssh_user=
- hosts: tc-password-explicit-ssh
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: "ansi-ssh-pass"
- hosts: tc-password-explicit-ssh
vars: {mitogen_via: tc-password-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: ""
- assert_equal:
left: out.result[1].kwargs.password
right: "ansi-ssh-pass"
# ansible_user=
- hosts: tc-password-explicit-pass
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: "ansi-pass"
- hosts: tc-password-explicit-pass
vars: {mitogen_via: tc-password-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: ""
- assert_equal:
left: out.result[1].kwargs.password
right: "ansi-pass"
# both; ansible_ssh_user= takes precedence according to play_context.py.
- hosts: tc-password-explicit-both
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: "c.b.a"
- hosts: tc-password-explicit-both
vars: {mitogen_via: tc-password-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.password
right: ""
- assert_equal:
left: out.result[1].kwargs.password
right: "c.b.a"

@ -0,0 +1,101 @@
# Each case is followed by mitogen_via= case to test hostvars pass.
# No port set
- name: integration/transport_config/port.yml
hosts: tc-port-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 1
- out.result[0].method == "ssh"
- out.result[0].kwargs.port == None
# Not set, mitogen_via=
- hosts: tc-port-explicit-ssh
vars: {mitogen_via: tc-port-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[0].kwargs.port == None
- out.result[1].method == "ssh"
- out.result[1].kwargs.port == 4321
# ansible_ssh_port=
- hosts: tc-port-explicit-ssh
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 1
- out.result[0].method == "ssh"
- out.result[0].kwargs.port == 4321
- hosts: tc-port-explicit-unset
vars: {mitogen_via: tc-port-explicit-ssh}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].kwargs.port == 4321
- out.result[1].method == "ssh"
- out.result[0].kwargs.port == None
# ansible_port=
- hosts: tc-port-explicit-port
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 1
- out.result[0].method == "ssh"
- out.result[0].kwargs.port == 1234
- hosts: tc-port-unset
vars: {mitogen_via: tc-port-explicit-port}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[0].kwargs.port == 1234
- out.result[1].method == "ssh"
- out.result[1].kwargs.port == None
# both, ssh takes precedence
- hosts: tc-port-both
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 1
- out.result[0].method == "ssh"
- out.result[0].kwargs.port == 1532
- hosts: tc-port-unset
vars: {mitogen_via: tc-port-both}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert:
that:
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[0].kwargs.port == 1532
- out.result[1].method == "ssh"
- out.result[1].kwargs.port == None

@ -0,0 +1,114 @@
# related: issue #511, #536
# Each case is followed by mitogen_via= case to test hostvars method.
# When no ansible_python_interpreter is set, executor/module_common.py chooses
# "/usr/bin/python".
- name: integration/transport_config/python_path.yml
hosts: tc-python-path-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["/usr/bin/python"]
- hosts: tc-python-path-hostvar
vars: {mitogen_via: tc-python-path-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["/usr/bin/python"]
- assert_equal:
left: out.result[1].kwargs.python_path
right: ["/hostvar/path/to/python"]
# Non-localhost with explicit ansible_python_interpreter
- hosts: tc-python-path-hostvar
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: [/hostvar/path/to/python]
- hosts: tc-python-path-unset
vars: {mitogen_via: tc-python-path-hostvar}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["/hostvar/path/to/python"]
- assert_equal:
left: out.result[1].kwargs.python_path
right: ["/usr/bin/python"]
# Implicit localhost gets ansible_python_interpreter=virtualenv interpreter
- hosts: localhost
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["{{ansible_playbook_python}}"]
- hosts: tc-python-path-unset
vars: {mitogen_via: localhost}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["{{ansible_playbook_python}}"]
- assert_equal:
left: out.result[1].kwargs.python_path
right: ["/usr/bin/python"]
# explicit local connections get the same treatment as everything else.
- hosts: tc-python-path-local-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["/usr/bin/python"]
- hosts: localhost
vars: {mitogen_via: tc-python-path-local-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["/usr/bin/python"]
- assert_equal:
left: out.result[1].kwargs.python_path
right: ["{{ansible_playbook_python}}"]
# explicit local connection with explicit interpreter
- hosts: tc-python-path-local-explicit
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["/a/b/c"]
- hosts: localhost
vars: {mitogen_via: tc-python-path-local-explicit}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
right: ["/a/b/c"]
- assert_equal:
left: out.result[1].kwargs.python_path
right: ["{{ansible_playbook_python}}"]

@ -0,0 +1,95 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# When no ansible_host/ansible_ssh_host= is set, hostname is same as inventory
# name.
- name: integration/transport_config/remote_addr.yml
hosts: tc-remote-addr-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "tc-remote-addr-unset"
- hosts: tc-remote-addr-unset
vars: {mitogen_via: tc-remote-addr-explicit-ssh}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "ansi.ssh.host"
- assert_equal:
left: out.result[1].kwargs.hostname
right: "tc-remote-addr-unset"
# ansible_ssh_host=
- hosts: tc-remote-addr-explicit-ssh
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "ansi.ssh.host"
- hosts: tc-remote-addr-explicit-ssh
vars: {mitogen_via: tc-remote-addr-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "tc-remote-addr-unset"
- assert_equal:
left: out.result[1].kwargs.hostname
right: "ansi.ssh.host"
# ansible_host=
- hosts: tc-remote-addr-explicit-host
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "ansi.host"
- hosts: tc-remote-addr-explicit-host
vars: {mitogen_via: tc-remote-addr-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "tc-remote-addr-unset"
- assert_equal:
left: out.result[1].kwargs.hostname
right: "ansi.host"
# both; ansible_ssh_host= takes precedence according to play_context.py.
- hosts: tc-remote-addr-explicit-both
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "a.b.c"
- hosts: tc-remote-addr-explicit-both
vars: {mitogen_via: tc-remote-addr-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.hostname
right: "tc-remote-addr-unset"
- assert_equal:
left: out.result[1].kwargs.hostname
right: "a.b.c"

@ -0,0 +1,96 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# When no ansible_user/ansible_ssh_user= is set, username is
# C.DEFAULT_REMOTE_USER.
- name: integration/transport_config/remote_user.yml
hosts: tc-remote-user-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
# We set DEFAULT_REMOTE_USER in our ansible.cfg
right: "ansible-cfg-remote-user"
- hosts: tc-remote-user-unset
vars: {mitogen_via: tc-remote-user-explicit-ssh}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
right: "ansi-ssh-user"
- assert_equal:
left: out.result[1].kwargs.username
right: "ansible-cfg-remote-user"
# ansible_ssh_user=
- hosts: tc-remote-user-explicit-ssh
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
right: "ansi-ssh-user"
- hosts: tc-remote-user-explicit-ssh
vars: {mitogen_via: tc-remote-user-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
right: "ansible-cfg-remote-user"
- assert_equal:
left: out.result[1].kwargs.username
right: "ansi-ssh-user"
# ansible_user=
- hosts: tc-remote-user-explicit-user
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
right: "ansi-user"
- hosts: tc-remote-user-explicit-host
vars: {mitogen_via: tc-remote-user-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
right: "ansible-cfg-remote-user"
- assert_equal:
left: out.result[1].kwargs.username
right: "ansi-user"
# both; ansible_ssh_user= takes precedence according to play_context.py.
- hosts: tc-remote-user-explicit-both
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
right: "c.b.a"
- hosts: tc-remote-user-explicit-both
vars: {mitogen_via: tc-remote-user-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.username
right: "ansible-cfg-remote-user"
- assert_equal:
left: out.result[1].kwargs.username
right: "c.b.a"

@ -0,0 +1,48 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# When no ansible_connection= is set, transport comes via ansible.cfg ("smart"
# is parsed away to either paramiko or ssh).
- name: integration/transport_config/transport.yml
hosts: tc-transport-unset
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].method
right: "ssh"
- hosts: tc-transport-local
vars: {mitogen_via: tc-transport-unset}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].method
right: "ssh"
- assert_equal:
left: out.result[1].method
right: "local"
# ansible_connection=local
- hosts: tc-transport-local
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].method
right: "local"
- hosts: tc-transport-unset
vars: {mitogen_via: tc-transport-local}
tasks:
- include: ../_mitogen_only.yml
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].method
right: "local"
- assert_equal:
left: out.result[1].method
right: "ssh"

@ -46,6 +46,10 @@ if '-i' in sys.argv:
extra['MITOGEN_INVENTORY_FILE'] = (
os.path.abspath(sys.argv[1 + sys.argv.index('-i')])
)
else:
extra['MITOGEN_INVENTORY_FILE'] = (
os.path.join(GIT_BASEDIR, 'tests/ansible/hosts')
)
args = ['ansible-playbook']
args += ['-e', json.dumps(extra)]

@ -11,11 +11,156 @@ import mitogen.parent
import ansible_mitogen.affinity
class NullFixedPolicy(ansible_mitogen.affinity.FixedPolicy):
def _set_cpu_mask(self, mask):
self.mask = mask
class FixedPolicyTest(testlib.TestCase):
klass = NullFixedPolicy
def test_assign_controller_1core(self):
# Uniprocessor .
policy = self.klass(cpu_count=1)
policy.assign_controller()
self.assertEquals(0x1, policy.mask)
def test_assign_controller_2core(self):
# Small SMP gets 1.. % cpu_count
policy = self.klass(cpu_count=2)
policy.assign_controller()
self.assertEquals(0x2, policy.mask)
policy.assign_controller()
self.assertEquals(0x2, policy.mask)
policy.assign_controller()
def test_assign_controller_3core(self):
# Small SMP gets 1.. % cpu_count
policy = self.klass(cpu_count=3)
policy.assign_controller()
self.assertEquals(0x2, policy.mask)
policy.assign_controller()
self.assertEquals(0x4, policy.mask)
policy.assign_controller()
self.assertEquals(0x2, policy.mask)
policy.assign_controller()
self.assertEquals(0x4, policy.mask)
policy.assign_controller()
def test_assign_controller_4core(self):
# Big SMP gets a dedicated core.
policy = self.klass(cpu_count=4)
policy.assign_controller()
self.assertEquals(0x2, policy.mask)
policy.assign_controller()
self.assertEquals(0x2, policy.mask)
def test_assign_muxprocess_1core(self):
# Uniprocessor .
policy = self.klass(cpu_count=1)
policy.assign_muxprocess()
self.assertEquals(0x1, policy.mask)
def test_assign_muxprocess_2core(self):
# Small SMP gets dedicated core.
policy = self.klass(cpu_count=2)
policy.assign_muxprocess()
self.assertEquals(0x1, policy.mask)
policy.assign_muxprocess()
self.assertEquals(0x1, policy.mask)
policy.assign_muxprocess()
def test_assign_muxprocess_3core(self):
# Small SMP gets a dedicated core.
policy = self.klass(cpu_count=3)
policy.assign_muxprocess()
self.assertEquals(0x1, policy.mask)
policy.assign_muxprocess()
self.assertEquals(0x1, policy.mask)
def test_assign_muxprocess_4core(self):
# Big SMP gets a dedicated core.
policy = self.klass(cpu_count=4)
policy.assign_muxprocess()
self.assertEquals(0x1, policy.mask)
policy.assign_muxprocess()
self.assertEquals(0x1, policy.mask)
def test_assign_worker_1core(self):
# Balance n % 1
policy = self.klass(cpu_count=1)
policy.assign_worker()
self.assertEquals(0x1, policy.mask)
policy.assign_worker()
self.assertEquals(0x1, policy.mask)
def test_assign_worker_2core(self):
# Balance n % 1
policy = self.klass(cpu_count=2)
policy.assign_worker()
self.assertEquals(0x2, policy.mask)
policy.assign_worker()
self.assertEquals(0x2, policy.mask)
def test_assign_worker_3core(self):
# Balance n % 1
policy = self.klass(cpu_count=3)
policy.assign_worker()
self.assertEquals(0x2, policy.mask)
policy.assign_worker()
self.assertEquals(0x4, policy.mask)
policy.assign_worker()
self.assertEquals(0x2, policy.mask)
def test_assign_worker_4core(self):
# Balance n % 1
policy = self.klass(cpu_count=4)
policy.assign_worker()
self.assertEquals(4, policy.mask)
policy.assign_worker()
self.assertEquals(8, policy.mask)
policy.assign_worker()
self.assertEquals(4, policy.mask)
def test_assign_subprocess_1core(self):
# allow all except reserved.
policy = self.klass(cpu_count=1)
policy.assign_subprocess()
self.assertEquals(0x1, policy.mask)
policy.assign_subprocess()
self.assertEquals(0x1, policy.mask)
def test_assign_subprocess_2core(self):
# allow all except reserved.
policy = self.klass(cpu_count=2)
policy.assign_subprocess()
self.assertEquals(0x2, policy.mask)
policy.assign_subprocess()
self.assertEquals(0x2, policy.mask)
def test_assign_subprocess_3core(self):
# allow all except reserved.
policy = self.klass(cpu_count=3)
policy.assign_subprocess()
self.assertEquals(0x2 + 0x4, policy.mask)
policy.assign_subprocess()
self.assertEquals(0x2 + 0x4, policy.mask)
def test_assign_subprocess_4core(self):
# allow all except reserved.
policy = self.klass(cpu_count=4)
policy.assign_subprocess()
self.assertEquals(0x4 + 0x8, policy.mask)
policy.assign_subprocess()
self.assertEquals(0x4 + 0x8, policy.mask)
@unittest2.skipIf(
reason='Linux/SMP only',
condition=(not (
os.uname()[0] == 'Linux' and
multiprocessing.cpu_count() >= 4
multiprocessing.cpu_count() > 2
))
)
class LinuxPolicyTest(testlib.TestCase):
@ -33,12 +178,15 @@ class LinuxPolicyTest(testlib.TestCase):
finally:
fp.close()
def test_set_clear(self):
before = self._get_cpus()
self.policy._set_cpu(3)
self.assertEquals(self._get_cpus(), 1 << 3)
self.policy._clear()
self.assertEquals(self._get_cpus(), before)
def test_set_cpu_mask(self):
self.policy._set_cpu_mask(0x1)
self.assertEquals(0x1, self._get_cpus())
self.policy._set_cpu_mask(0x2)
self.assertEquals(0x2, self._get_cpus())
self.policy._set_cpu_mask(0x3)
self.assertEquals(0x3, self._get_cpus())
def test_clear_on_popen(self):
tf = tempfile.NamedTemporaryFile()

@ -8,7 +8,12 @@ import sys
os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv)
os.environ['THIS_IS_STUB_SUDO'] = '1'
# This must be a child process and not exec() since Mitogen replaces its stderr
# descriptor, causing the last user of the slave PTY to close it, resulting in
# the master side indicating EIO.
subprocess.check_call(sys.argv[sys.argv.index('--') + 1:])
if os.environ.get('PREHISTORIC_SUDO'):
# issue #481: old versions of sudo did in fact use execve, thus we must
# have TTY handle preservation in core.py.
os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:])
else:
# This must be a child process and not exec() since Mitogen replaces its
# stderr descriptor, causing the last user of the slave PTY to close it,
# resulting in the master side indicating EIO.
subprocess.check_call(sys.argv[sys.argv.index('--') + 1:])

@ -0,0 +1,56 @@
import os
import tempfile
import unittest2
import mitogen.core
import mitogen.service
import testlib
from mitogen.core import b
def prepare():
# ensure module loading delay is complete before loading PushFileService.
pass
@mitogen.core.takes_router
def wait_for_file(path, router):
pool = mitogen.service.get_or_create_pool(router=router)
service = pool.get_service(u'mitogen.service.PushFileService')
return service.get(path)
class PropagateToTest(testlib.RouterMixin, testlib.TestCase):
klass = mitogen.service.PushFileService
def test_two_grandchild_one_intermediary(self):
tf = tempfile.NamedTemporaryFile()
path = mitogen.core.to_text(tf.name)
try:
tf.write(b('test'))
tf.flush()
interm = self.router.local(name='interm')
c1 = self.router.local(via=interm, name='c1')
c2 = self.router.local(via=interm)
c1.call(prepare)
c2.call(prepare)
service = self.klass(router=self.router)
service.propagate_to(context=c1, path=path)
service.propagate_to(context=c2, path=path)
s = c1.call(wait_for_file, path=path)
self.assertEquals(b('test'), s)
s = c2.call(wait_for_file, path=path)
self.assertEquals(b('test'), s)
finally:
tf.close()
if __name__ == '__main__':
unittest2.main()

@ -55,6 +55,15 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase):
'--'
])
def test_tty_preserved(self):
# issue #481
os.environ['PREHISTORIC_SUDO'] = '1'
try:
context, argv = self.run_sudo()
self.assertEquals('1', context.call(os.getenv, 'PREHISTORIC_SUDO'))
finally:
del os.environ['PREHISTORIC_SUDO']
class NonEnglishPromptTest(testlib.DockerMixin, testlib.TestCase):
# Only mitogen/debian-test has a properly configured sudo.

Loading…
Cancel
Save