Merge pull request #245 from dw/dmw

Log masking, unlinked contributors page, resolv.conf fix, unidirectional routing

.travis.yml

@@ -10,13 +10,16 @@ python:
 - "2.7"
 
 env:
-- MODE=mitogen MITOGEN_TEST_DISTRO=debian
+- MODE=mitogen DISTRO=debian
-- MODE=mitogen MITOGEN_TEST_DISTRO=centos
+- MODE=mitogen DISTRO=centos
-- MODE=debops_common ANSIBLE_VERSION=2.4.3.0
+- MODE=debops_common VER=2.4.3.0
-- MODE=debops_common ANSIBLE_VERSION=2.5.1
+- MODE=debops_common VER=2.5.1
-- MODE=ansible ANSIBLE_VERSION=2.4.3.0 MITOGEN_TEST_DISTRO=debian
+# Ansible tests.
-- MODE=ansible ANSIBLE_VERSION=2.5.1 MITOGEN_TEST_DISTRO=centos
+- MODE=ansible VER=2.4.3.0 DISTRO=debian
-- MODE=ansible ANSIBLE_VERSION=2.5.1 MITOGEN_TEST_DISTRO=debian
+- MODE=ansible VER=2.5.1 DISTRO=centos
+- MODE=ansible VER=2.5.1 DISTRO=debian
+# Sanity check our tests against vanilla Ansible, they should still pass.
+- MODE=ansible VER=2.5.1 DISTRO=debian STRATEGY=linear
 
 install:
 - pip install -r dev_requirements.txt

.travis/ansible_tests.sh

@@ -3,8 +3,9 @@
 
 TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}"
 TMPDIR="/tmp/ansible-tests-$$"
-ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.4.3.0}"
+ANSIBLE_VERSION="${VER:-2.4.3.0}"
-MITOGEN_TEST_DISTRO="${MITOGEN_TEST_DISTRO:-debian}"
+export ANSIBLE_STRATEGY="${STRATEGY:-mitogen_linear}"
+DISTRO="${DISTRO:-debian}"
 
 export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}"
 
@@ -30,7 +31,7 @@ docker run \
     --detach \
     --publish 0.0.0.0:2201:22/tcp \
     --name=target \
-    mitogen/${MITOGEN_TEST_DISTRO}-test
+    mitogen/${DISTRO}-test
 echo travis_fold:end:docker_setup
 
 
@@ -57,15 +58,8 @@ make -C ${TRAVIS_BUILD_DIR}/tests/ansible
 echo travis_fold:end:job_setup
 
 
-echo travis_fold:start:mitogen_linear
+echo travis_fold:start:ansible
-/usr/bin/time ./mitogen_ansible_playbook.sh \
-    all.yml \
-    -i "${TMPDIR}/hosts"
-echo travis_fold:end:mitogen_linear
-
-
-echo travis_fold:start:vanilla_ansible
 /usr/bin/time ./run_ansible_playbook.sh \
     all.yml \
     -i "${TMPDIR}/hosts"
-echo travis_fold:end:vanilla_ansible
+echo travis_fold:end:ansible

.travis/debops_common_tests.sh

@@ -4,8 +4,8 @@
 TMPDIR="/tmp/debops-$$"
 TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}"
 TARGET_COUNT="${TARGET_COUNT:-2}"
-ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.4.3.0}"
+ANSIBLE_VERSION="${VER:-2.4.3.0}"
-MITOGEN_TEST_DISTRO=debian  # Naturally DebOps only supports Debian.
+DISTRO=debian  # Naturally DebOps only supports Debian.
 
 export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}"
 
@@ -60,7 +60,7 @@ do
         --detach \
         --publish 0.0.0.0:$port:22/tcp \
         --name=target$i \
-        mitogen/${MITOGEN_TEST_DISTRO}-test
+        mitogen/${DISTRO}-test
 
     echo \
         target$i \

.travis/mitogen_tests.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -ex
 # Run the Mitogen tests.
 
-MITOGEN_TEST_DISTRO="${MITOGEN_TEST_DISTRO:-debian}"
+MITOGEN_TEST_DISTRO="${DISTRO:-debian}"
 MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests

ansible_mitogen/connection.py

@@ -58,6 +58,10 @@ def _connect_local(spec):
         }
     }
 
+def wrap_or_none(klass, value):
+    if value is not None:
+        return klass(value)
+
 
 def _connect_ssh(spec):
     if C.HOST_KEY_CHECKING:
@@ -71,7 +75,7 @@ def _connect_ssh(spec):
             'check_host_keys': check_host_keys,
             'hostname': spec['remote_addr'],
             'username': spec['remote_user'],
-            'password': spec['password'],
+            'password': wrap_or_none(mitogen.core.Secret, spec['password']),
             'port': spec['port'],
             'python_path': spec['python_path'],
             'identity_file': spec['private_key_file'],
@@ -142,7 +146,7 @@ def _connect_su(spec):
         'enable_lru': True,
         'kwargs': {
             'username': spec['become_user'],
-            'password': spec['become_pass'],
+            'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']),
             'python_path': spec['python_path'],
             'su_path': spec['become_exe'],
             'connect_timeout': spec['timeout'],
@@ -156,7 +160,7 @@ def _connect_sudo(spec):
         'enable_lru': True,
         'kwargs': {
             'username': spec['become_user'],
-            'password': spec['become_pass'],
+            'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']),
             'python_path': spec['python_path'],
             'sudo_path': spec['become_exe'],
             'connect_timeout': spec['timeout'],
@@ -171,7 +175,7 @@ def _connect_mitogen_su(spec):
         'method': 'su',
         'kwargs': {
             'username': spec['remote_user'],
-            'password': spec['password'],
+            'password': wrap_or_none(mitogen.core.Secret, spec['password']),
             'python_path': spec['python_path'],
             'su_path': spec['become_exe'],
             'connect_timeout': spec['timeout'],
@@ -185,7 +189,7 @@ def _connect_mitogen_sudo(spec):
         'method': 'sudo',
         'kwargs': {
             'username': spec['remote_user'],
-            'password': spec['password'],
+            'password': wrap_or_none(mitogen.core.Secret, spec['password']),
             'python_path': spec['python_path'],
             'sudo_path': spec['become_exe'],
             'connect_timeout': spec['timeout'],
@@ -581,7 +585,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         """
         self.call(ansible_mitogen.target.write_path,
                   mitogen.utils.cast(out_path),
-                  mitogen.utils.cast(data),
+                  mitogen.core.Blob(data),
                   mode=mode,
                   utimes=utimes)
 

ansible_mitogen/module_finder.py

@@ -0,0 +1,137 @@
+
+import imp
+import os
+import sys
+import mitogen.master
+
+
+class Name(object):
+    def __str__(self):
+        return self.identifier
+
+    def __repr__(self):
+        return 'Name(%r)' % (self.identifier,)
+
+    def __init__(self, identifier):
+        self.identifier = identifier
+
+    def head(self):
+        head, _, tail = self.identifier.partition('.')
+        return head
+
+    def tail(self):
+        head, _, tail = self.identifier.partition('.')
+        return tail
+
+    def pop_n(self, level):
+        name = self.identifier
+        for _ in xrange(level):
+            if '.' not in name:
+                return None
+            name, _, _ = self.identifier.rpartition('.')
+        return Name(name)
+
+    def append(self, part):
+        return Name('%s.%s' % (self.identifier, part))
+
+
+class Module(object):
+    def __init__(self, name, path, kind=imp.PY_SOURCE, parent=None):
+        self.name = Name(name)
+        self.path = path
+        if kind == imp.PKG_DIRECTORY:
+            self.path = os.path.join(self.path, '__init__.py')
+        self.kind = kind
+        self.parent = parent
+
+    def fullname(self):
+        bits = [str(self.name)]
+        while self.parent:
+            bits.append(str(self.parent.name))
+            self = self.parent
+        return '.'.join(reversed(bits))
+
+    def __repr__(self):
+        return 'Module(%r, path=%r, parent=%r)' % (
+            self.name,
+            self.path,
+            self.parent,
+        )
+
+    def dirname(self):
+        return os.path.dirname(self.path)
+
+    def code(self):
+        fp = open(self.path)
+        try:
+            return compile(fp.read(), str(self.name), 'exec')
+        finally:
+            fp.close()
+
+
+def find(name, path=(), parent=None):
+    """
+    (Name, search path) -> Module instance or None.
+    """
+    try:
+        tup = imp.find_module(name.head(), list(path))
+    except ImportError:
+        return parent
+
+    fp, path, (suffix, mode, kind) = tup
+    if fp:
+        fp.close()
+
+    module = Module(name.head(), path, kind, parent)
+    if name.tail():
+        return find_relative(module, Name(name.tail()), path)
+    return module
+
+
+def find_relative(parent, name, path=()):
+    path = [parent.dirname()] + list(path)
+    return find(name, path, parent=parent)
+
+
+def path_pop(s, n):
+    return os.pathsep.join(s.split(os.pathsep)[-n:])
+
+
+def scan(module, path):
+    scanner = mitogen.master.scan_code_imports(module.code())
+    for level, modname_s, fromlist in scanner:
+        modname = Name(modname_s)
+        if level == -1:
+            imported = find_relative(module, modname, path)
+        elif level:
+            subpath = [path_pop(module.dirname(), level)] + list(path)
+            imported = find(modname.pop_n(level), subpath)
+        else:
+            imported = find(modname.pop_n(level), path)
+
+        if imported and mitogen.master.is_stdlib_path(imported.path):
+            continue
+
+        if imported and fromlist:
+            have = False
+            for fromname_s in fromlist:
+                fromname = modname.append(fromname_s)
+                f_imported = find_relative(imported, fromname, path)
+                if f_imported and f_imported.fullname() == fromname.identifier:
+                    have = True
+                    yield fromname, f_imported, None
+            if have:
+                continue
+
+        if imported:
+            yield modname, imported
+
+
+module = Module(name='ansible_module_apt', path='/Users/dmw/src/mitogen/.venv/lib/python2.7/site-packages/ansible/modules/packaging/os/apt.py')
+path = tuple(sys.path)
+path = ('/Users/dmw/src/ansible/lib',) + path
+
+
+from pprint import pprint
+for name, imported in scan(module, sys.path):
+    print '%s: %s' % (name, imported and (str(name) == imported.fullname()))

ansible_mitogen/runner.py

@@ -37,6 +37,7 @@ how to build arguments for it, preseed related data, etc.
 
 from __future__ import absolute_import
 import cStringIO
+import ctypes
 import json
 import logging
 import os
@@ -57,6 +58,17 @@ except ImportError:
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
 
+# For tasks that modify /etc/resolv.conf, non-Debian derivative glibcs cache
+# resolv.conf at startup and never implicitly reload it. Cope with that via an
+# explicit call to res_init() on each task invocation. BSD-alikes export it
+# directly, Linux #defines it as "__res_init".
+libc = ctypes.CDLL(None)
+libc__res_init = None
+for symbol in 'res_init', '__res_init':
+    try:
+        libc__res_init = getattr(libc, symbol)
+    except AttributeError:
+        pass
 
 LOG = logging.getLogger(__name__)
 
@@ -397,6 +409,8 @@ class NewStyleRunner(ScriptRunner):
         # module, but this has never been a bug report. Instead act like an
         # interpreter that had its script piped on stdin.
         self._argv = TemporaryArgv([''])
+        if libc__res_init:
+            libc__res_init()
 
     def revert(self):
         self._argv.revert()

ansible_mitogen/services.py

@@ -255,7 +255,7 @@ class ContextService(mitogen.service.Service):
         except AttributeError:
             raise Error('unsupported method: %(transport)s' % spec)
 
-        context = method(via=via, **spec['kwargs'])
+        context = method(via=via, unidirectional=True, **spec['kwargs'])
         if via and spec.get('enable_lru'):
             self._update_lru(context, spec, via)
         else:
@@ -489,8 +489,11 @@ class FileService(mitogen.service.Service):
     # odd-sized messages waste one tiny write() per message on the trailer.
     # Therefore subtract 10 bytes pickle overhead + 24 bytes header.
     IO_SIZE = mitogen.core.CHUNK_SIZE - (mitogen.core.Stream.HEADER_LEN + (
-        len(mitogen.core.Message.pickled(' ' * mitogen.core.CHUNK_SIZE).data) -
+        len(
-        mitogen.core.CHUNK_SIZE
+            mitogen.core.Message.pickled(
+                mitogen.core.Blob(' ' * mitogen.core.CHUNK_SIZE)
+            ).data
+        ) - mitogen.core.CHUNK_SIZE
     ))
 
     def _schedule_pending_unlocked(self, state):
@@ -507,7 +510,7 @@ class FileService(mitogen.service.Service):
             s = fp.read(self.IO_SIZE)
             if s:
                 state.unacked += len(s)
-                sender.send(s)
+                sender.send(mitogen.core.Blob(s))
             else:
                 # File is done. Cause the target's receive loop to exit by
                 # closing the sender, close the file, and remove the job entry.

docs/ansible.rst

@@ -43,11 +43,31 @@ it can only ensure the module executes as quickly as possible.
   of magnitude** compared to SSH pipelining, with around 5x fewer frames
   traversing the network in a typical run.
 
-* **No writes to the target's filesystem occur**, unless explicitly triggered
+* **Fewer writes to the target filesystem occur**. In typical configurations,
-  by a playbook step. In all typical configurations, Ansible repeatedly
+  Ansible repeatedly rewrites and extracts ZIP files to multiple temporary
-  rewrites and extracts ZIP files to multiple temporary directories on the
+  directories on the target. Security issues relating to temporarily files in
-  target. Since no temporary files are used, security issues relating to those
+  cross-account scenarios are entirely avoided.
-  files in cross-account scenarios are entirely avoided.
+
+
+Installation
+------------
+
+1. Thoroughly review the documented behavioural differences.
+2. Verify Ansible 2.3/2.4/2.5 and Python 2.7 are listed in ``ansible --version``
+   output.
+3. Download and extract https://github.com/dw/mitogen/archive/master.zip
+4. Modify ``ansible.cfg``:
+
+   .. code-block:: dosini
+
+        [defaults]
+        strategy_plugins = /path/to/mitogen-master/ansible_mitogen/plugins/strategy
+        strategy = mitogen_linear
+
+   The ``strategy`` key is optional. If omitted, the
+   ``ANSIBLE_STRATEGY=mitogen_linear`` environment variable can be set on a
+   per-run basis. Like ``mitogen_linear``, the ``mitogen_free`` strategy exists
+   to mimic the ``free`` strategy.
 
 
 Demo
@@ -86,27 +106,6 @@ Testimonials
   can't quite believe it."
 
 
-Installation
-------------
-
-1. Thoroughly review the documented behavioural differences.
-2. Verify Ansible 2.3/2.4/2.5 and Python 2.7 are listed in ``ansible --version``
-   output.
-3. Download and extract https://github.com/dw/mitogen/archive/master.zip
-4. Modify ``ansible.cfg``:
-
-   .. code-block:: dosini
-
-        [defaults]
-        strategy_plugins = /path/to/mitogen-master/ansible_mitogen/plugins/strategy
-        strategy = mitogen_linear
-
-   The ``strategy`` key is optional. If omitted, the
-   ``ANSIBLE_STRATEGY=mitogen_linear`` environment variable can be set on a
-   per-run basis. Like ``mitogen_linear``, the ``mitogen_free`` strategy exists
-   to mimic the ``free`` strategy.
-
-
 Noteworthy Differences
 ----------------------
 

docs/contributors.rst

@@ -0,0 +1,122 @@
+
+Contributors
+============
+
+Mitogen is production ready exclusively thanks to the careful testing, gracious
+sponsorship and outstanding future-thinking of its early adopters.
+
+.. raw:: html
+
+    <!--
+
+.. image:: images/sponsors/cgi.svg
+.. image:: images/sponsors/securelink.svg
+.. image:: images/sponsors/seantis.svg
+.. raw:: html
+
+    -->
+
+    <div style="background: #efefef; padding: 16px; margin: 1.5em 0;">
+        <div style="float: left; padding: 8px 32px 16px 8px;">
+            <img src="_images/cgi.svg" height=110 width=238>
+        </div>
+        <div>
+            <p>
+            Founded in 1976, CGI is one of the world’s largest IT and business
+            consulting services firms, helping clients achieve their goals,
+            including becoming customer-centric digital organizations.
+            </p>
+
+            <p>
+            <br clear="all">
+            For global career opportunities, please visit <a
+            href="http://www.cgi.com/en/careers/working-at-cgi">http://www.cgi.com/en/careers/working-at-cgi</a>.
+            </p>
+
+            <p style="margin-bottom: 0px;">
+            To <a
+            href="https://cgi.njoyn.com/CGI/xweb/XWeb.asp?page=jobdetails&CLID=21001&SBDID=21814&jobid=J0118-0787">directly
+            apply</a> to a UK team currently using Mitogen, contact us
+            regarding <a
+            href="https://cgi.njoyn.com/CGI/xweb/XWeb.asp?page=jobdetails&CLID=21001&SBDID=21814&jobid=J0118-0787">Open
+            Source Developer/DevOps</a> opportunities.
+            </p>
+        </div>
+    </div>
+
+.. raw:: html
+
+    <table border="0" width="100%">
+    <tr>
+        <td width="160" style="padding: 12px; text-align: center;">
+            <a href="https://www.seantis.ch"
+                ><img src="_images/seantis.svg" width="160"></a>
+        </td>
+        <td style="padding: 12px;" valign=middle>
+            Seantis GmbH<br>
+            <a href="https://www.seantis.ch">www.seantis.ch</a>
+        </td>
+    </tr>
+    <tr>
+        <td width="160" style="padding: 12px; text-align: center;">
+            <a href="https://www.securelink.com/"
+                ><img src="_images/securelink.svg" width="160"></a>
+        </td>
+        <td style="padding: 12px;">
+            Secure Third-Party Remote Access for Highly Regulated Industries<br>
+            <a href="https://www.securelink.com/">www.securelink.com</a>
+        </td>
+    </tr>
+    </table>
+
+
+.. raw:: html
+
+    <h3>Private Sponsors</h3>
+
+    <ul style="line-height: 120% !important;">
+    <li>Donald Clark Jackson &mdash;
+        <em>Mitogen is an exciting project, and I am happy to support its
+        development.</em></li>
+    <li><a href="https://nuvini.com">Niels Hendriks</a></li>
+    <li><a href="https://uberspace.de/">Uberspace</a> &mdash;
+        <em>Shared hosting for command-line lovers</em></li>
+    </ul>
+
+
+    <h3>Defenders of Time</h3>
+
+    <ul>
+    <li>Icil &mdash; <em>Time saving, money saving...phenomenal! Keep going and
+        give us more. We await with anticipation.</em></li>
+    <li><a href="https://www.systemli.org/">systemli tech collective</a> &mdash;
+        <em>D.I.Y.</em></li>
+    </ul>
+
+
+.. raw:: html
+
+    <h3>Productivity Lovers</h3>
+
+    <ul>
+    <li>Alex Willmer</li>
+    <li><a href="https://underwhelm.net/">Dan Dorman</a> &mdash; - <em>When I truly understand my enemy … then in that very moment I also love him.</em></li>
+    <li><a href="https://www.deps.co/">Deps</a> &mdash; <em>Private Maven Repository Hosting for Java, Scala, Groovy, Clojure</em></li>
+    <li><a href="https://www.edport.co.uk/">Edward Wilson</a> &mdash; <em>To efficiency and beyond! I wish Mitogen and all who sail in her the best of luck.</em></li>
+    <li><a href="https://www.epartment.nl/">Epartment</a></li>
+    <li><a href="http://andrianaivo.org/">Fidy Andrianaivo</a> &mdash; <em>never let a human do an ansible job ;)</em></li>
+    <li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
+    <li>KennethC</li>
+    <li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>
+    <li>luto</li>
+    <li><a href="https://mayeu.me/">Mayeu a.k.a Matthieu Maury</a></li>
+    <li><a href="https://twitter.com/nathanhruby">@nathanhruby</a></li>
+    <li><a href="http://pageflows.com/">Ramy</a></li>
+    <li>Scott Vokes</li>
+    <li><a href="https://twitter.com/sirtux">Tom Eichhorn</a></li>
+    <li><a href="https://dotat.at/">Tony Finch</a></li>
+    <li>Tony Million &mdash; Never wear socks and sandles.</li>
+    <li>randy &mdash; <em>desperate for automation</em></li>
+    <li>Michael & Vicky Twomey-Lee</li>
+    <li><a href="http://www.wezm.net/">Wesley Moore</a></li>
+    </ul>

docs/getting_started.rst

@@ -320,6 +320,8 @@ remote procedure calls:
 
 User-defined types may not be used, except for:
 
+* :py:class:`mitogen.core.Blob`
+* :py:class:`mitogen.core.Secret`
 * :py:class:`mitogen.core.CallError`
 * :py:class:`mitogen.core.Context`
 * :py:class:`mitogen.core.Sender`

docs/images/sponsors/a.html

@@ -0,0 +1,3 @@
+<body bgcolor=black>
+
+<img src="cgi.svg" style="background: white">

docs/images/sponsors/cgi.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="65 4.5 302 141"><defs><style>.a{fill:#e31937;}</style></defs><title>CGI_Logo_color_v2</title><path class="a" d="M140.33,32c-27.4,0-44.2,21.4-44.2,43,0,26,21.2,43,44.4,43,15.4,0,30-6.8,42.4-17.8v32.2c-13,7.8-30.8,12.6-44.6,12.6-39.6,0-72.8-32.2-72.8-70,0-40,33.4-70,73-70,15.2,0,33,4.6,44.6,10.4V47C168.53,37.4,153.73,32,140.33,32Z"/><path class="a" d="M267.11,145c-39.8,0-73.4-31-73.4-70,0-39.4,33.4-70,75.2-70,15.2,0,34,4,45.6,9.4V45.8c-13.2-7.6-30.2-13.8-45-13.8-27.4,0-45.2,21.4-45.2,43,0,25.4,21,43.8,45.6,43.8a43.78,43.78,0,0,0,16.6-2.8V90.8h-22.4V64.2h51.2V135A116.82,116.82,0,0,1,267.11,145Z"/><path class="a" d="M337.67,142.2V7.8h28.8V142.2h-28.8Z"/></svg>

docs/images/sponsors/securelink.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="211.087" height="55.108">
+  <path fill="#007CBA" d="M153.115 35.018V21.074h3.336v11.175h8.62v2.769h-11.956zm14.683-13.944h3.343v13.943h-3.343V21.074zm6.151 13.944V21.074h5.613l8.496 11.175h.461V21.074h3.334v13.943h-5.723l-8.38-11.171h-.461v11.171h-3.34zm20.44 0V21.074h3.317v5.199h1.962l6.094-5.199h5.002l-8.217 6.528 8.54 7.415h-4.907l-6.445-5.965h-2.028v5.965h-3.318z"/>
+  <path fill="#FF5000" d="M48.995 34.595c-.684-.284-1.173-.713-1.468-1.297-.296-.563-.438-1.323-.438-2.233h3.41c0 .347.159.652.484.922.315.263.71.4 1.17.4h5.582c.745 0 1.292-.126 1.644-.367.35-.25.528-.646.528-1.207 0-.347-.062-.647-.178-.879-.114-.233-.346-.414-.687-.532-.337-.123-.801-.187-1.405-.187h-4.948c-.992 0-1.817-.06-2.462-.17-.65-.111-1.17-.323-1.565-.617-.404-.298-.689-.72-.873-1.273-.189-.547-.281-1.26-.281-2.138 0-.711.092-1.312.295-1.809.192-.508.49-.908.903-1.229.408-.309.933-.538 1.581-.684.646-.153 1.441-.22 2.385-.22h5.673c1.114 0 2.02.138 2.722.414.703.284 1.219.713 1.549 1.296.326.584.482 1.322.482 2.224h-3.332c-.037-.468-.162-.794-.383-.999-.222-.2-.661-.305-1.325-.305h-5.601c-1.076 0-1.615.48-1.615 1.447 0 .582.135.969.405 1.157.264.182.672.269 1.21.269h6.085c.878 0 1.595.044 2.139.133.542.089 1.003.269 1.388.539.378.277.664.7.863 1.252.192.569.282 1.323.282 2.28 0 .981-.159 1.791-.484 2.414-.32.627-.832 1.081-1.519 1.379-.688.3-1.586.443-2.689.443h-6.737c-1.177 0-2.107-.136-2.79-.423m16.639.423V21.074h14.242v2.63H68.961v3.06h10.577v2.431H68.961v3.192h10.915v2.631H65.634zm35.883-1.219c-.841-.811-1.267-2.058-1.267-3.745v-8.979h3.337v8.66c0 .942.202 1.593.595 1.961.398.365 1.053.554 1.977.554h4.604c.563 0 1.027-.085 1.377-.234.356-.17.618-.43.791-.812.172-.377.258-.872.258-1.487v-8.642h3.338v8.979c0 .936-.092 1.709-.277 2.339-.178.625-.467 1.139-.863 1.526-.395.378-.908.663-1.537.836-.627.176-1.398.263-2.31.263h-6.128c-1.756 0-3.052-.411-3.895-1.219m16.983 1.219V21.074h10.584c.782 0 1.461.067 2.053.22.598.146 1.107.396 1.545.765.436.358.767.862 1.012 1.502.234.642.354 1.447.354 2.414 0 .967-.18 1.724-.545 2.264-.363.547-.854 1.006-1.482 1.368.604.24 1.027.419 1.285.55.252.132.492.354.697.669.207.314.305.752.305 1.297v2.895h-3.353v-2.166c0-.7-.206-1.201-.604-1.512-.412-.31-1.089-.461-2.047-.461h-6.445v4.139H118.5zm18.039 0V21.074h14.252v2.63h-10.913v3.06h10.572v2.431h-10.572v3.192h10.913v2.631h-14.252zm-52.474-.477c-.735-.298-1.258-.803-1.571-1.51-.312-.713-.465-1.696-.465-2.956v-3.962c.015-1.356.189-2.401.515-3.128.329-.719.874-1.231 1.634-1.517.765-.291 1.847-.436 3.255-.436h5.485c1.19 0 2.183.132 2.959.401.785.262 1.383.746 1.824 1.449.427.701.646 1.673.646 2.933h-3.46c0-.733-.199-1.253-.589-1.557-.392-.302-1.037-.457-1.949-.457h-4.272c-.795 0-1.372.056-1.742.164-.364.104-.618.326-.762.653-.136.34-.208.871-.208 1.621v3.451c0 .693.087 1.226.269 1.595.179.364.456.615.837.741.377.137.906.199 1.586.199h4.273c.672 0 1.186-.058 1.543-.16.355-.104.619-.304.78-.599.17-.284.253-.721.253-1.29h3.459c-.009 1.274-.155 2.256-.423 2.937-.269.685-.765 1.162-1.477 1.452-.716.287-1.762.437-3.147.437h-6.209c-1.296.001-2.318-.157-3.044-.461m46.214-6.932c.279-.323.424-.874.432-1.634 0-.76-.141-1.309-.413-1.641-.271-.325-.769-.488-1.499-.488h-6.962v4.265h6.962c.704-.001 1.192-.169 1.48-.502M25.134 55.108l-2.936-2.976L33.18 41.317c.661-.721 1.245-1.514 1.752-2.386 3.771-6.555 1.531-14.927-5.016-18.709-3.846-2.225-8.336-2.361-12.133-.771 4.522-2.163 9.989-2.141 14.66.558 7.522 4.34 10.104 13.962 5.763 21.474-.593 1.037-1.3 1.987-2.076 2.826L25.134 55.108zm-11.708-3.67l-2.89-2.929 15.103-14.862c.515-.513.985-1.106 1.373-1.78 2.305-4.003.936-9.118-3.067-11.422-2.113-1.221-4.546-1.406-6.704-.727a.6456.6456 0 0 1 .091-.043c2.822-1.096 6.106-.952 8.94.682 4.966 2.879 6.674 9.232 3.797 14.203-.43.756-.948 1.435-1.536 2.04L13.426 51.438z"/>
+  <path fill="#007CBA" d="M7.865 35.094C.349 30.758-2.234 21.136 2.106 13.616c.599-1.035 1.296-1.979 2.078-2.827L15.184 0l2.923 2.976L7.132 13.785c-.655.719-1.245 1.52-1.748 2.388-3.78 6.548-1.534 14.931 5.017 18.709 3.842 2.228 8.33 2.356 12.127.766-4.517 2.166-9.994 2.147-14.663-.554m6.179-.353c-4.967-2.869-6.674-9.23-3.81-14.194.438-.759.956-1.441 1.54-2.04L26.893 3.659l2.88 2.931-15.1 14.865c-.523.519-.988 1.107-1.378 1.781-2.304 3.997-.925 9.116 3.071 11.427 2.114 1.213 4.548 1.406 6.706.722-.037.013-.061.033-.09.049-2.824 1.092-6.106.934-8.938-.693"/>
+</svg>

mitogen/core.py

@@ -81,8 +81,12 @@ IS_DEAD = 999
 PY3 = sys.version_info > (3,)
 if PY3:
     b = lambda s: s.encode('latin-1')
+    BytesType = bytes
+    UnicodeType = unicode
 else:
     b = str
+    BytesType = str
+    UnicodeType = unicode
 
 CHUNK_SIZE = 131072
 _tls = threading.local()
@@ -109,6 +113,25 @@ class LatchError(Error):
     pass
 
 
+class Blob(BytesType):
+    def __repr__(self):
+        return '[blob: %d bytes]' % len(self)
+
+    def __reduce__(self):
+        return (Blob, (BytesType(self),))
+
+
+class Secret(UnicodeType):
+    def __repr__(self):
+        return '[secret]'
+
+    def __str__(self):
+        return UnicodeType(self)
+
+    def __reduce__(self):
+        return (Secret, (UnicodeType(self),))
+
+
 class CallError(Error):
     def __init__(self, fmt=None, *args):
         if not isinstance(fmt, Exception):
@@ -310,7 +333,10 @@ class Message(object):
                 return self._unpickle_sender
             elif func == '_unpickle_context':
                 return self._unpickle_context
-
+            elif func == 'Blob':
+                return Blob
+            elif func == 'Secret':
+                return Secret
         raise StreamError('cannot unpickle %r/%r', module, func)
 
     @property
@@ -797,6 +823,11 @@ class Stream(BasicStream):
     #: :py:attr:`Message.auth_id` of every message received on this stream.
     auth_id = None
 
+    #: If not :data:`False`, indicates the stream has :attr:`auth_id` set and
+    #: its value is the same as :data:`mitogen.context_id` or appears in
+    #: :data:`mitogen.parent_ids`.
+    is_privileged = False
+
     def __init__(self, router, remote_id, **kwargs):
         self._router = router
         self.remote_id = remote_id
@@ -1239,6 +1270,7 @@ class IoLogger(BasicStream):
 class Router(object):
     context_class = Context
     max_message_size = 128 * 1048576
+    unidirectional = False
 
     def __init__(self, broker):
         self.broker = broker
@@ -1343,47 +1375,57 @@ class Router(object):
         except Exception:
             LOG.exception('%r._invoke(%r): %r crashed', self, msg, fn)
 
-    def _async_route(self, msg, stream=None):
+    def _async_route(self, msg, in_stream=None):
-        _vv and IOLOG.debug('%r._async_route(%r, %r)', self, msg, stream)
+        _vv and IOLOG.debug('%r._async_route(%r, %r)', self, msg, in_stream)
         if len(msg.data) > self.max_message_size:
             LOG.error('message too large (max %d bytes): %r',
                       self.max_message_size, msg)
             return
 
         # Perform source verification.
-        if stream:
+        if in_stream:
             parent = self._stream_by_id.get(mitogen.parent_id)
             expect = self._stream_by_id.get(msg.auth_id, parent)
-            if stream != expect:
+            if in_stream != expect:
                 LOG.error('%r: bad auth_id: got %r via %r, not %r: %r',
-                          self, msg.auth_id, stream, expect, msg)
+                          self, msg.auth_id, in_stream, expect, msg)
                 return
 
             if msg.src_id != msg.auth_id:
                 expect = self._stream_by_id.get(msg.src_id, parent)
-                if stream != expect:
+                if in_stream != expect:
                     LOG.error('%r: bad src_id: got %r via %r, not %r: %r',
-                              self, msg.src_id, stream, expect, msg)
+                              self, msg.src_id, in_stream, expect, msg)
                     return
 
-            if stream.auth_id is not None:
+            if in_stream.auth_id is not None:
-                msg.auth_id = stream.auth_id
+                msg.auth_id = in_stream.auth_id
 
         if msg.dst_id == mitogen.context_id:
-            return self._invoke(msg, stream)
+            return self._invoke(msg, in_stream)
 
-        stream = self._stream_by_id.get(msg.dst_id)
+        out_stream = self._stream_by_id.get(msg.dst_id)
-        if stream is None:
+        if out_stream is None:
-            stream = self._stream_by_id.get(mitogen.parent_id)
+            out_stream = self._stream_by_id.get(mitogen.parent_id)
 
-        if stream is None:
+        dead = False
+        if out_stream is None:
             LOG.error('%r: no route for %r, my ID is %r',
                       self, msg, mitogen.context_id)
+            dead = True
+
+        if in_stream and self.unidirectional and not dead and \
+           not (in_stream.is_privileged or out_stream.is_privileged):
+            LOG.error('routing mode prevents forward of %r from %r -> %r',
+                      msg, in_stream, out_stream)
+            dead = True
+
+        if dead:
             if msg.reply_to and not msg.is_dead:
                 msg.reply(Message.dead(), router=self)
             return
 
-        stream._send(msg)
+        out_stream._send(msg)
 
     def route(self, msg):
         self.broker.defer(self._async_route, msg)
@@ -1551,14 +1593,15 @@ class ExternalContext(object):
                 LOG.error('Stream had %d bytes after 2000ms', pending)
             self.broker.defer(stream.on_disconnect, self.broker)
 
-    def _setup_master(self, max_message_size, profiling, parent_id,
+    def _setup_master(self, max_message_size, profiling, unidirectional,
-                      context_id, in_fd, out_fd):
+                      parent_id, context_id, in_fd, out_fd):
         Router.max_message_size = max_message_size
         self.profiling = profiling
         if profiling:
             enable_profiling()
         self.broker = Broker()
         self.router = Router(self.broker)
+        self.router.undirectional = unidirectional
         self.router.add_handler(
             fn=self._on_shutdown_msg,
             handle=SHUTDOWN,
@@ -1694,11 +1737,11 @@ class ExternalContext(object):
         self.dispatch_stopped = True
 
     def main(self, parent_ids, context_id, debug, profiling, log_level,
-             max_message_size, version, in_fd=100, out_fd=1, core_src_fd=101,
+             unidirectional, max_message_size, version, in_fd=100, out_fd=1,
-             setup_stdio=True, setup_package=True, importer=None,
+             core_src_fd=101, setup_stdio=True, setup_package=True,
-             whitelist=(), blacklist=()):
+             importer=None, whitelist=(), blacklist=()):
-        self._setup_master(max_message_size, profiling, parent_ids[0],
+        self._setup_master(max_message_size, profiling, unidirectional,
-                           context_id, in_fd, out_fd)
+                           parent_ids[0], context_id, in_fd, out_fd)
         try:
             try:
                 self._setup_logging(debug, log_level)

mitogen/fakessh.py

@@ -349,6 +349,7 @@ def run(dest, router, args, deadline=None, econtext=None):
                 'out_fd': sock2.fileno(),
                 'parent_ids': parent_ids,
                 'profiling': getattr(router, 'profiling', False),
+                'unidirectional': getattr(router, 'unidirectional', False),
                 'setup_stdio': False,
                 'version': mitogen.__version__,
             },))

mitogen/fork.py

@@ -90,10 +90,11 @@ class Stream(mitogen.parent.Stream):
     on_fork = None
 
     def construct(self, old_router, max_message_size, on_fork=None,
-                  debug=False, profiling=False):
+                  debug=False, profiling=False, unidirectional=False):
         # fork method only supports a tiny subset of options.
         super(Stream, self).construct(max_message_size=max_message_size,
-                                      debug=debug, profiling=profiling)
+                                      debug=debug, profiling=profiling,
+                                      unidirectional=False)
         self.on_fork = on_fork
 
         responder = getattr(old_router, 'responder', None)

mitogen/master.py

@@ -319,21 +319,19 @@ class LogForwarder(object):
         return 'LogForwarder(%r)' % (self._router,)
 
 
-class ModuleFinder(object):
 _STDLIB_PATHS = _stdlib_paths()
 
-    def __init__(self):
-        #: Import machinery is expensive, keep :py:meth`:get_module_source`
-        #: results around.
-        self._found_cache = {}
 
-        #: Avoid repeated dependency scanning, which is expensive.
+def is_stdlib_path(path):
-        self._related_cache = {}
+    return any(
+        os.path.commonprefix((libpath, path)) == libpath
+        and 'site-packages' not in path
+        and 'dist-packages' not in path
+        for libpath in _STDLIB_PATHS
+    )
 
-    def __repr__(self):
-        return 'ModuleFinder()'
 
-    def is_stdlib_name(self, modname):
+def is_stdlib_name(modname):
     """Return ``True`` if `modname` appears to come from the standard
     library."""
     if imp.is_builtin(modname) != 0:
@@ -345,14 +343,20 @@ class ModuleFinder(object):
 
     # six installs crap with no __file__
     modpath = os.path.abspath(getattr(module, '__file__', ''))
-        if 'site-packages' in modpath:
+    return is_stdlib_path(modpath)
-            return False
 
-        for dirname in self._STDLIB_PATHS:
-            if os.path.commonprefix((dirname, modpath)) == dirname:
-                return True
 
-        return False
+class ModuleFinder(object):
+    def __init__(self):
+        #: Import machinery is expensive, keep :py:meth`:get_module_source`
+        #: results around.
+        self._found_cache = {}
+
+        #: Avoid repeated dependency scanning, which is expensive.
+        self._related_cache = {}
+
+    def __repr__(self):
+        return 'ModuleFinder()'
 
     def _looks_like_script(self, path):
         """
@@ -515,7 +519,7 @@ class ModuleFinder(object):
                 name
                 for name in maybe_names
                 if sys.modules.get(name) is not None
-                and not self.is_stdlib_name(name)
+                and not is_stdlib_name(name)
                 and 'six.moves' not in name  # TODO: crap
             )
         ))
@@ -674,7 +678,6 @@ class Broker(mitogen.core.Broker):
 
 class Router(mitogen.parent.Router):
     broker_class = Broker
-    debug = False
     profiling = False
 
     def __init__(self, broker=None, max_message_size=None):

mitogen/parent.py

@@ -563,7 +563,7 @@ class Stream(mitogen.core.Stream):
 
     def construct(self, max_message_size, remote_name=None, python_path=None,
                   debug=False, connect_timeout=None, profiling=False,
-                  old_router=None, **kwargs):
+                  unidirectional=False, old_router=None, **kwargs):
         """Get the named context running on the local machine, creating it if
         it does not exist."""
         super(Stream, self).construct(**kwargs)
@@ -585,6 +585,7 @@ class Stream(mitogen.core.Stream):
         self.remote_name = remote_name
         self.debug = debug
         self.profiling = profiling
+        self.unidirectional = unidirectional
         self.max_message_size = max_message_size
         self.connect_deadline = time.time() + self.connect_timeout
 
@@ -709,6 +710,7 @@ class Stream(mitogen.core.Stream):
             'context_id': self.remote_id,
             'debug': self.debug,
             'profiling': self.profiling,
+            'unidirectional': self.unidirectional,
             'log_level': get_log_level(),
             'whitelist': self._router.get_module_whitelist(),
             'blacklist': self._router.get_module_blacklist(),
@@ -1021,6 +1023,7 @@ class Router(mitogen.core.Router):
         klass = stream_by_method_name(method_name)
         kwargs.setdefault('debug', self.debug)
         kwargs.setdefault('profiling', self.profiling)
+        kwargs.setdefault('unidirectional', self.unidirectional)
 
         via = kwargs.pop('via', None)
         if via is not None:

mitogen/unix.py

@@ -89,6 +89,7 @@ class Listener(mitogen.core.BasicStream):
         stream.accept(sock.fileno(), sock.fileno())
         stream.name = 'unix_client.%d' % (pid,)
         stream.auth_id = mitogen.context_id
+        stream.is_privileged = True
         self._router.register(context, stream)
         sock.send(struct.pack('>LLL', context_id, mitogen.context_id,
                               os.getpid()))

mitogen/utils.py

@@ -101,19 +101,25 @@ def with_router(func):
     return wrapper
 
 
+PASSTHROUGH = (
+    int, float, bool,
+    type(None),
+    mitogen.core.Context,
+    mitogen.core.CallError,
+    mitogen.core.Blob,
+    mitogen.core.Secret,
+)
+
 def cast(obj):
     if isinstance(obj, dict):
         return {cast(k): cast(v) for k, v in obj.iteritems()}
     if isinstance(obj, (list, tuple)):
         return [cast(v) for v in obj]
-    if obj is None or isinstance(obj, (int, float)):
+    if isinstance(obj, PASSTHROUGH):
         return obj
     if isinstance(obj, unicode):
         return unicode(obj)
     if isinstance(obj, str):
         return str(obj)
-    if isinstance(obj, (mitogen.core.Context,
-                        mitogen.core.CallError)):
-        return obj
 
     raise TypeError("Cannot serialize: %r: %r" % (type(obj), obj))

tests/ansible/ansible.cfg

@@ -5,6 +5,7 @@ strategy_plugins = ../../ansible_mitogen/plugins/strategy
 action_plugins = lib/action
 callback_plugins = lib/callback
 library = lib/modules
+# module_utils = lib/module_utils
 retry_files_enabled = False
 forks = 50
 

tests/ansible/integration/all.yml

@@ -3,12 +3,14 @@
 # This playbook imports all tests that are known to work at present.
 #
 
-#- import_playbook: action/all.yml
+- import_playbook: action/all.yml
-#- import_playbook: async/all.yml
+- import_playbook: async/all.yml
-#- import_playbook: become/all.yml
+- import_playbook: become/all.yml
-#- import_playbook: connection_loader/all.yml
+- import_playbook: connection_loader/all.yml
-#- import_playbook: context_service/all.yml
+- import_playbook: context_service/all.yml
-#- import_playbook: playbook_semantics/all.yml
+#- import_playbook: module_utils/all.yml
-#- import_playbook: remote_tmp/all.yml
+- import_playbook: playbook_semantics/all.yml
-#- import_playbook: runner/all.yml
+- import_playbook: remote_tmp/all.yml
+- import_playbook: runner/all.yml
 - import_playbook: ssh/all.yml
+- import_playbook: glibc_caches/all.yml

tests/ansible/integration/glibc_caches/all.yml

@@ -0,0 +1,2 @@
+
+- import_playbook: resolv_conf.yml

tests/ansible/integration/glibc_caches/resolv_conf.yml

@@ -0,0 +1,38 @@
+
+# This cannot run against localhost, it damages /etc
+
+- name: integration/glibc_caches/resolv_conf.yml
+  gather_facts: true
+  become: true
+  hosts: test-targets
+  vars:
+    ansible_become_pass: has_sudo_pubkey_password
+  tasks:
+
+    - debug: msg={{hostvars}}
+    - mitogen_test_gethostbyname:
+        name: www.google.com
+      register: out
+      when: ansible_virtualization_type == "docker"
+
+    - shell: cp /etc/resolv.conf /tmp/resolv.conf
+      when: ansible_virtualization_type == "docker"
+
+    - shell: echo > /etc/resolv.conf
+      when: ansible_virtualization_type == "docker"
+
+    - mitogen_test_gethostbyname:
+        name: www.google.com
+      register: out
+      ignore_errors: true
+      when: ansible_virtualization_type == "docker"
+
+    - shell: cat /tmp/resolv.conf > /etc/resolv.conf
+      when: ansible_virtualization_type == "docker"
+
+    - assert:
+        that:
+        - out.failed
+        - '"Name or service not known" in out.msg or
+           "Temporary failure in name resolution" in out.msg'
+      when: ansible_virtualization_type == "docker"

tests/ansible/integration/module_utils/adjacent_to_playbook.yml

@@ -0,0 +1,17 @@
+# external2 is loaded from config path.
+# external1 is loaded from integration/module_utils/module_utils/..
+
+- name: integration/module_utils/adjacent_to_playbook.yml
+  hosts: test-targets
+  any_errors_fatal: true
+  tasks:
+
+  - custom_python_external_module:
+    register: out
+
+  - debug: msg={{out}}
+  - assert:
+      that:
+      - out.external1_path == "ansible/integration/module_utils/module_utils/external1.py"
+      - out.external2_path == "ansible/lib/module_utils/external2.py"
+

tests/ansible/integration/module_utils/adjacent_to_role.yml

@@ -0,0 +1,8 @@
+# external2 is loaded from config path.
+# external1 is loaded from integration/module_utils/roles/modrole/module_utils/..
+
+- name: integration/module_utils/adjacent_to_playbook.yml
+  hosts: test-targets
+  any_errors_fatal: true
+  roles:
+  - modrole

tests/ansible/integration/module_utils/all.yml

@@ -0,0 +1,6 @@
+
+- import_playbook: from_config_path.yml
+- import_playbook: from_config_path_pkg.yml
+- import_playbook: adjacent_to_playbook.yml
+- import_playbook: adjacent_to_role.yml
+- import_playbook: overrides_builtin.yml

tests/ansible/integration/module_utils/from_config_path.yml

@@ -0,0 +1,15 @@
+# external1 and external2 are loaded from config path.
+
+- name: integration/module_utils/from_config_path.yml
+  hosts: test-targets
+  any_errors_fatal: true
+  tasks:
+
+  - custom_python_external_module:
+    register: out
+
+  - assert:
+      that:
+      - out.external1_path == "ansible/lib/module_utils/external1.py"
+      - out.external2_path == "ansible/lib/module_utils/external2.py"
+

tests/ansible/integration/module_utils/from_config_path_pkg.yml

@@ -0,0 +1,14 @@
+# external1 and external2 are loaded from config path.
+
+- name: integration/module_utils/from_config_path.yml
+  hosts: test-targets
+  any_errors_fatal: true
+  tasks:
+
+  - custom_python_external_pkg:
+    register: out
+
+  - assert:
+      that:
+      - out.extmod_path == "ansible/lib/module_utils/externalpkg/extmod.py"
+

tests/ansible/integration/module_utils/module_utils/external1.py

@@ -0,0 +1,11 @@
+# I am ansible.module_utils.external1 for any module that does not have an
+# adjacent module_utils directory overriding the name, since I appear in the
+# 'module_utils' path in ansible.cfg.
+
+from ansible.module_utils import external2
+
+def path():
+    return "ansible/integration/module_utils/module_utils/external1.py"
+
+def path2():
+    return external2.path()

tests/ansible/integration/module_utils/overrides_builtin.yml

@@ -0,0 +1,6 @@
+
+- name: integration/module_utils/overrides_builtin.yml
+  hosts: test-targets
+  any_errors_fatal: true
+  roles:
+  - overrides_modrole

tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils import external3
+
+def main():
+    module = AnsibleModule(argument_spec={})
+    module.exit_json(external2_path=external3.path2(),
+                     external3_path=external3.path())
+
+if __name__ == '__main__':
+    main()

tests/ansible/integration/module_utils/roles/modrole/module_utils/external1.py

@@ -0,0 +1,12 @@
+# I am ansible.module_utils.external1 for any module that does not have an
+# adjacent module_utils directory overriding the name, since I appear in the
+# 'module_utils' path in ansible.cfg.
+
+from ansible.module_utils import external2
+
+def path():
+    return "ansible/integration/module_utils/roles/modroel/module_utils/external1.py"
+
+def path2():
+    return external2.path()
+

tests/ansible/integration/module_utils/roles/modrole/module_utils/external2.py

@@ -0,0 +1,3 @@
+
+def path():
+    return "integration/module_utils/roles/modrole/module_utils/external2.py"

tests/ansible/integration/module_utils/roles/modrole/module_utils/external3.py

@@ -0,0 +1,12 @@
+# I am ansible.module_utils.external1 for any module that does not have an
+# adjacent module_utils directory overriding the name, since I appear in the
+# 'module_utils' path in ansible.cfg.
+
+from ansible.module_utils import external2
+
+def path():
+    return "integration/module_utils/roles/modrole/module_utils/external3.py"
+
+def path2():
+    return external2.path()
+

tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+
+- uses_external3:
+  register: out
+
+- debug: msg={{out}}
+- assert:
+    that:
+    - out.external3_path == "integration/module_utils/roles/modrole/module_utils/external3.py"
+    - out.external2_path == "integration/module_utils/roles/modrole/module_utils/external2.py"

tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+import json
+from ansible.module_utils.basic import path
+
+def main():
+    print json.dumps({
+        'path': path()
+    })
+
+if __name__ == '__main__':
+    main()

tests/ansible/integration/module_utils/roles/overrides_modrole/module_utils/known_hosts.py

@@ -0,0 +1,4 @@
+# Override basic.py with our own thing.
+
+def path():
+    return 'ansible/integration/module_utils/roles/override_modrole/module_utils/basic.py'

tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+
+- uses_custom_known_hosts:
+  register: out
+
+- debug: msg={{out}}
+- assert:
+    that:
+    - out.path == "ansible/integration/module_utils/roles/override_modrole/module_utils/known_hosts.py"

tests/ansible/integration/remote_tmp/readonly_homedir.yml

@@ -17,4 +17,4 @@
     - name: Verify system temp directory was used.
       assert:
         that:
-          - out.__file__.startswith("/tmp/ansible_mitogen_")
+          - out.__file__.startswith("/tmp/ansible_")

tests/ansible/lib/module_utils/external1.py

@@ -0,0 +1,11 @@
+# I am ansible.module_utils.external1 for any module that does not have an
+# adjacent module_utils directory overriding the name, since I appear in the
+# 'module_utils' path in ansible.cfg.
+
+from ansible.module_utils import external2
+
+def path():
+    return "ansible/lib/module_utils/external1.py"
+
+def path2():
+    return external2.path()

tests/ansible/lib/module_utils/external2.py

@@ -0,0 +1,3 @@
+
+def path():
+    return "ansible/lib/module_utils/external2.py"

tests/ansible/lib/module_utils/externalpkg/extmod.py

@@ -0,0 +1,3 @@
+
+def path():
+    return 'ansible/lib/module_utils/externalpkg/extmod.py'

tests/ansible/lib/modules/custom_python_external_module.py

@@ -0,0 +1,13 @@
+#!/usr/bin/python
+# I expect the quote from modules2/module_utils/joker.py.
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils import external1
+
+def main():
+    module = AnsibleModule(argument_spec={})
+    module.exit_json(external1_path=external1.path(),
+                     external2_path=external1.path2())
+
+if __name__ == '__main__':
+    main()

tests/ansible/lib/modules/custom_python_external_pkg.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.externalpkg import extmod
+
+def main():
+    module = AnsibleModule(argument_spec={})
+    module.exit_json(extmod_path=extmod.path())
+
+if __name__ == '__main__':
+    main()

tests/ansible/lib/modules/mitogen_test_gethostbyname.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+
+# I am a module that indirectly depends on glibc cached /etc/resolv.conf state.
+
+import socket
+from ansible.module_utils.basic import AnsibleModule
+
+def main():
+    module = AnsibleModule(argument_spec={'name': {'type': 'str'}})
+    try:
+        module.exit_json(addr=socket.gethostbyname(module.params['name']))
+    except socket.error, e:
+        module.fail_json(msg=str(e))
+
+if __name__ == '__main__':
+    main()

tests/module_finder_test.py

@@ -22,33 +22,30 @@ class ReprTest(testlib.TestCase):
 
 
 class IsStdlibNameTest(testlib.TestCase):
-    klass = mitogen.master.ModuleFinder
+    func = staticmethod(mitogen.master.is_stdlib_name)
-
-    def call(self, fullname):
-        return self.klass().is_stdlib_name(fullname)
 
     def test_builtin(self):
         import sys
-        self.assertTrue(self.call('sys'))
+        self.assertTrue(self.func('sys'))
 
     def test_stdlib_1(self):
         import logging
-        self.assertTrue(self.call('logging'))
+        self.assertTrue(self.func('logging'))
 
     def test_stdlib_2(self):
         # virtualenv only symlinks some paths to its local site-packages
         # directory. Ensure both halves of the search path return the correct
         # result.
         import email
-        self.assertTrue(self.call('email'))
+        self.assertTrue(self.func('email'))
 
     def test_mitogen_core(self):
         import mitogen.core
-        self.assertFalse(self.call('mitogen.core'))
+        self.assertFalse(self.func('mitogen.core'))
 
     def test_mitogen_fakessh(self):
         import mitogen.fakessh
-        self.assertFalse(self.call('mitogen.fakessh'))
+        self.assertFalse(self.func('mitogen.fakessh'))
 
 
 class GetModuleViaPkgutilTest(testlib.TestCase):

tests/router_test.py

@@ -8,6 +8,7 @@ import unittest2
 
 import testlib
 import mitogen.master
+import mitogen.parent
 import mitogen.utils
 
 
@@ -15,6 +16,12 @@ def ping():
     return True
 
 
+@mitogen.core.takes_router
+def ping_context(other, router):
+    other = mitogen.parent.Context(router, other.context_id)
+    other.call(ping)
+
+
 @mitogen.core.takes_router
 def return_router_max_message_size(router):
     return router.max_message_size
@@ -50,7 +57,7 @@ class SourceVerifyTest(testlib.RouterMixin, unittest2.TestCase):
 
         self.broker.defer(self.router._async_route,
                           self.child2_msg,
-                          stream=self.child1_stream)
+                          in_stream=self.child1_stream)
 
         # Wait for IO loop to finish everything above.
         self.sync_with_broker()
@@ -270,5 +277,39 @@ class NoRouteTest(testlib.RouterMixin, testlib.TestCase):
         self.assertEquals(e.args[0], mitogen.core.ChannelError.local_msg)
 
 
+class UnidirectionalTest(testlib.RouterMixin, testlib.TestCase):
+    def test_siblings_cant_talk(self):
+        self.router.unidirectional = True
+        l1 = self.router.fork()
+        l2 = self.router.fork()
+        logs = testlib.LogCapturer()
+        logs.start()
+        e = self.assertRaises(mitogen.core.CallError,
+                              lambda: l2.call(ping_context, l1))
+
+        msg = 'mitogen.core.ChannelError: Channel closed by remote end.'
+        self.assertTrue(msg in str(e))
+        self.assertTrue('routing mode prevents forward of ' in logs.stop())
+
+    def test_auth_id_can_talk(self):
+        self.router.unidirectional = True
+        # One stream has auth_id stamped to that of the master, so it should be
+        # treated like a parent.
+        l1 = self.router.fork()
+        l1s = self.router.stream_by_id(l1.context_id)
+        l1s.auth_id = mitogen.context_id
+        l1s.is_privileged = True
+
+        l2 = self.router.fork()
+        logs = testlib.LogCapturer()
+        logs.start()
+        e = self.assertRaises(mitogen.core.CallError,
+                              lambda: l2.call(ping_context, l1))
+
+        msg = 'mitogen.core.CallError: Refused by policy.'
+        self.assertTrue(msg in str(e))
+        self.assertTrue('policy refused message: ' in logs.stop())
+
+
 if __name__ == '__main__':
     unittest2.main()

tests/testlib.py

@@ -22,6 +22,7 @@ if mitogen.is_master:  # TODO: shouldn't be necessary.
 DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
 sys.path.append(DATA_DIR)
 
+if mitogen.is_master:
     mitogen.utils.log_to_file()
 
 

tests/types_test.py

@@ -0,0 +1,72 @@
+
+import cStringIO
+
+import unittest2
+
+import mitogen.core
+
+
+class BlobTest(unittest2.TestCase):
+    klass = mitogen.core.Blob
+
+    def make(self):
+        return self.klass('x' * 128)
+
+    def test_repr(self):
+        blob = self.make()
+        self.assertEquals('[blob: 128 bytes]', repr(blob))
+
+    def test_decays_on_constructor(self):
+        blob = self.make()
+        self.assertEquals('x'*128, mitogen.core.BytesType(blob))
+
+    def test_decays_on_write(self):
+        blob = self.make()
+        io = cStringIO.StringIO()
+        io.write(blob)
+        self.assertEquals(128, io.tell())
+        self.assertEquals('x'*128, io.getvalue())
+
+    def test_message_roundtrip(self):
+        blob = self.make()
+        msg = mitogen.core.Message.pickled(blob)
+        blob2 = msg.unpickle()
+        self.assertEquals(type(blob), type(blob2))
+        self.assertEquals(repr(blob), repr(blob2))
+        self.assertEquals(mitogen.core.BytesType(blob),
+                          mitogen.core.BytesType(blob2))
+
+
+class SecretTest(unittest2.TestCase):
+    klass = mitogen.core.Secret
+
+    def make(self):
+        return self.klass('password')
+
+    def test_repr(self):
+        secret = self.make()
+        self.assertEquals('[secret]', repr(secret))
+
+    def test_decays_on_constructor(self):
+        secret = self.make()
+        self.assertEquals('password', mitogen.core.UnicodeType(secret))
+
+    def test_decays_on_write(self):
+        secret = self.make()
+        io = cStringIO.StringIO()
+        io.write(secret)
+        self.assertEquals(8, io.tell())
+        self.assertEquals('password', io.getvalue())
+
+    def test_message_roundtrip(self):
+        secret = self.make()
+        msg = mitogen.core.Message.pickled(secret)
+        secret2 = msg.unpickle()
+        self.assertEquals(type(secret), type(secret2))
+        self.assertEquals(repr(secret), repr(secret2))
+        self.assertEquals(mitogen.core.BytesType(secret),
+                          mitogen.core.BytesType(secret2))
+
+
+if __name__ == '__main__':
+    unittest2.main()