Merge remote-tracking branch 'origin/dmw'

d2eb01f tests: pin idna to last supporting 2.6-compat version.
b90889c tests: pin idna to last supporting 2.6-compat version.
9da6e6a tests: don't call Router.shutdown() twice.
b2b7e7b tests: file_service_test fixes
04755c3 issue #426: tighten up PushFileService types.
8fa3c74 issue #426: RouterMonitor format incorrect for 3->2 forward.
d8b9634 issue #426: PushFileService missing to_text() call.
1e9f344 issue #426: big hack so reset_connection has task_var access
18bfde5 issue #444: update Changelog.
835bead tests: allow running scripts from any subdir.
81c93e1 ci: remove duplicate /usr/bin/time call
ca9ae45 issue #426: TemporaryEnvironment must coerce to Unicode.
4bc0d0e issue #426: apply_mode_spec() must handle bytes.
a8921bb tests: fix scaling in fork_histogram
374a361 docs: try to fix CSS difference between local and rtfd
861be2e docs: wrap text around logos
b084d83 docs: fit SVG viewbox to ansible logo
98d06e2 docs: delete shame.rst to make room for new chapters.
5f3244a docs: import pcaps (using LFS) to regenerate charts.
6936b93 tests: import fork_histogram.py.
5a96d13 issue #426: fix all.yml sorting, one more delegate_to
bd82fa1 issue #426: fix low_level_execute_command.yml breakage.
a6e6bc4 issue #426: to_text filter.
d15f533 Turn on Travis build notifcations, but send them to IRC.
9d87f03 issue #426: disable Ansible smart transport.
49d37bf issue #426: remove LANG and LC_ALL during tests.
a6e6fd1 issue #426: more 2->3 test fixes.
67f710f issue #426: use delegate_to in fixup_perms2 and copy.yml
a67a436 docs: add #374 to Changelog.
2b229a6 docs: add thanks entry.

.ci/ansible_install.py

@@ -6,7 +6,7 @@ batches = [
     [
         # Must be installed separately, as PyNACL indirect requirement causes
         # newer version to be installed if done in a single pip run.
-        'pip install "pycparser<2.19"',
+        'pip install "pycparser<2.19" "idna<2.7"',
         'pip install '
             '-r tests/requirements.txt '
             '-r tests/ansible/requirements.txt',

.ci/ansible_tests.py

@@ -56,5 +56,5 @@ with ci_lib.Fold('job_setup'):
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
-    run('/usr/bin/time ./run_ansible_playbook.py %s -i "%s" %s',
+    run('./run_ansible_playbook.py %s -i "%s" %s',
         playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))

.ci/ci_lib.py

@@ -15,6 +15,13 @@ try:
 except ImportError:
     import urllib.parse as urlparse
 
+os.chdir(
+    os.path.join(
+        os.path.dirname(__file__),
+        '..'
+    )
+)
+
 
 #
 # check_output() monkeypatch cutpasted from testlib.py

.ci/mitogen_install.py

@@ -4,7 +4,7 @@ import ci_lib
 
 batches = [
     [
-        'pip install "pycparser<2.19"',
+        'pip install "pycparser<2.19" "idna<2.7"',
         'pip install -r tests/requirements.txt',
     ],
     [

.travis.yml

@@ -2,6 +2,7 @@ sudo: required
 
 notifications:
   email: false
+  irc: "chat.freenode.net#mitogen-builds"
 
 language: python
 

ansible_mitogen/connection.py

@@ -34,6 +34,7 @@ import os
 import pprint
 import random
 import stat
+import sys
 import time
 
 import jinja2.runtime
@@ -842,6 +843,19 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         self._mitogen_reset(mode='put')
         self._shutdown_broker()
 
+    def _reset_find_task_vars(self):
+        """
+        Monsterous hack: since "meta: reset_connection" does not run from an
+        action, we cannot capture task variables via :meth:`on_action_run`.
+        Instead walk the parent frames searching for the `all_vars` local from
+        StrategyBase._execute_meta(). If this fails, just leave task_vars
+        unset, likely causing the wrong configuration to be created.
+        """
+        frame = sys._getframe()
+        while frame and not self._task_vars:
+            self._task_vars = frame.f_locals.get('all_vars')
+            frame = frame.f_back
+
     reset_compat_msg = (
         'Mitogen only supports "reset_connection" on Ansible 2.5.6 or later'
     )
@@ -853,6 +867,9 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         the 'disconnected' state, and informs ContextService the connection is
         bad somehow, and should be shut down and discarded.
         """
+        if self._task_vars is None:
+            self._reset_find_task_vars()
+
         if self._play_context.remote_addr is None:
             # <2.5.6 incorrectly populate PlayContext for reset_connection
             # https://github.com/ansible/ansible/issues/27520

ansible_mitogen/planner.py

@@ -172,7 +172,7 @@ class BinaryPlanner(Planner):
         return module_common._is_binary(self._inv.module_source)
 
     def get_push_files(self):
-        return [self._inv.module_path]
+        return [mitogen.core.to_text(self._inv.module_path)]
 
     def get_kwargs(self, **kwargs):
         return super(BinaryPlanner, self).get_kwargs(
@@ -285,7 +285,7 @@ class NewStylePlanner(ScriptPlanner):
 
     def get_push_files(self):
         return super(NewStylePlanner, self).get_push_files() + [
-            path
+            mitogen.core.to_text(path)
             for fullname, path, is_pkg in self.get_module_map()['custom']
         ]
 

ansible_mitogen/runner.py

@@ -454,6 +454,8 @@ class TemporaryEnvironment(object):
         self.original = dict(os.environ)
         self.env = env or {}
         for key, value in iteritems(self.env):
+            key = mitogen.core.to_text(key)
+            value = mitogen.core.to_text(value)
             if value is None:
                 os.environ.pop(key, None)
             else:

ansible_mitogen/services.py

@@ -492,14 +492,16 @@ class ModuleDepService(mitogen.service.Service):
 
     def _get_builtin_names(self, builtin_path, resolved):
         return [
-            fullname
+            mitogen.core.to_text(fullname)
             for fullname, path, is_pkg in resolved
             if os.path.abspath(path).startswith(builtin_path)
         ]
 
     def _get_custom_tups(self, builtin_path, resolved):
         return [
-            (fullname, path, is_pkg)
+            (mitogen.core.to_text(fullname),
+             mitogen.core.to_text(path),
+             is_pkg)
             for fullname, path, is_pkg in resolved
             if not os.path.abspath(path).startswith(builtin_path)
         ]

ansible_mitogen/target.py

@@ -669,7 +669,7 @@ def apply_mode_spec(spec, mode):
     Given a symbolic file mode change specification in the style of chmod(1)
     `spec`, apply changes in the specification to the numeric file mode `mode`.
     """
-    for clause in spec.split(','):
+    for clause in mitogen.core.to_text(spec).split(','):
         match = CHMOD_CLAUSE_PAT.match(clause)
         who, op, perms = match.groups()
         for ch in who or 'a':

docs/_static/style.css

@@ -12,6 +12,17 @@ div.body li {
 }
 
 
+/*
+ * Undo the hyphens: auto in Sphinx basic.css.
+ */
+div.body p, div.body dd, div.body li, div.body blockquote {
+    -moz-hyphens: inherit;
+    -ms-hyphens: inherit;
+    -webkit-hyphens: inherit;
+    hyphens: inherit;
+}
+
+
 
 /*
  * Setting :width; on an image causes Sphinx to turn the image into a link, so
@@ -27,6 +38,12 @@ div.body li {
     width: 150px;
 }
 
+.mitogen-right-200 {
+    float: right;
+    padding-left: 8px;
+    width: 200px;
+}
+
 .mitogen-right-225 {
     float: right;
     padding-left: 8px;
@@ -50,3 +67,10 @@ div.body li {
     padding-left: 8px;
     width: 350px;
 }
+
+.mitogen-logo-wrap {
+    shape-margin: 8px;
+    shape-outside: polygon(
+        100% 0, 50% 10%, 24% 24%, 0% 50%, 24% 75%, 50% 90%, 100% 100%
+    );
+}

docs/ansible.rst

@@ -1,11 +1,10 @@
 
-.. image:: images/ansible/ansible_mitogen.svg
-    :class: mitogen-right-225
-
-
 Mitogen for Ansible
 ===================
 
+.. image:: images/ansible/ansible_mitogen.svg
+    :class: mitogen-right-200 mitogen-logo-wrap
+
 An extension to `Ansible`_ is included that implements connections over
 Mitogen, replacing embedded shell invocations with pure-Python equivalents
 invoked via highly efficient remote procedure calls to persistent interpreters

docs/changelog.rst

@@ -180,9 +180,10 @@ Fixes
   print a useful hint on failure, as no useful error is normally logged to the
   console by these tools.
 
-* `#391 <https://github.com/dw/mitogen/issues/391>`_: file transfer from 2.x
+* `#374 <https://github.com/dw/mitogen/issues/374>`_,
-  controllers to 3.x targets was broken due to a regression caused by
+  `#391 <https://github.com/dw/mitogen/issues/391>`_: file transfer and module
-  refactoring, and compounded by `#426
+  execution from 2.x controllers to 3.x targets was broken due to a regression
+  caused by refactoring, and compounded by `#426
   <https://github.com/dw/mitogen/issues/426>`_.
 
 * `#400 <https://github.com/dw/mitogen/issues/400>`_: work around a threading
@@ -289,6 +290,9 @@ Core Library
 * `#439 <https://github.com/dw/mitogen/issues/439>`_: descriptive errors are
   raised when attempting to invoke unsupported function types.
 
+* `#444 <https://github.com/dw/mitogen/issues/444>`_: messages regarding
+  unforwardable extension module are no longer logged as errors.
+
 * `#453 <https://github.com/dw/mitogen/issues/453>`_: the loggers used in
   children for standard IO redirection have propagation disabled, preventing
   accidental reconfiguration of the :mod:`logging` package in a child from
@@ -328,6 +332,7 @@ bug reports, testing, features and fixes in this release contributed by
 `Berend De Schouwer <https://github.com/berenddeschouwer>`_,
 `Brian Candler <https://github.com/candlerb>`_,
 `Duane Zamrok <https://github.com/dewthefifth>`_,
+`Eric Chang <https://github.com/changchichung>`_,
 `Guy Knights <https://github.com/knightsg>`_,
 `Jiří Vávra <https://github.com/Houbovo>`_,
 `Jonathan Rosser <https://github.com/jrosser>`_,

docs/index.rst

@@ -2,8 +2,6 @@
 Mitogen
 =======
 
-Mitogen is a Python library for writing distributed self-replicating programs.
-
 .. raw:: html
 
     <style>
@@ -13,7 +11,9 @@ Mitogen is a Python library for writing distributed self-replicating programs.
     </style>
 
 .. image:: images/mitogen.svg
-    :class: mitogen-right-225
+    :class: mitogen-right-200 mitogen-logo-wrap
+
+Mitogen is a Python library for writing distributed self-replicating programs.
 
 There is no requirement for installing packages, copying files around, writing
 shell snippets, upfront configuration, or providing any secondary link to a

docs/pcaps/.gitattributes

@@ -0,0 +1,3 @@
+*.pcapng filter=lfs diff=lfs merge=lfs -text
+run_hostname_100_times_mito.pcap.gz filter=lfs diff=lfs merge=lfs -text
+run_hostname_100_times_vanilla.pcap.gz filter=lfs diff=lfs merge=lfs -text

docs/pcaps/run_hostname_100_times_mito.pcap.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d9b4d4ff263003bd16e44c265783e7c1deff19950e453e3adeb8a6ab5052081
+size 175120

docs/pcaps/run_hostname_100_times_vanilla.pcap.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a993832501b7948a38c2e8ced3467d1cb04279a57ddd6afc735ed96ec509c08
+size 7623337

docs/shame.rst

@@ -1,90 +0,0 @@
-
-Importer Wall Of Shame
-----------------------
-
-The following modules and packages violate protocol or best practice in some way:
-
-* They run magic during ``__init.py__`` that makes life hard for Mitogen.
-  Executing code during module import is always bad, and Mitogen is a concrete
-  benchmark for why it's bad.
-
-* They install crap in :py:data:`sys.modules` that completely ignore or
-  partially implement the protocols laid out in PEP-302.
-
-* They "vendor" a third party package, either incompletely, using hacks visible
-  through the runtime's standard interfaces, or with ancient versions of code
-  that in turn mess with :py:data:`sys.modules` in some horrible way.
-
-Bugs will probably be filed for these in time, but it does not address the huge
-installed base of existing old software versions, so hacks are needed anyway.
-
-
-``pbr``
-=======
-
-It claims to use ``pkg_resources`` to read version information
-(``_get_version_from_pkg_metadata()``), which would result in PEP-302 being
-reused and everything just working wonderfully, but instead it actually does
-direct filesystem access.
-
-**What could it do instead?**
-
-* ``pkg_resources.resource_stream()``
-
-**What Mitogen is forced to do**
-
-When it sees ``pbr`` being loaded, it smodges the process environment with a
-``PBR_VERSION`` variable to override any attempt to auto-detect the version.
-This will probably break code I haven't seen yet.
-
-
-``pkg_resources``
-=================
-
-Anything that imports ``pkg_resources`` will eventually cause ``pkg_resources``
-to try and import and scan ``__main__`` for its ``__requires__`` attribute
-(``pkg_resources/__init__.py::_build_master()``). This breaks any app that is
-not expecting its ``__main__`` to suddenly be sucked over a network and
-injected into a remote process, like py.test.
-
-A future version of Mitogen might have a more general hack that doesn't import
-the master's ``__main__`` as ``__main__`` in the slave, avoiding all kinds of
-issues like these.
-
-**What could it do instead?**
-
-* Explicit is better than implicit: wait until the magical behaviour is
-  explicitly requested (i.e. an API call).
-
-* Use ``get("__main__")`` on :py:data:`sys.modules` rather than ``import``, but
-  this method isn't general enough, it only really helps tools like Mitogen.
-
-**What Mitogen is forced to do**
-
-Examine the stack during every attempt to import ``__main__`` and check if the
-requestee module is named ``pkg_resources``, if so then refuse the import.
-
-
-``six``
-=======
-
-The ``six`` module makes some effort to conform to PEP-302, but it is missing
-several critical pieces, e.g. the ``__loader__`` attribute. This not only
-breaks the Python standard library tooling (such as the :py:mod:`inspect`
-module), but also Mitogen. Newer versions of ``six`` improve things somewhat,
-but there are still outstanding issues preventing Mitogen from working with
-``six``.
-
-This package is sufficiently popular that it must eventually be supported. See
-`here for an example issue`_.
-
-.. _here for an example issue: https://github.com/dw/mitogen/issues/31
-
-**What could it do instead?**
-
-* Any custom hacks installed into :py:data:`sys.modules` should support the
-  protocols laid out in PEP-302.
-
-**What Mitogen is forced to do**
-
-Vendored versions of ``six`` currently don't work at all.

docs/toc.rst

@@ -14,7 +14,6 @@ Table Of Contents
    api
    examples
    internals
-   shame
 
 .. toctree::
    :hidden:

mitogen/parent.py

@@ -1626,7 +1626,7 @@ class RouteMonitor(object):
         """
         data = str(target_id)
         if name:
-            data = '%s:%s' % (target_id, mitogen.core.b(name))
+            data = '%s:%s' % (target_id, name)
         stream.send(
             mitogen.core.Message(
                 handle=handle,

mitogen/service.py

@@ -594,6 +594,7 @@ class PushFileService(Service):
         self._sent_by_stream = {}
 
     def get(self, path):
+        assert isinstance(path, mitogen.core.UnicodeType)
         self._lock.acquire()
         try:
             if path in self._cache:
@@ -641,7 +642,7 @@ class PushFileService(Service):
         with a set of small files and Python modules.
         """
         for path in paths:
-            self.propagate_to(context, path)
+            self.propagate_to(context, mitogen.core.to_text(path))
         self.router.responder.forward_modules(context, modules)
 
     @expose(policy=AllowParents())
@@ -670,7 +671,7 @@ class PushFileService(Service):
     @expose(policy=AllowParents())
     @no_reply()
     @arg_spec({
-        'path': mitogen.core.FsPathTypes,
+        'path': mitogen.core.UnicodeType,
         'data': mitogen.core.Blob,
         'context': mitogen.core.Context,
     })

tests/ansible/ansible.cfg

@@ -7,11 +7,15 @@ callback_plugins = lib/callback
 stdout_callback = nice_stdout
 vars_plugins = lib/vars
 library = lib/modules
+filter_plugins = lib/filters
 module_utils = lib/module_utils
 retry_files_enabled = False
 display_args_to_stdout = True
 forks = 100
 
+# On MacOS, "smart" with a password set causes Ansible to use paramiko.
+transport = ssh
+
 no_target_syslog = True
 
 # Required by integration/ssh/timeouts.yml

tests/ansible/integration/action/copy.yml

@@ -8,13 +8,13 @@
         dest: /tmp/copy-tiny-file
         content:
           this is a tiny file.
-      connection: local
+      delegate_to: localhost
 
     - copy:
         dest: /tmp/copy-large-file
         # Must be larger than Connection.SMALL_SIZE_LIMIT.
         content: "{% for x in range(200000) %}x{% endfor %}"
-      connection: local
+      delegate_to: localhost
 
     # end of making files
 

tests/ansible/integration/action/fixup_perms2__copy.yml

@@ -53,7 +53,7 @@
         state: absent
 
     - name: Create local test file.
-      connection: local
+      delegate_to: localhost
       copy:
         content: "weird mode"
         dest: "/tmp/weird-mode"

tests/ansible/integration/action/low_level_execute_command.yml

@@ -1,6 +1,6 @@
 # Verify the behaviour of _low_level_execute_command().
 
-- name: integration/action__low_level_execute_command.yml
+- name: integration/action/low_level_execute_command.yml
   hosts: test-targets
   any_errors_fatal: true
   tasks:
@@ -14,8 +14,8 @@
       assert:
         that:
           - 'raw.rc == 0'
-          - 'raw.stdout_lines == ["2"]'
+          - 'raw.stdout_lines[-1]|to_text == "2"'
-          - 'raw.stdout == "2"'
+          - 'raw.stdout[-1]|to_text == "2"'
 
     - name: Run raw module with sudo
       become: true
@@ -27,6 +27,12 @@
       assert:
         that:
         - raw.rc == 0
-        # WHY DOES VANILLA ANSIBLE INSERT NEWLINES HERE!?!?!?!?!?!ONE
+        # WHY DOES VANILLA INSERT NEWLINES HERE!?!?!?!?!?!ONE
-        - raw.stdout in ("\r\nroot\r\n", "root\r\n")
+        #- raw.stdout in ("\r\nroot\r\n", "root\r\n")
-        - raw.stdout_lines in (["", "root"], ["root"])
+        - '(raw.stdout|to_text).endswith("root\r\n")'
+        - |
+          raw.stdout_lines|to_text in (
+            ["\r\n"],
+            ["root\r\n"],
+            ["root"],
+          )

tests/ansible/integration/action/synchronize.yml

@@ -13,22 +13,22 @@
         dest: /tmp/synchronize-action-key
         src: ../../../data/docker/mitogen__has_sudo_pubkey.key
         mode: u=rw,go=
-      connection: local
+      delegate_to: localhost
 
     - file:
         path: /tmp/sync-test
         state: absent
-      connection: local
+      delegate_to: localhost
 
     - file:
         path: /tmp/sync-test
         state: directory
-      connection: local
+      delegate_to: localhost
 
     - copy:
         dest: /tmp/sync-test/item
         content: "item!"
-      connection: local
+      delegate_to: localhost
 
     - file:
         path: /tmp/sync-test.out

tests/ansible/integration/all.yml

@@ -7,7 +7,6 @@
 - import_playbook: async/all.yml
 - import_playbook: become/all.yml
 - import_playbook: connection/all.yml
-- import_playbook: stub_connections/all.yml
 - import_playbook: connection_loader/all.yml
 - import_playbook: context_service/all.yml
 - import_playbook: delegation/all.yml
@@ -18,3 +17,4 @@
 - import_playbook: runner/all.yml
 - import_playbook: ssh/all.yml
 - import_playbook: strategy/all.yml
+- import_playbook: stub_connections/all.yml

tests/ansible/integration/connection/_put_file.yml

@@ -3,7 +3,7 @@
 - shell: dd if=/dev/urandom of=/tmp/{{file_name}} bs=1024 count={{file_size}}
   args:
     creates: /tmp/{{file_name}}
-  connection: local
+  delegate_to: localhost
 
 - copy:
     dest: /tmp/{{file_name}}.out
@@ -11,7 +11,7 @@
 
 - stat: path=/tmp/{{file_name}}
   register: original
-  connection: local
+  delegate_to: localhost
 
 - stat: path=/tmp/{{file_name}}.out
   register: copied
@@ -19,4 +19,5 @@
 - assert:
     that:
     - original.stat.checksum == copied.stat.checksum
-    - (not is_mitogen) or (original.stat.mtime|int == copied.stat.mtime|int)
+    # Upstream does not preserve timestamps at al.
+    #- (not is_mitogen) or (original.stat.mtime|int == copied.stat.mtime|int)

tests/ansible/integration/connection/disconnect_during_module.yml

@@ -9,7 +9,7 @@
   - meta: end_play
     when: not is_mitogen
 
-  - connection: local
+  - delegate_to: localhost
     command: |
       ansible-playbook
       -i "{{inventory_file}}"

tests/ansible/integration/connection_loader/local_blemished.yml

@@ -1,13 +1,13 @@
 # Ensure 'local' connections are grabbed.
 
-- name: integration/connection_loader__local_blemished.yml
+- name: integration/connection_loader/local_blemished.yml
   hosts: test-targets
   any_errors_fatal: true
   tasks:
     - determine_strategy:
 
     - custom_python_detect_environment:
-      connection: local
+      delegate_to: localhost
       register: out
 
     - assert:

tests/ansible/lib/callback/fork_histogram.py

@@ -0,0 +1,92 @@
+
+# Monkey-patch os.fork() to produce a latency histogram on run completion.
+# Requires 'hdrhsitograms' PyPI module.
+
+from __future__ import print_function
+
+import os
+import resource
+import sys
+import time
+
+import ansible.plugins.callback
+import hdrh.histogram
+
+
+def get_fault_count(who=resource.RUSAGE_CHILDREN):
+    ru = resource.getrusage(who)
+    return ru.ru_minflt + ru.ru_majflt
+
+
+class CallbackModule(ansible.plugins.callback.CallbackBase):
+    hist = None
+
+    def v2_playbook_on_start(self, playbook):
+        if self.hist is not None:
+            return
+
+        self.hist = hdrh.histogram.HdrHistogram(1, int(1e6*60), 3)
+        self.fork_latency_sum_usec = 0.0
+        self.install()
+
+    def install(self):
+        self.faults_at_start = get_fault_count(resource.RUSAGE_SELF)
+        self.run_start_time = time.time()
+        self.real_fork = os.fork
+        os.fork = self.my_fork
+
+    self_fault_usec = 1.113664156753052
+    child_fault_usec = 4.734975610975617
+
+    dummy_heap_size = int(os.environ.get('FORK_STATS_FAKE_HEAP_MB', '0'))
+    dummy_heap = 'x' * (dummy_heap_size * 1048576)
+
+    def my_fork(self):
+        # doesnt count last child, oh well
+        now_faults = get_fault_count()
+        t0 = time.time()
+        try:
+            return self.real_fork()
+        finally:
+            latency_usec = (1e6 * (time.time() - t0))
+            self.fork_latency_sum_usec += latency_usec
+            self.hist.record_value(latency_usec)
+
+    def playbook_on_stats(self, stats):
+        self_faults = get_fault_count(resource.RUSAGE_SELF) - self.faults_at_start
+        child_faults = get_fault_count()
+        run_duration_sec = time.time() - self.run_start_time
+        fault_wastage_usec = (
+            ((self.self_fault_usec * self_faults) +
+             (self.child_fault_usec * child_faults))
+        )
+        fork_wastage = self.hist.get_total_count()
+        all_wastage_usec = ((2*self.fork_latency_sum_usec) + fault_wastage_usec)
+
+        print('--- Fork statistics ---')
+        print('Post-boot run duration: %.02f ms, %d total forks' % (
+            1000 * run_duration_sec,
+            self.hist.get_total_count(),
+        ))
+        print('Self faults during boot: %d, post-boot: %d, avg %d/child' % (
+            self.faults_at_start,
+            self_faults,
+            self_faults / self.hist.get_total_count(),
+        ))
+        print('Total child faults: %d, avg %d/child' % (
+            child_faults,
+            child_faults / self.hist.get_total_count(),
+        ))
+        print('Est. wastage on faults: %d ms, forks+faults+waits: %d ms (%.2f%%)' % (
+            fault_wastage_usec / 1000,
+            all_wastage_usec / 1000,
+            100 * (all_wastage_usec / (run_duration_sec * 1e6)),
+        ))
+        print('99th%% fork latency: %.03f msec, max %d new tasks/sec' % (
+            self.hist.get_value_at_percentile(99) / 1000.0,
+            1e6 / self.hist.get_value_at_percentile(99),
+        ))
+
+        self.hist.output_percentile_distribution(sys.stdout, 1000)
+        print('--- End fork statistics ---')
+        print()

tests/ansible/lib/filters/mitogen_tests.py

@@ -0,0 +1,27 @@
+
+from ansible.module_utils._text import to_text
+
+
+try:
+    Unicode = unicode
+except:
+    Unicode = str
+
+
+def to_text(s):
+    """
+    Ensure the str or unicode `s` is unicode, and strip away any subclass. Also
+    works on lists.
+    """
+    if isinstance(s, list):
+        return [to_text(ss) for ss in s]
+    if not isinstance(s, Unicode):
+        s = to_text(s)
+    return Unicode(s)
+
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'to_text': to_text,
+        }

tests/ansible/requirements.txt

@@ -2,5 +2,6 @@ ansible; python_version >= '2.7'
 ansible<2.7; python_version < '2.7'
 paramiko==2.3.2  # Last 2.6-compat version.
 google-api-python-client==1.6.5
+hdrhistogram==0.6.1
 PyYAML==3.11; python_version < '2.7'
 PyYAML==3.12; python_version >= '2.7'

tests/ansible/run_ansible_playbook.py

@@ -18,6 +18,12 @@ os.environ.setdefault(
     os.path.dirname(os.path.dirname(sys.executable))
 )
 
+# Set LANG and LC_ALL to C in order to avoid locale errors spammed by vanilla
+# during exec_command().
+os.environ.pop('LANG', None)
+os.environ.pop('LC_ALL', None)
+
+
 # Used by delegate_to.yml to ensure "sudo -E" preserves environment.
 os.environ['I_WAS_PRESERVED'] = '1'
 

tests/call_function_test.py

@@ -97,6 +97,7 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase):
         recv = self.local.call_async(time.sleep, 120)
         time.sleep(0.05)  # Ensure GIL is released
         self.broker.shutdown()
+        self.broker_shutdown = True
         exc = self.assertRaises(mitogen.core.ChannelError,
             lambda: recv.get())
         self.assertEquals(exc.args[0], self.router.respondent_disconnect_msg)

tests/file_service_test.py

@@ -9,72 +9,100 @@ import testlib
 class FetchTest(testlib.RouterMixin, testlib.TestCase):
     klass = mitogen.service.FileService
 
+    def replyable_msg(self, **kwargs):
+        recv = mitogen.core.Receiver(self.router, persist=False)
+        msg = mitogen.core.Message(
+            src_id=mitogen.context_id,
+            reply_to=recv.handle,
+            **kwargs
+        )
+        msg.router = self.router
+        return recv, msg
+
     def test_unauthorized(self):
         service = self.klass(self.router)
-        e = self.assertRaises(mitogen.service.Error,
+        recv, msg = self.replyable_msg()
-            lambda: service.fetch(
+        service.fetch(
-                path='/etc/shadow',
+            path='/etc/shadow',
-                sender=None,
+            sender=None,
-                msg=mitogen.core.Message(),
+            msg=msg,
-            )
         )
-
+        e = self.assertRaises(mitogen.core.CallError,
-        self.assertEquals(e.args[0], service.unregistered_msg)
+            lambda: recv.get().unpickle())
+        expect = service.unregistered_msg % ('/etc/shadow',)
+        self.assertTrue(expect in e.args[0])
+
+    def _validate_response(self, resp):
+        self.assertTrue(isinstance(resp, dict))
+        self.assertEquals('root', resp['owner'])
+        self.assertEquals('root', resp['group'])
+        self.assertTrue(isinstance(resp['mode'], int))
+        self.assertTrue(isinstance(resp['mtime'], float))
+        self.assertTrue(isinstance(resp['atime'], float))
+        self.assertTrue(isinstance(resp['size'], int))
 
     def test_path_authorized(self):
         recv = mitogen.core.Receiver(self.router)
         service = self.klass(self.router)
         service.register('/etc/passwd')
-        self.assertEquals(None, service.fetch(
+        recv, msg = self.replyable_msg()
+        service.fetch(
             path='/etc/passwd',
             sender=recv.to_sender(),
-            msg=mitogen.core.Message(),
+            msg=msg,
-        ))
+        )
+        self._validate_response(recv.get().unpickle())
 
     def test_root_authorized(self):
         recv = mitogen.core.Receiver(self.router)
         service = self.klass(self.router)
         service.register_prefix('/')
-        self.assertEquals(None, service.fetch(
+        recv, msg = self.replyable_msg()
+        service.fetch(
             path='/etc/passwd',
             sender=recv.to_sender(),
-            msg=mitogen.core.Message(),
+            msg=msg,
-        ))
+        )
+        self._validate_response(recv.get().unpickle())
 
     def test_prefix_authorized(self):
         recv = mitogen.core.Receiver(self.router)
         service = self.klass(self.router)
         service.register_prefix('/etc')
-        self.assertEquals(None, service.fetch(
+        recv, msg = self.replyable_msg()
+        service.fetch(
             path='/etc/passwd',
             sender=recv.to_sender(),
-            msg=mitogen.core.Message(),
+            msg=msg,
-        ))
+        )
+        self._validate_response(recv.get().unpickle())
 
     def test_prefix_authorized_abspath_bad(self):
         recv = mitogen.core.Receiver(self.router)
         service = self.klass(self.router)
         service.register_prefix('/etc')
-        self.assertEquals(None, service.fetch(
+        recv, msg = self.replyable_msg()
+        service.fetch(
             path='/etc/foo/bar/../../../passwd',
             sender=recv.to_sender(),
-            msg=mitogen.core.Message(),
+            msg=msg,
-        ))
+        )
+        self.assertEquals(None, recv.get().unpickle())
 
     def test_prefix_authorized_abspath_bad(self):
         recv = mitogen.core.Receiver(self.router)
         service = self.klass(self.router)
         service.register_prefix('/etc')
-        e = self.assertRaises(mitogen.service.Error,
+        recv, msg = self.replyable_msg()
-            lambda: service.fetch(
+        service.fetch(
-                path='/etc/../shadow',
+            path='/etc/../shadow',
-                sender=recv.to_sender(),
+            sender=recv.to_sender(),
-                msg=mitogen.core.Message(),
+            msg=msg,
-            )
         )
-
+        e = self.assertRaises(mitogen.core.CallError,
-        self.assertEquals(e.args[0], service.unregistered_msg)
+            lambda: recv.get().unpickle())
-
+        expect = service.unregistered_msg % ('/etc/../shadow',)
+        self.assertTrue(expect in e.args[0])
 
 
 if __name__ == '__main__':

tests/testlib.py

@@ -388,13 +388,15 @@ class DockerizedSshDaemon(object):
 
 class BrokerMixin(object):
     broker_class = mitogen.master.Broker
+    broker_shutdown = False
 
     def setUp(self):
         super(BrokerMixin, self).setUp()
         self.broker = self.broker_class()
 
     def tearDown(self):
-        self.broker.shutdown()
+        if not self.broker_shutdown:
+            self.broker.shutdown()
         self.broker.join()
         super(BrokerMixin, self).tearDown()