Initial commit.

.gitignore

@@ -0,0 +1,2 @@
+docs/_build
+*.egg-info

docs/Makefile

@@ -0,0 +1,71 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: changes
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."

docs/_static/style.css

@@ -0,0 +1,4 @@
+
+div.figure {
+    padding: 0;
+}

docs/_templates/github.html

@@ -0,0 +1,4 @@
+<p>
+<br>
+<a href="https://github.com/dw/mitogen/">GitHub Repository</a>
+</p>

docs/_templates/layout.html

@@ -0,0 +1,2 @@
+{% extends "!layout.html" %}
+{% set css_files = css_files + ['_static/style.css'] %}

docs/api.rst

@@ -0,0 +1,92 @@
+
+API Reference
+*************
+
+
+Package Layout
+==============
+
+
+mitogen Package
+---------------
+
+.. automodule:: mitogen
+
+.. autodata:: mitogen.slave
+.. autodata:: mitogen.context_id
+.. autodata:: mitogen.parent_id
+
+
+mitogen.core
+------------
+
+.. automodule:: mitogen.core
+
+
+mitogen.master
+--------------
+
+.. automodule:: mitogen.master
+
+
+mitogen.fakessh
+---------------
+
+.. automodule:: mitogen.fakessh
+
+.. autofunction:: mitogen.fakessh.run
+
+
+Router Class
+============
+
+.. autoclass:: mitogen.master.Router
+   :members:
+   :inherited-members:
+
+
+Broker Class
+============
+
+.. autoclass:: mitogen.master.Broker
+   :members:
+   :inherited-members:
+
+
+Context Class
+=============
+
+.. autoclass:: mitogen.master.Context
+   :members:
+   :inherited-members:
+
+
+Channel Class
+-------------
+
+.. autoclass:: mitogen.core.Channel
+   :members:
+
+
+Context Class
+-------------
+
+.. autoclass:: mitogen.master.Context
+   :members:
+
+
+Utility Functions
+=================
+
+.. automodule:: mitogen.utils
+  :members:
+
+
+Exceptions
+==========
+
+.. autoclass:: mitogen.core.Error
+.. autoclass:: mitogen.core.CallError
+.. autoclass:: mitogen.core.ChannelError
+.. autoclass:: mitogen.core.StreamError
+.. autoclass:: mitogen.core.TimeoutError

docs/conf.py

@@ -0,0 +1,23 @@
+import sys
+sys.path.append('..')
+
+author = u'David Wilson'
+copyright = u'2016, David Wilson'
+exclude_patterns = ['_build']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+html_show_sourcelink = False
+html_show_sphinx = False
+html_sidebars = {'**': ['globaltoc.html', 'github.html']}
+html_static_path = ['_static']
+html_theme = 'alabaster'
+htmlhelp_basename = 'mitogendoc'
+intersphinx_mapping = {'python': ('https://docs.python.org/2', None)}
+language = None
+master_doc = 'toc'
+project = u'Mitogen'
+pygments_style = 'sphinx'
+release = u'master'
+source_suffix = '.rst'
+templates_path = ['_templates']
+todo_include_todos = False
+version = u'master'

docs/examples.rst

@@ -0,0 +1,67 @@
+
+Examples
+========
+
+
+Recursively Nested Bootstrap
+----------------------------
+
+This demonstrates the library's ability to use slave contexts to recursively
+proxy connections to additional slave contexts, with a uniform API to any
+slave, and all features (function calls, import forwarding, stdio forwarding,
+log forwarding) functioning transparently.
+
+This example uses a chain of local contexts for clarity, however SSH and sudo
+contexts work identically.
+
+nested.py:
+
+.. code-block:: python
+
+    import os
+    import mitogen.utils
+
+    @mitogen.utils.run_with_router
+    def main(router):
+        mitogen.utils.log_to_file()
+
+        context = None
+        for x in range(1, 11):
+            print 'Connect local%d via %s' % (x, context)
+            context = router.local(via=context, name='local%d' % x)
+
+        context.call(os.system, 'pstree -s python -s mitogen')
+
+
+Output:
+
+.. code-block:: shell
+
+    $ python nested.py
+    Connect local1 via None
+    Connect local2 via Context(1, 'local1')
+    Connect local3 via Context(2, 'local2')
+    Connect local4 via Context(3, 'local3')
+    Connect local5 via Context(4, 'local4')
+    Connect local6 via Context(5, 'local5')
+    Connect local7 via Context(6, 'local6')
+    Connect local8 via Context(7, 'local7')
+    Connect local9 via Context(8, 'local8')
+    Connect local10 via Context(9, 'local9')
+    18:14:07 I ctx.local10: stdout: -+= 00001 root /sbin/launchd
+    18:14:07 I ctx.local10: stdout:  \-+= 08126 dmw /Applications/iTerm.app/Contents/MacOS/iTerm2
+    18:14:07 I ctx.local10: stdout:    \-+= 10638 dmw /Applications/iTerm.app/Contents/MacOS/iTerm2 --server bash --login
+    18:14:07 I ctx.local10: stdout:      \-+= 10639 dmw bash --login
+    18:14:07 I ctx.local10: stdout:        \-+= 13632 dmw python nested.py
+    18:14:07 I ctx.local10: stdout:          \-+- 13633 dmw mitogen:dmw@Eldil.local:13632
+    18:14:07 I ctx.local10: stdout:            \-+- 13635 dmw mitogen:dmw@Eldil.local:13633
+    18:14:07 I ctx.local10: stdout:              \-+- 13637 dmw mitogen:dmw@Eldil.local:13635
+    18:14:07 I ctx.local10: stdout:                \-+- 13639 dmw mitogen:dmw@Eldil.local:13637
+    18:14:07 I ctx.local10: stdout:                  \-+- 13641 dmw mitogen:dmw@Eldil.local:13639
+    18:14:07 I ctx.local10: stdout:                    \-+- 13643 dmw mitogen:dmw@Eldil.local:13641
+    18:14:07 I ctx.local10: stdout:                      \-+- 13645 dmw mitogen:dmw@Eldil.local:13643
+    18:14:07 I ctx.local10: stdout:                        \-+- 13647 dmw mitogen:dmw@Eldil.local:13645
+    18:14:07 I ctx.local10: stdout:                          \-+- 13649 dmw mitogen:dmw@Eldil.local:13647
+    18:14:07 I ctx.local10: stdout:                            \-+- 13651 dmw mitogen:dmw@Eldil.local:13649
+    18:14:07 I ctx.local10: stdout:                              \-+- 13653 dmw pstree -s python -s mitogen
+    18:14:07 I ctx.local10: stdout:                                \--- 13654 root ps -axwwo user,pid,ppid,pgid,command

docs/getting_started.rst

@@ -0,0 +1,5 @@
+
+Getting Started
+===============
+
+xxx

docs/howitworks.rst

@@ -0,0 +1,511 @@
+
+How Mitogen Works
+=================
+
+Some effort is required to accomplish the seemingly magical feat of
+bootstrapping a remote Python process without any software installed on the
+remote machine. The steps involved are unlikely to be immediately obvious to
+the casual reader, and they required several iterations to discover, so we
+document them thoroughly below.
+
+
+The UNIX First Stage
+--------------------
+
+To allow delivery of the bootstrap compressed using :py:mod:`zlib`, it is
+necessary for something on the remote to be prepared to decompress the payload
+and feed it to a Python interpreter. Since we would like to avoid writing an
+error-prone shell fragment to implement this, and since we must avoid writing
+to the remote machine's disk in case it is read-only, the Python process
+started on the remote machine by Mitogen immediately forks in order to
+implement the decompression.
+
+
+Python Command Line
+###################
+
+The Python command line sent to the host is a base64-encoded copy of the
+:py:meth:`mitogen.master.LocalStream._first_stage` function, which has been
+carefully optimized to reduce its size. Prior to base64 encoding,
+``CONTEXT_NAME`` is replaced with the desired context name in the function's
+source code.
+
+.. code::
+
+    python -c 'exec "xxx".decode("base64")'
+
+The command-line arranges for the Python interpreter to decode the base64'd
+component and execute it as Python code. Base64 is used since the first stage
+implementation contains newlines, and many special characters that may be
+interpreted by the system shell in use.
+
+
+Forking The First Stage
+#######################
+
+The first stage creates a UNIX pipe and saves a copy of the process's real
+``stdin`` file descriptor (used for communication with the master) so that it
+can be recovered by the bootstrapped process later. It then forks into a new
+process.
+
+After fork, the parent half overwrites its ``stdin`` with the read end of the
+pipe, and the child half writes the string ``EC0\n``, then begins reading the
+:py:mod:`zlib`-compressed payload supplied on ``stdin`` by the master, and
+writing the decompressed result to the write-end of the UNIX pipe.
+
+To allow recovery of ``stdin`` for reuse by the bootstrapped process for
+master<->slave communication, it is necessary for the first stage to avoid
+closing ``stdin`` or reading from it until until EOF. Therefore, the master
+sends the :py:mod:`zlib`-compressed payload prefixed with an integer size,
+allowing reading by the first stage of exactly the required bytes.
+
+
+Configuring argv[0]
+###################
+
+Forking provides us with an excellent opportunity for tidying up the eventual
+Python interpreter, in particular, restarting it using a fresh command-line to
+get rid of the large base64-encoded first stage parameter, and to replace
+**argv[0]** with something descriptive.
+
+After configuring its ``stdin`` to point to the read end of the pipe, the
+parent half of the fork re-executes Python, with **argv[0]** taken from the
+``CONTEXT_NAME`` variable earlier substituted into its source code. As no
+arguments are provided to this new execution of Python, and since ``stdin`` is
+connected to a pipe (whose write end is connected to the first stage), the
+Python interpreter begins reading source code to execute from the pipe
+connected to ``stdin``.
+
+
+Bootstrap Preparation
+#####################
+
+Now we have the mechanism in place to send a :py:mod:`zlib`-compressed script
+to the remote Python interpreter, it is time to choose what to send.
+
+The script sent is simply the source code for :py:mod:`mitogen.core`, with a
+single line suffixed to trigger execution of the
+:py:meth:`mitogen.core.ExternalContext.main` function. The encoded arguments
+to the main function include some additional details, such as the logging package
+level that was active in the parent process, and a random secret key that may
+later be used to generate HMAC signatures over the data frames that will be
+exchanged after bootstrap.
+
+After the script source code is prepared, it is passed through
+:py:func:`mitogen.master.minimize_source` to strip it of docstrings and
+comments, while preserving line numbers. This reduces the compressed payload
+by around 20%.
+
+
+Preserving The `mitogen.core` Source
+####################################
+
+One final trick is implemented in the first stage: after bootstrapping the new
+slave, it writes a duplicate copy of the :py:mod:`mitogen.core` source it just
+used to bootstrap it back into another pipe connected to the slave. The slave's
+module importer cache is initialized with a copy of the source, so that
+subsequent bootstraps of slave-of-slaves do not require the source to be
+fetched from the master a second time.
+
+
+Signalling Success
+##################
+
+Once the first stage has signalled ``EC0\n``, the master knows it is ready to
+receive the compressed bootstrap. After decompressing and writing the bootstrap
+source to its parent Python interpreter, the first stage writes the string
+``EC1\n`` to ``stdout`` before exiting. The master process waits for this
+string before considering bootstrap successful and the child's ``stdio`` ready
+to receive messages.
+
+
+ExternalContext.main()
+----------------------
+
+.. automethod:: mitogen.core.ExternalContext.main
+
+
+Generating A Synthetic `mitogen` Package
+########################################
+
+Since the bootstrap consists of the :py:mod:`mitogen.core` source code, and
+this code is loaded by Python by way of its main script (``__main__`` module),
+initially the module layout in the slave will be incorrect.
+
+The first step taken after bootstrap is to rearrange :py:data:`sys.modules` slightly
+so that :py:mod:`mitogen.core` appears in the correct location, and all
+classes defined in that module have their ``__module__`` attribute fixed up
+such that :py:mod:`cPickle` correctly serializes instance module names.
+
+Once a synthetic :py:mod:`mitogen` package and :py:mod:`mitogen.core` module
+have been generated, the bootstrap **deletes** `sys.modules['__main__']`, so
+that any attempt to import it (by :py:mod:`cPickle`) will cause the import to
+be satisfied by fetching the master's actual ``__main__`` module. This is
+necessary to allow master programs to be written as a self-contained Python
+script.
+
+
+Reaping The First Stage
+#######################
+
+After the bootstrap has called :py:func:`os.dup` on the copy of the ``stdin``
+file descriptor saved by the first stage, it is closed.
+
+Additionally, since the first stage was forked prior to re-executing the Python
+interpreter, it will exist as a zombie process until the parent process reaps
+it. Therefore the bootstrap must call :py:func:`os.wait` soon after startup.
+
+
+Setup Logging
+#############
+
+The slave's :py:mod:`logging` package root logger is configured to have the
+same log level as the root logger in the master, and
+:py:class:`mitogen.core.LogHandler` is installed to forward logs to the master
+context's :py:data:`FORWARD_LOG <mitogen.core.FORWARD_LOG>` handle.
+
+The log level is copied into the slave to avoid generating a potentially large
+amount of network IO forwarding logs that will simply be filtered away once
+they reach the master.
+
+
+The Module Importer
+###################
+
+An instance of :py:class:`mitogen.core.Importer` is installed in
+:py:data:`sys.meta_path`, where Python's ``import`` statement will execute it
+before attempting to find a module locally.
+
+
+Standard IO Redirection
+#######################
+
+Two instances of :py:class:`mitogen.core.IoLogger` are created, one for
+``stdout`` and one for ``stderr``. This class creates a UNIX pipe whose read
+end is added to the IO multiplexer, and whose write end is used to overwrite
+the handles inherited during process creation.
+
+Even without IO redirection, something must replace ``stdin`` and ``stdout``,
+otherwise it is possible for the stream used for communication between the
+master and slave to be accidentally corrupted by subprocesses run by user code.
+
+The inherited ``stdin`` is replaced by a file descriptor pointing to
+``/dev/null``.
+
+Finally Python's :py:data:`sys.stdout` is reopened to ensure line buffering is
+active, so that ``print`` statements and suchlike promptly appear in the logs.
+
+
+Function Call Dispatch
+######################
+
+After all initialization is complete, the slave's main thread sits in a loop
+reading from a :py:class:`Channel <mitogen.core.Channel>` connected to the
+:py:data:`CALL_FUNCTION <mitogen.core.CALL_FUNCTION>` handle. This handle is
+written to by
+:py:meth:`call_with_deadline() <mitogen.master.Context.call_with_deadline>`
+and :py:meth:`call() <mitogen.master.Context.call>`.
+
+
+Shutdown
+########
+
+When the master signals the :py:data:`CALL_FUNCTION
+<mitogen.core.CALL_FUNCTION>` :py:class:`Channel <mitogen.core.Channel>` is
+closed, the slave calls :py:meth:`shutdown() <mitogen.core.Broker.shutdown>`
+followed by :py:meth:`wait() <mitogen.core.Broker.wait>` on its own broker,
+triggering graceful shutdown.
+
+During shutdown, the master will wait a few seconds for slaves to disconnect
+gracefully before force disconnecting them, while the slaves will use that time
+to call :py:meth:`socket.shutdown(SHUT_WR) <socket.socket.shutdown>` on their
+:py:class:`IoLogger <mitogen.core.IoLogger>` socket's write ends before
+draining any remaining data buffered on the read ends.
+
+An alternative approach is to wait until the socket is completely closed, with
+some hard timeout, but this necessitates greater discipline than is common in
+infrastructure code (how often have you forgotten to redirect stderr to
+``/dev/null``?), so needless irritating delays would often be experienced
+during program termination.
+
+If the main thread (responsible for function call dispatch) fails to trigger
+shutdown (because some user function is hanging), then the eventual force
+disconnection by the master will cause the IO multiplexer thread to enter
+shutdown by itself.
+
+
+.. _stream-protocol:
+
+Stream Protocol
+---------------
+
+Once connected, a basic framing protocol is used to communicate between
+master and slave:
+
++--------------------+------+------------------------------------------------------+
+| Field              | Size | Description                                          |
++====================+======+======================================================+
+| ``dst_id``         | 2    | Integer target context ID.                           |
++--------------------+------+------------------------------------------------------+
+| ``src_id``         | 2    | Integer source context ID.                           |
++--------------------+------+------------------------------------------------------+
+| ``handle``         | 4    | Integer target handle in recipient.                  |
++--------------------+------+------------------------------------------------------+
+| ``reply_to``       | 4    | Integer response target ID.                          |
++--------------------+------+------------------------------------------------------+
+| ``length``         | 4    | Message length                                       |
++--------------------+------+------------------------------------------------------+
+| ``data``           | n/a  | Pickled message data.                                |
++--------------------+------+------------------------------------------------------+
+
+Masters listen on the following handles:
+
+.. data:: mitogen.core.FORWARD_LOG
+
+    Receives `(logger_name, level, msg)` 3-tuples and writes them to the
+    master's ``mitogen.ctx.<context_name>`` logger.
+
+.. data:: mitogen.core.GET_MODULE
+
+    Receives `(reply_to, fullname)` 2-tuples, looks up the source code for the
+    module named ``fullname``, and writes the source along with some metadata
+    back to the handle ``reply_to``. If lookup fails, ``None`` is sent instead.
+
+
+Slaves listen on the following handles:
+
+.. data:: mitogen.core.CALL_FUNCTION
+
+    Receives `(with_context, mod_name, class_name, func_name, args, kwargs)`
+    5-tuples from
+    :py:meth:`call_with_deadline() <mitogen.master.Context.call_with_deadline>`,
+    imports ``mod_name``, then attempts to execute
+    `class_name.func_name(\*args, \**kwargs)`.
+
+    When this channel is closed (by way of sending ``_DEAD`` to it), the
+    slave's main thread begins graceful shutdown of its own `Broker` and
+    `Router`. Each slave is responsible for sending ``_DEAD`` to each of its
+    directly connected slaves in response to the master sending ``_DEAD`` to
+    it, and arranging for the connection to its parent context to be closed
+    shortly thereafter.
+
+.. data:: mitogen.core.ADD_ROUTE
+
+    Receives `(target_id, via_id)` integer tuples, describing how messages
+    arriving at this context on any Stream should be forwarded on the stream
+    associated with the Context `via_id` such that they are eventually
+    delivered to the target Context.
+
+    This message is necessary to inform intermediary contexts of the existence
+    of a downstream Context, as they do not otherwise parse traffic they are
+    fowarding to their downstream contexts that may cause new contexts to be
+    established.
+
+    Given a chain `master -> ssh1 -> sudo1`, no `ADD_ROUTE` message is
+    necessary, since :py:class:`mitogen.core.Router` in the `ssh` context can
+    arrange to update its routes while setting up the new slave during
+    `proxy_connect()`.
+
+    However, given a chain like `master -> ssh1 -> sudo1 -> ssh2 -> sudo2`,
+    `ssh1` requires an `ADD_ROUTE` for `ssh2`, and both `ssh1` and `sudo1`
+    require an `ADD_ROUTE` for `sudo2`, as neither directly dealt with its
+    establishment.
+
+
+Slaves that have ever been used to create a descendent child context also
+listen on the following handles:
+
+.. data:: mitogen.core.GET_MODULE
+
+    As with master's ``GET_MODULE``, except this implementation
+    (:py:class:`mitogen.master.ModuleForwarder`) serves responses using
+    :py:class:`mitogen.core.Importer`'s cache before forwarding the request to
+    its parent context. The response is cached by each context in turn before
+    being forwarded on to the slave context that originally made the request.
+    In this way, the master need never re-send a module it has already sent to
+    a direct descendant.
+
+
+Additional handles are created to receive the result of every function call
+triggered by :py:meth:`call_with_deadline() <mitogen.master.Context.call_with_deadline>`.
+
+
+Sentinel Value
+##############
+
+.. autodata:: mitogen.core._DEAD
+
+The special value :py:data:`mitogen.core._DEAD` is used to signal
+disconnection or closure of the remote end. It is used internally by
+:py:class:`Channel <mitogen.core.Channel>` and also passed to any function
+still registered with :py:meth:`add_handler()
+<mitogen.core.Router.add_handler>` during Broker shutdown.
+
+
+Use of Pickle
+#############
+
+The current implementation uses the Python :py:mod:`cPickle` module, with
+mitigations to prevent untrusted slaves from triggering code excution in the
+master. The primary reason for using :py:mod:`cPickle` is that it is
+computationally efficient, and avoids including a potentially large body of
+serialization code in the bootstrap.
+
+The pickler active in slave contexts will instantiate any class, however in the
+master it is initially restricted to only permitting
+:py:class:`CallError <mitogen.core.CallError>` and :py:data:`_DEAD
+<mitogen.core._DEAD>`. While not recommended, it is possible to register more
+using :py:meth:`mitogen.master.LocalStream.allow_class`.
+
+The choice of Pickle is one area to be revisited later. All accounts suggest it
+cannot be used securely, however few of those accounts appear to be expert, and
+none mention any additional attacks that would not be prevented by using a
+restrictive class whitelist.
+
+.. note::
+
+    Since unpickling may trigger module loads, it is not possible to
+    deserialize data on the broker thread, as this will result in recursion
+    leading to a deadlock. Therefore any internal services (module loader,
+    logging forwarder, etc.) must rely on simple string formats, or only
+    perform serialization from within the broker thread.
+
+
+The IO Multiplexer
+------------------
+
+Since we must include our IO multiplexer as part of the bootstrap,
+off-the-shelf implementations are for the most part entirely inappropriate. For
+example, a minimal copy of Twisted weighs in at around 440KiB and is composed
+of approximately 115 files. Even if we could arrange for an entire Python
+package to be transferred during bootstrap, this minimal configuration is
+massive in comparison to Mitogen's solution, multiplies quickly in the
+presence of many machines, and would require manually splitting up the parts of
+Twisted that we would like to use.
+
+
+Message Routing
+---------------
+
+Routing assumes it is impossible to construct a tree such that one of a
+context's parents will not know the ID of a target the context is attempting to
+communicate with.
+
+When :py:class:`mitogen.core.Router` receives a message, it checks the IDs
+associated with its directly connected streams for a potential route. If any
+stream matches, either because it directly connects to the target ID, or
+because the master sent an ``ADD_ROUTE`` message associating it, then the
+message will be forwarded down the tree using that stream.
+
+If the message does not match any ``ADD_ROUTE`` message or stream, instead it
+is forwarded upwards to the immediate parent, and recursively by each parent in
+turn until one is reached that knows how to forward the message down the tree.
+
+When the master establishes a new context via an existing child context, it
+sends corresponding ``ADD_ROUTE`` messages to each indirect parent between the
+context and the root.
+
+
+Example
+#######
+
+.. image:: images/context-tree.png
+
+In the diagram, when ``master`` is creating the ``sudo:node12b:webapp``
+context, it must send ``ADD_ROUTE`` messages to ``rack12``, ``dc1``,
+``bastion``, and itself; ``node12b`` does not require an ``ADD_ROUTE`` message
+since it has a stream directly connected to the new context.
+
+When ``sudo:node22a:webapp`` wants to send a message to
+``sudo:node12b:webapp``, the message will be routed as follows:
+
+``sudo:node22a:webapp -> node22a -> rack22 -> dc2 -> bastion -> dc1 -> rack12 -> node12b -> sudo:node12b:webapp``
+
+.. image:: images/route.png
+
+
+Future
+######
+
+The current routing approach is incomplete, since routes to downstream contexts
+are not propagated upwards when a descendant of the master context establishes
+a new child context, but that is okay for now, since child contexts cannot
+currently allocate new context IDs anyway.
+
+
+Differences Between Master And Slave Brokers
+############################################
+
+The main difference between :py:class:`mitogen.core.Broker` and
+:py:class:`mitogen.master.Broker` is that when the stream connection to the
+parent is lost in a slave, the broker will trigger its own shutdown.
+
+
+The Module Importer
+-------------------
+
+:py:class:`mitogen.core.Importer` is still a work in progress, as there
+are a variety of approaches to implementing it, and the present implementation
+is not pefectly efficient in every case.
+
+It operates by intercepting ``import`` statements via `sys.meta_path`, asking
+Python if it can satisfy the import by itself, and if not, indicating to Python
+that it is capable of loading the module.
+
+In :py:meth:`load_module() <mitogen.core.Importer.load_module>` an RPC is
+started to the parent context, requesting the module source code. Once the
+source is fetched, the method builds a new module object using the best
+practice documented in PEP-302.
+
+
+Minimizing Roundtrips
+#####################
+
+In Python 2.x where relative imports are the default, a large number of import
+requests will be made for modules that do not exist. For example:
+
+.. code-block:: python
+
+    # mypkg/__init__.py
+
+    import sys
+    import os
+
+In Python 2.x, Python will first try to load ``mypkg.sys`` and ``mypkg.os``,
+which do not exist, before falling back on :py:mod:`sys` and :py:mod:`os`.
+
+These negative imports present a challenge, as they introduce a large number of
+pointless network roundtrips. Therefore in addition to the
+:py:mod:`zlib`-compressed source, for packages the master sends along a list of
+child modules known to exist.
+
+Before indicating it can satisfy an import request,
+:py:class:`mitogen.core.Importer` first checks to see if the module belongs to
+a package it has previously imported, and if so, ignores the request if the
+module does not appear in the enumeration of child modules belonging to the
+package.
+
+
+Child Module Enumeration
+########################
+
+Package children are enumerated using :py:func:`pkgutil.iter_modules`.
+
+
+Use Of Threads
+--------------
+
+The package always runs the IO multiplexer in a thread. This is so the
+multiplexer retains control flow in order to shut down gracefully, say, if the
+user's code has hung and the master context has disconnected.
+
+While it is possible for the IO multiplexer to recover control of a hung
+function call on UNIX using for example :py:mod:`signal.SIGALRM <signal>`, this
+mechanism is not portable to non-UNIX operating systems, and does not work in
+every case, for example when Python blocks signals during a variety of
+:py:mod:`threading` package operations.
+
+At some point it is likely Mitogen will be extended to support starting slaves
+running on Windows. When that happens, it would be nice if the process model on
+Windows and UNIX did not differ, and in fact the code used on both were
+identical.

docs/images/billing.graphml

@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="189.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="47.984375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="239.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="70.55078125" x="34.724609375" y="5.93359375">ssh:bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="289.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="32.48828125" x="53.755859375" y="5.93359375">sudo<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="339.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="71.423828125" x="34.2880859375" y="5.93359375">ssh:billing0<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="389.0"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="131.740234375" x="4.1298828125" y="5.93359375">run-nightly-billing.py<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n1" target="n2">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n2" target="n3">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n3" target="n4">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/images/context-tree.graphml

@@ -0,0 +1,497 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="0.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="13.484375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="50.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="47.072265625" x="11.9638671875" y="5.93359375">bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="121.5" y="100.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc1<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="486.0" y="100.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc2<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="40.5" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack11<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n5">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="202.5" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack12<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n6">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="81.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node11a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n7">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="0.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node11b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n8">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="162.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node12a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n9">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="243.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node12b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n10">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="526.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node21a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n11">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="607.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node21b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n12">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="445.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node22a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n13">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="364.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node22b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n14">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="567.0" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack21<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n15">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="405.0" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack22<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n16">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="152.0" x="202.5" y="250.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="137.083984375" x="7.4580078125" y="5.93359375">sudo:node12b:webapp<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n17">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="152.0" x="405.0" y="250.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="136.158203125" x="7.9208984375" y="5.93359375">sudo:node22a:webapp<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n1" target="n3">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n1" target="n2">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n2" target="n4">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e4" source="n4" target="n7">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e5" source="n4" target="n6">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e6" source="n5" target="n9">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e7" source="n5" target="n8">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e8" source="n2" target="n5">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e9" source="n15" target="n13">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e10" source="n15" target="n12">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e11" source="n3" target="n14">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e12" source="n14" target="n10">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e13" source="n14" target="n11">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e14" source="n9" target="n16">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e15" source="n3" target="n15">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e16" source="n12" target="n17">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/images/fakessh.graphml

@@ -0,0 +1,331 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="156.0" y="-15.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="47.072265625" x="29.4638671875" y="5.93359375">bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="20.0" y="60.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="86.482421875" x="9.7587890625" y="5.93359375">ssh.webserver<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="20.0" y="141.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="81.501953125" x="12.2490234375" y="5.93359375">sudo.webapp<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="156.0" y="222.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="48.419921875" x="28.7900390625" y="5.93359375">fakessh<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="156.0" y="141.0"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="34.890625" x="35.5546875" y="5.93359375">rsync<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n5">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="20.0" y="222.0"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="88.09375" x="8.953125" y="5.93359375">rsync --server<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n6">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="156.0" y="60.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="81.033203125" x="12.4833984375" y="5.93359375">ssh.fileserver<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n7">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="106.0" x="156.0" y="-90.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="30.984375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="-53.0" sy="0.0" tx="0.0" ty="-15.0">
+            <y:Point x="73.0" y="0.0"/>
+          </y:Path>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n1" target="n2">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="15.0" tx="0.0" ty="-15.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n0" target="n6">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="15.0" tx="0.0" ty="-15.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n3" target="n6">
+      <data key="d9"/>
+      <data key="d10">
+        <y:ArcEdge>
+          <y:Path sx="53.0" sy="0.0" tx="53.0" ty="0.0">
+            <y:Point x="313.97589111328125" y="156.0"/>
+          </y:Path>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:Arc height="-51.97590637207031" ratio="-1.283355712890625" type="fixedRatio"/>
+        </y:ArcEdge>
+      </data>
+    </edge>
+    <edge id="e4" source="n0" target="n1">
+      <data key="d9"/>
+      <data key="d10">
+        <y:ArcEdge>
+          <y:Path sx="-53.0" sy="0.0" tx="46.522850036621094" ty="-15.0078125">
+            <y:Point x="120.11017608642578" y="19.263574600219727"/>
+          </y:Path>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:Arc height="-20.658016204833984" ratio="-1.176903247833252" type="fixedRatio"/>
+        </y:ArcEdge>
+      </data>
+    </edge>
+    <edge id="e5" source="n1" target="n2">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="44.41933213706437" sy="15.025426211222324" tx="44.41933213706437" ty="-14.9921875"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e6" source="n2" target="n5">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="44.30647864807331" sy="14.996030393496994" tx="44.30647864807331" ty="-15.0234375"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e7" source="n6" target="n4">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e8" source="n6" target="n0">
+      <data key="d9"/>
+      <data key="d10">
+        <y:ArcEdge>
+          <y:Path sx="53.0" sy="0.0" tx="53.0" ty="0.0">
+            <y:Point x="298.25" y="37.5"/>
+          </y:Path>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:Arc height="-36.25" ratio="-1.9333332777023315" type="fixedRatio"/>
+        </y:ArcEdge>
+      </data>
+    </edge>
+    <edge id="e9" source="n0" target="n0">
+      <data key="d9"/>
+      <data key="d10">
+        <y:ArcEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="209.0" y="0.0"/>
+          </y:Path>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:Arc height="0.0" ratio="1.0" type="fixedRatio"/>
+        </y:ArcEdge>
+      </data>
+    </edge>
+    <edge id="e10" source="n7" target="n7">
+      <data key="d9"/>
+      <data key="d10">
+        <y:ArcEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="209.0" y="-75.0"/>
+          </y:Path>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:Arc height="0.0" ratio="1.0" type="fixedRatio"/>
+        </y:ArcEdge>
+      </data>
+    </edge>
+    <edge id="e11" source="n7" target="n0">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="-6.5"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e12" source="n4" target="n3">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="15.0" tx="0.0" ty="-15.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e13" source="n2" target="n5">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/images/route.graphml

@@ -0,0 +1,637 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="0.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="13.484375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="50.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="47.072265625" x="11.9638671875" y="5.93359375">bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="121.5" y="100.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc1<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="486.0" y="100.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc2<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="40.5" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack11<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n5">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="202.5" y="150.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack12<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n6">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="81.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node11a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n7">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="0.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node11b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n8">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="162.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node12a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n9">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="243.0" y="200.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node12b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n10">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="526.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node21a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n11">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="607.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node21b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n12">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="445.5" y="200.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node22a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n13">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="364.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node22b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n14">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="567.0" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack21<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n15">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="405.0" y="150.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack22<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n16">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="152.0" x="202.5" y="250.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="137.083984375" x="7.4580078125" y="5.93359375">sudo:node12b:webapp<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n17">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="152.0" x="405.0" y="250.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="136.158203125" x="7.9208984375" y="5.93359375">sudo:node22a:webapp<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n1" target="n3">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n1" target="n2">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n2" target="n4">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e4" source="n4" target="n7">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e5" source="n4" target="n6">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e6" source="n5" target="n9">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e7" source="n5" target="n8">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e8" source="n2" target="n5">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e9" source="n15" target="n13">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e10" source="n15" target="n12">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e11" source="n3" target="n14">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e12" source="n14" target="n10">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e13" source="n14" target="n11">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e14" source="n9" target="n16">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e15" source="n3" target="n15">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e16" source="n12" target="n17">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e17" source="n16" target="n9">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="345.5" y="215.0"/>
+          </y:Path>
+          <y:LineStyle color="#339966" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e18" source="n9" target="n5">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="33.5" ty="0.0">
+            <y:Point x="344.25" y="165.0"/>
+          </y:Path>
+          <y:LineStyle color="#339966" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e19" source="n5" target="n2">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="339.25" y="115.0"/>
+          </y:Path>
+          <y:LineStyle color="#008000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e20" source="n2" target="n1">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="339.25" y="94.5"/>
+          </y:Path>
+          <y:LineStyle color="#008000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e21" source="n1" target="n3">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="384.5" y="115.0"/>
+          </y:Path>
+          <y:LineStyle color="#008000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e22" source="n3" target="n15">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="374.5" y="140.5"/>
+            <y:Point x="374.5" y="165.0"/>
+          </y:Path>
+          <y:LineStyle color="#008000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e23" source="n15" target="n12">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="500.5" y="165.0"/>
+          </y:Path>
+          <y:LineStyle color="#008000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e24" source="n12" target="n17">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="40.5" ty="-5.5">
+            <y:Point x="521.5" y="240.5"/>
+          </y:Path>
+          <y:LineStyle color="#008000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/index.rst

@@ -0,0 +1,357 @@
+
+Mitogen
+=======
+
+Mitogen is a Python library for writing distributed self-replicating programs.
+
+.. raw:: html
+
+    <style>
+        .warning code {
+            background-color: rgba(0, 0, 0, 0.1);
+        }
+    </style>
+
+.. warning::
+
+    This is alpha-quality code. If you intend to use it, be aware of how little
+    real world testing it has received, the total absence of any systematic
+    tests, and the nightmare-level difficulty of debugging hangs in a tree of
+    processes running identical code straddling multiple thread and machine
+    boundaries! ``router.enable_debug()`` is your friend.
+
+    If you think you have a use for this software, please `drop me an e-mail`_
+    so that expectations and bug fixes can be managed sensibly.
+
+    .. _drop me an e-mail: dw@botanicus.net
+
+.. image:: images/cell_division.png
+    :align: right
+
+There is no requirement for installing packages, copying files around, writing
+shell snippets, upfront configuration, or providing any secondary link to a
+remote machine aside from an SSH connection. Due to its origins for use in
+managing potentially damaged infrastructure, the **remote machine need not even
+have free disk space or a writeable filesystem**.
+
+It is not intended as a generic RPC framework; the goal is to provide a robust
+and efficient low-level API on which tools like `Salt`_, `Ansible`_, or
+`Fabric`_ can be built, and while the API is quite friendly and comparable to
+`Fabric`_, ultimately it is not intended for direct use by consumer software.
+
+.. _Salt: https://docs.saltstack.com/en/latest/
+.. _Ansible: http://docs.ansible.com/
+.. _Fabric: http://docs.fabfile.org/en/
+
+The focus is to centralize and perfect the intricate dance required to run
+Python code safely and efficiently on a remote machine, while **avoiding
+temporary files or large chunks of error-prone shell scripts**, and supporting
+common privilege escalation techniques like `sudo`, potentially in combination
+with exotic connection methods such as WMI, `telnet`, or console-over-IPMI.
+
+
+Automatic Bootstrap
+###################
+
+Mitogen's main feature is enabling your Python program to bootstrap and
+communicate with new copies of itself under its control running on remote
+machines, **using only an existing installed Python interpreter and SSH
+client**, something that by default can be found on almost all contemporary
+machines in the wild. To accomplish bootstrap, Mitogen uses a single 600 byte
+SSH command line and 6KB of its own source code sent to stdin of the remote SSH
+connection.
+
+.. code::
+
+    $ python preamble_size.py
+    SSH command size: 576
+    Preamble size: 6360 (6.21KiB)
+    mitogen.master size: 4104 (4.01KiB)
+    mitogen.ssh size: 295 (0.29KiB)
+    mtiogen.sudo size: 1210 (1.18KiB)
+
+Once bootstrapped, the remote process is configured with a customizable
+**argv[0]**, readily visible to system administrators of the remote machine
+using the UNIX **ps** command:
+
+.. code::
+
+    20051 ?        Ss     0:00  \_ sshd: dmw [priv]
+    20053 ?        S      0:00  |   \_ sshd: dmw@notty
+    20054 ?        Ssl    0:00  |       \_ mitogen:dmw@Eldil.home:22476
+    20103 ?        S      0:00  |           \_ tar zxvf myapp.tar.gz
+
+The example context was started by UID ``dmw`` on host ``Eldil.home``, process
+ID ``22476``.
+
+
+IO Multiplexer
+##############
+
+The bootstrap includes a compact IO multiplexer (like Twisted or asyncio) that
+allows it to perform work in the background while executing your program's
+code. For example, the remote context can be used to **connect to a new user on
+the remote machine using sudo**, or as an intermediary for extending the
+program's domain of control outward to other machines, enabling your program to
+**manipulate machines behind a firewall**, or enable its **data plane to cohere
+to your network topology**.
+
+.. image:: images/billing.png
+    :align: right
+
+.. code::
+
+    bastion_host = router.ssh(
+        hostname='jump-box.mycorp.com'
+    )
+
+    ssh_account = router.sudo(
+        via=bastion_host,
+        username='user_with_magic_ssh_key',
+        password='sudo password',
+    )
+
+    internal_box = router.ssh(
+        via=ssh_account,
+        hostname='billing0.internal.mycorp.com'
+    )
+
+    internal_box.call(os.system, './run-nightly-billing.py')
+
+The multiplexer also ensures the remote process is terminated if your Python
+program crashes, communication is lost, or the application code running in the
+context has hung.
+
+
+Module Forwarder
+################
+
+In addition to an IO multiplexer, slaves are configured with a custom `PEP-302
+importer`_ that forwards requests for unknown Python modules back to the host
+program. When your program asks a context to execute code from an unknown
+module, all requisite modules are transferred automatically and imported
+entirely in RAM without need for further configuration.
+
+.. _PEP-302 importer: https://www.python.org/dev/peps/pep-0302/
+
+.. code-block:: python
+
+    import myapp.mypkg.mymodule
+
+    # myapp/__init__.py, myapp/mypkg/__init__.py, and myapp/mypkg/mymodule.py
+    # are transferred automatically.
+    print context.call(myapp.mymodule.my_function)
+
+As the forwarder reuses the import mechanism, it should integrate cleanly with
+any tool such as `py2exe`_ that correctly implement the protocols in PEP-302,
+allowing truly single file applications to run across multiple machines without
+further effort.
+
+.. _py2exe: http://www.py2exe.org/
+
+
+SSH Client Emulation
+####################
+
+.. image:: images/fakessh.png
+    :align: right
+
+Support is included for starting subprocesses with a modified environment, that
+cause their attempt to use SSH to be redirected back into the host program. In
+this way tools like `rsync`, `sftp`, and `scp` can efficiently reuse the host
+program's existing connection to the remote machine, including any
+firewall/user account hopping in use, with no additional configuration.
+
+Scenarios that were not previously possible with these tools are enabled, such
+as running `sftp` and `rsync` over a `sudo` session, to an account the user
+cannot otherwise directly log into, including in restrictive environments that
+for example enforce an interactive TTY and account password.
+
+.. code-block:: python
+
+    bastion = router.ssh(hostname='bastion.mycorp.com')
+    webserver = router.ssh(via=bastion, hostname='webserver')
+    webapp = router.sudo(via=webserver, username='webapp')
+    fileserver = router.ssh(via=bastion, hostname='fileserver')
+
+    # Transparently tunnelled over fileserver -> .. -> sudo.webapp link
+    fileserver.call(mitogen.fakessh.run, webapp, [
+        'rsync', 'appdata', 'appserver:appdata'
+    ])
+
+
+Inter-slave Message Routing
+###########################
+
+.. image:: images/route.png
+
+Slaves may communicate autonomously without direct interaction with the master,
+allowing a wide variety of complex data and control flows to be expressed using
+the links between the processes.
+
+
+Logging Forwarder
+#################
+
+The bootstrap configures the remote process's Python logging package to forward
+all logs back to the local process, enabling management of program logs in one
+location.
+
+.. code::
+
+    18:15:29 D mitogen.ctx.k3: mitogen: Importer.find_module('mitogen.zlib')
+    18:15:29 D mitogen.ctx.k3: mitogen: _dispatch_calls((1002L, False, 'posix', None, 'system', ('ls -l /proc/self/fd',), {}))
+
+
+Stdio Forwarder
+###############
+
+To ease porting of crusty old infrastructure scripts to Python, the bootstrap
+redirects stdio for itself and any child processes back into the logging
+framework. This allows use of functions as basic as **os.system('hostname;
+uptime')** without further need to capture or manage output.
+
+.. code::
+
+   18:17:28 D mitogen.ctx.k3: mitogen: _dispatch_calls((1002L, False, 'posix', None, 'system', ('hostname; uptime',), {}))
+   18:17:56 I mitogen.ctx.k3: stdout: k3
+   18:17:56 I mitogen.ctx.k3: stdout: 17:37:10 up 562 days,  2:25,  5 users,  load average: 1.24, 1.13, 1.14
+
+
+Blocking Code Friendly
+######################
+
+Within each process, a private thread runs the I/O multiplexer, leaving the
+main thread and any additional application threads free to perform useful work.
+
+While Mitogen is internally asynchronous, it hides this asynchrony from
+consumer code. This is since writing asynchronous code is mostly a foreign
+concept to the target application of managing infrastructure. It should be
+possible to rewrite a shell script in Python without significant restructuring,
+or mind-bending feats of comprehension to understand control flow.
+
+Before:
+
+.. code-block:: sh
+
+    #!/bin/bash
+    # Install our application.
+
+    tar zxvf app.tar.gz
+
+After:
+
+.. code-block:: python
+
+    def install_app():
+        """
+        Install our application.
+        """
+        os.system('tar zxvf app.tar.gz')
+
+    context.call(install_app)
+
+Or even:
+
+.. code-block:: python
+
+    context.call(os.system, 'tar zxvf app.tar.gz')
+
+Exceptions raised by function calls are propagated back to the parent program,
+and timeouts can be configured to ensure failed calls do not block progress of
+the parent.
+
+
+Support For Single File Programs
+################################
+
+Programs that are self-contained within a single Python script are supported.
+External contexts are configured such that any attempt to execute a function
+from the main Python script will correctly cause that script to be imported as
+usual into the slave process.
+
+.. code-block:: python
+
+    #!/usr/bin/env python
+    """
+    Install our application on a remote machine.
+
+    Usage:
+        install_app.py <hostname>
+
+    Where:
+        <hostname>  Hostname to install to.
+    """
+    import os
+    import sys
+
+    import mitogen
+
+
+    def install_app():
+        os.system('tar zxvf my_app.tar.gz')
+
+
+    def main(broker):
+        if len(sys.argv) != 2:
+            print __doc__
+            sys.exit(1)
+
+        context = mitogen.ssh.connect(broker, sys.argv[1])
+        context.call(install_app)
+
+    if __name__ == '__main__' and mitogen.master:
+        import mitogen.utils
+        mitogen.utils.run_with_broker(main)
+
+
+Event-driven IO
+###############
+
+Code running in a remote context can be connected to a *Channel*. Channels are
+used to send data asynchronously back to the parent, without further need for
+the parent to poll for changes. This is useful for monitoring systems managing
+a large fleet of machines, or to alert the parent of unexpected state changes.
+
+.. code-block:: python
+
+    def tail_log_file(channel, path='/var/log/messages'):
+        """
+        Forward new lines in a log file to the parent.
+        """
+        size = os.path.getsize(path)
+
+        while channel.open():
+            new_size = os.path.getsize(path)
+            if new_size == size:
+                time.sleep(1)
+                continue
+            elif new_size < size:
+                size = 0
+
+            fp = file(path, 'r')
+            fp.seek(size)
+            channel.send(fp.read(new_size - size))
+            fp.close()
+            size = new_size
+
+
+Compatibility
+#############
+
+The package is written using syntax compatible all the way back to **Python
+2.4** released November 2004, making it suitable for managing a fleet of
+potentially ancient corporate hardware. For example Mitogen can be used out of
+the box against Red Hat Enterprise Linux 5, released in 2007.
+
+There is currently no support for Python 3, and no solid plan for supporting it
+any time soon. Due to constraints on implementation size and desire for
+compatibility with ancient Python versions, conventional porting methods such
+as ``six.py`` are likely to be unsuitable.
+
+
+Zero Dependencies
+#################
+
+Mitogen is implemented entirely using the standard library functionality and
+interfaces that were available in Python 2.4.

docs/internals.rst

@@ -0,0 +1,61 @@
+
+Internal API Reference
+**********************
+
+
+mitogen.core
+============
+
+
+Side Class
+----------
+
+.. autoclass:: mitogen.core.Side
+   :members:
+
+
+Stream Classes
+--------------
+
+.. autoclass:: mitogen.core.BasicStream
+   :members:
+
+.. autoclass:: mitogen.core.Stream
+   :members:
+
+.. autoclass:: mitogen.master.Stream
+   :members:
+
+.. autoclass:: mitogen.ssh.Stream
+   :members:
+
+
+Other Stream Subclasses
+-----------------------
+
+.. autoclass:: mitogen.core.IoLogger
+   :members:
+
+.. autoclass:: mitogen.core.Waker
+   :members:
+
+
+
+ExternalContext Class
+---------------------
+
+.. autoclass:: mitogen.core.ExternalContext
+
+
+mitogen.master
+===============
+
+.. autoclass:: mitogen.master.ProcessMonitor
+
+
+Helper Functions
+----------------
+
+.. autofunction:: mitogen.master.create_child
+.. autofunction:: mitogen.master.get_child_modules
+.. autofunction:: mitogen.master.minimize_source

docs/toc.rst

@@ -0,0 +1,13 @@
+
+Table Of Contents
+=================
+
+.. toctree::
+   :maxdepth: 2
+
+   index
+   howitworks
+   examples
+   getting_started
+   api
+   internals

examples/ansible_demo.py

@@ -0,0 +1,110 @@
+"""
+Minimal demo of running an Ansible module via mitogen.
+"""
+
+import json
+import logging
+import time
+
+import mitogen
+import mitogen.master
+import mitogen.utils
+
+# Prevent accident import of an Ansible module from hanging on stdin read.
+import ansible.module_utils.basic
+ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
+
+
+class Exit(Exception):
+    """
+    Raised when a module exits with success.
+    """
+    def __init__(self, dct):
+        self.dct = dct
+
+
+class ModuleError(Exception):
+    """
+    Raised when a module voluntarily indicates failure via .fail_json().
+    """
+    def __init__(self, msg, dct):
+        Exception.__init__(self, msg)
+        self.dct = dct
+
+
+def wtf_exit_json(self, **kwargs):
+    """
+    Replace AnsibleModule.exit_json() with something that doesn't try to
+    suicide the process or JSON-encode the dictionary. Instead, cause Exit to
+    be raised, with a `dct` attribute containing the successful result
+    dictionary.
+    """
+    self.add_path_info(kwargs)
+    kwargs.setdefault('changed', False)
+    kwargs.setdefault('invocation', {
+        'module_args': self.params
+    })
+    kwargs = ansible.module_utils.basic.remove_values(kwargs, self.no_log_values)
+    self.do_cleanup_files()
+    raise Exit(kwargs)
+
+
+def wtf_fail_json(self, **kwargs):
+    """
+    Replace AnsibleModule.fail_json() with something that raises ModuleError,
+    which includes a `dct` attribute.
+    """
+    self.add_path_info(kwargs)
+    kwargs.setdefault('failed', True)
+    kwargs.setdefault('invocation', {
+        'module_args': self.params
+    })
+    kwargs = ansible.module_utils.basic.remove_values(kwargs, self.no_log_values)
+    self.do_cleanup_files()
+    raise ModuleError(kwargs.get('msg'), kwargs)
+
+
+def run_module(module, raw_params=None, args=None):
+    """
+    Set up the process environment in preparation for running an Ansible
+    module. The monkey-patches the Ansible libraries in various places to
+    prevent it from trying to kill the process on completion, and to prevent it
+    from reading sys.stdin.
+    """
+    if args is None:
+        args = {}
+    if raw_params is not None:
+        args['_raw_params'] = raw_params
+
+    ansible.module_utils.basic.AnsibleModule.exit_json = wtf_exit_json
+    ansible.module_utils.basic.AnsibleModule.fail_json = wtf_fail_json
+    ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({
+        'ANSIBLE_MODULE_ARGS': args
+    })
+
+    try:
+        mod = __import__(module, {}, {}, [''])
+        # Ansible modules begin execution on import, because they're crap from
+        # hell. Thus the above __import__ will cause either Exit or
+        # ModuleError to be raised. If we reach the line below, the module did
+        # not execute and must already have been imported for a previous
+        # invocation, so we need to invoke main explicitly.
+        mod.main()
+    except Exit, e:
+        return e.dct
+
+
+def main(router):
+    fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s'
+    datefmt = '%H:%M:%S'
+    level = logging.DEBUG
+    level = logging.INFO
+    logging.basicConfig(level=level, format=fmt, datefmt=datefmt)
+
+    context = mitogen.master.connect(broker)
+    print context.call(run_module, 'ansible.modules.core.system.setup')
+    for x in xrange(10):
+        print context.call(run_module, 'ansible.modules.core.commands.command', 'hostname')
+
+if __name__ == '__main__' and not mitogen.slave:
+    mitogen.utils.run_with_router(main)

mitogen/__init__.py

@@ -0,0 +1,33 @@
+"""
+On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would
+be expected. On the slave, it is built dynamically during startup.
+"""
+
+#: This is ``False`` in slave contexts. It is used in single-file Python
+#: programs to avoid reexecuting the program's :py:func:`main` function in the
+#: slave. For example:
+#:
+#:      .. code-block:: python
+#:
+#:          def do_work():
+#:              os.system('hostname')
+#:
+#:          def main(broker):
+#:              context = mitogen.master.connect(broker)
+#:              context.call(do_work)  # Causes slave to import __main__.
+#:
+#:          if __name__ == '__main__' and mitogen.master:
+#:              import mitogen.utils
+#:              mitogen.utils.run_with_broker(main)
+#:
+master = True
+
+
+#: This is ``0`` in a master, otherwise it is a master-generated ID unique to
+#: the slave context used for message routing.
+context_id = 0
+
+
+#: This is ``None`` in a master, otherwise it is the master-generated ID unique
+#: to the slave's parent context.
+parent_id = None

mitogen/ansible/connection.py

@@ -0,0 +1,64 @@
+"""
+Basic Ansible connection plug-in mostly useful for testing functionality,
+due to Ansible's use of the multiprocessing package a lot more work is required
+to share the mitogen SSH connection across tasks.
+
+Enable it by:
+
+    $ cat ansible.cfg
+    [defaults]
+    connection_plugins = plugins/connection
+
+    $ mkdir -p plugins/connection
+    $ cat > plugins/connection/mitogen_conn.py <<-EOF
+    from mitogen.ansible.connection import Connection
+    EOF
+"""
+
+import mitogen.master
+import mitogen.ssh
+import mitogen.utils
+from mitogen.ansible import helpers
+
+import ansible.plugins.connection
+
+
+class Connection(ansible.plugins.connection.ConnectionBase):
+    broker = None
+    context = None
+
+    become_methods = []
+    transport = 'mitogen'
+
+    @property
+    def connected(self):
+        return self.broker is not None
+
+    def _connect(self):
+        if self.connected:
+            return
+        self.broker = mitogen.master.Broker()
+        if self._play_context.remote_addr == 'localhost':
+            self.context = mitogen.master.connect(self.broker)
+        else:
+            self.context = mitogen.ssh.connect(broker,
+                self._play_context.remote_addr)
+
+    def exec_command(self, cmd, in_data=None, sudoable=True):
+        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
+        if in_data:
+            raise AnsibleError("does not support module pipelining")
+
+        return self.context.call(helpers.exec_command, cmd, in_data)
+
+    def fetch_file(self, in_path, out_path):
+        output = self.context.call(helpers.read_path, in_path)
+        helpers.write_path(out_path, output)
+
+    def put_file(self, in_path, out_path):
+        self.context.call(helpers.write_path, out_path,
+                          helpers.read_path(in_path))
+
+    def close(self):
+        self.broker.shutdown()
+        self.broker.join()

mitogen/ansible/helpers.py

@@ -0,0 +1,28 @@
+"""
+Ansible is so poorly layered that attempting to import anything under
+ansible.plugins automatically triggers import of __main__, which causes
+remote execution of the ansible command-line tool. :(
+
+So here we define helpers in some sanely layered package where the entirety of
+Ansible won't be imported.
+"""
+
+import subprocess
+
+
+def exec_command(cmd, in_data=None):
+    proc = subprocess.Popen(cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+        shell=True)
+    stdout, stderr = proc.communicate(in_data)
+    return proc.returncode, stdout, stderr
+
+
+def read_path(path):
+    return file(path, 'rb').read()
+
+
+def write_path(path, s):
+    open(path, 'wb').write(s)

mitogen/compat/pkgutil.py

@@ -0,0 +1,591 @@
+"""Utilities to support packages."""
+
+# NOTE: This module must remain compatible with Python 2.3, as it is shared
+# by setuptools for distribution with Python 2.3 and up.
+
+import os
+import sys
+import imp
+import os.path
+from types import ModuleType
+
+__all__ = [
+    'get_importer', 'iter_importers', 'get_loader', 'find_loader',
+    'walk_packages', 'iter_modules', 'get_data',
+    'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
+]
+
+def read_code(stream):
+    # This helper is needed in order for the PEP 302 emulation to
+    # correctly handle compiled files
+    import marshal
+
+    magic = stream.read(4)
+    if magic != imp.get_magic():
+        return None
+
+    stream.read(4) # Skip timestamp
+    return marshal.load(stream)
+
+
+def simplegeneric(func):
+    """Make a trivial single-dispatch generic function"""
+    registry = {}
+    def wrapper(*args, **kw):
+        ob = args[0]
+        try:
+            cls = ob.__class__
+        except AttributeError:
+            cls = type(ob)
+        try:
+            mro = cls.__mro__
+        except AttributeError:
+            try:
+                class cls(cls, object):
+                    pass
+                mro = cls.__mro__[1:]
+            except TypeError:
+                mro = object,   # must be an ExtensionClass or some such  :(
+        for t in mro:
+            if t in registry:
+                return registry[t](*args, **kw)
+        else:
+            return func(*args, **kw)
+    try:
+        wrapper.__name__ = func.__name__
+    except (TypeError, AttributeError):
+        pass    # Python 2.3 doesn't allow functions to be renamed
+
+    def register(typ, func=None):
+        if func is None:
+            return lambda f: register(typ, f)
+        registry[typ] = func
+        return func
+
+    wrapper.__dict__ = func.__dict__
+    wrapper.__doc__ = func.__doc__
+    wrapper.register = register
+    return wrapper
+
+
+def walk_packages(path=None, prefix='', onerror=None):
+    """Yields (module_loader, name, ispkg) for all modules recursively
+    on path, or, if path is None, all accessible modules.
+
+    'path' should be either None or a list of paths to look for
+    modules in.
+
+    'prefix' is a string to output on the front of every module name
+    on output.
+
+    Note that this function must import all *packages* (NOT all
+    modules!) on the given path, in order to access the __path__
+    attribute to find submodules.
+
+    'onerror' is a function which gets called with one argument (the
+    name of the package which was being imported) if any exception
+    occurs while trying to import a package.  If no onerror function is
+    supplied, ImportErrors are caught and ignored, while all other
+    exceptions are propagated, terminating the search.
+
+    Examples:
+
+    # list all modules python can access
+    walk_packages()
+
+    # list all submodules of ctypes
+    walk_packages(ctypes.__path__, ctypes.__name__+'.')
+    """
+
+    def seen(p, m={}):
+        if p in m:
+            return True
+        m[p] = True
+
+    for importer, name, ispkg in iter_modules(path, prefix):
+        yield importer, name, ispkg
+
+        if ispkg:
+            try:
+                __import__(name)
+            except ImportError:
+                if onerror is not None:
+                    onerror(name)
+            except Exception:
+                if onerror is not None:
+                    onerror(name)
+                else:
+                    raise
+            else:
+                path = getattr(sys.modules[name], '__path__', None) or []
+
+                # don't traverse path items we've seen before
+                path = [p for p in path if not seen(p)]
+
+                for item in walk_packages(path, name+'.', onerror):
+                    yield item
+
+
+def iter_modules(path=None, prefix=''):
+    """Yields (module_loader, name, ispkg) for all submodules on path,
+    or, if path is None, all top-level modules on sys.path.
+
+    'path' should be either None or a list of paths to look for
+    modules in.
+
+    'prefix' is a string to output on the front of every module name
+    on output.
+    """
+
+    if path is None:
+        importers = iter_importers()
+    else:
+        importers = map(get_importer, path)
+
+    yielded = {}
+    for i in importers:
+        for name, ispkg in iter_importer_modules(i, prefix):
+            if name not in yielded:
+                yielded[name] = 1
+                yield i, name, ispkg
+
+
+#@simplegeneric
+def iter_importer_modules(importer, prefix=''):
+    if not hasattr(importer, 'iter_modules'):
+        return []
+    return importer.iter_modules(prefix)
+
+iter_importer_modules = simplegeneric(iter_importer_modules)
+
+
+class ImpImporter:
+    """PEP 302 Importer that wraps Python's "classic" import algorithm
+
+    ImpImporter(dirname) produces a PEP 302 importer that searches that
+    directory.  ImpImporter(None) produces a PEP 302 importer that searches
+    the current sys.path, plus any modules that are frozen or built-in.
+
+    Note that ImpImporter does not currently support being used by placement
+    on sys.meta_path.
+    """
+
+    def __init__(self, path=None):
+        self.path = path
+
+    def find_module(self, fullname, path=None):
+        # Note: we ignore 'path' argument since it is only used via meta_path
+        subname = fullname.split(".")[-1]
+        if subname != fullname and self.path is None:
+            return None
+        if self.path is None:
+            path = None
+        else:
+            path = [os.path.realpath(self.path)]
+        try:
+            file, filename, etc = imp.find_module(subname, path)
+        except ImportError:
+            return None
+        return ImpLoader(fullname, file, filename, etc)
+
+    def iter_modules(self, prefix=''):
+        if self.path is None or not os.path.isdir(self.path):
+            return
+
+        yielded = {}
+        import inspect
+        try:
+            filenames = os.listdir(self.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            modname = inspect.getmodulename(fn)
+            if modname=='__init__' or modname in yielded:
+                continue
+
+            path = os.path.join(self.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                try:
+                    dircontents = os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    dircontents = []
+                for fn in dircontents:
+                    subname = inspect.getmodulename(fn)
+                    if subname=='__init__':
+                        ispkg = True
+                        break
+                else:
+                    continue    # not a package
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield prefix + modname, ispkg
+
+
+class ImpLoader:
+    """PEP 302 Loader that wraps Python's "classic" import algorithm
+    """
+    code = source = None
+
+    def __init__(self, fullname, file, filename, etc):
+        self.file = file
+        self.filename = filename
+        self.fullname = fullname
+        self.etc = etc
+
+    def load_module(self, fullname):
+        self._reopen()
+        try:
+            mod = imp.load_module(fullname, self.file, self.filename, self.etc)
+        finally:
+            if self.file:
+                self.file.close()
+        # Note: we don't set __loader__ because we want the module to look
+        # normal; i.e. this is just a wrapper for standard import machinery
+        return mod
+
+    def get_data(self, pathname):
+        return open(pathname, "rb").read()
+
+    def _reopen(self):
+        if self.file and self.file.closed:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                self.file = open(self.filename, 'rU')
+            elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
+                self.file = open(self.filename, 'rb')
+
+    def _fix_name(self, fullname):
+        if fullname is None:
+            fullname = self.fullname
+        elif fullname != self.fullname:
+            raise ImportError("Loader for module %s cannot handle "
+                              "module %s" % (self.fullname, fullname))
+        return fullname
+
+    def is_package(self, fullname):
+        fullname = self._fix_name(fullname)
+        return self.etc[2]==imp.PKG_DIRECTORY
+
+    def get_code(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        if self.code is None:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                source = self.get_source(fullname)
+                self.code = compile(source, self.filename, 'exec')
+            elif mod_type==imp.PY_COMPILED:
+                self._reopen()
+                try:
+                    self.code = read_code(self.file)
+                finally:
+                    self.file.close()
+            elif mod_type==imp.PKG_DIRECTORY:
+                self.code = self._get_delegate().get_code()
+        return self.code
+
+    def get_source(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        if self.source is None:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                self._reopen()
+                try:
+                    self.source = self.file.read()
+                finally:
+                    self.file.close()
+            elif mod_type==imp.PY_COMPILED:
+                if os.path.exists(self.filename[:-1]):
+                    f = open(self.filename[:-1], 'rU')
+                    self.source = f.read()
+                    f.close()
+            elif mod_type==imp.PKG_DIRECTORY:
+                self.source = self._get_delegate().get_source()
+        return self.source
+
+
+    def _get_delegate(self):
+        return ImpImporter(self.filename).find_module('__init__')
+
+    def get_filename(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        mod_type = self.etc[2]
+        if self.etc[2]==imp.PKG_DIRECTORY:
+            return self._get_delegate().get_filename()
+        elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
+            return self.filename
+        return None
+
+
+try:
+    import zipimport
+    from zipimport import zipimporter
+
+    def iter_zipimport_modules(importer, prefix=''):
+        dirlist = zipimport._zip_directory_cache[importer.archive].keys()
+        dirlist.sort()
+        _prefix = importer.prefix
+        plen = len(_prefix)
+        yielded = {}
+        import inspect
+        for fn in dirlist:
+            if not fn.startswith(_prefix):
+                continue
+
+            fn = fn[plen:].split(os.sep)
+
+            if len(fn)==2 and fn[1].startswith('__init__.py'):
+                if fn[0] not in yielded:
+                    yielded[fn[0]] = 1
+                    yield fn[0], True
+
+            if len(fn)!=1:
+                continue
+
+            modname = inspect.getmodulename(fn[0])
+            if modname=='__init__':
+                continue
+
+            if modname and '.' not in modname and modname not in yielded:
+                yielded[modname] = 1
+                yield prefix + modname, False
+
+    iter_importer_modules.register(zipimporter, iter_zipimport_modules)
+
+except ImportError:
+    pass
+
+
+def get_importer(path_item):
+    """Retrieve a PEP 302 importer for the given path item
+
+    The returned importer is cached in sys.path_importer_cache
+    if it was newly created by a path hook.
+
+    If there is no importer, a wrapper around the basic import
+    machinery is returned. This wrapper is never inserted into
+    the importer cache (None is inserted instead).
+
+    The cache (or part of it) can be cleared manually if a
+    rescan of sys.path_hooks is necessary.
+    """
+    try:
+        importer = sys.path_importer_cache[path_item]
+    except KeyError:
+        for path_hook in sys.path_hooks:
+            try:
+                importer = path_hook(path_item)
+                break
+            except ImportError:
+                pass
+        else:
+            importer = None
+        sys.path_importer_cache.setdefault(path_item, importer)
+
+    if importer is None:
+        try:
+            importer = ImpImporter(path_item)
+        except ImportError:
+            importer = None
+    return importer
+
+
+def iter_importers(fullname=""):
+    """Yield PEP 302 importers for the given module name
+
+    If fullname contains a '.', the importers will be for the package
+    containing fullname, otherwise they will be importers for sys.meta_path,
+    sys.path, and Python's "classic" import machinery, in that order.  If
+    the named module is in a package, that package is imported as a side
+    effect of invoking this function.
+
+    Non PEP 302 mechanisms (e.g. the Windows registry) used by the
+    standard import machinery to find files in alternative locations
+    are partially supported, but are searched AFTER sys.path. Normally,
+    these locations are searched BEFORE sys.path, preventing sys.path
+    entries from shadowing them.
+
+    For this to cause a visible difference in behaviour, there must
+    be a module or package name that is accessible via both sys.path
+    and one of the non PEP 302 file system mechanisms. In this case,
+    the emulation will find the former version, while the builtin
+    import mechanism will find the latter.
+
+    Items of the following types can be affected by this discrepancy:
+        imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
+    """
+    if fullname.startswith('.'):
+        raise ImportError("Relative module names not supported")
+    if '.' in fullname:
+        # Get the containing package's __path__
+        pkg = '.'.join(fullname.split('.')[:-1])
+        if pkg not in sys.modules:
+            __import__(pkg)
+        path = getattr(sys.modules[pkg], '__path__', None) or []
+    else:
+        for importer in sys.meta_path:
+            yield importer
+        path = sys.path
+    for item in path:
+        yield get_importer(item)
+    if '.' not in fullname:
+        yield ImpImporter()
+
+def get_loader(module_or_name):
+    """Get a PEP 302 "loader" object for module_or_name
+
+    If the module or package is accessible via the normal import
+    mechanism, a wrapper around the relevant part of that machinery
+    is returned.  Returns None if the module cannot be found or imported.
+    If the named module is not already imported, its containing package
+    (if any) is imported, in order to establish the package __path__.
+
+    This function uses iter_importers(), and is thus subject to the same
+    limitations regarding platform-specific special import locations such
+    as the Windows registry.
+    """
+    if module_or_name in sys.modules:
+        module_or_name = sys.modules[module_or_name]
+    if isinstance(module_or_name, ModuleType):
+        module = module_or_name
+        loader = getattr(module, '__loader__', None)
+        if loader is not None:
+            return loader
+        fullname = module.__name__
+    else:
+        fullname = module_or_name
+    return find_loader(fullname)
+
+def find_loader(fullname):
+    """Find a PEP 302 "loader" object for fullname
+
+    If fullname contains dots, path must be the containing package's __path__.
+    Returns None if the module cannot be found or imported. This function uses
+    iter_importers(), and is thus subject to the same limitations regarding
+    platform-specific special import locations such as the Windows registry.
+    """
+    for importer in iter_importers(fullname):
+        loader = importer.find_module(fullname)
+        if loader is not None:
+            return loader
+
+    return None
+
+
+def extend_path(path, name):
+    """Extend a package's path.
+
+    Intended use is to place the following code in a package's __init__.py:
+
+        from pkgutil import extend_path
+        __path__ = extend_path(__path__, __name__)
+
+    This will add to the package's __path__ all subdirectories of
+    directories on sys.path named after the package.  This is useful
+    if one wants to distribute different parts of a single logical
+    package as multiple directories.
+
+    It also looks for *.pkg files beginning where * matches the name
+    argument.  This feature is similar to *.pth files (see site.py),
+    except that it doesn't special-case lines starting with 'import'.
+    A *.pkg file is trusted at face value: apart from checking for
+    duplicates, all entries found in a *.pkg file are added to the
+    path, regardless of whether they are exist the filesystem.  (This
+    is a feature.)
+
+    If the input path is not a list (as is the case for frozen
+    packages) it is returned unchanged.  The input path is not
+    modified; an extended copy is returned.  Items are only appended
+    to the copy at the end.
+
+    It is assumed that sys.path is a sequence.  Items of sys.path that
+    are not (unicode or 8-bit) strings referring to existing
+    directories are ignored.  Unicode items of sys.path that cause
+    errors when used as filenames may cause this function to raise an
+    exception (in line with os.path.isdir() behavior).
+    """
+
+    if not isinstance(path, list):
+        # This could happen e.g. when this is called from inside a
+        # frozen package.  Return the path unchanged in that case.
+        return path
+
+    pname = os.path.join(*name.split('.')) # Reconstitute as relative path
+    # Just in case os.extsep != '.'
+    sname = os.extsep.join(name.split('.'))
+    sname_pkg = sname + os.extsep + "pkg"
+    init_py = "__init__" + os.extsep + "py"
+
+    path = path[:] # Start with a copy of the existing path
+
+    for dir in sys.path:
+        if not isinstance(dir, basestring) or not os.path.isdir(dir):
+            continue
+        subdir = os.path.join(dir, pname)
+        # XXX This may still add duplicate entries to path on
+        # case-insensitive filesystems
+        initfile = os.path.join(subdir, init_py)
+        if subdir not in path and os.path.isfile(initfile):
+            path.append(subdir)
+        # XXX Is this the right thing for subpackages like zope.app?
+        # It looks for a file named "zope.app.pkg"
+        pkgfile = os.path.join(dir, sname_pkg)
+        if os.path.isfile(pkgfile):
+            try:
+                f = open(pkgfile)
+            except IOError, msg:
+                sys.stderr.write("Can't open %s: %s\n" %
+                                 (pkgfile, msg))
+            else:
+                for line in f:
+                    line = line.rstrip('\n')
+                    if not line or line.startswith('#'):
+                        continue
+                    path.append(line) # Don't check for existence!
+                f.close()
+
+    return path
+
+def get_data(package, resource):
+    """Get a resource from a package.
+
+    This is a wrapper round the PEP 302 loader get_data API. The package
+    argument should be the name of a package, in standard module format
+    (foo.bar). The resource argument should be in the form of a relative
+    filename, using '/' as the path separator. The parent directory name '..'
+    is not allowed, and nor is a rooted name (starting with a '/').
+
+    The function returns a binary string, which is the contents of the
+    specified resource.
+
+    For packages located in the filesystem, which have already been imported,
+    this is the rough equivalent of
+
+        d = os.path.dirname(sys.modules[package].__file__)
+        data = open(os.path.join(d, resource), 'rb').read()
+
+    If the package cannot be located or loaded, or it uses a PEP 302 loader
+    which does not support get_data(), then None is returned.
+    """
+
+    loader = get_loader(package)
+    if loader is None or not hasattr(loader, 'get_data'):
+        return None
+    mod = sys.modules.get(package) or loader.load_module(package)
+    if mod is None or not hasattr(mod, '__file__'):
+        return None
+
+    # Modify the resource name to be compatible with the loader.get_data
+    # signature - an os.path format "filename" starting with the dirname of
+    # the package's __file__
+    parts = resource.split('/')
+    parts.insert(0, os.path.dirname(mod.__file__))
+    resource_name = os.path.join(*parts)
+    return loader.get_data(resource_name)

mitogen/fakessh.py

@@ -0,0 +1,377 @@
+"""
+fakessh is a stream implementation that starts a local subprocess with its
+environment modified such that ``PATH`` searches for `ssh` return an mitogen
+implementation of the SSH command. When invoked, this tool arranges for the
+command line supplied by the calling program to be executed in a context
+already established by the master process, reusing the master's (possibly
+proxied) connection to that context.
+
+This allows tools like `rsync` and `scp` to transparently reuse the connections
+and tunnels already established by the host program to connect to a target
+machine, without wasteful redundant SSH connection setup, 3-way handshakes,
+or firewall hopping configurations, and enables these tools to be used in
+impossible scenarios, such as over `sudo` with ``requiretty`` enabled.
+
+The fake `ssh` command source is written to a temporary file on disk, and
+consists of a copy of the :py:mod:`mitogen.core` source code (just like any
+other child context), with a line appended to cause it to connect back to the
+host process over an FD it inherits. As there is no reliance on an existing
+filesystem file, it is possible for child contexts to use fakessh.
+
+As a consequence of connecting back through an inherited FD, only one SSH
+invocation is possible, which is fine for tools like `rsync`, however in future
+this restriction will be lifted.
+
+Sequence:
+
+    1. ``fakessh`` Context and Stream created by parent context. The stream's
+       buffer has a `_fakessh_main()` ``CALL_FUNCTION`` enqueued.
+    2. Target program (`rsync/scp/sftp`) invoked, which internally executes
+       `ssh` from ``PATH``.
+    3. :py:mod:`mitogen.core` bootstrap begins, recovers the stream FD
+       inherited via the target program, established itself as the fakessh
+       context.
+    4. `_fakessh_main()` ``CALL_FUNCTION`` is read by fakessh context,
+        a. sets up :py:class:`mitogen.fakessh.IoPump` for stdio, registers
+           stdin_handle for local context.
+        b. Enqueues ``CALL_FUNCTION`` for `_start_slave()` invoked in target context,
+            i. the program from the `ssh` command line is started
+            ii. sets up :py:class:`mitogen.fakessh.IoPump` for `ssh` command
+                line process's stdio pipes
+            iii. returns `(control_handle, stdin_handle)` to `_fakessh_main()`
+    5. `_fakessh_main()` receives control/stdin handles from from `_start_slave()`,
+        a. registers remote's stdin_handle with local IoPump
+        b. sends `("start", local_stdin_handle)` to remote's control_handle
+        c. registers local IoPump with Broker
+        d. loops waiting for 'local stdout closed && remote stdout closed'
+    6. `_start_slave()` control channel receives `("start", stdin_handle)`,
+        a. registers remote's stdin_handle with local IoPump
+        b. registers local IoPump with Broker
+        c. loops waiting for 'local stdout closed && remote stdout closed'
+"""
+
+import getopt
+import inspect
+import logging
+import os
+import shutil
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+
+import mitogen.core
+import mitogen.master
+
+from mitogen.core import LOG, IOLOG
+
+
+SSH_GETOPTS = (
+    "1246ab:c:e:fgi:kl:m:no:p:qstvx"
+    "ACD:E:F:I:KL:MNO:PQ:R:S:TVw:W:XYy"
+)
+
+_mitogen = None
+
+
+class IoPump(mitogen.core.BasicStream):
+    _output_buf = ''
+    _closed = False
+
+    def __init__(self, process, broker, stdin_fd, stdout_fd):
+        self.process = process
+        self._broker = broker
+        self.receive_side = mitogen.core.Side(self, stdout_fd)
+        self.transmit_side = mitogen.core.Side(self, stdin_fd)
+
+    def write(self, s):
+        self._output_buf += s
+        self._broker.start_transmit(self)
+
+    def close(self):
+        self._closed = True
+        # If local process hasn't exitted yet, ensure its write buffer is
+        # drained before lazily triggering disconnect in on_transmit.
+        if self.transmit_side.fd is not None:
+            self._broker.start_transmit(self)
+
+    def on_shutdown(self, broker):
+        self.close()
+
+    def on_transmit(self, broker):
+        written = self.transmit_side.write(self._output_buf)
+        IOLOG.debug('%r.on_transmit() -> len %r', self, written)
+        if written is None:
+            self.on_disconnect(broker)
+        else:
+            self._output_buf = self._output_buf[written:]
+
+        if not self._output_buf:
+            broker.stop_transmit(self)
+            if self._closed:
+                self.on_disconnect(broker)
+
+    def on_receive(self, broker):
+        s = self.receive_side.read()
+        IOLOG.debug('%r.on_receive() -> len %r', self, len(s))
+        if s:
+            mitogen.core.fire(self, 'receive', s)
+        else:
+            self.on_disconnect(broker)
+
+    def __repr__(self):
+        return 'IoPump(%r)' % (
+            self.process,
+        )
+
+
+class Process(object):
+    """
+    Manages the lifetime and pipe connections of the SSH command running in the
+    slave.
+    """
+    def __init__(self, router, stdin_fd, stdout_fd, proc=None):
+        self.router = router
+        self.stdin_fd = stdin_fd
+        self.stdout_fd = stdout_fd
+        self.proc = proc
+        self.control_handle = router.add_handler(self._on_control)
+        self.stdin_handle = router.add_handler(self._on_stdin)
+        self.pump = IoPump(self, router.broker, stdin_fd, stdout_fd)
+        self.stdin = None
+        self.control = None
+        self.wake_event = threading.Event()
+
+        mitogen.core.listen(self.pump, 'disconnect', self._on_pump_disconnect)
+        mitogen.core.listen(self.pump, 'receive', self._on_pump_receive)
+
+        if proc:
+            pmon = mitogen.master.ProcessMonitor.instance()
+            pmon.add(proc.pid, self._on_proc_exit)
+
+    def __repr__(self):
+        return 'Process(%r, %r)' % (self.stdin_fd, self.stdout_fd)
+
+    def _on_proc_exit(self, status):
+        LOG.debug('%r._on_proc_exit(%r)', self, status)
+        self.control.put(('exit', status))
+
+    def _on_stdin(self, msg):
+        if msg == mitogen.core._DEAD:
+            return
+
+        data = msg.unpickle()
+        IOLOG.debug('%r._on_stdin(%r)', self, data)
+
+        if data == mitogen.core._DEAD:
+            self.pump.close()
+        else:
+            self.pump.write(data)
+
+    def _on_control(self, msg):
+        if msg != mitogen.core._DEAD:
+            command, arg = msg.unpickle()
+            LOG.debug('%r._on_control(%r, %s)', self, command, arg)
+
+            func = getattr(self, '_on_%s' % (command,), None)
+            if func:
+                return func(msg, arg)
+
+            LOG.warning('%r: unknown command %r', self, command)
+
+    def _on_start(self, msg, arg):
+        dest = mitogen.core.Context(self.router, msg.src_id)
+        self.control = mitogen.core.Sender(dest, arg[0])
+        self.stdin = mitogen.core.Sender(dest, arg[1])
+        self.router.broker.start_receive(self.pump)
+
+    def _on_exit(self, msg, arg):
+        LOG.debug('on_exit: proc = %r', self.proc)
+        if self.proc:
+            self.proc.terminate()
+        else:
+            self.router.broker.shutdown()
+
+    def _on_pump_receive(self, s):
+        IOLOG.info('%r._on_pump_receive()', self)
+        self.stdin.put(s)
+
+    def _on_pump_disconnect(self):
+        LOG.debug('%r._on_pump_disconnect()', self)
+        mitogen.core.fire(self, 'disconnect')
+        self.stdin.close()
+        self.wake_event.set()
+
+    def start_master(self, stdin, control):
+        self.stdin = stdin
+        self.control = control
+        control.put(('start', (self.control_handle, self.stdin_handle)))
+        self.router.broker.start_receive(self.pump)
+
+    def wait(self):
+        while not self.wake_event.wait(0.1):
+            pass
+
+
+def _start_slave(mitogen_, src_id, args):
+    """
+    This runs in the target context, it is invoked by _fakessh_main running in
+    the fakessh context immediately after startup. It starts the slave process
+    (the the point where it has a stdin_handle to target but not stdout_chan to
+    write to), and waits for main to.
+    """
+    LOG.debug('_start_slave(%r, %r)', mitogen_, args)
+
+    proc = subprocess.Popen(args,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+    )
+
+    process = Process(mitogen_.router,
+        proc.stdin.fileno(),
+        proc.stdout.fileno(),
+        proc,
+    )
+
+    return process.control_handle, process.stdin_handle
+
+
+#
+# SSH client interface.
+#
+
+
+def exit():
+    _mitogen.broker.shutdown()
+
+
+def die(msg, *args):
+    if args:
+        msg %= args
+    print msg
+    exit()
+
+
+def parse_args():
+    hostname = None
+    remain = sys.argv[1:]
+    allopts = []
+    restarted = 0
+
+    while remain and restarted < 2:
+        opts, args = getopt.getopt(remain, SSH_GETOPTS)
+        remain = remain[:]  # getopt bug!
+        allopts += opts
+        if not args:
+            break
+
+        if not hostname:
+            hostname = args.pop(0)
+            remain = remain[remain.index(hostname) + 1:]
+
+        restarted += 1
+
+    return hostname, allopts, args
+
+
+def _fakessh_main(mitogen_, dest_context_id):
+    hostname, opts, args = parse_args()
+    if not hostname:
+        die('Missing hostname')
+
+    for opt, optarg in opts:
+        if 0 and opt == '-s':
+            subsystem = True
+        else:
+            LOG.debug('Warning option %s %s is ignored.', opt, optarg)
+
+    LOG.debug('hostname: %r', hostname)
+    LOG.debug('opts: %r', opts)
+    LOG.debug('args: %r', args)
+
+    dest = mitogen.master.Context(mitogen_.router, dest_context_id)
+    control_handle, stdin_handle = dest.call_with_deadline(None, True,
+        _start_slave, mitogen.context_id, args)
+
+    LOG.debug('_fakessh_main: received control_handle=%r, stdin_handle=%r',
+              control_handle, stdin_handle)
+
+    process = Process(mitogen_.router, 1, 0)
+    process.start_master(
+        stdin=mitogen.core.Sender(dest, stdin_handle),
+        control=mitogen.core.Sender(dest, control_handle),
+    )
+    process.wait()
+    process.control.put(('exit', None))
+
+
+#
+# Public API.
+#
+
+def run(dest, router, args, deadline=None):
+    """
+    Run the command specified by the argument vector `args` such that ``PATH``
+    searches for SSH by the command will cause its attempt to use SSH to
+    execute a remote program to be redirected to use mitogen to execute that
+    program using the context `dest` instead.
+
+    :param mitogen.core.Context dest:
+        The destination context to execute the SSH command line in.
+
+    :param mitogen.core.Router router:
+
+    :param list[str] args:
+        Command line arguments for local program, e.g. ``['rsync', '/tmp', 'remote:/tmp']``
+    """
+    context_id = router.context_id_counter.next()
+    fakessh = mitogen.master.Context(router, context_id)
+    fakessh.name = 'fakessh'
+
+    sock1, sock2 = socket.socketpair()
+    mitogen.core.set_cloexec(sock1.fileno())
+
+    stream = mitogen.core.Stream(router, context_id, fakessh.key)
+    stream.name = 'fakessh'
+    stream.accept(sock1.fileno(), sock1.fileno())
+    router.register(fakessh, stream)
+
+    # Held in socket buffer until process is booted.
+    fakessh.call_async(True, _fakessh_main, dest.context_id)
+
+    tmp_path = tempfile.mkdtemp(prefix='mitogen_fakessh')
+    try:
+        ssh_path = os.path.join(tmp_path, 'ssh')
+        fp = file(ssh_path, 'w')
+        try:
+            fp.write('#!/usr/bin/env python\n')
+            fp.write(inspect.getsource(mitogen.core))
+            fp.write('\n')
+            fp.write('ExternalContext().main%r\n' % ((
+                mitogen.context_id,             # parent_id
+                context_id,                     # context_id
+                fakessh.key,                    # key
+                router.debug,                   # debug
+                logging.getLogger().level,      # log_level
+                sock2.fileno(),                 # in_fd
+                sock2.fileno(),                 # out_fd
+                None,                           # core_src_fd
+                False,                          # setup_stdio
+            ),))
+        finally:
+            fp.close()
+
+        os.chmod(ssh_path, 0755)
+        env = os.environ.copy()
+        env.update({
+            'PATH': '%s:%s' % (tmp_path, env.get('PATH', '')),
+            'ARGV0': `[sys.executable]`,
+            'SSH_PATH': ssh_path,
+        })
+
+        proc = subprocess.Popen(args, env=env)
+        proc.wait()
+    finally:
+        shutil.rmtree(tmp_path)

mitogen/master.py

@@ -0,0 +1,642 @@
+"""
+This module implements functionality required by master processes, such as
+starting new contexts via SSH. Its size is also restricted, since it must be
+sent to any context that will be used to establish additional child contexts.
+"""
+
+import errno
+import getpass
+import imp
+import inspect
+import itertools
+import logging
+import os
+import pkgutil
+import re
+import select
+import signal
+import socket
+import sys
+import textwrap
+import time
+import types
+import zlib
+
+if not hasattr(pkgutil, 'find_loader'):
+    # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has
+    # been kept intentionally 2.3 compatible so we can reuse it.
+    from mitogen.compat import pkgutil
+
+import mitogen.core
+
+
+LOG = logging.getLogger('mitogen')
+IOLOG = logging.getLogger('mitogen.io')
+RLOG = logging.getLogger('ctx')
+
+DOCSTRING_RE = re.compile(r'""".+?"""', re.M | re.S)
+COMMENT_RE = re.compile(r'^[ ]*#[^\n]*$', re.M)
+IOLOG_RE = re.compile(r'^[ ]*IOLOG.debug\(.+?\)$', re.M)
+
+PERMITTED_CLASSES = set([
+    ('mitogen.core', 'CallError'),
+    ('mitogen.core', 'Dead'),
+])
+
+
+def minimize_source(source):
+    """Remove comments and docstrings from Python `source`, preserving line
+    numbers and syntax of empty blocks."""
+    subber = lambda match: '""' + ('\n' * match.group(0).count('\n'))
+    source = DOCSTRING_RE.sub(subber, source)
+    source = COMMENT_RE.sub('', source)
+    return source.replace('    ', '\t')
+
+
+def get_child_modules(path, fullname):
+    """Return the canonical names of all submodules of a package `module`."""
+    it = pkgutil.iter_modules([os.path.dirname(path)])
+    return ['%s.%s' % (fullname, name) for _, name, _ in it]
+
+
+def create_child(*args):
+    """Create a child process whose stdin/stdout is connected to a socket,
+    returning `(pid, socket_obj)`."""
+    parentfp, childfp = socket.socketpair()
+    pid = os.fork()
+    if not pid:
+        os.dup2(childfp.fileno(), 0)
+        os.dup2(childfp.fileno(), 1)
+        childfp.close()
+        parentfp.close()
+        os.execvp(args[0], args)
+        raise SystemExit
+
+    childfp.close()
+    LOG.debug('create_child() child %d fd %d, parent %d, args %r',
+              pid, parentfp.fileno(), os.getpid(), args)
+    return pid, os.dup(parentfp.fileno())
+
+
+def write_all(fd, s):
+    written = 0
+    while written < len(s):
+        rc = os.write(fd, buffer(s, written))
+        if not rc:
+            raise IOError('short write')
+        written += rc
+    return written
+
+
+def read_with_deadline(fd, size, deadline):
+    timeout = deadline - time.time()
+    if timeout > 0:
+        rfds, _, _ = select.select([fd], [], [], timeout)
+        if rfds:
+            return os.read(fd, size)
+
+    raise mitogen.core.TimeoutError('read timed out')
+
+
+def iter_read(fd, deadline):
+    if deadline is not None:
+        LOG.debug('Warning: iter_read(.., deadline=...) unimplemented')
+
+    bits = []
+    while True:
+        s, disconnected = mitogen.core.io_op(os.read, fd, 4096)
+        if disconnected:
+            s = ''
+
+        if not s:
+            raise mitogen.core.StreamError(
+                'EOF on stream; last 100 bytes received: %r' %
+                (''.join(bits)[-100:],)
+            )
+
+        bits.append(s)
+        yield s
+
+
+def discard_until(fd, s, deadline):
+    for buf in iter_read(fd, deadline):
+        if buf.endswith(s):
+            return
+
+
+class LogForwarder(object):
+    def __init__(self, router):
+        self._router = router
+        self._cache = {}
+        router.add_handler(self._on_forward_log, mitogen.core.FORWARD_LOG)
+
+    def _on_forward_log(self, msg):
+        if msg == mitogen.core._DEAD:
+            return
+
+        logger = self._cache.get(msg.src_id)
+        if logger is None:
+            context = self._router.context_by_id(msg.src_id)
+            if context is None:
+                LOG.error('FORWARD_LOG received from src_id %d', msg.src_id)
+                return
+
+            name = '%s.%s' % (RLOG.name, context.name)
+            self._cache[msg.src_id] = logger = logging.getLogger(name)
+
+        name, level_s, s = msg.data.split('\x00', 2)
+        logger.log(int(level_s), '%s: %s', name, s)
+
+    def __repr__(self):
+        return 'LogForwarder(%r)' % (self._router,)
+
+
+class ModuleResponder(object):
+    def __init__(self, router):
+        self._router = router
+        router.add_handler(self._on_get_module, mitogen.core.GET_MODULE)
+
+    def __repr__(self):
+        return 'ModuleResponder(%r)' % (self._router,)
+
+    def _get_module_via_pkgutil(self, fullname):
+        """Attempt to fetch source code via pkgutil. In an ideal world, this
+        would be the only required implementation of get_module()."""
+        loader = pkgutil.find_loader(fullname)
+        LOG.debug('pkgutil.find_loader(%r) -> %r', fullname, loader)
+        if not loader:
+            return
+
+        try:
+            path = loader.get_filename(fullname)
+            source = loader.get_source(fullname)
+            if path is not None and source is not None:
+                return path, source, loader.is_package(fullname)
+        except AttributeError:
+            return
+
+    def _get_module_via_sys_modules(self, fullname):
+        """Attempt to fetch source code via sys.modules. This is specifically
+        to support __main__, but it may catch a few more cases."""
+        if fullname not in sys.modules:
+            LOG.debug('%r does not appear in sys.modules', fullname)
+            return
+
+        is_pkg = hasattr(sys.modules[fullname], '__path__')
+        try:
+            source = inspect.getsource(sys.modules[fullname])
+        except IOError:
+            # Work around inspect.getsourcelines() bug.
+            if not is_pkg:
+                raise
+            source = '\n'
+
+        return (sys.modules[fullname].__file__.rstrip('co'),
+                source,
+                hasattr(sys.modules[fullname], '__path__'))
+
+    def _get_module_via_parent_enumeration(self, fullname):
+        """Attempt to fetch source code by examining the module's (hopefully
+        less insane) parent package. Required for ansible.compat.six."""
+        pkgname, _, modname = fullname.rpartition('.')
+        pkg = sys.modules.get(pkgname)
+        if pkg is None or not hasattr(pkg, '__file__'):
+            return
+
+        pkg_path = os.path.dirname(pkg.__file__)
+        try:
+            fp, path, ext = imp.find_module(modname, [pkg_path])
+            LOG.error('%r', (fp, path, ext))
+            return path, fp.read(), False
+        except ImportError, e:
+            LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e)
+
+    get_module_methods = [_get_module_via_pkgutil,
+                          _get_module_via_sys_modules,
+                          _get_module_via_parent_enumeration]
+
+    def _on_get_module(self, msg):
+        LOG.debug('%r.get_module(%r)', self, msg)
+        if msg == mitogen.core._DEAD:
+            return
+
+        fullname = msg.data
+        try:
+            for method in self.get_module_methods:
+                tup = method(self, fullname)
+                if tup:
+                    break
+
+            try:
+                path, source, is_pkg = tup
+            except TypeError:
+                raise ImportError('could not find %r' % (fullname,))
+
+            LOG.debug('%s found %r: (%r, .., %r)',
+                      method.__name__, fullname, path, is_pkg)
+            if is_pkg:
+                pkg_present = get_child_modules(path, fullname)
+                LOG.debug('get_child_modules(%r, %r) -> %r',
+                          path, fullname, pkg_present)
+            else:
+                pkg_present = None
+
+            compressed = zlib.compress(source)
+            self._router.route(
+                mitogen.core.Message.pickled(
+                    (pkg_present, path, compressed),
+                    dst_id=msg.src_id,
+                    handle=msg.reply_to,
+                )
+            )
+        except Exception:
+            LOG.debug('While importing %r', fullname, exc_info=True)
+            self._router.route(
+                mitogen.core.Message.pickled(
+                    None,
+                    dst_id=msg.src_id,
+                    handle=msg.reply_to,
+                )
+            )
+
+
+class ModuleForwarder(object):
+    """
+    Respond to GET_MODULE requests in a slave by forwarding the request to our
+    parent context, or satisfying the request from our local Importer cache.
+    """
+    def __init__(self, router, parent_context, importer):
+        self.router = router
+        self.parent_context = parent_context
+        self.importer = importer
+        router.add_handler(self._on_get_module, mitogen.core.GET_MODULE)
+
+    def __repr__(self):
+        return 'ModuleForwarder(%r)' % (self.router,)
+
+    def _on_get_module(self, msg):
+        LOG.debug('%r._on_get_module(%r)', self, msg)
+        if msg == mitogen.core._DEAD:
+            return
+
+        fullname = msg.data
+        cached = self.importer._cache.get(fullname)
+        if cached:
+            LOG.debug('%r._on_get_module(): using cached %r', self, fullname)
+            self.router.route(
+                mitogen.core.Message.pickled(
+                    cached,
+                    dst_id=msg.src_id,
+                    handle=msg.reply_to,
+                )
+            )
+        else:
+            LOG.debug('%r._on_get_module(): requesting %r', self, fullname)
+            self.parent_context.send(
+                mitogen.core.Message(
+                    data=msg.data,
+                    handle=mitogen.core.GET_MODULE,
+                    reply_to=self.router.add_handler(
+                        lambda m: self._on_got_source(m, msg),
+                        persist=False
+                    )
+                )
+            )
+
+    def _on_got_source(self, msg, original_msg):
+        LOG.debug('%r._on_got_source(%r, %r)', self, msg, original_msg)
+        fullname = original_msg.data
+        self.importer._cache[fullname] = msg.unpickle()
+        self.router.route(
+            mitogen.core.Message(
+                data=msg.data,
+                dst_id=original_msg.src_id,
+                handle=original_msg.reply_to,
+            )
+        )
+
+
+class Message(mitogen.core.Message):
+    """
+    Message subclass that controls unpickling.
+    """
+    def _find_global(self, module_name, class_name):
+        """Return the class implementing `module_name.class_name` or raise
+        `StreamError` if the module is not whitelisted."""
+        if (module_name, class_name) not in PERMITTED_CLASSES:
+            raise mitogen.core.StreamError(
+                '%r attempted to unpickle %r in module %r',
+                self._context, class_name, module_name)
+        return getattr(sys.modules[module_name], class_name)
+
+
+class Stream(mitogen.core.Stream):
+    """
+    Base for streams capable of starting new slaves.
+    """
+    message_class = Message
+
+    #: The path to the remote Python interpreter.
+    python_path = 'python2.7'
+
+    #: True to cause context to write verbose /tmp/mitogen.<pid>.log.
+    debug = False
+
+    def construct(self, remote_name=None, python_path=None, debug=False, **kwargs):
+        """Get the named context running on the local machine, creating it if
+        it does not exist."""
+        super(Stream, self).construct(**kwargs)
+        if python_path:
+            self.python_path = python_path
+
+        if remote_name is None:
+            remote_name = '%s@%s:%d'
+            remote_name %= (getpass.getuser(), socket.gethostname(), os.getpid())
+        self.remote_name = remote_name
+        self.debug = debug
+
+    def on_shutdown(self, broker):
+        """Request the slave gracefully shut itself down."""
+        LOG.debug('%r closing CALL_FUNCTION channel', self)
+        self.send(
+            mitogen.core.Message.pickled(
+                mitogen.core._DEAD,
+                src_id=mitogen.context_id,
+                dst_id=self.remote_id,
+                handle=mitogen.core.CALL_FUNCTION
+            )
+        )
+
+    # base64'd and passed to 'python -c'. It forks, dups 0->100, creates a
+    # pipe, then execs a new interpreter with a custom argv. 'CONTEXT_NAME' is
+    # replaced with the context name. Optimized for size.
+    def _first_stage():
+        import os,sys,zlib
+        R,W=os.pipe()
+        R2,W2=os.pipe()
+        if os.fork():
+            os.dup2(0,100)
+            os.dup2(R,0)
+            os.dup2(R2,101)
+            for f in R,R2,W,W2: os.close(f)
+            os.environ['ARGV0'] = `[sys.executable]`
+            os.execv(sys.executable,['mitogen:CONTEXT_NAME'])
+        else:
+            os.write(1, 'EC0\n')
+            C = zlib.decompress(sys.stdin.read(input()))
+            os.fdopen(W,'w',0).write(C)
+            os.fdopen(W2,'w',0).write('%s\n%s' % (len(C),C))
+            os.write(1, 'EC1\n')
+            sys.exit(0)
+
+    def get_boot_command(self):
+        source = inspect.getsource(self._first_stage)
+        source = textwrap.dedent('\n'.join(source.strip().split('\n')[1:]))
+        source = source.replace('    ', '\t')
+        source = source.replace('CONTEXT_NAME', self.remote_name)
+        encoded = source.encode('base64').replace('\n', '')
+        return [self.python_path, '-c',
+                'exec("%s".decode("base64"))' % (encoded,)]
+
+    def get_preamble(self):
+        source = inspect.getsource(mitogen.core)
+        source += '\nExternalContext().main%r\n' % ((
+            mitogen.context_id,       # parent_id
+            self.remote_id,            # context_id
+            self.key,
+            self.debug,
+            LOG.level or logging.getLogger().level or logging.INFO,
+        ),)
+
+        compressed = zlib.compress(minimize_source(source))
+        return str(len(compressed)) + '\n' + compressed
+
+    create_child = staticmethod(create_child)
+
+    def connect(self):
+        LOG.debug('%r.connect()', self)
+        pid, fd = self.create_child(*self.get_boot_command())
+        self.name = 'local.%s' % (pid,)
+        self.receive_side = mitogen.core.Side(self, fd)
+        self.transmit_side = mitogen.core.Side(self, os.dup(fd))
+        LOG.debug('%r.connect(): child process stdin/stdout=%r',
+                  self, self.receive_side.fd)
+
+        self._connect_bootstrap()
+
+    def _ec0_received(self):
+        LOG.debug('%r._ec0_received()', self)
+        write_all(self.transmit_side.fd, self.get_preamble())
+        discard_until(self.receive_side.fd, 'EC1\n', time.time() + 10.0)
+
+    def _connect_bootstrap(self):
+        discard_until(self.receive_side.fd, 'EC0\n', time.time() + 10.0)
+        self._ec0_received()
+
+
+class Broker(mitogen.core.Broker):
+    shutdown_timeout = 5.0
+
+
+class Context(mitogen.core.Context):
+    via = None
+
+    def on_disconnect(self, broker):
+        """
+        Override base behaviour of triggering Broker shutdown on parent stream
+        disconnection.
+        """
+        mitogen.core.fire(self, 'disconnect')
+
+    def _discard_result(self, msg):
+        data = msg.unpickle()
+        if isinstance(data, Exception):
+            try:
+                raise data
+            except Exception:
+                LOG.exception('_discard_result')
+        else:
+            LOG.debug('_discard_result: %r', data)
+
+    def call_async(self, with_context, fn, *args, **kwargs):
+        LOG.debug('%r.call_async(%r, %r, *%r, **%r)',
+                  self, with_context, fn, args, kwargs)
+
+        if isinstance(fn, types.MethodType) and \
+           isinstance(fn.im_self, (type, types.ClassType)):
+            klass = fn.im_self.__name__
+        else:
+            klass = None
+
+        call = (with_context, fn.__module__, klass, fn.__name__, args, kwargs)
+        self.send(
+            mitogen.core.Message.pickled(
+                call,
+                handle=mitogen.core.CALL_FUNCTION,
+                reply_to=self.router.add_handler(self._discard_result),
+            )
+        )
+
+    def call_with_deadline(self, deadline, with_context, fn, *args, **kwargs):
+        """Invoke `fn([context,] *args, **kwargs)` in the external context.
+
+        If `with_context` is ``True``, pass its
+        :py:class:`ExternalContext <mitogen.core.ExternalContext>` instance as
+        the first parameter.
+
+        If `deadline` is not ``None``, expire the call after `deadline`
+        seconds. If `deadline` is ``None``, the invocation may block
+        indefinitely."""
+        LOG.debug('%r.call_with_deadline(%r, %r, %r, *%r, **%r)',
+                  self, deadline, with_context, fn, args, kwargs)
+
+        if isinstance(fn, types.MethodType) and \
+           isinstance(fn.im_self, (type, types.ClassType)):
+            klass = fn.im_self.__name__
+        else:
+            klass = None
+
+        call = (with_context, fn.__module__, klass, fn.__name__, args, kwargs)
+        response = self.send_await(
+            mitogen.core.Message.pickled(
+                call,
+                handle=mitogen.core.CALL_FUNCTION
+            ),
+            deadline
+        )
+
+        decoded = response.unpickle()
+        if isinstance(decoded, mitogen.core.CallError):
+            raise decoded
+        return decoded
+
+    def call(self, fn, *args, **kwargs):
+        """Invoke `fn(*args, **kwargs)` in the external context."""
+        return self.call_with_deadline(None, False, fn, *args, **kwargs)
+
+
+def _proxy_connect(mitogen, name, context_id, klass, kwargs):
+    if not isinstance(mitogen.router, Router):  # TODO
+        mitogen.router.__class__ = Router  # TODO
+        LOG.debug('_proxy_connect(): constructing ModuleForwarder')
+        ModuleForwarder(mitogen.router, mitogen.parent, mitogen.importer)
+
+    context = mitogen.router._connect(
+        context_id,
+        klass,
+        name=name,
+        **kwargs
+    )
+    return context.name
+
+
+class Router(mitogen.core.Router):
+    context_id_counter = itertools.count(1)
+
+    debug = False
+
+    def __init__(self, *args, **kwargs):
+        super(Router, self).__init__(*args, **kwargs)
+        self.responder = ModuleResponder(self)
+        self.log_forwarder = LogForwarder(self)
+
+    def enable_debug(self):
+        """
+        Cause this context and any descendant child contexts to write debug
+        logs to /tmp/mitogen.<pid>.log.
+        """
+        mitogen.core.enable_debug_logging()
+        self.debug = True
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, e_type, e_val, tb):
+        self.broker.shutdown()
+        self.broker.join()
+
+    def context_by_id(self, context_id):
+        return self._context_by_id.get(context_id)
+
+    def local(self, **kwargs):
+        return self.connect(Stream, **kwargs)
+
+    def sudo(self, **kwargs):
+        import mitogen.sudo
+        return self.connect(mitogen.sudo.Stream, **kwargs)
+
+    def ssh(self, **kwargs):
+        import mitogen.ssh
+        return self.connect(mitogen.ssh.Stream, **kwargs)
+
+    def _connect(self, context_id, klass, name=None, **kwargs):
+        context = Context(self, context_id)
+        stream = klass(self, context.context_id, context.key, **kwargs)
+        if name is not None:
+            stream.name = name
+        stream.connect()
+        context.name = stream.name
+        self.register(context, stream)
+        return context
+
+    def connect(self, klass, name=None, **kwargs):
+        kwargs.setdefault('debug', self.debug)
+
+        via = kwargs.pop('via', None)
+        if via is not None:
+            return self.proxy_connect(via, klass, name=name, **kwargs)
+
+        context_id = self.context_id_counter.next()
+        return self._connect(context_id, klass, name=name, **kwargs)
+
+    def proxy_connect(self, via_context, klass, name=None, **kwargs):
+        context_id = self.context_id_counter.next()
+        # Must be added prior to _proxy_connect() to avoid a race.
+        self.add_route(context_id, via_context.context_id)
+        name = via_context.call_with_deadline(None, True,
+            _proxy_connect, name, context_id, klass, kwargs
+        )
+        # name = '%s.%s' % (via_context.name, name)
+        context = Context(self, context_id, name=name)
+        context.via = via_context
+
+        child = via_context
+        parent = via_context.via
+        while parent is not None:
+            LOG.debug('Adding route to %r for %r via %r', parent, context, child)
+            parent.send(
+                mitogen.core.Message(
+                    data='%s\x00%s' % (context_id, child.context_id),
+                    handle=mitogen.core.ADD_ROUTE,
+                )
+            )
+            child = parent
+            parent = parent.via
+
+        self._context_by_id[context.context_id] = context
+        return context
+
+
+class ProcessMonitor(object):
+    def __init__(self):
+        # pid -> callback()
+        self.callback_by_pid = {}
+        signal.signal(signal.SIGCHLD, self._on_sigchld)
+
+    def _on_sigchld(self, _signum, _frame):
+        for pid, callback in self.callback_by_pid.items():
+            pid, status = os.waitpid(pid, os.WNOHANG)
+            if pid:
+                callback(status)
+                del self.callback_by_pid[pid]
+
+    def add(self, pid, callback):
+        self.callback_by_pid[pid] = callback
+
+    _instance = None
+
+    @classmethod
+    def instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance

mitogen/ssh.py

@@ -0,0 +1,33 @@
+"""
+Functionality to allow establishing new slave contexts over an SSH connection.
+"""
+
+import commands
+
+import mitogen.master
+
+
+class Stream(mitogen.master.Stream):
+    python_path = 'python'
+
+    #: The path to the SSH binary.
+    ssh_path = 'ssh'
+
+    def construct(self, hostname, username=None, ssh_path=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        self.hostname = hostname
+        self.username = username
+        if ssh_path:
+            self.ssh_path = ssh_path
+
+    def get_boot_command(self):
+        bits = [self.ssh_path]
+        if self.username:
+            bits += ['-l', self.username]
+        bits.append(self.hostname)
+        base = super(Stream, self).get_boot_command()
+        return bits + map(commands.mkarg, base)
+
+    def connect(self):
+        super(Stream, self).connect()
+        self.name = 'ssh.' + self.hostname

mitogen/sudo.py

@@ -0,0 +1,155 @@
+
+import logging
+import os
+import pty
+import termios
+import time
+
+import mitogen.core
+import mitogen.master
+
+
+LOG = logging.getLogger(__name__)
+PASSWORD_PROMPT = 'password'
+
+
+class PasswordError(mitogen.core.Error):
+    pass
+
+
+def flags(names):
+    """Return the result of ORing a set of (space separated) :py:mod:`termios`
+    module constants together."""
+    return sum(getattr(termios, name) for name in names.split())
+
+
+def cfmakeraw((iflag, oflag, cflag, lflag, ispeed, ospeed, cc)):
+    """Given a list returned by :py:func:`termios.tcgetattr`, return a list
+    that has been modified in the same manner as the `cfmakeraw()` C library
+    function."""
+    iflag &= ~flags('IGNBRK BRKINT PARMRK ISTRIP INLCR IGNCR ICRNL IXON')
+    oflag &= ~flags('OPOST IXOFF')
+    lflag &= ~flags('ECHO ECHOE ECHONL ICANON ISIG IEXTEN')
+    cflag &= ~flags('CSIZE PARENB')
+    cflag |= flags('CS8')
+
+    iflag = 0
+    oflag = 0
+    lflag = 0
+    return [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]
+
+
+def disable_echo(fd):
+    old = termios.tcgetattr(fd)
+    new = cfmakeraw(old)
+    flags = (
+        termios.TCSAFLUSH |
+        getattr(termios, 'TCSASOFT', 0)
+    )
+    termios.tcsetattr(fd, flags, new)
+
+
+def close_nonstandard_fds():
+    for fd in xrange(3, 1024):
+        try:
+            os.close(fd)
+        except OSError:
+            pass
+
+
+def tty_create_child(*args):
+    """
+    Return a file descriptor connected to the master end of a pseudo-terminal,
+    whose slave end is connected to stdin/stdout/stderr of a new child process.
+    The child is created such that the pseudo-terminal becomes its controlling
+    TTY, ensuring access to /dev/tty returns a new file descriptor open on the
+    slave end.
+
+    :param args:
+        execl() arguments.
+    """
+    master_fd, slave_fd = os.openpty()
+    disable_echo(master_fd)
+    disable_echo(slave_fd)
+
+    pid = os.fork()
+    if not pid:
+        os.dup2(slave_fd, 0)
+        os.dup2(slave_fd, 1)
+        os.dup2(slave_fd, 2)
+        close_nonstandard_fds()
+        os.setsid()
+        os.close(os.open(os.ttyname(1), os.O_RDWR))
+        os.execvp(args[0], args)
+        raise SystemExit
+
+    os.close(slave_fd)
+    LOG.debug('tty_create_child() child %d fd %d, parent %d, args %r',
+              pid, master_fd, os.getpid(), args)
+    return pid, master_fd
+
+
+class Stream(mitogen.master.Stream):
+    create_child = staticmethod(tty_create_child)
+    sudo_path = 'sudo'
+    password = None
+
+    def construct(self, username=None, sudo_path=None, password=None, **kwargs):
+        """
+        Get the named sudo context, creating it if it does not exist.
+
+        :param mitogen.core.Broker broker:
+            The broker that will own the context.
+
+        :param str username:
+            Username to pass to sudo as the ``-u`` parameter, defaults to ``root``.
+
+        :param str sudo_path:
+            Filename or complete path to the sudo binary. ``PATH`` will be searched
+            if given as a filename. Defaults to ``sudo``.
+
+        :param str python_path:
+            Filename or complete path to the Python binary. ``PATH`` will be
+            searched if given as a filename. Defaults to :py:data:`sys.executable`.
+
+        :param str password:
+            The password to use when authenticating to sudo. Depending on the sudo
+            configuration, this is either the current account password or the
+            target account password. :py:class:`mitogen.sudo.PasswordError` will
+            be raised if sudo requests a password but none is provided.
+
+        """
+        super(Stream, self).construct(**kwargs)
+        self.username = username or 'root'
+        if sudo_path:
+            self.sudo_path = sudo_path
+        if password:
+            self.password = password
+        self.name = 'sudo.' + self.username
+
+    def get_boot_command(self):
+        bits = [self.sudo_path, '-u', self.username]
+        bits = bits + super(Stream, self).get_boot_command()
+        LOG.debug('sudo command line: %r', bits)
+        return bits
+
+    password_incorrect_msg = 'sudo password is incorrect'
+    password_required_msg = 'sudo password is required'
+
+    def _connect_bootstrap(self):
+        password_sent = False
+        for buf in mitogen.master.iter_read(self.receive_side.fd,
+                                             time.time() + 10.0):
+            LOG.debug('%r: received %r', self, buf)
+            if buf.endswith('EC0\n'):
+                return self._ec0_received()
+            elif PASSWORD_PROMPT in buf.lower():
+                if self.password is None:
+                    raise PasswordError(self.password_required_msg)
+                if password_sent:
+                    raise PasswordError(self.password_incorrect_msg)
+                LOG.debug('sending password')
+                os.write(self.transmit_side.fd, self.password + '\n')
+                password_sent = True
+        else:
+            raise mitogen.core.StreamError('bootstrap failed')

mitogen/tcp.py

@@ -0,0 +1,42 @@
+"""
+Functionality to allow a slave context to reconnect back to its master using a
+plain TCP connection.
+"""
+
+import socket
+
+import mitogen.core
+
+
+class Listener(mitogen.core.BasicStream):
+    def __init__(self, broker, address=None, backlog=30):
+        self._broker = broker
+        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self._sock.bind(address or ('0.0.0.0', 0))
+        self._sock.listen(backlog)
+        mitogen.core.set_cloexec(self._sock.fileno())
+        self.address = self._sock.getsockname()
+        self.receive_side = mitogen.core.Side(self, self._sock.fileno())
+        broker.start_receive(self)
+
+    def on_receive(self, broker):
+        sock, addr = self._sock.accept()
+        context = Context(self._broker, name=addr)
+        stream = mitogen.core.Stream(context)
+        stream.accept(sock.fileno(), sock.fileno())
+
+
+def listen(broker, address=None, backlog=30):
+    """Listen on `address` for connections from newly spawned contexts."""
+    return Listener(broker, address, backlog)
+
+
+def connect(context):
+    """Connect to a Broker at the address specified in our associated
+    Context."""
+    LOG.debug('%s.connect()', __name__)
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    self.receive_side = mitogen.core.Side(self, sock.fileno())
+    self.transmit_side = mitogen.core.Side(self, sock.fileno())
+    sock.connect(self._context.parent_addr)
+    self.enqueue(0, self._context.name)

mitogen/utils.py

@@ -0,0 +1,78 @@
+"""
+A random assortment of utility functions useful on masters and slaves.
+"""
+
+import logging
+import sys
+
+import mitogen
+import mitogen.core
+import mitogen.master
+
+
+LOG = logging.getLogger('mitogen')
+
+
+def disable_site_packages():
+    """Remove all entries mentioning site-packages or Extras from the system
+    path. Used primarily for testing on OS X within a virtualenv, where OS X
+    bundles some ancient version of the 'six' module."""
+    for entry in sys.path[:]:
+        if 'site-packages' in entry or 'Extras' in entry:
+            sys.path.remove(entry)
+
+
+def log_to_tmp():
+    import os
+    log_to_file(path='/tmp/mitogen.%s.log' % (os.getpid(),))
+
+
+def log_to_file(path=None, io=True, level=logging.INFO):
+    """Install a new :py:class:`logging.Handler` writing applications logs to
+    the filesystem. Useful when debugging slave IO problems."""
+    log = logging.getLogger('')
+    if path:
+        fp = open(path, 'w', 1)
+        mitogen.core.set_cloexec(fp.fileno())
+    else:
+        fp = sys.stderr
+
+    log.setLevel(level)
+    if io:
+        logging.getLogger('mitogen.io').setLevel(level)
+
+    fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s'
+    datefmt = '%H:%M:%S'
+    handler = logging.StreamHandler(fp)
+    handler.formatter = logging.Formatter(fmt, datefmt)
+    log.handlers.insert(0, handler)
+
+
+def run_with_router(func, *args, **kwargs):
+    """Arrange for `func(broker, *args, **kwargs)` to run with a temporary
+    :py:class:`mitogen.master.Router`, ensuring the Router and Broker are
+    correctly shut down during normal or exceptional return."""
+    broker = mitogen.master.Broker()
+    router = mitogen.master.Router(broker)
+    try:
+        return func(router, *args, **kwargs)
+    finally:
+        broker.shutdown()
+        broker.join()
+
+
+def with_router(func):
+    """Decorator version of :py:func:`run_with_broker`. Example:
+
+    .. code-block:: python
+
+        @with_broker
+        def do_stuff(broker, arg):
+            pass
+
+        do_stuff(blah, 123)
+    """
+    def wrapper(*args, **kwargs):
+        return run_with_router(func, *args, **kwargs)
+    wrapper.func_name = func.func_name
+    return wrapper

preamble_size.py

@@ -0,0 +1,32 @@
+"""
+Print the size of a typical SSH command line and the bootstrap code sent to new
+contexts.
+"""
+
+import inspect
+import zlib
+
+import mitogen.master
+import mitogen.ssh
+import mitogen.sudo
+
+broker = mitogen.master.Broker()
+
+router = mitogen.core.Router(broker)
+context = mitogen.master.Context(router, 0)
+stream = mitogen.ssh.Stream(router, 0, context.key, hostname='foo')
+broker.shutdown()
+
+print 'SSH command size: %s' % (len(' '.join(stream.get_boot_command())),)
+print 'Preamble size: %s (%.2fKiB)' % (
+    len(stream.get_preamble()),
+    len(stream.get_preamble()) / 1024.0,
+)
+
+for mod in (
+        mitogen.master,
+        mitogen.ssh,
+        mitogen.sudo,
+        ):
+    sz = len(zlib.compress(mitogen.master.minimize_source(inspect.getsource(mod))))
+    print '%s size: %s (%.2fKiB)' % (mod.__name__, sz, sz / 1024.0)

setup.py

@@ -0,0 +1,13 @@
+
+from distutils.core import setup
+
+setup(
+    name = 'mitogen',
+    version = '0.0.0',
+    description = 'Library for writing distributed self-replicating programs. THIS PACKAGE IS INCOMPLETE. IT IS BEING UPLOADED BECAUSE PYPI MAINTAINERS BROKE THE REGISTER COMMAND',
+    author = 'David Wilson',
+    license = 'OpenLDAP BSD',
+    url = 'http://github.com/dw/mitogen/',
+    py_packages = ['Mitogen'],
+    zip_safe = False
+)

tests/data/fakessh.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+import optparse
+import os
+import shlex
+import sys
+
+parser = optparse.OptionParser()
+parser.add_option('--user', '-l', action='store')
+parser.disable_interspersed_args()
+
+opts, args = parser.parse_args(sys.argv[1:])
+args.pop(0)  # hostname
+args = [''.join(shlex.split(s)) for s in args]
+print args
+os.execvp(args[0], args)

tests/data/plain_old_module.py

@@ -0,0 +1,10 @@
+"""
+I am a plain old module with no interesting dependencies or import machinery
+fiddlery.
+"""
+
+import math
+
+
+def pow(x, y):
+    return x ** y

tests/data/self_contained_program.py

@@ -0,0 +1,22 @@
+"""
+I am a self-contained program!
+"""
+
+import mitogen.master
+
+
+def repr_stuff():
+    return repr([__name__, 50])
+
+
+def main():
+    broker = mitogen.master.Broker()
+    try:
+        context = mitogen.master.connect(broker)
+        print context.call(repr_stuff)
+    finally:
+        broker.shutdown()
+        broker.join()
+
+if __name__ == '__main__' and not mitogen.slave:
+    main()

tests/data/simple_pkg/a.py

@@ -0,0 +1,6 @@
+
+import simple_pkg.b
+
+
+def subtract_one_add_two(n):
+    return simple_pkg.b.subtract_one(n) + 2

tests/data/simple_pkg/b.py

@@ -0,0 +1,3 @@
+
+def subtract_one(n):
+    return n - 1

tests/data/six_brokenpkg/__init__.py

@@ -0,0 +1,56 @@
+# (c) 2014, Toshio Kuratomi <tkuratomi@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+'''
+Compat six library.  RHEL7 has python-six 1.3.0 which is too old
+'''
+# The following makes it easier for us to script updates of the bundled code
+_BUNDLED_METADATA = { "pypi_name": "six", "version": "1.10.0" }
+
+import os.path
+
+try:
+    import six as _system_six
+    print('unsocks', _system_six)
+except ImportError, e:
+    print('cocks', e)
+    _system_six = None
+
+if _system_six:
+    # If we need some things from even newer versions of six, then we need to
+    # use our bundled copy instead
+
+       if ( # Added in six-1.8.0
+            not hasattr(_system_six.moves, 'shlex_quote') or
+            # Added in six-1.4.0
+            not hasattr(_system_six, 'byte2int') or
+            not hasattr(_system_six, 'add_metaclass') or
+            not hasattr(_system_six.moves, 'urllib')
+            ):
+
+        _system_six = False
+
+if _system_six:
+    six = _system_six
+else:
+    from . import _six as six
+six_py_file = '{0}.py'.format(os.path.splitext(six.__file__)[0])
+exec(open(six_py_file, 'rb').read())

tests/data/six_brokenpkg/_six.py

@@ -0,0 +1,868 @@
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+# Copyright (c) 2010-2015 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import absolute_import
+
+import functools
+import itertools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson <benjamin@python.org>"
+__version__ = "1.10.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform.startswith("java"):
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+        del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)  # Invokes __set__.
+        try:
+            # This is a bit ugly, but it avoids running this again by
+            # removing this descriptor.
+            delattr(obj.__class__, self.name)
+        except AttributeError:
+            pass
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+    def __getattr__(self, attr):
+        _module = self._resolve()
+        value = getattr(_module, attr)
+        setattr(self, attr, value)
+        return value
+
+
+class _LazyModule(types.ModuleType):
+
+    def __init__(self, name):
+        super(_LazyModule, self).__init__(name)
+        self.__doc__ = self.__class__.__doc__
+
+    def __dir__(self):
+        attrs = ["__doc__", "__name__"]
+        attrs += [attr.name for attr in self._moved_attributes]
+        return attrs
+
+    # Subclasses should override this
+    _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+
+    """
+    A meta path importer to import six.moves and its submodules.
+
+    This class implements a PEP302 finder and loader. It should be compatible
+    with Python 2.5 and all existing versions of Python3
+    """
+
+    def __init__(self, six_module_name):
+        self.name = six_module_name
+        self.known_modules = {}
+
+    def _add_module(self, mod, *fullnames):
+        for fullname in fullnames:
+            self.known_modules[self.name + "." + fullname] = mod
+
+    def _get_module(self, fullname):
+        return self.known_modules[self.name + "." + fullname]
+
+    def find_module(self, fullname, path=None):
+        if fullname in self.known_modules:
+            return self
+        return None
+
+    def __get_module(self, fullname):
+        try:
+            return self.known_modules[fullname]
+        except KeyError:
+            raise ImportError("This loader does not know module " + fullname)
+
+    def load_module(self, fullname):
+        try:
+            # in case of a reload
+            return sys.modules[fullname]
+        except KeyError:
+            pass
+        mod = self.__get_module(fullname)
+        if isinstance(mod, MovedModule):
+            mod = mod._resolve()
+        else:
+            mod.__loader__ = self
+        sys.modules[fullname] = mod
+        return mod
+
+    def is_package(self, fullname):
+        """
+        Return true, if the named module is a package.
+
+        We need this method to get correct spec objects with
+        Python 3.4 (see PEP451)
+        """
+        return hasattr(self.__get_module(fullname), "__path__")
+
+    def get_code(self, fullname):
+        """Return None
+
+        Required, if is_package is implemented"""
+        self.__get_module(fullname)  # eventually raises ImportError
+        return None
+    get_source = get_code  # same as get_code
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+
+    """Lazy loading of moved objects"""
+    __path__ = []  # mark as package
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("intern", "__builtin__", "sys"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("UserDict", "UserDict", "collections"),
+    MovedAttribute("UserList", "UserList", "collections"),
+    MovedAttribute("UserString", "UserString", "collections"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("_thread", "thread", "_thread"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+]
+# Add windows specific modules.
+if sys.platform == "win32":
+    _moved_attributes += [
+        MovedModule("winreg", "_winreg"),
+    ]
+
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+    if isinstance(attr, MovedModule):
+        _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("quote", "urllib", "urllib.parse"),
+    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("urlencode", "urllib", "urllib.parse"),
+    MovedAttribute("splitquery", "urllib", "urllib.parse"),
+    MovedAttribute("splittag", "urllib", "urllib.parse"),
+    MovedAttribute("splituser", "urllib", "urllib.parse"),
+    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+    setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+                      "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+    MovedAttribute("URLError", "urllib2", "urllib.error"),
+    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+    setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+                      "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+    MovedAttribute("urlopen", "urllib2", "urllib.request"),
+    MovedAttribute("install_opener", "urllib2", "urllib.request"),
+    MovedAttribute("build_opener", "urllib2", "urllib.request"),
+    MovedAttribute("pathname2url", "urllib", "urllib.request"),
+    MovedAttribute("url2pathname", "urllib", "urllib.request"),
+    MovedAttribute("getproxies", "urllib", "urllib.request"),
+    MovedAttribute("Request", "urllib2", "urllib.request"),
+    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+    MovedAttribute("URLopener", "urllib", "urllib.request"),
+    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+]
+for attr in _urllib_request_moved_attributes:
+    setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+                      "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+    MovedAttribute("addbase", "urllib", "urllib.response"),
+    MovedAttribute("addclosehook", "urllib", "urllib.response"),
+    MovedAttribute("addinfo", "urllib", "urllib.response"),
+    MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+    setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+                      "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+                      "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+
+    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+    __path__ = []  # mark as package
+    parse = _importer._get_module("moves.urllib_parse")
+    error = _importer._get_module("moves.urllib_error")
+    request = _importer._get_module("moves.urllib_request")
+    response = _importer._get_module("moves.urllib_response")
+    robotparser = _importer._get_module("moves.urllib_robotparser")
+
+    def __dir__(self):
+        return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+                      "moves.urllib")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_closure = "__closure__"
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+    _func_globals = "__globals__"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_closure = "func_closure"
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+    _func_globals = "func_globals"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    create_bound_method = types.MethodType
+
+    def create_unbound_method(func, cls):
+        return func
+
+    Iterator = object
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    def create_bound_method(func, obj):
+        return types.MethodType(func, obj, obj.__class__)
+
+    def create_unbound_method(func, cls):
+        return types.MethodType(func, None, cls)
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+    def iterkeys(d, **kw):
+        return iter(d.keys(**kw))
+
+    def itervalues(d, **kw):
+        return iter(d.values(**kw))
+
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+
+    def iterlists(d, **kw):
+        return iter(d.lists(**kw))
+
+    viewkeys = operator.methodcaller("keys")
+
+    viewvalues = operator.methodcaller("values")
+
+    viewitems = operator.methodcaller("items")
+else:
+    def iterkeys(d, **kw):
+        return d.iterkeys(**kw)
+
+    def itervalues(d, **kw):
+        return d.itervalues(**kw)
+
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)
+
+    def iterlists(d, **kw):
+        return d.iterlists(**kw)
+
+    viewkeys = operator.methodcaller("viewkeys")
+
+    viewvalues = operator.methodcaller("viewvalues")
+
+    viewitems = operator.methodcaller("viewitems")
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+         "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+         "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+
+    def u(s):
+        return s
+    unichr = chr
+    import struct
+    int2byte = struct.Struct(">B").pack
+    del struct
+    byte2int = operator.itemgetter(0)
+    indexbytes = operator.getitem
+    iterbytes = iter
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+    _assertCountEqual = "assertCountEqual"
+    if sys.version_info[1] <= 1:
+        _assertRaisesRegex = "assertRaisesRegexp"
+        _assertRegex = "assertRegexpMatches"
+    else:
+        _assertRaisesRegex = "assertRaisesRegex"
+        _assertRegex = "assertRegex"
+else:
+    def b(s):
+        return s
+    # Workaround for standalone backslash
+
+    def u(s):
+        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+    unichr = unichr
+    int2byte = chr
+
+    def byte2int(bs):
+        return ord(bs[0])
+
+    def indexbytes(buf, i):
+        return ord(buf[i])
+    iterbytes = functools.partial(itertools.imap, ord)
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+    _assertCountEqual = "assertItemsEqual"
+    _assertRaisesRegex = "assertRaisesRegexp"
+    _assertRegex = "assertRegexpMatches"
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+def assertCountEqual(self, *args, **kwargs):
+    return getattr(self, _assertCountEqual)(*args, **kwargs)
+
+
+def assertRaisesRegex(self, *args, **kwargs):
+    return getattr(self, _assertRaisesRegex)(*args, **kwargs)
+
+
+def assertRegex(self, *args, **kwargs):
+    return getattr(self, _assertRegex)(*args, **kwargs)
+
+
+if PY3:
+    exec_ = getattr(moves.builtins, "exec")
+
+    def reraise(tp, value, tb=None):
+        if value is None:
+            value = tp()
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+else:
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+    exec_("""def reraise(tp, value, tb=None):
+    raise tp, value, tb
+""")
+
+
+if sys.version_info[:2] == (3, 2):
+    exec_("""def raise_from(value, from_value):
+    if from_value is None:
+        raise value
+    raise value from from_value
+""")
+elif sys.version_info[:2] > (3, 2):
+    exec_("""def raise_from(value, from_value):
+    raise value from from_value
+""")
+else:
+    def raise_from(value, from_value):
+        raise value
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+    def print_(*args, **kwargs):
+        """The new-style print function for Python 2.4 and 2.5."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            # If the file has an encoding, encode unicode with it.
+            if (isinstance(fp, file) and
+                    isinstance(data, unicode) and
+                    fp.encoding is not None):
+                errors = getattr(fp, "errors", None)
+                if errors is None:
+                    errors = "strict"
+                data = data.encode(fp.encoding, errors)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+if sys.version_info[:2] < (3, 3):
+    _print = print_
+
+    def print_(*args, **kwargs):
+        fp = kwargs.get("file", sys.stdout)
+        flush = kwargs.pop("flush", False)
+        _print(*args, **kwargs)
+        if flush and fp is not None:
+            fp.flush()
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+              updated=functools.WRAPPER_UPDATES):
+        def wrapper(f):
+            f = functools.wraps(wrapped, assigned, updated)(f)
+            f.__wrapped__ = wrapped
+            return f
+        return wrapper
+else:
+    wraps = functools.wraps
+
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(meta):
+
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+    """Class decorator for creating a class with a metaclass."""
+    def wrapper(cls):
+        orig_vars = cls.__dict__.copy()
+        slots = orig_vars.get('__slots__')
+        if slots is not None:
+            if isinstance(slots, str):
+                slots = [slots]
+            for slots_var in slots:
+                orig_vars.pop(slots_var)
+        orig_vars.pop('__dict__', None)
+        orig_vars.pop('__weakref__', None)
+        return metaclass(cls.__name__, cls.__bases__, orig_vars)
+    return wrapper
+
+
+def python_2_unicode_compatible(klass):
+    """
+    A decorator that defines __unicode__ and __str__ methods under Python 2.
+    Under Python 3 it does nothing.
+
+    To support Python 2 and 3 with a single code base, define a __str__ method
+    returning text and apply this decorator to the class.
+    """
+    if PY2:
+        if '__str__' not in klass.__dict__:
+            raise ValueError("@python_2_unicode_compatible cannot be applied "
+                             "to %s because it doesn't define __str__()." %
+                             klass.__name__)
+        klass.__unicode__ = klass.__str__
+        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+    return klass
+
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = []  # required for PEP 302 and PEP 451
+__package__ = __name__  # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+    for i, importer in enumerate(sys.meta_path):
+        # Here's some real nastiness: Another "instance" of the six module might
+        # be floating around. Therefore, we can't use isinstance() to check for
+        # the six meta path importer, since the other six instance will have
+        # inserted an importer with different class.
+        if (type(importer).__name__ == "_SixMetaPathImporter" and
+                importer.name == __name__):
+            del sys.meta_path[i]
+            break
+    del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)

tests/data/webproject/manage.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webproject.settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

tests/data/webproject/serve_django_app.py

@@ -0,0 +1,37 @@
+
+import os
+import sys
+
+import mitogen
+import mitogen.master
+import mitogen.utils
+
+
+import sys
+sys.path.insert(0, '..')
+
+
+def serve_django_app(settings_name):
+    os.listdir = lambda path: []
+
+    os.environ['DJANGO_SETTINGS_MODULE'] = settings_name
+    import django
+    args = ['manage.py', 'runserver', '0:9191', '--noreload']
+    from django.conf import settings
+    #settings.configure()
+    django.setup()
+    from django.core.management.commands import runserver
+    runserver.Command().run_from_argv(args)
+    #django.core.management.execute_from_command_line(args)
+
+
+def main(broker):
+    import logging
+    mitogen.utils.log_to_file(level=logging.INFO, io=False)
+    context = mitogen.master.connect(broker)
+    context.call(os.chdir, '/')
+    #context.call(mitogen.utils.log_to_file, '/tmp/log')
+    context.call(serve_django_app, 'webproject.settings')
+
+if __name__ == '__main__' and not mitogen.slave:
+    mitogen.utils.run_with_broker(main)

tests/data/webproject/webapp/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

tests/data/webproject/webapp/apps.py

@@ -0,0 +1,7 @@
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class WebappConfig(AppConfig):
+    name = 'webapp'

tests/data/webproject/webapp/models.py

@@ -0,0 +1,5 @@
+from __future__ import unicode_literals
+
+from django.db import models
+
+# Create your models here.

tests/data/webproject/webapp/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

tests/data/webproject/webapp/views.py

@@ -0,0 +1,9 @@
+from django.http import HTTPResponse
+
+
+def index(request):
+    return HTTPResponse('hello, world')
+
+
+def shutdown(request):
+     raise KeyboardInterrupt

tests/data/webproject/webproject/settings.py

@@ -0,0 +1,122 @@
+"""
+Django settings for webproject project.
+
+Generated by 'django-admin startproject' using Django 1.9.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.9/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.9/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'db)-3@sbmu5d91tpio#h*9=iew@12-n1bh-de!xbrb0e4la(9n'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+AUTH_PASSWORD_VALIDATORS = ['cats']
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+]
+
+MIDDLEWARE_CLASSES = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'webproject.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'webproject.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ''
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.9/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.9/howto/static-files/
+
+STATIC_URL = '/static/'

tests/data/webproject/webproject/urls.py

@@ -0,0 +1,24 @@
+"""webproject URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/1.9/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.conf.urls import url, include
+    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
+"""
+from django.conf.urls import url
+from django.contrib import admin
+import webapp.views
+
+urlpatterns = [
+    url(r'^$', webapp.views.index),
+    url(r'^shutdown/$', webapp.views.shutdown),
+    url(r'^admin/', admin.site.urls),
+]

tests/data/webproject/webproject/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for webproject project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webproject.settings")
+
+application = get_wsgi_application()

tests/importer_test.py

@@ -0,0 +1,127 @@
+
+import email.utils
+import sys
+import types
+import unittest
+import zlib
+
+import mock
+import pytest
+
+import mitogen.core
+import testlib
+
+
+class ImporterMixin(object):
+    modname = None
+
+    def setUp(self):
+        super(ImporterMixin, self).setUp()
+        self.context = mock.Mock()
+        self.importer = mitogen.core.Importer(self.context)
+
+    def tearDown(self):
+        sys.modules.pop(self.modname, None)
+        super(ImporterMixin, self).tearDown()
+
+
+class LoadModuleTest(ImporterMixin, unittest.TestCase):
+    data = zlib.compress("data = 1\n\n")
+    path = 'fake_module.py'
+    modname = 'fake_module'
+    response = (None, path, data)
+
+    def test_no_such_module(self):
+        self.context.enqueue_await_reply.return_value = None
+        self.assertRaises(ImportError,
+            lambda: self.importer.load_module(self.modname))
+
+    def test_module_added_to_sys_modules(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertTrue(sys.modules[self.modname] is mod)
+        self.assertTrue(isinstance(mod, types.ModuleType))
+
+    def test_module_file_set(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertEquals(mod.__file__, 'master:' + self.path)
+
+    def test_module_loader_set(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertTrue(mod.__loader__ is self.importer)
+
+    def test_module_package_unset(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertTrue(mod.__package__ is None)
+
+
+class LoadSubmoduleTest(ImporterMixin, unittest.TestCase):
+    data = zlib.compress("data = 1\n\n")
+    path = 'fake_module.py'
+    modname = 'mypkg.fake_module'
+    response = (None, path, data)
+
+    def test_module_package_unset(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertEquals(mod.__package__, 'mypkg')
+
+
+class LoadModulePackageTest(ImporterMixin, unittest.TestCase):
+    data = zlib.compress("func = lambda: 1\n\n")
+    path = 'fake_pkg/__init__.py'
+    modname = 'fake_pkg'
+    response = ([], path, data)
+
+    def test_module_file_set(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertEquals(mod.__file__, 'master:' + self.path)
+
+    def test_get_filename(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        filename = mod.__loader__.get_filename(self.modname)
+        self.assertEquals('master:fake_pkg/__init__.py', filename)
+
+    def test_get_source(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        source = mod.__loader__.get_source(self.modname)
+        self.assertEquals(source, zlib.decompress(self.data))
+
+    def test_module_loader_set(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertTrue(mod.__loader__ is self.importer)
+
+    def test_module_path_present(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertEquals(mod.__path__, [])
+
+    def test_module_package_set(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertEquals(mod.__package__, self.modname)
+
+    def test_module_data(self):
+        self.context.enqueue_await_reply.return_value = self.response
+        mod = self.importer.load_module(self.modname)
+        self.assertTrue(isinstance(mod.func, types.FunctionType))
+        self.assertEquals(mod.func.__module__, self.modname)
+
+
+class EmailParseAddrSysTest(testlib.BrokerMixin, unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def initdir(self, caplog):
+        self.caplog = caplog
+
+    def test_sys_module_not_fetched(self):
+        # An old version of core.Importer would request the email.sys module
+        # while executing email.utils.parseaddr(). Ensure this needless
+        # roundtrip has not reappeared.
+        pass

tests/responder_test.py

@@ -0,0 +1,67 @@
+
+import mock
+import subprocess
+import unittest
+import sys
+
+import mitogen.master
+import testlib
+
+import plain_old_module
+import simple_pkg.a
+
+
+class GoodModulesTest(testlib.BrokerMixin, unittest.TestCase):
+    def test_plain_old_module(self):
+        # The simplest case: a top-level module with no interesting imports or
+        # package machinery damage.
+        context = mitogen.master.connect(self.broker)
+        self.assertEquals(256, context.call(plain_old_module.pow, 2, 8))
+
+    def test_simple_pkg(self):
+        # Ensure success of a simple package containing two submodules, one of
+        # which imports the other.
+        context = mitogen.master.connect(self.broker)
+        self.assertEquals(3,
+            context.call(simple_pkg.a.subtract_one_add_two, 2))
+
+    def test_self_contained_program(self):
+        # Ensure a program composed of a single script can be imported
+        # successfully.
+        args = [sys.executable, testlib.data_path('self_contained_program.py')]
+        output = subprocess.check_output(args)
+        self.assertEquals(output, "['__main__', 50]\n")
+
+
+class BrokenModulesTest(unittest.TestCase):
+    def test_obviously_missing(self):
+        # Ensure we don't crash in the case of a module legitimately being
+        # unavailable. Should never happen in the real world.
+
+        context = mock.Mock()
+        responder = mitogen.master.ModuleResponder(context)
+        responder.get_module((50, 'non_existent_module'))
+        self.assertEquals(1, len(context.enqueue.mock_calls))
+
+        call = context.enqueue.mock_calls[0]
+        reply_to, data = call[1]
+        self.assertEquals(50, reply_to)
+        self.assertTrue(data is None)
+
+    def test_ansible_six_messed_up_path(self):
+        # The copy of six.py shipped with Ansible appears in a package whose
+        # __path__ subsequently ends up empty, which prevents pkgutil from
+        # finding its submodules. After ansible.compat.six is initialized in
+        # the parent, attempts to execute six/__init__.py on the slave will
+        # cause an attempt to request ansible.compat.six._six from the master.
+        import six_brokenpkg
+
+        context = mock.Mock()
+        responder = mitogen.master.ModuleResponder(context)
+        responder.get_module((50, 'six_brokenpkg._six'))
+        self.assertEquals(1, len(context.enqueue.mock_calls))
+
+        call = context.enqueue.mock_calls[0]
+        reply_to, data = call[1]
+        self.assertEquals(50, reply_to)
+        self.assertTrue(isinstance(data, tuple))

tests/ssh_test.py

@@ -0,0 +1,25 @@
+
+import unittest
+
+import mitogen
+import mitogen.master
+import mitogen.ssh
+import mitogen.utils
+
+import testlib
+
+
+def add(x, y):
+    return x + y
+
+
+class SshTest(unittest.TestCase):
+    def test_okay(self):
+        @mitogen.utils.run_with_broker
+        def test(broker):
+            context = mitogen.ssh.connect(broker,
+                hostname='hostname',
+                ssh_path=testlib.data_path('fakessh.py'))
+            context.call(mitogen.utils.log_to_file, '/tmp/log')
+            context.call(mitogen.utils.disable_site_packages)
+            self.assertEquals(3, context.call(add, 1, 2))

tests/testlib.py

@@ -0,0 +1,34 @@
+
+import os
+import sys
+import unittest
+
+import mock
+
+import mitogen.master
+
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
+sys.path.append(DATA_DIR)
+
+
+def set_debug():
+    import logging
+    logging.getLogger('mitogen').setLevel(logging.DEBUG)
+
+
+def data_path(suffix):
+    return os.path.join(DATA_DIR, suffix)
+
+
+class BrokerMixin(object):
+    broker_class = mitogen.master.Broker
+
+    def setUp(self):
+        super(BrokerMixin, self).setUp()
+        self.broker = self.broker_class()
+
+    def tearDown(self):
+        self.broker.shutdown()
+        self.broker.join()
+        super(BrokerMixin, self).tearDown()

tests/timing_test.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+import socket
+import time
+import unittest
+
+import mitogen.master
+import mitogen.utils
+
+
+@mitogen.utils.with_broker
+def do_stuff(broker):
+    context = mitogen.master.connect(broker)
+    t0 = time.time()
+    ncalls = 1000
+    for x in xrange(ncalls):
+        context.call(socket.gethostname)
+    return (1e6 * (time.time() - t0)) / ncalls
+
+
+class LocalContextTimingTest(unittest.TestCase):
+    def test_timing(self):
+        self.assertLess(do_stuff(), 1000)

tests/utils_test.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+import unittest
+
+import mitogen.master
+import mitogen.utils
+
+
+def func0(broker):
+    return broker
+
+
+@mitogen.utils.with_broker
+def func(broker):
+    return broker
+
+
+class RunWithBrokerTest(unittest.TestCase):
+    # test_shutdown_on_exception
+    # test_shutdown_on_success
+
+    def test_run_with_broker(self):
+        broker = mitogen.utils.run_with_broker(func0)
+        self.assertTrue(isinstance(broker, mitogen.master.Broker))
+        self.assertFalse(broker._thread.isAlive())
+
+
+class WithBrokerTest(unittest.TestCase):
+    def test_with_broker(self):
+        broker = func()
+        self.assertTrue(isinstance(broker, mitogen.master.Broker))
+        self.assertFalse(broker._thread.isAlive())

todo/six-module-import-error.py

@@ -0,0 +1,332 @@
+[11:46:20 Eldil!8 mitogen] py.test tests/ssh_test.py
+=============================================================================== test session starts ================================================================================
+platform darwin -- Python 2.7.10, pytest-2.8.6, py-1.4.31, pluggy-0.3.1
+rootdir: /Users/dmw/src/mitogen, inifile:
+plugins: capturelog-0.7, timeout-1.0.0
+collected 1 items
+
+tests/ssh_test.py F
+
+===================================================================================== FAILURES =====================================================================================
+________________________________________________________________________________ SshTest.test_okay _________________________________________________________________________________
+
+self = <ssh_test.SshTest testMethod=test_okay>
+
+    def test_okay(self):
+>       @mitogen.utils.run_with_broker
+        def test(broker):
+
+tests/ssh_test.py:18:
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+mitogen/utils.py:52: in run_with_broker
+    return func(broker, *args, **kwargs)
+tests/ssh_test.py:25: in test
+    self.assertEquals(3, context.call(add, 1, 2))
+mitogen/master.py:319: in call
+    return self.call_with_deadline(None, False, fn, *args, **kwargs)
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+self = Context('hostname', 'hostname'), deadline = None, with_context = False, fn = <function add at 0x1023630c8>, args = (1, 2), kwargs = {}, klass = None
+call = (False, 'ssh_test', None, 'add', (1, 2), {})
+result = CallError('call failed: __builtin__.str: call failed: exceptions.KeyError: \'p...ern/__init__.py", line 43, in load_module\n    mod = sys.modules[extant]\n\n',)
+
+    def call_with_deadline(self, deadline, with_context, fn, *args, **kwargs):
+        """Invoke `fn([context,] *args, **kwargs)` in the external context.
+
+            If `with_context` is ``True``, pass its
+            :py:class:`ExternalContext <mitogen.core.ExternalContext>` instance as
+            the first parameter.
+
+            If `deadline` is not ``None``, expire the call after `deadline`
+            seconds. If `deadline` is ``None``, the invocation may block
+            indefinitely."""
+        LOG.debug('%r.call_with_deadline(%r, %r, %r, *%r, **%r)',
+                  self, deadline, with_context, fn, args, kwargs)
+
+        if isinstance(fn, types.MethodType) and \
+           isinstance(fn.im_self, (type, types.ClassType)):
+            klass = fn.im_self.__name__
+        else:
+            klass = None
+
+        call = (with_context, fn.__module__, klass, fn.__name__, args, kwargs)
+        result = self.enqueue_await_reply(mitogen.core.CALL_FUNCTION,
+                                          deadline, call)
+        if isinstance(result, mitogen.core.CallError):
+>           raise result
+E           CallError: call failed: __builtin__.str: call failed: exceptions.KeyError: 'pkg_resources._vendor.six.moves.'
+E             File "<stdin>", line 862, in _dispatch_calls
+E             File "<stdin>", line 220, in load_module
+E             File "master:/Users/dmw/src/mitogen/tests/ssh_test.py", line 9, in <module>
+E               import testlib
+E             File "<stdin>", line 220, in load_module
+E             File "master:/Users/dmw/src/mitogen/tests/testlib.py", line 6, in <module>
+E               import mock
+E             File "<stdin>", line 220, in load_module
+E             File "master:/Users/dmw/.venv/lib/python2.7/site-packages/mock/__init__.py", line 2, in <module>
+E               import mock.mock as _mock
+E             File "<stdin>", line 220, in load_module
+E             File "master:/Users/dmw/.venv/lib/python2.7/site-packages/mock/mock.py", line 69, in <module>
+E               from pbr.version import VersionInfo
+E             File "<stdin>", line 220, in load_module
+E             File "master:/Users/dmw/.venv/lib/python2.7/site-packages/pbr/version.py", line 25, in <module>
+E               import pkg_resources
+E             File "<stdin>", line 220, in load_module
+E             File "master:/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/__init__.py", line 49, in <module>
+E               from pkg_resources.extern.six.moves import urllib, map, filter
+E             File "<stdin>", line 190, in find_module
+E             File "master:/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/extern/__init__.py", line 43, in load_module
+E               mod = sys.modules[extant]
+
+mitogen/master.py:314: CallError
+----------------------------------------------------------------------------------- Captured log -----------------------------------------------------------------------------------
+master.py                  265 DEBUG    Stream(Context('hostname', 'hostname')).connect()
+master.py                   67 DEBUG    create_child() child 41405 fd 12, parent 41402, args ('/Users/dmw/src/mitogen/tests/data/fakessh.py', 'hostname', " 'python'", " '-c'", ' \'exec("aW1wb3J0IG9zLHN5cyx6bGliClIsVz1vcy5waXBlKCkKaWYgb3MuZm9yaygpOgoJb3MuZHVwMigwLDEwMCkKCW9zLmR1cDIoUiwwKQoJb3MuY2xvc2UoUikKCW9zLmNsb3NlKFcpCglvcy5leGVjdihzeXMuZXhlY3V0YWJsZSxbJ2Vjb250ZXh0OmRtd0BFbGRpbC5ob21lOjQxNDAyJ10pCmVsc2U6Cglvcy5mZG9wZW4oVywnd2InLDApLndyaXRlKHpsaWIuZGVjb21wcmVzcyhzeXMuc3RkaW4ucmVhZChpbnB1dCgpKSkpCglwcmludCgnT0snKQoJc3lzLmV4aXQoMCk=".decode("base64"))\'')
+master.py                  270 DEBUG    Stream(Context('hostname', 'hostname')).connect(): child process stdin/stdout=13
+core.py                    679 DEBUG    Broker().register(Context('hostname', 'hostname')) -> r=<Side of Stream(Context('hostname', 'hostname')) fd 13> w=<Side of Stream(Context('hostname', 'hostname')) fd 12>
+master.py                  302 DEBUG    Context('hostname', 'hostname').call_with_deadline(None, False, <function log_to_file at 0x1022e6b18>, *('/tmp/log',), **{})
+core.py                    516 DEBUG    Context('hostname', 'hostname').enqueue_await_reply(101, None, (False, 'mitogen.utils', None, 'log_to_file', ('/tmp/log',), {})) -> reply handle 1000
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1000, 'mitogen.utils'))
+master.py                  100 DEBUG    pkgutil.find_loader('mitogen.utils') -> <pkgutil.ImpLoader instance at 0x10236f518>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'mitogen.utils': ('/Users/dmw/src/mitogen/mitogen/utils.py', .., False)
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1001, 'mitogen.master'))
+master.py                  100 DEBUG    pkgutil.find_loader('mitogen.master') -> <pkgutil.ImpLoader instance at 0x10236f1b8>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'mitogen.master': ('/Users/dmw/src/mitogen/mitogen/master.py', .., False)
+master.py                  302 DEBUG    Context('hostname', 'hostname').call_with_deadline(None, False, <function disable_site_packages at 0x1022b56e0>, *(), **{})
+core.py                    516 DEBUG    Context('hostname', 'hostname').enqueue_await_reply(101, None, (False, 'mitogen.utils', None, 'disable_site_packages', (), {})) -> reply handle 1001
+master.py                   84 DEBUG    mitogen: _dispatch_calls((1001, False, 'mitogen.utils', None, 'disable_site_packages', (), {}))
+master.py                  302 DEBUG    Context('hostname', 'hostname').call_with_deadline(None, False, <function add at 0x1023630c8>, *(1, 2), **{})
+core.py                    516 DEBUG    Context('hostname', 'hostname').enqueue_await_reply(101, None, (False, 'ssh_test', None, 'add', (1, 2), {})) -> reply handle 1002
+master.py                   84 DEBUG    mitogen: _dispatch_calls((1002, False, 'ssh_test', None, 'add', (1, 2), {}))
+master.py                   84 DEBUG    mitogen: Importer().find_module('ssh_test')
+master.py                   84 DEBUG    mitogen: find_module('ssh_test') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('ssh_test')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('ssh_test',)) -> reply handle 1002
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1002, 'ssh_test'))
+master.py                  100 DEBUG    pkgutil.find_loader('ssh_test') -> <_pytest.assertion.rewrite.AssertionRewritingHook object at 0x10222b5d0>
+master.py                  170 DEBUG    _get_module_via_sys_modules found 'ssh_test': ('/Users/dmw/src/mitogen/tests/ssh_test.py', .., False)
+master.py                   84 DEBUG    mitogen: Importer().find_module('unittest')
+master.py                   84 DEBUG    mitogen: Importer(): 'unittest' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('mitogen.ssh')
+master.py                   84 DEBUG    mitogen: find_module('mitogen.ssh') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('mitogen.ssh')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('mitogen.ssh',)) -> reply handle 1003
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1003, 'mitogen.ssh'))
+master.py                  100 DEBUG    pkgutil.find_loader('mitogen.ssh') -> <pkgutil.ImpLoader instance at 0x1023662d8>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'mitogen.ssh': ('/Users/dmw/src/mitogen/mitogen/ssh.py', .., False)
+master.py                   84 DEBUG    mitogen: Importer().find_module('mitogen.commands')
+master.py                   84 DEBUG    mitogen: Importer(): master doesn't know 'mitogen.commands'
+master.py                   84 DEBUG    mitogen: Importer().find_module('commands')
+master.py                   84 DEBUG    mitogen: Importer(): 'commands' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('testlib')
+master.py                   84 DEBUG    mitogen: find_module('testlib') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('testlib')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('testlib',)) -> reply handle 1004
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1004, 'testlib'))
+master.py                  100 DEBUG    pkgutil.find_loader('testlib') -> <pkgutil.ImpLoader instance at 0x10235c9e0>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'testlib': ('/Users/dmw/src/mitogen/tests/testlib.py', .., False)
+master.py                   84 DEBUG    mitogen: Importer().find_module('mock')
+master.py                   84 DEBUG    mitogen: find_module('mock') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('mock')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('mock',)) -> reply handle 1005
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1005, 'mock'))
+master.py                  100 DEBUG    pkgutil.find_loader('mock') -> <pkgutil.ImpLoader instance at 0x10235c710>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'mock': ('/Users/dmw/.venv/lib/python2.7/site-packages/mock/__init__.py', .., True)
+master.py                  174 DEBUG    get_child_modules('/Users/dmw/.venv/lib/python2.7/site-packages/mock/__init__.py', 'mock') -> ['mock.mock', 'mock.tests']
+master.py                   84 DEBUG    mitogen: Importer().find_module('mock.mock')
+master.py                   84 DEBUG    mitogen: find_module('mock.mock') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('mock.mock')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('mock.mock',)) -> reply handle 1006
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1006, 'mock.mock'))
+master.py                  100 DEBUG    pkgutil.find_loader('mock.mock') -> <pkgutil.ImpLoader instance at 0x1027a82d8>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'mock.mock': ('/Users/dmw/.venv/lib/python2.7/site-packages/mock/mock.py', .., False)
+master.py                   84 DEBUG    mitogen: Importer().find_module('builtins')
+master.py                   84 DEBUG    mitogen: find_module('builtins') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('builtins')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('builtins',)) -> reply handle 1007
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1007, 'builtins'))
+master.py                  100 DEBUG    pkgutil.find_loader('builtins') -> None
+master.py                  116 DEBUG    'builtins' does not appear in sys.modules
+master.py                  182 DEBUG    While importing 'builtins'
+Traceback (most recent call last):
+  File "/Users/dmw/src/mitogen/mitogen/master.py", line 167, in get_module
+    raise ImportError('could not find %r' % (fullname,))
+ImportError: could not find 'builtins'
+master.py                   84 DEBUG    mitogen: Importer().find_module('six')
+master.py                   84 DEBUG    mitogen: find_module('six') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('six')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('six',)) -> reply handle 1008
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1008, 'six'))
+master.py                  100 DEBUG    pkgutil.find_loader('six') -> <pkgutil.ImpLoader instance at 0x1027a8878>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'six': ('/Users/dmw/.venv/lib/python2.7/site-packages/six.py', .., False)
+master.py                   84 DEBUG    mitogen: Importer().find_module('pbr')
+master.py                   84 DEBUG    mitogen: find_module('pbr') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('pbr')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('pbr',)) -> reply handle 1009
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1009, 'pbr'))
+master.py                  100 DEBUG    pkgutil.find_loader('pbr') -> <pkgutil.ImpLoader instance at 0x1027a8b00>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'pbr': ('/Users/dmw/.venv/lib/python2.7/site-packages/pbr/__init__.py', .., True)
+master.py                  174 DEBUG    get_child_modules('/Users/dmw/.venv/lib/python2.7/site-packages/pbr/__init__.py', 'pbr') -> ['pbr.builddoc', 'pbr.cmd', 'pbr.core', 'pbr.extra_files', 'pbr.find_package', 'pbr.git', 'pbr.hooks', 'pbr.options', 'pbr.packaging', 'pbr.pbr_json', 'pbr.testr_command', 'pbr.tests', 'pbr.util', 'pbr.version']
+master.py                   84 DEBUG    mitogen: Importer().find_module('pbr.version')
+master.py                   84 DEBUG    mitogen: find_module('pbr.version') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('pbr.version')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('pbr.version',)) -> reply handle 1010
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1010, 'pbr.version'))
+master.py                  100 DEBUG    pkgutil.find_loader('pbr.version') -> <pkgutil.ImpLoader instance at 0x1027b1128>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'pbr.version': ('/Users/dmw/.venv/lib/python2.7/site-packages/pbr/version.py', .., False)
+master.py                   84 DEBUG    mitogen: Importer().find_module('pbr.itertools')
+master.py                   84 DEBUG    mitogen: Importer(): master doesn't know 'pbr.itertools'
+master.py                   84 DEBUG    mitogen: Importer().find_module('pbr.operator')
+master.py                   84 DEBUG    mitogen: Importer(): master doesn't know 'pbr.operator'
+master.py                   84 DEBUG    mitogen: Importer().find_module('pbr.sys')
+master.py                   84 DEBUG    mitogen: Importer(): master doesn't know 'pbr.sys'
+master.py                   84 DEBUG    mitogen: Importer().find_module('pbr.pkg_resources')
+master.py                   84 DEBUG    mitogen: Importer(): master doesn't know 'pbr.pkg_resources'
+master.py                   84 DEBUG    mitogen: Importer().find_module('pkg_resources')
+master.py                   84 DEBUG    mitogen: find_module('pkg_resources') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('pkg_resources')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('pkg_resources',)) -> reply handle 1011
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1011, 'pkg_resources'))
+master.py                  100 DEBUG    pkgutil.find_loader('pkg_resources') -> <pkgutil.ImpLoader instance at 0x1027b1248>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'pkg_resources': ('/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/__init__.py', .., True)
+master.py                  174 DEBUG    get_child_modules('/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/__init__.py', 'pkg_resources') -> ['pkg_resources._vendor', 'pkg_resources.extern']
+master.py                   84 DEBUG    mitogen: Importer().find_module('io')
+master.py                   84 DEBUG    mitogen: Importer(): 'io' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('zipfile')
+master.py                   84 DEBUG    mitogen: Importer(): 'zipfile' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('symbol')
+master.py                   84 DEBUG    mitogen: Importer(): 'symbol' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('platform')
+master.py                   84 DEBUG    mitogen: Importer(): 'platform' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('plistlib')
+master.py                   84 DEBUG    mitogen: Importer(): 'plistlib' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('email')
+master.py                   84 DEBUG    mitogen: Importer(): 'email' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.parser')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.parser' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.warnings')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.warnings' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.cStringIO')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.cStringIO' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.feedparser')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.feedparser' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.re')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.re' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.errors')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.errors' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.message')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.message' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.uu')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.uu' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('uu')
+master.py                   84 DEBUG    mitogen: Importer(): 'uu' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.binascii')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.binascii' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.charset')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.charset' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.codecs')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.codecs' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.base64mime')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.base64mime' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.utils')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.utils' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.os')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.os' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.time')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.time' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.base64')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.base64' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('base64')
+master.py                   84 DEBUG    mitogen: Importer(): 'base64' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.random')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.random' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.socket')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.socket' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.urllib')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.urllib' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('urllib')
+master.py                   84 DEBUG    mitogen: Importer(): 'urllib' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('email._parseaddr')
+master.py                   84 DEBUG    mitogen: Importer(): 'email._parseaddr' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.calendar')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.calendar' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('calendar')
+master.py                   84 DEBUG    mitogen: Importer(): 'calendar' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.quopri')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.quopri' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('quopri')
+master.py                   84 DEBUG    mitogen: Importer(): 'quopri' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.encoders')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.encoders' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.quoprimime')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.quoprimime' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.string')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.string' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('email.iterators')
+master.py                   84 DEBUG    mitogen: Importer(): 'email.iterators' is submodule of a package we did not load
+master.py                   84 DEBUG    mitogen: Importer().find_module('tempfile')
+master.py                   84 DEBUG    mitogen: Importer(): 'tempfile' is available locally
+master.py                   84 DEBUG    mitogen: Importer().find_module('_imp')
+master.py                   84 DEBUG    mitogen: find_module('_imp') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('_imp')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('_imp',)) -> reply handle 1012
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1012, '_imp'))
+master.py                  100 DEBUG    pkgutil.find_loader('_imp') -> None
+master.py                  116 DEBUG    '_imp' does not appear in sys.modules
+master.py                  182 DEBUG    While importing '_imp'
+Traceback (most recent call last):
+  File "/Users/dmw/src/mitogen/mitogen/master.py", line 167, in get_module
+    raise ImportError('could not find %r' % (fullname,))
+ImportError: could not find '_imp'
+master.py                   84 DEBUG    mitogen: Importer().find_module('pkg_resources.extern')
+master.py                   84 DEBUG    mitogen: find_module('pkg_resources.extern') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('pkg_resources.extern')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('pkg_resources.extern',)) -> reply handle 1013
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1013, 'pkg_resources.extern'))
+master.py                  100 DEBUG    pkgutil.find_loader('pkg_resources.extern') -> <pkgutil.ImpLoader instance at 0x1027c7290>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'pkg_resources.extern': ('/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/extern/__init__.py', .., True)
+master.py                  174 DEBUG    get_child_modules('/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/extern/__init__.py', 'pkg_resources.extern') -> []
+master.py                   84 DEBUG    mitogen: Importer().find_module('pkg_resources.extern.sys')
+master.py                   84 DEBUG    mitogen: Importer(): master doesn't know 'pkg_resources.extern.sys'
+master.py                   84 DEBUG    mitogen: Importer().find_module('pkg_resources.extern.six')
+master.py                   84 DEBUG    mitogen: Importer(): master doesn't know 'pkg_resources.extern.six'
+master.py                   84 DEBUG    mitogen: Importer().find_module('pkg_resources._vendor')
+master.py                   84 DEBUG    mitogen: find_module('pkg_resources._vendor') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('pkg_resources._vendor')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('pkg_resources._vendor',)) -> reply handle 1014
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1014, 'pkg_resources._vendor'))
+master.py                  100 DEBUG    pkgutil.find_loader('pkg_resources._vendor') -> <pkgutil.ImpLoader instance at 0x1027c7680>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'pkg_resources._vendor': ('/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/_vendor/__init__.py', .., True)
+master.py                  174 DEBUG    get_child_modules('/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/_vendor/__init__.py', 'pkg_resources._vendor') -> ['pkg_resources._vendor.packaging', 'pkg_resources._vendor.six']
+master.py                   84 DEBUG    mitogen: Importer().find_module('pkg_resources._vendor.six')
+master.py                   84 DEBUG    mitogen: find_module('pkg_resources._vendor.six') returning self
+master.py                   84 DEBUG    mitogen: Importer.load_module('pkg_resources._vendor.six')
+master.py                   84 DEBUG    mitogen: Context('master').enqueue_await_reply(100, None, ('pkg_resources._vendor.six',)) -> reply handle 1015
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module((1015, 'pkg_resources._vendor.six'))
+master.py                  100 DEBUG    pkgutil.find_loader('pkg_resources._vendor.six') -> <pkgutil.ImpLoader instance at 0x1027c73f8>
+master.py                  170 DEBUG    _get_module_via_pkgutil found 'pkg_resources._vendor.six': ('/Users/dmw/.venv/lib/python2.7/site-packages/pkg_resources/_vendor/six.py', .., False)
+master.py                   84 DEBUG    mitogen: Importer().find_module('pkg_resources.extern.six.moves')
+core.py                    760 DEBUG    Broker().shutdown()
+master.py                  202 DEBUG    Stream(Context('hostname', 'hostname')) closing CALL_FUNCTION channel
+core.py                    337 DEBUG    Waker(Broker()).on_shutdown()
+core.py                    330 DEBUG    Waker(Broker()).on_disconnect()
+master.py                   84 DEBUG    mitogen: Waker(Broker()).on_shutdown()
+master.py                   84 DEBUG    mitogen: Waker(Broker()).on_disconnect()
+master.py                   84 DEBUG    mitogen: <IoLogger stderr>.on_shutdown()
+master.py                   84 DEBUG    mitogen: <IoLogger stdout>.on_shutdown()
+master.py                   84 DEBUG    mitogen: Stream(Context('master')).on_shutdown(Broker())
+master.py                   84 DEBUG    mitogen: ExternalContext.main() normal exit
+master.py                   84 DEBUG    mitogen: Broker().shutdown()
+master.py                   84 DEBUG    mitogen: <IoLogger stdout>.on_receive()
+master.py                   84 DEBUG    mitogen: <IoLogger stdout>.on_disconnect()
+master.py                   84 DEBUG    mitogen: <IoLogger stderr>.on_receive()
+master.py                   84 DEBUG    mitogen: <IoLogger stderr>.on_disconnect()
+core.py                    330 DEBUG    Stream(Context('hostname', 'hostname')).on_disconnect()
+core.py                    486 DEBUG    Context('hostname', 'hostname').on_shutdown(Broker())
+core.py                    488 DEBUG    Context('hostname', 'hostname').on_disconnect(): killing 100: <bound method ModuleResponder.get_module of ModuleResponder(Context('hostname', 'hostname'))>
+master.py                  153 DEBUG    ModuleResponder(Context('hostname', 'hostname')).get_module(<Dead>)
+core.py                    488 DEBUG    Context('hostname', 'hostname').on_disconnect(): killing 102: <bound method LogForwarder.forward_log of <mitogen.master.LogForwarder object at 0x10236a750>>
+------------------------------------------------------------------------------ Captured stdout setup -------------------------------------------------------------------------------
+[<TestCaseFunction 'test_okay'>]
+=================================================================== 1 failed, 1 pytest-warnings in 0.57 seconds ====================================================================
+[22:43:16 Eldil!8 mitogen] n