diff --git a/docs/_templates/github.html b/docs/_templates/github.html index 2735c31f..f26b0c86 100644 --- a/docs/_templates/github.html +++ b/docs/_templates/github.html @@ -1,4 +1,4 @@


-GitHub Repository +GitHub Repository

diff --git a/docs/api.rst b/docs/api.rst index 61681596..81533558 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -7,40 +7,40 @@ Package Layout ============== -econtext Package ----------------- +mitogen Package +--------------- -.. automodule:: econtext +.. automodule:: mitogen -.. autodata:: econtext.slave -.. autodata:: econtext.context_id -.. autodata:: econtext.parent_id +.. autodata:: mitogen.slave +.. autodata:: mitogen.context_id +.. autodata:: mitogen.parent_id -econtext.core -------------- +mitogen.core +------------ -.. automodule:: econtext.core +.. automodule:: mitogen.core -econtext.master ---------------- +mitogen.master +-------------- -.. automodule:: econtext.master +.. automodule:: mitogen.master -econtext.fakessh +mitogen.fakessh --------------- -.. automodule:: econtext.fakessh +.. automodule:: mitogen.fakessh -.. autofunction:: econtext.fakessh.run +.. autofunction:: mitogen.fakessh.run Router Class ============ -.. autoclass:: econtext.master.Router +.. autoclass:: mitogen.master.Router :members: :inherited-members: @@ -48,7 +48,7 @@ Router Class Broker Class ============ -.. autoclass:: econtext.master.Broker +.. autoclass:: mitogen.master.Broker :members: :inherited-members: @@ -56,7 +56,7 @@ Broker Class Context Class ============= -.. autoclass:: econtext.master.Context +.. autoclass:: mitogen.master.Context :members: :inherited-members: @@ -64,29 +64,29 @@ Context Class Channel Class ------------- -.. autoclass:: econtext.core.Channel +.. autoclass:: mitogen.core.Channel :members: Context Class ------------- -.. autoclass:: econtext.master.Context +.. autoclass:: mitogen.master.Context :members: Utility Functions ================= -.. automodule:: econtext.utils +.. automodule:: mitogen.utils :members: Exceptions ========== -.. autoclass:: econtext.core.Error -.. autoclass:: econtext.core.CallError -.. autoclass:: econtext.core.ChannelError -.. autoclass:: econtext.core.StreamError -.. autoclass:: econtext.core.TimeoutError +.. autoclass:: mitogen.core.Error +.. autoclass:: mitogen.core.CallError +.. autoclass:: mitogen.core.ChannelError +.. autoclass:: mitogen.core.StreamError +.. autoclass:: mitogen.core.TimeoutError diff --git a/docs/conf.py b/docs/conf.py index 16d41277..59a46065 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,11 +10,11 @@ html_show_sphinx = False html_sidebars = {'**': ['globaltoc.html', 'github.html']} html_static_path = ['_static'] html_theme = 'alabaster' -htmlhelp_basename = 'econtextdoc' +htmlhelp_basename = 'mitogendoc' intersphinx_mapping = {'python': ('https://docs.python.org/2', None)} language = None master_doc = 'toc' -project = u'econtext' +project = u'Mitogen' pygments_style = 'sphinx' release = u'master' source_suffix = '.rst' diff --git a/docs/examples.rst b/docs/examples.rst index 8538744e..b1d29589 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -6,7 +6,6 @@ Examples Recursively Nested Bootstrap ---------------------------- - This demonstrates the library's ability to use slave contexts to recursively proxy connections to additional slave contexts, with a uniform API to any slave, and all features (function calls, import forwarding, stdio forwarding, @@ -20,18 +19,18 @@ nested.py: .. code-block:: python import os - import econtext.utils + import mitogen.utils - @econtext.utils.run_with_router + @mitogen.utils.run_with_router def main(router): - econtext.utils.log_to_file() + mitogen.utils.log_to_file() context = None for x in range(1, 11): print 'Connect local%d via %s' % (x, context) context = router.local(via=context, name='local%d' % x) - context.call(os.system, 'pstree -s python -s econtext') + context.call(os.system, 'pstree -s python -s mitogen') Output: @@ -54,15 +53,15 @@ Output: 18:14:07 I ctx.local10: stdout: \-+= 10638 dmw /Applications/iTerm.app/Contents/MacOS/iTerm2 --server bash --login 18:14:07 I ctx.local10: stdout: \-+= 10639 dmw bash --login 18:14:07 I ctx.local10: stdout: \-+= 13632 dmw python nested.py - 18:14:07 I ctx.local10: stdout: \-+- 13633 dmw econtext:dmw@Eldil.local:13632 - 18:14:07 I ctx.local10: stdout: \-+- 13635 dmw econtext:dmw@Eldil.local:13633 - 18:14:07 I ctx.local10: stdout: \-+- 13637 dmw econtext:dmw@Eldil.local:13635 - 18:14:07 I ctx.local10: stdout: \-+- 13639 dmw econtext:dmw@Eldil.local:13637 - 18:14:07 I ctx.local10: stdout: \-+- 13641 dmw econtext:dmw@Eldil.local:13639 - 18:14:07 I ctx.local10: stdout: \-+- 13643 dmw econtext:dmw@Eldil.local:13641 - 18:14:07 I ctx.local10: stdout: \-+- 13645 dmw econtext:dmw@Eldil.local:13643 - 18:14:07 I ctx.local10: stdout: \-+- 13647 dmw econtext:dmw@Eldil.local:13645 - 18:14:07 I ctx.local10: stdout: \-+- 13649 dmw econtext:dmw@Eldil.local:13647 - 18:14:07 I ctx.local10: stdout: \-+- 13651 dmw econtext:dmw@Eldil.local:13649 - 18:14:07 I ctx.local10: stdout: \-+- 13653 dmw pstree -s python -s econtext + 18:14:07 I ctx.local10: stdout: \-+- 13633 dmw mitogen:dmw@Eldil.local:13632 + 18:14:07 I ctx.local10: stdout: \-+- 13635 dmw mitogen:dmw@Eldil.local:13633 + 18:14:07 I ctx.local10: stdout: \-+- 13637 dmw mitogen:dmw@Eldil.local:13635 + 18:14:07 I ctx.local10: stdout: \-+- 13639 dmw mitogen:dmw@Eldil.local:13637 + 18:14:07 I ctx.local10: stdout: \-+- 13641 dmw mitogen:dmw@Eldil.local:13639 + 18:14:07 I ctx.local10: stdout: \-+- 13643 dmw mitogen:dmw@Eldil.local:13641 + 18:14:07 I ctx.local10: stdout: \-+- 13645 dmw mitogen:dmw@Eldil.local:13643 + 18:14:07 I ctx.local10: stdout: \-+- 13647 dmw mitogen:dmw@Eldil.local:13645 + 18:14:07 I ctx.local10: stdout: \-+- 13649 dmw mitogen:dmw@Eldil.local:13647 + 18:14:07 I ctx.local10: stdout: \-+- 13651 dmw mitogen:dmw@Eldil.local:13649 + 18:14:07 I ctx.local10: stdout: \-+- 13653 dmw pstree -s python -s mitogen 18:14:07 I ctx.local10: stdout: \--- 13654 root ps -axwwo user,pid,ppid,pgid,command diff --git a/docs/history.rst b/docs/history.rst index 01787f7b..1ef13427 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -6,7 +6,7 @@ History And Future History ####### -The first version of econtext was written in late 2006 for use in an +The first version of Mitogen was written in late 2006 for use in an infrastructure management program, however at the time I lacked the pragmatism necessary for pushing my little design from concept to finished implementation. I tired of it when no way could be found to unify every communication style @@ -35,14 +35,14 @@ shocked to discover it writing temporary files everywhere, and uploading a All contemporary Devops tooling -Searching around for something to play with, I came across my forgotten -``src/econtext`` directory and somehow in a few hours managed to squash most of -the race conditions and logic bugs that were preventing reliable operation, -write the IO and log forwarders, rewrite the module importer, move from +Searching around for something to play with, I came across my forgotten project +directory and somehow in a few hours managed to squash most of the race +conditions and logic bugs that were preventing reliable operation, write the IO +and log forwarders, rewrite the module importer, move from :py:func:`select.poll` to :py:func:`select.select`, and even refactor the special cases out of the main loop. -So there you have it. As of writing :py:mod:`econtext.core` consists of 681 +So there you have it. As of writing :py:mod:`mitogen.core` consists of 681 source lines, and those 681 lines have taken over a decade to write. I have long had a preference for avoiding infrastructure work commercially, not least for the inescapable depression induced by considering the wasted effort across @@ -55,4 +55,4 @@ Future `View the issue list on GitHub`_ -.. _View the issue list on GitHub: https://github.com/dw/econtext/issues?q=is%3Aopen%20is%3Aissue%20-label%3Abug +.. _View the issue list on GitHub: https://github.com/dw/mitogen/issues?q=is%3Aopen%20is%3Aissue%20-label%3Abug diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 16c8189a..71a1c778 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -1,6 +1,6 @@ -How econtext Works -================== +How Mitogen Works +================= Some effort is required to accomplish the seemingly magical feat of bootstrapping a remote Python process without any software installed on the @@ -17,7 +17,7 @@ necessary for something on the remote to be prepared to decompress the payload and feed it to a Python interpreter. Since we would like to avoid writing an error-prone shell fragment to implement this, and since we must avoid writing to the remote machine's disk in case it is read-only, the Python process -started on the remote machine by ``econtext`` immediately forks in order to +started on the remote machine by Mitogen immediately forks in order to implement the decompression. @@ -25,7 +25,7 @@ Python Command Line ################### The Python command line sent to the host is a base64-encoded copy of the -:py:meth:`econtext.master.LocalStream._first_stage` function, which has been +:py:meth:`mitogen.master.LocalStream._first_stage` function, which has been carefully optimized to reduce its size. Prior to base64 encoding, ``CONTEXT_NAME`` is replaced with the desired context name in the function's source code. @@ -50,8 +50,8 @@ process. After fork, the parent half overwrites its ``stdin`` with the read end of the pipe, and the child half writes the string ``EC0\n``, then begins reading the -:py:mod:`zlib`-compressed payload supplied on ``stdin`` by the econtext master, -and writing the decompressed result to the write-end of the UNIX pipe. +:py:mod:`zlib`-compressed payload supplied on ``stdin`` by the master, and +writing the decompressed result to the write-end of the UNIX pipe. To allow recovery of ``stdin`` for reuse by the bootstrapped process for master<->slave communication, it is necessary for the first stage to avoid @@ -83,25 +83,25 @@ Bootstrap Preparation Now we have the mechanism in place to send a :py:mod:`zlib`-compressed script to the remote Python interpreter, it is time to choose what to send. -The script sent is simply the source code for :py:mod:`econtext.core`, with a +The script sent is simply the source code for :py:mod:`mitogen.core`, with a single line suffixed to trigger execution of the -:py:meth:`econtext.core.ExternalContext.main` function. The encoded arguments +:py:meth:`mitogen.core.ExternalContext.main` function. The encoded arguments to the main function include some additional details, such as the logging package level that was active in the parent process, and a random secret key that may later be used to generate HMAC signatures over the data frames that will be exchanged after bootstrap. After the script source code is prepared, it is passed through -:py:func:`econtext.master.minimize_source` to strip it of docstrings and +:py:func:`mitogen.master.minimize_source` to strip it of docstrings and comments, while preserving line numbers. This reduces the compressed payload by around 20%. -Preserving The `econtext.core` Source -##################################### +Preserving The `mitogen.core` Source +#################################### One final trick is implemented in the first stage: after bootstrapping the new -slave, it writes a duplicate copy of the :py:mod:`econtext.core` source it just +slave, it writes a duplicate copy of the :py:mod:`mitogen.core` source it just used to bootstrap it back into another pipe connected to the slave. The slave's module importer cache is initialized with a copy of the source, so that subsequent bootstraps of slave-of-slaves do not require the source to be @@ -122,26 +122,26 @@ to receive messages. ExternalContext.main() ---------------------- -.. automethod:: econtext.core.ExternalContext.main +.. automethod:: mitogen.core.ExternalContext.main -Generating A Synthetic `econtext` Package -######################################### +Generating A Synthetic `mitogen` Package +######################################## -Since the bootstrap consists of the :py:mod:`econtext.core` source code, and +Since the bootstrap consists of the :py:mod:`mitogen.core` source code, and this code is loaded by Python by way of its main script (``__main__`` module), initially the module layout in the slave will be incorrect. The first step taken after bootstrap is to rearrange :py:data:`sys.modules` slightly -so that :py:mod:`econtext.core` appears in the correct location, and all +so that :py:mod:`mitogen.core` appears in the correct location, and all classes defined in that module have their ``__module__`` attribute fixed up such that :py:mod:`cPickle` correctly serializes instance module names. -Once a synthetic :py:mod:`econtext` package and :py:mod:`econtext.core` module +Once a synthetic :py:mod:`mitogen` package and :py:mod:`mitogen.core` module have been generated, the bootstrap **deletes** `sys.modules['__main__']`, so that any attempt to import it (by :py:mod:`cPickle`) will cause the import to -be satisfied by fetching the econtext master's actual ``__main__`` module. This -is necessary to allow master programs to be written as a self-contained Python +be satisfied by fetching the master's actual ``__main__`` module. This is +necessary to allow master programs to be written as a self-contained Python script. @@ -161,8 +161,8 @@ Setup Logging The slave's :py:mod:`logging` package root logger is configured to have the same log level as the root logger in the master, and -:py:class:`econtext.core.LogHandler` is installed to forward logs to the master -context's :py:data:`FORWARD_LOG ` handle. +:py:class:`mitogen.core.LogHandler` is installed to forward logs to the master +context's :py:data:`FORWARD_LOG ` handle. The log level is copied into the slave to avoid generating a potentially large amount of network IO forwarding logs that will simply be filtered away once @@ -172,7 +172,7 @@ they reach the master. The Module Importer ################### -An instance of :py:class:`econtext.core.Importer` is installed in +An instance of :py:class:`mitogen.core.Importer` is installed in :py:data:`sys.meta_path`, where Python's ``import`` statement will execute it before attempting to find a module locally. @@ -180,7 +180,7 @@ before attempting to find a module locally. Standard IO Redirection ####################### -Two instances of :py:class:`econtext.core.IoLogger` are created, one for +Two instances of :py:class:`mitogen.core.IoLogger` are created, one for ``stdout`` and one for ``stderr``. This class creates a UNIX pipe whose read end is added to the IO multiplexer, and whose write end is used to overwrite the handles inherited during process creation. @@ -200,26 +200,26 @@ Function Call Dispatch ###################### After all initialization is complete, the slave's main thread sits in a loop -reading from a :py:class:`Channel ` connected to the -:py:data:`CALL_FUNCTION ` handle. This handle is +reading from a :py:class:`Channel ` connected to the +:py:data:`CALL_FUNCTION ` handle. This handle is written to by -:py:meth:`call_with_deadline() ` -and :py:meth:`call() `. +:py:meth:`call_with_deadline() ` +and :py:meth:`call() `. Shutdown ######## When the master signals the :py:data:`CALL_FUNCTION -` :py:class:`Channel ` is -closed, the slave calls :py:meth:`shutdown() ` -followed by :py:meth:`wait() ` on its own broker, +` :py:class:`Channel ` is +closed, the slave calls :py:meth:`shutdown() ` +followed by :py:meth:`wait() ` on its own broker, triggering graceful shutdown. During shutdown, the master will wait a few seconds for slaves to disconnect gracefully before force disconnecting them, while the slaves will use that time to call :py:meth:`socket.shutdown(SHUT_WR) ` on their -:py:class:`IoLogger ` socket's write ends before +:py:class:`IoLogger ` socket's write ends before draining any remaining data buffered on the read ends. An alternative approach is to wait until the socket is completely closed, with @@ -260,12 +260,12 @@ master and slave: Masters listen on the following handles: -.. data:: econtext.core.FORWARD_LOG +.. data:: mitogen.core.FORWARD_LOG Receives `(logger_name, level, msg)` 3-tuples and writes them to the - master's ``econtext.ctx.`` logger. + master's ``mitogen.ctx.`` logger. -.. data:: econtext.core.GET_MODULE +.. data:: mitogen.core.GET_MODULE Receives `(reply_to, fullname)` 2-tuples, looks up the source code for the module named ``fullname``, and writes the source along with some metadata @@ -274,11 +274,11 @@ Masters listen on the following handles: Slaves listen on the following handles: -.. data:: econtext.core.CALL_FUNCTION +.. data:: mitogen.core.CALL_FUNCTION Receives `(with_context, mod_name, class_name, func_name, args, kwargs)` 5-tuples from - :py:meth:`call_with_deadline() `, + :py:meth:`call_with_deadline() `, imports ``mod_name``, then attempts to execute `class_name.func_name(\*args, \**kwargs)`. @@ -289,7 +289,7 @@ Slaves listen on the following handles: it, and arranging for the connection to its parent context to be closed shortly thereafter. -.. data:: econtext.core.ADD_ROUTE +.. data:: mitogen.core.ADD_ROUTE Receives `(target_id, via_id)` integer tuples, describing how messages arriving at this context on any Stream should be forwarded on the stream @@ -302,7 +302,7 @@ Slaves listen on the following handles: established. Given a chain `master -> ssh1 -> sudo1`, no `ADD_ROUTE` message is - necessary, since :py:class:`econtext.core.Router` in the `ssh` context can + necessary, since :py:class:`mitogen.core.Router` in the `ssh` context can arrange to update its routes while setting up the new slave during `proxy_connect()`. @@ -315,11 +315,11 @@ Slaves listen on the following handles: Slaves that have ever been used to create a descendent child context also listen on the following handles: -.. data:: econtext.core.GET_MODULE +.. data:: mitogen.core.GET_MODULE As with master's ``GET_MODULE``, except this implementation - (:py:class:`econtext.master.ModuleForwarder`) serves responses using - :py:class:`econtext.core.Importer`'s cache before forwarding the request to + (:py:class:`mitogen.master.ModuleForwarder`) serves responses using + :py:class:`mitogen.core.Importer`'s cache before forwarding the request to its parent context. The response is cached by each context in turn before being forwarded on to the slave context that originally made the request. In this way, the master need never re-send a module it has already sent to @@ -327,19 +327,19 @@ listen on the following handles: Additional handles are created to receive the result of every function call -triggered by :py:meth:`call_with_deadline() `. +triggered by :py:meth:`call_with_deadline() `. Sentinel Value ############## -.. autodata:: econtext.core._DEAD +.. autodata:: mitogen.core._DEAD -The special value :py:data:`econtext.core._DEAD` is used to signal +The special value :py:data:`mitogen.core._DEAD` is used to signal disconnection or closure of the remote end. It is used internally by -:py:class:`Channel ` and also passed to any function +:py:class:`Channel ` and also passed to any function still registered with :py:meth:`add_handler() -` during Broker shutdown. +` during Broker shutdown. Use of Pickle @@ -353,9 +353,9 @@ serialization code in the bootstrap. The pickler active in slave contexts will instantiate any class, however in the master it is initially restricted to only permitting -:py:class:`CallError ` and :py:data:`_DEAD -`. While not recommended, it is possible to register more -using :py:meth:`econtext.master.LocalStream.allow_class`. +:py:class:`CallError ` and :py:data:`_DEAD +`. While not recommended, it is possible to register more +using :py:meth:`mitogen.master.LocalStream.allow_class`. The choice of Pickle is one area to be revisited later. All accounts suggest it cannot be used securely, however few of those accounts appear to be expert, and @@ -379,7 +379,7 @@ off-the-shelf implementations are for the most part entirely inappropriate. For example, a minimal copy of Twisted weighs in at around 440KiB and is composed of approximately 115 files. Even if we could arrange for an entire Python package to be transferred during bootstrap, this minimal configuration is -massive in comparison to econtext's solution, multiplies quickly in the +massive in comparison to Mitogen's solution, multiplies quickly in the presence of many machines, and would require manually splitting up the parts of Twisted that we would like to use. @@ -391,7 +391,7 @@ Routing assumes it is impossible to construct a tree such that one of a context's parents will not know the ID of a target the context is attempting to communicate with. -When :py:class:`econtext.core.Router` receives a message, it checks the IDs +When :py:class:`mitogen.core.Router` receives a message, it checks the IDs associated with its directly connected streams for a potential route. If any stream matches, either because it directly connects to the target ID, or because the master sent an ``ADD_ROUTE`` message associating it, then the @@ -436,15 +436,15 @@ currently allocate new context IDs anyway. Differences Between Master And Slave Brokers ############################################ -The main difference between :py:class:`econtext.core.Broker` and -:py:class:`econtext.master.Broker` is that when the stream connection to the +The main difference between :py:class:`mitogen.core.Broker` and +:py:class:`mitogen.master.Broker` is that when the stream connection to the parent is lost in a slave, the broker will trigger its own shutdown. The Module Importer ------------------- -:py:class:`econtext.core.Importer` is still a work in progress, as there +:py:class:`mitogen.core.Importer` is still a work in progress, as there are a variety of approaches to implementing it, and the present implementation is not pefectly efficient in every case. @@ -452,7 +452,7 @@ It operates by intercepting ``import`` statements via `sys.meta_path`, asking Python if it can satisfy the import by itself, and if not, indicating to Python that it is capable of loading the module. -In :py:meth:`load_module() ` an RPC is +In :py:meth:`load_module() ` an RPC is started to the parent context, requesting the module source code. Once the source is fetched, the method builds a new module object using the best practice documented in PEP-302. @@ -480,7 +480,7 @@ pointless network roundtrips. Therefore in addition to the child modules known to exist. Before indicating it can satisfy an import request, -:py:class:`econtext.core.Importer` first checks to see if the module belongs to +:py:class:`mitogen.core.Importer` first checks to see if the module belongs to a package it has previously imported, and if so, ignores the request if the module does not appear in the enumeration of child modules belonging to the package. @@ -505,7 +505,7 @@ mechanism is not portable to non-UNIX operating systems, and does not work in every case, for example when Python blocks signals during a variety of :py:mod:`threading` package operations. -At some point it is likely econtext will be extended to support starting slaves +At some point it is likely Mitogen will be extended to support starting slaves running on Windows. When that happens, it would be nice if the process model on Windows and UNIX did not differ, and in fact the code used on both were identical. diff --git a/docs/index.rst b/docs/index.rst index 712836b4..6a55b552 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,15 +1,29 @@ -Python Execution Contexts -========================= +Mitogen +======= -**6KiB of sugar and no fat** +Mitogen is a Python library for writing distributed self-replicating programs. +.. raw:: html -Introduction ------------- + -``econtext`` is a library for writing distributed self-replicating programs in -Python. +.. warning:: + + This is alpha-quality code. If you intend to use it, be aware of how little + real world testing it has received, the total absence of any systematic + tests, and the nightmare-level difficulty of debugging hangs in a tree of + processes running identical code straddling multiple thread and machine + boundaries! ``router.enable_debug()`` is your friend. + + If you think you have a use for this software, please `drop me an e-mail`_ + so that expectations and bug fixes can be managed sensibly. + + .. _drop me an e-mail: dw@botanicus.net There is no requirement for installing packages, copying files around, writing shell snippets, upfront configuration, or providing any secondary link to a @@ -33,36 +47,14 @@ common privilege escalation techniques like `sudo`, potentially in combination with exotic connection methods such as WMI, `telnet`, or console-over-IPMI. -.. raw:: html - - - -.. warning:: - - This is alpha-quality code. If you intend to use it, be aware of how little - real world testing it has received, the total absence of any systematic - tests, and the nightmare-level difficulty of debugging hangs in a tree of - processes running identical code straddling multiple thread and machine - boundaries! ``router.enable_debug()`` is your friend. - - If you think you have a use for this software, please `drop me an e-mail`_ - so that expectations and bug fixes can be managed sensibly. - - .. _drop me an e-mail: dw@botanicus.net - - Automatic Bootstrap ################### -The package's main feature is enabling your Python program to bootstrap and +Mitogen's main feature is enabling your Python program to bootstrap and communicate with new copies of itself under its control running on remote machines, **using only an existing installed Python interpreter and SSH client**, something that by default can be found on almost all contemporary -machines in the wild. To accomplish bootstrap, econtext uses a single 600 byte +machines in the wild. To accomplish bootstrap, Mitogen uses a single 600 byte SSH command line and 6KB of its own source code sent to stdin of the remote SSH connection. @@ -71,9 +63,9 @@ connection. $ python preamble_size.py SSH command size: 576 Preamble size: 6360 (6.21KiB) - econtext.master size: 4104 (4.01KiB) - econtext.ssh size: 295 (0.29KiB) - econtext.sudo size: 1210 (1.18KiB) + mitogen.master size: 4104 (4.01KiB) + mitogen.ssh size: 295 (0.29KiB) + mtiogen.sudo size: 1210 (1.18KiB) Once bootstrapped, the remote process is configured with a customizable **argv[0]**, readily visible to system administrators of the remote machine @@ -83,7 +75,7 @@ using the UNIX **ps** command: 20051 ? Ss 0:00 \_ sshd: dmw [priv] 20053 ? S 0:00 | \_ sshd: dmw@notty - 20054 ? Ssl 0:00 | \_ econtext:dmw@Eldil.home:22476 + 20054 ? Ssl 0:00 | \_ mitogen:dmw@Eldil.home:22476 20103 ? S 0:00 | \_ tar zxvf myapp.tar.gz The example context was started by UID ``dmw`` on host ``Eldil.home``, process @@ -125,6 +117,33 @@ program crashes, communication is lost, or the application code running in the context has hung. +Module Forwarder +################ + +In addition to an IO multiplexer, slaves are configured with a custom `PEP-302 +importer`_ that forwards requests for unknown Python modules back to the host +program. When your program asks a context to execute code from an unknown +module, all requisite modules are transferred automatically and imported +entirely in RAM without need for further configuration. + +.. _PEP-302 importer: https://www.python.org/dev/peps/pep-0302/ + +.. code-block:: python + + import myapp.mypkg.mymodule + + # myapp/__init__.py, myapp/mypkg/__init__.py, and myapp/mypkg/mymodule.py + # are transferred automatically. + print context.call(myapp.mymodule.my_function) + +As the forwarder reuses the import mechanism, it should integrate cleanly with +any tool such as `py2exe`_ that correctly implement the protocols in PEP-302, +allowing truly single file applications to run across multiple machines without +further effort. + +.. _py2exe: http://www.py2exe.org/ + + SSH Client Emulation #################### @@ -135,12 +154,12 @@ Support is included for starting subprocesses with a modified environment, that cause their attempt to use SSH to be redirected back into the host program. In this way tools like `rsync`, `sftp`, and `scp` can efficiently reuse the host program's existing connection to the remote machine, including any -firewall/user account hopping in use, with zero additional configuration. +firewall/user account hopping in use, with no additional configuration. Scenarios that were not previously possible with these tools are enabled, such -as running sftp and rsync over a sudo session, to an account the user cannot -otherwise directly log into, including in restrictive environments that for -example enforce an interactive sudo TTY and account password. +as running `sftp` and `rsync` over a `sudo` session, to an account the user +cannot otherwise directly log into, including in restrictive environments that +for example enforce an interactive TTY and account password. .. code-block:: python @@ -150,39 +169,11 @@ example enforce an interactive sudo TTY and account password. fileserver = router.ssh(via=bastion, hostname='fileserver') # Transparently tunnelled over fileserver -> .. -> sudo.webapp link - fileserver.call(econtext.fakessh.run, webapp, [ + fileserver.call(mitogen.fakessh.run, webapp, [ 'rsync', 'appdata', 'appserver:appdata' ]) -Module Forwarder -################ - -In addition to an IO multiplexer, the external context is configured with a -custom `PEP-302 importer`_ that forwards requests for unknown Python modules -back to the host program. When your program asks an external context to execute -code from an unknown module, all requisite modules are transferred -automatically and imported entirely in RAM without need for further -configuration. - -.. _PEP-302 importer: https://www.python.org/dev/peps/pep-0302/ - -.. code-block:: python - - import myapp.mypkg.mymodule - - # myapp/__init__.py, myapp/mypkg/__init__.py, and myapp/mypkg/mymodule.py - # are transferred automatically. - print context.call(myapp.mymodule.my_function) - -As the forwarder reuses the import mechanism, it should integrate cleanly with -any tool such as `py2exe`_ that correctly implement the protocols in PEP-302, -allowing truly single file applications to run across multiple machines without -further effort. - -.. _py2exe: http://www.py2exe.org/ - - Logging Forwarder ################# @@ -192,8 +183,8 @@ location. .. code:: - 18:15:29 D econtext.ctx.k3: econtext: Importer.find_module('econtext.zlib') - 18:15:29 D econtext.ctx.k3: econtext: _dispatch_calls((1002L, False, 'posix', None, 'system', ('ls -l /proc/self/fd',), {})) + 18:15:29 D mitogen.ctx.k3: mitogen: Importer.find_module('mitogen.zlib') + 18:15:29 D mitogen.ctx.k3: mitogen: _dispatch_calls((1002L, False, 'posix', None, 'system', ('ls -l /proc/self/fd',), {})) Stdio Forwarder @@ -206,9 +197,9 @@ uptime')** without further need to capture or manage output. .. code:: - 18:17:28 D econtext.ctx.k3: econtext: _dispatch_calls((1002L, False, 'posix', None, 'system', ('hostname; uptime',), {})) - 18:17:56 I econtext.ctx.k3: stdout: k3 - 18:17:56 I econtext.ctx.k3: stdout: 17:37:10 up 562 days, 2:25, 5 users, load average: 1.24, 1.13, 1.14 + 18:17:28 D mitogen.ctx.k3: mitogen: _dispatch_calls((1002L, False, 'posix', None, 'system', ('hostname; uptime',), {})) + 18:17:56 I mitogen.ctx.k3: stdout: k3 + 18:17:56 I mitogen.ctx.k3: stdout: 17:37:10 up 562 days, 2:25, 5 users, load average: 1.24, 1.13, 1.14 Blocking Code Friendly @@ -217,7 +208,7 @@ Blocking Code Friendly Within each process, a private thread runs the I/O multiplexer, leaving the main thread and any additional application threads free to perform useful work. -While econtext is internally asynchronous it hides this asynchrony from +While Mitogen is internally asynchronous, it hides this asynchrony from consumer code. This is since writing asynchronous code is mostly a foreign concept to the target application of managing infrastructure. It should be possible to rewrite a shell script in Python without significant restructuring, @@ -278,7 +269,7 @@ usual into the slave process. import os import sys - import econtext + import mitogen def install_app(): @@ -290,12 +281,12 @@ usual into the slave process. print __doc__ sys.exit(1) - context = econtext.ssh.connect(broker, sys.argv[1]) + context = mitogen.ssh.connect(broker, sys.argv[1]) context.call(install_app) - if __name__ == '__main__' and not econtext.slave: - import econtext.utils - econtext.utils.run_with_broker(main) + if __name__ == '__main__' and not mitogen.slave: + import mitogen.utils + mitogen.utils.run_with_broker(main) Event-driven IO @@ -334,7 +325,7 @@ Compatibility The package is written using syntax compatible all the way back to **Python 2.4** released November 2004, making it suitable for managing a fleet of -potentially ancient corporate hardware. For example econtext can be used out of +potentially ancient corporate hardware. For example Mitogen can be used out of the box against Red Hat Enterprise Linux 5, released in 2007. There is currently no support for Python 3, and no solid plan for supporting it @@ -346,5 +337,5 @@ as ``six.py`` are likely to be unsuitable. Zero Dependencies ################# -Econtext is implemented entirely using the standard library functionality and +Mitogen is implemented entirely using the standard library functionality and interfaces that were available in Python 2.4. diff --git a/docs/internals.rst b/docs/internals.rst index dc8df018..aec3f775 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -3,40 +3,40 @@ Internal API Reference ********************** -econtext.core -============= +mitogen.core +============ Side Class ---------- -.. autoclass:: econtext.core.Side +.. autoclass:: mitogen.core.Side :members: Stream Classes -------------- -.. autoclass:: econtext.core.BasicStream +.. autoclass:: mitogen.core.BasicStream :members: -.. autoclass:: econtext.core.Stream +.. autoclass:: mitogen.core.Stream :members: -.. autoclass:: econtext.master.Stream +.. autoclass:: mitogen.master.Stream :members: -.. autoclass:: econtext.ssh.Stream +.. autoclass:: mitogen.ssh.Stream :members: Other Stream Subclasses ----------------------- -.. autoclass:: econtext.core.IoLogger +.. autoclass:: mitogen.core.IoLogger :members: -.. autoclass:: econtext.core.Waker +.. autoclass:: mitogen.core.Waker :members: @@ -44,18 +44,18 @@ Other Stream Subclasses ExternalContext Class --------------------- -.. autoclass:: econtext.core.ExternalContext +.. autoclass:: mitogen.core.ExternalContext -econtext.master +mitogen.master =============== -.. autoclass:: econtext.master.ProcessMonitor +.. autoclass:: mitogen.master.ProcessMonitor Helper Functions ---------------- -.. autofunction:: econtext.master.create_child -.. autofunction:: econtext.master.get_child_modules -.. autofunction:: econtext.master.minimize_source +.. autofunction:: mitogen.master.create_child +.. autofunction:: mitogen.master.get_child_modules +.. autofunction:: mitogen.master.minimize_source diff --git a/econtext/__init__.py b/mitogen/__init__.py similarity index 72% rename from econtext/__init__.py rename to mitogen/__init__.py index d9481b00..11bd98e4 100644 --- a/econtext/__init__.py +++ b/mitogen/__init__.py @@ -1,5 +1,5 @@ """ -On the econtext master, this is imported from ``econtext/__init__.py`` as would +On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would be expected. On the slave, it is built dynamically during startup. """ @@ -13,12 +13,12 @@ be expected. On the slave, it is built dynamically during startup. #: os.system('hostname') #: #: def main(broker): -#: context = econtext.master.connect(broker) +#: context = mitogen.master.connect(broker) #: context.call(do_work) # Causes slave to import __main__. #: -#: if __name__ == '__main__' and not econtext.slave: -#: import econtext.utils -#: econtext.utils.run_with_broker(main) +#: if __name__ == '__main__' and not mitogen.slave: +#: import mitogen.utils +#: mitogen.utils.run_with_broker(main) #: slave = False diff --git a/econtext/ansible/__init__.py b/mitogen/ansible/__init__.py similarity index 100% rename from econtext/ansible/__init__.py rename to mitogen/ansible/__init__.py diff --git a/econtext/ansible/connection.py b/mitogen/ansible/connection.py similarity index 75% rename from econtext/ansible/connection.py rename to mitogen/ansible/connection.py index ef6bd2bf..4070707f 100644 --- a/econtext/ansible/connection.py +++ b/mitogen/ansible/connection.py @@ -1,7 +1,7 @@ """ Basic Ansible connection plug-in mostly useful for testing functionality, due to Ansible's use of the multiprocessing package a lot more work is required -to share the econtext SSH connection across tasks. +to share the mitogen SSH connection across tasks. Enable it by: @@ -10,15 +10,15 @@ Enable it by: connection_plugins = plugins/connection $ mkdir -p plugins/connection - $ cat > plugins/connection/econtext_conn.py <<-EOF - from econtext.ansible.connection import Connection + $ cat > plugins/connection/mitogen_conn.py <<-EOF + from mitogen.ansible.connection import Connection EOF """ -import econtext.master -import econtext.ssh -import econtext.utils -from econtext.ansible import helpers +import mitogen.master +import mitogen.ssh +import mitogen.utils +from mitogen.ansible import helpers import ansible.plugins.connection @@ -28,7 +28,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): context = None become_methods = [] - transport = 'econtext' + transport = 'mitogen' @property def connected(self): @@ -37,11 +37,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): def _connect(self): if self.connected: return - self.broker = econtext.master.Broker() + self.broker = mitogen.master.Broker() if self._play_context.remote_addr == 'localhost': - self.context = econtext.master.connect(self.broker) + self.context = mitogen.master.connect(self.broker) else: - self.context = econtext.ssh.connect(broker, + self.context = mitogen.ssh.connect(broker, self._play_context.remote_addr) def exec_command(self, cmd, in_data=None, sudoable=True): diff --git a/econtext/ansible/helpers.py b/mitogen/ansible/helpers.py similarity index 100% rename from econtext/ansible/helpers.py rename to mitogen/ansible/helpers.py diff --git a/econtext/compat/__init__.py b/mitogen/compat/__init__.py similarity index 100% rename from econtext/compat/__init__.py rename to mitogen/compat/__init__.py diff --git a/econtext/compat/pkgutil.py b/mitogen/compat/pkgutil.py similarity index 100% rename from econtext/compat/pkgutil.py rename to mitogen/compat/pkgutil.py diff --git a/econtext/core.py b/mitogen/core.py similarity index 94% rename from econtext/core.py rename to mitogen/core.py index 70033573..d22cdb19 100644 --- a/econtext/core.py +++ b/mitogen/core.py @@ -24,8 +24,8 @@ import traceback import zlib -LOG = logging.getLogger('econtext') -IOLOG = logging.getLogger('econtext.io') +LOG = logging.getLogger('mitogen') +IOLOG = logging.getLogger('mitogen.io') IOLOG.setLevel(logging.INFO) GET_MODULE = 100 @@ -36,14 +36,14 @@ ADD_ROUTE = 103 CHUNK_SIZE = 16384 -if __name__ == 'econtext.core': +if __name__ == 'mitogen.core': # When loaded using import mechanism, ExternalContext.main() will not have - # a chance to set the synthetic econtext global, so just import it here. - import econtext + # a chance to set the synthetic mitogen global, so just import it here. + import mitogen else: # When loaded as __main__, ensure classes and functions gain a __module__ # attribute consistent with the host process, so that pickling succeeds. - __name__ = 'econtext.core' + __name__ = 'mitogen.core' class Error(Exception): @@ -53,7 +53,7 @@ class Error(Exception): class CallError(Error): - """Raised when :py:meth:`Context.call() ` + """Raised when :py:meth:`Context.call() ` fails. A copy of the traceback from the external context is appended to the exception message. """ @@ -113,7 +113,7 @@ def io_op(func, *args): this will be replaced later by a 'goodbye' message to avoid reading from a disconnected endpoint, allowing for more robust error reporting. - When connected over a socket (e.g. econtext.master.create_child()), + When connected over a socket (e.g. mitogen.master.create_child()), ECONNRESET may be triggered by any read or write. """ try: @@ -129,7 +129,7 @@ def enable_debug_logging(): root = logging.getLogger() root.setLevel(logging.DEBUG) IOLOG.setLevel(logging.DEBUG) - fp = open('/tmp/econtext.%s.log' % (os.getpid(),), 'w', 1) + fp = open('/tmp/mitogen.%s.log' % (os.getpid(),), 'w', 1) set_cloexec(fp.fileno()) handler = logging.StreamHandler(fp) handler.formatter = logging.Formatter( @@ -147,7 +147,7 @@ class Message(object): data = None def __init__(self, **kwargs): - self.src_id = econtext.context_id + self.src_id = mitogen.context_id vars(self).update(kwargs) _find_global = None @@ -287,22 +287,22 @@ class Importer(object): """ def __init__(self, context, core_src): self._context = context - self._present = {'econtext': [ - 'econtext.ansible', - 'econtext.compat', - 'econtext.compat.pkgutil', - 'econtext.fakessh', - 'econtext.master', - 'econtext.ssh', - 'econtext.sudo', - 'econtext.utils', + self._present = {'mitogen': [ + 'mitogen.ansible', + 'mitogen.compat', + 'mitogen.compat.pkgutil', + 'mitogen.fakessh', + 'mitogen.master', + 'mitogen.ssh', + 'mitogen.sudo', + 'mitogen.utils', ]} self.tls = threading.local() self._cache = {} if core_src: - self._cache['econtext.core'] = ( + self._cache['mitogen.core'] = ( None, - 'econtext/core.py', + 'mitogen/core.py', zlib.compress(core_src), ) @@ -381,7 +381,7 @@ class LogHandler(logging.Handler): self.local = threading.local() def emit(self, rec): - if rec.name == 'econtext.io' or \ + if rec.name == 'mitogen.io' or \ getattr(self.local, 'in_emit', False): return @@ -500,7 +500,7 @@ class BasicStream(object): class Stream(BasicStream): """ - :py:class:`BasicStream` subclass implementing econtext's :ref:`stream + :py:class:`BasicStream` subclass implementing mitogen's :ref:`stream protocol `. """ _input_buf = '' @@ -625,7 +625,7 @@ class Context(object): be called from any thread.""" msg.dst_id = self.context_id if msg.src_id is None: - msg.src_id = econtext.context_id + msg.src_id = mitogen.context_id self.router.route(msg) def send_await(self, msg, deadline=None): @@ -835,16 +835,16 @@ class Router(object): def _async_route(self, msg): IOLOG.debug('%r._async_route(%r)', self, msg) - if msg.dst_id == econtext.context_id: + if msg.dst_id == mitogen.context_id: return self._invoke(msg) stream = self._stream_by_id.get(msg.dst_id) if stream is None: - stream = self._stream_by_id.get(econtext.parent_id) + stream = self._stream_by_id.get(mitogen.parent_id) if stream is None: LOG.error('%r: no route for %r, my ID is %r', - self, msg, econtext.context_id) + self, msg, mitogen.context_id) return stream.send(msg) @@ -881,7 +881,7 @@ class Broker(object): self._waker = Waker(self) self.start_receive(self._waker) self._thread = threading.Thread(target=self._broker_main, - name='econtext-broker') + name='mitogen-broker') self._thread.start() def defer(self, func, *args, **kwargs): @@ -1005,24 +1005,24 @@ class ExternalContext(object): .. attribute:: broker - The :py:class:`econtext.core.Broker` instance. + The :py:class:`mitogen.core.Broker` instance. .. attribute:: context - The :py:class:`econtext.core.Context` instance. + The :py:class:`mitogen.core.Context` instance. .. attribute:: channel - The :py:class:`econtext.core.Channel` over which + The :py:class:`mitogen.core.Channel` over which :py:data:`CALL_FUNCTION` requests are received. .. attribute:: stdout_log - The :py:class:`econtext.core.IoLogger` connected to ``stdout``. + The :py:class:`mitogen.core.IoLogger` connected to ``stdout``. .. attribute:: importer - The :py:class:`econtext.core.Importer` instance. + The :py:class:`mitogen.core.Importer` instance. .. attribute:: stdout_log @@ -1080,19 +1080,19 @@ class ExternalContext(object): sys.meta_path.append(self.importer) def _setup_package(self, context_id, parent_id): - global econtext - econtext = imp.new_module('econtext') - econtext.__package__ = 'econtext' - econtext.__path__ = [] - econtext.__loader__ = self.importer - econtext.slave = True - econtext.context_id = context_id - econtext.parent_id = parent_id - econtext.core = sys.modules['__main__'] - econtext.core.__file__ = 'x/econtext/core.py' # For inspect.getsource() - econtext.core.__loader__ = self.importer - sys.modules['econtext'] = econtext - sys.modules['econtext.core'] = econtext.core + global mitogen + mitogen = imp.new_module('mitogen') + mitogen.__package__ = 'mitogen' + mitogen.__path__ = [] + mitogen.__loader__ = self.importer + mitogen.slave = True + mitogen.context_id = context_id + mitogen.parent_id = parent_id + mitogen.core = sys.modules['__main__'] + mitogen.core.__file__ = 'x/mitogen/core.py' # For inspect.getsource() + mitogen.core.__loader__ = self.importer + sys.modules['mitogen'] = mitogen + sys.modules['mitogen.core'] = mitogen.core del sys.modules['__main__'] def _setup_stdio(self): diff --git a/econtext/fakessh.py b/mitogen/fakessh.py similarity index 82% rename from econtext/fakessh.py rename to mitogen/fakessh.py index 8286b20a..d313477d 100644 --- a/econtext/fakessh.py +++ b/mitogen/fakessh.py @@ -1,6 +1,6 @@ """ fakessh is a stream implementation that starts a local subprocess with its -environment modified such that ``PATH`` searches for `ssh` return an econtext +environment modified such that ``PATH`` searches for `ssh` return an mitogen implementation of the SSH command. When invoked, this tool arranges for the command line supplied by the calling program to be executed in a context already established by the master process, reusing the master's (possibly @@ -13,7 +13,7 @@ or firewall hopping configurations, and enables these tools to be used in impossible scenarios, such as over `sudo` with ``requiretty`` enabled. The fake `ssh` command source is written to a temporary file on disk, and -consists of a copy of the :py:mod:`econtext.core` source code (just like any +consists of a copy of the :py:mod:`mitogen.core` source code (just like any other child context), with a line appended to cause it to connect back to the host process over an FD it inherits. As there is no reliance on an existing filesystem file, it is possible for child contexts to use fakessh. @@ -28,15 +28,15 @@ Sequence: buffer has a `_fakessh_main()` ``CALL_FUNCTION`` enqueued. 2. Target program (`rsync/scp/sftp`) invoked, which internally executes `ssh` from ``PATH``. - 3. :py:mod:`econtext.core` bootstrap begins, recovers the stream FD + 3. :py:mod:`mitogen.core` bootstrap begins, recovers the stream FD inherited via the target program, established itself as the fakessh context. 4. `_fakessh_main()` ``CALL_FUNCTION`` is read by fakessh context, - a. sets up :py:class:`econtext.fakessh.IoPump` for stdio, registers + a. sets up :py:class:`mitogen.fakessh.IoPump` for stdio, registers stdin_handle for local context. b. Enqueues ``CALL_FUNCTION`` for `_start_slave()` invoked in target context, i. the program from the `ssh` command line is started - ii. sets up :py:class:`econtext.fakessh.IoPump` for `ssh` command + ii. sets up :py:class:`mitogen.fakessh.IoPump` for `ssh` command line process's stdio pipes iii. returns `(control_handle, stdin_handle)` to `_fakessh_main()` 5. `_fakessh_main()` receives control/stdin handles from from `_start_slave()`, @@ -62,10 +62,10 @@ import sys import tempfile import threading -import econtext.core -import econtext.master +import mitogen.core +import mitogen.master -from econtext.core import LOG, IOLOG +from mitogen.core import LOG, IOLOG SSH_GETOPTS = ( @@ -73,18 +73,18 @@ SSH_GETOPTS = ( "ACD:E:F:I:KL:MNO:PQ:R:S:TVw:W:XYy" ) -_econtext = None +_mitogen = None -class IoPump(econtext.core.BasicStream): +class IoPump(mitogen.core.BasicStream): _output_buf = '' _closed = False def __init__(self, process, broker, stdin_fd, stdout_fd): self.process = process self._broker = broker - self.receive_side = econtext.core.Side(self, stdout_fd) - self.transmit_side = econtext.core.Side(self, stdin_fd) + self.receive_side = mitogen.core.Side(self, stdout_fd) + self.transmit_side = mitogen.core.Side(self, stdin_fd) def write(self, s): self._output_buf += s @@ -117,7 +117,7 @@ class IoPump(econtext.core.BasicStream): s = self.receive_side.read() IOLOG.debug('%r.on_receive() -> len %r', self, len(s)) if s: - econtext.core.fire(self, 'receive', s) + mitogen.core.fire(self, 'receive', s) else: self.on_disconnect(broker) @@ -144,11 +144,11 @@ class Process(object): self.control = None self.wake_event = threading.Event() - econtext.core.listen(self.pump, 'disconnect', self._on_pump_disconnect) - econtext.core.listen(self.pump, 'receive', self._on_pump_receive) + mitogen.core.listen(self.pump, 'disconnect', self._on_pump_disconnect) + mitogen.core.listen(self.pump, 'receive', self._on_pump_receive) if proc: - pmon = econtext.master.ProcessMonitor.instance() + pmon = mitogen.master.ProcessMonitor.instance() pmon.add(proc.pid, self._on_proc_exit) def __repr__(self): @@ -159,19 +159,19 @@ class Process(object): self.control.put(('exit', status)) def _on_stdin(self, msg): - if msg == econtext.core._DEAD: + if msg == mitogen.core._DEAD: return data = msg.unpickle() IOLOG.debug('%r._on_stdin(%r)', self, data) - if data == econtext.core._DEAD: + if data == mitogen.core._DEAD: self.pump.close() else: self.pump.write(data) def _on_control(self, msg): - if msg != econtext.core._DEAD: + if msg != mitogen.core._DEAD: command, arg = msg.unpickle() LOG.debug('%r._on_control(%r, %s)', self, command, arg) @@ -182,9 +182,9 @@ class Process(object): LOG.warning('%r: unknown command %r', self, command) def _on_start(self, msg, arg): - dest = econtext.core.Context(self.router, msg.src_id) - self.control = econtext.core.Sender(dest, arg[0]) - self.stdin = econtext.core.Sender(dest, arg[1]) + dest = mitogen.core.Context(self.router, msg.src_id) + self.control = mitogen.core.Sender(dest, arg[0]) + self.stdin = mitogen.core.Sender(dest, arg[1]) self.router.broker.start_receive(self.pump) def _on_exit(self, msg, arg): @@ -200,7 +200,7 @@ class Process(object): def _on_pump_disconnect(self): LOG.debug('%r._on_pump_disconnect()', self) - econtext.core.fire(self, 'disconnect') + mitogen.core.fire(self, 'disconnect') self.stdin.close() self.wake_event.set() @@ -215,21 +215,21 @@ class Process(object): pass -def _start_slave(econtext_, src_id, args): +def _start_slave(mitogen_, src_id, args): """ This runs in the target context, it is invoked by _fakessh_main running in the fakessh context immediately after startup. It starts the slave process (the the point where it has a stdin_handle to target but not stdout_chan to write to), and waits for main to. """ - LOG.debug('_start_slave(%r, %r)', econtext_, args) + LOG.debug('_start_slave(%r, %r)', mitogen_, args) proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) - process = Process(econtext_.router, + process = Process(mitogen_.router, proc.stdin.fileno(), proc.stdout.fileno(), proc, @@ -244,7 +244,7 @@ def _start_slave(econtext_, src_id, args): def exit(): - _econtext.broker.shutdown() + _mitogen.broker.shutdown() def die(msg, *args): @@ -276,7 +276,7 @@ def parse_args(): return hostname, allopts, args -def _fakessh_main(econtext_, dest_context_id): +def _fakessh_main(mitogen_, dest_context_id): hostname, opts, args = parse_args() if not hostname: die('Missing hostname') @@ -291,17 +291,17 @@ def _fakessh_main(econtext_, dest_context_id): LOG.debug('opts: %r', opts) LOG.debug('args: %r', args) - dest = econtext.master.Context(econtext_.router, dest_context_id) + dest = mitogen.master.Context(mitogen_.router, dest_context_id) control_handle, stdin_handle = dest.call_with_deadline(None, True, - _start_slave, econtext.context_id, args) + _start_slave, mitogen.context_id, args) LOG.debug('_fakessh_main: received control_handle=%r, stdin_handle=%r', control_handle, stdin_handle) - process = Process(econtext_.router, 1, 0) + process = Process(mitogen_.router, 1, 0) process.start_master( - stdin=econtext.core.Sender(dest, stdin_handle), - control=econtext.core.Sender(dest, control_handle), + stdin=mitogen.core.Sender(dest, stdin_handle), + control=mitogen.core.Sender(dest, control_handle), ) process.wait() process.control.put(('exit', None)) @@ -315,25 +315,25 @@ def run(dest, router, args, deadline=None): """ Run the command specified by the argument vector `args` such that ``PATH`` searches for SSH by the command will cause its attempt to use SSH to - execute a remote program to be redirected to use econtext to execute that + execute a remote program to be redirected to use mitogen to execute that program using the context `dest` instead. - :param econtext.core.Context dest: + :param mitogen.core.Context dest: The destination context to execute the SSH command line in. - :param econtext.core.Router router: + :param mitogen.core.Router router: :param list[str] args: Command line arguments for local program, e.g. ``['rsync', '/tmp', 'remote:/tmp']`` """ context_id = router.context_id_counter.next() - fakessh = econtext.master.Context(router, context_id) + fakessh = mitogen.master.Context(router, context_id) fakessh.name = 'fakessh' sock1, sock2 = socket.socketpair() - econtext.core.set_cloexec(sock1.fileno()) + mitogen.core.set_cloexec(sock1.fileno()) - stream = econtext.core.Stream(router, context_id, fakessh.key) + stream = mitogen.core.Stream(router, context_id, fakessh.key) stream.name = 'fakessh' stream.accept(sock1.fileno(), sock1.fileno()) router.register(fakessh, stream) @@ -341,16 +341,16 @@ def run(dest, router, args, deadline=None): # Held in socket buffer until process is booted. fakessh.call_async(True, _fakessh_main, dest.context_id) - tmp_path = tempfile.mkdtemp(prefix='econtext_fakessh') + tmp_path = tempfile.mkdtemp(prefix='mitogen_fakessh') try: ssh_path = os.path.join(tmp_path, 'ssh') fp = file(ssh_path, 'w') try: fp.write('#!/usr/bin/env python\n') - fp.write(inspect.getsource(econtext.core)) + fp.write(inspect.getsource(mitogen.core)) fp.write('\n') fp.write('ExternalContext().main%r\n' % (( - econtext.context_id, # parent_id + mitogen.context_id, # parent_id context_id, # context_id fakessh.key, # key router.debug, # debug diff --git a/econtext/master.py b/mitogen/master.py similarity index 87% rename from econtext/master.py rename to mitogen/master.py index de8c15d2..237fc680 100644 --- a/econtext/master.py +++ b/mitogen/master.py @@ -25,13 +25,13 @@ import zlib if not hasattr(pkgutil, 'find_loader'): # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has # been kept intentionally 2.3 compatible so we can reuse it. - from econtext.compat import pkgutil + from mitogen.compat import pkgutil -import econtext.core +import mitogen.core -LOG = logging.getLogger('econtext') -IOLOG = logging.getLogger('econtext.io') +LOG = logging.getLogger('mitogen') +IOLOG = logging.getLogger('mitogen.io') RLOG = logging.getLogger('ctx') DOCSTRING_RE = re.compile(r'""".+?"""', re.M | re.S) @@ -39,8 +39,8 @@ COMMENT_RE = re.compile(r'^[ ]*#[^\n]*$', re.M) IOLOG_RE = re.compile(r'^[ ]*IOLOG.debug\(.+?\)$', re.M) PERMITTED_CLASSES = set([ - ('econtext.core', 'CallError'), - ('econtext.core', 'Dead'), + ('mitogen.core', 'CallError'), + ('mitogen.core', 'Dead'), ]) @@ -95,7 +95,7 @@ def read_with_deadline(fd, size, deadline): if rfds: return os.read(fd, size) - raise econtext.core.TimeoutError('read timed out') + raise mitogen.core.TimeoutError('read timed out') def iter_read(fd, deadline): @@ -104,12 +104,12 @@ def iter_read(fd, deadline): bits = [] while True: - s, disconnected = econtext.core.io_op(os.read, fd, 4096) + s, disconnected = mitogen.core.io_op(os.read, fd, 4096) if disconnected: s = '' if not s: - raise econtext.core.StreamError( + raise mitogen.core.StreamError( 'EOF on stream; last 100 bytes received: %r' % (''.join(bits)[-100:],) ) @@ -128,10 +128,10 @@ class LogForwarder(object): def __init__(self, router): self._router = router self._cache = {} - router.add_handler(self._on_forward_log, econtext.core.FORWARD_LOG) + router.add_handler(self._on_forward_log, mitogen.core.FORWARD_LOG) def _on_forward_log(self, msg): - if msg == econtext.core._DEAD: + if msg == mitogen.core._DEAD: return logger = self._cache.get(msg.src_id) @@ -154,7 +154,7 @@ class LogForwarder(object): class ModuleResponder(object): def __init__(self, router): self._router = router - router.add_handler(self._on_get_module, econtext.core.GET_MODULE) + router.add_handler(self._on_get_module, mitogen.core.GET_MODULE) def __repr__(self): return 'ModuleResponder(%r)' % (self._router,) @@ -217,7 +217,7 @@ class ModuleResponder(object): def _on_get_module(self, msg): LOG.debug('%r.get_module(%r)', self, msg) - if msg == econtext.core._DEAD: + if msg == mitogen.core._DEAD: return fullname = msg.data @@ -243,7 +243,7 @@ class ModuleResponder(object): compressed = zlib.compress(source) self._router.route( - econtext.core.Message.pickled( + mitogen.core.Message.pickled( (pkg_present, path, compressed), dst_id=msg.src_id, handle=msg.reply_to, @@ -252,7 +252,7 @@ class ModuleResponder(object): except Exception: LOG.debug('While importing %r', fullname, exc_info=True) self._router.route( - econtext.core.Message.pickled( + mitogen.core.Message.pickled( None, dst_id=msg.src_id, handle=msg.reply_to, @@ -269,14 +269,14 @@ class ModuleForwarder(object): self.router = router self.parent_context = parent_context self.importer = importer - router.add_handler(self._on_get_module, econtext.core.GET_MODULE) + router.add_handler(self._on_get_module, mitogen.core.GET_MODULE) def __repr__(self): return 'ModuleForwarder(%r)' % (self.router,) def _on_get_module(self, msg): LOG.debug('%r._on_get_module(%r)', self, msg) - if msg == econtext.core._DEAD: + if msg == mitogen.core._DEAD: return fullname = msg.data @@ -284,7 +284,7 @@ class ModuleForwarder(object): if cached: LOG.debug('%r._on_get_module(): using cached %r', self, fullname) self.router.route( - econtext.core.Message.pickled( + mitogen.core.Message.pickled( cached, dst_id=msg.src_id, handle=msg.reply_to, @@ -293,9 +293,9 @@ class ModuleForwarder(object): else: LOG.debug('%r._on_get_module(): requesting %r', self, fullname) self.parent_context.send( - econtext.core.Message( + mitogen.core.Message( data=msg.data, - handle=econtext.core.GET_MODULE, + handle=mitogen.core.GET_MODULE, reply_to=self.router.add_handler( lambda m: self._on_got_source(m, msg), persist=False @@ -308,7 +308,7 @@ class ModuleForwarder(object): fullname = original_msg.data self.importer._cache[fullname] = msg.unpickle() self.router.route( - econtext.core.Message( + mitogen.core.Message( data=msg.data, dst_id=original_msg.src_id, handle=original_msg.reply_to, @@ -316,7 +316,7 @@ class ModuleForwarder(object): ) -class Message(econtext.core.Message): +class Message(mitogen.core.Message): """ Message subclass that controls unpickling. """ @@ -324,13 +324,13 @@ class Message(econtext.core.Message): """Return the class implementing `module_name.class_name` or raise `StreamError` if the module is not whitelisted.""" if (module_name, class_name) not in PERMITTED_CLASSES: - raise econtext.core.StreamError( + raise mitogen.core.StreamError( '%r attempted to unpickle %r in module %r', self._context, class_name, module_name) return getattr(sys.modules[module_name], class_name) -class Stream(econtext.core.Stream): +class Stream(mitogen.core.Stream): """ Base for streams capable of starting new slaves. """ @@ -339,7 +339,7 @@ class Stream(econtext.core.Stream): #: The path to the remote Python interpreter. python_path = 'python2.7' - #: True to cause context to write verbose /tmp/econtext..log. + #: True to cause context to write verbose /tmp/mitogen..log. debug = False def construct(self, remote_name=None, python_path=None, debug=False, **kwargs): @@ -359,11 +359,11 @@ class Stream(econtext.core.Stream): """Request the slave gracefully shut itself down.""" LOG.debug('%r closing CALL_FUNCTION channel', self) self.send( - econtext.core.Message.pickled( - econtext.core._DEAD, - src_id=econtext.context_id, + mitogen.core.Message.pickled( + mitogen.core._DEAD, + src_id=mitogen.context_id, dst_id=self.remote_id, - handle=econtext.core.CALL_FUNCTION + handle=mitogen.core.CALL_FUNCTION ) ) @@ -380,7 +380,7 @@ class Stream(econtext.core.Stream): os.dup2(R2,101) for f in R,R2,W,W2: os.close(f) os.environ['ARGV0'] = `[sys.executable]` - os.execv(sys.executable,['econtext:CONTEXT_NAME']) + os.execv(sys.executable,['mitogen:CONTEXT_NAME']) else: os.write(1, 'EC0\n') C = zlib.decompress(sys.stdin.read(input())) @@ -399,9 +399,9 @@ class Stream(econtext.core.Stream): 'exec("%s".decode("base64"))' % (encoded,)] def get_preamble(self): - source = inspect.getsource(econtext.core) + source = inspect.getsource(mitogen.core) source += '\nExternalContext().main%r\n' % (( - econtext.context_id, # parent_id + mitogen.context_id, # parent_id self.remote_id, # context_id self.key, self.debug, @@ -417,8 +417,8 @@ class Stream(econtext.core.Stream): LOG.debug('%r.connect()', self) pid, fd = self.create_child(*self.get_boot_command()) self.name = 'local.%s' % (pid,) - self.receive_side = econtext.core.Side(self, fd) - self.transmit_side = econtext.core.Side(self, os.dup(fd)) + self.receive_side = mitogen.core.Side(self, fd) + self.transmit_side = mitogen.core.Side(self, os.dup(fd)) LOG.debug('%r.connect(): child process stdin/stdout=%r', self, self.receive_side.fd) @@ -434,11 +434,11 @@ class Stream(econtext.core.Stream): self._ec0_received() -class Broker(econtext.core.Broker): +class Broker(mitogen.core.Broker): shutdown_timeout = 5.0 -class Context(econtext.core.Context): +class Context(mitogen.core.Context): via = None def on_disconnect(self, broker): @@ -446,7 +446,7 @@ class Context(econtext.core.Context): Override base behaviour of triggering Broker shutdown on parent stream disconnection. """ - econtext.core.fire(self, 'disconnect') + mitogen.core.fire(self, 'disconnect') def _discard_result(self, msg): data = msg.unpickle() @@ -470,9 +470,9 @@ class Context(econtext.core.Context): call = (with_context, fn.__module__, klass, fn.__name__, args, kwargs) self.send( - econtext.core.Message.pickled( + mitogen.core.Message.pickled( call, - handle=econtext.core.CALL_FUNCTION, + handle=mitogen.core.CALL_FUNCTION, reply_to=self.router.add_handler(self._discard_result), ) ) @@ -481,7 +481,7 @@ class Context(econtext.core.Context): """Invoke `fn([context,] *args, **kwargs)` in the external context. If `with_context` is ``True``, pass its - :py:class:`ExternalContext ` instance as + :py:class:`ExternalContext ` instance as the first parameter. If `deadline` is not ``None``, expire the call after `deadline` @@ -498,15 +498,15 @@ class Context(econtext.core.Context): call = (with_context, fn.__module__, klass, fn.__name__, args, kwargs) response = self.send_await( - econtext.core.Message.pickled( + mitogen.core.Message.pickled( call, - handle=econtext.core.CALL_FUNCTION + handle=mitogen.core.CALL_FUNCTION ), deadline ) decoded = response.unpickle() - if isinstance(decoded, econtext.core.CallError): + if isinstance(decoded, mitogen.core.CallError): raise decoded return decoded @@ -515,13 +515,13 @@ class Context(econtext.core.Context): return self.call_with_deadline(None, False, fn, *args, **kwargs) -def _proxy_connect(econtext, name, context_id, klass, kwargs): - if not isinstance(econtext.router, Router): # TODO - econtext.router.__class__ = Router # TODO +def _proxy_connect(mitogen, name, context_id, klass, kwargs): + if not isinstance(mitogen.router, Router): # TODO + mitogen.router.__class__ = Router # TODO LOG.debug('_proxy_connect(): constructing ModuleForwarder') - ModuleForwarder(econtext.router, econtext.parent, econtext.importer) + ModuleForwarder(mitogen.router, mitogen.parent, mitogen.importer) - context = econtext.router._connect( + context = mitogen.router._connect( context_id, klass, name=name, @@ -530,7 +530,7 @@ def _proxy_connect(econtext, name, context_id, klass, kwargs): return context.name -class Router(econtext.core.Router): +class Router(mitogen.core.Router): context_id_counter = itertools.count(1) debug = False @@ -543,9 +543,9 @@ class Router(econtext.core.Router): def enable_debug(self): """ Cause this context and any descendant child contexts to write debug - logs to /tmp/econtext..log. + logs to /tmp/mitogen..log. """ - econtext.core.enable_debug_logging() + mitogen.core.enable_debug_logging() self.debug = True def __enter__(self): @@ -562,12 +562,12 @@ class Router(econtext.core.Router): return self.connect(Stream, **kwargs) def sudo(self, **kwargs): - import econtext.sudo - return self.connect(econtext.sudo.Stream, **kwargs) + import mitogen.sudo + return self.connect(mitogen.sudo.Stream, **kwargs) def ssh(self, **kwargs): - import econtext.ssh - return self.connect(econtext.ssh.Stream, **kwargs) + import mitogen.ssh + return self.connect(mitogen.ssh.Stream, **kwargs) def _connect(self, context_id, klass, name=None, **kwargs): context = Context(self, context_id) @@ -605,9 +605,9 @@ class Router(econtext.core.Router): while parent is not None: LOG.debug('Adding route to %r for %r via %r', parent, context, child) parent.send( - econtext.core.Message( + mitogen.core.Message( data='%s\x00%s' % (context_id, child.context_id), - handle=econtext.core.ADD_ROUTE, + handle=mitogen.core.ADD_ROUTE, ) ) child = parent diff --git a/econtext/ssh.py b/mitogen/ssh.py similarity index 93% rename from econtext/ssh.py rename to mitogen/ssh.py index beb4ca39..250a13f1 100644 --- a/econtext/ssh.py +++ b/mitogen/ssh.py @@ -4,10 +4,10 @@ Functionality to allow establishing new slave contexts over an SSH connection. import commands -import econtext.master +import mitogen.master -class Stream(econtext.master.Stream): +class Stream(mitogen.master.Stream): python_path = 'python' #: The path to the SSH binary. diff --git a/econtext/sudo.py b/mitogen/sudo.py similarity index 92% rename from econtext/sudo.py rename to mitogen/sudo.py index 45b13635..51c59186 100644 --- a/econtext/sudo.py +++ b/mitogen/sudo.py @@ -5,15 +5,15 @@ import pty import termios import time -import econtext.core -import econtext.master +import mitogen.core +import mitogen.master LOG = logging.getLogger(__name__) PASSWORD_PROMPT = 'password' -class PasswordError(econtext.core.Error): +class PasswordError(mitogen.core.Error): pass @@ -89,7 +89,7 @@ def tty_create_child(*args): return pid, master_fd -class Stream(econtext.master.Stream): +class Stream(mitogen.master.Stream): create_child = staticmethod(tty_create_child) sudo_path = 'sudo' password = None @@ -98,7 +98,7 @@ class Stream(econtext.master.Stream): """ Get the named sudo context, creating it if it does not exist. - :param econtext.core.Broker broker: + :param mitogen.core.Broker broker: The broker that will own the context. :param str username: @@ -115,7 +115,7 @@ class Stream(econtext.master.Stream): :param str password: The password to use when authenticating to sudo. Depending on the sudo configuration, this is either the current account password or the - target account password. :py:class:`econtext.sudo.PasswordError` will + target account password. :py:class:`mitogen.sudo.PasswordError` will be raised if sudo requests a password but none is provided. """ @@ -138,7 +138,7 @@ class Stream(econtext.master.Stream): def _connect_bootstrap(self): password_sent = False - for buf in econtext.master.iter_read(self.receive_side.fd, + for buf in mitogen.master.iter_read(self.receive_side.fd, time.time() + 10.0): LOG.debug('%r: received %r', self, buf) if buf.endswith('EC0\n'): @@ -152,4 +152,4 @@ class Stream(econtext.master.Stream): os.write(self.transmit_side.fd, self.password + '\n') password_sent = True else: - raise econtext.core.StreamError('bootstrap failed') + raise mitogen.core.StreamError('bootstrap failed') diff --git a/econtext/tcp.py b/mitogen/tcp.py similarity index 74% rename from econtext/tcp.py rename to mitogen/tcp.py index b45688a8..b0b7c6d3 100644 --- a/econtext/tcp.py +++ b/mitogen/tcp.py @@ -5,24 +5,24 @@ plain TCP connection. import socket -import econtext.core +import mitogen.core -class Listener(econtext.core.BasicStream): +class Listener(mitogen.core.BasicStream): def __init__(self, broker, address=None, backlog=30): self._broker = broker self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock.bind(address or ('0.0.0.0', 0)) self._sock.listen(backlog) - econtext.core.set_cloexec(self._sock.fileno()) + mitogen.core.set_cloexec(self._sock.fileno()) self.address = self._sock.getsockname() - self.receive_side = econtext.core.Side(self, self._sock.fileno()) + self.receive_side = mitogen.core.Side(self, self._sock.fileno()) broker.start_receive(self) def on_receive(self, broker): sock, addr = self._sock.accept() context = Context(self._broker, name=addr) - stream = econtext.core.Stream(context) + stream = mitogen.core.Stream(context) stream.accept(sock.fileno(), sock.fileno()) @@ -36,7 +36,7 @@ def connect(context): Context.""" LOG.debug('%s.connect()', __name__) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.receive_side = econtext.core.Side(self, sock.fileno()) - self.transmit_side = econtext.core.Side(self, sock.fileno()) + self.receive_side = mitogen.core.Side(self, sock.fileno()) + self.transmit_side = mitogen.core.Side(self, sock.fileno()) sock.connect(self._context.parent_addr) self.enqueue(0, self._context.name) diff --git a/econtext/utils.py b/mitogen/utils.py similarity index 80% rename from econtext/utils.py rename to mitogen/utils.py index ce76897f..51f9a3c7 100644 --- a/econtext/utils.py +++ b/mitogen/utils.py @@ -5,12 +5,12 @@ A random assortment of utility functions useful on masters and slaves. import logging import sys -import econtext -import econtext.core -import econtext.master +import mitogen +import mitogen.core +import mitogen.master -LOG = logging.getLogger('econtext') +LOG = logging.getLogger('mitogen') def disable_site_packages(): @@ -24,7 +24,7 @@ def disable_site_packages(): def log_to_tmp(): import os - log_to_file(path='/tmp/econtext.%s.log' % (os.getpid(),)) + log_to_file(path='/tmp/mitogen.%s.log' % (os.getpid(),)) def log_to_file(path=None, io=True, level=logging.INFO): @@ -33,13 +33,13 @@ def log_to_file(path=None, io=True, level=logging.INFO): log = logging.getLogger('') if path: fp = open(path, 'w', 1) - econtext.core.set_cloexec(fp.fileno()) + mitogen.core.set_cloexec(fp.fileno()) else: fp = sys.stderr log.setLevel(level) if io: - logging.getLogger('econtext.io').setLevel(level) + logging.getLogger('mitogen.io').setLevel(level) fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s' datefmt = '%H:%M:%S' @@ -50,10 +50,10 @@ def log_to_file(path=None, io=True, level=logging.INFO): def run_with_router(func, *args, **kwargs): """Arrange for `func(broker, *args, **kwargs)` to run with a temporary - :py:class:`econtext.master.Router`, ensuring the Router and Broker are + :py:class:`mitogen.master.Router`, ensuring the Router and Broker are correctly shut down during normal or exceptional return.""" - broker = econtext.master.Broker() - router = econtext.master.Router(broker) + broker = mitogen.master.Broker() + router = mitogen.master.Router(broker) try: return func(router, *args, **kwargs) finally: diff --git a/setup.py b/setup.py index 153f39fb..0ce5f345 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,12 @@ from distutils.core import setup setup( - name = 'econtext', - version = '0.0.0-master', - description = "Remote Code Execution Contexts", + name = 'mitogen', + version = '0.0.0', + description = 'Library for writing distributed self-replicating programs. THIS PACKAGE IS INCOMPLETE. IT IS BEING UPLOADED BECAUSE PYPI MAINTAINERS BROKE THE REGISTER COMMAND', author = 'David Wilson', license = 'OpenLDAP BSD', - url = 'http://github.com/dw/econtext/', - py_packages = ['econtext'], + url = 'http://github.com/dw/mitogen/', + py_packages = ['Mitogen'], zip_safe = False )