From feb16543057222b6161daac3e381fa18238fb9fb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 4 Aug 2019 18:25:28 +0100 Subject: [PATCH] docs: many more internals.rst tidyups --- docs/internals.rst | 103 ++++++++++++++++++++++++++++++--------------- mitogen/core.py | 20 ++++++--- mitogen/master.py | 34 ++++++++++----- mitogen/parent.py | 35 ++++++++++++++- 4 files changed, 138 insertions(+), 54 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index fc6206e0..c9021d31 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2,6 +2,11 @@ Internal API Reference ********************** +.. note:: + + Internal APIs are subject to rapid change even across minor releases. This + page exists to help users modify and extend the library. + .. toctree:: :hidden: @@ -40,16 +45,22 @@ Latch :members: -PidfulStreamHandler -=================== +Logging +======= + +See also :class:`mitogen.core.IoLoggerProtocol`. + +.. currentmodule:: mitogen.master +.. autoclass:: LogForwarder + :members: .. currentmodule:: mitogen.core .. autoclass:: PidfulStreamHandler :members: -Stream & Side -============= +Stream, Side & Protocol +======================= .. currentmodule:: mitogen.core .. autoclass:: Stream @@ -59,10 +70,6 @@ Stream & Side .. autoclass:: Side :members: - -Protocol -======== - .. currentmodule:: mitogen.core .. autoclass:: Protocol :members: @@ -120,72 +127,100 @@ Connection / Options :members: -Importer -======== +Import Mechanism +================ .. currentmodule:: mitogen.core .. autoclass:: Importer :members: - -ModuleResponder -=============== - .. currentmodule:: mitogen.master .. autoclass:: ModuleResponder :members: +.. currentmodule:: mitogen.parent +.. autoclass:: ModuleForwarder + :members: + -RouteMonitor -============ +Module Finders +============== -.. currentmodule:: mitogen.parent -.. autoclass:: RouteMonitor +.. currentmodule:: mitogen.master +.. autoclass:: ModuleFinder :members: +.. currentmodule:: mitogen.master +.. autoclass:: FinderMethod + :members: -TimerList -========= +.. currentmodule:: mitogen.master +.. autoclass:: DefectivePython3xMainMethod + :members: -.. currentmodule:: mitogen.parent -.. autoclass:: TimerList +.. currentmodule:: mitogen.master +.. autoclass:: PkgutilMethod + :members: + +.. currentmodule:: mitogen.master +.. autoclass:: SysModulesMethod :members: +.. currentmodule:: mitogen.master +.. autoclass:: ParentEnumerationMethod + :members: -Timer -===== + +Routing Management +================== .. currentmodule:: mitogen.parent -.. autoclass:: Timer +.. autoclass:: RouteMonitor :members: -Forwarder -========= +Timer Management +================ .. currentmodule:: mitogen.parent -.. autoclass:: ModuleForwarder +.. autoclass:: TimerList + :members: + +.. currentmodule:: mitogen.parent +.. autoclass:: Timer :members: -ExternalContext -=============== +Child Implementation +==================== .. currentmodule:: mitogen.core .. autoclass:: ExternalContext :members: +.. currentmodule:: mitogen.core +.. autoclass:: Dispatcher + :members: -Process -======= + +Process Management +================== + +.. currentmodule:: mitogen.parent +.. autoclass:: Process + :members: .. currentmodule:: mitogen.parent +.. autoclass:: PopenProcess + :members: + +.. currentmodule:: mitogen.fork .. autoclass:: Process :members: -Helpers -======= +Helper Functions +================ Subprocess Functions diff --git a/mitogen/core.py b/mitogen/core.py index c9b1f9df..cf0b97fd 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -3440,30 +3440,36 @@ class ExternalContext(object): """ External context implementation. + This class contains the main program implementation for new children. It is + responsible for setting up everything about the process environment, import + hooks, standard IO redirection, logging, configuring a :class:`Router` and + :class:`Broker`, and finally arranging for :class:`Dispatcher` to take over + the main thread after initialization is complete. + .. attribute:: broker + The :class:`mitogen.core.Broker` instance. .. attribute:: context + The :class:`mitogen.core.Context` instance. .. attribute:: channel + The :class:`mitogen.core.Channel` over which :data:`CALL_FUNCTION` requests are received. - .. attribute:: stdout_log - The :class:`mitogen.core.IoLogger` connected to ``stdout``. - .. attribute:: importer + The :class:`mitogen.core.Importer` instance. .. attribute:: stdout_log - The :class:`IoLogger` connected to ``stdout``. + + The :class:`IoLogger` connected to :data:`sys.stdout`. .. attribute:: stderr_log - The :class:`IoLogger` connected to ``stderr``. - .. method:: _dispatch_calls - Implementation for the main thread in every child context. + The :class:`IoLogger` connected to :data:`sys.stderr`. """ detached = False diff --git a/mitogen/master.py b/mitogen/master.py index cb4452a1..69a6a7db 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -430,8 +430,8 @@ class FinderMethod(object): def find(self, fullname): """ - Accept a canonical module name and return `(path, source, is_pkg)` - tuples, where: + Accept a canonical module name as would be found in :data:`sys.modules` + and return a `(path, source, is_pkg)` tuple, where: * `path`: Unicode string containing path to source file. * `source`: Bytestring containing source file's content. @@ -447,10 +447,13 @@ class DefectivePython3xMainMethod(FinderMethod): """ Recent versions of Python 3.x introduced an incomplete notion of importer specs, and in doing so created permanent asymmetry in the - :mod:`pkgutil` interface handling for the `__main__` module. Therefore - we must handle `__main__` specially. + :mod:`pkgutil` interface handling for the :mod:`__main__` module. Therefore + we must handle :mod:`__main__` specially. """ def find(self, fullname): + """ + Find :mod:`__main__` using its :data:`__file__` attribute. + """ if fullname != '__main__': return None @@ -477,6 +480,9 @@ class PkgutilMethod(FinderMethod): be the only required implementation of get_module(). """ def find(self, fullname): + """ + Find `fullname` using :func:`pkgutil.find_loader`. + """ try: # Pre-'import spec' this returned None, in Python3.6 it raises # ImportError. @@ -522,10 +528,13 @@ class PkgutilMethod(FinderMethod): class SysModulesMethod(FinderMethod): """ - Attempt to fetch source code via sys.modules. This is specifically to - support __main__, but it may catch a few more cases. + Attempt to fetch source code via :data:`sys.modules`. This was originally + specifically to support :mod:`__main__`, but it may catch a few more cases. """ def find(self, fullname): + """ + Find `fullname` using its :data:`__file__` attribute. + """ module = sys.modules.get(fullname) LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) if getattr(module, '__name__', None) != fullname: @@ -566,14 +575,17 @@ class ParentEnumerationMethod(FinderMethod): """ Attempt to fetch source code by examining the module's (hopefully less insane) parent package. Required for older versions of - ansible.compat.six and plumbum.colors, and Ansible 2.8 - ansible.module_utils.distro. + :mod:`ansible.compat.six`, :mod:`plumbum.colors`, and Ansible 2.8 + :mod:`ansible.module_utils.distro`. - For cases like module_utils.distro, this must handle cases where a package - transmuted itself into a totally unrelated module during import and vice - versa. + For cases like :mod:`ansible.module_utils.distro`, this must handle cases + where a package transmuted itself into a totally unrelated module during + import and vice versa. """ def find(self, fullname): + """ + See implementation for a description of how this works. + """ if fullname not in sys.modules: # Don't attempt this unless a module really exists in sys.modules, # else we could return junk. diff --git a/mitogen/parent.py b/mitogen/parent.py index 22e40610..bba29e7e 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -2049,8 +2049,8 @@ class RouteMonitor(object): def notice_stream(self, stream): """ When this parent is responsible for a new directly connected child - stream, we're also responsible for broadcasting DEL_ROUTE upstream - if/when that child disconnects. + stream, we're also responsible for broadcasting + :data:`mitogen.core.DEL_ROUTE` upstream when that child disconnects. """ self._routes_by_stream[stream] = set([stream.protocol.remote_id]) self._propagate_up(mitogen.core.ADD_ROUTE, stream.protocol.remote_id, @@ -2357,6 +2357,22 @@ class Router(mitogen.core.Router): class Process(object): + """ + Process objects contain asynchronous logic for reaping children, and + keeping track of their stdio descriptors. + + This base class is extended by :class:`PopenProcess` and + :class:`mitogen.fork.Process`. + + :param int pid: + The process ID. + :param file stdin: + File object attached to standard input. + :param file stdout: + File object attached to standard output. + :param file stderr: + File object attached to standard error, or :data:`None`. + """ _delays = [0.05, 0.15, 0.3, 1.0, 5.0, 10.0] name = None @@ -2376,6 +2392,14 @@ class Process(object): ) def poll(self): + """ + Fetch the child process exit status, or :data:`None` if it is still + running. This should be overridden by subclasses. + + :returns: + Exit status in the style of the :attr:`subprocess.Popen.returncode` + attribute, i.e. with signals represented by a negative integer. + """ raise NotImplementedError() def _signal_child(self, signum): @@ -2430,8 +2454,15 @@ class Process(object): class PopenProcess(Process): + """ + :class:`Process` subclass wrapping a :class:`subprocess.Popen` object. + + :param subprocess.Popen proc: + The subprocess. + """ def __init__(self, proc, stdin, stdout, stderr=None): super(PopenProcess, self).__init__(proc.pid, stdin, stdout, stderr) + #: The subprocess. self.proc = proc def poll(self):