docs: many more fixes/merges.

docs/api.rst

@@ -238,16 +238,16 @@ Router Class
     .. method:: add_handler (fn, handle=None, persist=True, respondent=None, policy=None)
 
         Invoke `fn(msg)` for each Message sent to `handle` from this context.
-        Unregister after one invocation if `persist` is ``False``. If `handle`
-        is ``None``, a new handle is allocated and returned.
+        Unregister after one invocation if `persist` is :data:`False`. If
+        `handle` is :data:`None`, a new handle is allocated and returned.
 
         :param int handle:
-            If not ``None``, an explicit handle to register, usually one of the
-            ``mitogen.core.*`` constants. If unspecified, a new unused handle
-            will be allocated.
+            If not :data:`None`, an explicit handle to register, usually one of
+            the ``mitogen.core.*`` constants. If unspecified, a new unused
+            handle will be allocated.
 
         :param bool persist:
-            If ``False``, the handler will be unregistered after a single
+            If :data:`False`, the handler will be unregistered after a single
             message has been received.
 
         :param mitogen.core.Context respondent:
@@ -281,7 +281,8 @@ Router Class
             sender indicating refusal occurred.
 
         :return:
-            `handle`, or if `handle` was ``None``, the newly allocated handle.
+            `handle`, or if `handle` was :data:`None`, the newly allocated
+            handle.
 
     .. method:: del_handler (handle)
 
@@ -300,10 +301,10 @@ Router Class
         called from the I/O multiplexer thread.
 
         :param mitogen.core.Stream stream:
-            If not ``None``, a reference to the stream the message arrived on.
-            Used for performing source route verification, to ensure sensitive
-            messages such as ``CALL_FUNCTION`` arrive only from trusted
-            contexts.
+            If not :data:`None`, a reference to the stream the message arrived
+            on. Used for performing source route verification, to ensure
+            sensitive messages such as ``CALL_FUNCTION`` arrive only from
+            trusted contexts.
 
     .. method:: route(msg)
 
@@ -515,8 +516,8 @@ Router Class
             otherwise.
 
         :param mitogen.core.Context via:
-            If not ``None``, arrange for construction to occur via RPCs made to
-            the context `via`, and for :py:data:`ADD_ROUTE
+            If not :data:`None`, arrange for construction to occur via RPCs
+            made to the context `via`, and for :py:data:`ADD_ROUTE
             <mitogen.core.ADD_ROUTE>` messages to be generated as appropriate.
 
             .. code-block:: python
@@ -567,7 +568,7 @@ Router Class
             :data:`None`, which Docker interprets as ``root``.
         :param str image:
             Image tag to use to construct a temporary container. Defaults to
-            ``None``.
+            :data:`None`.
         :param str docker_path:
             Filename or complete path to the Docker binary. ``PATH`` will be
             searched if given as a filename. Defaults to ``docker``.
@@ -596,7 +597,7 @@ Router Class
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 
         :param str container:
-            Existing container to connect to. Defaults to ``None``.
+            Existing container to connect to. Defaults to :data:`None`.
         :param str lxc_attach_path:
             Filename or complete path to the ``lxc-attach`` binary. ``PATH``
             will be searched if given as a filename. Defaults to
@@ -610,7 +611,7 @@ Router Class
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 
         :param str container:
-            Existing container to connect to. Defaults to ``None``.
+            Existing container to connect to. Defaults to :data:`None`.
         :param str lxc_path:
             Filename or complete path to the ``lxc`` binary. ``PATH`` will be
             searched if given as a filename. Defaults to ``lxc``.
@@ -790,7 +791,7 @@ Context Class
         handle which is placed in the message's `reply_to`.
 
         :param bool persist:
-            If ``False``, the handler will be unregistered after a single
+            If :data:`False`, the handler will be unregistered after a single
             message has been received.
 
         :param mitogen.core.Message msg:
@@ -809,7 +810,7 @@ Context Class
             The message.
 
         :param float deadline:
-            If not ``None``, seconds before timing out waiting for a reply.
+            If not :data:`None`, seconds before timing out waiting for a reply.
 
         :raises mitogen.core.TimeoutError:
             No message was received and `deadline` passed.
@@ -931,8 +932,8 @@ Receiver Class
         Router to register the handler on.
 
     :param int handle:
-        If not ``None``, an explicit handle to register, otherwise an unused
-        handle is chosen.
+        If not :data:`None`, an explicit handle to register, otherwise an
+        unused handle is chosen.
 
     :param bool persist:
         If :data:`True`, do not unregister the receiver's handler after the
@@ -940,13 +941,13 @@ Receiver Class
 
     :param mitogen.core.Context respondent:
         Reference to the context this receiver is receiving from. If not
-        ``None``, arranges for the receiver to receive a dead message if
+        :data:`None`, arranges for the receiver to receive a dead message if
         messages can no longer be routed to the context, due to disconnection
         or exit.
 
     .. attribute:: notify = None
 
-        If not ``None``, a reference to a function invoked as
+        If not :data:`None`, a reference to a function invoked as
         `notify(receiver)` when a new message is delivered to this receiver.
         Used by :py:class:`mitogen.select.Select` to implement waiting on
         multiple receivers.
@@ -1000,7 +1001,7 @@ Receiver Class
         Sleep waiting for a message to arrive on this receiver.
 
         :param float timeout:
-            If not ``None``, specifies a timeout in seconds.
+            If not :data:`None`, specifies a timeout in seconds.
 
         :raises mitogen.core.ChannelError:
             The remote end indicated the channel should be closed, or
@@ -1183,10 +1184,10 @@ Select Class
         message may be posted at any moment between :py:meth:`empty` and
         :py:meth:`get`.
 
-        :py:meth:`empty` may return ``False`` even when :py:meth:`get` would
-        block if another thread has drained a receiver added to this select.
-        This can be avoided by only consuming each receiver from a single
-        thread.
+        :py:meth:`empty` may return :data:`False` even when :py:meth:`get`
+        would block if another thread has drained a receiver added to this
+        select. This can be avoided by only consuming each receiver from a
+        single thread.
 
     .. py:method:: __iter__ (self)
 
@@ -1370,8 +1371,8 @@ A random assortment of utility functions useful on masters and children.
     variables. See :ref:`logging-env-vars`.
 
     :param str path:
-        If not ``None``, a filesystem path to write logs to. Otherwise, logs
-        are written to :py:data:`sys.stderr`.
+        If not :data:`None`, a filesystem path to write logs to. Otherwise,
+        logs are written to :py:data:`sys.stderr`.
 
     :param bool io:
         If :data:`True`, include extremely verbose IO logs in the output.

docs/howitworks.rst

@@ -332,7 +332,7 @@ Masters listen on the following handles:
     Receives the name of a module to load `fullname`, locates the source code
     for `fullname`, and routes one or more :py:data:`LOAD_MODULE` messages back
     towards the sender of the :py:data:`GET_MODULE` request. If lookup fails,
-    ``None`` is sent instead.
+    :data:`None` is sent instead.
 
     See :ref:`import-preloading` for a deeper discussion of
     :py:data:`GET_MODULE`/:py:data:`LOAD_MODULE`.
@@ -355,12 +355,13 @@ Children listen on the following handles:
 
     Receives `(pkg_present, path, compressed, related)` tuples, composed of:
 
-    * **pkg_present**: Either ``None`` for a plain ``.py`` module, or a list of
-      canonical names of submodules existing witin this package. For example, a
-      :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would return a
-      list like: `["mitogen.core", "mitogen.fakessh", "mitogen.master", ..]`.
-      This list is used by children to avoid generating useless round-trips due
-      to Python 2.x's :keyword:`import` statement behavior.
+    * **pkg_present**: Either :data:`None` for a plain ``.py`` module, or a
+      list of canonical names of submodules existing witin this package. For
+      example, a :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would
+      return a list like: `["mitogen.core", "mitogen.fakessh",
+      "mitogen.master", ..]`. This list is used by children to avoid generating
+      useless round-trips due to Python 2.x's :keyword:`import` statement
+      behavior.
     * **path**: Original filesystem where the module was found on the master.
     * **compressed**: :py:mod:`zlib`-compressed module source code.
     * **related**: list of canonical module names on which this module appears

docs/internals.rst

@@ -53,24 +53,24 @@ Side Class
 
     .. attribute:: fd
 
-        Integer file descriptor to perform IO on, or ``None`` if
+        Integer file descriptor to perform IO on, or :data:`None` if
         :py:meth:`close` has been called.
 
     .. attribute:: keep_alive
 
-        If ``True``, causes presence of this side in :py:class:`Broker`'s
+        If :data:`True`, causes presence of this side in :py:class:`Broker`'s
         active reader set to defer shutdown until the side is disconnected.
 
     .. method:: fileno
 
-        Return :py:attr:`fd` if it is not ``None``, otherwise raise
+        Return :py:attr:`fd` if it is not :data:`None`, otherwise raise
         :py:class:`StreamError`. This method is implemented so that
         :py:class:`Side` can be used directly by :py:func:`select.select`.
 
     .. method:: close
 
-        Call :py:func:`os.close` on :py:attr:`fd` if it is not ``None``, then
-        set it to ``None``.
+        Call :py:func:`os.close` on :py:attr:`fd` if it is not :data:`None`,
+        then set it to :data:`None`.
 
     .. method:: read (n=CHUNK_SIZE)
 
@@ -97,7 +97,8 @@ Side Class
         in a 0-sized write.
 
         :returns:
-            Number of bytes written, or ``None`` if disconnection was detected.
+            Number of bytes written, or :data:`None` if disconnection was
+            detected.
 
 
 Stream Classes
@@ -305,55 +306,25 @@ mitogen.master
 
 
 Blocking I/O Functions
-----------------------
+======================
 
 These functions exist to support the blocking phase of setting up a new
 context. They will eventually be replaced with asynchronous equivalents.
 
 
-.. currentmodule:: mitogen.master
-
-.. function:: iter_read(fd, deadline=None)
-
-    Return a generator that arranges for up to 4096-byte chunks to be read at a
-    time from the file descriptor `fd` until the generator is destroyed.
-
-    :param fd:
-        File descriptor to read from.
-
-    :param deadline:
-        If not ``None``, an absolute UNIX timestamp after which timeout should
-        occur.
-
-    :raises mitogen.core.TimeoutError:
-        Attempt to read beyond deadline.
-
-    :raises mitogen.core.StreamError:
-        Attempt to read past end of file.
-
-
-.. currentmodule:: mitogen.master
-
-.. function:: write_all (fd, s, deadline=None)
-
-    Arrange for all of bytestring `s` to be written to the file descriptor
-    `fd`.
-
-    :param int fd:
-        File descriptor to write to.
-
-    :param bytes s:
-        Bytestring to write to file descriptor.
+.. currentmodule:: mitogen.parent
+.. autofunction:: discard_until
+.. autofunction:: iter_read
+.. autofunction:: write_all
 
-    :param float deadline:
-        If not ``None``, an absolute UNIX timestamp after which timeout should
-        occur.
 
-    :raises mitogen.core.TimeoutError:
-        Bytestring could not be written entirely before deadline was exceeded.
+Subprocess Creation Functions
+=============================
 
-    :raises mitogen.core.StreamError:
-        File descriptor was disconnected before write could complete.
+.. currentmodule:: mitogen.parent
+.. autofunction:: create_child
+.. autofunction:: hybrid_tty_create_child
+.. autofunction:: tty_create_child
 
 
 Helper Functions
@@ -368,42 +339,11 @@ Helper Functions
 .. autofunction:: io_op
 
 
-
-.. currentmodule:: mitogen.parent
-
-.. autofunction:: create_child
-.. autofunction:: tty_create_child
-.. autofunction:: hybrid_tty_create_child
-
-
 .. currentmodule:: mitogen.master
-
-.. function:: get_child_modules (path)
-
-    Return the suffixes of submodules directly neated beneath of the package
-    directory at `path`.
-
-    :param str path:
-        Path to the module's source code on disk, or some PEP-302-recognized
-        equivalent. Usually this is the module's ``__file__`` attribute, but
-        is specified explicitly to avoid loading the module.
-
-    :return:
-        List of submodule name suffixes.
-
+.. autofunction:: get_child_modules
 
 .. currentmodule:: mitogen.minify
-
-.. autofunction:: minimize_source (source)
-
-    Remove comments and docstrings from Python `source`, preserving line
-    numbers and syntax of empty blocks.
-
-    :param str source:
-        The source to minimize.
-
-    :returns str:
-        The minimized source.
+.. autofunction:: minimize_source
 
 
 Signals

mitogen/core.py

@@ -1013,7 +1013,7 @@ class Stream(BasicStream):
     :py:class:`BasicStream` subclass implementing mitogen's :ref:`stream
     protocol <stream-protocol>`.
     """
-    #: If not ``None``, :py:class:`Router` stamps this into
+    #: If not :data:`None`, :py:class:`Router` stamps this into
     #: :py:attr:`Message.auth_id` of every message received on this stream.
     auth_id = None
 

mitogen/master.py

@@ -84,6 +84,17 @@ def _stdlib_paths():
 
 
 def get_child_modules(path):
+    """Return the suffixes of submodules directly neated beneath of the package
+    directory at `path`.
+
+    :param str path:
+        Path to the module's source code on disk, or some PEP-302-recognized
+        equivalent. Usually this is the module's ``__file__`` attribute, but
+        is specified explicitly to avoid loading the module.
+
+    :return:
+        List of submodule name suffixes.
+    """
     it = pkgutil.iter_modules([os.path.dirname(path)])
     return [to_text(name) for _, name, _ in it]
 
@@ -276,7 +287,7 @@ def is_stdlib_path(path):
 
 
 def is_stdlib_name(modname):
-    """Return ``True`` if `modname` appears to come from the standard
+    """Return :data:`True` if `modname` appears to come from the standard
     library."""
     if imp.is_builtin(modname) != 0:
         return True
@@ -412,8 +423,8 @@ class ModuleFinder(object):
         source code.
 
         :returns:
-            Tuple of `(module path, source text, is package?)`, or ``None`` if
-            the source cannot be found.
+            Tuple of `(module path, source text, is package?)`, or :data:`None`
+            if the source cannot be found.
         """
         tup = self._found_cache.get(fullname)
         if tup:

mitogen/minify.py

@@ -48,10 +48,16 @@ except ImportError:
 
 @lru_cache()
 def minimize_source(source):
-    """Remove most comments and docstrings from Python source code.
+    """Remove comments and docstrings from Python `source`, preserving line
+    numbers and syntax of empty blocks.
+
+    :param str source:
+        The source to minimize.
+
+    :returns str:
+        The minimized source.
     """
-    if not isinstance(source, mitogen.core.UnicodeType):
-        source = source.decode('utf-8')
+    source = mitogen.core.to_text(source)
     tokens = tokenize.generate_tokens(StringIO(source).readline)
     tokens = strip_comments(tokens)
     tokens = strip_docstrings(tokens)

mitogen/parent.py

@@ -297,6 +297,22 @@ def hybrid_tty_create_child(args):
 
 
 def write_all(fd, s, deadline=None):
+    """Arrange for all of bytestring `s` to be written to the file descriptor
+    `fd`.
+
+    :param int fd:
+        File descriptor to write to.
+    :param bytes s:
+        Bytestring to write to file descriptor.
+    :param float deadline:
+        If not :data:`None`, absolute UNIX timestamp after which timeout should
+        occur.
+
+    :raises mitogen.core.TimeoutError:
+        Bytestring could not be written entirely before deadline was exceeded.
+    :raises mitogen.core.StreamError:
+        File descriptor was disconnected before write could complete.
+    """
     timeout = None
     written = 0
     poller = PREFERRED_POLLER()
@@ -325,6 +341,20 @@ def write_all(fd, s, deadline=None):
 
 
 def iter_read(fds, deadline=None):
+    """Return a generator that arranges for up to 4096-byte chunks to be read
+    at a time from the file descriptor `fd` until the generator is destroyed.
+
+    :param int fd:
+        File descriptor to read from.
+    :param float deadline:
+        If not :data:`None`, an absolute UNIX timestamp after which timeout
+        should occur.
+
+    :raises mitogen.core.TimeoutError:
+        Attempt to read beyond deadline.
+    :raises mitogen.core.StreamError:
+        Attempt to read past end of file.
+    """
     poller = PREFERRED_POLLER()
     for fd in fds:
         poller.start_receive(fd)
@@ -359,6 +389,24 @@ def iter_read(fds, deadline=None):
 
 
 def discard_until(fd, s, deadline):
+    """Read chunks from `fd` until one is encountered that ends with `s`. This
+    is used to skip output produced by ``/etc/profile``, ``/etc/motd`` and
+    mandatory SSH banners while waiting for :attr:`Stream.EC0_MARKER` to
+    appear, indicating the first stage is ready to receive the compressed
+    :mod:`mitogen.core` source.
+
+    :param int fd:
+        File descriptor to read from.
+    :param bytes s:
+        Marker string to discard until encountered.
+    :param float deadline:
+        Absolute UNIX timestamp after which timeout should occur.
+
+    :raises mitogen.core.TimeoutError:
+        Attempt to read beyond deadline.
+    :raises mitogen.core.StreamError:
+        Attempt to read past end of file.
+    """
     for buf in iter_read([fd], deadline):
         if IOLOG.level == logging.DEBUG:
             for line in buf.splitlines():
@@ -980,7 +1028,9 @@ class Stream(mitogen.core.Stream):
             self._reap_child()
             raise
 
-    #: For ssh.py, this must be at least max(len('password'), len('debug1:'))
+    #: Sentinel value emitted by the first stage to indicate it is ready to
+    #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have
+    #: length of at least `max(len('password'), len('debug1:'))`
     EC0_MARKER = mitogen.core.b('MITO000\n')
     EC1_MARKER = mitogen.core.b('MITO001\n')
 

mitogen/service.py

@@ -372,8 +372,9 @@ class DeduplicatingInvoker(Invoker):
 
 
 class Service(object):
-    #: Sentinel object to suppress reply generation, since returning ``None``
-    #: will trigger a response message containing the pickled ``None``.
+    #: Sentinel object to suppress reply generation, since returning
+    #: :data:`None` will trigger a response message containing the pickled
+    #: :data:`None`.
     NO_REPLY = object()
 
     invoker_class = Invoker