From 442d88e3d7d9b1be19e52e2fd92d58f9b30aae11 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 14:33:55 +0100 Subject: [PATCH] docs: many more fixes/merges. --- docs/api.rst | 59 +++++++++++++------------- docs/howitworks.rst | 15 +++---- docs/internals.rst | 100 +++++++++----------------------------------- mitogen/core.py | 2 +- mitogen/master.py | 17 ++++++-- mitogen/minify.py | 12 ++++-- mitogen/parent.py | 52 ++++++++++++++++++++++- mitogen/service.py | 5 ++- 8 files changed, 136 insertions(+), 126 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index d605e128..7a4c130c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -238,16 +238,16 @@ Router Class .. method:: add_handler (fn, handle=None, persist=True, respondent=None, policy=None) Invoke `fn(msg)` for each Message sent to `handle` from this context. - Unregister after one invocation if `persist` is ``False``. If `handle` - is ``None``, a new handle is allocated and returned. + Unregister after one invocation if `persist` is :data:`False`. If + `handle` is :data:`None`, a new handle is allocated and returned. :param int handle: - If not ``None``, an explicit handle to register, usually one of the - ``mitogen.core.*`` constants. If unspecified, a new unused handle - will be allocated. + If not :data:`None`, an explicit handle to register, usually one of + the ``mitogen.core.*`` constants. If unspecified, a new unused + handle will be allocated. :param bool persist: - If ``False``, the handler will be unregistered after a single + If :data:`False`, the handler will be unregistered after a single message has been received. :param mitogen.core.Context respondent: @@ -281,7 +281,8 @@ Router Class sender indicating refusal occurred. :return: - `handle`, or if `handle` was ``None``, the newly allocated handle. + `handle`, or if `handle` was :data:`None`, the newly allocated + handle. .. method:: del_handler (handle) @@ -300,10 +301,10 @@ Router Class called from the I/O multiplexer thread. :param mitogen.core.Stream stream: - If not ``None``, a reference to the stream the message arrived on. - Used for performing source route verification, to ensure sensitive - messages such as ``CALL_FUNCTION`` arrive only from trusted - contexts. + If not :data:`None`, a reference to the stream the message arrived + on. Used for performing source route verification, to ensure + sensitive messages such as ``CALL_FUNCTION`` arrive only from + trusted contexts. .. method:: route(msg) @@ -515,8 +516,8 @@ Router Class otherwise. :param mitogen.core.Context via: - If not ``None``, arrange for construction to occur via RPCs made to - the context `via`, and for :py:data:`ADD_ROUTE + If not :data:`None`, arrange for construction to occur via RPCs + made to the context `via`, and for :py:data:`ADD_ROUTE ` messages to be generated as appropriate. .. code-block:: python @@ -567,7 +568,7 @@ Router Class :data:`None`, which Docker interprets as ``root``. :param str image: Image tag to use to construct a temporary container. Defaults to - ``None``. + :data:`None`. :param str docker_path: Filename or complete path to the Docker binary. ``PATH`` will be searched if given as a filename. Defaults to ``docker``. @@ -596,7 +597,7 @@ Router Class Accepts all parameters accepted by :py:meth:`local`, in addition to: :param str container: - Existing container to connect to. Defaults to ``None``. + Existing container to connect to. Defaults to :data:`None`. :param str lxc_attach_path: Filename or complete path to the ``lxc-attach`` binary. ``PATH`` will be searched if given as a filename. Defaults to @@ -610,7 +611,7 @@ Router Class Accepts all parameters accepted by :py:meth:`local`, in addition to: :param str container: - Existing container to connect to. Defaults to ``None``. + Existing container to connect to. Defaults to :data:`None`. :param str lxc_path: Filename or complete path to the ``lxc`` binary. ``PATH`` will be searched if given as a filename. Defaults to ``lxc``. @@ -790,7 +791,7 @@ Context Class handle which is placed in the message's `reply_to`. :param bool persist: - If ``False``, the handler will be unregistered after a single + If :data:`False`, the handler will be unregistered after a single message has been received. :param mitogen.core.Message msg: @@ -809,7 +810,7 @@ Context Class The message. :param float deadline: - If not ``None``, seconds before timing out waiting for a reply. + If not :data:`None`, seconds before timing out waiting for a reply. :raises mitogen.core.TimeoutError: No message was received and `deadline` passed. @@ -931,8 +932,8 @@ Receiver Class Router to register the handler on. :param int handle: - If not ``None``, an explicit handle to register, otherwise an unused - handle is chosen. + If not :data:`None`, an explicit handle to register, otherwise an + unused handle is chosen. :param bool persist: If :data:`True`, do not unregister the receiver's handler after the @@ -940,13 +941,13 @@ Receiver Class :param mitogen.core.Context respondent: Reference to the context this receiver is receiving from. If not - ``None``, arranges for the receiver to receive a dead message if + :data:`None`, arranges for the receiver to receive a dead message if messages can no longer be routed to the context, due to disconnection or exit. .. attribute:: notify = None - If not ``None``, a reference to a function invoked as + If not :data:`None`, a reference to a function invoked as `notify(receiver)` when a new message is delivered to this receiver. Used by :py:class:`mitogen.select.Select` to implement waiting on multiple receivers. @@ -1000,7 +1001,7 @@ Receiver Class Sleep waiting for a message to arrive on this receiver. :param float timeout: - If not ``None``, specifies a timeout in seconds. + If not :data:`None`, specifies a timeout in seconds. :raises mitogen.core.ChannelError: The remote end indicated the channel should be closed, or @@ -1183,10 +1184,10 @@ Select Class message may be posted at any moment between :py:meth:`empty` and :py:meth:`get`. - :py:meth:`empty` may return ``False`` even when :py:meth:`get` would - block if another thread has drained a receiver added to this select. - This can be avoided by only consuming each receiver from a single - thread. + :py:meth:`empty` may return :data:`False` even when :py:meth:`get` + would block if another thread has drained a receiver added to this + select. This can be avoided by only consuming each receiver from a + single thread. .. py:method:: __iter__ (self) @@ -1370,8 +1371,8 @@ A random assortment of utility functions useful on masters and children. variables. See :ref:`logging-env-vars`. :param str path: - If not ``None``, a filesystem path to write logs to. Otherwise, logs - are written to :py:data:`sys.stderr`. + If not :data:`None`, a filesystem path to write logs to. Otherwise, + logs are written to :py:data:`sys.stderr`. :param bool io: If :data:`True`, include extremely verbose IO logs in the output. diff --git a/docs/howitworks.rst b/docs/howitworks.rst index b14ceab7..1d45647f 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -332,7 +332,7 @@ Masters listen on the following handles: Receives the name of a module to load `fullname`, locates the source code for `fullname`, and routes one or more :py:data:`LOAD_MODULE` messages back towards the sender of the :py:data:`GET_MODULE` request. If lookup fails, - ``None`` is sent instead. + :data:`None` is sent instead. See :ref:`import-preloading` for a deeper discussion of :py:data:`GET_MODULE`/:py:data:`LOAD_MODULE`. @@ -355,12 +355,13 @@ Children listen on the following handles: Receives `(pkg_present, path, compressed, related)` tuples, composed of: - * **pkg_present**: Either ``None`` for a plain ``.py`` module, or a list of - canonical names of submodules existing witin this package. For example, a - :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would return a - list like: `["mitogen.core", "mitogen.fakessh", "mitogen.master", ..]`. - This list is used by children to avoid generating useless round-trips due - to Python 2.x's :keyword:`import` statement behavior. + * **pkg_present**: Either :data:`None` for a plain ``.py`` module, or a + list of canonical names of submodules existing witin this package. For + example, a :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would + return a list like: `["mitogen.core", "mitogen.fakessh", + "mitogen.master", ..]`. This list is used by children to avoid generating + useless round-trips due to Python 2.x's :keyword:`import` statement + behavior. * **path**: Original filesystem where the module was found on the master. * **compressed**: :py:mod:`zlib`-compressed module source code. * **related**: list of canonical module names on which this module appears diff --git a/docs/internals.rst b/docs/internals.rst index 7c3809bc..7db4262a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -53,24 +53,24 @@ Side Class .. attribute:: fd - Integer file descriptor to perform IO on, or ``None`` if + Integer file descriptor to perform IO on, or :data:`None` if :py:meth:`close` has been called. .. attribute:: keep_alive - If ``True``, causes presence of this side in :py:class:`Broker`'s + If :data:`True`, causes presence of this side in :py:class:`Broker`'s active reader set to defer shutdown until the side is disconnected. .. method:: fileno - Return :py:attr:`fd` if it is not ``None``, otherwise raise + Return :py:attr:`fd` if it is not :data:`None`, otherwise raise :py:class:`StreamError`. This method is implemented so that :py:class:`Side` can be used directly by :py:func:`select.select`. .. method:: close - Call :py:func:`os.close` on :py:attr:`fd` if it is not ``None``, then - set it to ``None``. + Call :py:func:`os.close` on :py:attr:`fd` if it is not :data:`None`, + then set it to :data:`None`. .. method:: read (n=CHUNK_SIZE) @@ -97,7 +97,8 @@ Side Class in a 0-sized write. :returns: - Number of bytes written, or ``None`` if disconnection was detected. + Number of bytes written, or :data:`None` if disconnection was + detected. Stream Classes @@ -305,55 +306,25 @@ mitogen.master Blocking I/O Functions ----------------------- +====================== These functions exist to support the blocking phase of setting up a new context. They will eventually be replaced with asynchronous equivalents. -.. currentmodule:: mitogen.master - -.. function:: iter_read(fd, deadline=None) - - Return a generator that arranges for up to 4096-byte chunks to be read at a - time from the file descriptor `fd` until the generator is destroyed. - - :param fd: - File descriptor to read from. - - :param deadline: - If not ``None``, an absolute UNIX timestamp after which timeout should - occur. - - :raises mitogen.core.TimeoutError: - Attempt to read beyond deadline. - - :raises mitogen.core.StreamError: - Attempt to read past end of file. - - -.. currentmodule:: mitogen.master - -.. function:: write_all (fd, s, deadline=None) - - Arrange for all of bytestring `s` to be written to the file descriptor - `fd`. - - :param int fd: - File descriptor to write to. - - :param bytes s: - Bytestring to write to file descriptor. +.. currentmodule:: mitogen.parent +.. autofunction:: discard_until +.. autofunction:: iter_read +.. autofunction:: write_all - :param float deadline: - If not ``None``, an absolute UNIX timestamp after which timeout should - occur. - :raises mitogen.core.TimeoutError: - Bytestring could not be written entirely before deadline was exceeded. +Subprocess Creation Functions +============================= - :raises mitogen.core.StreamError: - File descriptor was disconnected before write could complete. +.. currentmodule:: mitogen.parent +.. autofunction:: create_child +.. autofunction:: hybrid_tty_create_child +.. autofunction:: tty_create_child Helper Functions @@ -368,42 +339,11 @@ Helper Functions .. autofunction:: io_op - -.. currentmodule:: mitogen.parent - -.. autofunction:: create_child -.. autofunction:: tty_create_child -.. autofunction:: hybrid_tty_create_child - - .. currentmodule:: mitogen.master - -.. function:: get_child_modules (path) - - Return the suffixes of submodules directly neated beneath of the package - directory at `path`. - - :param str path: - Path to the module's source code on disk, or some PEP-302-recognized - equivalent. Usually this is the module's ``__file__`` attribute, but - is specified explicitly to avoid loading the module. - - :return: - List of submodule name suffixes. - +.. autofunction:: get_child_modules .. currentmodule:: mitogen.minify - -.. autofunction:: minimize_source (source) - - Remove comments and docstrings from Python `source`, preserving line - numbers and syntax of empty blocks. - - :param str source: - The source to minimize. - - :returns str: - The minimized source. +.. autofunction:: minimize_source Signals diff --git a/mitogen/core.py b/mitogen/core.py index 03b892f0..c651b85c 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1013,7 +1013,7 @@ class Stream(BasicStream): :py:class:`BasicStream` subclass implementing mitogen's :ref:`stream protocol `. """ - #: If not ``None``, :py:class:`Router` stamps this into + #: If not :data:`None`, :py:class:`Router` stamps this into #: :py:attr:`Message.auth_id` of every message received on this stream. auth_id = None diff --git a/mitogen/master.py b/mitogen/master.py index d057f7f1..671bee85 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -84,6 +84,17 @@ def _stdlib_paths(): def get_child_modules(path): + """Return the suffixes of submodules directly neated beneath of the package + directory at `path`. + + :param str path: + Path to the module's source code on disk, or some PEP-302-recognized + equivalent. Usually this is the module's ``__file__`` attribute, but + is specified explicitly to avoid loading the module. + + :return: + List of submodule name suffixes. + """ it = pkgutil.iter_modules([os.path.dirname(path)]) return [to_text(name) for _, name, _ in it] @@ -276,7 +287,7 @@ def is_stdlib_path(path): def is_stdlib_name(modname): - """Return ``True`` if `modname` appears to come from the standard + """Return :data:`True` if `modname` appears to come from the standard library.""" if imp.is_builtin(modname) != 0: return True @@ -412,8 +423,8 @@ class ModuleFinder(object): source code. :returns: - Tuple of `(module path, source text, is package?)`, or ``None`` if - the source cannot be found. + Tuple of `(module path, source text, is package?)`, or :data:`None` + if the source cannot be found. """ tup = self._found_cache.get(fullname) if tup: diff --git a/mitogen/minify.py b/mitogen/minify.py index 26ecf62f..a261bf6a 100644 --- a/mitogen/minify.py +++ b/mitogen/minify.py @@ -48,10 +48,16 @@ except ImportError: @lru_cache() def minimize_source(source): - """Remove most comments and docstrings from Python source code. + """Remove comments and docstrings from Python `source`, preserving line + numbers and syntax of empty blocks. + + :param str source: + The source to minimize. + + :returns str: + The minimized source. """ - if not isinstance(source, mitogen.core.UnicodeType): - source = source.decode('utf-8') + source = mitogen.core.to_text(source) tokens = tokenize.generate_tokens(StringIO(source).readline) tokens = strip_comments(tokens) tokens = strip_docstrings(tokens) diff --git a/mitogen/parent.py b/mitogen/parent.py index 5ff6ced8..ef6c69b2 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -297,6 +297,22 @@ def hybrid_tty_create_child(args): def write_all(fd, s, deadline=None): + """Arrange for all of bytestring `s` to be written to the file descriptor + `fd`. + + :param int fd: + File descriptor to write to. + :param bytes s: + Bytestring to write to file descriptor. + :param float deadline: + If not :data:`None`, absolute UNIX timestamp after which timeout should + occur. + + :raises mitogen.core.TimeoutError: + Bytestring could not be written entirely before deadline was exceeded. + :raises mitogen.core.StreamError: + File descriptor was disconnected before write could complete. + """ timeout = None written = 0 poller = PREFERRED_POLLER() @@ -325,6 +341,20 @@ def write_all(fd, s, deadline=None): def iter_read(fds, deadline=None): + """Return a generator that arranges for up to 4096-byte chunks to be read + at a time from the file descriptor `fd` until the generator is destroyed. + + :param int fd: + File descriptor to read from. + :param float deadline: + If not :data:`None`, an absolute UNIX timestamp after which timeout + should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ poller = PREFERRED_POLLER() for fd in fds: poller.start_receive(fd) @@ -359,6 +389,24 @@ def iter_read(fds, deadline=None): def discard_until(fd, s, deadline): + """Read chunks from `fd` until one is encountered that ends with `s`. This + is used to skip output produced by ``/etc/profile``, ``/etc/motd`` and + mandatory SSH banners while waiting for :attr:`Stream.EC0_MARKER` to + appear, indicating the first stage is ready to receive the compressed + :mod:`mitogen.core` source. + + :param int fd: + File descriptor to read from. + :param bytes s: + Marker string to discard until encountered. + :param float deadline: + Absolute UNIX timestamp after which timeout should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ for buf in iter_read([fd], deadline): if IOLOG.level == logging.DEBUG: for line in buf.splitlines(): @@ -980,7 +1028,9 @@ class Stream(mitogen.core.Stream): self._reap_child() raise - #: For ssh.py, this must be at least max(len('password'), len('debug1:')) + #: Sentinel value emitted by the first stage to indicate it is ready to + #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have + #: length of at least `max(len('password'), len('debug1:'))` EC0_MARKER = mitogen.core.b('MITO000\n') EC1_MARKER = mitogen.core.b('MITO001\n') diff --git a/mitogen/service.py b/mitogen/service.py index 923ec04a..f1ccadde 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -372,8 +372,9 @@ class DeduplicatingInvoker(Invoker): class Service(object): - #: Sentinel object to suppress reply generation, since returning ``None`` - #: will trigger a response message containing the pickled ``None``. + #: Sentinel object to suppress reply generation, since returning + #: :data:`None` will trigger a response message containing the pickled + #: :data:`None`. NO_REPLY = object() invoker_class = Invoker