From a561fb79e5bdcc60b79fea48e79ec5f437c75dda Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 14:15:37 +0100 Subject: [PATCH] docs: merge more docs back into mitogen/core.py. --- docs/api.rst | 32 ++++------------- docs/conf.py | 2 +- docs/internals.rst | 47 +++++++++--------------- docs/signals.rst | 12 ++----- mitogen/core.py | 89 ++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 109 insertions(+), 73 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index ed9509fc..d605e128 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1411,29 +1411,9 @@ Exceptions .. currentmodule:: mitogen.core -.. class:: Error (fmt, \*args) - - Base for all exceptions raised by Mitogen. - -.. class:: CallError (e) - - Raised when :py:meth:`Context.call() ` fails. - A copy of the traceback from the external context is appended to the - exception message. - -.. class:: ChannelError (fmt, \*args) - - Raised when a channel dies or has been closed. - -.. class:: LatchError (fmt, \*args) - - Raised when an attempt is made to use a :py:class:`mitogen.core.Latch` that - has been marked closed. - -.. class:: StreamError (fmt, \*args) - - Raised when a stream cannot be established. - -.. class:: TimeoutError (fmt, \*args) - - Raised when a timeout occurs on a stream. +.. autoclass:: Error +.. autoclass:: CallError +.. autoclass:: ChannelError +.. autoclass:: LatchError +.. autoclass:: StreamError +.. autoclass:: TimeoutError diff --git a/docs/conf.py b/docs/conf.py index 57adf597..abb6e97e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,7 @@ html_theme_options = { 'head_font_family': "Georgia, serif", } htmlhelp_basename = 'mitogendoc' -intersphinx_mapping = {'python': ('https://docs.python.org/2', None)} +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} language = None master_doc = 'toc' project = u'Mitogen' diff --git a/docs/internals.rst b/docs/internals.rst index 3d4d4130..7c3809bc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -8,17 +8,20 @@ Internal API Reference signals -mitogen.core -============ - - Latch Class ------------ +=========== .. currentmodule:: mitogen.core +.. autoclass:: Latch + :members: -.. autoclass:: Latch () +PidfulStreamHandler Class +========================= + +.. currentmodule:: mitogen.core +.. autoclass:: PidfulStreamHandler + :members: Side Class @@ -354,40 +357,22 @@ context. They will eventually be replaced with asynchronous equivalents. Helper Functions ----------------- +================ .. currentmodule:: mitogen.core +.. autofunction:: to_text +.. autofunction:: has_parent_authority +.. autofunction:: set_cloexec +.. autofunction:: set_nonblock +.. autofunction:: set_block +.. autofunction:: io_op -.. function:: io_op (func, \*args) - - Wrap a function that may raise :py:class:`OSError`, trapping common error - codes relating to disconnection events in various subsystems: - - * When performing IO against a TTY, disconnection of the remote end is - signalled by :py:data:`errno.EIO`. - * When performing IO against a socket, disconnection of the remote end is - signalled by :py:data:`errno.ECONNRESET`. - * When performing IO against a pipe, disconnection of the remote end is - signalled by :py:data:`errno.EPIPE`. - - :returns: - Tuple of `(return_value, disconnected)`, where `return_value` is the - return value of `func(\*args)`, and `disconnected` is ``True`` if - disconnection was detected, otherwise ``False``. .. currentmodule:: mitogen.parent .. autofunction:: create_child - - -.. currentmodule:: mitogen.parent - .. autofunction:: tty_create_child - - -.. currentmodule:: mitogen.parent - .. autofunction:: hybrid_tty_create_child diff --git a/docs/signals.rst b/docs/signals.rst index 1c41353a..19533bb1 100644 --- a/docs/signals.rst +++ b/docs/signals.rst @@ -19,16 +19,10 @@ functions registered to receive it will be called back. Functions --------- -.. function:: mitogen.core.listen (obj, name, func) - - Arrange for `func(\*args, \*\*kwargs)` to be invoked when the named signal - is fired by `obj`. - -.. function:: mitogen.core.fire (obj, name, \*args, \*\*kwargs) - - Arrange for `func(\*args, \*\*kwargs)` to be invoked for every function - registered for the named signal on `obj`. +.. currentmodule:: mitogen.core +.. autofunction:: listen +.. autofunction:: fire List diff --git a/mitogen/core.py b/mitogen/core.py index 261f2621..03b892f0 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -131,6 +131,13 @@ else: class Error(Exception): + """Base for all exceptions raised by Mitogen. + + :param str fmt: + Exception text, or format string if `args` is non-empty. + :param tuple args: + Format string arguments. + """ def __init__(self, fmt=None, *args): if args: fmt %= args @@ -140,10 +147,13 @@ class Error(Exception): class LatchError(Error): - pass + """Raised when an attempt is made to use a :py:class:`mitogen.core.Latch` + that has been marked closed.""" class Blob(BytesType): + """A serializable bytes subclass whose content is summarized in repr() + output, making it suitable for logging binary data.""" def __repr__(self): return '[blob: %d bytes]' % len(self) @@ -152,6 +162,8 @@ class Blob(BytesType): class Secret(UnicodeType): + """A serializable unicode subclass whose content is masked in repr() + output, making it suitable for logging passwords.""" def __repr__(self): return '[secret]' @@ -165,6 +177,10 @@ class Secret(UnicodeType): class Kwargs(dict): + """A serializable dict subclass that indicates the contained keys should be + be coerced to Unicode on Python 3 as required. Python 2 produces keyword + argument dicts whose keys are bytestrings, requiring a helper to ensure + compatibility with Python 3.""" if PY3: def __init__(self, dct): for k, v in dct.items(): @@ -181,6 +197,10 @@ class Kwargs(dict): class CallError(Error): + """Serializable :class:`Error` subclass raised when + :py:meth:`Context.call() ` fails. A copy of + the traceback from the external context is appended to the exception + message.""" def __init__(self, fmt=None, *args): if not isinstance(fmt, BaseException): Error.__init__(self, fmt, *args) @@ -207,37 +227,52 @@ def _unpickle_call_error(s): class ChannelError(Error): + """Raised when a channel dies or has been closed.""" remote_msg = 'Channel closed by remote end.' local_msg = 'Channel closed by local end.' class StreamError(Error): - pass + """Raised when a stream cannot be established.""" class TimeoutError(Error): - pass + """Raised when a timeout occurs on a stream.""" def to_text(o): - if isinstance(o, UnicodeType): - return UnicodeType(o) + """Coerce `o` to Unicode by decoding it from UTF-8 if it is an instance of + :class:`bytes`, otherwise pass it to the :class:`str` constructor. The + returned object is always a plain :class:`str`, any subclass is removed.""" if isinstance(o, BytesType): return o.decode('utf-8') return UnicodeType(o) def has_parent_authority(msg, _stream=None): + """Policy function for use with :class:`Receiver` and + :meth:`Router.add_handler` that requires incoming messages to originate + from a parent context, or on a :class:`Stream` whose :attr:`auth_id + ` has been set to that of a parent context or the current + context.""" return (msg.auth_id == mitogen.context_id or msg.auth_id in mitogen.parent_ids) def listen(obj, name, func): + """ + Arrange for `func(*args, **kwargs)` to be invoked when the named signal is + fired by `obj`. + """ signals = vars(obj).setdefault('_signals', {}) signals.setdefault(name, []).append(func) def fire(obj, name, *args, **kwargs): + """ + Arrange for `func(*args, **kwargs)` to be invoked for every function + registered for the named signal on `obj`. + """ signals = vars(obj).get('_signals', {}) return [func(*args, **kwargs) for func in signals.get(name, ())] @@ -253,7 +288,8 @@ def takes_router(func): def is_blacklisted_import(importer, fullname): - """Return ``True`` if `fullname` is part of a blacklisted package, or if + """ + Return :data:`True` if `fullname` is part of a blacklisted package, or if any packages have been whitelisted and `fullname` is not part of one. NB: @@ -266,22 +302,51 @@ def is_blacklisted_import(importer, fullname): def set_cloexec(fd): + """Set the file descriptor `fd` to automatically close on + :func:`os.execve`. This has no effect on file descriptors inherited across + :func:`os.fork`, they must be explicitly closed through some other means, + such as :func:`mitogen.fork.on_fork`.""" flags = fcntl.fcntl(fd, fcntl.F_GETFD) assert fd > 2 fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) def set_nonblock(fd): + """Set the file descriptor `fd` to non-blocking mode. For most underlying + file types, this causes :func:`os.read` or :func:`os.write` to raise + :class:`OSError` with :data:`errno.EAGAIN` rather than block the thread + when the underlying kernel buffer is exhausted.""" flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) def set_block(fd): + """Inverse of :func:`set_nonblock`, i.e. cause `fd` to block the thread + when the underlying kernel buffer is exhausted.""" flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) def io_op(func, *args): + """Wrap `func(*args)` that may raise :class:`select.error`, + :class:`IOError`, or :class:`OSError`, trapping UNIX error codes relating + to disconnection and retry events in various subsystems: + + * When a signal is delivered to the process on Python 2, system call retry + is signalled through :data:`errno.EINTR`. The invocation is automatically + restarted. + * When performing IO against a TTY, disconnection of the remote end is + signalled by :data:`errno.EIO`. + * When performing IO against a socket, disconnection of the remote end is + signalled by :data:`errno.ECONNRESET`. + * When performing IO against a pipe, disconnection of the remote end is + signalled by :data:`errno.EPIPE`. + + :returns: + Tuple of `(return_value, disconnected)`, where `return_value` is the + return value of `func(\*args)`, and `disconnected` is :data:`True` if + disconnection was detected, otherwise :data:`False`. + """ while True: try: return func(*args), False @@ -296,7 +361,19 @@ def io_op(func, *args): class PidfulStreamHandler(logging.StreamHandler): + """A :class:`logging.StreamHandler` subclass used when + :meth:`Router.enable_debug() ` has been + called, or the `debug` parameter was specified during context construction. + Verifies the process ID has not changed on each call to :meth:`emit`, + reopening the associated log file when a change is detected. + + This ensures logging to the per-process output files happens correctly even + when uncooperative third party components call :func:`os.fork`. + """ + #: PID that last opened the log file. open_pid = None + + #: Output path template. template = '/tmp/mitogen.%s.%s.log' def _reopen(self):