From c607831561d6057f6000d903fd5d6d1cb147cea5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 14 Aug 2016 11:38:07 +0100 Subject: [PATCH] SlaveModuleImporter->Importer, enable intersphinx, finish howitworks.rst. --- docs/conf.py | 3 ++- docs/howitworks.rst | 53 ++++++++++++++++++++++++++++++++++++--------- docs/index.rst | 2 +- econtext/core.py | 8 +++---- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8da56957..37541150 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,13 +4,14 @@ import os author = u'David Wilson' copyright = u'2016, David Wilson' exclude_patterns = ['_build'] -extensions = ['sphinx.ext.autodoc'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] html_show_sourcelink = False html_show_sphinx = False html_sidebars = {'**': ['globaltoc.html']} html_static_path = ['_static'] html_theme = 'alabaster' htmlhelp_basename = 'econtextdoc' +intersphinx_mapping = {'python': ('https://docs.python.org/2', None)} language = None master_doc = 'index' project = u'econtext' diff --git a/docs/howitworks.rst b/docs/howitworks.rst index e53866f4..de1d6646 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -133,10 +133,6 @@ is necessary to allow master programs to be written as a self-contained Python script. -Setup The Broker And Master Context -################################### - - Reaping The First Stage ####################### @@ -164,7 +160,7 @@ they reach the master. The Module Importer ################### -An instance of :py:class:`econtext.core.SlaveModuleImporter` is installed in +An instance of :py:class:`econtext.core.Importer` is installed in `sys.meta_path`, where Python's ``import`` statement will execute it before attempting to find a module locally. @@ -293,23 +289,60 @@ Twisted that we would like to use. Differences Between Master And Slave Brokers ############################################ -* Self destruct. +The main difference between :py:class:`econtext.core.Broker` and +:py:class:`econtext.master.Broker` is that when the stream connection to the +parent is lost in a slave, the broker will trigger its own shutdown. The Module Importer ------------------- +:py:class:`econtext.core.Importer` is still a work in progress, as there +are a variety of approaches to implementing it, and the present implementation +is not pefectly efficient in every case. + +It operates by intercepting ``import`` statements via `sys.meta_path`, asking +Python if it can satisfy the import by itself, and if not, indicating to Python +that it is capable of loading the module. + +In :py:meth:`load_module() ` an RPC is +started to the master, requesting the module source code. Once the source is +fetched, the method builds a new module object using the best practice +documented in PEP-302. + + Minimizing Roundtrips ##################### +In Python 2.x where relative imports are the default, a large number of import +requests will be made for modules that do not exist. For example: -Child Package Enumeration -######################### +.. code-block:: python + # mypkg/__init__.py -Negative Cache Hits -################### + import sys + import os + +In Python 2.x, Python will first try to load ``mypkg.sys`` and ``mypkg.os``, +which do not exist, before falling back on :py:mod:`sys` and :py:mod:`os`. + +These negative imports present a challenge, as they introduce a large number of +pointless network roundtrips. Therefore in addition to the zlib-compressed +source, for packages the master sends along a list of child modules known to +exist. + +Before indicating it can satisfy an import request, +:py:class:`econtext.core.Importer` first checks to see if the module belongs to +a package it has previously imported, and if so, ignores the request if the +module does not appear in the enumeration of child modules belonging to the +package. + + +Child Module Enumeration +######################## +Package children are enumerated using the :py:mod:`pkgutil` module. Use Of Threads diff --git a/docs/index.rst b/docs/index.rst index 738a3b58..b39348cd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -114,7 +114,7 @@ location. .. code:: - 18:15:29 D econtext.ctx.k3: econtext: SlaveModuleImporter.find_module('econtext.zlib') + 18:15:29 D econtext.ctx.k3: econtext: Importer.find_module('econtext.zlib') 18:15:29 D econtext.ctx.k3: econtext: _dispatch_calls((1002L, False, 'posix', None, 'system', ('ls -l /proc/self/fd',), {})) diff --git a/econtext/core.py b/econtext/core.py index 624c275e..963db99e 100644 --- a/econtext/core.py +++ b/econtext/core.py @@ -138,7 +138,7 @@ class Channel(object): return 'Channel(%r, %r)' % (self._context, self._handle) -class SlaveModuleImporter(object): +class Importer(object): """ Import protocol implementation that fetches modules from the parent process. @@ -152,7 +152,7 @@ class SlaveModuleImporter(object): self._ignore = [] def find_module(self, fullname, path=None): - LOG.debug('SlaveModuleImporter.find_module(%r)', fullname) + LOG.debug('Importer.find_module(%r)', fullname) if fullname in self._ignore: return None @@ -174,7 +174,7 @@ class SlaveModuleImporter(object): self._lock.release() def load_module(self, fullname): - LOG.debug('SlaveModuleImporter.load_module(%r)', fullname) + LOG.debug('Importer.load_module(%r)', fullname) ret = self._context.enqueue_await_reply(GET_MODULE, None, (fullname,)) if ret is None: raise ImportError('Master does not have %r' % (fullname,)) @@ -707,7 +707,7 @@ class ExternalContext(object): LOG.debug('Connected to %s', self.context) def _setup_importer(self): - self.importer = SlaveModuleImporter(self.context) + self.importer = Importer(self.context) sys.meta_path.append(self.importer) def _setup_stdio(self):