importer: warn on duplicate request, simplify preload logic

* Children should never generate a request for a module that has already
  been sent, however there are a variety of edge cases where, e.g.
  asynchronous calls are made into unloaded modules in a set of
  children, causing those children to request modules (and deps) in a
  different order, which might break deduplication. So add a warning to
  catch when this happens, so we can figure out how to handle it.
  Meanwhile it's only a warning since in the worst case, this just adds
  needless latency.

* Don't bother treating sent packages separately, there doesn't seem to
  be any need for this (after docs are updated to match how preloading
  actually works now).
wip-fakessh-exit-status
David Wilson 7 years ago
parent 2e729e54cc
commit 2848d35aff

@ -464,7 +464,7 @@ class ModuleResponder(object):
path, source, is_pkg = self._finder.get_module_source(fullname)
if source is None:
LOG.error('_build_tuple(%r): could not locate source', fullname)
tup = fullname, None, None, None, None
tup = fullname, None, None, None, ()
self._cache[fullname] = tup
return tup
@ -505,17 +505,16 @@ class ModuleResponder(object):
stream = self._router.stream_by_id(msg.src_id)
fullname = msg.data
if fullname in stream.sent_modules:
LOG.warning('_on_get_module(): dup request for %r from %r',
fullname, stream)
try:
tup = self._build_tuple(fullname)
for name in tup[4] or (): # related
if name == fullname:
# Must be sent last
continue
for name in tup[4]: # related
parent_pkg, _, _ = name.partition('.')
if parent_pkg != fullname and parent_pkg not in stream.sent_packages:
# Parent hasn't been required, so don't load this guy yet.
if parent_pkg != fullname and parent_pkg not in stream.sent_modules:
# Parent hasn't been sent, so don't load submodule yet.
continue
if name in stream.sent_modules:
@ -523,11 +522,7 @@ class ModuleResponder(object):
continue
self._send_load_module(stream, msg, name)
self._send_load_module(stream, msg, fullname)
if tup[1] is not None:
# It's a package, record the fact it was sent.
stream.sent_packages.add(fullname)
except Exception:
LOG.debug('While importing %r', fullname, exc_info=True)

@ -267,7 +267,6 @@ class Stream(mitogen.core.Stream):
def __init__(self, *args, **kwargs):
super(Stream, self).__init__(*args, **kwargs)
self.sent_modules = set(['mitogen', 'mitogen.core'])
self.sent_packages = set(['mitogen'])
def construct(self, remote_name=None, python_path=None, debug=False,
profiling=False, **kwargs):

Loading…
Cancel
Save