importer: warn on duplicate request, simplify preload logic

* Children should never generate a request for a module that has already
  been sent, however there are a variety of edge cases where, e.g.
  asynchronous calls are made into unloaded modules in a set of
  children, causing those children to request modules (and deps) in a
  different order, which might break deduplication. So add a warning to
  catch when this happens, so we can figure out how to handle it.
  Meanwhile it's only a warning since in the worst case, this just adds
  needless latency.

* Don't bother treating sent packages separately, there doesn't seem to
  be any need for this (after docs are updated to match how preloading
  actually works now).

mitogen/master.py

@@ -464,7 +464,7 @@ class ModuleResponder(object):
         path, source, is_pkg = self._finder.get_module_source(fullname)
         if source is None:
             LOG.error('_build_tuple(%r): could not locate source', fullname)
-            tup = fullname, None, None, None, None
+            tup = fullname, None, None, None, ()
             self._cache[fullname] = tup
             return tup
 
@@ -505,17 +505,16 @@ class ModuleResponder(object):
 
         stream = self._router.stream_by_id(msg.src_id)
         fullname = msg.data
+        if fullname in stream.sent_modules:
+            LOG.warning('_on_get_module(): dup request for %r from %r',
+                       fullname, stream)
 
         try:
             tup = self._build_tuple(fullname)
-            for name in tup[4] or ():  # related
+            for name in tup[4]:  # related
-                if name == fullname:
-                    # Must be sent last
-                    continue
-
                 parent_pkg, _, _ = name.partition('.')
-                if parent_pkg != fullname and parent_pkg not in stream.sent_packages:
+                if parent_pkg != fullname and parent_pkg not in stream.sent_modules:
-                    # Parent hasn't been required, so don't load this guy yet.
+                    # Parent hasn't been sent, so don't load submodule yet.
                     continue
 
                 if name in stream.sent_modules:
@@ -523,11 +522,7 @@ class ModuleResponder(object):
                     continue
 
                 self._send_load_module(stream, msg, name)
-
             self._send_load_module(stream, msg, fullname)
-            if tup[1] is not None:
-                # It's a package, record the fact it was sent.
-                stream.sent_packages.add(fullname)
 
         except Exception:
             LOG.debug('While importing %r', fullname, exc_info=True)

mitogen/parent.py

@@ -267,7 +267,6 @@ class Stream(mitogen.core.Stream):
     def __init__(self, *args, **kwargs):
         super(Stream, self).__init__(*args, **kwargs)
         self.sent_modules = set(['mitogen', 'mitogen.core'])
-        self.sent_packages = set(['mitogen'])
 
     def construct(self, remote_name=None, python_path=None, debug=False,
                   profiling=False, **kwargs):