issue #536: rework how 2.3-compatible simplejson is served

Regardless of the version of simplejson loaded in the master, load up
the ModuleResponder cache with our 2.4-compatible version.

To cope with simplejson being loaded due to modules like ec2_group that
try to import it before importing 'json', also update target.py to
remove it from the whitelist if a local 'json' module import succeeds.

ansible_mitogen/process.py

@@ -236,6 +236,41 @@ class MuxProcess(object):
         if secs:
             mitogen.debug.dump_to_logger(secs=secs)
 
+    def _setup_simplejson(self, responder):
+        """
+        We support serving simplejson for Python 2.4 targets on Ansible 2.3, at
+        least so the package's own CI Docker scripts can run without external
+        help, however newer versions of simplejson no longer support Python
+        2.4. Therefore override any installed/loaded version with a
+        2.4-compatible version we ship in the compat/ directory.
+        """
+        responder.whitelist_prefix('simplejson')
+
+        # issue #536: must be at end of sys.path, in case existing newer
+        # version is already loaded.
+        compat_path = os.path.join(os.path.dirname(__file__), 'compat')
+        sys.path.append(compat_path)
+
+        for fullname, is_pkg, suffix in (
+            (u'simplejson', True, '__init__.py'),
+            (u'simplejson.decoder', False, 'decoder.py'),
+            (u'simplejson.encoder', False, 'encoder.py'),
+            (u'simplejson.scanner', False, 'scanner.py'),
+        ):
+            path = os.path.join(compat_path, 'simplejson', suffix)
+            fp = open(path, 'rb')
+            try:
+                source = fp.read()
+            finally:
+                fp.close()
+
+            responder.add_source_override(
+                fullname=fullname,
+                path=path,
+                source=source,
+                is_pkg=is_pkg,
+            )
+
     def _setup_responder(self, responder):
         """
         Configure :class:`mitogen.master.ModuleResponder` to only permit
@@ -243,9 +278,7 @@ class MuxProcess(object):
         """
         responder.whitelist_prefix('ansible')
         responder.whitelist_prefix('ansible_mitogen')
-        responder.whitelist_prefix('simplejson')
+        self._setup_simplejson(responder)
-        simplejson_path = os.path.join(os.path.dirname(__file__), 'compat')
-        sys.path.insert(0, simplejson_path)
 
         # Ansible 2.3 is compatible with Python 2.4 targets, however
         # ansible/__init__.py is not. Instead, executor/module_common.py writes

ansible_mitogen/target.py

@@ -377,6 +377,11 @@ def init_child(econtext, log_level, candidate_temp_dirs):
     LOG.setLevel(log_level)
     logging.getLogger('ansible_mitogen').setLevel(log_level)
 
+    # issue #536: if the json module is available, remove simplejson from the
+    # importer whitelist to avoid confusing certain Ansible modules.
+    if json.__name__ == 'json':
+        econtext.importer.whitelist.remove('simplejson')
+
     global _fork_parent
     if FORK_SUPPORTED:
         mitogen.parent.upgrade_router(econtext)

mitogen/core.py

@@ -1125,6 +1125,11 @@ class Importer(object):
         self.whitelist = list(whitelist) or ['']
         self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST
 
+        # Preserve copies of the original server-supplied whitelist/blacklist
+        # for later use by children.
+        self.master_whitelist = self.whitelist[:]
+        self.master_blacklist = self.blacklist[:]
+
         # Presence of an entry in this map indicates in-flight GET_MODULE.
         self._callbacks = {}
         self._cache = {}

mitogen/master.py

@@ -485,8 +485,10 @@ class ModuleFinder(object):
         return path, source, is_pkg
 
     def _get_module_via_sys_modules(self, fullname):
-        """Attempt to fetch source code via sys.modules. This is specifically
+        """
-        to support __main__, but it may catch a few more cases."""
+        Attempt to fetch source code via sys.modules. This is specifically to
+        support __main__, but it may catch a few more cases.
+        """
         module = sys.modules.get(fullname)
         LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module)
         if not isinstance(module, types.ModuleType):
@@ -883,10 +885,13 @@ class ModuleResponder(object):
         if msg.is_dead:
             return
 
-        LOG.debug('%r._on_get_module(%r)', self, msg.data)
-        self.get_module_count += 1
         stream = self._router.stream_by_id(msg.src_id)
+        if stream is None:
+            return
+
         fullname = msg.data.decode()
+        LOG.debug('%s requested module %s', stream.name, fullname)
+        self.get_module_count += 1
         if fullname in stream.sent_modules:
             LOG.warning('_on_get_module(): dup request for %r from %r',
                         fullname, stream)

mitogen/parent.py

@@ -2054,12 +2054,12 @@ class Router(mitogen.core.Router):
     def get_module_blacklist(self):
         if mitogen.context_id == 0:
             return self.responder.blacklist
-        return self.importer.blacklist
+        return self.importer.master_blacklist
 
     def get_module_whitelist(self):
         if mitogen.context_id == 0:
             return self.responder.whitelist
-        return self.importer.whitelist
+        return self.importer.master_whitelist
 
     def allocate_id(self):
         return self.id_allocator.allocate()