From 7ff4e6694cefcfae3a3577915e7058d072a86609 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 13 Feb 2019 02:16:49 +0000 Subject: [PATCH] issue #536: rework how 2.3-compatible simplejson is served Regardless of the version of simplejson loaded in the master, load up the ModuleResponder cache with our 2.4-compatible version. To cope with simplejson being loaded due to modules like ec2_group that try to import it before importing 'json', also update target.py to remove it from the whitelist if a local 'json' module import succeeds. --- ansible_mitogen/process.py | 39 +++++++++++++++++++++++++++++++++++--- ansible_mitogen/target.py | 5 +++++ mitogen/core.py | 5 +++++ mitogen/master.py | 13 +++++++++---- mitogen/parent.py | 4 ++-- 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 8137af9c..32f4a012 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -236,6 +236,41 @@ class MuxProcess(object): if secs: mitogen.debug.dump_to_logger(secs=secs) + def _setup_simplejson(self, responder): + """ + We support serving simplejson for Python 2.4 targets on Ansible 2.3, at + least so the package's own CI Docker scripts can run without external + help, however newer versions of simplejson no longer support Python + 2.4. Therefore override any installed/loaded version with a + 2.4-compatible version we ship in the compat/ directory. + """ + responder.whitelist_prefix('simplejson') + + # issue #536: must be at end of sys.path, in case existing newer + # version is already loaded. + compat_path = os.path.join(os.path.dirname(__file__), 'compat') + sys.path.append(compat_path) + + for fullname, is_pkg, suffix in ( + (u'simplejson', True, '__init__.py'), + (u'simplejson.decoder', False, 'decoder.py'), + (u'simplejson.encoder', False, 'encoder.py'), + (u'simplejson.scanner', False, 'scanner.py'), + ): + path = os.path.join(compat_path, 'simplejson', suffix) + fp = open(path, 'rb') + try: + source = fp.read() + finally: + fp.close() + + responder.add_source_override( + fullname=fullname, + path=path, + source=source, + is_pkg=is_pkg, + ) + def _setup_responder(self, responder): """ Configure :class:`mitogen.master.ModuleResponder` to only permit @@ -243,9 +278,7 @@ class MuxProcess(object): """ responder.whitelist_prefix('ansible') responder.whitelist_prefix('ansible_mitogen') - responder.whitelist_prefix('simplejson') - simplejson_path = os.path.join(os.path.dirname(__file__), 'compat') - sys.path.insert(0, simplejson_path) + self._setup_simplejson(responder) # Ansible 2.3 is compatible with Python 2.4 targets, however # ansible/__init__.py is not. Instead, executor/module_common.py writes diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 01877e34..58b715ce 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -377,6 +377,11 @@ def init_child(econtext, log_level, candidate_temp_dirs): LOG.setLevel(log_level) logging.getLogger('ansible_mitogen').setLevel(log_level) + # issue #536: if the json module is available, remove simplejson from the + # importer whitelist to avoid confusing certain Ansible modules. + if json.__name__ == 'json': + econtext.importer.whitelist.remove('simplejson') + global _fork_parent if FORK_SUPPORTED: mitogen.parent.upgrade_router(econtext) diff --git a/mitogen/core.py b/mitogen/core.py index a48e13ed..9f329d88 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1125,6 +1125,11 @@ class Importer(object): self.whitelist = list(whitelist) or [''] self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST + # Preserve copies of the original server-supplied whitelist/blacklist + # for later use by children. + self.master_whitelist = self.whitelist[:] + self.master_blacklist = self.blacklist[:] + # Presence of an entry in this map indicates in-flight GET_MODULE. self._callbacks = {} self._cache = {} diff --git a/mitogen/master.py b/mitogen/master.py index 257fb81b..18323844 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -485,8 +485,10 @@ class ModuleFinder(object): return path, source, is_pkg def _get_module_via_sys_modules(self, fullname): - """Attempt to fetch source code via sys.modules. This is specifically - to support __main__, but it may catch a few more cases.""" + """ + Attempt to fetch source code via sys.modules. This is specifically to + support __main__, but it may catch a few more cases. + """ module = sys.modules.get(fullname) LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) if not isinstance(module, types.ModuleType): @@ -883,10 +885,13 @@ class ModuleResponder(object): if msg.is_dead: return - LOG.debug('%r._on_get_module(%r)', self, msg.data) - self.get_module_count += 1 stream = self._router.stream_by_id(msg.src_id) + if stream is None: + return + fullname = msg.data.decode() + LOG.debug('%s requested module %s', stream.name, fullname) + self.get_module_count += 1 if fullname in stream.sent_modules: LOG.warning('_on_get_module(): dup request for %r from %r', fullname, stream) diff --git a/mitogen/parent.py b/mitogen/parent.py index 91a4e5eb..814d20e3 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -2054,12 +2054,12 @@ class Router(mitogen.core.Router): def get_module_blacklist(self): if mitogen.context_id == 0: return self.responder.blacklist - return self.importer.blacklist + return self.importer.master_blacklist def get_module_whitelist(self): if mitogen.context_id == 0: return self.responder.whitelist - return self.importer.whitelist + return self.importer.master_whitelist def allocate_id(self): return self.id_allocator.allocate()