From 4d940f08aeae0c82462da1ed6d39463f72c84b66 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 16 Feb 2018 07:28:35 +0545 Subject: [PATCH] importer: drop redundant prefix from pkg_present For the 52 submodules of ansible.modules.system, this produced a 1602 byte pkg_present list. After stripping it becomes 406 bytes, and the entire LOAD_MODULE size drops from 1988 bytes to 792 bytes (-60%). For the 68 submodules of ansible.module_utils, 1902 bytes pkg_present becomes 474 bytes (-75%), and LOAD_MODULE size drops from 2867 bytes to 1439 bytes (-49%). In a simple test running Ansible's "setup" module followed by its "apt" module, wire bytes sent drops from 140,357 to 135,531 (-3.4%). --- docs/internals.rst | 11 ++++------- mitogen/core.py | 21 +++++++++++---------- mitogen/master.py | 6 +++--- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index e2281a47..5dbf431f 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -352,21 +352,18 @@ Helper Functions .. currentmodule:: mitogen.master -.. function:: get_child_modules (path, fullname) +.. function:: get_child_modules (path) - Return the canonical names of all submodules of a package `module`. + Return the suffixes of submodules directly neated beneath of the package + directory at `path`. :param str path: Path to the module's source code on disk, or some PEP-302-recognized equivalent. Usually this is the module's ``__file__`` attribute, but is specified explicitly to avoid loading the module. - :param str fullname: - The module's canonical name. This is the module's ``__name__`` - attribute, but is specified explicitly to avoid loading the module. - :return: - List of canonical submodule names. + List of submodule name suffixes. .. currentmodule:: mitogen.parent diff --git a/mitogen/core.py b/mitogen/core.py index 36661b96..2e1ad8aa 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -402,14 +402,14 @@ class Importer(object): def __init__(self, router, context, core_src, whitelist=(), blacklist=()): self._context = context self._present = {'mitogen': [ - 'mitogen.compat', - 'mitogen.compat.pkgutil', - 'mitogen.fakessh', - 'mitogen.master', - 'mitogen.parent', - 'mitogen.ssh', - 'mitogen.sudo', - 'mitogen.utils', + 'compat', + 'compat.pkgutil', + 'fakessh', + 'master', + 'parent', + 'ssh', + 'sudo', + 'utils', ]} self._lock = threading.Lock() self.whitelist = whitelist or [''] @@ -444,9 +444,10 @@ class Importer(object): _tls.running = True fullname = fullname.rstrip('.') try: - pkgname, _, _ = fullname.rpartition('.') + pkgname, dot, _ = fullname.rpartition('.') _v and LOG.debug('%r.find_module(%r)', self, fullname) - if fullname not in self._present.get(pkgname, (fullname,)): + suffix = fullname[len(pkgname+dot):] + if suffix not in self._present.get(pkgname, (suffix,)): _v and LOG.debug('%r: master doesn\'t know %r', self, fullname) return None diff --git a/mitogen/master.py b/mitogen/master.py index 6fbfaf8a..a0a85a8e 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -56,9 +56,9 @@ from mitogen.core import LOG RLOG = logging.getLogger('mitogen.ctx') -def get_child_modules(path, fullname): +def get_child_modules(path): it = pkgutil.iter_modules([os.path.dirname(path)]) - return ['%s.%s' % (fullname, name) for _, name, _ in it] + return [name for _, name, _ in it] def scan_code_imports(co, LOAD_CONST=dis.opname.index('LOAD_CONST'), @@ -489,7 +489,7 @@ class ModuleResponder(object): raise ImportError('could not find %r' % (fullname,)) if is_pkg: - pkg_present = get_child_modules(path, fullname) + pkg_present = get_child_modules(path) LOG.debug('_build_tuple(%r, %r) -> %r', path, fullname, pkg_present) else: