From f191f050bf0e0910e845bdf54ad23185246ef388 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Nov 2025 21:37:42 +0000 Subject: [PATCH 1/2] mitogen: Log why a module is sent or not sent by ModuleResponder This should not change the logic --- docs/changelog.rst | 2 ++ mitogen/master.py | 41 +++++++++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ab048938..a0d27c1c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,8 @@ In progress (unreleased) * :gh:issue:`1118` CI: Add OS release coverage: CentOS 5 * :gh:issue:`1118` CI: Add OS release coverage: Debian 12 * :gh:issue:`1118` CI: Add OS release coverage: Ubuntu 22.04, Ubuntu 24.04 +* :gh:issue:`1124` :mod:`mitogen`: Log why a module is sent or not sent by + :class:`mitogen.master.ModuleResponder` v0.3.33 (2025-11-22) diff --git a/mitogen/master.py b/mitogen/master.py index 3b896504..4de32586 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -931,6 +931,31 @@ class ModuleFinder(object): fullname, _, _ = str_rpartition(to_text(fullname), u'.') yield fullname + def _reject_related_module(self, requested_fullname, related_fullname): + def _log_reject(reason): + LOG.debug( + '%r: Rejected related module %s of requested module %s: %s', + self, related_fullname, requested_fullname, reason, + ) + return reason + + try: + related_module = sys.modules[related_fullname] + except KeyError: + return _log_reject('sys.modules entry absent') + + # Python 2.x "indirection entry" + if related_module is None: + return _log_reject('sys.modules entry is None') + + if is_stdlib_name(related_fullname): + return _log_reject('stdlib module') + + if 'six.moves' in related_fullname: + return _log_reject('six.moves avoidence') + + return False + def find_related_imports(self, fullname): """ Return a list of non-stdlib modules that are directly imported by @@ -973,9 +998,7 @@ class ModuleFinder(object): set( mitogen.core.to_text(name) for name in maybe_names - if sys.modules.get(name) is not None - and not is_stdlib_name(name) - and u'six.moves' not in name # TODO: crap + if not self._reject_related_module(fullname, name) ) )) @@ -1138,7 +1161,7 @@ class ModuleResponder(object): self._cache[fullname] = tup return tup - def _send_load_module(self, stream, fullname): + def _send_load_module(self, stream, fullname, reason): if fullname not in stream.protocol.sent_modules: tup = self._build_tuple(fullname) msg = mitogen.core.Message.pickled( @@ -1146,8 +1169,10 @@ class ModuleResponder(object): dst_id=stream.protocol.remote_id, handle=mitogen.core.LOAD_MODULE, ) - self._log.debug('sending %s (%.2f KiB) to %s', - fullname, len(msg.data) / 1024.0, stream.name) + self._log.debug( + 'sending %s %s (%.2f KiB) to %s', + reason, fullname, len(msg.data) / 1024.0, stream.name, + ) self._router._async_route(msg) stream.protocol.sent_modules.add(fullname) if tup[2] is not None: @@ -1178,8 +1203,8 @@ class ModuleResponder(object): # Parent hasn't been sent, so don't load submodule yet. continue - self._send_load_module(stream, name) - self._send_load_module(stream, fullname) + self._send_load_module(stream, name, 'related') + self._send_load_module(stream, fullname, 'requested') except Exception: LOG.debug('While importing %r', fullname, exc_info=True) self._send_module_load_failed(stream, fullname) From 83b6cdb616ad838f431a7cac94717f6adb4c4698 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Nov 2025 22:16:31 +0000 Subject: [PATCH 2/2] ansible_mitogen: Speedup startup by not sending `__main__` On my laptop his reduces the time to execute `ansible -mping ...` by approx 300 ms with `strategy=mitogen_linear`. Until this commit Mitogen was unnecessarily sending large chunks of Ansible from the controller to targets, due to `__main__` being identified as a related module of `ansible.module_utils.basic`, and resolving to something within `ansible.cli...`. On Ansible target hosts executing any Ansible Module `__main__` is imported by `ansible.module_utils.basic` as part of Ansible's module delivery mechanism. When `mitogen.master.ModuleResponder` (on the controller) processes the request for `ansible.module_utils.basic` from the target, it scans `ansible.module_utils.basic` for related imports and finds `__main__`. However `__main__` on the controller is not the same module as `__main__` on the target. On the controller it is a module in `ansible.cli...` that implements one of the ansible commands (e.g. `ansible`, `ansible-playbook`). --- docs/changelog.rst | 2 ++ mitogen/master.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a0d27c1c..1824f6f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,8 @@ In progress (unreleased) * :gh:issue:`1118` CI: Add OS release coverage: Ubuntu 22.04, Ubuntu 24.04 * :gh:issue:`1124` :mod:`mitogen`: Log why a module is sent or not sent by :class:`mitogen.master.ModuleResponder` +* :gh:issue:`1124` :mod:`ansible_mitogen`: Speedup startup by not sending + ``__main__`` as a related module v0.3.33 (2025-11-22) diff --git a/mitogen/master.py b/mitogen/master.py index 4de32586..e495197d 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -843,6 +843,12 @@ class ModuleFinder(object): related modules likely needed by a child context requesting the original module. """ + + # Fullnames of modules that should not be sent as a related module + _related_modules_denylist = frozenset({ + '__main__', + }) + def __init__(self): #: Import machinery is expensive, keep :py:meth`:get_module_source` #: results around. @@ -954,6 +960,9 @@ class ModuleFinder(object): if 'six.moves' in related_fullname: return _log_reject('six.moves avoidence') + if related_fullname in self._related_modules_denylist: + return _log_reject('on denylist') + return False def find_related_imports(self, fullname):