From 9ce6a43329af581e0ce8e913df8db99208f9219b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 1 Dec 2025 09:59:38 +0000 Subject: [PATCH 01/17] Begin 0.3.36dev --- mitogen/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 92297521..88e018d0 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 35) +__version__ = (0, 3, 36, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent From b105877f4d041bb92435561418c01dd656a5bff0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 2 Dec 2025 12:58:39 +0000 Subject: [PATCH 02/17] mitogen: Re-declare Python 2.4 compatibility With CentOS 5 now covered by the Mitogen unit tests I'm content to reverse/clarify 104865e86632429c9c7408e7c6b14ab35ef43067 --- docs/changelog.rst | 1 + docs/index.rst | 10 +++++++--- setup.py | 5 ++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ffab849..c8e9c59d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ +* :gh:issue:`1237` :mod:`mitogen`: Re-declare Python 2.4 compatibility v0.3.35 (2025-12-01) -------------------- diff --git a/docs/index.rst b/docs/index.rst index 32083db0..220a7fff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -332,12 +332,16 @@ a large fleet of machines, or to alert the parent of unexpected state changes. Compatibility ############# -Mitogen is compatible with **Python 2.4** released November 2004, making it +``mitogen.*`` is compatible with Python 2.4 - 2.7 and 3.6 onward; making it suitable for managing a fleet of potentially ancient corporate hardware, such as Red Hat Enterprise Linux 5, released in 2007. -Every combination of Python 3.x/2.x parent and child should be possible, -however at present only Python 2.4, 2.6, 2.7 and 3.6 are tested automatically. +Every combination of Python 3.x/2.x parent and child should be possible. +Automated testing cannot cover every combination, automated testing tries to +cover the extemities (e.g. Python 3.14 parent -> Python 2.4 child). + +``ansible_mitogen.*`` is compatible with Python 2.7 and 3.6 onward; making it +suitable for Ansible 2.10 onward. Zero Dependencies diff --git a/setup.py b/setup.py index ad60847e..047b7e86 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ setup( license = 'BSD-3-Clause', url = 'https://github.com/mitogen-hq/mitogen/', packages = find_packages(exclude=['tests', 'examples']), - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + python_requires='>=2.4, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', zip_safe = False, classifiers = [ 'Environment :: Console', @@ -91,6 +91,9 @@ setup( 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX', 'Programming Language :: Python', + 'Programming Language :: Python :: 2.4', + 'Programming Language :: Python :: 2.5', + 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', From 9b4688247872ae7dcf1b13d760559efd6ff548fd Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 2 Dec 2025 12:59:58 +0000 Subject: [PATCH 03/17] ansible_mitogen: Remove a use of ansible.module_utils.six --- ansible_mitogen/target.py | 7 +++---- docs/changelog.rst | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 6e128af4..199f3a14 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -40,7 +40,6 @@ import errno import grp import json import logging -import operator import os import pty import pwd @@ -66,8 +65,6 @@ if not sys.modules.get(str('__main__')): import ansible.module_utils.json_utils -from ansible.module_utils.six.moves import reduce - import ansible_mitogen.runner @@ -718,7 +715,9 @@ def apply_mode_spec(spec, mode): mask = CHMOD_MASKS[ch] bits = CHMOD_BITS[ch] cur_perm_bits = mode & mask - new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0) + new_perm_bits = 0 + for perm in perms: + new_perm_bits |= bits[perm] mode &= ~mask if op == '=': mode |= new_perm_bits diff --git a/docs/changelog.rst b/docs/changelog.rst index c8e9c59d..ce87cb01 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,8 @@ In progress (unreleased) ------------------------ * :gh:issue:`1237` :mod:`mitogen`: Re-declare Python 2.4 compatibility +* :gh:issue:`1385` :mod:`ansible_mitogen`: Remove a use of + ``ansible.module_utils.six`` v0.3.35 (2025-12-01) -------------------- From 823d1d8b47a07bff7a7323b358e53538f9dc3946 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 2 Dec 2025 13:00:46 +0000 Subject: [PATCH 04/17] docs: Document Ansible 13 (ansible-core 2.20) support --- docs/ansible_detailed.rst | 2 ++ docs/changelog.rst | 2 ++ tox.ini | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index e692c43a..ff5f9dcb 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -145,6 +145,8 @@ Noteworthy Differences +-----------------+ 3.11 - 3.14 | | 12 | | +-----------------+-----------------+ + | 13 | 3.12 - 3.14 | + +-----------------+-----------------+ Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index ce87cb01..7b02c56e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ In progress (unreleased) * :gh:issue:`1237` :mod:`mitogen`: Re-declare Python 2.4 compatibility * :gh:issue:`1385` :mod:`ansible_mitogen`: Remove a use of ``ansible.module_utils.six`` +* :gh:issue:`1354` docs: Document Ansible 13 (ansible-core 2.20) support + v0.3.35 (2025-12-01) -------------------- diff --git a/tox.ini b/tox.ini index 28c97496..3ef977ac 100644 --- a/tox.ini +++ b/tox.ini @@ -63,7 +63,7 @@ envlist = py{27,36}-m_ans-ans{2.10,3,4} py{311}-m_ans-ans{2.10,3-5} py{313}-m_ans-ans{6-9} - py{314}-m_ans-ans{10-12} + py{314}-m_ans-ans{10-13} py{27,36,314}-m_mtg report, From 64a581b2aced72b07d905fb5a4a9da6ba7815825 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 2 Dec 2025 13:06:05 +0000 Subject: [PATCH 05/17] tests: Add Ubuntu 16.04 to image_prep inventory I missed this when committing what built 2025.02 iamges --- tests/image_prep/hosts.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/image_prep/hosts.ini b/tests/image_prep/hosts.ini index 254643a7..0c57e914 100644 --- a/tests/image_prep/hosts.ini +++ b/tests/image_prep/hosts.ini @@ -43,6 +43,7 @@ centos7 centos8 debian9 debian10 +ubuntu1604 ubuntu1804 [ansible_11] From 073fc48afc0cc344026959aa84172a05e8ff29d8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 30 Jul 2023 11:24:12 +0100 Subject: [PATCH 06/17] tests: Remove BlacklistTest stubs (covered by ImporterBlacklistTest) --- tests/responder_test.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/responder_test.py b/tests/responder_test.py index d1e65816..1e376078 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -198,21 +198,3 @@ class ForwardTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(2+os_fork, self.router.responder.good_load_module_count) self.assertLess(10000, self.router.responder.good_load_module_size) self.assertGreater(40000, self.router.responder.good_load_module_size) - - -class BlacklistTest(testlib.TestCase): - @unittest.skip('implement me') - def test_whitelist_no_blacklist(self): - assert 0 - - @unittest.skip('implement me') - def test_whitelist_has_blacklist(self): - assert 0 - - @unittest.skip('implement me') - def test_blacklist_no_whitelist(self): - assert 0 - - @unittest.skip('implement me') - def test_blacklist_has_whitelist(self): - assert 0 From ccaaf4b7fe2d59dc86a35cc5b5fe8d9d0b6a271f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 30 Jul 2023 11:32:59 +0100 Subject: [PATCH 07/17] mitogen: Clarify blacklisted ModuleNotFoundError message Previous phrasing was misleading - it implied a given module was explicitly on the blacklist, even if it was due to a restrictive whitelist and the blacklist was empty. Arguably the blacklist/whitelist semantics are themselves misleading. A redesign is tempting. --- docs/changelog.rst | 2 ++ mitogen/core.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7b02c56e..af756f37 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,8 @@ In progress (unreleased) * :gh:issue:`1385` :mod:`ansible_mitogen`: Remove a use of ``ansible.module_utils.six`` * :gh:issue:`1354` docs: Document Ansible 13 (ansible-core 2.20) support +* :gh:issue:`1354` :mod:`mitogen`: Clarify error message when a module + request would be refused by allow or deny listing v0.3.35 (2025-12-01) diff --git a/mitogen/core.py b/mitogen/core.py index 441743d4..90499fac 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -541,6 +541,7 @@ def is_blacklisted_import(importer, fullname): any packages have been whitelisted and `fullname` is not part of one. NB: + - The default whitelist is `['']` which matches any module name. - If a package is on both lists, then it is treated as blacklisted. - If any package is whitelisted, then all non-whitelisted packages are treated as blacklisted. @@ -1536,9 +1537,8 @@ class Importer(object): return importlib.machinery.ModuleSpec(fullname, loader=self) blacklisted_msg = ( - '%r is present in the Mitogen importer blacklist, therefore this ' - 'context will not attempt to request it from the master, as the ' - 'request will always be refused.' + 'A %r request would be refused by the Mitogen master. The module is ' + 'on the deny list (blacklist) or not on the allow list (whitelist).' ) pkg_resources_msg = ( 'pkg_resources is prohibited from importing __main__, as it causes ' From 0a559ec8d861f691578ac28686c235090f43d43d Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:34:59 +0000 Subject: [PATCH 08/17] testlib: Fix hanging tests When I run $ MITOGEN_LOG_LEVEL=debug SKIP_ANSIBLE=1 ./run_tests -v -k first_stage_test.CommandLineTest in a interactive Shell (with a tty), it ends in a hanging process as the `have_python2` and `have_python3` ends up ends up in an interactive Python shell. Therefore check the Python version instead. Signed-off-by: Marc Hartmayer --- tests/testlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testlib.py b/tests/testlib.py index 15016964..e9104000 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -172,11 +172,11 @@ def _have_cmd(args): def have_python2(): - return _have_cmd(['python2']) + return _have_cmd(['python2', '--version']) def have_python3(): - return _have_cmd(['python3']) + return _have_cmd(['python3', '--version']) def have_sudo_nopassword(): From 733f4bca81d14da8af70cef61da74f3cab813503 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 30 Oct 2025 15:56:10 +0000 Subject: [PATCH 09/17] mitogen: Allow line comments in first stage, strip them. --- mitogen/parent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 6e30b1c6..71641d36 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1463,8 +1463,9 @@ class Connection(object): return [self.options.python_path] def get_boot_command(self): - source = inspect.getsource(self._first_stage) - source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:])) + lines = inspect.getsourcelines(self._first_stage)[0][2:] + # Remove line comments, leading indentation, trailing newline + source = textwrap.dedent(''.join(s for s in lines if '#' not in s))[:-1] source = source.replace(' ', ' ') compressor = zlib.compressobj( zlib.Z_BEST_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, From 606a21fb2765a44b66a2eb299fd454bbe07f510d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 9 Dec 2025 13:46:57 +0000 Subject: [PATCH 10/17] mitogen: Add first line comments to _first_stage() --- mitogen/parent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 71641d36..e91504ce 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1415,7 +1415,6 @@ class Connection(object): # W: write side of interpreter stdin. # r: read side of core_src FD. # w: write side of core_src FD. - # C: the decompressed core source. # Final os.close(STDOUT_FILENO) to avoid --py-debug build corrupting stream with # "[1234 refs]" during exit. @@ -1437,8 +1436,12 @@ class Connection(object): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:%s)'%sys.argv[2]) os.write(1,'MITO000\n'.encode()) + # Read `len(compressed preamble)` bytes sent by our Mitogen parent. + # `select()` handles non-blocking stdin (e.g. sudo + log_output). + # `C` accumulates compressed bytes. C=''.encode() while int(sys.argv[3])-len(C)and select.select([0],[],[]):C+=os.read(0,int(sys.argv[3])-len(C)) + # Raises `zlib.error` if compressed preamble is truncated or invalid C=zlib.decompress(C) f=os.fdopen(W,'wb',0) f.write(C) From fdaf09c4d6780059c677b0078dc2d9b255fede7f Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 19:26:59 +0000 Subject: [PATCH 11/17] mitogen/parent: Fix typo Signed-off-by: Marc Hartmayer --- mitogen/parent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index e91504ce..9da5d58a 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1416,7 +1416,7 @@ class Connection(object): # r: read side of core_src FD. # w: write side of core_src FD. - # Final os.close(STDOUT_FILENO) to avoid --py-debug build corrupting stream with + # Final os.close(STDERR_FILENO) to avoid --py-debug build corrupting stream with # "[1234 refs]" during exit. @staticmethod def _first_stage(): From 2d9e90acf914fe838adf062222dba748a31b7035 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Mon, 15 Dec 2025 10:00:17 +0000 Subject: [PATCH 12/17] parent_test: Refactor `wait_for_child` Signed-off-by: Marc Hartmayer --- tests/parent_test.py | 22 ++-------------------- tests/testlib.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/parent_test.py b/tests/parent_test.py index 558a89b6..dfacffb1 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -2,7 +2,6 @@ import errno import fcntl import os import signal -import sys import time import unittest @@ -21,23 +20,6 @@ except NameError: from io import FileIO as file -def wait_for_child(pid, timeout=1.0): - deadline = mitogen.core.now() + timeout - while timeout < mitogen.core.now(): - try: - target_pid, status = os.waitpid(pid, os.WNOHANG) - if target_pid == pid: - return - except OSError: - e = sys.exc_info()[1] - if e.args[0] == errno.ECHILD: - return - - time.sleep(0.05) - - assert False, "wait_for_child() timed out" - - @mitogen.core.takes_econtext def call_func_in_sibling(ctx, econtext, sync_sender): recv = ctx.call_async(time.sleep, 99999) @@ -106,7 +88,7 @@ class ReapChildTest(testlib.RouterMixin, testlib.TestCase): self.assertRaises(mitogen.core.TimeoutError, lambda: conn.connect(context=mitogen.core.Context(None, 1234)) ) - wait_for_child(conn.proc.pid) + testlib.wait_for_child(conn.proc.pid) e = self.assertRaises(OSError, lambda: os.kill(conn.proc.pid, 0) ) @@ -165,7 +147,7 @@ class ContextTest(testlib.RouterMixin, testlib.TestCase): local = self.router.local() pid = local.call(os.getpid) local.shutdown(wait=True) - wait_for_child(pid) + testlib.wait_for_child(pid) self.assertRaises(OSError, lambda: os.kill(pid, 0)) diff --git a/tests/testlib.py b/tests/testlib.py index e9104000..868aaa84 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -86,6 +86,23 @@ if faulthandler is not None: mitogen.core.LOG.propagate = True +def wait_for_child(pid, timeout=1.0): + deadline = mitogen.core.now() + timeout + while timeout < mitogen.core.now(): + try: + target_pid, status = os.waitpid(pid, os.WNOHANG) + if target_pid == pid: + return + except OSError: + e = sys.exc_info()[1] + if e.args[0] == errno.ECHILD: + return + + time.sleep(0.05) + + assert False, "wait_for_child() timed out" + + def base_executable(executable=None): '''Return the path of the Python executable used to create the virtualenv. ''' From 0ab5b425d8258d27afd17655f7d6cca11550b9aa Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:58:40 +0000 Subject: [PATCH 13/17] first_stage_test: Refactor the test Use testlib.subprocess instead of subprocess and make the test description a docstring that can be used by the test runner. Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 2576ec14..28c60a91 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,5 +1,3 @@ -import subprocess - import mitogen.core import mitogen.parent from mitogen.core import b @@ -17,28 +15,32 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # * 3.x starting 2.7 def test_valid_syntax(self): - options = mitogen.parent.Options(max_message_size=123) - conn = mitogen.parent.Connection(options, self.router) - conn.context = mitogen.core.Context(None, 123) - args = conn.get_boot_command() + """Test valid syntax + + The boot command should write an ECO marker to stdout, read the + preamble from stdin, then execute it. - # The boot command should write an ECO marker to stdout, read the - # preamble from stdin, then execute it. + This test attaches /dev/zero to stdin to create a specific failure - # This test attaches /dev/zero to stdin to create a specific failure - # 1. Fork child reads bytes of NUL (`b'\0'`) - # 2. Fork child crashes (trying to decompress the junk data) - # 3. Fork child's file descriptors (write pipes) are closed by the OS - # 4. Fork parent does `dup(, )` and `exec()` - # 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) - # 6. Python runs `''` (a valid script) and exits with success + 1. Fork child reads bytes of NUL (`b'\0'`) + 2. Fork child crashes (trying to decompress the junk data) + 3. Fork child's file descriptors (write pipes) are closed by the OS + 4. Fork parent does `dup(, )` and `exec()` + 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) + 6. Python runs `''` (a valid script) and exits with success + """ + + options = mitogen.parent.Options(max_message_size=123) + conn = mitogen.parent.Connection(options, self.router) + conn.context = mitogen.core.Context(None, 123) fp = open("/dev/zero", "r") try: - proc = subprocess.Popen(args, + proc = testlib.subprocess.Popen( + args=conn.get_boot_command(), stdin=fp, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=testlib.subprocess.PIPE, + stderr=testlib.subprocess.PIPE, ) stdout, stderr = proc.communicate() self.assertEqual(0, proc.returncode) From f7ca6af62d9f8968b696a565a77d4b1349494cdd Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Wed, 17 Dec 2025 08:40:51 +0000 Subject: [PATCH 14/17] first_stage_test: Open /dev/zero in binary mode Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 28c60a91..3f8a3f50 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -34,7 +34,7 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): options = mitogen.parent.Options(max_message_size=123) conn = mitogen.parent.Connection(options, self.router) conn.context = mitogen.core.Context(None, 123) - fp = open("/dev/zero", "r") + fp = open("/dev/zero", "rb") try: proc = testlib.subprocess.Popen( args=conn.get_boot_command(), From f5195edf08a9dca39db03c79f6d87231c05a718f Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:59:51 +0000 Subject: [PATCH 15/17] first_stage_test: Add more tests + test_non_blocking_stdin + test_blocking_stdin + test_premature_eof + test_broker_connect_eof_error + test_broker_connect_timeout_because_blocking_read(self): Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 313 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 3f8a3f50..354f7479 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,3 +1,7 @@ +import errno +import operator +import os + import mitogen.core import mitogen.parent from mitogen.core import b @@ -5,6 +9,269 @@ from mitogen.core import b import testlib +def create_child_using_pipes(args, blocking, preexec_fn=None): + """ + Create a child process whose stdin/stdout/stderr is connected to a pipe. + + :param list args: + Program argument vector. + :param bool blocking: + If :data:`True`, the sockets use blocking IO, otherwise non-blocking. + :param function preexec_fn: + If not :data:`None`, a function to run within the post-fork child + before executing the target program. + :returns: + :class:`PopenProcess` instance. + """ + + parent_rfp, child_wfp = mitogen.core.pipe(blocking) + child_rfp, parent_wfp = mitogen.core.pipe(blocking) + stderr_r, stderr = mitogen.core.pipe(blocking=blocking) + mitogen.core.set_cloexec(stderr_r.fileno()) + try: + proc = testlib.subprocess.Popen( + args=args, + stdin=child_rfp, + stdout=child_wfp, + stderr=stderr, + close_fds=True, + preexec_fn=preexec_fn, + ) + except Exception: + parent_rfp.close() + parent_wfp.close() + stderr_r.close() + raise + finally: + child_rfp.close() + child_wfp.close() + stderr.close() + + return mitogen.parent.PopenProcess( + proc=proc, + stdin=parent_wfp, + stdout=parent_rfp, + stderr=stderr_r, + ) + + +def create_child_using_sockets(args, blocking, size=None, preexec_fn=None): + """ + Create a child process whose stdin/stdout is connected to a socket and stderr to a pipe. + + :param list args: + Program argument vector. + :param bool blocking: + If :data:`True`, the sockets use blocking IO, otherwise non-blocking. + :param int size: + If not :data:`None`, use the value as the socket buffer size. + :param function preexec_fn: + If not :data:`None`, a function to run within the post-fork child + before executing the target program. + :returns: + :class:`PopenProcess` instance. + """ + + parent_rw_fp, child_rw_fp = mitogen.parent.create_socketpair(size=size, blocking=blocking) + stderr_r, stderr = mitogen.core.pipe(blocking=blocking) + mitogen.core.set_cloexec(stderr_r.fileno()) + try: + proc = testlib.subprocess.Popen( + args=args, + stdin=child_rw_fp, + stdout=child_rw_fp, + stderr=stderr, + close_fds=True, + preexec_fn=preexec_fn, + ) + except Exception: + parent_rw_fp.close() + stderr_r.close() + raise + finally: + child_rw_fp.close() + stderr.close() + + return mitogen.parent.PopenProcess( + proc=proc, + stdin=parent_rw_fp, + stdout=parent_rw_fp, + stderr=stderr_r, + ) + + +class DummyConnectionBlocking(mitogen.parent.Connection): + """Dummy blocking IO connection""" + + create_child = staticmethod(create_child_using_sockets) + name_prefix = "dummy_blocking" + + #: Dictionary of extra kwargs passed to :attr:`create_child`. + #: Use a size smaller than the conn.get_preamble() size so multiple + #: read-calls are needed in the first stage. + create_child_args = {"blocking": True, "size": 4096} + + +class DummyConnectionNonBlocking(mitogen.parent.Connection): + """Dummy non-blocking IO connection""" + + create_child = staticmethod(create_child_using_sockets) + name_prefix = "dummy_non_blocking" + + #: Dictionary of extra kwargs passed to :attr:`create_child`. + #: Use a size smaller than the conn.get_preamble() size so multiple + #: read-calls are needed in the first stage. + create_child_args = {"blocking": False, "size": 4096} + + +class DummyConnectionEOFRead(mitogen.parent.Connection): + """Dummy connection that triggers an EOF-read(STDIN) in the first_stage""" + + name_prefix = "dummy_eof_read" + + #: Dictionary of extra kwargs passed to :attr:`create_child`. + create_child_args = {"blocking": True} + + @staticmethod + def create_child(*a, **kw): + proc = create_child_using_pipes(*a, **kw) + # Close the pipe -> results in an EOF-read(STDIN) in the first_stage + proc.stdin.close() + # Whatever the parent writes to the child, drop it. + proc.stdin = open("/dev/null", "wb") + return proc + + +class DummyConnectionEndlessBlockingRead(mitogen.parent.Connection): + """Dummy connection that triggers a non-returning read(STDIN) call in the + first_stage. + + """ + + name_prefix = "dummy_endless_blocking_read" + + #: Dictionary of extra kwargs passed to :attr:`create_child`. + create_child_args = {"blocking": True} + + @staticmethod + def create_child(*a, **kw): + proc = create_child_using_pipes(*a, **kw) + # Keep the pipe open by having a reference to it, otherwise it would be + # automatically closed by the garbage collector. + proc._mitogen_test_orig_stdin = proc.stdin + # Whatever the parent writes to the child, drop it -> read from STDOUT + # blocks forever in the fork child as no data could be read. + proc.stdin = open("/dev/null", "wb") + return proc + + +class ConnectionTest(testlib.RouterMixin, testlib.TestCase): + def test_non_blocking_stdin(self): + """Test that first stage works with non-blocking STDIN + + The boot command should read the preamble from STDIN, write all ECO + markers to STDOUT, and then execute the preamble. + + This test writes the complete preamble to non-blocking STDIN. + + 1. Fork child reads from non-blocking STDIN + 2. Fork child writes all data as expected by the protocol. + 3. A context call works as expected. + + """ + with testlib.LogCapturer() as _: + ctx = self.router._connect(DummyConnectionNonBlocking, connect_timeout=0.5) + self.assertEqual(3, ctx.call(operator.add, 1, 2)) + + def test_blocking_stdin(self): + """Test that first stage works with blocking STDIN + + The boot command should read the preamble from STDIN, write all ECO + markers to STDOUT, and then execute the preamble. + + This test writes the complete preamble to blocking STDIN. + + 1. Fork child reads from blocking STDIN + 2. Fork child writes all data as expected by the protocol. + 3. A context call works as expected. + + """ + with testlib.LogCapturer() as _: + ctx = self.router._connect(DummyConnectionBlocking, connect_timeout=0.5) + self.assertEqual(3, ctx.call(operator.add, 1, 2)) + + def test_broker_connect_eof_error(self): + """Test that broker takes care about EOF errors in the first stage + + The boot command should write an ECO marker to stdout, try to read the + preamble from STDIN. This read returns with an EOF and the process exits. + + This test writes closes the pipe for STDIN of the fork child to enforce an EOF read call. + 1. Fork child reads from STDIN and reads an EOF and breaks the read-loop + 2. Decompressing the received data results in an error + 3. The child process exits + 4. The streams get disconnected -> mitogen.parent.EofError is raised + + """ + + with testlib.LogCapturer() as _: + e = self.assertRaises(mitogen.parent.EofError, + self.router._connect, DummyConnectionEOFRead, connect_timeout=0.5) + self.assertIn("Error -5 while decompressing data", str(e)) + + # Test that a TimeoutError is raised by the broker and all resources + # are cleaned up. + options = mitogen.parent.Options( + old_router=self.router, + max_message_size=self.router.max_message_size, + connect_timeout=0.5, + ) + conn = DummyConnectionEOFRead(options, router=self.router) + e = self.assertRaises(mitogen.parent.EofError, + conn.connect, context=mitogen.core.Context(None, 1234)) + self.assertIn("Error -5 while decompressing data", str(e)) + # Ensure the child process is reaped if the connection times out. + testlib.wait_for_child(conn.proc.pid) + e = self.assertRaises(OSError, + os.kill, conn.proc.pid, 0) + self.assertEqual(e.args[0], errno.ESRCH) + + def test_broker_connect_timeout_because_endless_blocking_read(self): + """Test that broker takes care about connection timeouts + + The boot command should write an ECO marker to stdout, try to read the + preamble from STDIN. This read blocks forever as the parent does write + all the data to /dev/null instead of the pipe. The broker should then + raise a TimeoutError as the child needs too much time. + + This test writes no data to STDIN of the fork child to enforce a blocking read call. + 1. Fork child tries to read from STDIN, but blocks forever. + 2. Parent connection timeout timer pops up and the parent cleans up + everything from the child (e.g. kills the child process). + 3. TimeoutError is raised in the connect call + + """ + with testlib.LogCapturer() as _: + # Ensure the child process is reaped if the connection times out. + options = mitogen.parent.Options( + old_router=self.router, + max_message_size=self.router.max_message_size, + connect_timeout=0.5, + ) + + conn = DummyConnectionEndlessBlockingRead(options, router=self.router) + try: + self.assertRaises(mitogen.core.TimeoutError, + lambda: conn.connect(context=mitogen.core.Context(None, 1234)) + ) + testlib.wait_for_child(conn.proc.pid) + e = self.assertRaises(OSError, + os.kill, conn.proc.pid, 0) + self.assertEqual(e.args[0], errno.ESRCH) + finally: + conn.proc._mitogen_test_orig_stdin.close() + + class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # Ensure this version of Python produces a command line that is sufficient # to bootstrap this version of Python. @@ -52,3 +319,49 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): ) finally: fp.close() + + def test_premature_eof(self): + """The boot command should write an ECO marker to stdout, read the + preamble from stdin, then execute it. + + This test writes some data to STDIN and closes it then to create an + EOF situation. + 1. Fork child tries to read from STDIN, but stops as EOF is received. + 2. Fork child crashes (trying to decompress the junk data) + 3. Fork child's file descriptors (write pipes) are closed by the OS + 4. Fork parent does `dup(, )` and `exec()` + 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) + 6. Python runs `''` (a valid script) and exits with success""" + + options = mitogen.parent.Options(max_message_size=123) + conn = mitogen.parent.Connection(options, self.router) + conn.context = mitogen.core.Context(None, 123) + proc = testlib.subprocess.Popen( + args=conn.get_boot_command(), + stdout=testlib.subprocess.PIPE, + stderr=testlib.subprocess.PIPE, + stdin=testlib.subprocess.PIPE, + ) + + # Do not send all of the data from the preamble + proc.stdin.write(conn.get_preamble()[:-128]) + proc.stdin.close() + try: + returncode = proc.wait(timeout=1) + except testlib.subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=3) + self.fail("First stage did not handle EOF on STDIN") + try: + self.assertEqual(0, returncode) + self.assertEqual( + proc.stdout.read(), + mitogen.parent.BootstrapProtocol.EC0_MARKER + b("\n"), + ) + self.assertIn( + b("Error -5 while decompressing data"), + proc.stderr.read(), + ) + finally: + proc.stdout.close() + proc.stderr.close() From 8807cd53beb12ff9a022a9fa355e989e31fcc871 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:57:48 +0000 Subject: [PATCH 16/17] mitogen: first_stage: Break the while loop in case of EOF The current implementation can cause an infinite loop, leading to a process that hangs and consumes 100% CPU. This occurs because the EOF condition is not handled properly, resulting in repeated select(...) and read(...) calls. The fix is to properly handle the EOF condition and break out of the loop when it occurs. -SSH command size: 822 +SSH command size: 838 Preamble (mitogen.core + econtext) size: 18226 (17.80KiB) -mitogen.parent 99062 96.7KiB 51235 50.0KiB 51.7% 12936 12.6KiB 13.1% +mitogen.parent 99240 96.9KiB 51244 50.0KiB 51.6% 12956 12.7KiB 13.1% Fixes: https://github.com/mitogen-hq/mitogen/issues/1348 Signed-off-by: Marc Hartmayer --- docs/changelog.rst | 1 + mitogen/parent.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index af756f37..6d35b7b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,7 @@ In progress (unreleased) * :gh:issue:`1354` docs: Document Ansible 13 (ansible-core 2.20) support * :gh:issue:`1354` :mod:`mitogen`: Clarify error message when a module request would be refused by allow or deny listing +* :gh:issue:`1348` :mod:`mitogen`: Fix hanging process with 100% CPU usage v0.3.35 (2025-12-01) diff --git a/mitogen/parent.py b/mitogen/parent.py index 9da5d58a..97681653 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1436,11 +1436,16 @@ class Connection(object): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:%s)'%sys.argv[2]) os.write(1,'MITO000\n'.encode()) + # Size of the compressed core source to be read + n=int(sys.argv[3]) # Read `len(compressed preamble)` bytes sent by our Mitogen parent. # `select()` handles non-blocking stdin (e.g. sudo + log_output). # `C` accumulates compressed bytes. C=''.encode() - while int(sys.argv[3])-len(C)and select.select([0],[],[]):C+=os.read(0,int(sys.argv[3])-len(C)) + # data chunk + V='V' + # Stop looping if no more data is needed or EOF is detected (empty bytes). + while n-len(C) and V:select.select([0],[],[]);V=os.read(0,n-len(C));C+=V # Raises `zlib.error` if compressed preamble is truncated or invalid C=zlib.decompress(C) f=os.fdopen(W,'wb',0) From 4111224161e739a453b79b0c355df77e6f02612b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 18 Dec 2025 17:21:45 +0000 Subject: [PATCH 17/17] Prepare v0.3.36 --- docs/changelog.rst | 4 ++++ mitogen/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6d35b7b2..b6f66c28 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,10 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ + +v0.3.36 (2025-12-01) +-------------------- + * :gh:issue:`1237` :mod:`mitogen`: Re-declare Python 2.4 compatibility * :gh:issue:`1385` :mod:`ansible_mitogen`: Remove a use of ``ansible.module_utils.six`` diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 88e018d0..03694786 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 36, 'dev') +__version__ = (0, 3, 36) #: This is :data:`False` in slave contexts. Previously it was used to prevent