From 07d10780105ffc7cd8e097f18701c12f97ac512e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 20 Aug 2025 22:29:10 +0100 Subject: [PATCH 01/13] Begin v0.3.28dev --- docs/changelog.rst | 4 ++++ mitogen/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1b196b58..70d411ee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,10 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +In progress (unreleased) +------------------------ + + v0.3.27 (2025-08-20) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 2dab027a..cc3cd425 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 27) +__version__ = (0, 3, 28, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent From dc7fae973b7eafd74d95e84f2aaa43213ef912df Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 12 Aug 2025 18:11:32 +0100 Subject: [PATCH 02/13] CI: Fix ci_lib and test_lib have_() when exits abnormally We were not raising CalledProcessError when exit status != 0. --- .ci/ci_lib.py | 1 + tests/testlib.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 8193a34c..6a771662 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -58,6 +58,7 @@ def _have_cmd(args): try: subprocess.run( args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + check=True, ) except OSError as exc: if exc.errno == errno.ENOENT: diff --git a/tests/testlib.py b/tests/testlib.py index a0e27d95..b91739e6 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -160,6 +160,7 @@ def _have_cmd(args): try: subprocess.run( args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + check=True, ) except OSError as exc: if exc.errno == errno.ENOENT: From 5abdde1117356a146aebf6d4e94a4adde1aef07f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Aug 2025 16:56:18 +0100 Subject: [PATCH 03/13] CI: Report sudo version on Ansible targets --- docs/changelog.rst | 2 ++ tests/ansible/setup/report_targets.yml | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 70d411ee..dac924d8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ +* :gh:issue:`1306` CI: Report sudo version on Ansible targets + v0.3.27 (2025-08-20) -------------------- diff --git a/tests/ansible/setup/report_targets.yml b/tests/ansible/setup/report_targets.yml index 5aa67124..c2d9e3d3 100644 --- a/tests/ansible/setup/report_targets.yml +++ b/tests/ansible/setup/report_targets.yml @@ -13,3 +13,23 @@ - debug: {var: ansible_facts.osversion} - debug: {var: ansible_facts.python} - debug: {var: ansible_facts.system} + +- name: Check target versions + hosts: localhost:test-targets + check_mode: false + tasks: + - name: Get command versions + command: + cmd: "{{ item.cmd }}" + changed_when: false + check_mode: false + loop: + - cmd: sudo -V + register: command_versions + + - name: Show command versions + debug: + msg: | + cmd: {{ item.item.cmd }} + {{ item.stdout }} + loop: "{{ command_versions.results }}" From e4e2c6caaf06edf2a346c150565fd08a6616e7dc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 20 Aug 2025 23:46:25 +0100 Subject: [PATCH 04/13] CI: Move sudo test users defaults into /etc/sudoers.d Prep for reusing it in non-Ansible tests --- docs/changelog.rst | 1 + tests/image_prep/_user_accounts.yml | 15 +++++++-------- tests/image_prep/files/sudoers_defaults | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 tests/image_prep/files/sudoers_defaults diff --git a/docs/changelog.rst b/docs/changelog.rst index dac924d8..0e52d365 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,7 @@ In progress (unreleased) ------------------------ * :gh:issue:`1306` CI: Report sudo version on Ansible targets +* :gh:issue:`1306` CI: Move sudo test users defaults into ``/etc/sudoers.d`` v0.3.27 (2025-08-20) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 225d4a53..a1701e55 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -157,15 +157,14 @@ owner: mitogen__has_sudo_pubkey group: mitogen__group - - name: Configure sudoers defaults - blockinfile: - path: /etc/sudoers - marker: "# {mark} Mitogen test defaults" - block: | - Defaults>mitogen__pw_required targetpw - Defaults>mitogen__require_tty requiretty - Defaults>mitogen__require_tty_pw_required requiretty,targetpw + - name: Configure sudoers + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: ug=r,o= validate: '/usr/sbin/visudo -cf %s' + with_items: + - {src: sudoers_defaults, dest: /etc/sudoers.d/mitogen_test_defaults} - name: Configure sudoers users blockinfile: diff --git a/tests/image_prep/files/sudoers_defaults b/tests/image_prep/files/sudoers_defaults new file mode 100644 index 00000000..3ad7a6d4 --- /dev/null +++ b/tests/image_prep/files/sudoers_defaults @@ -0,0 +1,3 @@ +Defaults>mitogen__pw_required targetpw +Defaults>mitogen__require_tty requiretty +Defaults>mitogen__require_tty_pw_required requiretty,targetpw From 30d8a38a3b93626f2090e5847b001a280e817c29 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Aug 2025 17:08:47 +0100 Subject: [PATCH 05/13] preamble_size: Consolidate table formatting, align columns better Before ./preamble_size.py SSH command size: 759 Bootstrap (mitogen.core) size: 18227 (17.80KiB) Original Minimized Compressed mitogen.parent 98853 96.5KiB 51103 49.9KiB 51.7% 12881 12.6KiB 13.0% mitogen.fork 8445 8.2KiB 4139 4.0KiB 49.0% 1652 1.6KiB 19.6% mitogen.ssh 10827 10.6KiB 6893 6.7KiB 63.7% 2099 2.0KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41581 40.6KiB 22398 21.9KiB 53.9% 5847 5.7KiB 14.1% mitogen.fakessh 15767 15.4KiB 8149 8.0KiB 51.7% 2676 2.6KiB 17.0% mitogen.master 55317 54.0KiB 28846 28.2KiB 52.1% 7528 7.4KiB 13.6% After: SSH command size: 759 Bootstrap (mitogen.core) size: 18227 (17.80KiB) Original Minimized Compressed mitogen.parent 98853 96.5KiB 51103 49.9KiB 51.7% 12881 12.6KiB 13.0% mitogen.fork 8445 8.2KiB 4139 4.0KiB 49.0% 1652 1.6KiB 19.6% mitogen.ssh 10827 10.6KiB 6893 6.7KiB 63.7% 2099 2.0KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41581 40.6KiB 22398 21.9KiB 53.9% 5847 5.7KiB 14.1% mitogen.fakessh 15767 15.4KiB 8149 8.0KiB 51.7% 2676 2.6KiB 17.0% mitogen.master 55317 54.0KiB 28846 28.2KiB 52.1% 7528 7.4KiB 13.6% --- preamble_size.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/preamble_size.py b/preamble_size.py index efb46eae..2225658a 100755 --- a/preamble_size.py +++ b/preamble_size.py @@ -18,6 +18,16 @@ import mitogen.service import mitogen.ssh import mitogen.sudo + +class Table(object): + HEADERS = (' ', 'Original', 'Minimized', 'Compressed') + HEAD_FMT = '{:20} {:^15} {:^19} {:^19}' + ROW_FMT = '%-20s %6i %5.1fKiB %5i %4.1fKiB %4.1f%% %5i %4.1fKiB %4.1f%%' + + def header(self): + return self.HEAD_FMT.format(*self.HEADERS) + + router = mitogen.master.Router() context = mitogen.parent.Context(router, 0) options = mitogen.ssh.Options(max_message_size=0, hostname='foo') @@ -36,16 +46,8 @@ if '--dump' in sys.argv: exit() -print( - ' ' - ' ' - ' Original ' - ' ' - ' Minimized ' - ' ' - ' Compressed ' -) - +table = Table() +print(table.header()) for mod in ( mitogen.parent, mitogen.fork, @@ -63,13 +65,7 @@ for mod in ( compressed = zlib.compress(minimized.encode(), 9) compressed_size = len(compressed) print( - '%-25s' - ' ' - '%5i %4.1fKiB' - ' ' - '%5i %4.1fKiB %.1f%%' - ' ' - '%5i %4.1fKiB %.1f%%' + table.ROW_FMT % ( mod.__name__, original_size, From 936b08dd082116efc69070296091a8fda2ec057a Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Aug 2025 17:13:00 +0100 Subject: [PATCH 06/13] preamble_size: Include mitogen.core and clarify bootstrap size After: SSH command size: 759 Preamble (mitogen.core + econtext) size: 18227 (17.80KiB) Original Minimized Compressed mitogen.core 152218 148.7KiB 68437 66.8KiB 45.0% 18124 17.7KiB 11.9% mitogen.parent 98853 96.5KiB 51103 49.9KiB 51.7% 12881 12.6KiB 13.0% mitogen.fork 8445 8.2KiB 4139 4.0KiB 49.0% 1652 1.6KiB 19.6% mitogen.ssh 10827 10.6KiB 6893 6.7KiB 63.7% 2099 2.0KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41581 40.6KiB 22398 21.9KiB 53.9% 5847 5.7KiB 14.1% mitogen.fakessh 15767 15.4KiB 8149 8.0KiB 51.7% 2676 2.6KiB 17.0% mitogen.master 55317 54.0KiB 28846 28.2KiB 52.1% 7528 7.4KiB 13.6% --- preamble_size.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/preamble_size.py b/preamble_size.py index 2225658a..adea488f 100755 --- a/preamble_size.py +++ b/preamble_size.py @@ -8,6 +8,7 @@ import inspect import sys import zlib +import mitogen.core import mitogen.fakessh import mitogen.fork import mitogen.master @@ -35,7 +36,7 @@ conn = mitogen.ssh.Connection(options, router) conn.context = context print('SSH command size: %s' % (len(' '.join(conn.get_boot_command())),)) -print('Bootstrap (mitogen.core) size: %s (%.2fKiB)' % ( +print('Preamble (mitogen.core + econtext) size: %s (%.2fKiB)' % ( len(conn.get_preamble()), len(conn.get_preamble()) / 1024.0, )) @@ -49,6 +50,7 @@ if '--dump' in sys.argv: table = Table() print(table.header()) for mod in ( + mitogen.core, mitogen.parent, mitogen.fork, mitogen.ssh, From 3dfaf83ce71f28bfbc3528fcaada070dac1c74a6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Aug 2025 17:17:34 +0100 Subject: [PATCH 07/13] preamble_size: Fix variability of command & preamble(?) size Previously the command size could very depanding on the current username, hostname, and process pid. Before ``` SSH command size: 759 Preamble (mitogen.core + econtext) size: 18227 (17.80KiB) ... ``` After SSH command size: 755 Preamble (mitogen.core + econtext) size: 18227 (17.80KiB) ... ``` --- docs/changelog.rst | 1 + preamble_size.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0e52d365..efa01375 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ In progress (unreleased) * :gh:issue:`1306` CI: Report sudo version on Ansible targets * :gh:issue:`1306` CI: Move sudo test users defaults into ``/etc/sudoers.d`` +* :gh:issue:`1306` preamble_size: Fix variability of measured command size v0.3.27 (2025-08-20) diff --git a/preamble_size.py b/preamble_size.py index adea488f..a696b2b5 100755 --- a/preamble_size.py +++ b/preamble_size.py @@ -31,7 +31,11 @@ class Table(object): router = mitogen.master.Router() context = mitogen.parent.Context(router, 0) -options = mitogen.ssh.Options(max_message_size=0, hostname='foo') +options = mitogen.ssh.Options( + hostname='foo', + max_message_size=0, + remote_name='alice@host:1234', +) conn = mitogen.ssh.Connection(options, router) conn.context = context From 76f6eb741dc88e07a671e4b9b472b5a27fd5b3fa Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Aug 2025 17:34:46 +0100 Subject: [PATCH 08/13] tests: Count bytes written in stdio_test.StdIOTest This is mainly for peace of mind. With all this non-blocking IO investigation I'm getting a bit paranoid wrt file objects. refs #712 --- docs/changelog.rst | 1 + tests/data/stdio_checks.py | 19 +++++++++++++++++-- tests/stdio_test.py | 4 ++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index efa01375..dbf653dc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,7 @@ In progress (unreleased) * :gh:issue:`1306` CI: Report sudo version on Ansible targets * :gh:issue:`1306` CI: Move sudo test users defaults into ``/etc/sudoers.d`` * :gh:issue:`1306` preamble_size: Fix variability of measured command size +* :gh:issue:`1306` tests: Count bytes written in ``stdio_test.StdIOTest`` v0.3.27 (2025-08-20) diff --git a/tests/data/stdio_checks.py b/tests/data/stdio_checks.py index 5bfe8063..a67e2689 100644 --- a/tests/data/stdio_checks.py +++ b/tests/data/stdio_checks.py @@ -3,9 +3,24 @@ import os import sys +def _shout_stdout_py3(size): + nwritten = sys.stdout.write('A' * size) + return nwritten + + +def _shout_stdout_py2(size): + shout = 'A' * size + nwritten = 0 + while nwritten < size: + nwritten += os.write(sys.stdout.fileno(), shout[-nwritten:]) + return nwritten + + def shout_stdout(size): - sys.stdout.write('A' * size) - return 'success' + if sys.version_info > (3, 0): + return _shout_stdout_py3(size) + else: + return _shout_stdout_py2(size) def file_is_blocking(fobj): diff --git a/tests/stdio_test.py b/tests/stdio_test.py index da8edd8e..3c0e91d1 100644 --- a/tests/stdio_test.py +++ b/tests/stdio_test.py @@ -15,8 +15,8 @@ class StdIOTest(testlib.RouterMixin, testlib.TestCase): """ size = 1 * 2**20 context = self.router.local() - result = context.call(stdio_checks.shout_stdout, size) - self.assertEqual('success', result) + nwritten = context.call(stdio_checks.shout_stdout, size) + self.assertEqual(nwritten, size) def test_stdio_is_blocking(self): context = self.router.local() From c508bfb58bca496ea9e436ed2ac60b678910b829 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Aug 2025 17:39:40 +0100 Subject: [PATCH 09/13] tests: Check stdio is blocking in sudo contexts refs #712 --- docs/changelog.rst | 1 + tests/stdio_test.py | 28 +++++++++++++++++++++++----- tests/testlib.py | 9 +++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dbf653dc..956e5dc9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ In progress (unreleased) * :gh:issue:`1306` CI: Move sudo test users defaults into ``/etc/sudoers.d`` * :gh:issue:`1306` preamble_size: Fix variability of measured command size * :gh:issue:`1306` tests: Count bytes written in ``stdio_test.StdIOTest`` +* :gh:issue:`1306` tests: Check stdio is blocking in sudo contexts v0.3.27 (2025-08-20) diff --git a/tests/stdio_test.py b/tests/stdio_test.py index 3c0e91d1..d60cf31b 100644 --- a/tests/stdio_test.py +++ b/tests/stdio_test.py @@ -1,28 +1,46 @@ +import unittest + import testlib import stdio_checks -class StdIOTest(testlib.RouterMixin, testlib.TestCase): +class StdIOMixin(testlib.RouterMixin): """ Test that stdin, stdout, and stderr conform to common expectations, such as blocking IO. """ - def test_can_write_stdout_1_mib(self): + def check_can_write_stdout_1_mib(self, context): """ Writing to stdout should not raise EAGAIN. Regression test for https://github.com/mitogen-hq/mitogen/issues/712. """ size = 1 * 2**20 - context = self.router.local() nwritten = context.call(stdio_checks.shout_stdout, size) self.assertEqual(nwritten, size) - def test_stdio_is_blocking(self): - context = self.router.local() + def check_stdio_is_blocking(self, context): stdin_blocking, stdout_blocking, stderr_blocking = context.call( stdio_checks.stdio_is_blocking, ) self.assertTrue(stdin_blocking) self.assertTrue(stdout_blocking) self.assertTrue(stderr_blocking) + + +class LocalTest(StdIOMixin, testlib.TestCase): + def test_can_write_stdout_1_mib(self): + self.check_can_write_stdout_1_mib(self.router.local()) + + def test_stdio_is_blocking(self): + self.check_stdio_is_blocking(self.router.local()) + + +class SudoTest(StdIOMixin, testlib.TestCase): + @unittest.skipIf(not testlib.have_sudo_nopassword(), 'Needs passwordless sudo') + def test_can_write_stdout_1_mib(self): + self.check_can_write_stdout_1_mib(self.router.sudo()) + + @unittest.skipIf(not testlib.have_sudo_nopassword(), 'Needs passwordless sudo') + def test_stdio_is_blocking(self): + self.check_stdio_is_blocking(self.router.sudo()) diff --git a/tests/testlib.py b/tests/testlib.py index b91739e6..20b6e7c7 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -179,6 +179,15 @@ def have_python3(): return _have_cmd(['python3']) +def have_sudo_nopassword(): + """ + Return True if we can run `sudo` with no password, otherwise False. + + Any cached credentials are ignored. + """ + return _have_cmd(['sudo', '-kn', 'true']) + + def retry(fn, on, max_attempts, delay): for i in range(max_attempts): try: From 85d6046f2fbf4e80c472f7a06596bf8ad1fcd9e6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 20 Aug 2025 23:47:53 +0100 Subject: [PATCH 10/13] mitogen: Fix non-blocking IO errors in first stage of bootstrap When /etc/sudoers has log_output (or similar) enabled the process spawned by `ctx.sudo()` via `mitogen.parent.Connection.start_child()` receives a stdin that is in non-blocking mode. The immediate symptom is that `os.openfd(0, ...).read(n)` sometimes returns `None`, causing the first stage to raise an unhandled TypeError. The fix (for now) is to use `select.select()` in a while loop to read stdin. This increases the command size slightly, but I think it's a reasonable tradeoff until/unless the cause is more fully understood. All CI tests are now run with sudoers log_output enabled, in order to catch regressions. `first_stage_test.CommandLineTest` has been amended, because it relied on implementation details of the bootstrap process that are no longer true. Before ``` SSH command size: 755 Preamble (mitogen.core + econtext) size: 18227 (17.80KiB) Original Minimized Compressed mitogen.core 152218 148.7KiB 68437 66.8KiB 45.0% 18124 17.7KiB 11.9% mitogen.parent 98853 96.5KiB 51103 49.9KiB 51.7% 12881 12.6KiB 13.0% mitogen.fork 8445 8.2KiB 4139 4.0KiB 49.0% 1652 1.6KiB 19.6% mitogen.ssh 10827 10.6KiB 6893 6.7KiB 63.7% 2099 2.0KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41581 40.6KiB 22398 21.9KiB 53.9% 5847 5.7KiB 14.1% mitogen.fakessh 15767 15.4KiB 8149 8.0KiB 51.7% 2676 2.6KiB 17.0% mitogen.master 55317 54.0KiB 28846 28.2KiB 52.1% 7528 7.4KiB 13.6% ``` After ``` SSH command size: 798 Preamble (mitogen.core + econtext) size: 18227 (17.80KiB) Original Minimized Compressed mitogen.core 152218 148.7KiB 68437 66.8KiB 45.0% 18124 17.7KiB 11.9% mitogen.parent 98944 96.6KiB 51180 50.0KiB 51.7% 12910 12.6KiB 13.0% mitogen.fork 8445 8.2KiB 4139 4.0KiB 49.0% 1652 1.6KiB 19.6% mitogen.ssh 10827 10.6KiB 6893 6.7KiB 63.7% 2099 2.0KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41581 40.6KiB 22398 21.9KiB 53.9% 5847 5.7KiB 14.1% mitogen.fakessh 15767 15.4KiB 8149 8.0KiB 51.7% 2676 2.6KiB 17.0% mitogen.master 55317 54.0KiB 28846 28.2KiB 52.1% 7528 7.4KiB 13.6% ``` --- .ci/ci_lib.py | 2 ++ .ci/mitogen_tests.py | 9 +++++++++ docs/changelog.rst | 2 ++ mitogen/parent.py | 11 ++++++----- tests/first_stage_test.py | 24 ++++++++++++++++-------- tests/image_prep/files/sudoers_defaults | 4 ++++ 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 6a771662..c50c76da 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -41,6 +41,8 @@ IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'ghcr.io/mitogen-hq/%(distro)s-test:2021', ) +SUDOERS_DEFAULTS_SRC = './tests/image_prep/files/sudoers_defaults' +SUDOERS_DEFAULTS_DEST = '/etc/sudoers.d/mitogen_test_defaults' TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key') diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 47aa2444..2ecddc31 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -2,6 +2,7 @@ # Run the Mitogen tests. import os +import subprocess import ci_lib @@ -13,6 +14,14 @@ os.environ.update({ if not ci_lib.have_docker(): os.environ['SKIP_DOCKER_TESTS'] = '1' +subprocess.check_call( + "umask 0022; sudo cp '%s' '%s'" + % (ci_lib.SUDOERS_DEFAULTS_SRC, ci_lib.SUDOERS_DEFAULTS_DEST), + shell=True, +) +subprocess.check_call(['sudo', 'visudo', '-cf', ci_lib.SUDOERS_DEFAULTS_DEST]) +subprocess.check_call(['sudo', '-l']) + interesting = ci_lib.get_interesting_procs() ci_lib.run('./run_tests -v') ci_lib.check_stray_processes(interesting) diff --git a/docs/changelog.rst b/docs/changelog.rst index 956e5dc9..99378e1a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ +* :gh:issue:`1306` :mod:`ansible_mitogen`: Fix non-blocking IO errors in + first stage of bootstrap * :gh:issue:`1306` CI: Report sudo version on Ansible targets * :gh:issue:`1306` CI: Move sudo test users defaults into ``/etc/sudoers.d`` * :gh:issue:`1306` preamble_size: Fix variability of measured command size diff --git a/mitogen/parent.py b/mitogen/parent.py index 7ac14fb1..1a23df18 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1440,9 +1440,9 @@ class Connection(object): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') os.write(1,'MITO000\n'.encode()) - fp=os.fdopen(0,'rb') - C=zlib.decompress(fp.read(PREAMBLE_COMPRESSED_LEN)) - fp.close() + C=''.encode() + while PREAMBLE_COMPRESSED_LEN-len(C)and select.select([0],[],[]):C+=os.read(0,PREAMBLE_COMPRESSED_LEN-len(C)) + C=zlib.decompress(C) fp=os.fdopen(W,'wb',0) fp.write(C) fp.close() @@ -1478,11 +1478,12 @@ class Connection(object): # Just enough to decode, decompress, and exec the first stage. # Priorities: wider compatibility, faster startup, shorter length. - # `import os` here, instead of stage 1, to save a few bytes. # `sys.path=...` for https://github.com/python/cpython/issues/115911. + # `import os,select` here (not stage 1) to save a few bytes overall. return self.get_python_argv() + [ '-c', - 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;' + 'import sys;sys.path=[p for p in sys.path if p];' + 'import binascii,os,select,zlib;' 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),), ] diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index ad7165b3..e06f453f 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,5 +1,6 @@ import subprocess +import mitogen.core import mitogen.parent from mitogen.core import b @@ -21,14 +22,18 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): conn.context = mitogen.core.Context(None, 123) args = conn.get_boot_command() - # Executing the boot command will print "EC0" and expect to read from - # stdin, which will fail because it's pointing at /dev/null, causing - # the forked child to crash with an EOFError and disconnect its write - # pipe. The forked and freshly execed parent will get a 0-byte read - # from the pipe, which is a valid script, and therefore exit indicating - # success. + # The boot command should write an ECO marker to stdout, read the + # preamble from stdin, then execute it. - fp = open("/dev/null", "r") + # This test attaches /dev/zero to stdin to create a specific failure + # 1. Fork child reads PREAMBLE_COMPRESSED_LEN bytes of junk (all `\0`) + # 2. Fork child crashes (trying to decompress the junk data) + # 3. Fork child's file descriptors (write pipes) are closed by the OS + # 4. Fork parent does `dup(, )` and `exec()` + # 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) + # 6. Python runs `''` (a valid script) and exits with success + + fp = open("/dev/zero", "r") try: proc = subprocess.Popen(args, stdin=fp, @@ -39,6 +44,9 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(0, proc.returncode) self.assertEqual(stdout, mitogen.parent.BootstrapProtocol.EC0_MARKER+b('\n')) - self.assertIn(b("Error -5 while decompressing data"), stderr) + self.assertIn( + b("Error -3 while decompressing data"), # Unknown compression method + stderr, + ) finally: fp.close() diff --git a/tests/image_prep/files/sudoers_defaults b/tests/image_prep/files/sudoers_defaults index 3ad7a6d4..8090fe24 100644 --- a/tests/image_prep/files/sudoers_defaults +++ b/tests/image_prep/files/sudoers_defaults @@ -1,3 +1,7 @@ +# Testing non-blocking stdio during bootstrap +# https://github.com/mitogen-hq/mitogen/issues/1306 +Defaults log_output + Defaults>mitogen__pw_required targetpw Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw From 59d5d74abd03b4caa7e5ec9c9e6662d7546e2a28 Mon Sep 17 00:00:00 2001 From: Jarl Gullberg Date: Sat, 13 Sep 2025 21:48:03 +0200 Subject: [PATCH 11/13] Add FreeIPA client modules to the always-fork list. --- ansible_mitogen/planner.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 5d0fbd78..b4c9d9b8 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -347,6 +347,25 @@ class NewStylePlanner(ScriptPlanner): 'freeipa.ansible_freeipa.ipaautomountlocation', 'freeipa.ansible_freeipa.ipaautomountmap', 'freeipa.ansible_freeipa.ipacert', + 'freeipa.ansible_freeipa.ipaclient_api', + 'freeipa.ansible_freeipa.ipaclient_fix_ca', + 'freeipa.ansible_freeipa.ipaclient_fstore', + 'freeipa.ansible_freeipa.ipaclient_get_otp', + 'freeipa.ansible_freeipa.ipaclient_ipa_conf', + 'freeipa.ansible_freeipa.ipaclient_join', + 'freeipa.ansible_freeipa.ipaclient_set_hostname', + 'freeipa.ansible_freeipa.ipaclient_setup_automount', + 'freeipa.ansible_freeipa.ipaclient_setup_certmonger', + 'freeipa.ansible_freeipa.ipaclient_setup_firefox', + 'freeipa.ansible_freeipa.ipaclient_setup_krb5', + 'freeipa.ansible_freeipa.ipaclient_setup_nis', + 'freeipa.ansible_freeipa.ipaclient_setup_nss', + 'freeipa.ansible_freeipa.ipaclient_setup_ntp', + 'freeipa.ansible_freeipa.ipaclient_setup_ssh', + 'freeipa.ansible_freeipa.ipaclient_setup_sshd', + 'freeipa.ansible_freeipa.ipaclient_temp_krb5', + 'freeipa.ansible_freeipa.ipaclient_test', + 'freeipa.ansible_freeipa.ipaclient_test_keytab', 'freeipa.ansible_freeipa.ipaconfig', 'freeipa.ansible_freeipa.ipadelegation', 'freeipa.ansible_freeipa.ipadnsconfig', From 2736f38c4babf3c83c7b0f41a07fb0a5612953c1 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 17 Sep 2025 10:46:28 +0100 Subject: [PATCH 12/13] docs: Changelog for FreeIPA client modules -> ALWAYS_FORK_MODULES --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 99378e1a..f99f220a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,8 @@ In progress (unreleased) * :gh:issue:`1306` preamble_size: Fix variability of measured command size * :gh:issue:`1306` tests: Count bytes written in ``stdio_test.StdIOTest`` * :gh:issue:`1306` tests: Check stdio is blocking in sudo contexts +* :gh:issue:`1327` :mod:`ansible_mitogen`: Add FreeIPA client modules to the + always-fork list v0.3.27 (2025-08-20) From 9f9b37d1ada998ad8d6e329d1956eb7f4f4a7658 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 17 Sep 2025 11:04:10 +0100 Subject: [PATCH 13/13] Prepare v0.3.28 --- docs/changelog.rst | 4 ++-- mitogen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f99f220a..600206f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -In progress (unreleased) ------------------------- +v0.3.28 (2025-09-17) +-------------------- * :gh:issue:`1306` :mod:`ansible_mitogen`: Fix non-blocking IO errors in first stage of bootstrap diff --git a/mitogen/__init__.py b/mitogen/__init__.py index cc3cd425..76c83cab 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 28, 'dev') +__version__ = (0, 3, 28) #: This is :data:`False` in slave contexts. Previously it was used to prevent