Merge pull request #1270 from moreati/stable

Release 0.3.24

.github/workflows/tests.yml

@@ -4,7 +4,7 @@ name: Tests
 
 # env:
 #  ANSIBLE_VERBOSITY: 3
-#   MITOGEN_LOG_LEVEL: DEBUG
+#  MITOGEN_LOG_LEVEL: DEBUG
 
 on:
   pull_request:
@@ -19,6 +19,7 @@ jobs:
     name: u2204 ${{ matrix.tox_env }}
     # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
     runs-on: ubuntu-22.04
+    timeout-minutes: 25
 
     strategy:
       fail-fast: false
@@ -140,6 +141,7 @@ jobs:
     name: u2404 ${{ matrix.tox_env }}
     # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md
     runs-on: ubuntu-24.04
+    timeout-minutes: 25
 
     strategy:
       fail-fast: false

ansible_mitogen/target.py

@@ -42,6 +42,7 @@ import json
 import logging
 import operator
 import os
+import pty
 import pwd
 import re
 import signal
@@ -120,7 +121,7 @@ def subprocess__Popen__close_fds(self, but):
             continue
 
         fd = int(name, 10)
-        if fd > 2 and fd != but:
+        if fd > pty.STDERR_FILENO and fd != but:
             try:
                 os.close(fd)
             except OSError:

docs/changelog.rst

@@ -18,6 +18,13 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+v0.3.24 (2025-05-29)
+--------------------
+
+* :gh:issue:`1268` :mod:`mitogen` Only close stdin, stdout, and stderr file
+  descriptors (0, 1, and 2) if they were open at process startup.
+
+
 v0.3.23 (2025-04-28)
 --------------------
 

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 23)
+__version__ = (0, 3, 24)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/core.py

@@ -74,6 +74,7 @@ import logging
 import os
 import pickle as py_pickle
 import pstats
+import pty
 import signal
 import socket
 import struct
@@ -544,8 +545,17 @@ def set_cloexec(fd):
     they must be explicitly closed through some other means, such as
     :func:`mitogen.fork.on_fork`.
     """
+    stdfds = [
+        stdfd
+        for stdio, stdfd in [
+            (sys.stdin, pty.STDIN_FILENO),
+            (sys.stdout, pty.STDOUT_FILENO),
+            (sys.stderr, pty.STDERR_FILENO),
+        ]
+        if stdio is not None and not stdio.closed
+    ]
+    assert fd not in stdfds, 'fd %r is one of the stdio fds: %r' % (fd, stdfds)
     flags = fcntl.fcntl(fd, fcntl.F_GETFD)
-    assert fd > 2, 'fd %r <= 2' % (fd,)
     fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
 
 
@@ -4019,7 +4029,9 @@ class ExternalContext(object):
         in_fp = os.fdopen(os.dup(in_fd), 'rb', 0)
         os.close(in_fd)
 
-        out_fp = os.fdopen(os.dup(self.config.get('out_fd', 1)), 'wb', 0)
+        out_fd = self.config.get('out_fd', pty.STDOUT_FILENO)
+        out_fd2 = os.dup(out_fd)
+        out_fp = os.fdopen(out_fd2, 'wb', 0)
         self.stream = MitogenProtocol.build_stream(
             self.router,
             parent_id,
@@ -4103,7 +4115,13 @@ class ExternalContext(object):
         Open /dev/null to replace stdio temporarily. In case of odd startup,
         assume we may be allocated a standard handle.
         """
-        for stdfd, mode in ((0, os.O_RDONLY), (1, os.O_RDWR), (2, os.O_RDWR)):
+        for stdio, stdfd, mode in [
+                (sys.stdin, pty.STDIN_FILENO, os.O_RDONLY),
+                (sys.stdout, pty.STDOUT_FILENO, os.O_RDWR),
+                (sys.stderr, pty.STDERR_FILENO, os.O_RDWR),
+        ]:
+            if stdio is None:
+                continue
             fd = os.open('/dev/null', mode)
             if fd != stdfd:
                 os.dup2(fd, stdfd)
@@ -4119,8 +4137,9 @@ class ExternalContext(object):
         avoid receiving SIGHUP.
         """
         try:
-            if os.isatty(2):
+            if os.isatty(pty.STDERR_FILENO):
-                self.reserve_tty_fp = os.fdopen(os.dup(2), 'r+b', 0)
+                reserve_tty_fd = os.dup(pty.STDERR_FILENO)
+                self.reserve_tty_fp = os.fdopen(reserve_tty_fd, 'r+b', 0)
                 set_cloexec(self.reserve_tty_fp.fileno())
         except OSError:
             pass
@@ -4140,13 +4159,18 @@ class ExternalContext(object):
         self._nullify_stdio()
 
         self.loggers = []
-        for name, fd in (('stdout', 1), ('stderr', 2)):
+        for stdio, stdfd, name in [
-            log = IoLoggerProtocol.build_stream(name, fd)
+                (sys.stdout, pty.STDOUT_FILENO, 'stdout'),
+                (sys.stderr, pty.STDERR_FILENO, 'stderr'),
+        ]:
+            if stdio is None:
+                continue
+            log = IoLoggerProtocol.build_stream(name, stdfd)
             self.broker.start_receive(log)
             self.loggers.append(log)
 
         # Reopen with line buffering.
-        sys.stdout = os.fdopen(1, 'w', 1)
+        sys.stdout = os.fdopen(pty.STDOUT_FILENO, 'w', 1)
 
     def main(self):
         self._setup_master()

mitogen/fakessh.py

@@ -95,6 +95,7 @@ Sequence:
 import getopt
 import inspect
 import os
+import pty
 import shutil
 import socket
 import subprocess
@@ -354,8 +355,9 @@ def _fakessh_main(dest_context_id, econtext):
               control_handle, stdin_handle)
 
     process = Process(econtext.router,
-                      stdin=os.fdopen(1, 'w+b', 0),
+        stdin=os.fdopen(pty.STDOUT_FILENO, 'w+b', 0),
-                      stdout=os.fdopen(0, 'r+b', 0))
+        stdout=os.fdopen(pty.STDIN_FILENO, 'r+b', 0),
+    )
     process.start_master(
         stdin=mitogen.core.Sender(dest, stdin_handle),
         control=mitogen.core.Sender(dest, control_handle),

mitogen/parent.py

@@ -43,6 +43,7 @@ import inspect
 import logging
 import os
 import re
+import pty
 import signal
 import socket
 import struct
@@ -361,13 +362,13 @@ def create_child(args, merge_stdio=False, stderr_pipe=False,
         escalates_privilege=escalates_privilege
     )
 
-    stderr = None
-    stderr_r = None
     if merge_stdio:
-        stderr = child_wfp
+        stderr_r, stderr = None, child_wfp
     elif stderr_pipe:
         stderr_r, stderr = mitogen.core.pipe()
         mitogen.core.set_cloexec(stderr_r.fileno())
+    else:
+        stderr_r, stderr = None, None
 
     try:
         proc = popen(
@@ -406,12 +407,14 @@ def _acquire_controlling_tty():
     if sys.platform in ('linux', 'linux2'):
         # On Linux, the controlling tty becomes the first tty opened by a
         # process lacking any prior tty.
-        os.close(os.open(os.ttyname(2), os.O_RDWR))
+        tty_path = os.ttyname(pty.STDERR_FILENO)
+        tty_fd = os.open(tty_path, os.O_RDWR)
+        os.close(tty_fd)
     if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL and not IS_SOLARIS:
         # #550: prehistoric WSL does not like TIOCSCTTY.
         # On BSD an explicit ioctl is required. For some inexplicable reason,
         # Python 2.6 on Travis also requires it.
-        fcntl.ioctl(2, termios.TIOCSCTTY)
+        fcntl.ioctl(pty.STDERR_FILENO, termios.TIOCSCTTY)
 
 
 def _linux_broken_devpts_openpty():
@@ -1415,7 +1418,7 @@ class Connection(object):
     #   w: write side of core_src FD.
     #   C: the decompressed core source.
 
-    # Final os.close(2) to avoid --py-debug build from corrupting stream with
+    # Final os.close(STDOUT_FILENO) to avoid --py-debug build corrupting stream with
     # "[1234 refs]" during exit.
     @staticmethod
     def _first_stage():
@@ -1683,9 +1686,7 @@ class Connection(object):
 
         LOG.debug('child for %r started: pid:%r stdin:%r stdout:%r stderr:%r',
                   self, self.proc.pid,
-                  self.proc.stdin.fileno(),
+                  self.proc.stdin, self.proc.stdout, self.proc.stderr)
-                  self.proc.stdout.fileno(),
-                  self.proc.stderr and self.proc.stderr.fileno())
 
         self.stdio_stream = self._setup_stdio_stream()
         if self.context.name is None:

tests/ansible/hosts/group_vars/all.yml

@@ -12,12 +12,12 @@ become_unpriv_available: >-
   {{-
     (
       not is_mitogen
-      and ansible_facts.distribution in ["MacOSX"]
+      and is_macos_controller
       and ansible_version.full is version("2.11", ">=", strict=True)
     )
     or (
       is_mitogen
-      and not ansible_facts.distribution in ["MacOSX"]
+      and not is_macos_controller
     )
     or (
       is_mitogen

tests/ansible/integration/async/multiple_items_loop.yml

@@ -19,7 +19,7 @@
       jid: "{{ item.ansible_job_id }}"
     become: yes
     register: out
-    until: out.finished
+    until: out is finished
     retries: 30
     with_items:
       - "{{ jobs.results }}"

tests/ansible/integration/async/result_binary_producing_json.yml

@@ -34,7 +34,7 @@
     async_status:
       jid: "{{job.ansible_job_id}}"
     register: result
-    until: result.finished
+    until: result is finished
     retries: 100000
     delay: 0
 

tests/ansible/integration/async/runner_new_process.yml

@@ -30,7 +30,7 @@
     async_status:
       jid: "{{async_proc1.ansible_job_id}}"
     register: async_result1
-    until: async_result1.finished
+    until: async_result1 is finished
     retries: 100000
     delay: 0
 
@@ -44,7 +44,7 @@
     async_status:
       jid: "{{async_proc2.ansible_job_id}}"
     register: async_result2
-    until: async_result2.finished
+    until: async_result2 is finished
     retries: 100000
     delay: 0
 

tests/ansible/integration/async/runner_one_job.yml

@@ -30,7 +30,7 @@
     async_status:
       jid: "{{job1.ansible_job_id}}"
     register: result1
-    until: result1.finished
+    until: result1 is finished
     retries: 100000
     delay: 0
 

tests/ansible/integration/async/runner_timeout_then_polling.yml

@@ -19,7 +19,7 @@
     async_status:
       jid: "{{job.ansible_job_id}}"
     register: result
-    until: result.finished
+    until: result is finished
     retries: 500
     delay: 0
     ignore_errors: true

tests/ansible/integration/async/runner_two_simultaneous_jobs.yml

@@ -40,7 +40,7 @@
     async_status:
       jid: "{{job1.ansible_job_id}}"
     register: result1
-    until: result1.finished
+    until: result1 is finished
     retries: 5
     delay: 1
 
@@ -48,7 +48,7 @@
     async_status:
       jid: "{{job2.ansible_job_id}}"
     register: result2
-    until: result2.finished
+    until: result2 is finished
     retries: 5
     delay: 1
 

tests/ansible/integration/connection/disconnect_during_module.yml

@@ -13,6 +13,7 @@
     vars:
       ansible_python_interpreter: "{{ ansible_playbook_python }}"
     environment:
+      ANSIBLE_STRATEGY:  "{{ lookup('env', 'ANSIBLE_STRATEGY') | mandatory }}"
       ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
     command: |
       ansible-playbook

tests/ansible/integration/connection_delegation/stack_construction.yml

@@ -24,10 +24,12 @@
     - local_action: custom_python_detect_environment
       register: local_env
   tags:
+    - mitogen_only
     - stack_construction
 
 
-- hosts: cd-normal
+- name: stack_construction.yml, cd-normal
+  hosts: cd-normal
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - mitogen_get_stack:
@@ -52,7 +54,8 @@
     - stack_construction
 
 
-- hosts: cd-normal
+- name: stack_construction.yml, cd-normal, delegate_to=cd-alias
+  hosts: cd-normal
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: ../_expected_ssh_port.yml
@@ -96,7 +99,8 @@
     - stack_construction
 
 
-- hosts: cd-alias
+- name: stack_construction.yml, cd-alias
+  hosts: cd-alias
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: ../_expected_ssh_port.yml
@@ -139,7 +143,8 @@
     - stack_construction
 
 
-- hosts: cd-normal-normal
+- name: stack_construction.yml, cd-normal-normal
+  hosts: cd-normal-normal
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: ../_expected_ssh_port.yml
@@ -193,7 +198,8 @@
     - stack_construction
 
 
-- hosts: cd-normal-alias
+- name: stack_construction.yml, cd-normal-alias
+  hosts: cd-normal-alias
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: ../_expected_ssh_port.yml
@@ -263,7 +269,8 @@
     - stack_construction
 
 
-- hosts: cd-newuser-normal-normal
+- name: stack_construction.yml, cd-newuser-normal-normal
+  hosts: cd-newuser-normal-normal
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: ../_expected_ssh_port.yml
@@ -317,7 +324,8 @@
     - stack_construction
 
 
-- hosts: cd-newuser-normal-normal
+- name: stack_construction.yml, cd-newuser-normal-normal, delegate_to=cd-alias
+  hosts: cd-newuser-normal-normal
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: ../_expected_ssh_port.yml
@@ -361,7 +369,8 @@
     - stack_construction
 
 
-- hosts: cd-newuser-normal-normal
+- name: stack_construction.yml, cd-newuser-normal-normal, local_action
+  hosts: cd-newuser-normal-normal
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - local_action: mitogen_get_stack
@@ -381,7 +390,8 @@
     - stack_construction
 
 
-- hosts: cd-newuser-doas-normal
+- name: stack_construction.yml, cd-newuser-doas-normal
+  hosts: cd-newuser-doas-normal
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - mitogen_get_stack:

tests/ansible/integration/runner/crashy_new_style_module.yml

@@ -16,7 +16,7 @@
       assert:
         that:
         - not out.changed
-        - out.rc == 1
+        - out is failed
         # https://github.com/ansible/ansible/commit/62d8c8fde6a76d9c567ded381e9b34dad69afcd6
         - out.msg is match(msg_pattern)
         - (out.module_stdout == "" and out.module_stderr is search(tb_pattern))

tests/ansible/integration/runner/missing_module.yml

@@ -5,6 +5,7 @@
     - name: Run missing_module
       connection: local
       environment:
+        ANSIBLE_STRATEGY:  "{{ lookup('env', 'ANSIBLE_STRATEGY') | mandatory }}"
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       vars:
         ansible_python_interpreter: "{{ ansible_playbook_python }}"

tests/ansible/integration/ssh/timeouts.yml

@@ -14,6 +14,7 @@
       connection: local
       environment:
         ANSIBLE_SSH_TIMEOUT: 10
+        ANSIBLE_STRATEGY:  "{{ lookup('env', 'ANSIBLE_STRATEGY') | mandatory }}"
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       vars:
         ansible_python_interpreter: "{{ ansible_playbook_python }}"

tests/ansible/integration/ssh/variables.yml

@@ -14,6 +14,8 @@
       -o "ControlPath /tmp/mitogen-ansible-test-{{18446744073709551615|random}}"
 
   tasks:
+    - include_tasks: ../_mitogen_only.yml
+
     - name: ansible_user, ansible_ssh_private_key_file
       shell: >
         ANSIBLE_ANY_ERRORS_FATAL=false

tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml

@@ -2,6 +2,8 @@
 - name: integration/strategy/mixed_vanilla_mitogen.yml (linear->mitogen->linear)
   hosts: test-targets[0]
   tasks:
+    - include_tasks: ../_mitogen_only.yml
+
     - connection: local
       environment:
         ANSIBLE_PYTHON_INTERPRETER: "{{ ansible_playbook_python }}"

tests/ansible/integration/stub_connections/setns_lxc.yml

@@ -15,6 +15,7 @@
 
     - name: Run stub-lxc-info.py
       environment:
+        ANSIBLE_STRATEGY:  "{{ lookup('env', 'ANSIBLE_STRATEGY') | mandatory }}"
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible"

tests/ansible/integration/stub_connections/setns_lxd.yml

@@ -15,6 +15,7 @@
 
     - name: Run ansible stub-lxc.py
       environment:
+        ANSIBLE_STRATEGY:  "{{ lookup('env', 'ANSIBLE_STRATEGY') | mandatory }}"
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible"

tests/ansible/regression/become_test.yml

@@ -2,7 +2,6 @@
   hosts: test-targets:&linux_containers
   become: true
   become_user: mitogen__pw_required
-  strategy: mitogen_linear
   tasks:
     - command: whoami
       changed_when: false

tests/ansible/regression/issue_1087__template_streamerror.yml

@@ -20,6 +20,7 @@
           vars:
             ansible_python_interpreter: "{{ ansible_playbook_python }}"
           environment:
+            ANSIBLE_STRATEGY:  "{{ lookup('env', 'ANSIBLE_STRATEGY') | mandatory }}"
             ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
           command:
             cmd: >

tests/ansible/regression/issue_766__get_with_context.yml

@@ -1,4 +1,4 @@
-# https://github.com/mitogen-hq/mitogen/issues/776
+# https://github.com/mitogen-hq/mitogen/issues/766
 ---
 - name: regression/issue_766__get_with_context.yml
   hosts: localhost
@@ -7,7 +7,7 @@
   gather_facts: true
   vars:
     netconf_container_image: ghcr.io/mitogen-hq/sysrepo-netopeer2:latest
-    netconf_container_name: sysprep
+    netconf_container_name: sysrepo
     netconf_container_port: 8030
 
   tasks:

tests/ansible/regression/issue_952__ask_become_pass.yml

@@ -9,6 +9,7 @@
       vars:
         ansible_python_interpreter: "{{ ansible_playbook_python }}"
       environment:
+        ANSIBLE_STRATEGY:  "{{ lookup('env', 'ANSIBLE_STRATEGY') | mandatory }}"
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       expect:
         command: >

tests/ansible/run_ansible_playbook.py

@@ -2,6 +2,7 @@
 # Wrap ansible-playbook, setting up some test of the test environment.
 import json
 import os
+import platform
 import sys
 
 GIT_BASEDIR = os.path.dirname(
@@ -36,6 +37,7 @@ os.environ['PATH'] = '%s%s%s' % (
 )
 
 extra = {
+    'is_macos_controller': platform.system() == 'Darwin',
     'is_mitogen': os.environ.get('ANSIBLE_STRATEGY', '').startswith('mitogen'),
     'git_basedir': GIT_BASEDIR,
 }