From 552819e7658d8a8764a3d4d8976e9ac3ba6c49c6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 23 Nov 2021 23:46:47 +0000 Subject: [PATCH 1/4] mitogen.parent: Detect and avoid Python2.7 wrapper on macOS 11 & 12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this errors such as the following occur ``` ✗ MITOGEN_LOG_LEVEL=DEBUG python3 foo.py Python: execv: (null): No such file or directory Traceback (most recent call last): File "foo.py", line 16, in target = router.local(python_path='/usr/bin/python2.7', debug=True) File "/Users/alex/src/mitogen2/mitogen/parent.py", line 2486, in local return self.connect(u'local', **kwargs) File "/Users/alex/src/mitogen2/mitogen/parent.py", line 2446, in connect return self._connect(klass, **mitogen.core.Kwargs(kwargs)) File "/Users/alex/src/mitogen2/mitogen/parent.py", line 2426, in _connect conn.connect(context=context) File "/Users/alex/src/mitogen2/mitogen/parent.py", line 1708, in connect raise self.exception mitogen.parent.EofError: EOF on stream; last 100 lines received: MITO000 MITO001 ``` Before ``` $ ./preamble_size.py SSH command size: 625 Bootstrap (mitogen.core) size: 17007 (16.61KiB) Original Minimized Compressed mitogen.parent 97496 95.2KiB 50355 49.2KiB 51.6% 12663 12.4KiB 13.0% mitogen.fork 8436 8.2KiB 4130 4.0KiB 49.0% 1648 1.6KiB 19.5% mitogen.ssh 10892 10.6KiB 6952 6.8KiB 63.8% 2113 2.1KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41644 40.7KiB 22431 21.9KiB 53.9% 5886 5.7KiB 14.1% mitogen.fakessh 15599 15.2KiB 8011 7.8KiB 51.4% 2624 2.6KiB 16.8% mitogen.master 48732 47.6KiB 24569 24.0KiB 50.4% 6768 6.6KiB 13.9% ``` After ``` $ ./preamble_size.py SSH command size: 705 Bootstrap (mitogen.core) size: 17007 (16.61KiB) Original Minimized Compressed mitogen.parent 97885 95.6KiB 50516 49.3KiB 51.6% 12728 12.4KiB 13.0% mitogen.fork 8436 8.2KiB 4130 4.0KiB 49.0% 1648 1.6KiB 19.5% mitogen.ssh 10892 10.6KiB 6952 6.8KiB 63.8% 2113 2.1KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41644 40.7KiB 22431 21.9KiB 53.9% 5886 5.7KiB 14.1% mitogen.fakessh 15599 15.2KiB 8011 7.8KiB 51.4% 2624 2.6KiB 16.8% mitogen.master 48733 47.6KiB 24570 24.0KiB 50.4% 6771 6.6KiB 13.9% ``` --- docs/changelog.rst | 1 + mitogen/parent.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 787dd1c4..9165dbda 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file v0.3.1.dev0 (unreleased) ------------------------ +* :gh:issue:`774` Fix bootstrap failures on macOS 11.x and 12.x, involving Python 2.7 wrapper * :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11) * :gh:issue:`869` Continuous Integration tests are now run with Tox * :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04 diff --git a/mitogen/parent.py b/mitogen/parent.py index 44ff9aba..c3efe28a 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1412,9 +1412,12 @@ class Connection(object): # * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version # switcher as /usr/bin/python, which introspects argv0. To workaround # it we redirect attempts to call /usr/bin/python with an explicit - # call to /usr/bin/python2.7. macOS 10.15+ (Darwin 19+) removed it. - # On these versions /usr/bin/python is a symlink to - # /System/Library/Frameworks/Python.framework/Versions/2.7/.../Python + # call to /usr/bin/python2.7. macOS 10.15 (Darwin 19) removed it. + # * macOS 11.x (Darwin 20, Big Sur) and macOS 12.x (Darwin 21, Montery) + # do something slightly different. The Python executable is patched to + # perform an extra execvp(). I don't fully understand the details, but + # setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it. + # * macOS 13.x (Darwin 22?) may remove python 2.x entirely. # # Locals: # R: read side of interpreter stdin. @@ -1437,7 +1440,8 @@ class Connection(object): os.close(r) os.close(W) os.close(w) - if sys.executable+sys.platform=='/usr/bin/pythondarwin'and os.uname()[2]<'19':sys.executable+='2.7' + if os.uname()[0]=='Darwin'and os.uname()[2][:2]<'19'and sys.executable=='/usr/bin/python':sys.executable='/usr/bin/python2.7' + if os.uname()[0]=='Darwin'and os.uname()[2][:2]in'2021'and sys.version[:3]=='2.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1' os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') os.write(1,'MITO000\n'.encode()) From 8276b81b7d9fc649ded73484f9340c86c7c8a719 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 26 Nov 2021 21:51:16 +0000 Subject: [PATCH 2/4] ci: Account for pre-existing children in process leak checks --- tests/testlib.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/tests/testlib.py b/tests/testlib.py index d40ce573..ef35e13e 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,4 +1,5 @@ +import errno import logging import os import random @@ -11,6 +12,7 @@ import threading import time import traceback +import psutil import unittest2 import mitogen.core @@ -67,7 +69,6 @@ def get_fd_count(): """ Return the number of FDs open by this process. """ - import psutil return psutil.Process().num_fds() @@ -334,6 +335,10 @@ class TestCase(unittest2.TestCase): # Broker() instantiations in setUp() etc. mitogen.fork.on_fork() cls._fd_count_before = get_fd_count() + # Ignore children started by external packages - in particular + # multiprocessing.resource_tracker.main()`, started when some Ansible + # versions instantiate a `multithreading.Lock()`. + cls._children_before = frozenset(psutil.Process().children()) super(TestCase, cls).setUpClass() ALLOWED_THREADS = set([ @@ -361,7 +366,10 @@ class TestCase(unittest2.TestCase): def _teardown_check_fds(self): mitogen.core.Latch._on_fork() if get_fd_count() != self._fd_count_before: - import os; os.system('lsof +E -w -p %s | grep -vw mem' % (os.getpid(),)) + if sys.platform == 'linux': + os.system('lsof +E -w -p %i | grep -vw mem' % (os.getpid(),)) + else: + os.system('lsof -w -p %i | grep -vw mem' % (os.getpid(),)) assert 0, "%s leaked FDs. Count before: %s, after: %s" % ( self, self._fd_count_before, get_fd_count(), ) @@ -374,19 +382,33 @@ class TestCase(unittest2.TestCase): if self.no_zombie_check: return + # pid=0: Wait for any child process in the same process group as us. + # WNOHANG: Don't block if no processes ready to report status. try: pid, status = os.waitpid(0, os.WNOHANG) - except OSError: - return # ECHILD + except OSError as e: + # ECHILD: there are no child processes in our group. + if e.errno == errno.ECHILD: + return + raise if pid: assert 0, "%s failed to reap subprocess %d (status %d)." % ( self, pid, status ) - print('') - print('Children of unit test process:') - os.system('ps uww --ppid ' + str(os.getpid())) + children_after = frozenset(psutil.Process().children()) + children_leaked = children_after.difference(self._children_before) + if not children_leaked: + return + + print('Leaked children of unit test process:') + os.system('ps -o "user,pid,%%cpu,%%mem,vsz,rss,tty,stat,start,time,command" -ww -p %s' + % (','.join(str(p.pid) for p in children_leaked),)) + if self._children_before: + print('Pre-existing children of unit test process:') + os.system('ps -o "user,pid,%%cpu,%%mem,vsz,rss,tty,stat,start,time,command" -ww -p %s' + % (','.join(str(p.pid) for p in self._children_before),)) assert 0, "%s leaked still-running subprocesses." % (self,) def tearDown(self): From cadd393cd74d27cb2e86d41b7c69cd7c39501e51 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 1 Dec 2021 21:01:07 +0000 Subject: [PATCH 3/4] ci: Expand macOS coverage to 10.14, latest release on Azure Pipelines 10.14 will be retired on Azure mid December 2021. I want to get so test runs while we can. --- .ci/azure-pipelines-steps.yml | 3 +- .ci/azure-pipelines.yml | 67 ++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index fbf3d8fa..50f1a16b 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -9,11 +9,12 @@ steps: displayName: Install python inputs: versionSpec: '$(python.version)' + condition: ne(variables['python.version'], '') - script: python -mpip install tox displayName: Install tooling -- script: tox -e "$(tox.env)" +- script: python -mtox -e "$(tox.env)" displayName: "Run tests" env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 8045514e..81ce05f2 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -9,8 +9,27 @@ #ANSIBLE_VERBOSITY: 3 jobs: +- job: Mac1014 + # vanilla Ansible is really slow + timeoutInMinutes: 120 + steps: + - template: azure-pipelines-steps.yml + pool: + # https://github.com/actions/virtual-environments/blob/main/images/macos/ + vmImage: macOS-10.14 + strategy: + matrix: + Mito_27: + python.version: '2.7' + tox.env: py27-mode_mitogen + Mito_36: + python.version: '3.6' + tox.env: py36-mode_mitogen + Mito_39: + python.version: '3.9' + tox.env: py39-mode_mitogen -- job: Mac +- job: Mac1015 # vanilla Ansible is really slow timeoutInMinutes: 120 steps: @@ -23,6 +42,12 @@ jobs: Mito_27: python.version: '2.7' tox.env: py27-mode_mitogen + Mito_36: + python.version: '3.6' + tox.env: py36-mode_mitogen + Mito_39: + python.version: '3.9' + tox.env: py39-mode_mitogen # TODO: test python3, python3 tests are broken Loc_27_210: @@ -52,6 +77,46 @@ jobs: STRATEGY: linear ANSIBLE_SKIP_TAGS: resource_intensive +- job: MacLatest + # vanilla Ansible is really slow + timeoutInMinutes: 120 + steps: + - template: azure-pipelines-steps.yml + pool: + # https://github.com/actions/virtual-environments/blob/main/images/macos/ + vmImage: macOS-latest + strategy: + matrix: + Mito_27: + tox.env: py27-mode_mitogen + Mito_37: + python.version: '3.7' + tox.env: py37-mode_mitogen + Mito_39: + python.version: '3.9' + tox.env: py39-mode_mitogen + + # TODO: test python3, python3 tests are broken + Loc_27_210: + tox.env: py27-mode_localhost-ansible2.10 + Loc_27_3: + tox.env: py27-mode_localhost-ansible3 + Loc_27_4: + tox.env: py27-mode_localhost-ansible4 + + # NOTE: this hangs when ran in Ubuntu 18.04 + Van_27_210: + tox.env: py27-mode_localhost-ansible2.10 + STRATEGY: linear + ANSIBLE_SKIP_TAGS: resource_intensive + Van_27_3: + tox.env: py27-mode_localhost-ansible3 + STRATEGY: linear + ANSIBLE_SKIP_TAGS: resource_intensive + Van_27_4: + tox.env: py27-mode_localhost-ansible4 + STRATEGY: linear + ANSIBLE_SKIP_TAGS: resource_intensive - job: Linux pool: From 9a19f0ed16044ee467ab511be3b132d42ed1cae2 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 23 Nov 2021 23:49:00 +0000 Subject: [PATCH 4/4] mitogen.master: Fix stdlib discovery when sys.prefix contains symlinks This can be he case when Python has been installed with Homebrew. --- mitogen/master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/master.py b/mitogen/master.py index e54795cb..806120d6 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -108,7 +108,7 @@ def _stdlib_paths(): ] prefixes = (getattr(sys, a, None) for a in attr_candidates) version = 'python%s.%s' % sys.version_info[0:2] - s = set(os.path.abspath(os.path.join(p, 'lib', version)) + s = set(os.path.realpath(os.path.join(p, 'lib', version)) for p in prefixes if p is not None) # When running 'unit2 tests/module_finder_test.py' in a Py2 venv on Ubuntu