From 4b39013ef43bd22791bbab870d731e26e7c9b6aa Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 10 Jan 2021 21:00:06 +0000 Subject: [PATCH 1/4] tests: Compatiblity shim for threading.Thread.is_alive() On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9. On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6). --- tests/testlib.py | 16 +++++++++++++++- tests/utils_test.py | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/testlib.py b/tests/testlib.py index ace8f0a2..ee76a26d 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -103,6 +103,18 @@ if hasattr(subprocess.Popen, 'terminate'): Popen__terminate = subprocess.Popen.terminate +def threading__thread_is_alive(thread): + """Return whether the thread is alive (Python version compatibility shim). + + On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9). + On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6). + """ + try: + return thread.is_alive() + except AttributeError: + return thread.isAlive() + + def wait_for_port( host, port, @@ -334,7 +346,9 @@ class TestCase(unittest2.TestCase): for thread in threading.enumerate(): name = thread.getName() # Python 2.4: enumerate() may return stopped threads. - assert (not thread.isAlive()) or name in self.ALLOWED_THREADS, \ + assert \ + not threading__thread_is_alive(thread) \ + or name in self.ALLOWED_THREADS, \ 'Found thread %r still running after tests.' % (name,) counts[name] = counts.get(name, 0) + 1 diff --git a/tests/utils_test.py b/tests/utils_test.py index a70b23dc..b5204a3c 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -31,14 +31,14 @@ class RunWithRouterTest(testlib.TestCase): def test_run_with_broker(self): router = mitogen.utils.run_with_router(func0) self.assertIsInstance(router, mitogen.master.Router) - self.assertFalse(router.broker._thread.isAlive()) + self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread)) class WithRouterTest(testlib.TestCase): def test_with_broker(self): router = func() self.assertIsInstance(router, mitogen.master.Router) - self.assertFalse(router.broker._thread.isAlive()) + self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread)) class Dict(dict): pass From 1e72ebaf8bb1d69a4abeb7fea9cac09325681805 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Jan 2021 19:34:44 +0000 Subject: [PATCH 2/4] tests: Update test coverage to Python 3.9 The Travis Linux distribution must be upgraded because the Trusty (Ubuntu 14.04) image does not have Python 3.9. Xenial (Ubuntu 16.04) is the earliest version that offers Python 3.9. I have not chosen a later release, in order to aid restoration of Python 2.4 - 2.6 tests. --- .ci/azure-pipelines.yml | 11 +++++++++++ .travis.yml | 8 ++++++-- docs/changelog.rst | 1 + tests/ansible/requirements.txt | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 397c52f3..5f687be4 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -66,6 +66,12 @@ jobs: DISTRO: debian VER: 2.10.0 + Mito39Debian_27: + python.version: '3.9' + MODE: mitogen + DISTRO: debian + VER: 2.10.0 + #Py26CentOS7: #python.version: '2.7' #MODE: mitogen @@ -117,3 +123,8 @@ jobs: python.version: '3.5' MODE: ansible VER: 2.10.0 + + Ansible_210_39: + python.version: '3.9' + MODE: ansible + VER: 2.10.0 diff --git a/.travis.yml b/.travis.yml index aafb4413..7ee98677 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: trusty +dist: xenial # Ubuntu 16.04 LTS notifications: email: false @@ -17,7 +17,6 @@ cache: - /home/travis/virtualenv install: -- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v - pip install -U pip==20.2.1 - .ci/${MODE}_install.py @@ -53,6 +52,9 @@ matrix: - python: "3.6" env: MODE=ansible VER=2.10.0 # 2.10 -> {debian, centos6, centos7} + - python: "3.9" + env: MODE=ansible VER=2.10.0 + # 2.10 -> {debian, centos6, centos7} - python: "2.7" env: MODE=ansible VER=2.10.0 # 2.10 -> {debian, centos6, centos7} @@ -73,6 +75,8 @@ matrix: #env: MODE=mitogen DISTRO=centos6 - python: "3.6" env: MODE=mitogen DISTROS=centos7 VER=2.10.0 + - python: "3.9" + env: MODE=mitogen DISTROS=centos7 VER=2.10.0 # 2.6 -> 2.7 # - python: "2.6" # env: MODE=mitogen DISTROS=centos7 VER=2.10.0 diff --git a/docs/changelog.rst b/docs/changelog.rst index 4cb8d6fe..48fbb083 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -40,6 +40,7 @@ v0.2.10 (unreleased) timeout, when using recent OpenSSH client versions. * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` +* :gh:issue:`775` Test with Python 3.9 v0.2.9 (2019-11-02) diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index c0386cd8..2c3c87c8 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,4 @@ paramiko==2.3.2 # Last 2.6-compat version. hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' -PyYAML==3.13; python_version >= '2.7' +PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) From ceb0a9446728a7095e2f4ff37159d29cc55f1cad Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 16 Jan 2021 13:17:48 +0000 Subject: [PATCH 3/4] Explain why the Python 2.x thread module is blacklisted Based on the original commit I believe it is only an optimization. However I could be wrong. I intend to request review of this part. --- mitogen/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mitogen/core.py b/mitogen/core.py index 4dd44925..4dcc3d77 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1269,6 +1269,9 @@ class Importer(object): # a negative round-trip. 'builtins', '__builtin__', + + # Python 2.x module that was renamed to _thread in 3.x. + # This entry avoids a roundtrip on 2.x -> 3.x. 'thread', # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but From bce3bab3e866229778e975ba0539111a9f2362af Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 16 Jan 2021 13:24:37 +0000 Subject: [PATCH 4/4] Add the msvcrt moduleto the default module deny list Commit https://github.com/python/cpython/commit/880d42a3b247 (first released in Python 3.8a0) moved an import of msvcrt from an if block, into a try/except block. So now the import is tried even on Linux or MacOS. https://docs.python.org/3/library/msvcrt.html is a Windows specific builtin. --- docs/changelog.rst | 1 + mitogen/core.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48fbb083..99c30798 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -41,6 +41,7 @@ v0.2.10 (unreleased) * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` * :gh:issue:`775` Test with Python 3.9 +* :gh:issue:`775` Add msvcrt to the default module deny list v0.2.9 (2019-11-02) diff --git a/mitogen/core.py b/mitogen/core.py index 4dcc3d77..9a00ce65 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1270,6 +1270,10 @@ class Importer(object): 'builtins', '__builtin__', + # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries + # to import of this Windows-only builtin module. + 'msvcrt', + # Python 2.x module that was renamed to _thread in 3.x. # This entry avoids a roundtrip on 2.x -> 3.x. 'thread',