From e8af2c239c467f94e3e8b42ed00949c6c6d652b6 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Mon, 18 Jan 2021 13:46:31 -0800 Subject: [PATCH 01/19] fix 0.3.0-rc.0 version reporting --- mitogen/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/__init__.py b/mitogen/__init__.py index f18c5a90..21a9eda7 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 2, 9) +__version__ = (0, 3, "0-rc.0") #: This is :data:`False` in slave contexts. Previously it was used to prevent From a64e966ab9cc932e0776219a83eb64f00e5b426d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 18 Jan 2021 00:02:07 +0000 Subject: [PATCH 02/19] tests: Add unofficial Tox environments for Ansible tests These are not part of the official testing regime (tests run for pull requests). I find them convenient for local development. Limitations - Python 2.7+ only. No Python 2.4, 2.5, or 2.6. - Requires Pythons pre-installed (e.g. DeadSnakes, pyenv) - No coverage of alternate controller OS (e.g. MacOS) The environments tested by default are py27-mode_ansible-ansible2.10 py36-mode_ansible-ansible2.10 py39-mode_ansible-ansible2.10 py27-mode_mitogen py36-mode_mitogen py39-mode_mitogen py27-mode_mitogen-distro_centos7 py36-mode_mitogen-distro_centos7 py39-mode_mitogen-distro_centos7 --- .ci/ansible_tests.py | 2 - .../playbook_semantics/delegate_to.yml | 12 +++ ...ue_152__local_action_wrong_interpreter.yml | 2 +- tox.ini | 76 +++++++++++++++---- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index c81f9539..b2aa3199 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -66,8 +66,6 @@ with ci_lib.Fold('job_setup'): run("sudo apt-get update") run("sudo apt-get install -y sshpass") - run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'") - run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'") with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml index 4d4da028..23b7168c 100644 --- a/tests/ansible/integration/playbook_semantics/delegate_to.yml +++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml @@ -51,10 +51,14 @@ shell: whoami > /tmp/delegate_to.yml.txt delegate_to: localhost become: true + tags: + - requires_local_sudo - name: "delegate_to, sudo" assert: that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + tags: + - requires_local_sudo - name: "delegate_to, sudo" file: @@ -62,6 +66,8 @@ state: absent delegate_to: localhost become: true + tags: + - requires_local_sudo # @@ -71,10 +77,14 @@ shell: whoami > /tmp/delegate_to.yml.txt connection: local become: true + tags: + - requires_local_sudo - name: "connection:local, sudo" assert: that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + tags: + - requires_local_sudo - name: "connection:local, sudo" file: @@ -82,3 +92,5 @@ state: absent connection: local become: true + tags: + - requires_local_sudo diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index 5de67ab9..8a8f0068 100644 --- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -15,7 +15,7 @@ content: | #!/bin/bash export CUSTOM_INTERPRETER=1 - exec python2.7 "$@" + exec python "$@" - custom_python_detect_environment: vars: diff --git a/tox.ini b/tox.ini index 8a4ef364..3227672f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,44 +1,88 @@ +# This file is a local convenience. It is not a substitute for the full CI +# suite, and does not cover the full range of Python versions for Mitogen. + +# I use this on Ubuntu 20.04, with the following additions +# +# sudo add-apt-repository ppa:deadsnakes/ppa +# sudo apt update +# sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass + +# Last version to support each python version +# +# tox vir'env pip ansible coverage +# ========== ======== ======== ======== ======== ======== +# python2.4 1.4 1.8 1.1 ??? +# python2.5 1.6.1 1.9.1 1.3.1 ??? +# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 4.5.4 + [tox] envlist = init, - py26, - py27, - py35, - py36, - py37, + py{27,36,39}-mode_ansible-ansible2.10, + py{27,36,39}-mode_mitogen, + py{27,36,39}-mode_mitogen-distro_centos7, report, +requires = + tox-factor [testenv] -usedevelop = True -deps = - -r{toxinidir}/dev_requirements.txt - -r{toxinidir}/tests/ansible/requirements.txt - +basepython = + py26: python2.6 + py27: python2.7 + py36: python3.6 + py37: python3.7 + py38: python3.8 + py39: python3.9 +commands_pre = + mode_ansible: {toxinidir}/.ci/ansible_install.py + mode_debops_common: {toxinidir}/.ci/debops_common_install.py + mode_mitogen: {toxinidir}/.ci/mitogen_install.py commands = - {posargs:bash run_tests} -whitelist_externals = - bash + mode_ansible: {toxinidir}/.ci/ansible_tests.py \ + --skip-tags requires_local_sudo + mode_debops_common: {toxinidir}/.ci/debops_common_tests.py + mode_mitogen: {toxinidir}/.ci/mitogen_tests.py +passenv = + HOME setenv = NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 + ansible2.3: VER=2.3.3.0 + ansible2.4: VER=2.4.6.0 + ansible2.8: VER=2.8.3 + ansible2.9: VER=2.9.6 + ansible2.10: VER=2.10.0 + distro_centos5: DISTRO=centos5 + distro_centos6: DISTRO=centos6 + distro_centos7: DISTRO=centos7 + distro_debian: DISTRO=debian + distro_debianpy3: DISTRO=debian-py3 + distros_centos5: DISTROS=centos5 + distros_debian: DISTROS=debian + mode_ansible: MODE=ansible + mode_debops_common: MODE=debops_common + mode_mitogen: MODE=mitogen + strategy_linear: STRATEGY=linear [testenv:init] +basepython = python3 commands = coverage erase deps = - coverage + coverage==4.5.4 [testenv:report] +basepython = python3 commands = coverage html echo "coverage report is at file://{toxinidir}/htmlcov/index.html" deps = - coverage + coverage==4.5.4 whitelist_externals = echo [testenv:docs] -basepython = python +basepython = python3 changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From bb271d8d004a3ea21fd53347406e1368e4746bc8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 20 Jan 2021 20:58:35 +0000 Subject: [PATCH 03/19] tests: Tag resource intensive tasks, to skip when strategy=linear The ansible_mitogen test suite takes over an hour when Ansible is not accelerated by Mitogen. This change aims to reduce that by skipping tests with a large number of iterations when the linear strategy is chosen. The tagged tests are intended to uncover Mitogen resource leaks. Since Mitogen is not invoked when strategy=linear, the slight reduction in test coverage is an acceptable trade off. --- .ci/azure-pipelines.yml | 1 + tests/ansible/all.yml | 1 + tests/ansible/bench/file_transfer.yml | 3 +++ tests/ansible/bench/includes.yml | 2 ++ tests/ansible/bench/loop-100-copies.yml | 5 +++++ tests/ansible/bench/loop-100-items.yml | 2 ++ tests/ansible/bench/loop-100-tasks.yml | 2 ++ tests/ansible/regression/issue_140__thread_pileup.yml | 5 +++++ tests/ansible/setup/all.yml | 1 + tests/ansible/setup/report.yml | 8 ++++++++ tests/ansible/soak/file_service.yml | 2 ++ 11 files changed, 32 insertions(+) create mode 100644 tests/ansible/setup/all.yml create mode 100644 tests/ansible/setup/report.yml diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index d436f175..397c52f3 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -30,6 +30,7 @@ jobs: MODE: localhost_ansible VER: 2.10.0 STRATEGY: linear + ANSIBLE_SKIP_TAGS: resource_intensive - job: Linux diff --git a/tests/ansible/all.yml b/tests/ansible/all.yml index e074a384..06f3acdb 100644 --- a/tests/ansible/all.yml +++ b/tests/ansible/all.yml @@ -1,3 +1,4 @@ +- include: setup/all.yml - include: regression/all.yml - include: integration/all.yml diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml index 2ca46f1c..f6702f58 100644 --- a/tests/ansible/bench/file_transfer.yml +++ b/tests/ansible/bench/file_transfer.yml @@ -66,3 +66,6 @@ copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + + tags: + - resource_intensive diff --git a/tests/ansible/bench/includes.yml b/tests/ansible/bench/includes.yml index 4f50113a..96079874 100644 --- a/tests/ansible/bench/includes.yml +++ b/tests/ansible/bench/includes.yml @@ -2,3 +2,5 @@ tasks: - include_tasks: _includes.yml with_sequence: start=1 end=1000 + tags: + - resource_intensive diff --git a/tests/ansible/bench/loop-100-copies.yml b/tests/ansible/bench/loop-100-copies.yml index 0f4d3600..e25ae552 100644 --- a/tests/ansible/bench/loop-100-copies.yml +++ b/tests/ansible/bench/loop-100-copies.yml @@ -24,3 +24,8 @@ mode: 0644 with_filetree: /tmp/filetree.in when: item.state == 'file' + loop_control: + label: "/tmp/filetree.out/{{ item.path }}" + + tags: + - resource_intensive diff --git a/tests/ansible/bench/loop-100-items.yml b/tests/ansible/bench/loop-100-items.yml index c071c100..e711301d 100644 --- a/tests/ansible/bench/loop-100-items.yml +++ b/tests/ansible/bench/loop-100-items.yml @@ -8,3 +8,5 @@ tasks: - command: hostname with_sequence: start=1 end="{{end|default(100)}}" + tags: + - resource_intensive diff --git a/tests/ansible/bench/loop-100-tasks.yml b/tests/ansible/bench/loop-100-tasks.yml index bf6e31b8..4a76c4fe 100644 --- a/tests/ansible/bench/loop-100-tasks.yml +++ b/tests/ansible/bench/loop-100-tasks.yml @@ -110,3 +110,5 @@ - command: hostname - command: hostname - command: hostname + tags: + - resource_intensive diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index a9826d23..78d5c7b1 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -29,3 +29,8 @@ mode: 0644 with_filetree: /tmp/filetree.in when: item.state == 'file' + loop_control: + label: "/tmp/filetree.out/{{ item.path }}" + + tags: + - resource_intensive diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml new file mode 100644 index 00000000..c51fa295 --- /dev/null +++ b/tests/ansible/setup/all.yml @@ -0,0 +1 @@ +- include: report.yml diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report.yml new file mode 100644 index 00000000..7ccd049a --- /dev/null +++ b/tests/ansible/setup/report.yml @@ -0,0 +1,8 @@ +- name: Report runtime settings + hosts: localhost + gather_facts: false + tasks: + - debug: {var: ansible_forks} + - debug: {var: ansible_run_tags} + - debug: {var: ansible_skip_tags} + - debug: {var: ansible_version.full} diff --git a/tests/ansible/soak/file_service.yml b/tests/ansible/soak/file_service.yml index 0640233a..65b10b2d 100644 --- a/tests/ansible/soak/file_service.yml +++ b/tests/ansible/soak/file_service.yml @@ -4,3 +4,5 @@ content: "{% for x in range(126977) %}x{% endfor %}" - include: _file_service_loop.yml with_sequence: start=1 end=100 + tags: + - resource_intensive From adbed36dd9fc71fc9ac0cfee32e8dd90d93ac36a Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 21 Jan 2021 19:34:47 +0000 Subject: [PATCH 04/19] tests: Avoid double docker pull on Azure Pipes --- .ci/prep_azure.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index e236e3e7..80dbf485 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -86,12 +86,4 @@ if need_to_fix_psycopg2: batches.append(venv_steps) - -if ci_lib.have_docker(): - batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),)] - for distro in ci_lib.DISTROS - ) - - ci_lib.run_batches(batches) From 1c7e1078092f0581676aa2cc0f83c218084a1a32 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 22 Jan 2021 00:31:19 +0000 Subject: [PATCH 05/19] tests: Use AWS Elastic Container registry for test images This replaces use of Docker Hub, which now rate limits API calls and causing CI builds to fail. refs #791 --- .ci/ci_lib.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index f735f6a1..285d7f19 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -184,7 +184,11 @@ def get_docker_hostname(): def image_for_distro(distro): - return 'mitogen/%s-test' % (distro.partition('-')[0],) + """Return the container image name or path for a test distro name. + + The returned value is suitable for use with `docker pull`. + """ + return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],) def make_containers(name_prefix='', port_offset=0): From 99235071a341d80c71612a7680ae93e166217951 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 22 Jan 2021 23:16:25 +0000 Subject: [PATCH 06/19] tests: docstrings for ci_lib --- .ci/ci_lib.py | 104 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index f735f6a1..bcbe695e 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -84,12 +84,28 @@ if 'TRAVIS_HOME' in os.environ: # ----------------- def _argv(s, *args): + """Interpolate a command line using *args, return an argv style list. + + >>> _argv('git commit -m "Use frobnicate 2.0 (fixes #%d)"', 1234) + ['git', commit', '-m', 'Use frobnicate 2.0 (fixes #1234)'] + """ if args: s %= args return shlex.split(s) def run(s, *args, **kwargs): + """ Run a command, with arguments, and print timing information + + >>> rc = run('echo "%s %s"', 'foo', 'bar') + Running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + foo bar + 0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k + 0inputs+0outputs (0major+71minor)pagefaults 0swaps + Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + >>> rc + 0 + """ argv = ['/usr/bin/time', '--'] + _argv(s, *args) print('Running: %s' % (argv,)) try: @@ -102,12 +118,36 @@ def run(s, *args, **kwargs): return ret -def run_batches(batches): - combine = lambda batch: 'set -x; ' + (' && '.join( +def combine(batch): + """ + >>> combine(['ls -l', 'echo foo']) + 'set -x; ( ls -l; ) && ( echo foo; )' + """ + return 'set -x; ' + (' && '.join( '( %s; )' % (cmd,) for cmd in batch )) + +def run_batches(batches): + """ Run shell commands grouped into batches, showing an execution trace. + + Raise AssertionError if any command has exits with a non-zero status. + + >>> run_batches([['echo foo', 'true']]) + + echo foo + foo + + true + >>> run_batches([['true', 'echo foo'], ['false']]) + + true + + echo foo + foo + + false + Traceback (most recent call last): + File "...", line ..., in + File "...", line ..., in run_batches + AssertionError + """ procs = [ subprocess.Popen(combine(batch), shell=True) for batch in batches @@ -116,12 +156,28 @@ def run_batches(batches): def get_output(s, *args, **kwargs): + """ + Print and run command line s, %-interopolated using *args. Return stdout. + + >>> s = get_output('echo "%s %s"', 'foo', 'bar') + Running: ['echo', 'foo bar'] + >>> s + 'foo bar\n' + """ argv = _argv(s, *args) print('Running: %s' % (argv,)) return subprocess.check_output(argv, **kwargs) def exists_in_path(progname): + """ + Return True if proganme exists in $PATH. + + >>> exists_in_path('echo') + True + >>> exists_in_path('kwyjibo') # Only found in North American cartoons + False + """ return any(os.path.exists(os.path.join(dirname, progname)) for dirname in os.environ['PATH'].split(os.pathsep)) @@ -136,6 +192,18 @@ class TempDir(object): class Fold(object): + """ + Bracket a section of stdout with travis_fold markers. + + This allows the section to be collapsed or expanded in Travis CI web UI. + + >>> with Fold('stage 1'): + ... print('Frobnicate the frobnitz') + ... + travis_fold:start:stage 1 + Frobnicate the frobnitz + travis_fold:end:stage 1 + """ def __init__(self, name): self.name = name @@ -175,6 +243,8 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( ) def get_docker_hostname(): + """Return the hostname where the docker daemon is running. + """ url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -184,10 +254,32 @@ def get_docker_hostname(): def image_for_distro(distro): + """Return the docker image for running `distro` in a container. + + >>> image_for_distro('centos5') + 'mitogen/centos5-test' + >>> image_for_distro('centos5-something_custom') + 'mitogen/centos5-test' + """ return 'mitogen/%s-test' % (distro.partition('-')[0],) def make_containers(name_prefix='', port_offset=0): + """ + >>> import pprint + >>> BASE_PORT=2200; DISTROS=['debian', 'centos6'] + >>> pprint.pprint(make_containers()) + [{'distro': 'debian', + 'hostname': 'localhost', + 'name': 'target-debian-1', + 'port': 2201, + 'python_path': '/usr/bin/python'}, + {'distro': 'centos6', + 'hostname': 'localhost', + 'name': 'target-centos6-2', + 'port': 2202, + 'python_path': '/usr/bin/python'}] + """ docker_hostname = get_docker_hostname() firstbit = lambda s: (s+'-').split('-')[0] secondbit = lambda s: (s+'-').split('-')[1] @@ -260,6 +352,14 @@ def get_interesting_procs(container_name=None): def start_containers(containers): + """Run docker containers in the background, with sshd on specified ports. + + >>> containers = start_containers([ + ... {'distro': 'debian', 'hostname': 'localhost', + ... 'name': 'target-debian-1', 'port': 2201, + ... 'python_path': '/usr/bin/python'}, + ... ]) + """ if os.environ.get('KEEP'): return From 25446cd69814d9c8c10f0d20047af12f3ee47662 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 22 Jan 2021 23:31:21 +0000 Subject: [PATCH 07/19] tests: Throttle docker pulls for Ansible tests I hope this will prevent AWS Elastic Container Registry returning 429: Too Many Requests error, due to the burst of activity from multiple VMs, when a CI build begins. --- .ci/ansible_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 906961db..bb659f8a 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -17,7 +17,7 @@ batches = [ ] batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),)] + ['docker pull %s' % (ci_lib.image_for_distro(distro),), 'sleep 1'] for distro in ci_lib.DISTROS ) From 6b5a46ccd0db58f89d19f5b05fd0f2628b722c44 Mon Sep 17 00:00:00 2001 From: Michael Weigle Date: Tue, 25 Aug 2020 19:03:45 -0400 Subject: [PATCH 08/19] Fixes RuntimeWarning for improper bufferring on local connection --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 4dd44925..5deac358 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -3860,7 +3860,7 @@ class ExternalContext(object): else: core_src_fd = self.config.get('core_src_fd', 101) if core_src_fd: - fp = os.fdopen(core_src_fd, 'rb', 1) + fp = os.fdopen(core_src_fd, 'rb', 0) try: core_src = fp.read() # Strip "ExternalContext.main()" call from last line. From 4b39013ef43bd22791bbab870d731e26e7c9b6aa Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 10 Jan 2021 21:00:06 +0000 Subject: [PATCH 09/19] tests: Compatiblity shim for threading.Thread.is_alive() On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9. On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6). --- tests/testlib.py | 16 +++++++++++++++- tests/utils_test.py | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/testlib.py b/tests/testlib.py index ace8f0a2..ee76a26d 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -103,6 +103,18 @@ if hasattr(subprocess.Popen, 'terminate'): Popen__terminate = subprocess.Popen.terminate +def threading__thread_is_alive(thread): + """Return whether the thread is alive (Python version compatibility shim). + + On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9). + On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6). + """ + try: + return thread.is_alive() + except AttributeError: + return thread.isAlive() + + def wait_for_port( host, port, @@ -334,7 +346,9 @@ class TestCase(unittest2.TestCase): for thread in threading.enumerate(): name = thread.getName() # Python 2.4: enumerate() may return stopped threads. - assert (not thread.isAlive()) or name in self.ALLOWED_THREADS, \ + assert \ + not threading__thread_is_alive(thread) \ + or name in self.ALLOWED_THREADS, \ 'Found thread %r still running after tests.' % (name,) counts[name] = counts.get(name, 0) + 1 diff --git a/tests/utils_test.py b/tests/utils_test.py index a70b23dc..b5204a3c 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -31,14 +31,14 @@ class RunWithRouterTest(testlib.TestCase): def test_run_with_broker(self): router = mitogen.utils.run_with_router(func0) self.assertIsInstance(router, mitogen.master.Router) - self.assertFalse(router.broker._thread.isAlive()) + self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread)) class WithRouterTest(testlib.TestCase): def test_with_broker(self): router = func() self.assertIsInstance(router, mitogen.master.Router) - self.assertFalse(router.broker._thread.isAlive()) + self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread)) class Dict(dict): pass From 1e72ebaf8bb1d69a4abeb7fea9cac09325681805 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 15 Jan 2021 19:34:44 +0000 Subject: [PATCH 10/19] tests: Update test coverage to Python 3.9 The Travis Linux distribution must be upgraded because the Trusty (Ubuntu 14.04) image does not have Python 3.9. Xenial (Ubuntu 16.04) is the earliest version that offers Python 3.9. I have not chosen a later release, in order to aid restoration of Python 2.4 - 2.6 tests. --- .ci/azure-pipelines.yml | 11 +++++++++++ .travis.yml | 8 ++++++-- docs/changelog.rst | 1 + tests/ansible/requirements.txt | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 397c52f3..5f687be4 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -66,6 +66,12 @@ jobs: DISTRO: debian VER: 2.10.0 + Mito39Debian_27: + python.version: '3.9' + MODE: mitogen + DISTRO: debian + VER: 2.10.0 + #Py26CentOS7: #python.version: '2.7' #MODE: mitogen @@ -117,3 +123,8 @@ jobs: python.version: '3.5' MODE: ansible VER: 2.10.0 + + Ansible_210_39: + python.version: '3.9' + MODE: ansible + VER: 2.10.0 diff --git a/.travis.yml b/.travis.yml index aafb4413..7ee98677 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: trusty +dist: xenial # Ubuntu 16.04 LTS notifications: email: false @@ -17,7 +17,6 @@ cache: - /home/travis/virtualenv install: -- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v - pip install -U pip==20.2.1 - .ci/${MODE}_install.py @@ -53,6 +52,9 @@ matrix: - python: "3.6" env: MODE=ansible VER=2.10.0 # 2.10 -> {debian, centos6, centos7} + - python: "3.9" + env: MODE=ansible VER=2.10.0 + # 2.10 -> {debian, centos6, centos7} - python: "2.7" env: MODE=ansible VER=2.10.0 # 2.10 -> {debian, centos6, centos7} @@ -73,6 +75,8 @@ matrix: #env: MODE=mitogen DISTRO=centos6 - python: "3.6" env: MODE=mitogen DISTROS=centos7 VER=2.10.0 + - python: "3.9" + env: MODE=mitogen DISTROS=centos7 VER=2.10.0 # 2.6 -> 2.7 # - python: "2.6" # env: MODE=mitogen DISTROS=centos7 VER=2.10.0 diff --git a/docs/changelog.rst b/docs/changelog.rst index 4cb8d6fe..48fbb083 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -40,6 +40,7 @@ v0.2.10 (unreleased) timeout, when using recent OpenSSH client versions. * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` +* :gh:issue:`775` Test with Python 3.9 v0.2.9 (2019-11-02) diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index c0386cd8..2c3c87c8 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,4 @@ paramiko==2.3.2 # Last 2.6-compat version. hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' -PyYAML==3.13; python_version >= '2.7' +PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) From ceb0a9446728a7095e2f4ff37159d29cc55f1cad Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 16 Jan 2021 13:17:48 +0000 Subject: [PATCH 11/19] Explain why the Python 2.x thread module is blacklisted Based on the original commit I believe it is only an optimization. However I could be wrong. I intend to request review of this part. --- mitogen/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mitogen/core.py b/mitogen/core.py index 4dd44925..4dcc3d77 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1269,6 +1269,9 @@ class Importer(object): # a negative round-trip. 'builtins', '__builtin__', + + # Python 2.x module that was renamed to _thread in 3.x. + # This entry avoids a roundtrip on 2.x -> 3.x. 'thread', # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but From bce3bab3e866229778e975ba0539111a9f2362af Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 16 Jan 2021 13:24:37 +0000 Subject: [PATCH 12/19] Add the msvcrt moduleto the default module deny list Commit https://github.com/python/cpython/commit/880d42a3b247 (first released in Python 3.8a0) moved an import of msvcrt from an if block, into a try/except block. So now the import is tried even on Linux or MacOS. https://docs.python.org/3/library/msvcrt.html is a Windows specific builtin. --- docs/changelog.rst | 1 + mitogen/core.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48fbb083..99c30798 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -41,6 +41,7 @@ v0.2.10 (unreleased) * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` * :gh:issue:`775` Test with Python 3.9 +* :gh:issue:`775` Add msvcrt to the default module deny list v0.2.9 (2019-11-02) diff --git a/mitogen/core.py b/mitogen/core.py index 4dcc3d77..9a00ce65 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1270,6 +1270,10 @@ class Importer(object): 'builtins', '__builtin__', + # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries + # to import of this Windows-only builtin module. + 'msvcrt', + # Python 2.x module that was renamed to _thread in 3.x. # This entry avoids a roundtrip on 2.x -> 3.x. 'thread', From f0ffd1616cd99c4ad99a986cd7e4a75dec2fcf6d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 20:19:56 +0000 Subject: [PATCH 13/19] Declare universal wheel support This mean the package is single source compatible with Python 2.x and 3.x. There is no need to build separate wheels. --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index bf012c6b..08919787 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,6 @@ +[bdist_wheel] +universal=1 + [coverage:run] branch = true source = From 737e71202403ef5e7366c25270d9b09910fc1ec6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 20:20:30 +0000 Subject: [PATCH 14/19] setup: Update project URL --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c3257996..f28d7fb1 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ setup( description = 'Library for writing distributed self-replicating programs.', author = 'David Wilson', license = 'New BSD', - url = 'https://github.com/dw/mitogen/', + url = 'https://github.com/mitogen-hq/mitogen/', packages = find_packages(exclude=['tests', 'examples']), zip_safe = False, classifiers = [ From 0a28549c47e22b03edeb2a1e5a705916c6a71a94 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 20:21:35 +0000 Subject: [PATCH 15/19] setup: Declare supported Python versions (2.4-2.7, 3.6+) --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index f28d7fb1..87601907 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( license = 'New BSD', url = 'https://github.com/mitogen-hq/mitogen/', packages = find_packages(exclude=['tests', 'examples']), + python_requires='>=2.4, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4', zip_safe = False, classifiers = [ 'Environment :: Console', @@ -55,11 +56,16 @@ setup( 'License :: OSI Approved :: BSD License', 'Operating System :: POSIX', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.4', 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', From a6387f69a8f387e6e4a2d552b9d10c96da1e34d0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 20:35:06 +0000 Subject: [PATCH 16/19] setup: Declare MacOS/OSX support --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 87601907..6134b7eb 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( 'Environment :: Console', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', + 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX', 'Programming Language :: Python', 'Programming Language :: Python :: 2', From 28ea0067c66f85804aa263f370afae4d08ec040e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 20:35:49 +0000 Subject: [PATCH 17/19] setup: Add long_description, based on README --- setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.py b/setup.py index 6134b7eb..34a3ba0a 100644 --- a/setup.py +++ b/setup.py @@ -40,10 +40,20 @@ def grep_version(): return '.'.join(map(str, eval(s))) +def long_description(): + here = os.path.dirname(__file__) + readme_path = os.path.join(here, 'README.md') + with open(readme_path) as fp: + readme = fp.read() + return readme + + setup( name = 'mitogen', version = grep_version(), description = 'Library for writing distributed self-replicating programs.', + long_description = long_description(), + long_description_content_type='text/markdown', author = 'David Wilson', license = 'New BSD', url = 'https://github.com/mitogen-hq/mitogen/', From 9b1617f26f03d6b90065bbdb71412c216f3c2e1a Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 20:39:04 +0000 Subject: [PATCH 18/19] Bump version to 0.3.0rc1 --- mitogen/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/__init__.py b/mitogen/__init__.py index f18c5a90..9e709d7d 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 2, 9) +__version__ = (0, 3, 0, 'rc', 1) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/setup.py b/setup.py index 34a3ba0a..bd105147 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def grep_version(): for line in fp: if line.startswith('__version__'): _, _, s = line.partition('=') - return '.'.join(map(str, eval(s))) + return '%i.%i.%i%s%i' % eval(s) def long_description(): From a71a2915b82ccff2d0b0cbfbe33e9e70c98cb9c0 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Sat, 30 Jan 2021 23:39:26 -0800 Subject: [PATCH 19/19] change version to having separate tuple entries for each --- mitogen/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 21a9eda7..3c5b997f 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, "0-rc.0") +__version__ = (0 , 3, 0, 'rc', 0) #: This is :data:`False` in slave contexts. Previously it was used to prevent