diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 906961db..bb659f8a 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -17,7 +17,7 @@ batches = [ ] batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),)] + ['docker pull %s' % (ci_lib.image_for_distro(distro),), 'sleep 1'] for distro in ci_lib.DISTROS ) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index d436f175..5f687be4 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -30,6 +30,7 @@ jobs: MODE: localhost_ansible VER: 2.10.0 STRATEGY: linear + ANSIBLE_SKIP_TAGS: resource_intensive - job: Linux @@ -65,6 +66,12 @@ jobs: DISTRO: debian VER: 2.10.0 + Mito39Debian_27: + python.version: '3.9' + MODE: mitogen + DISTRO: debian + VER: 2.10.0 + #Py26CentOS7: #python.version: '2.7' #MODE: mitogen @@ -116,3 +123,8 @@ jobs: python.version: '3.5' MODE: ansible VER: 2.10.0 + + Ansible_210_39: + python.version: '3.9' + MODE: ansible + VER: 2.10.0 diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index f735f6a1..98e8bd0a 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -84,12 +84,28 @@ if 'TRAVIS_HOME' in os.environ: # ----------------- def _argv(s, *args): + """Interpolate a command line using *args, return an argv style list. + + >>> _argv('git commit -m "Use frobnicate 2.0 (fixes #%d)"', 1234) + ['git', commit', '-m', 'Use frobnicate 2.0 (fixes #1234)'] + """ if args: s %= args return shlex.split(s) def run(s, *args, **kwargs): + """ Run a command, with arguments, and print timing information + + >>> rc = run('echo "%s %s"', 'foo', 'bar') + Running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + foo bar + 0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k + 0inputs+0outputs (0major+71minor)pagefaults 0swaps + Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + >>> rc + 0 + """ argv = ['/usr/bin/time', '--'] + _argv(s, *args) print('Running: %s' % (argv,)) try: @@ -102,12 +118,36 @@ def run(s, *args, **kwargs): return ret -def run_batches(batches): - combine = lambda batch: 'set -x; ' + (' && '.join( +def combine(batch): + """ + >>> combine(['ls -l', 'echo foo']) + 'set -x; ( ls -l; ) && ( echo foo; )' + """ + return 'set -x; ' + (' && '.join( '( %s; )' % (cmd,) for cmd in batch )) + +def run_batches(batches): + """ Run shell commands grouped into batches, showing an execution trace. + + Raise AssertionError if any command has exits with a non-zero status. + + >>> run_batches([['echo foo', 'true']]) + + echo foo + foo + + true + >>> run_batches([['true', 'echo foo'], ['false']]) + + true + + echo foo + foo + + false + Traceback (most recent call last): + File "...", line ..., in + File "...", line ..., in run_batches + AssertionError + """ procs = [ subprocess.Popen(combine(batch), shell=True) for batch in batches @@ -116,12 +156,28 @@ def run_batches(batches): def get_output(s, *args, **kwargs): + """ + Print and run command line s, %-interopolated using *args. Return stdout. + + >>> s = get_output('echo "%s %s"', 'foo', 'bar') + Running: ['echo', 'foo bar'] + >>> s + 'foo bar\n' + """ argv = _argv(s, *args) print('Running: %s' % (argv,)) return subprocess.check_output(argv, **kwargs) def exists_in_path(progname): + """ + Return True if proganme exists in $PATH. + + >>> exists_in_path('echo') + True + >>> exists_in_path('kwyjibo') # Only found in North American cartoons + False + """ return any(os.path.exists(os.path.join(dirname, progname)) for dirname in os.environ['PATH'].split(os.pathsep)) @@ -136,6 +192,18 @@ class TempDir(object): class Fold(object): + """ + Bracket a section of stdout with travis_fold markers. + + This allows the section to be collapsed or expanded in Travis CI web UI. + + >>> with Fold('stage 1'): + ... print('Frobnicate the frobnitz') + ... + travis_fold:start:stage 1 + Frobnicate the frobnitz + travis_fold:end:stage 1 + """ def __init__(self, name): self.name = name @@ -175,6 +243,8 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( ) def get_docker_hostname(): + """Return the hostname where the docker daemon is running. + """ url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -184,10 +254,34 @@ def get_docker_hostname(): def image_for_distro(distro): - return 'mitogen/%s-test' % (distro.partition('-')[0],) + """Return the container image name or path for a test distro name. + + The returned value is suitable for use with `docker pull`. + + >>> image_for_distro('centos5') + 'public.ecr.aws/n5z0e8q9/centos5-test' + >>> image_for_distro('centos5-something_custom') + 'public.ecr.aws/n5z0e8q9/centos5-test' + """ + return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],) def make_containers(name_prefix='', port_offset=0): + """ + >>> import pprint + >>> BASE_PORT=2200; DISTROS=['debian', 'centos6'] + >>> pprint.pprint(make_containers()) + [{'distro': 'debian', + 'hostname': 'localhost', + 'name': 'target-debian-1', + 'port': 2201, + 'python_path': '/usr/bin/python'}, + {'distro': 'centos6', + 'hostname': 'localhost', + 'name': 'target-centos6-2', + 'port': 2202, + 'python_path': '/usr/bin/python'}] + """ docker_hostname = get_docker_hostname() firstbit = lambda s: (s+'-').split('-')[0] secondbit = lambda s: (s+'-').split('-')[1] @@ -260,6 +354,14 @@ def get_interesting_procs(container_name=None): def start_containers(containers): + """Run docker containers in the background, with sshd on specified ports. + + >>> containers = start_containers([ + ... {'distro': 'debian', 'hostname': 'localhost', + ... 'name': 'target-debian-1', 'port': 2201, + ... 'python_path': '/usr/bin/python'}, + ... ]) + """ if os.environ.get('KEEP'): return diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index e236e3e7..80dbf485 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -86,12 +86,4 @@ if need_to_fix_psycopg2: batches.append(venv_steps) - -if ci_lib.have_docker(): - batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),)] - for distro in ci_lib.DISTROS - ) - - ci_lib.run_batches(batches) diff --git a/.travis.yml b/.travis.yml index aafb4413..7ee98677 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: trusty +dist: xenial # Ubuntu 16.04 LTS notifications: email: false @@ -17,7 +17,6 @@ cache: - /home/travis/virtualenv install: -- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v - pip install -U pip==20.2.1 - .ci/${MODE}_install.py @@ -53,6 +52,9 @@ matrix: - python: "3.6" env: MODE=ansible VER=2.10.0 # 2.10 -> {debian, centos6, centos7} + - python: "3.9" + env: MODE=ansible VER=2.10.0 + # 2.10 -> {debian, centos6, centos7} - python: "2.7" env: MODE=ansible VER=2.10.0 # 2.10 -> {debian, centos6, centos7} @@ -73,6 +75,8 @@ matrix: #env: MODE=mitogen DISTRO=centos6 - python: "3.6" env: MODE=mitogen DISTROS=centos7 VER=2.10.0 + - python: "3.9" + env: MODE=mitogen DISTROS=centos7 VER=2.10.0 # 2.6 -> 2.7 # - python: "2.6" # env: MODE=mitogen DISTROS=centos7 VER=2.10.0 diff --git a/docs/changelog.rst b/docs/changelog.rst index 4cb8d6fe..99c30798 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -40,6 +40,8 @@ v0.2.10 (unreleased) timeout, when using recent OpenSSH client versions. * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` +* :gh:issue:`775` Test with Python 3.9 +* :gh:issue:`775` Add msvcrt to the default module deny list v0.2.9 (2019-11-02) diff --git a/mitogen/core.py b/mitogen/core.py index 4dd44925..802ac45e 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1269,6 +1269,13 @@ class Importer(object): # a negative round-trip. 'builtins', '__builtin__', + + # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries + # to import of this Windows-only builtin module. + 'msvcrt', + + # Python 2.x module that was renamed to _thread in 3.x. + # This entry avoids a roundtrip on 2.x -> 3.x. 'thread', # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but @@ -3860,7 +3867,7 @@ class ExternalContext(object): else: core_src_fd = self.config.get('core_src_fd', 101) if core_src_fd: - fp = os.fdopen(core_src_fd, 'rb', 1) + fp = os.fdopen(core_src_fd, 'rb', 0) try: core_src = fp.read() # Strip "ExternalContext.main()" call from last line. diff --git a/tests/ansible/all.yml b/tests/ansible/all.yml index e074a384..06f3acdb 100644 --- a/tests/ansible/all.yml +++ b/tests/ansible/all.yml @@ -1,3 +1,4 @@ +- include: setup/all.yml - include: regression/all.yml - include: integration/all.yml diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml index 2ca46f1c..f6702f58 100644 --- a/tests/ansible/bench/file_transfer.yml +++ b/tests/ansible/bench/file_transfer.yml @@ -66,3 +66,6 @@ copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + + tags: + - resource_intensive diff --git a/tests/ansible/bench/includes.yml b/tests/ansible/bench/includes.yml index 4f50113a..96079874 100644 --- a/tests/ansible/bench/includes.yml +++ b/tests/ansible/bench/includes.yml @@ -2,3 +2,5 @@ tasks: - include_tasks: _includes.yml with_sequence: start=1 end=1000 + tags: + - resource_intensive diff --git a/tests/ansible/bench/loop-100-copies.yml b/tests/ansible/bench/loop-100-copies.yml index 0f4d3600..e25ae552 100644 --- a/tests/ansible/bench/loop-100-copies.yml +++ b/tests/ansible/bench/loop-100-copies.yml @@ -24,3 +24,8 @@ mode: 0644 with_filetree: /tmp/filetree.in when: item.state == 'file' + loop_control: + label: "/tmp/filetree.out/{{ item.path }}" + + tags: + - resource_intensive diff --git a/tests/ansible/bench/loop-100-items.yml b/tests/ansible/bench/loop-100-items.yml index c071c100..e711301d 100644 --- a/tests/ansible/bench/loop-100-items.yml +++ b/tests/ansible/bench/loop-100-items.yml @@ -8,3 +8,5 @@ tasks: - command: hostname with_sequence: start=1 end="{{end|default(100)}}" + tags: + - resource_intensive diff --git a/tests/ansible/bench/loop-100-tasks.yml b/tests/ansible/bench/loop-100-tasks.yml index bf6e31b8..4a76c4fe 100644 --- a/tests/ansible/bench/loop-100-tasks.yml +++ b/tests/ansible/bench/loop-100-tasks.yml @@ -110,3 +110,5 @@ - command: hostname - command: hostname - command: hostname + tags: + - resource_intensive diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index a9826d23..78d5c7b1 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -29,3 +29,8 @@ mode: 0644 with_filetree: /tmp/filetree.in when: item.state == 'file' + loop_control: + label: "/tmp/filetree.out/{{ item.path }}" + + tags: + - resource_intensive diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index c0386cd8..2c3c87c8 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,4 @@ paramiko==2.3.2 # Last 2.6-compat version. hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' -PyYAML==3.13; python_version >= '2.7' +PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml new file mode 100644 index 00000000..c51fa295 --- /dev/null +++ b/tests/ansible/setup/all.yml @@ -0,0 +1 @@ +- include: report.yml diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report.yml new file mode 100644 index 00000000..7ccd049a --- /dev/null +++ b/tests/ansible/setup/report.yml @@ -0,0 +1,8 @@ +- name: Report runtime settings + hosts: localhost + gather_facts: false + tasks: + - debug: {var: ansible_forks} + - debug: {var: ansible_run_tags} + - debug: {var: ansible_skip_tags} + - debug: {var: ansible_version.full} diff --git a/tests/ansible/soak/file_service.yml b/tests/ansible/soak/file_service.yml index 0640233a..65b10b2d 100644 --- a/tests/ansible/soak/file_service.yml +++ b/tests/ansible/soak/file_service.yml @@ -4,3 +4,5 @@ content: "{% for x in range(126977) %}x{% endfor %}" - include: _file_service_loop.yml with_sequence: start=1 end=100 + tags: + - resource_intensive diff --git a/tests/testlib.py b/tests/testlib.py index ace8f0a2..ee76a26d 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -103,6 +103,18 @@ if hasattr(subprocess.Popen, 'terminate'): Popen__terminate = subprocess.Popen.terminate +def threading__thread_is_alive(thread): + """Return whether the thread is alive (Python version compatibility shim). + + On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9). + On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6). + """ + try: + return thread.is_alive() + except AttributeError: + return thread.isAlive() + + def wait_for_port( host, port, @@ -334,7 +346,9 @@ class TestCase(unittest2.TestCase): for thread in threading.enumerate(): name = thread.getName() # Python 2.4: enumerate() may return stopped threads. - assert (not thread.isAlive()) or name in self.ALLOWED_THREADS, \ + assert \ + not threading__thread_is_alive(thread) \ + or name in self.ALLOWED_THREADS, \ 'Found thread %r still running after tests.' % (name,) counts[name] = counts.get(name, 0) + 1 diff --git a/tests/utils_test.py b/tests/utils_test.py index a70b23dc..b5204a3c 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -31,14 +31,14 @@ class RunWithRouterTest(testlib.TestCase): def test_run_with_broker(self): router = mitogen.utils.run_with_router(func0) self.assertIsInstance(router, mitogen.master.Router) - self.assertFalse(router.broker._thread.isAlive()) + self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread)) class WithRouterTest(testlib.TestCase): def test_with_broker(self): router = func() self.assertIsInstance(router, mitogen.master.Router) - self.assertFalse(router.broker._thread.isAlive()) + self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread)) class Dict(dict): pass