Merge branch 'master' into fix0.3.0naming

.ci/ansible_install.py

@@ -17,7 +17,7 @@ batches = [
 ]
 
 batches.extend(
-    ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
+    ['docker pull %s' % (ci_lib.image_for_distro(distro),), 'sleep 1']
     for distro in ci_lib.DISTROS
 )
 

.ci/azure-pipelines.yml

@@ -30,6 +30,7 @@ jobs:
         MODE: localhost_ansible
         VER: 2.10.0
         STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
 
 
 - job: Linux
@@ -65,6 +66,12 @@ jobs:
         DISTRO: debian
         VER: 2.10.0
 
+      Mito39Debian_27:
+        python.version: '3.9'
+        MODE: mitogen
+        DISTRO: debian
+        VER: 2.10.0
+
       #Py26CentOS7:
         #python.version: '2.7'
         #MODE: mitogen
@@ -116,3 +123,8 @@ jobs:
         python.version: '3.5'
         MODE: ansible
         VER: 2.10.0
+
+      Ansible_210_39:
+        python.version: '3.9'
+        MODE: ansible
+        VER: 2.10.0

.ci/ci_lib.py

@@ -84,12 +84,28 @@ if 'TRAVIS_HOME' in os.environ:
 # -----------------
 
 def _argv(s, *args):
+    """Interpolate a command line using *args, return an argv style list.
+
+    >>> _argv('git commit -m "Use frobnicate 2.0 (fixes #%d)"', 1234)
+    ['git', commit', '-m', 'Use frobnicate 2.0 (fixes #1234)']
+    """
     if args:
         s %= args
     return shlex.split(s)
 
 
 def run(s, *args, **kwargs):
+    """ Run a command, with arguments, and print timing information
+
+    >>> rc = run('echo "%s %s"', 'foo', 'bar')
+    Running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    foo bar
+    0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k
+    0inputs+0outputs (0major+71minor)pagefaults 0swaps
+    Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    >>> rc
+    0
+    """
     argv = ['/usr/bin/time', '--'] + _argv(s, *args)
     print('Running: %s' % (argv,))
     try:
@@ -102,12 +118,36 @@ def run(s, *args, **kwargs):
     return ret
 
 
-def run_batches(batches):
+def combine(batch):
-    combine = lambda batch: 'set -x; ' + (' && '.join(
+    """
+    >>> combine(['ls -l', 'echo foo'])
+    'set -x; ( ls -l; ) && ( echo foo; )'
+    """
+    return 'set -x; ' + (' && '.join(
         '( %s; )' % (cmd,)
         for cmd in batch
     ))
 
+
+def run_batches(batches):
+    """ Run shell commands grouped into batches, showing an execution trace.
+
+    Raise AssertionError if any command has exits with a non-zero status.
+
+    >>> run_batches([['echo foo', 'true']])
+    + echo foo
+    foo
+    + true
+    >>> run_batches([['true', 'echo foo'], ['false']])
+    + true
+    + echo foo
+    foo
+    + false
+    Traceback (most recent call last):
+    File "...", line ..., in <module>
+    File "...", line ..., in run_batches
+    AssertionError
+    """
     procs = [
         subprocess.Popen(combine(batch), shell=True)
         for batch in batches
@@ -116,12 +156,28 @@ def run_batches(batches):
 
 
 def get_output(s, *args, **kwargs):
+    """
+    Print and run command line s, %-interopolated using *args. Return stdout.
+
+    >>> s = get_output('echo "%s %s"', 'foo', 'bar')
+    Running: ['echo', 'foo bar']
+    >>> s
+    'foo bar\n'
+    """
     argv = _argv(s, *args)
     print('Running: %s' % (argv,))
     return subprocess.check_output(argv, **kwargs)
 
 
 def exists_in_path(progname):
+    """
+    Return True if proganme exists in $PATH.
+
+    >>> exists_in_path('echo')
+    True
+    >>> exists_in_path('kwyjibo') # Only found in North American cartoons
+    False
+    """
     return any(os.path.exists(os.path.join(dirname, progname))
                for dirname in os.environ['PATH'].split(os.pathsep))
 
@@ -136,6 +192,18 @@ class TempDir(object):
 
 
 class Fold(object):
+    """
+    Bracket a section of stdout with travis_fold markers.
+
+    This allows the section to be collapsed or expanded in Travis CI web UI.
+
+    >>> with Fold('stage 1'):
+    ...     print('Frobnicate the frobnitz')
+    ...
+    travis_fold:start:stage 1
+    Frobnicate the frobnitz
+    travis_fold:end:stage 1
+    """
     def __init__(self, name):
         self.name = name
 
@@ -175,6 +243,8 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
 )
 
 def get_docker_hostname():
+    """Return the hostname where the docker daemon is running.
+    """
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -184,10 +254,34 @@ def get_docker_hostname():
 
 
 def image_for_distro(distro):
-    return 'mitogen/%s-test' % (distro.partition('-')[0],)
+    """Return the container image name or path for a test distro name.
+
+    The returned value is suitable for use with `docker pull`.
+
+    >>> image_for_distro('centos5')
+    'public.ecr.aws/n5z0e8q9/centos5-test'
+    >>> image_for_distro('centos5-something_custom')
+    'public.ecr.aws/n5z0e8q9/centos5-test'
+    """
+    return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],)
 
 
 def make_containers(name_prefix='', port_offset=0):
+    """
+    >>> import pprint
+    >>> BASE_PORT=2200; DISTROS=['debian', 'centos6']
+    >>> pprint.pprint(make_containers())
+    [{'distro': 'debian',
+      'hostname': 'localhost',
+      'name': 'target-debian-1',
+      'port': 2201,
+      'python_path': '/usr/bin/python'},
+     {'distro': 'centos6',
+      'hostname': 'localhost',
+      'name': 'target-centos6-2',
+      'port': 2202,
+      'python_path': '/usr/bin/python'}]
+    """
     docker_hostname = get_docker_hostname()
     firstbit = lambda s: (s+'-').split('-')[0]
     secondbit = lambda s: (s+'-').split('-')[1]
@@ -260,6 +354,14 @@ def get_interesting_procs(container_name=None):
 
 
 def start_containers(containers):
+    """Run docker containers in the background, with sshd on specified ports.
+
+    >>> containers = start_containers([
+    ...     {'distro': 'debian', 'hostname': 'localhost',
+    ...      'name': 'target-debian-1', 'port': 2201,
+    ...      'python_path': '/usr/bin/python'},
+    ... ])
+    """
     if os.environ.get('KEEP'):
         return
 

.ci/prep_azure.py

@@ -86,12 +86,4 @@ if need_to_fix_psycopg2:
 
 batches.append(venv_steps)
 
-
-if ci_lib.have_docker():
-    batches.extend(
-        ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
-        for distro in ci_lib.DISTROS
-    )
-
-
 ci_lib.run_batches(batches)

.travis.yml

@@ -1,5 +1,5 @@
 sudo: required
-dist: trusty
+dist: xenial  # Ubuntu 16.04 LTS
 
 notifications:
   email: false
@@ -17,7 +17,6 @@ cache:
   - /home/travis/virtualenv
 
 install:
-- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v
 - pip install -U pip==20.2.1
 - .ci/${MODE}_install.py
 
@@ -53,6 +52,9 @@ matrix:
     - python: "3.6"
       env: MODE=ansible VER=2.10.0
     # 2.10 -> {debian, centos6, centos7}
+    - python: "3.9"
+      env: MODE=ansible VER=2.10.0
+    # 2.10 -> {debian, centos6, centos7}
     - python: "2.7"
       env: MODE=ansible VER=2.10.0
     # 2.10 -> {debian, centos6, centos7}
@@ -73,6 +75,8 @@ matrix:
       #env: MODE=mitogen DISTRO=centos6
     - python: "3.6"
       env: MODE=mitogen DISTROS=centos7 VER=2.10.0
+    - python: "3.9"
+      env: MODE=mitogen DISTROS=centos7 VER=2.10.0
     # 2.6 -> 2.7
     # - python: "2.6"
     #   env: MODE=mitogen DISTROS=centos7 VER=2.10.0

docs/changelog.rst

@@ -40,6 +40,8 @@ v0.2.10 (unreleased)
   timeout, when using recent OpenSSH client versions.
 * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in
   :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
+* :gh:issue:`775` Test with Python 3.9
+* :gh:issue:`775` Add msvcrt to the default module deny list
 
 
 v0.2.9 (2019-11-02)

mitogen/core.py

@@ -1269,6 +1269,13 @@ class Importer(object):
         # a negative round-trip.
         'builtins',
         '__builtin__',
+
+        # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries
+        # to import of this Windows-only builtin module.
+        'msvcrt',
+
+        # Python 2.x module that was renamed to _thread in 3.x.
+        # This entry avoids a roundtrip on 2.x -> 3.x.
         'thread',
 
         # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but
@@ -3860,7 +3867,7 @@ class ExternalContext(object):
         else:
             core_src_fd = self.config.get('core_src_fd', 101)
             if core_src_fd:
-                fp = os.fdopen(core_src_fd, 'rb', 1)
+                fp = os.fdopen(core_src_fd, 'rb', 0)
                 try:
                     core_src = fp.read()
                     # Strip "ExternalContext.main()" call from last line.

tests/ansible/all.yml

@@ -1,3 +1,4 @@
+- include: setup/all.yml
 - include: regression/all.yml
 - include: integration/all.yml
 

tests/ansible/bench/file_transfer.yml

@@ -66,3 +66,6 @@
     copy:
       src: /tmp/bigbigfile.in
       dest: /tmp/bigbigfile.out
+
+  tags:
+  - resource_intensive

tests/ansible/bench/includes.yml

@@ -2,3 +2,5 @@
   tasks:
   - include_tasks: _includes.yml
     with_sequence: start=1 end=1000
+  tags:
+    - resource_intensive

tests/ansible/bench/loop-100-copies.yml

@@ -24,3 +24,8 @@
       mode: 0644
     with_filetree: /tmp/filetree.in
     when: item.state == 'file'
+    loop_control:
+      label: "/tmp/filetree.out/{{ item.path }}"
+
+  tags:
+    - resource_intensive

tests/ansible/bench/loop-100-items.yml

@@ -8,3 +8,5 @@
   tasks:
     - command: hostname
       with_sequence: start=1 end="{{end|default(100)}}"
+  tags:
+    - resource_intensive

tests/ansible/bench/loop-100-tasks.yml

@@ -110,3 +110,5 @@
     - command: hostname
     - command: hostname
     - command: hostname
+  tags:
+    - resource_intensive

tests/ansible/regression/issue_140__thread_pileup.yml

@@ -29,3 +29,8 @@
       mode: 0644
     with_filetree: /tmp/filetree.in
     when: item.state == 'file'
+    loop_control:
+      label: "/tmp/filetree.out/{{ item.path }}"
+
+  tags:
+  - resource_intensive

tests/ansible/requirements.txt

@@ -1,4 +1,4 @@
 paramiko==2.3.2  # Last 2.6-compat version.
 hdrhistogram==0.6.1
 PyYAML==3.11; python_version < '2.7'
-PyYAML==3.13; python_version >= '2.7'
+PyYAML==5.3.1; python_version >= '2.7'  # Latest release (Jan 2021)

tests/ansible/setup/all.yml

@@ -0,0 +1 @@
+- include: report.yml

tests/ansible/setup/report.yml

@@ -0,0 +1,8 @@
+- name: Report runtime settings
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - debug: {var: ansible_forks}
+    - debug: {var: ansible_run_tags}
+    - debug: {var: ansible_skip_tags}
+    - debug: {var: ansible_version.full}

tests/ansible/soak/file_service.yml

@@ -4,3 +4,5 @@
       content: "{% for x in range(126977) %}x{% endfor %}"
   - include: _file_service_loop.yml
     with_sequence: start=1 end=100
+  tags:
+    - resource_intensive

tests/testlib.py

@@ -103,6 +103,18 @@ if hasattr(subprocess.Popen, 'terminate'):
     Popen__terminate = subprocess.Popen.terminate
 
 
+def threading__thread_is_alive(thread):
+    """Return whether the thread is alive (Python version compatibility shim).
+
+    On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9).
+    On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6).
+    """
+    try:
+        return thread.is_alive()
+    except AttributeError:
+        return thread.isAlive()
+
+
 def wait_for_port(
         host,
         port,
@@ -334,7 +346,9 @@ class TestCase(unittest2.TestCase):
         for thread in threading.enumerate():
             name = thread.getName()
             # Python 2.4: enumerate() may return stopped threads.
-            assert (not thread.isAlive()) or name in self.ALLOWED_THREADS, \
+            assert \
+                not threading__thread_is_alive(thread) \
+                or name in self.ALLOWED_THREADS, \
                 'Found thread %r still running after tests.' % (name,)
             counts[name] = counts.get(name, 0) + 1
 

tests/utils_test.py

@@ -31,14 +31,14 @@ class RunWithRouterTest(testlib.TestCase):
     def test_run_with_broker(self):
         router = mitogen.utils.run_with_router(func0)
         self.assertIsInstance(router, mitogen.master.Router)
-        self.assertFalse(router.broker._thread.isAlive())
+        self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread))
 
 
 class WithRouterTest(testlib.TestCase):
     def test_with_broker(self):
         router = func()
         self.assertIsInstance(router, mitogen.master.Router)
-        self.assertFalse(router.broker._thread.isAlive())
+        self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread))
 
 
 class Dict(dict): pass