Merge branch 'master' into distro-vs-distros

.ci/ansible_install.py

@@ -17,7 +17,7 @@ batches = [
 ]
 
 batches.extend(
-    ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
+    ['docker pull %s' % (ci_lib.image_for_distro(distro),), 'sleep 1']
     for distro in ci_lib.DISTROS
 )
 

.ci/ansible_tests.py

@@ -66,8 +66,6 @@ with ci_lib.Fold('job_setup'):
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")
 
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'")
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'")
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')

.ci/azure-pipelines.yml

@@ -29,6 +29,7 @@ jobs:
         MODE: localhost_ansible
         VER: 2.10.0
         STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
 
 
 - job: Linux
@@ -61,6 +62,12 @@ jobs:
         MODE: mitogen
         DISTRO: debian
 
+      Mito39Debian_27:
+        python.version: '3.9'
+        MODE: mitogen
+        DISTRO: debian
+        VER: 2.10.0
+
       #Py26CentOS7:
         #python.version: '2.7'
         #MODE: mitogen
@@ -112,3 +119,8 @@ jobs:
         python.version: '3.5'
         MODE: ansible
         VER: 2.10.0
+
+      Ansible_210_39:
+        python.version: '3.9'
+        MODE: ansible
+        VER: 2.10.0

.ci/ci_lib.py

@@ -84,12 +84,28 @@ if 'TRAVIS_HOME' in os.environ:
 # -----------------
 
 def _argv(s, *args):
+    """Interpolate a command line using *args, return an argv style list.
+
+    >>> _argv('git commit -m "Use frobnicate 2.0 (fixes #%d)"', 1234)
+    ['git', commit', '-m', 'Use frobnicate 2.0 (fixes #1234)']
+    """
     if args:
         s %= args
     return shlex.split(s)
 
 
 def run(s, *args, **kwargs):
+    """ Run a command, with arguments, and print timing information
+
+    >>> rc = run('echo "%s %s"', 'foo', 'bar')
+    Running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    foo bar
+    0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k
+    0inputs+0outputs (0major+71minor)pagefaults 0swaps
+    Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    >>> rc
+    0
+    """
     argv = ['/usr/bin/time', '--'] + _argv(s, *args)
     print('Running: %s' % (argv,))
     try:
@@ -102,12 +118,36 @@ def run(s, *args, **kwargs):
     return ret
 
 
-def run_batches(batches):
+def combine(batch):
-    combine = lambda batch: 'set -x; ' + (' && '.join(
+    """
+    >>> combine(['ls -l', 'echo foo'])
+    'set -x; ( ls -l; ) && ( echo foo; )'
+    """
+    return 'set -x; ' + (' && '.join(
         '( %s; )' % (cmd,)
         for cmd in batch
     ))
 
+
+def run_batches(batches):
+    """ Run shell commands grouped into batches, showing an execution trace.
+
+    Raise AssertionError if any command has exits with a non-zero status.
+
+    >>> run_batches([['echo foo', 'true']])
+    + echo foo
+    foo
+    + true
+    >>> run_batches([['true', 'echo foo'], ['false']])
+    + true
+    + echo foo
+    foo
+    + false
+    Traceback (most recent call last):
+    File "...", line ..., in <module>
+    File "...", line ..., in run_batches
+    AssertionError
+    """
     procs = [
         subprocess.Popen(combine(batch), shell=True)
         for batch in batches
@@ -116,12 +156,28 @@ def run_batches(batches):
 
 
 def get_output(s, *args, **kwargs):
+    """
+    Print and run command line s, %-interopolated using *args. Return stdout.
+
+    >>> s = get_output('echo "%s %s"', 'foo', 'bar')
+    Running: ['echo', 'foo bar']
+    >>> s
+    'foo bar\n'
+    """
     argv = _argv(s, *args)
     print('Running: %s' % (argv,))
     return subprocess.check_output(argv, **kwargs)
 
 
 def exists_in_path(progname):
+    """
+    Return True if proganme exists in $PATH.
+
+    >>> exists_in_path('echo')
+    True
+    >>> exists_in_path('kwyjibo') # Only found in North American cartoons
+    False
+    """
     return any(os.path.exists(os.path.join(dirname, progname))
                for dirname in os.environ['PATH'].split(os.pathsep))
 
@@ -136,6 +192,18 @@ class TempDir(object):
 
 
 class Fold(object):
+    """
+    Bracket a section of stdout with travis_fold markers.
+
+    This allows the section to be collapsed or expanded in Travis CI web UI.
+
+    >>> with Fold('stage 1'):
+    ...     print('Frobnicate the frobnitz')
+    ...
+    travis_fold:start:stage 1
+    Frobnicate the frobnitz
+    travis_fold:end:stage 1
+    """
     def __init__(self, name):
         self.name = name
 
@@ -178,6 +246,8 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
 )
 
 def get_docker_hostname():
+    """Return the hostname where the docker daemon is running.
+    """
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -187,10 +257,34 @@ def get_docker_hostname():
 
 
 def image_for_distro(distro):
-    return 'mitogen/%s-test' % (distro.partition('-')[0],)
+    """Return the container image name or path for a test distro name.
+
+    The returned value is suitable for use with `docker pull`.
+
+    >>> image_for_distro('centos5')
+    'public.ecr.aws/n5z0e8q9/centos5-test'
+    >>> image_for_distro('centos5-something_custom')
+    'public.ecr.aws/n5z0e8q9/centos5-test'
+    """
+    return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],)
 
 
 def make_containers(name_prefix='', port_offset=0):
+    """
+    >>> import pprint
+    >>> BASE_PORT=2200; DISTROS=['debian', 'centos6']
+    >>> pprint.pprint(make_containers())
+    [{'distro': 'debian',
+      'hostname': 'localhost',
+      'name': 'target-debian-1',
+      'port': 2201,
+      'python_path': '/usr/bin/python'},
+     {'distro': 'centos6',
+      'hostname': 'localhost',
+      'name': 'target-centos6-2',
+      'port': 2202,
+      'python_path': '/usr/bin/python'}]
+    """
     docker_hostname = get_docker_hostname()
     firstbit = lambda s: (s+'-').split('-')[0]
     secondbit = lambda s: (s+'-').split('-')[1]
@@ -263,6 +357,14 @@ def get_interesting_procs(container_name=None):
 
 
 def start_containers(containers):
+    """Run docker containers in the background, with sshd on specified ports.
+
+    >>> containers = start_containers([
+    ...     {'distro': 'debian', 'hostname': 'localhost',
+    ...      'name': 'target-debian-1', 'port': 2201,
+    ...      'python_path': '/usr/bin/python'},
+    ... ])
+    """
     if os.environ.get('KEEP'):
         return
 

.ci/prep_azure.py

@@ -86,12 +86,4 @@ if need_to_fix_psycopg2:
 
 batches.append(venv_steps)
 
-
-if ci_lib.have_docker():
-    batches.extend(
-        ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
-        for distro in ci_lib.DISTROS
-    )
-
-
 ci_lib.run_batches(batches)

.travis.yml

@@ -1,5 +1,5 @@
 sudo: required
-dist: trusty
+dist: xenial  # Ubuntu 16.04 LTS
 
 notifications:
   email: false
@@ -17,7 +17,6 @@ cache:
   - /home/travis/virtualenv
 
 install:
-- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v
 - pip install -U pip==20.2.1
 - .ci/${MODE}_install.py
 
@@ -53,6 +52,9 @@ matrix:
     - python: "3.6"
       env: MODE=ansible VER=2.10.0
     # 2.10 -> {debian, centos6, centos7}
+    - python: "3.9"
+      env: MODE=ansible VER=2.10.0
+    # 2.10 -> {debian, centos6, centos7}
     - python: "2.7"
       env: MODE=ansible VER=2.10.0
     # 2.10 -> {debian, centos6, centos7}
@@ -73,6 +75,8 @@ matrix:
       #env: MODE=mitogen DISTRO=centos6
     - python: "3.6"
       env: MODE=mitogen DISTRO=centos7
+    - python: "3.9"
+      env: MODE=mitogen DISTRO=centos7
     # 2.6 -> 2.7
     # - python: "2.6"
     #   env: MODE=mitogen DISTRO=centos7

docs/changelog.rst

@@ -40,6 +40,8 @@ v0.2.10 (unreleased)
   timeout, when using recent OpenSSH client versions.
 * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in
   :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
+* :gh:issue:`775` Test with Python 3.9
+* :gh:issue:`775` Add msvcrt to the default module deny list
 
 
 v0.2.9 (2019-11-02)

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 2, 9)
+__version__ = (0, 3, 0, 'rc', 1)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/core.py

@@ -1269,6 +1269,13 @@ class Importer(object):
         # a negative round-trip.
         'builtins',
         '__builtin__',
+
+        # On some Python releases (e.g. 3.8, 3.9) the subprocess module tries
+        # to import of this Windows-only builtin module.
+        'msvcrt',
+
+        # Python 2.x module that was renamed to _thread in 3.x.
+        # This entry avoids a roundtrip on 2.x -> 3.x.
         'thread',
 
         # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but
@@ -3860,7 +3867,7 @@ class ExternalContext(object):
         else:
             core_src_fd = self.config.get('core_src_fd', 101)
             if core_src_fd:
-                fp = os.fdopen(core_src_fd, 'rb', 1)
+                fp = os.fdopen(core_src_fd, 'rb', 0)
                 try:
                     core_src = fp.read()
                     # Strip "ExternalContext.main()" call from last line.

setup.cfg

@@ -1,3 +1,6 @@
+[bdist_wheel]
+universal=1
+
 [coverage:run]
 branch = true
 source =

setup.py

@@ -37,29 +37,46 @@ def grep_version():
         for line in fp:
             if line.startswith('__version__'):
                 _, _, s = line.partition('=')
-                return '.'.join(map(str, eval(s)))
+                return '%i.%i.%i%s%i' % eval(s)
+
+
+def long_description():
+    here = os.path.dirname(__file__)
+    readme_path = os.path.join(here, 'README.md')
+    with open(readme_path) as fp:
+        readme = fp.read()
+    return readme
 
 
 setup(
     name = 'mitogen',
     version = grep_version(),
     description = 'Library for writing distributed self-replicating programs.',
+    long_description = long_description(),
+    long_description_content_type='text/markdown',
     author = 'David Wilson',
     license = 'New BSD',
-    url = 'https://github.com/dw/mitogen/',
+    url = 'https://github.com/mitogen-hq/mitogen/',
     packages = find_packages(exclude=['tests', 'examples']),
+    python_requires='>=2.4, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4',
     zip_safe = False,
     classifiers = [
         'Environment :: Console',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: BSD License',
+        'Operating System :: MacOS :: MacOS X',
         'Operating System :: POSIX',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.4',
         'Programming Language :: Python :: 2.5',
         'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: Implementation :: CPython',
         'Topic :: System :: Distributed Computing',
         'Topic :: System :: Systems Administration',

tests/ansible/all.yml

@@ -1,3 +1,4 @@
+- include: setup/all.yml
 - include: regression/all.yml
 - include: integration/all.yml
 

tests/ansible/bench/file_transfer.yml

@@ -66,3 +66,6 @@
     copy:
       src: /tmp/bigbigfile.in
       dest: /tmp/bigbigfile.out
+
+  tags:
+  - resource_intensive

tests/ansible/bench/includes.yml

@@ -2,3 +2,5 @@
   tasks:
   - include_tasks: _includes.yml
     with_sequence: start=1 end=1000
+  tags:
+    - resource_intensive

tests/ansible/bench/loop-100-copies.yml

@@ -24,3 +24,8 @@
       mode: 0644
     with_filetree: /tmp/filetree.in
     when: item.state == 'file'
+    loop_control:
+      label: "/tmp/filetree.out/{{ item.path }}"
+
+  tags:
+    - resource_intensive

tests/ansible/bench/loop-100-items.yml

@@ -8,3 +8,5 @@
   tasks:
     - command: hostname
       with_sequence: start=1 end="{{end|default(100)}}"
+  tags:
+    - resource_intensive

tests/ansible/bench/loop-100-tasks.yml

@@ -110,3 +110,5 @@
     - command: hostname
     - command: hostname
     - command: hostname
+  tags:
+    - resource_intensive

tests/ansible/integration/playbook_semantics/delegate_to.yml

@@ -51,10 +51,14 @@
       shell: whoami > /tmp/delegate_to.yml.txt
       delegate_to: localhost
       become: true
+      tags:
+        - requires_local_sudo
 
     - name: "delegate_to, sudo"
       assert:
         that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+      tags:
+        - requires_local_sudo
 
     - name: "delegate_to, sudo"
       file:
@@ -62,6 +66,8 @@
         state: absent
       delegate_to: localhost
       become: true
+      tags:
+        - requires_local_sudo
 
 
     #
@@ -71,10 +77,14 @@
       shell: whoami > /tmp/delegate_to.yml.txt
       connection: local
       become: true
+      tags:
+        - requires_local_sudo
 
     - name: "connection:local, sudo"
       assert:
         that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+      tags:
+        - requires_local_sudo
 
     - name: "connection:local, sudo"
       file:
@@ -82,3 +92,5 @@
         state: absent
       connection: local
       become: true
+      tags:
+        - requires_local_sudo

tests/ansible/regression/issue_140__thread_pileup.yml

@@ -29,3 +29,8 @@
       mode: 0644
     with_filetree: /tmp/filetree.in
     when: item.state == 'file'
+    loop_control:
+      label: "/tmp/filetree.out/{{ item.path }}"
+
+  tags:
+  - resource_intensive

tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml

@@ -15,7 +15,7 @@
         content: |
           #!/bin/bash
           export CUSTOM_INTERPRETER=1
-          exec python2.7 "$@"
+          exec python "$@"
 
     - custom_python_detect_environment:
       vars:

tests/ansible/requirements.txt

@@ -1,4 +1,4 @@
 paramiko==2.3.2  # Last 2.6-compat version.
 hdrhistogram==0.6.1
 PyYAML==3.11; python_version < '2.7'
-PyYAML==3.13; python_version >= '2.7'
+PyYAML==5.3.1; python_version >= '2.7'  # Latest release (Jan 2021)

tests/ansible/setup/all.yml

@@ -0,0 +1 @@
+- include: report.yml

tests/ansible/setup/report.yml

@@ -0,0 +1,8 @@
+- name: Report runtime settings
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - debug: {var: ansible_forks}
+    - debug: {var: ansible_run_tags}
+    - debug: {var: ansible_skip_tags}
+    - debug: {var: ansible_version.full}

tests/ansible/soak/file_service.yml

@@ -4,3 +4,5 @@
       content: "{% for x in range(126977) %}x{% endfor %}"
   - include: _file_service_loop.yml
     with_sequence: start=1 end=100
+  tags:
+    - resource_intensive

tests/testlib.py

@@ -103,6 +103,18 @@ if hasattr(subprocess.Popen, 'terminate'):
     Popen__terminate = subprocess.Popen.terminate
 
 
+def threading__thread_is_alive(thread):
+    """Return whether the thread is alive (Python version compatibility shim).
+
+    On Python >= 3.8 thread.isAlive() is deprecated (removed in Python 3.9).
+    On Python <= 2.5 thread.is_alive() isn't present (added in Python 2.6).
+    """
+    try:
+        return thread.is_alive()
+    except AttributeError:
+        return thread.isAlive()
+
+
 def wait_for_port(
         host,
         port,
@@ -334,7 +346,9 @@ class TestCase(unittest2.TestCase):
         for thread in threading.enumerate():
             name = thread.getName()
             # Python 2.4: enumerate() may return stopped threads.
-            assert (not thread.isAlive()) or name in self.ALLOWED_THREADS, \
+            assert \
+                not threading__thread_is_alive(thread) \
+                or name in self.ALLOWED_THREADS, \
                 'Found thread %r still running after tests.' % (name,)
             counts[name] = counts.get(name, 0) + 1
 

tests/utils_test.py

@@ -31,14 +31,14 @@ class RunWithRouterTest(testlib.TestCase):
     def test_run_with_broker(self):
         router = mitogen.utils.run_with_router(func0)
         self.assertIsInstance(router, mitogen.master.Router)
-        self.assertFalse(router.broker._thread.isAlive())
+        self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread))
 
 
 class WithRouterTest(testlib.TestCase):
     def test_with_broker(self):
         router = func()
         self.assertIsInstance(router, mitogen.master.Router)
-        self.assertFalse(router.broker._thread.isAlive())
+        self.assertFalse(testlib.threading__thread_is_alive(router.broker._thread))
 
 
 class Dict(dict): pass

tox.ini

@@ -1,44 +1,88 @@
+# This file is a local convenience. It is not a substitute for the full CI
+# suite, and does not cover the full range of Python versions for Mitogen.
+
+# I use this on Ubuntu 20.04, with the following additions
+#
+#   sudo add-apt-repository ppa:deadsnakes/ppa
+#   sudo apt update
+#   sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass
+
+# Last version to support each python version
+#
+#            tox      vir'env  pip      ansible  coverage
+# ========== ======== ======== ======== ======== ========
+# python2.4  1.4      1.8      1.1      ???
+# python2.5  1.6.1    1.9.1    1.3.1    ???
+# python2.6  2.9.1    15.2.0   9.0.3    2.6.20   4.5.4
+
 [tox]
 envlist =
     init,
-    py26,
+    py{27,36,39}-mode_ansible-ansible2.10,
-    py27,
+    py{27,36,39}-mode_mitogen,
-    py35,
+    py{27,36,39}-mode_mitogen-distro_centos7,
-    py36,
-    py37,
     report,
+requires =
+    tox-factor
 
 [testenv]
-usedevelop = True
+basepython =
-deps =
+    py26: python2.6
-    -r{toxinidir}/dev_requirements.txt
+    py27: python2.7
-    -r{toxinidir}/tests/ansible/requirements.txt
+    py36: python3.6
-
+    py37: python3.7
+    py38: python3.8
+    py39: python3.9
+commands_pre =
+    mode_ansible: {toxinidir}/.ci/ansible_install.py
+    mode_debops_common: {toxinidir}/.ci/debops_common_install.py
+    mode_mitogen: {toxinidir}/.ci/mitogen_install.py
 commands =
-    {posargs:bash run_tests}
+    mode_ansible: {toxinidir}/.ci/ansible_tests.py \
-whitelist_externals =
+                        --skip-tags requires_local_sudo
-    bash
+    mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
+    mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
+passenv =
+    HOME
 setenv =
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
+    ansible2.3: VER=2.3.3.0
+    ansible2.4: VER=2.4.6.0
+    ansible2.8: VER=2.8.3
+    ansible2.9: VER=2.9.6
+    ansible2.10: VER=2.10.0
+    distro_centos5: DISTRO=centos5
+    distro_centos6: DISTRO=centos6
+    distro_centos7: DISTRO=centos7
+    distro_debian: DISTRO=debian
+    distro_debianpy3: DISTRO=debian-py3
+    distros_centos5: DISTROS=centos5
+    distros_debian: DISTROS=debian
+    mode_ansible: MODE=ansible
+    mode_debops_common: MODE=debops_common
+    mode_mitogen: MODE=mitogen
+    strategy_linear: STRATEGY=linear
 
 [testenv:init]
+basepython = python3
 commands =
     coverage erase
 deps =
-    coverage
+    coverage==4.5.4
 
 [testenv:report]
+basepython = python3
 commands =
     coverage html
     echo "coverage report is at file://{toxinidir}/htmlcov/index.html"
 deps =
-    coverage
+    coverage==4.5.4
 whitelist_externals =
     echo
 
 [testenv:docs]
-basepython = python
+basepython = python3
 changedir = docs
 commands =
     sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html