Pythonize, split out and parallelize all jobs.

.ci/ansible_install.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+import ci_lib
+
+batches = [
+    [
+        # Must be installed separately, as PyNACL indirect requirement causes
+        # newer version to be installed if done in a single pip run.
+        'pip install "pycparser<2.19"',
+        'pip install '
+            '-r tests/requirements.txt '
+            '-r tests/ansible/requirements.txt',
+    ]
+]
+
+batches.extend(
+    ['docker pull mitogen/%s-test' % (distro,)]
+    for distro in ci_lib.DISTROS
+)
+
+ci_lib.run_batches(batches)

.ci/ansible_tests.py

@@ -8,53 +8,42 @@ import ci_lib
 from ci_lib import run
 
 
-BASE_PORT = 2201
 TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
 HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
 
 
+with ci_lib.Fold('unit_tests'):
+    os.environ['SKIP_MITOGEN'] = '1'
+    ci_lib.run('./run_tests -v')
+
+
 with ci_lib.Fold('docker_setup'):
-    for i, distro in enumerate(ci_lib.DISTROS):
+    containers = ci_lib.make_containers()
-        try:
+    ci_lib.start_containers(containers)
-            run("docker rm -f target-%s", distro)
-        except: pass
-
-        run("""
-            docker run
-            --rm
-            --detach
-            --publish 0.0.0.0:%s:22/tcp
-            --hostname=target-%s
-            --name=target-%s
-            mitogen/%s-test
-        """, BASE_PORT + i, distro, distro, distro)
 
 
 with ci_lib.Fold('job_setup'):
-    os.chdir(TESTS_DIR)
-    os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
-
-    run("pip install -qr requirements.txt")  # tests/ansible/requirements
     # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
     run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION)
 
+    os.chdir(TESTS_DIR)
+    os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
+
     run("mkdir %s", HOSTS_DIR)
     run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR)
 
-    docker_hostname = ci_lib.get_docker_hostname()
     with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp:
         fp.write('[test-targets]\n')
-        for i, distro in enumerate(ci_lib.DISTROS):
+        fp.writelines(
-            fp.write("target-%s "
+            "%(name)s "
-                     "ansible_host=%s "
+            "ansible_host=%(hostname)s "
-                     "ansible_port=%s "
+            "ansible_port=%(port)s "
-                     "ansible_user=mitogen__has_sudo_nopw "
+            "ansible_user=mitogen__has_sudo_nopw "
-                     "ansible_password=has_sudo_nopw_password"
+            "ansible_password=has_sudo_nopw_password"
-                     "\n" % (
+            "\n"
-                distro,
+            % container
-                docker_hostname,
+            for container in containers
-                BASE_PORT + i,
+        )
-            ))
 
     # Build the binaries.
     # run("make -C %s", TESTS_DIR)

.ci/azure-pipelines.yml

@@ -76,11 +76,8 @@ jobs:
   - script: .ci/prep_azure.py
     displayName: "Install requirements."
 
-  - script: |
+  - script: .ci/$(MODE)_install.py
-      export TRAVIS_BUILD_DIR=`pwd`
+    displayName: "Install requirements."
-      if [ -f ".ci/$(MODE)_tests.sh" ]; then
+
-        .ci/$(MODE)_tests.sh;
+  - script: .ci/$(MODE)_tests.py
-      else
-        .ci/$(MODE)_tests.py;
-      fi
     displayName: Run tests.

.ci/ci_lib.py

@@ -4,12 +4,17 @@ from __future__ import print_function
 
 import atexit
 import os
-import subprocess
-import sys
 import shlex
 import shutil
+import subprocess
+import sys
 import tempfile
 
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+
 
 #
 # check_output() monkeypatch cutpasted from testlib.py
@@ -60,13 +65,26 @@ def _argv(s, *args):
 
 
 def run(s, *args, **kwargs):
-    argv = _argv(s, *args)
+    argv = ['/usr/bin/time', '--'] + _argv(s, *args)
     print('Running: %s' % (argv,))
     ret = subprocess.check_call(argv, **kwargs)
     print('Finished running: %s' % (argv,))
     return ret
 
 
+def run_batches(batches):
+    combine = lambda batch: 'set -x; ' + (' && '.join(
+        '( %s; )' % (cmd,)
+        for cmd in batch
+    ))
+
+    procs = [
+        subprocess.Popen(combine(batch), shell=True)
+        for batch in batches
+    ]
+    assert [proc.wait() for proc in procs] == [0] * len(procs)
+
+
 def get_output(s, *args, **kwargs):
     argv = _argv(s, *args)
     print('Running: %s' % (argv,))
@@ -103,7 +121,10 @@ os.environ.setdefault('ANSIBLE_STRATEGY',
     os.environ.get('STRATEGY', 'mitogen_linear'))
 ANSIBLE_VERSION = os.environ.get('VER', '2.6.2')
 GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+DISTRO = os.environ.get('DISTRO', 'debian')
 DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split()
+TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
+BASE_PORT = 2200
 TMP = TempDir().path
 
 os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
@@ -113,10 +134,44 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
 )
 
 def get_docker_hostname():
-    return subprocess.check_output([
+    url = os.environ.get('DOCKER_HOST')
-        sys.executable,
+    if url in (None, 'http+docker://localunixsocket'):
-        os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'),
+        return 'localhost'
-    ]).decode().strip()
+
+    parsed = urlparse.urlparse(url)
+    return parsed.netloc.partition(':')[0]
+
+
+def make_containers():
+    docker_hostname = get_docker_hostname()
+    return [
+        {
+            "distro": distro,
+            "name": "target-%s-%s" % (distro, i),
+            "hostname": docker_hostname,
+            "port": BASE_PORT + i,
+        }
+        for i, distro in enumerate(DISTROS, 1)
+    ]
+
+
+def start_containers(containers):
+    run_batches([
+        [
+            "docker rm -f %(name)s || true" % container,
+            "docker run "
+                "--rm "
+                "--detach "
+                "--publish 0.0.0.0:%(port)s:22/tcp "
+                "--hostname=%(name)s "
+                "--name=%(name)s "
+                "mitogen/%(distro)s-test "
+            % container
+        ]
+        for container in containers
+    ])
+    return containers
+
 
 # SSH passes these through to the container when run interactively, causing
 # stdout to get messed up with libc warnings.

.ci/debops_common_install.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+import ci_lib
+
+# Naturally DebOps only supports Debian.
+ci_lib.DISTROS = ['debian']
+
+ci_lib.run_batches([
+    [
+        # Must be installed separately, as PyNACL indirect requirement causes
+        # newer version to be installed if done in a single pip run.
+        'pip install "pycparser<2.19"',
+        'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
+    ],
+    [
+        'docker pull mitogen/debian-test',
+    ],
+])

.ci/debops_common_tests.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import os
+
+import ci_lib
+
+
+# DebOps only supports Debian.
+ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
+
+project_dir = os.path.join(ci_lib.TMP, 'project')
+key_file = os.path.join(
+    ci_lib.GIT_ROOT,
+    'tests/data/docker/mitogen__has_sudo_pubkey.key',
+)
+vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
+inventory_path = 'ansible/inventory/hosts'
+docker_hostname = ci_lib.get_docker_hostname()
+
+
+with ci_lib.Fold('docker_setup'):
+    containers = ci_lib.make_containers()
+    ci_lib.start_containers(containers)
+
+
+with ci_lib.Fold('job_setup'):
+    ci_lib.run('debops-init %s', project_dir)
+    os.chdir(project_dir)
+
+    with open('.debops.cfg', 'w') as fp:
+        fp.write(
+            "[ansible defaults]\n"
+            "strategy_plugins = %s/ansible_mitogen/plugins/strategy\n"
+            "strategy = mitogen_linear\n"
+            % (ci_lib.GIT_ROOT,)
+        )
+
+    ci_lib.run('chmod go= %s', key_file)
+    with open(vars_path, 'w') as fp:
+        fp.write(
+            "ansible_python_interpreter: /usr/bin/python2.7\n"
+            "\n"
+            "ansible_user: mitogen__has_sudo_pubkey\n"
+            "ansible_become_pass: has_sudo_pubkey_password\n"
+            "ansible_ssh_private_key_file: %s\n"
+            "\n"
+            # Speed up slow DH generation.
+            "dhparam__bits: ['128', '64']\n"
+            % (key_file,)
+        )
+
+    with open(inventory_path, 'a') as fp:
+        fp.writelines(
+            '%(name)s '
+            'ansible_host=%(hostname)s '
+            'ansible_port=%(port)d '
+            '\n'
+            % container
+            for container in containers
+        )
+
+    print()
+    print(' echo --- ansible/inventory/hosts: ---')
+    ci_lib.run('cat ansible/inventory/hosts')
+    print('---')
+    print()
+
+    # Now we have real host key checking, we need to turn it off
+    os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+
+
+with ci_lib.Fold('first_run'):
+    ci_lib.run('debops common')
+
+
+with ci_lib.Fold('second_run'):
+    ci_lib.run('debops common')

.ci/debops_common_tests.sh

@@ -1,90 +0,0 @@
-#!/bin/bash -ex
-# Run some invocations of DebOps.
-
-TMPDIR="/tmp/debops-$$"
-TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}"
-TARGET_COUNT="${TARGET_COUNT:-2}"
-ANSIBLE_VERSION="${VER:-2.6.1}"
-DISTRO=debian  # Naturally DebOps only supports Debian.
-
-export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}"
-
-function on_exit()
-{
-    echo travis_fold:start:cleanup
-    [ "$KEEP" ] || {
-        rm -rf "$TMPDIR" || true
-        for i in $(seq $TARGET_COUNT)
-        do
-            docker kill target$i || true
-        done
-    }
-    echo travis_fold:end:cleanup
-}
-
-trap on_exit EXIT
-mkdir "$TMPDIR"
-
-
-echo travis_fold:start:job_setup
-pip install -qqqU debops==0.7.2 ansible==${ANSIBLE_VERSION} |cat
-debops-init "$TMPDIR/project"
-cd "$TMPDIR/project"
-
-cat > .debops.cfg <<-EOF
-[ansible defaults]
-strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy
-strategy = mitogen_linear
-EOF
-
-chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
-
-cat > ansible/inventory/group_vars/debops_all_hosts.yml <<-EOF
-ansible_python_interpreter: /usr/bin/python2.7
-
-ansible_user: mitogen__has_sudo_pubkey
-ansible_become_pass: has_sudo_pubkey_password
-ansible_ssh_private_key_file: ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
-
-# Speed up slow DH generation.
-dhparam__bits: ["128", "64"]
-EOF
-
-DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)"
-
-for i in $(seq $TARGET_COUNT)
-do
-    port=$((2200 + $i))
-    docker run \
-        --rm \
-        --detach \
-        --publish 0.0.0.0:$port:22/tcp \
-        --name=target$i \
-        mitogen/${DISTRO}-test
-
-    echo \
-        target$i \
-        ansible_host=$DOCKER_HOSTNAME \
-        ansible_port=$port \
-        >> ansible/inventory/hosts
-done
-
-echo
-echo --- ansible/inventory/hosts: ----
-cat ansible/inventory/hosts
-echo ---
-
-# Now we have real host key checking, we need to turn it off. :)
-export ANSIBLE_HOST_KEY_CHECKING=False
-
-echo travis_fold:end:job_setup
-
-
-echo travis_fold:start:first_run
-/usr/bin/time debops common "$@"
-echo travis_fold:end:first_run
-
-
-echo travis_fold:start:second_run
-/usr/bin/time debops common "$@"
-echo travis_fold:end:second_run

.ci/mitogen_install.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+import ci_lib
+
+batches = [
+    [
+        'pip install "pycparser<2.19"',
+        'pip install -r tests/requirements.txt',
+    ],
+    [
+        'docker pull mitogen/%s-test' % (ci_lib.DISTRO,),
+    ]
+]
+
+ci_lib.run_batches(batches)

.ci/mitogen_tests.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# Run the Mitogen tests.
+
+import os
+
+import ci_lib
+
+os.environ.update({
+    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
+    'MITOGEN_LOG_LEVEL': 'debug',
+    'SKIP_ANSIBLE': '1',
+})
+
+ci_lib.run('./run_tests -v')

.ci/mitogen_tests.sh

@@ -1,5 +0,0 @@
-#!/bin/bash -ex
-# Run the Mitogen tests.
-
-MITOGEN_TEST_DISTRO="${DISTRO:-debian}"
-MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -v

.ci/prep_azure.py

@@ -1,36 +1,22 @@
 #!/usr/bin/env python
-# Run preparation steps in parallel.
 
-import subprocess
 import ci_lib
 
-subprocess.check_call(
+batches = []
+batches.append([
     'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
-    shell=True,
+    'sudo add-apt-repository ppa:deadsnakes/ppa',
-)
+    'sudo apt-get update',
+    'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
+])
 
-procs = [
+batches.append([
-    subprocess.Popen(
+    'pip install -r dev_requirements.txt',
-        'pip install -r dev_requirements.txt 2>&1 | cat',
+])
-        shell=True,
-    ),
-    subprocess.Popen(
-        """
-        sudo add-apt-repository ppa:deadsnakes/ppa && \
-        ( sudo apt-get update 2>&1 | cat ) && \
-        sudo apt-get -y install \
-            python2.6 python2.6-dev libsasl2-dev libldap2-dev 2>&1 | cat
-        """,
-        shell=True,
-    )
-]
 
-procs += [
+batches.extend(
-    subprocess.Popen(
+    ['docker pull mitogen/%s-test' % (distro,)]
-        'docker pull mitogen/%s-test 2>&1 | cat' % (distro,),
-        shell=True
-    )
     for distro in ci_lib.DISTROS
-]
+)
 
-assert [proc.wait() for proc in procs] == [0] * len(procs)
+ci_lib.run_batches(batches)

.travis.yml

@@ -1,7 +1,4 @@
 sudo: required
-addons:
-  apt:
-    update: true
 
 notifications:
   email: false
@@ -14,20 +11,10 @@ cache:
   - /home/travis/virtualenv
 
 install:
-# |cat to disable progress bar.
+- .ci/${MODE}_install.py
-- pip install -r dev_requirements.txt |cat
 
 script:
-- |
+- .ci/${MODE}_tests.py
-  if [ -f "${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh" ]; then
-    ${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh;
-  else
-    ${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.py;
-  fi
-
-
-services:
-  - docker
 
 
 # To avoid matrix explosion, just test against oldest->newest and

run_tests

@@ -1,4 +1,4 @@
-#/usr/bin/env bash
+#!/usr/bin/env bash
 
 echo '----- ulimits -----'
 ulimit -a

tests/show_docker_hostname.py

@@ -1,9 +0,0 @@
-#!/usr/bin/env python
-
-"""
-For use by the Travis scripts, just print out the hostname of the Docker
-daemon from the environment.
-"""
-
-import testlib
-print(testlib.get_docker_host())