Pythonize, split out and parallelize all jobs.

issue510
David Wilson 6 years ago
parent b60a6d0f3b
commit 2a6dbb038f

@ -0,0 +1,21 @@
#!/usr/bin/env python
import ci_lib
batches = [
[
# Must be installed separately, as PyNACL indirect requirement causes
# newer version to be installed if done in a single pip run.
'pip install "pycparser<2.19"',
'pip install '
'-r tests/requirements.txt '
'-r tests/ansible/requirements.txt',
]
]
batches.extend(
['docker pull mitogen/%s-test' % (distro,)]
for distro in ci_lib.DISTROS
)
ci_lib.run_batches(batches)

@ -8,53 +8,42 @@ import ci_lib
from ci_lib import run from ci_lib import run
BASE_PORT = 2201
TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
with ci_lib.Fold('unit_tests'):
os.environ['SKIP_MITOGEN'] = '1'
ci_lib.run('./run_tests -v')
with ci_lib.Fold('docker_setup'): with ci_lib.Fold('docker_setup'):
for i, distro in enumerate(ci_lib.DISTROS): containers = ci_lib.make_containers()
try: ci_lib.start_containers(containers)
run("docker rm -f target-%s", distro)
except: pass
run("""
docker run
--rm
--detach
--publish 0.0.0.0:%s:22/tcp
--hostname=target-%s
--name=target-%s
mitogen/%s-test
""", BASE_PORT + i, distro, distro, distro)
with ci_lib.Fold('job_setup'): with ci_lib.Fold('job_setup'):
os.chdir(TESTS_DIR)
os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
run("pip install -qr requirements.txt") # tests/ansible/requirements
# Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION)
os.chdir(TESTS_DIR)
os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
run("mkdir %s", HOSTS_DIR) run("mkdir %s", HOSTS_DIR)
run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR) run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR)
docker_hostname = ci_lib.get_docker_hostname()
with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp: with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp:
fp.write('[test-targets]\n') fp.write('[test-targets]\n')
for i, distro in enumerate(ci_lib.DISTROS): fp.writelines(
fp.write("target-%s " "%(name)s "
"ansible_host=%s " "ansible_host=%(hostname)s "
"ansible_port=%s " "ansible_port=%(port)s "
"ansible_user=mitogen__has_sudo_nopw " "ansible_user=mitogen__has_sudo_nopw "
"ansible_password=has_sudo_nopw_password" "ansible_password=has_sudo_nopw_password"
"\n" % ( "\n"
distro, % container
docker_hostname, for container in containers
BASE_PORT + i, )
))
# Build the binaries. # Build the binaries.
# run("make -C %s", TESTS_DIR) # run("make -C %s", TESTS_DIR)

@ -76,11 +76,8 @@ jobs:
- script: .ci/prep_azure.py - script: .ci/prep_azure.py
displayName: "Install requirements." displayName: "Install requirements."
- script: | - script: .ci/$(MODE)_install.py
export TRAVIS_BUILD_DIR=`pwd` displayName: "Install requirements."
if [ -f ".ci/$(MODE)_tests.sh" ]; then
.ci/$(MODE)_tests.sh; - script: .ci/$(MODE)_tests.py
else
.ci/$(MODE)_tests.py;
fi
displayName: Run tests. displayName: Run tests.

@ -4,12 +4,17 @@ from __future__ import print_function
import atexit import atexit
import os import os
import subprocess
import sys
import shlex import shlex
import shutil import shutil
import subprocess
import sys
import tempfile import tempfile
try:
import urlparse
except ImportError:
import urllib.parse as urlparse
# #
# check_output() monkeypatch cutpasted from testlib.py # check_output() monkeypatch cutpasted from testlib.py
@ -60,13 +65,26 @@ def _argv(s, *args):
def run(s, *args, **kwargs): def run(s, *args, **kwargs):
argv = _argv(s, *args) argv = ['/usr/bin/time', '--'] + _argv(s, *args)
print('Running: %s' % (argv,)) print('Running: %s' % (argv,))
ret = subprocess.check_call(argv, **kwargs) ret = subprocess.check_call(argv, **kwargs)
print('Finished running: %s' % (argv,)) print('Finished running: %s' % (argv,))
return ret return ret
def run_batches(batches):
combine = lambda batch: 'set -x; ' + (' && '.join(
'( %s; )' % (cmd,)
for cmd in batch
))
procs = [
subprocess.Popen(combine(batch), shell=True)
for batch in batches
]
assert [proc.wait() for proc in procs] == [0] * len(procs)
def get_output(s, *args, **kwargs): def get_output(s, *args, **kwargs):
argv = _argv(s, *args) argv = _argv(s, *args)
print('Running: %s' % (argv,)) print('Running: %s' % (argv,))
@ -103,7 +121,10 @@ os.environ.setdefault('ANSIBLE_STRATEGY',
os.environ.get('STRATEGY', 'mitogen_linear')) os.environ.get('STRATEGY', 'mitogen_linear'))
ANSIBLE_VERSION = os.environ.get('VER', '2.6.2') ANSIBLE_VERSION = os.environ.get('VER', '2.6.2')
GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
DISTRO = os.environ.get('DISTRO', 'debian')
DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split()
TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
BASE_PORT = 2200
TMP = TempDir().path TMP = TempDir().path
os.environ['PYTHONDONTWRITEBYTECODE'] = 'x' os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
@ -113,10 +134,44 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
) )
def get_docker_hostname(): def get_docker_hostname():
return subprocess.check_output([ url = os.environ.get('DOCKER_HOST')
sys.executable, if url in (None, 'http+docker://localunixsocket'):
os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'), return 'localhost'
]).decode().strip()
parsed = urlparse.urlparse(url)
return parsed.netloc.partition(':')[0]
def make_containers():
docker_hostname = get_docker_hostname()
return [
{
"distro": distro,
"name": "target-%s-%s" % (distro, i),
"hostname": docker_hostname,
"port": BASE_PORT + i,
}
for i, distro in enumerate(DISTROS, 1)
]
def start_containers(containers):
run_batches([
[
"docker rm -f %(name)s || true" % container,
"docker run "
"--rm "
"--detach "
"--publish 0.0.0.0:%(port)s:22/tcp "
"--hostname=%(name)s "
"--name=%(name)s "
"mitogen/%(distro)s-test "
% container
]
for container in containers
])
return containers
# SSH passes these through to the container when run interactively, causing # SSH passes these through to the container when run interactively, causing
# stdout to get messed up with libc warnings. # stdout to get messed up with libc warnings.

@ -0,0 +1,18 @@
#!/usr/bin/env python
import ci_lib
# Naturally DebOps only supports Debian.
ci_lib.DISTROS = ['debian']
ci_lib.run_batches([
[
# Must be installed separately, as PyNACL indirect requirement causes
# newer version to be installed if done in a single pip run.
'pip install "pycparser<2.19"',
'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
],
[
'docker pull mitogen/debian-test',
],
])

@ -0,0 +1,78 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import ci_lib
# DebOps only supports Debian.
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
project_dir = os.path.join(ci_lib.TMP, 'project')
key_file = os.path.join(
ci_lib.GIT_ROOT,
'tests/data/docker/mitogen__has_sudo_pubkey.key',
)
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts'
docker_hostname = ci_lib.get_docker_hostname()
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers()
ci_lib.start_containers(containers)
with ci_lib.Fold('job_setup'):
ci_lib.run('debops-init %s', project_dir)
os.chdir(project_dir)
with open('.debops.cfg', 'w') as fp:
fp.write(
"[ansible defaults]\n"
"strategy_plugins = %s/ansible_mitogen/plugins/strategy\n"
"strategy = mitogen_linear\n"
% (ci_lib.GIT_ROOT,)
)
ci_lib.run('chmod go= %s', key_file)
with open(vars_path, 'w') as fp:
fp.write(
"ansible_python_interpreter: /usr/bin/python2.7\n"
"\n"
"ansible_user: mitogen__has_sudo_pubkey\n"
"ansible_become_pass: has_sudo_pubkey_password\n"
"ansible_ssh_private_key_file: %s\n"
"\n"
# Speed up slow DH generation.
"dhparam__bits: ['128', '64']\n"
% (key_file,)
)
with open(inventory_path, 'a') as fp:
fp.writelines(
'%(name)s '
'ansible_host=%(hostname)s '
'ansible_port=%(port)d '
'\n'
% container
for container in containers
)
print()
print(' echo --- ansible/inventory/hosts: ---')
ci_lib.run('cat ansible/inventory/hosts')
print('---')
print()
# Now we have real host key checking, we need to turn it off
os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
with ci_lib.Fold('first_run'):
ci_lib.run('debops common')
with ci_lib.Fold('second_run'):
ci_lib.run('debops common')

@ -1,90 +0,0 @@
#!/bin/bash -ex
# Run some invocations of DebOps.
TMPDIR="/tmp/debops-$$"
TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}"
TARGET_COUNT="${TARGET_COUNT:-2}"
ANSIBLE_VERSION="${VER:-2.6.1}"
DISTRO=debian # Naturally DebOps only supports Debian.
export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}"
function on_exit()
{
echo travis_fold:start:cleanup
[ "$KEEP" ] || {
rm -rf "$TMPDIR" || true
for i in $(seq $TARGET_COUNT)
do
docker kill target$i || true
done
}
echo travis_fold:end:cleanup
}
trap on_exit EXIT
mkdir "$TMPDIR"
echo travis_fold:start:job_setup
pip install -qqqU debops==0.7.2 ansible==${ANSIBLE_VERSION} |cat
debops-init "$TMPDIR/project"
cd "$TMPDIR/project"
cat > .debops.cfg <<-EOF
[ansible defaults]
strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy
strategy = mitogen_linear
EOF
chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
cat > ansible/inventory/group_vars/debops_all_hosts.yml <<-EOF
ansible_python_interpreter: /usr/bin/python2.7
ansible_user: mitogen__has_sudo_pubkey
ansible_become_pass: has_sudo_pubkey_password
ansible_ssh_private_key_file: ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
# Speed up slow DH generation.
dhparam__bits: ["128", "64"]
EOF
DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)"
for i in $(seq $TARGET_COUNT)
do
port=$((2200 + $i))
docker run \
--rm \
--detach \
--publish 0.0.0.0:$port:22/tcp \
--name=target$i \
mitogen/${DISTRO}-test
echo \
target$i \
ansible_host=$DOCKER_HOSTNAME \
ansible_port=$port \
>> ansible/inventory/hosts
done
echo
echo --- ansible/inventory/hosts: ----
cat ansible/inventory/hosts
echo ---
# Now we have real host key checking, we need to turn it off. :)
export ANSIBLE_HOST_KEY_CHECKING=False
echo travis_fold:end:job_setup
echo travis_fold:start:first_run
/usr/bin/time debops common "$@"
echo travis_fold:end:first_run
echo travis_fold:start:second_run
/usr/bin/time debops common "$@"
echo travis_fold:end:second_run

@ -0,0 +1,15 @@
#!/usr/bin/env python
import ci_lib
batches = [
[
'pip install "pycparser<2.19"',
'pip install -r tests/requirements.txt',
],
[
'docker pull mitogen/%s-test' % (ci_lib.DISTRO,),
]
]
ci_lib.run_batches(batches)

@ -0,0 +1,14 @@
#!/usr/bin/env python
# Run the Mitogen tests.
import os
import ci_lib
os.environ.update({
'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
'MITOGEN_LOG_LEVEL': 'debug',
'SKIP_ANSIBLE': '1',
})
ci_lib.run('./run_tests -v')

@ -1,5 +0,0 @@
#!/bin/bash -ex
# Run the Mitogen tests.
MITOGEN_TEST_DISTRO="${DISTRO:-debian}"
MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -v

@ -1,36 +1,22 @@
#!/usr/bin/env python #!/usr/bin/env python
# Run preparation steps in parallel.
import subprocess
import ci_lib import ci_lib
subprocess.check_call( batches = []
batches.append([
'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
shell=True, 'sudo add-apt-repository ppa:deadsnakes/ppa',
) 'sudo apt-get update',
'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
])
procs = [ batches.append([
subprocess.Popen( 'pip install -r dev_requirements.txt',
'pip install -r dev_requirements.txt 2>&1 | cat', ])
shell=True,
),
subprocess.Popen(
"""
sudo add-apt-repository ppa:deadsnakes/ppa && \
( sudo apt-get update 2>&1 | cat ) && \
sudo apt-get -y install \
python2.6 python2.6-dev libsasl2-dev libldap2-dev 2>&1 | cat
""",
shell=True,
)
]
procs += [ batches.extend(
subprocess.Popen( ['docker pull mitogen/%s-test' % (distro,)]
'docker pull mitogen/%s-test 2>&1 | cat' % (distro,),
shell=True
)
for distro in ci_lib.DISTROS for distro in ci_lib.DISTROS
] )
assert [proc.wait() for proc in procs] == [0] * len(procs) ci_lib.run_batches(batches)

@ -1,7 +1,4 @@
sudo: required sudo: required
addons:
apt:
update: true
notifications: notifications:
email: false email: false
@ -14,20 +11,10 @@ cache:
- /home/travis/virtualenv - /home/travis/virtualenv
install: install:
# |cat to disable progress bar. - .ci/${MODE}_install.py
- pip install -r dev_requirements.txt |cat
script: script:
- | - .ci/${MODE}_tests.py
if [ -f "${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh" ]; then
${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh;
else
${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.py;
fi
services:
- docker
# To avoid matrix explosion, just test against oldest->newest and # To avoid matrix explosion, just test against oldest->newest and

@ -1,4 +1,4 @@
#/usr/bin/env bash #!/usr/bin/env bash
echo '----- ulimits -----' echo '----- ulimits -----'
ulimit -a ulimit -a

@ -1,9 +0,0 @@
#!/usr/bin/env python
"""
For use by the Travis scripts, just print out the hostname of the Docker
daemon from the environment.
"""
import testlib
print(testlib.get_docker_host())
Loading…
Cancel
Save