Pythonize, split out and parallelize all jobs.
David Wilson
6 years ago
parent
b60a6d0f3b
commit
2a6dbb038f
@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import ci_lib
|
||||
|
||||
batches = [
|
||||
[
|
||||
# Must be installed separately, as PyNACL indirect requirement causes
|
||||
# newer version to be installed if done in a single pip run.
|
||||
'pip install "pycparser<2.19"',
|
||||
'pip install '
|
||||
'-r tests/requirements.txt '
|
||||
'-r tests/ansible/requirements.txt',
|
||||
]
|
||||
]
|
||||
|
||||
batches.extend(
|
||||
['docker pull mitogen/%s-test' % (distro,)]
|
||||
for distro in ci_lib.DISTROS
|
||||
)
|
||||
|
||||
ci_lib.run_batches(batches)
|
||||
@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import ci_lib
|
||||
|
||||
# Naturally DebOps only supports Debian.
|
||||
ci_lib.DISTROS = ['debian']
|
||||
|
||||
ci_lib.run_batches([
|
||||
[
|
||||
# Must be installed separately, as PyNACL indirect requirement causes
|
||||
# newer version to be installed if done in a single pip run.
|
||||
'pip install "pycparser<2.19"',
|
||||
'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
|
||||
],
|
||||
[
|
||||
'docker pull mitogen/debian-test',
|
||||
],
|
||||
])
|
||||
@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from __future__ import print_function
|
||||
import os
|
||||
|
||||
import ci_lib
|
||||
|
||||
|
||||
# DebOps only supports Debian.
|
||||
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
|
||||
|
||||
project_dir = os.path.join(ci_lib.TMP, 'project')
|
||||
key_file = os.path.join(
|
||||
ci_lib.GIT_ROOT,
|
||||
'tests/data/docker/mitogen__has_sudo_pubkey.key',
|
||||
)
|
||||
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
|
||||
inventory_path = 'ansible/inventory/hosts'
|
||||
docker_hostname = ci_lib.get_docker_hostname()
|
||||
|
||||
|
||||
with ci_lib.Fold('docker_setup'):
|
||||
containers = ci_lib.make_containers()
|
||||
ci_lib.start_containers(containers)
|
||||
|
||||
|
||||
with ci_lib.Fold('job_setup'):
|
||||
ci_lib.run('debops-init %s', project_dir)
|
||||
os.chdir(project_dir)
|
||||
|
||||
with open('.debops.cfg', 'w') as fp:
|
||||
fp.write(
|
||||
"[ansible defaults]\n"
|
||||
"strategy_plugins = %s/ansible_mitogen/plugins/strategy\n"
|
||||
"strategy = mitogen_linear\n"
|
||||
% (ci_lib.GIT_ROOT,)
|
||||
)
|
||||
|
||||
ci_lib.run('chmod go= %s', key_file)
|
||||
with open(vars_path, 'w') as fp:
|
||||
fp.write(
|
||||
"ansible_python_interpreter: /usr/bin/python2.7\n"
|
||||
"\n"
|
||||
"ansible_user: mitogen__has_sudo_pubkey\n"
|
||||
"ansible_become_pass: has_sudo_pubkey_password\n"
|
||||
"ansible_ssh_private_key_file: %s\n"
|
||||
"\n"
|
||||
# Speed up slow DH generation.
|
||||
"dhparam__bits: ['128', '64']\n"
|
||||
% (key_file,)
|
||||
)
|
||||
|
||||
with open(inventory_path, 'a') as fp:
|
||||
fp.writelines(
|
||||
'%(name)s '
|
||||
'ansible_host=%(hostname)s '
|
||||
'ansible_port=%(port)d '
|
||||
'\n'
|
||||
% container
|
||||
for container in containers
|
||||
)
|
||||
|
||||
print()
|
||||
print(' echo --- ansible/inventory/hosts: ---')
|
||||
ci_lib.run('cat ansible/inventory/hosts')
|
||||
print('---')
|
||||
print()
|
||||
|
||||
# Now we have real host key checking, we need to turn it off
|
||||
os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
|
||||
|
||||
|
||||
with ci_lib.Fold('first_run'):
|
||||
ci_lib.run('debops common')
|
||||
|
||||
|
||||
with ci_lib.Fold('second_run'):
|
||||
ci_lib.run('debops common')
|
||||
@ -1,90 +0,0 @@
|
||||
#!/bin/bash -ex
|
||||
# Run some invocations of DebOps.
|
||||
|
||||
TMPDIR="/tmp/debops-$$"
|
||||
TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}"
|
||||
TARGET_COUNT="${TARGET_COUNT:-2}"
|
||||
ANSIBLE_VERSION="${VER:-2.6.1}"
|
||||
DISTRO=debian # Naturally DebOps only supports Debian.
|
||||
|
||||
export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}"
|
||||
|
||||
function on_exit()
|
||||
{
|
||||
echo travis_fold:start:cleanup
|
||||
[ "$KEEP" ] || {
|
||||
rm -rf "$TMPDIR" || true
|
||||
for i in $(seq $TARGET_COUNT)
|
||||
do
|
||||
docker kill target$i || true
|
||||
done
|
||||
}
|
||||
echo travis_fold:end:cleanup
|
||||
}
|
||||
|
||||
trap on_exit EXIT
|
||||
mkdir "$TMPDIR"
|
||||
|
||||
|
||||
echo travis_fold:start:job_setup
|
||||
pip install -qqqU debops==0.7.2 ansible==${ANSIBLE_VERSION} |cat
|
||||
debops-init "$TMPDIR/project"
|
||||
cd "$TMPDIR/project"
|
||||
|
||||
cat > .debops.cfg <<-EOF
|
||||
[ansible defaults]
|
||||
strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy
|
||||
strategy = mitogen_linear
|
||||
EOF
|
||||
|
||||
chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
|
||||
|
||||
cat > ansible/inventory/group_vars/debops_all_hosts.yml <<-EOF
|
||||
ansible_python_interpreter: /usr/bin/python2.7
|
||||
|
||||
ansible_user: mitogen__has_sudo_pubkey
|
||||
ansible_become_pass: has_sudo_pubkey_password
|
||||
ansible_ssh_private_key_file: ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
|
||||
|
||||
# Speed up slow DH generation.
|
||||
dhparam__bits: ["128", "64"]
|
||||
EOF
|
||||
|
||||
DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)"
|
||||
|
||||
for i in $(seq $TARGET_COUNT)
|
||||
do
|
||||
port=$((2200 + $i))
|
||||
docker run \
|
||||
--rm \
|
||||
--detach \
|
||||
--publish 0.0.0.0:$port:22/tcp \
|
||||
--name=target$i \
|
||||
mitogen/${DISTRO}-test
|
||||
|
||||
echo \
|
||||
target$i \
|
||||
ansible_host=$DOCKER_HOSTNAME \
|
||||
ansible_port=$port \
|
||||
>> ansible/inventory/hosts
|
||||
done
|
||||
|
||||
echo
|
||||
echo --- ansible/inventory/hosts: ----
|
||||
cat ansible/inventory/hosts
|
||||
echo ---
|
||||
|
||||
# Now we have real host key checking, we need to turn it off. :)
|
||||
export ANSIBLE_HOST_KEY_CHECKING=False
|
||||
|
||||
echo travis_fold:end:job_setup
|
||||
|
||||
|
||||
echo travis_fold:start:first_run
|
||||
/usr/bin/time debops common "$@"
|
||||
echo travis_fold:end:first_run
|
||||
|
||||
|
||||
echo travis_fold:start:second_run
|
||||
/usr/bin/time debops common "$@"
|
||||
echo travis_fold:end:second_run
|
||||
@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import ci_lib
|
||||
|
||||
batches = [
|
||||
[
|
||||
'pip install "pycparser<2.19"',
|
||||
'pip install -r tests/requirements.txt',
|
||||
],
|
||||
[
|
||||
'docker pull mitogen/%s-test' % (ci_lib.DISTRO,),
|
||||
]
|
||||
]
|
||||
|
||||
ci_lib.run_batches(batches)
|
||||
@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python
|
||||
# Run the Mitogen tests.
|
||||
|
||||
import os
|
||||
|
||||
import ci_lib
|
||||
|
||||
os.environ.update({
|
||||
'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
|
||||
'MITOGEN_LOG_LEVEL': 'debug',
|
||||
'SKIP_ANSIBLE': '1',
|
||||
})
|
||||
|
||||
ci_lib.run('./run_tests -v')
|
||||
@ -1,5 +0,0 @@
|
||||
#!/bin/bash -ex
|
||||
# Run the Mitogen tests.
|
||||
|
||||
MITOGEN_TEST_DISTRO="${DISTRO:-debian}"
|
||||
MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -v
|
||||
@ -1,36 +1,22 @@
|
||||
#!/usr/bin/env python
|
||||
# Run preparation steps in parallel.
|
||||
|
||||
import subprocess
|
||||
import ci_lib
|
||||
|
||||
subprocess.check_call(
|
||||
batches = []
|
||||
batches.append([
|
||||
'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
|
||||
shell=True,
|
||||
)
|
||||
'sudo add-apt-repository ppa:deadsnakes/ppa',
|
||||
'sudo apt-get update',
|
||||
'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
|
||||
])
|
||||
|
||||
procs = [
|
||||
subprocess.Popen(
|
||||
'pip install -r dev_requirements.txt 2>&1 | cat',
|
||||
shell=True,
|
||||
),
|
||||
subprocess.Popen(
|
||||
"""
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa && \
|
||||
( sudo apt-get update 2>&1 | cat ) && \
|
||||
sudo apt-get -y install \
|
||||
python2.6 python2.6-dev libsasl2-dev libldap2-dev 2>&1 | cat
|
||||
""",
|
||||
shell=True,
|
||||
)
|
||||
]
|
||||
batches.append([
|
||||
'pip install -r dev_requirements.txt',
|
||||
])
|
||||
|
||||
procs += [
|
||||
subprocess.Popen(
|
||||
'docker pull mitogen/%s-test 2>&1 | cat' % (distro,),
|
||||
shell=True
|
||||
)
|
||||
batches.extend(
|
||||
['docker pull mitogen/%s-test' % (distro,)]
|
||||
for distro in ci_lib.DISTROS
|
||||
]
|
||||
)
|
||||
|
||||
assert [proc.wait() for proc in procs] == [0] * len(procs)
|
||||
ci_lib.run_batches(batches)
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
For use by the Travis scripts, just print out the hostname of the Docker
|
||||
daemon from the environment.
|
||||
"""
|
||||
|
||||
import testlib
|
||||
print(testlib.get_docker_host())
|
||||
Loading…
Reference in New Issue