Merge remote-tracking branch 'origin/dmw' into HEAD
Refactored most CI stuff.issue510
David Wilson
7 years ago
commit
ef06e7e16a
@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import ci_lib
|
||||
|
||||
batches = [
|
||||
[
|
||||
# Must be installed separately, as PyNACL indirect requirement causes
|
||||
# newer version to be installed if done in a single pip run.
|
||||
'pip install "pycparser<2.19"',
|
||||
'pip install '
|
||||
'-r tests/requirements.txt '
|
||||
'-r tests/ansible/requirements.txt',
|
||||
]
|
||||
]
|
||||
|
||||
batches.extend(
|
||||
['docker pull mitogen/%s-test' % (distro,)]
|
||||
for distro in ci_lib.DISTROS
|
||||
)
|
||||
|
||||
ci_lib.run_batches(batches)
|
||||
@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import ci_lib
|
||||
|
||||
# Naturally DebOps only supports Debian.
|
||||
ci_lib.DISTROS = ['debian']
|
||||
|
||||
ci_lib.run_batches([
|
||||
[
|
||||
# Must be installed separately, as PyNACL indirect requirement causes
|
||||
# newer version to be installed if done in a single pip run.
|
||||
'pip install "pycparser<2.19"',
|
||||
'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
|
||||
],
|
||||
[
|
||||
'docker pull mitogen/debian-test',
|
||||
],
|
||||
])
|
||||
@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from __future__ import print_function
|
||||
import os
|
||||
|
||||
import ci_lib
|
||||
|
||||
|
||||
# DebOps only supports Debian.
|
||||
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
|
||||
|
||||
project_dir = os.path.join(ci_lib.TMP, 'project')
|
||||
key_file = os.path.join(
|
||||
ci_lib.GIT_ROOT,
|
||||
'tests/data/docker/mitogen__has_sudo_pubkey.key',
|
||||
)
|
||||
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
|
||||
inventory_path = 'ansible/inventory/hosts'
|
||||
docker_hostname = ci_lib.get_docker_hostname()
|
||||
|
||||
|
||||
with ci_lib.Fold('docker_setup'):
|
||||
containers = ci_lib.make_containers()
|
||||
ci_lib.start_containers(containers)
|
||||
|
||||
|
||||
with ci_lib.Fold('job_setup'):
|
||||
ci_lib.run('debops-init %s', project_dir)
|
||||
os.chdir(project_dir)
|
||||
|
||||
with open('.debops.cfg', 'w') as fp:
|
||||
fp.write(
|
||||
"[ansible defaults]\n"
|
||||
"strategy_plugins = %s/ansible_mitogen/plugins/strategy\n"
|
||||
"strategy = mitogen_linear\n"
|
||||
% (ci_lib.GIT_ROOT,)
|
||||
)
|
||||
|
||||
ci_lib.run('chmod go= %s', key_file)
|
||||
with open(vars_path, 'w') as fp:
|
||||
fp.write(
|
||||
"ansible_python_interpreter: /usr/bin/python2.7\n"
|
||||
"\n"
|
||||
"ansible_user: mitogen__has_sudo_pubkey\n"
|
||||
"ansible_become_pass: has_sudo_pubkey_password\n"
|
||||
"ansible_ssh_private_key_file: %s\n"
|
||||
"\n"
|
||||
# Speed up slow DH generation.
|
||||
"dhparam__bits: ['128', '64']\n"
|
||||
% (key_file,)
|
||||
)
|
||||
|
||||
with open(inventory_path, 'a') as fp:
|
||||
fp.writelines(
|
||||
'%(name)s '
|
||||
'ansible_host=%(hostname)s '
|
||||
'ansible_port=%(port)d '
|
||||
'\n'
|
||||
% container
|
||||
for container in containers
|
||||
)
|
||||
|
||||
print()
|
||||
print(' echo --- ansible/inventory/hosts: ---')
|
||||
ci_lib.run('cat ansible/inventory/hosts')
|
||||
print('---')
|
||||
print()
|
||||
|
||||
# Now we have real host key checking, we need to turn it off
|
||||
os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
|
||||
|
||||
|
||||
with ci_lib.Fold('first_run'):
|
||||
ci_lib.run('debops common')
|
||||
|
||||
|
||||
with ci_lib.Fold('second_run'):
|
||||
ci_lib.run('debops common')
|
||||
@ -1,90 +0,0 @@
|
||||
#!/bin/bash -ex
|
||||
# Run some invocations of DebOps.
|
||||
|
||||
TMPDIR="/tmp/debops-$$"
|
||||
TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}"
|
||||
TARGET_COUNT="${TARGET_COUNT:-2}"
|
||||
ANSIBLE_VERSION="${VER:-2.6.1}"
|
||||
DISTRO=debian # Naturally DebOps only supports Debian.
|
||||
|
||||
export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}"
|
||||
|
||||
function on_exit()
|
||||
{
|
||||
echo travis_fold:start:cleanup
|
||||
[ "$KEEP" ] || {
|
||||
rm -rf "$TMPDIR" || true
|
||||
for i in $(seq $TARGET_COUNT)
|
||||
do
|
||||
docker kill target$i || true
|
||||
done
|
||||
}
|
||||
echo travis_fold:end:cleanup
|
||||
}
|
||||
|
||||
trap on_exit EXIT
|
||||
mkdir "$TMPDIR"
|
||||
|
||||
|
||||
echo travis_fold:start:job_setup
|
||||
pip install -qqqU debops==0.7.2 ansible==${ANSIBLE_VERSION} |cat
|
||||
debops-init "$TMPDIR/project"
|
||||
cd "$TMPDIR/project"
|
||||
|
||||
cat > .debops.cfg <<-EOF
|
||||
[ansible defaults]
|
||||
strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy
|
||||
strategy = mitogen_linear
|
||||
EOF
|
||||
|
||||
chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
|
||||
|
||||
cat > ansible/inventory/group_vars/debops_all_hosts.yml <<-EOF
|
||||
ansible_python_interpreter: /usr/bin/python2.7
|
||||
|
||||
ansible_user: mitogen__has_sudo_pubkey
|
||||
ansible_become_pass: has_sudo_pubkey_password
|
||||
ansible_ssh_private_key_file: ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key
|
||||
|
||||
# Speed up slow DH generation.
|
||||
dhparam__bits: ["128", "64"]
|
||||
EOF
|
||||
|
||||
DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)"
|
||||
|
||||
for i in $(seq $TARGET_COUNT)
|
||||
do
|
||||
port=$((2200 + $i))
|
||||
docker run \
|
||||
--rm \
|
||||
--detach \
|
||||
--publish 0.0.0.0:$port:22/tcp \
|
||||
--name=target$i \
|
||||
mitogen/${DISTRO}-test
|
||||
|
||||
echo \
|
||||
target$i \
|
||||
ansible_host=$DOCKER_HOSTNAME \
|
||||
ansible_port=$port \
|
||||
>> ansible/inventory/hosts
|
||||
done
|
||||
|
||||
echo
|
||||
echo --- ansible/inventory/hosts: ----
|
||||
cat ansible/inventory/hosts
|
||||
echo ---
|
||||
|
||||
# Now we have real host key checking, we need to turn it off. :)
|
||||
export ANSIBLE_HOST_KEY_CHECKING=False
|
||||
|
||||
echo travis_fold:end:job_setup
|
||||
|
||||
|
||||
echo travis_fold:start:first_run
|
||||
/usr/bin/time debops common "$@"
|
||||
echo travis_fold:end:first_run
|
||||
|
||||
|
||||
echo travis_fold:start:second_run
|
||||
/usr/bin/time debops common "$@"
|
||||
echo travis_fold:end:second_run
|
||||
@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import ci_lib
|
||||
|
||||
batches = [
|
||||
[
|
||||
'pip install "pycparser<2.19"',
|
||||
'pip install -r tests/requirements.txt',
|
||||
],
|
||||
[
|
||||
'docker pull mitogen/%s-test' % (ci_lib.DISTRO,),
|
||||
]
|
||||
]
|
||||
|
||||
ci_lib.run_batches(batches)
|
||||
@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python
|
||||
# Run the Mitogen tests.
|
||||
|
||||
import os
|
||||
|
||||
import ci_lib
|
||||
|
||||
os.environ.update({
|
||||
'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
|
||||
'MITOGEN_LOG_LEVEL': 'debug',
|
||||
'SKIP_ANSIBLE': '1',
|
||||
})
|
||||
|
||||
ci_lib.run('./run_tests -v')
|
||||
@ -1,5 +0,0 @@
|
||||
#!/bin/bash -ex
|
||||
# Run the Mitogen tests.
|
||||
|
||||
MITOGEN_TEST_DISTRO="${DISTRO:-debian}"
|
||||
MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -v
|
||||
@ -1,36 +1,22 @@
|
||||
#!/usr/bin/env python
|
||||
# Run preparation steps in parallel.
|
||||
|
||||
import subprocess
|
||||
import ci_lib
|
||||
|
||||
subprocess.check_call(
|
||||
batches = []
|
||||
batches.append([
|
||||
'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
|
||||
shell=True,
|
||||
)
|
||||
'sudo add-apt-repository ppa:deadsnakes/ppa',
|
||||
'sudo apt-get update',
|
||||
'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
|
||||
])
|
||||
|
||||
procs = [
|
||||
subprocess.Popen(
|
||||
'pip install -r dev_requirements.txt 2>&1 | cat',
|
||||
shell=True,
|
||||
),
|
||||
subprocess.Popen(
|
||||
"""
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa && \
|
||||
( sudo apt-get update 2>&1 | cat ) && \
|
||||
sudo apt-get -y install \
|
||||
python2.6 python2.6-dev libsasl2-dev libldap2-dev 2>&1 | cat
|
||||
""",
|
||||
shell=True,
|
||||
)
|
||||
]
|
||||
batches.append([
|
||||
'pip install -r dev_requirements.txt',
|
||||
])
|
||||
|
||||
procs += [
|
||||
subprocess.Popen(
|
||||
'docker pull mitogen/%s-test 2>&1 | cat' % (distro,),
|
||||
shell=True
|
||||
)
|
||||
batches.extend(
|
||||
['docker pull mitogen/%s-test' % (distro,)]
|
||||
for distro in ci_lib.DISTROS
|
||||
]
|
||||
)
|
||||
|
||||
assert [proc.wait() for proc in procs] == [0] * len(procs)
|
||||
ci_lib.run_batches(batches)
|
||||
|
||||
@ -1,19 +1,11 @@
|
||||
-r docs/docs-requirements.txt
|
||||
# This file is no longer used by CI jobs, it's mostly for interactive use.
|
||||
# Instead CI jobs grab the relevant sub-requirement.
|
||||
|
||||
# mitogen_tests
|
||||
-r tests/requirements.txt
|
||||
|
||||
# ansible_tests
|
||||
-r tests/ansible/requirements.txt
|
||||
psutil==5.4.8
|
||||
coverage==4.5.1
|
||||
Django==1.6.11 # Last version supporting 2.6.
|
||||
mock==2.0.0
|
||||
pytz==2018.5
|
||||
cffi==1.11.2 # Random pin to try and fix pyparser==2.18 not having effect
|
||||
pycparser==2.18 # Last version supporting 2.6.
|
||||
faulthandler==3.1; python_version < '3.3' # used by testlib
|
||||
pytest-catchlog==1.2.2
|
||||
pytest==3.1.2
|
||||
PyYAML==3.11; python_version < '2.7'
|
||||
PyYAML==3.12; python_version >= '2.7'
|
||||
timeoutcontext==1.2.0
|
||||
unittest2==1.1.0
|
||||
# Fix InsecurePlatformWarning while creating py26 tox environment
|
||||
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
|
||||
urllib3[secure]; python_version < '2.7.9'
|
||||
|
||||
# readthedocs
|
||||
-r docs/requirements.txt
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
Sphinx==1.7.1
|
||||
sphinx-autobuild==0.6.0 # Last version to support Python 2.6
|
||||
sphinxcontrib-programoutput==0.11
|
||||
alabaster==0.7.10
|
||||
@ -0,0 +1,35 @@
|
||||
# /etc/environment
|
||||
|
||||
- file:
|
||||
path: /etc/environment
|
||||
state: absent
|
||||
become: true
|
||||
|
||||
- shell: echo $MAGIC_ETC_ENV
|
||||
register: echo
|
||||
|
||||
- assert:
|
||||
that: echo.stdout == ""
|
||||
|
||||
- copy:
|
||||
dest: /etc/environment
|
||||
content: |
|
||||
MAGIC_ETC_ENV=555
|
||||
become: true
|
||||
|
||||
- shell: echo $MAGIC_ETC_ENV
|
||||
register: echo
|
||||
|
||||
- assert:
|
||||
that: echo.stdout == "555"
|
||||
|
||||
- file:
|
||||
path: /etc/environment
|
||||
state: absent
|
||||
become: true
|
||||
|
||||
- shell: echo $MAGIC_ETC_ENV
|
||||
register: echo
|
||||
|
||||
- assert:
|
||||
that: echo.stdout == ""
|
||||
@ -0,0 +1,32 @@
|
||||
# ~/.pam_environment
|
||||
|
||||
- file:
|
||||
path: ~/.pam_environment
|
||||
state: absent
|
||||
|
||||
- shell: echo $MAGIC_PAM_ENV
|
||||
register: echo
|
||||
|
||||
- assert:
|
||||
that: echo.stdout == ""
|
||||
|
||||
- copy:
|
||||
dest: ~/.pam_environment
|
||||
content: |
|
||||
MAGIC_PAM_ENV=321
|
||||
|
||||
- shell: echo $MAGIC_PAM_ENV
|
||||
register: echo
|
||||
|
||||
- assert:
|
||||
that: echo.stdout == "321"
|
||||
|
||||
- file:
|
||||
path: ~/.pam_environment
|
||||
state: absent
|
||||
|
||||
- shell: echo $MAGIC_PAM_ENV
|
||||
register: echo
|
||||
|
||||
- assert:
|
||||
that: echo.stdout == ""
|
||||
@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import pprint
|
||||
|
||||
pprint.pprint(dict(os.environ))
|
||||
@ -1,2 +0,0 @@
|
||||
#!/bin/bash
|
||||
set
|
||||
@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
# This script exists to test the behavior of Stream.python_path being set to a
|
||||
# list. It sets an environmnt variable that we can detect, then executes any
|
||||
# arguments passed to it.
|
||||
export EXECUTED_VIA_ENV_WRAPPER=1
|
||||
if [ "${1:0:1}" == "-" ]; then
|
||||
exec "$PYTHON" "$@"
|
||||
else
|
||||
export ENV_WRAPPER_FIRST_ARG="$1"
|
||||
shift
|
||||
exec "$@"
|
||||
fi
|
||||
@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv)
|
||||
os.environ['THIS_IS_STUB_PYTHON'] = '1'
|
||||
|
||||
if sys.argv[1].startswith('-'):
|
||||
os.execvp(sys.executable, [sys.executable] + sys.argv[1:])
|
||||
else:
|
||||
os.environ['STUB_PYTHON_FIRST_ARG'] = sys.argv.pop(1)
|
||||
os.execvp(sys.executable, sys.argv[1:])
|
||||
@ -0,0 +1,15 @@
|
||||
psutil==5.4.8
|
||||
coverage==4.5.1
|
||||
Django==1.6.11 # Last version supporting 2.6.
|
||||
mock==2.0.0
|
||||
pytz==2018.5
|
||||
cffi==1.11.2 # Random pin to try and fix pyparser==2.18 not having effect
|
||||
pycparser==2.18 # Last version supporting 2.6.
|
||||
faulthandler==3.1; python_version < '3.3' # used by testlib
|
||||
pytest-catchlog==1.2.2
|
||||
pytest==3.1.2
|
||||
timeoutcontext==1.2.0
|
||||
unittest2==1.1.0
|
||||
# Fix InsecurePlatformWarning while creating py26 tox environment
|
||||
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
|
||||
urllib3[secure]; python_version < '2.7.9'
|
||||
@ -1,9 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
For use by the Travis scripts, just print out the hostname of the Docker
|
||||
daemon from the environment.
|
||||
"""
|
||||
|
||||
import testlib
|
||||
print(testlib.get_docker_host())
|
||||
Loading…
Reference in New Issue