Merge pull request #869 from moreati/azure-tox

ci: Switch Continuous Integration to Tox
pull/860/head
Alex Willmer 3 years ago committed by GitHub
commit 2885f62c45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -28,8 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
### Environment Variables ### Environment Variables
* `VER`: Ansible version the `_install` script should install. Default changes
over time.
* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`

@ -3,21 +3,9 @@
import ci_lib import ci_lib
batches = [ batches = [
[
'pip install '
'-r tests/requirements.txt '
'-r tests/ansible/requirements.txt',
# encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error
'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
],
[ [
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
] ]
] ]
batches[-1].extend([
'docker pull %s' % (ci_lib.image_for_distro(distro),)
for distro in ci_lib.DISTROS
])
ci_lib.run_batches(batches) ci_lib.run_batches(batches)

@ -5,20 +5,17 @@ parameters:
sign: false sign: false
steps: steps:
- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py" - task: UsePythonVersion@0
displayName: "Run prep_azure.py" displayName: Install python
inputs:
versionSpec: '$(python.version)'
- script: | - script: python -mpip install tox
echo "##vso[task.prependpath]/tmp/venv/bin" displayName: Install tooling
displayName: activate venv - script: tox -e $(tox.env)
displayName: "Run tests"
- script: .ci/$(MODE)_install.py
displayName: "Run $(MODE)_install.py"
env: env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
- script: .ci/$(MODE)_tests.py
displayName: "Run $(MODE)_tests.py"

@ -16,62 +16,46 @@ jobs:
steps: steps:
- template: azure-pipelines-steps.yml - template: azure-pipelines-steps.yml
pool: pool:
# https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md
vmImage: macOS-10.15 vmImage: macOS-10.15
strategy: strategy:
matrix: matrix:
Mito27_27: Mito27_27:
python.version: '2.7' python.version: '2.7'
MODE: mitogen tox.env: py27-mode_mitogen
# TODO: test python3, python3 tests are broken # TODO: test python3, python3 tests are broken
Ans210_27: Ans210_27:
python.version: '2.7' python.version: '2.7'
MODE: localhost_ansible tox.env: py27-mode_localhost-ansible2.10
VER: 2.10.0
# NOTE: this hangs when ran in Ubuntu 18.04 # NOTE: this hangs when ran in Ubuntu 18.04
Vanilla_210_27: Vanilla_210_27:
python.version: '2.7' python.version: '2.7'
MODE: localhost_ansible tox.env: py27-mode_localhost-ansible2.10
VER: 2.10.0
STRATEGY: linear STRATEGY: linear
ANSIBLE_SKIP_TAGS: resource_intensive ANSIBLE_SKIP_TAGS: resource_intensive
- job: Linux - job: Linux
pool: pool:
# https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
vmImage: "Ubuntu 18.04" vmImage: "Ubuntu 18.04"
steps: steps:
- template: azure-pipelines-steps.yml - template: azure-pipelines-steps.yml
strategy: strategy:
matrix: matrix:
#
# Confirmed working
#
Mito27Debian_27: Mito27Debian_27:
python.version: '2.7' python.version: '2.7'
MODE: mitogen tox.env: py27-mode_mitogen-distro_debian9
DISTRO: debian9
#MitoPy27CentOS6_26:
#python.version: '2.7'
#MODE: mitogen
#DISTRO: centos6
Mito36CentOS6_26: Mito36CentOS6_26:
python.version: '3.6' python.version: '3.6'
MODE: mitogen tox.env: py36-mode_mitogen-distro_centos6
DISTRO: centos6
Mito37Debian_27:
python.version: '3.7'
MODE: mitogen
DISTRO: debian9
Mito39Debian_27: Mito39Debian_27:
python.version: '3.9' python.version: '3.9'
MODE: mitogen tox.env: py39-mode_mitogen-distro_debian9
DISTRO: debian9
VER: 2.10.0
#Py26CentOS7: #Py26CentOS7:
#python.version: '2.7' #python.version: '2.7'
@ -117,15 +101,12 @@ jobs:
Ansible_210_27: Ansible_210_27:
python.version: '2.7' python.version: '2.7'
MODE: ansible tox.env: py27-mode_ansible-ansible2.10
VER: 2.10.0
Ansible_210_35: Ansible_210_36:
python.version: '3.5' python.version: '3.6'
MODE: ansible tox.env: py36-mode_ansible-ansible2.10
VER: 2.10.0
Ansible_210_39: Ansible_210_39:
python.version: '3.9' python.version: '3.9'
MODE: ansible tox.env: py39-mode_ansible-ansible2.10
VER: 2.10.0

@ -59,12 +59,6 @@ def have_docker():
return proc.wait() == 0 return proc.wait() == 0
# -----------------
# Force line buffering on stdout.
sys.stdout = os.fdopen(1, 'w', 1)
def _argv(s, *args): def _argv(s, *args):
"""Interpolate a command line using *args, return an argv style list. """Interpolate a command line using *args, return an argv style list.
@ -195,13 +189,11 @@ class Fold(object):
os.environ.setdefault('ANSIBLE_STRATEGY', os.environ.setdefault('ANSIBLE_STRATEGY',
os.environ.get('STRATEGY', 'mitogen_linear')) os.environ.get('STRATEGY', 'mitogen_linear'))
# Ignoreed when MODE=mitogen
ANSIBLE_VERSION = os.environ.get('VER', '2.6.2')
GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
# Used only when MODE=mitogen # Used only when MODE=mitogen
DISTRO = os.environ.get('DISTRO', 'debian') DISTRO = os.environ.get('DISTRO', 'debian9')
# Used only when MODE=ansible # Used only when MODE=ansible
DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
BASE_PORT = 2200 BASE_PORT = 2200
TMP = TempDir().path TMP = TempDir().path
@ -254,11 +246,13 @@ def make_containers(name_prefix='', port_offset=0):
>>> pprint.pprint(make_containers()) >>> pprint.pprint(make_containers())
[{'distro': 'debian', [{'distro': 'debian',
'hostname': 'localhost', 'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/debian-test',
'name': 'target-debian-1', 'name': 'target-debian-1',
'port': 2201, 'port': 2201,
'python_path': '/usr/bin/python'}, 'python_path': '/usr/bin/python'},
{'distro': 'centos6', {'distro': 'centos6',
'hostname': 'localhost', 'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
'name': 'target-centos6-2', 'name': 'target-centos6-2',
'port': 2202, 'port': 2202,
'python_path': '/usr/bin/python'}] 'python_path': '/usr/bin/python'}]
@ -280,6 +274,7 @@ def make_containers(name_prefix='', port_offset=0):
for x in range(count): for x in range(count):
lst.append({ lst.append({
"distro": firstbit(distro), "distro": firstbit(distro),
"image": image_for_distro(distro),
"name": name_prefix + ("target-%s-%s" % (distro, i)), "name": name_prefix + ("target-%s-%s" % (distro, i)),
"hostname": docker_hostname, "hostname": docker_hostname,
"port": BASE_PORT + i + port_offset, "port": BASE_PORT + i + port_offset,
@ -358,7 +353,7 @@ def start_containers(containers):
"--publish 0.0.0.0:%(port)s:22/tcp " "--publish 0.0.0.0:%(port)s:22/tcp "
"--hostname=%(name)s " "--hostname=%(name)s "
"--name=%(name)s " "--name=%(name)s "
"mitogen/%(distro)s-test " "%(image)s"
% container % container
] ]
for container in containers for container in containers

@ -7,11 +7,10 @@ ci_lib.DISTROS = ['debian']
ci_lib.run_batches([ ci_lib.run_batches([
[ [
'pip install -qqq "debops[ansible]==2.1.2" "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION), 'pip install -qqq "debops[ansible]==2.1.2"',
], ],
[ [
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
'docker pull %s' % (ci_lib.image_for_distro('debian'),),
], ],
]) ])

@ -3,12 +3,6 @@
import ci_lib import ci_lib
batches = [ batches = [
[
'pip install '
'-r tests/requirements.txt '
'-r tests/ansible/requirements.txt',
'pip install -q "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
]
] ]
ci_lib.run_batches(batches) ci_lib.run_batches(batches)

@ -3,15 +3,11 @@
import ci_lib import ci_lib
batches = [ batches = [
[
'pip install -r tests/requirements.txt',
]
] ]
if ci_lib.have_docker(): if ci_lib.have_docker():
batches.append([ batches.append([
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
]) ])

@ -5,7 +5,6 @@ import ci_lib
batches = [ batches = [
[ [
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
], ],
[ [
'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',

@ -1,95 +0,0 @@
#!/usr/bin/env python
import os
import sys
import ci_lib
batches = []
if 0 and os.uname()[0] == 'Linux':
batches += [
[
"sudo chown `whoami`: ~",
"chmod u=rwx,g=rx,o= ~",
"sudo mkdir /var/run/sshd",
"sudo /etc/init.d/ssh start",
"mkdir -p ~/.ssh",
"chmod u=rwx,go= ~/.ssh",
"ssh-keyscan -H localhost >> ~/.ssh/known_hosts",
"chmod u=rw,go= ~/.ssh/known_hosts",
"cat tests/data/docker/mitogen__has_sudo_pubkey.key > ~/.ssh/id_rsa",
"chmod u=rw,go= ~/.ssh/id_rsa",
"cat tests/data/docker/mitogen__has_sudo_pubkey.key.pub > ~/.ssh/authorized_keys",
"chmod u=rw,go=r ~/.ssh/authorized_keys",
]
]
# setup venv, need all python commands in 1 list to be subprocessed at the same time
venv_steps = []
need_to_fix_psycopg2 = False
is_python3 = os.environ['PYTHONVERSION'].startswith('3')
# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
# broken symlinks, incorrect permissions and missing codecs. So we use the
# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
# stuff into. The virtualenv can probably be removed again, but this was a
# hard-fought battle and for now I am tired of this crap.
if ci_lib.have_apt():
venv_steps.extend([
'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
'sudo add-apt-repository ppa:deadsnakes/ppa',
'sudo apt-get update',
'sudo apt-get -y install '
'python{pv} '
'python{pv}-dev '
'libsasl2-dev '
'libldap2-dev '
.format(pv=os.environ['PYTHONVERSION']),
'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}'
.format(pv=os.environ['PYTHONVERSION'])
])
if is_python3:
venv_steps.append('sudo apt-get -y install python{pv}-venv'.format(pv=os.environ['PYTHONVERSION']))
# TODO: somehow `Mito36CentOS6_26` has both brew and apt installed https://dev.azure.com/dw-mitogen/Mitogen/_build/results?buildId=1031&view=logs&j=7bdbcdc6-3d3e-568d-ccf8-9ddca1a9623a&t=73d379b6-4eea-540f-c97e-046a2f620483
elif is_python3 and ci_lib.have_brew():
# Mac's System Integrity Protection prevents symlinking /usr/bin
# and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
# so we'll use /usr/local/bin/python for everything
# /usr/local/bin/python2.7 already exists!
need_to_fix_psycopg2 = True
venv_steps.append(
'brew install python@{pv} postgresql'
.format(pv=os.environ['PYTHONVERSION'])
)
# need wheel before building virtualenv because of bdist_wheel and setuptools deps
venv_steps.append('/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools'.format(pv=os.environ['PYTHONVERSION']))
if os.environ['PYTHONVERSION'].startswith('2'):
venv_steps.extend([
'/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(pv=os.environ['PYTHONVERSION']),
'/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(pv=os.environ['PYTHONVERSION'])
])
else:
venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(pv=os.environ['PYTHONVERSION']))
# fixes https://stackoverflow.com/questions/59595649/can-not-install-psycopg2-on-macos-catalina https://github.com/Azure/azure-cli/issues/12854#issuecomment-619213863
if need_to_fix_psycopg2:
venv_steps.append('/tmp/venv/bin/pip3 install psycopg2==2.8.5 psycopg2-binary')
venv_steps.extend([
# pbr is a transitive setup_requires of hdrhistogram. If it's not already
# installed then setuptools attempts to use easy_install, which fails.
'/tmp/venv/bin/pip install pbr==5.6.0',
])
batches.append(venv_steps)
ci_lib.run_batches(batches)

@ -21,6 +21,9 @@ To avail of fixes in an unreleased version, please download a ZIP file
v0.3.1.dev0 (unreleased) v0.3.1.dev0 (unreleased)
------------------------ ------------------------
* :gh:issue:`869` Continuous Integration tests are now run with Tox
* :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04
v0.3.0 (2021-11-24) v0.3.0 (2021-11-24)
------------------- -------------------

@ -23,15 +23,15 @@
# pip --no-python-version-warning # pip --no-python-version-warning
# pip --disable-pip-version-check # pip --disable-pip-version-check
# TODO distros=-py3
[tox] [tox]
envlist = envlist =
init, init,
py{27,35,39}-mode_ansible-distros_centos, py{27,36,39}-mode_ansible-ansible2.10,
py{27,35,39}-mode_ansible-distros_debian, py{27,36,39}-mode_mitogen-distro_centos{6,7,8},
py{27,35,39}-mode_ansible-distros_ubuntu, py{27,36,39}-mode_mitogen-distro_debian{9,10,11},
py{27,35,39}-mode_mitogen-distro_centos{6,7,8}, py{27,36,39}-mode_mitogen-distro_ubuntu{1604,1804,2004},
py{27,35,39}-mode_mitogen-distro_debian{9,10,11},
py{27,35,39}-mode_mitogen-distro_ubuntu{1604,1804,2004},
report, report,
requires = requires =
tox-factor tox-factor
@ -45,29 +45,37 @@ basepython =
py37: python3.7 py37: python3.7
py38: python3.8 py38: python3.8
py39: python3.9 py39: python3.9
deps =
-r{toxinidir}/tests/requirements.txt
mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
ansible2.3: ansible=2.3.3.0
ansible2.4: ansible=2.4.6.0
ansible2.8: ansible=2.8.3
ansible2.9: ansible=2.9.6
ansible2.10: ansible-base<2.10.14
ansible2.10: ansible==2.10.0
install_command = install_command =
python -m pip --no-python-version-warning install {opts} {packages} python -m pip --no-python-version-warning install {opts} {packages}
commands_pre = commands_pre =
mode_ansible: {toxinidir}/.ci/ansible_install.py mode_ansible: {toxinidir}/.ci/ansible_install.py
mode_debops_common: {toxinidir}/.ci/debops_common_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py
mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py
mode_mitogen: {toxinidir}/.ci/mitogen_install.py mode_mitogen: {toxinidir}/.ci/mitogen_install.py
commands = commands =
mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_ansible: {toxinidir}/.ci/ansible_tests.py
mode_debops_common: {toxinidir}/.ci/debops_common_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
mode_localhost: {toxinidir}/.ci/localhost_ansible_tests.py
mode_mitogen: {toxinidir}/.ci/mitogen_tests.py mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
passenv = passenv =
ANSIBLE_* ANSIBLE_*
AWS_ACCESS_KEY_ID
AWS_DEFAULT_REGION
AWS_SECRET_ACCESS_KEY
HOME HOME
setenv = setenv =
ANSIBLE_SKIP_TAGS = requires_local_sudo
NOCOVERAGE_ERASE = 1 NOCOVERAGE_ERASE = 1
NOCOVERAGE_REPORT = 1 NOCOVERAGE_REPORT = 1
VER=2.10.5 # Only applicable to MODE=mitogen
ansible2.3: VER=2.3.3.0
ansible2.4: VER=2.4.6.0
ansible2.8: VER=2.8.3
ansible2.9: VER=2.9.6
ansible2.10: VER=2.10.0
distro_centos5: DISTRO=centos5 distro_centos5: DISTRO=centos5
distro_centos6: DISTRO=centos6 distro_centos6: DISTRO=centos6
distro_centos7: DISTRO=centos7 distro_centos7: DISTRO=centos7
@ -78,6 +86,7 @@ setenv =
distro_ubuntu1604: DISTRO=ubuntu1604 distro_ubuntu1604: DISTRO=ubuntu1604
distro_ubuntu1804: DISTRO=ubuntu1804 distro_ubuntu1804: DISTRO=ubuntu1804
distro_ubuntu2004: DISTRO=ubuntu2004 distro_ubuntu2004: DISTRO=ubuntu2004
# Note the plural, only applicable to MODE=ansible
distros_centos: DISTROS=centos6 centos7 centos8 distros_centos: DISTROS=centos6 centos7 centos8
distros_centos5: DISTROS=centos5 distros_centos5: DISTROS=centos5
distros_centos6: DISTROS=centos6 distros_centos6: DISTROS=centos6
@ -95,6 +104,12 @@ setenv =
mode_debops_common: MODE=debops_common mode_debops_common: MODE=debops_common
mode_mitogen: MODE=mitogen mode_mitogen: MODE=mitogen
strategy_linear: STRATEGY=linear strategy_linear: STRATEGY=linear
whitelist_externals =
docker
docker-credential-secretservice
echo
gpg2
pass
[testenv:init] [testenv:init]
basepython = python3 basepython = python3

Loading…
Cancel
Save