From c5c65ab0d2716d5131b3fbcd23b8d69d6519c94a Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 10 Nov 2021 01:05:37 +0000 Subject: [PATCH] ci: Switch Continuous Integration to Tox Also - Simplifies adding support for additional Ansible versions - Unifies Python package versioning in CI and local test environments - Matches Python versions tested, with those declared in setup.py - Expands targets covered by automated Ansible tests to - centos6, centos8 - debian9, debian11 - ubuntu1604, ubuntu2004 --- .ci/README.md | 2 - .ci/ansible_install.py | 12 ---- .ci/azure-pipelines-steps.yml | 19 +++---- .ci/azure-pipelines.yml | 47 +++++----------- .ci/ci_lib.py | 17 ++---- .ci/debops_common_install.py | 3 +- .ci/localhost_ansible_install.py | 6 -- .ci/mitogen_install.py | 4 -- .ci/mitogen_py24_install.py | 1 - .ci/prep_azure.py | 95 -------------------------------- docs/changelog.rst | 3 + tox.ini | 41 +++++++++----- 12 files changed, 60 insertions(+), 190 deletions(-) delete mode 100755 .ci/prep_azure.py diff --git a/.ci/README.md b/.ci/README.md index 4d033602..67a3805b 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,8 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `VER`: Ansible version the `_install` script should install. Default changes - over time. * `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 63dbe563..900303f6 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -3,21 +3,9 @@ import ci_lib batches = [ - [ - 'pip install ' - '-r tests/requirements.txt ' - '-r tests/ansible/requirements.txt', - # encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error - 'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION) - ], [ 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', ] ] -batches[-1].extend([ - 'docker pull %s' % (ci_lib.image_for_distro(distro),) - for distro in ci_lib.DISTROS -]) - ci_lib.run_batches(batches) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 98d1146a..3249da41 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -5,20 +5,17 @@ parameters: sign: false steps: -- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py" - displayName: "Run prep_azure.py" +- task: UsePythonVersion@0 + displayName: Install python + inputs: + versionSpec: '$(python.version)' -- script: | - echo "##vso[task.prependpath]/tmp/venv/bin" +- script: python -mpip install tox + displayName: Install tooling - displayName: activate venv - -- script: .ci/$(MODE)_install.py - displayName: "Run $(MODE)_install.py" +- script: tox -e $(tox.env) + displayName: "Run tests" env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) - -- script: .ci/$(MODE)_tests.py - displayName: "Run $(MODE)_tests.py" diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index bde7fa87..dd9e9c68 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -16,62 +16,46 @@ jobs: steps: - template: azure-pipelines-steps.yml pool: + # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md vmImage: macOS-10.15 strategy: matrix: Mito27_27: python.version: '2.7' - MODE: mitogen + tox.env: py27-mode_mitogen + # TODO: test python3, python3 tests are broken Ans210_27: python.version: '2.7' - MODE: localhost_ansible - VER: 2.10.0 + tox.env: py27-mode_localhost-ansible2.10 # NOTE: this hangs when ran in Ubuntu 18.04 Vanilla_210_27: python.version: '2.7' - MODE: localhost_ansible - VER: 2.10.0 + tox.env: py27-mode_localhost-ansible2.10 STRATEGY: linear ANSIBLE_SKIP_TAGS: resource_intensive - job: Linux pool: + # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md vmImage: "Ubuntu 18.04" steps: - template: azure-pipelines-steps.yml strategy: matrix: - # - # Confirmed working - # Mito27Debian_27: python.version: '2.7' - MODE: mitogen - DISTRO: debian9 - - #MitoPy27CentOS6_26: - #python.version: '2.7' - #MODE: mitogen - #DISTRO: centos6 + tox.env: py27-mode_mitogen-distro_debian9 Mito36CentOS6_26: python.version: '3.6' - MODE: mitogen - DISTRO: centos6 - - Mito37Debian_27: - python.version: '3.7' - MODE: mitogen - DISTRO: debian9 + tox.env: py36-mode_mitogen-distro_centos6 Mito39Debian_27: python.version: '3.9' - MODE: mitogen - DISTRO: debian9 - VER: 2.10.0 + tox.env: py39-mode_mitogen-distro_debian9 #Py26CentOS7: #python.version: '2.7' @@ -117,15 +101,12 @@ jobs: Ansible_210_27: python.version: '2.7' - MODE: ansible - VER: 2.10.0 + tox.env: py27-mode_ansible-ansible2.10 - Ansible_210_35: - python.version: '3.5' - MODE: ansible - VER: 2.10.0 + Ansible_210_36: + python.version: '3.6' + tox.env: py36-mode_ansible-ansible2.10 Ansible_210_39: python.version: '3.9' - MODE: ansible - VER: 2.10.0 + tox.env: py39-mode_ansible-ansible2.10 diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 428d3ab9..1d9b0d73 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -59,12 +59,6 @@ def have_docker(): return proc.wait() == 0 -# ----------------- - -# Force line buffering on stdout. -sys.stdout = os.fdopen(1, 'w', 1) - - def _argv(s, *args): """Interpolate a command line using *args, return an argv style list. @@ -195,13 +189,11 @@ class Fold(object): os.environ.setdefault('ANSIBLE_STRATEGY', os.environ.get('STRATEGY', 'mitogen_linear')) -# Ignoreed when MODE=mitogen -ANSIBLE_VERSION = os.environ.get('VER', '2.6.2') GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # Used only when MODE=mitogen -DISTRO = os.environ.get('DISTRO', 'debian') +DISTRO = os.environ.get('DISTRO', 'debian9') # Used only when MODE=ansible -DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() +DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) BASE_PORT = 2200 TMP = TempDir().path @@ -254,11 +246,13 @@ def make_containers(name_prefix='', port_offset=0): >>> pprint.pprint(make_containers()) [{'distro': 'debian', 'hostname': 'localhost', + 'image': 'public.ecr.aws/n5z0e8q9/debian-test', 'name': 'target-debian-1', 'port': 2201, 'python_path': '/usr/bin/python'}, {'distro': 'centos6', 'hostname': 'localhost', + 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] @@ -280,6 +274,7 @@ def make_containers(name_prefix='', port_offset=0): for x in range(count): lst.append({ "distro": firstbit(distro), + "image": image_for_distro(distro), "name": name_prefix + ("target-%s-%s" % (distro, i)), "hostname": docker_hostname, "port": BASE_PORT + i + port_offset, @@ -358,7 +353,7 @@ def start_containers(containers): "--publish 0.0.0.0:%(port)s:22/tcp " "--hostname=%(name)s " "--name=%(name)s " - "mitogen/%(distro)s-test " + "%(image)s" % container ] for container in containers diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 50d556cc..75124fe3 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -7,11 +7,10 @@ ci_lib.DISTROS = ['debian'] ci_lib.run_batches([ [ - 'pip install -qqq "debops[ansible]==2.1.2" "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION), + 'pip install -qqq "debops[ansible]==2.1.2"', ], [ 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', - 'docker pull %s' % (ci_lib.image_for_distro('debian'),), ], ]) diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py index e70bcf18..d08ddafc 100755 --- a/.ci/localhost_ansible_install.py +++ b/.ci/localhost_ansible_install.py @@ -3,12 +3,6 @@ import ci_lib batches = [ - [ - 'pip install ' - '-r tests/requirements.txt ' - '-r tests/ansible/requirements.txt', - 'pip install -q "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION) - ] ] ci_lib.run_batches(batches) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index ab9bb2f2..b0b1eb14 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -3,15 +3,11 @@ import ci_lib batches = [ - [ - 'pip install -r tests/requirements.txt', - ] ] if ci_lib.have_docker(): batches.append([ 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', - 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ]) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index 92294aab..bd6ecb24 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -5,7 +5,6 @@ import ci_lib batches = [ [ 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', - 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py deleted file mode 100755 index fd8fb694..00000000 --- a/.ci/prep_azure.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -import os -import sys - -import ci_lib - -batches = [] - -if 0 and os.uname()[0] == 'Linux': - batches += [ - [ - "sudo chown `whoami`: ~", - "chmod u=rwx,g=rx,o= ~", - - "sudo mkdir /var/run/sshd", - "sudo /etc/init.d/ssh start", - - "mkdir -p ~/.ssh", - "chmod u=rwx,go= ~/.ssh", - - "ssh-keyscan -H localhost >> ~/.ssh/known_hosts", - "chmod u=rw,go= ~/.ssh/known_hosts", - - "cat tests/data/docker/mitogen__has_sudo_pubkey.key > ~/.ssh/id_rsa", - "chmod u=rw,go= ~/.ssh/id_rsa", - - "cat tests/data/docker/mitogen__has_sudo_pubkey.key.pub > ~/.ssh/authorized_keys", - "chmod u=rw,go=r ~/.ssh/authorized_keys", - ] - ] - -# setup venv, need all python commands in 1 list to be subprocessed at the same time -venv_steps = [] - -need_to_fix_psycopg2 = False - -is_python3 = os.environ['PYTHONVERSION'].startswith('3') - -# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage, -# broken symlinks, incorrect permissions and missing codecs. So we use the -# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our -# stuff into. The virtualenv can probably be removed again, but this was a -# hard-fought battle and for now I am tired of this crap. -if ci_lib.have_apt(): - venv_steps.extend([ - 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', - 'sudo add-apt-repository ppa:deadsnakes/ppa', - 'sudo apt-get update', - 'sudo apt-get -y install ' - 'python{pv} ' - 'python{pv}-dev ' - 'libsasl2-dev ' - 'libldap2-dev ' - .format(pv=os.environ['PYTHONVERSION']), - 'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}' - .format(pv=os.environ['PYTHONVERSION']) - ]) - if is_python3: - venv_steps.append('sudo apt-get -y install python{pv}-venv'.format(pv=os.environ['PYTHONVERSION'])) -# TODO: somehow `Mito36CentOS6_26` has both brew and apt installed https://dev.azure.com/dw-mitogen/Mitogen/_build/results?buildId=1031&view=logs&j=7bdbcdc6-3d3e-568d-ccf8-9ddca1a9623a&t=73d379b6-4eea-540f-c97e-046a2f620483 -elif is_python3 and ci_lib.have_brew(): - # Mac's System Integrity Protection prevents symlinking /usr/bin - # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html - # so we'll use /usr/local/bin/python for everything - # /usr/local/bin/python2.7 already exists! - need_to_fix_psycopg2 = True - venv_steps.append( - 'brew install python@{pv} postgresql' - .format(pv=os.environ['PYTHONVERSION']) - ) - -# need wheel before building virtualenv because of bdist_wheel and setuptools deps -venv_steps.append('/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools'.format(pv=os.environ['PYTHONVERSION'])) - -if os.environ['PYTHONVERSION'].startswith('2'): - venv_steps.extend([ - '/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(pv=os.environ['PYTHONVERSION']), - '/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(pv=os.environ['PYTHONVERSION']) - ]) -else: - venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(pv=os.environ['PYTHONVERSION'])) -# fixes https://stackoverflow.com/questions/59595649/can-not-install-psycopg2-on-macos-catalina https://github.com/Azure/azure-cli/issues/12854#issuecomment-619213863 -if need_to_fix_psycopg2: - venv_steps.append('/tmp/venv/bin/pip3 install psycopg2==2.8.5 psycopg2-binary') - -venv_steps.extend([ - # pbr is a transitive setup_requires of hdrhistogram. If it's not already - # installed then setuptools attempts to use easy_install, which fails. - '/tmp/venv/bin/pip install pbr==5.6.0', -]) - -batches.append(venv_steps) - -ci_lib.run_batches(batches) diff --git a/docs/changelog.rst b/docs/changelog.rst index 75917494..5d617432 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,9 @@ To avail of fixes in an unreleased version, please download a ZIP file v0.3.1.dev0 (unreleased) ------------------------ +* :gh:issue:`869` Continuous Integration tests are now run with Tox +* :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04 + v0.3.0 (2021-11-24) ------------------- diff --git a/tox.ini b/tox.ini index eda4e567..d6a7eccc 100644 --- a/tox.ini +++ b/tox.ini @@ -23,15 +23,15 @@ # pip --no-python-version-warning # pip --disable-pip-version-check +# TODO distros=-py3 + [tox] envlist = init, - py{27,35,39}-mode_ansible-distros_centos, - py{27,35,39}-mode_ansible-distros_debian, - py{27,35,39}-mode_ansible-distros_ubuntu, - py{27,35,39}-mode_mitogen-distro_centos{6,7,8}, - py{27,35,39}-mode_mitogen-distro_debian{9,10,11}, - py{27,35,39}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{27,36,39}-mode_ansible-ansible2.10, + py{27,36,39}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,39}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,39}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, requires = tox-factor @@ -45,29 +45,37 @@ basepython = py37: python3.7 py38: python3.8 py39: python3.9 +deps = + -r{toxinidir}/tests/requirements.txt + mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt + ansible2.3: ansible=2.3.3.0 + ansible2.4: ansible=2.4.6.0 + ansible2.8: ansible=2.8.3 + ansible2.9: ansible=2.9.6 + ansible2.10: ansible-base<2.10.14 + ansible2.10: ansible==2.10.0 install_command = python -m pip --no-python-version-warning install {opts} {packages} commands_pre = mode_ansible: {toxinidir}/.ci/ansible_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py + mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py mode_mitogen: {toxinidir}/.ci/mitogen_install.py commands = mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py + mode_localhost: {toxinidir}/.ci/localhost_ansible_tests.py mode_mitogen: {toxinidir}/.ci/mitogen_tests.py passenv = ANSIBLE_* + AWS_ACCESS_KEY_ID + AWS_DEFAULT_REGION + AWS_SECRET_ACCESS_KEY HOME setenv = - ANSIBLE_SKIP_TAGS = requires_local_sudo NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 - VER=2.10.5 - ansible2.3: VER=2.3.3.0 - ansible2.4: VER=2.4.6.0 - ansible2.8: VER=2.8.3 - ansible2.9: VER=2.9.6 - ansible2.10: VER=2.10.0 + # Only applicable to MODE=mitogen distro_centos5: DISTRO=centos5 distro_centos6: DISTRO=centos6 distro_centos7: DISTRO=centos7 @@ -78,6 +86,7 @@ setenv = distro_ubuntu1604: DISTRO=ubuntu1604 distro_ubuntu1804: DISTRO=ubuntu1804 distro_ubuntu2004: DISTRO=ubuntu2004 + # Note the plural, only applicable to MODE=ansible distros_centos: DISTROS=centos6 centos7 centos8 distros_centos5: DISTROS=centos5 distros_centos6: DISTROS=centos6 @@ -95,6 +104,12 @@ setenv = mode_debops_common: MODE=debops_common mode_mitogen: MODE=mitogen strategy_linear: STRATEGY=linear +whitelist_externals = + docker + docker-credential-secretservice + echo + gpg2 + pass [testenv:init] basepython = python3