From 0b86c4e45fd353cb60de1cd6f12362b6e66e2a98 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 5 Nov 2018 21:09:54 +0000 Subject: [PATCH] tests: basic (incomplete) Azure Pipelines config --- .ci/ansible_tests.py | 3 +- .ci/azure-pipelines.yml | 93 +++++++++++++++++++++++++++-------------- .ci/ci_lib.py | 34 +++++++-------- .ci/mitogen_tests.sh | 2 +- .ci/prep_azure.py | 36 ++++++++++++++++ 5 files changed, 118 insertions(+), 50 deletions(-) create mode 100755 .ci/prep_azure.py diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index e664ec8b..d7730ba6 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -41,6 +41,7 @@ with ci_lib.Fold('job_setup'): run("mkdir %s", HOSTS_DIR) run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR) + docker_hostname = ci_lib.get_docker_hostname() with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp: fp.write('[test-targets]\n') for i, distro in enumerate(ci_lib.DISTROS): @@ -51,7 +52,7 @@ with ci_lib.Fold('job_setup'): "ansible_password=has_sudo_nopw_password" "\n" % ( distro, - ci_lib.DOCKER_HOSTNAME, + docker_hostname, BASE_PORT + i, )) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 59d5d931..2a22bdad 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -5,51 +5,82 @@ jobs: -- job: 'Test' +- job: 'MitogenTests' pool: vmImage: 'Ubuntu 16.04' strategy: matrix: - Python27: + Mitogen27Debian_27: python.version: '2.7' - Python35: - python.version: '3.5' - Python36: + MODE: mitogen + DISTRO: debian + + MitogenPy27CentOS6_26: + python.version: '2.7' + MODE: mitogen + DISTRO: centos6 + + #Py26CentOS7: + #python.version: '2.7' + #MODE: mitogen + #DISTRO: centos6 + + Mitogen36CentOS6_26: python.version: '3.6' - Python37: - python.version: '3.7' - maxParallel: 4 + MODE: mitogen + DISTRO: centos6 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' + DebOps_2460_27_27: + python.version: '2.7' + MODE: debops_common + VER: 2.4.6.0 - - script: python -m pip install --upgrade pip && pip install -r requirements.txt - displayName: 'Install dependencies' + DebOps_262_36_27: + python.version: '3.6' + MODE: debops_common + VER: 2.6.2 - - script: | - pip install pytest - pytest tests --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest' + Ansible_2460_26: + python.version: '2.7' + MODE: ansible + VER: 2.4.6.0 - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() + Ansible_262_26: + python.version: '2.7' + MODE: ansible + VER: 2.6.2 -- job: 'Publish' - dependsOn: 'Test' - pool: - vmImage: 'Ubuntu 16.04' + Ansible_2460_36: + python.version: '3.6' + MODE: ansible + VER: 2.4.6.0 + + Ansible_262_36: + python.version: '3.6' + MODE: ansible + VER: 2.6.2 + + Vanilla_262_27: + python.version: '2.7' + MODE: ansible + VER: 2.6.2 + DISTROS: debian + STRATEGY: linear steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.x' + versionSpec: '$(python.version)' architecture: 'x64' - - script: python setup.py sdist - displayName: 'Build sdist' + - script: .ci/prep_azure.py + displayName: "Install requirements." + + - script: | + export TRAVIS_BUILD_DIR=`pwd` + if [ -f ".ci/$(MODE)_tests.sh" ]; then + .ci/$(MODE)_tests.sh; + else + .ci/$(MODE)_tests.py; + fi + displayName: Run tests. diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index f5778161..d7d6e09b 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -10,8 +10,6 @@ import shlex import shutil import tempfile -import os -os.system('curl -H Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/machine-type') # # check_output() monkeypatch cutpasted from testlib.py @@ -37,20 +35,21 @@ if not hasattr(subprocess, 'check_output'): # Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars. -proc = subprocess.Popen( - args=['stdbuf', '-oL', 'cat'], - stdin=subprocess.PIPE -) +if sys.platform.startswith('linux'): + proc = subprocess.Popen( + args=['stdbuf', '-oL', 'cat'], + stdin=subprocess.PIPE + ) -os.dup2(proc.stdin.fileno(), 1) -os.dup2(proc.stdin.fileno(), 2) + os.dup2(proc.stdin.fileno(), 1) + os.dup2(proc.stdin.fileno(), 2) -def cleanup_travis_junk(stdout=sys.stdout, stderr=sys.stderr, proc=proc): - stdout.close() - stderr.close() - proc.terminate() + def cleanup_travis_junk(stdout=sys.stdout, stderr=sys.stderr, proc=proc): + stdout.close() + stderr.close() + proc.terminate() -atexit.register(cleanup_travis_junk) + atexit.register(cleanup_travis_junk) # ----------------- @@ -113,10 +112,11 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( GIT_ROOT ) -DOCKER_HOSTNAME = subprocess.check_output([ - sys.executable, - os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'), -]).decode().strip() +def get_docker_hostname(): + return subprocess.check_output([ + sys.executable, + os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'), + ]).decode().strip() # SSH passes these through to the container when run interactively, causing # stdout to get messed up with libc warnings. diff --git a/.ci/mitogen_tests.sh b/.ci/mitogen_tests.sh index db393d73..33ee16ba 100755 --- a/.ci/mitogen_tests.sh +++ b/.ci/mitogen_tests.sh @@ -2,4 +2,4 @@ # Run the Mitogen tests. MITOGEN_TEST_DISTRO="${DISTRO:-debian}" -MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -vvv +MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -v diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py new file mode 100755 index 00000000..164e04e3 --- /dev/null +++ b/.ci/prep_azure.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Run preparation steps in parallel. + +import subprocess +import ci_lib + +subprocess.check_call( + 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', + shell=True, +) + +procs = [ + subprocess.Popen( + 'pip install -r dev_requirements.txt 2>&1 | cat', + shell=True, + ), + subprocess.Popen( + """ + sudo add-apt-repository ppa:deadsnakes/ppa && \ + ( sudo apt-get update 2>&1 | cat ) && \ + sudo apt-get -y install \ + python2.6 python2.6-dev libsasl2-dev libldap2-dev 2>&1 | cat + """, + shell=True, + ) +] + +procs += [ + subprocess.Popen( + 'docker pull mitogen/%s-test 2>&1 | cat' % (distro,), + shell=True + ) + for distro in ci_lib.DISTROS +] + +assert [proc.wait() for proc in procs] == [0] * len(procs)