From a717c5406cec70c2a3d0f250e2870aeabfb0ef1e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 6 Nov 2018 04:43:56 +0000 Subject: [PATCH 1/6] tests: split etc_environment test in two Turns out nobody supports ~/.pam_environment any more. Keep the behaviour around for the time being. --- .../runner/_etc_environment_global.yml | 35 +++++++++ .../runner/_etc_environment_user.yml | 32 ++++++++ .../integration/runner/etc_environment.yml | 77 ++----------------- 3 files changed, 73 insertions(+), 71 deletions(-) create mode 100644 tests/ansible/integration/runner/_etc_environment_global.yml create mode 100644 tests/ansible/integration/runner/_etc_environment_user.yml diff --git a/tests/ansible/integration/runner/_etc_environment_global.yml b/tests/ansible/integration/runner/_etc_environment_global.yml new file mode 100644 index 00000000..e24a038a --- /dev/null +++ b/tests/ansible/integration/runner/_etc_environment_global.yml @@ -0,0 +1,35 @@ +# /etc/environment + +- file: + path: /etc/environment + state: absent + become: true + +- shell: echo $MAGIC_ETC_ENV + register: echo + +- assert: + that: echo.stdout == "" + +- copy: + dest: /etc/environment + content: | + MAGIC_ETC_ENV=555 + become: true + +- shell: echo $MAGIC_ETC_ENV + register: echo + +- assert: + that: echo.stdout == "555" + +- file: + path: /etc/environment + state: absent + become: true + +- shell: echo $MAGIC_ETC_ENV + register: echo + +- assert: + that: echo.stdout == "" diff --git a/tests/ansible/integration/runner/_etc_environment_user.yml b/tests/ansible/integration/runner/_etc_environment_user.yml new file mode 100644 index 00000000..ca1dc5cc --- /dev/null +++ b/tests/ansible/integration/runner/_etc_environment_user.yml @@ -0,0 +1,32 @@ +# ~/.pam_environment + +- file: + path: ~/.pam_environment + state: absent + +- shell: echo $MAGIC_PAM_ENV + register: echo + +- assert: + that: echo.stdout == "" + +- copy: + dest: ~/.pam_environment + content: | + MAGIC_PAM_ENV=321 + +- shell: echo $MAGIC_PAM_ENV + register: echo + +- assert: + that: echo.stdout == "321" + +- file: + path: ~/.pam_environment + state: absent + +- shell: echo $MAGIC_PAM_ENV + register: echo + +- assert: + that: echo.stdout == "" diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml index 0037698a..7eb405cb 100644 --- a/tests/ansible/integration/runner/etc_environment.yml +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -3,78 +3,13 @@ # but less likely to brick a development workstation - name: integration/runner/etc_environment.yml - hosts: test-targets + hosts: test-targets[0] any_errors_fatal: true gather_facts: true tasks: - # ~/.pam_environment + - include_tasks: _etc_environment_user.yml + when: ansible_system == "Linux" and is_mitogen - - file: - path: ~/.pam_environment - state: absent - - - shell: echo $MAGIC_PAM_ENV - register: echo - - - assert: - that: echo.stdout == "" - - - copy: - dest: ~/.pam_environment - content: | - MAGIC_PAM_ENV=321 - - - shell: echo $MAGIC_PAM_ENV - register: echo - - - assert: - that: echo.stdout == "321" - - - file: - path: ~/.pam_environment - state: absent - - - shell: echo $MAGIC_PAM_ENV - register: echo - - - assert: - that: echo.stdout == "" - - - # /etc/environment - - meta: end_play - when: ansible_virtualization_type != "docker" - - - file: - path: /etc/environment - state: absent - become: true - - - shell: echo $MAGIC_ETC_ENV - register: echo - - - assert: - that: echo.stdout == "" - - - copy: - dest: /etc/environment - content: | - MAGIC_ETC_ENV=555 - become: true - - - shell: echo $MAGIC_ETC_ENV - register: echo - - - assert: - that: echo.stdout == "555" - - - file: - path: /etc/environment - state: absent - become: true - - - shell: echo $MAGIC_ETC_ENV - register: echo - - - assert: - that: echo.stdout == "" + - include_tasks: _etc_environment_global.yml + # Don't destroy laptops. + when: ansible_virtualization_type == "docker" From 8f1f3de1231f424dce269fef1e14d99a9c26cdda Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 6 Nov 2018 12:43:12 +0000 Subject: [PATCH 2/6] tests: Pythonize env_wrapper.sh, clean up local_test --- tests/data/env_wrapper.sh | 12 ------------ tests/data/stubs/stub-python.py | 15 +++++++++++++++ tests/local_test.py | 29 ++++++++++------------------- 3 files changed, 25 insertions(+), 31 deletions(-) delete mode 100755 tests/data/env_wrapper.sh create mode 100755 tests/data/stubs/stub-python.py diff --git a/tests/data/env_wrapper.sh b/tests/data/env_wrapper.sh deleted file mode 100755 index afb523f0..00000000 --- a/tests/data/env_wrapper.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# This script exists to test the behavior of Stream.python_path being set to a -# list. It sets an environmnt variable that we can detect, then executes any -# arguments passed to it. -export EXECUTED_VIA_ENV_WRAPPER=1 -if [ "${1:0:1}" == "-" ]; then - exec "$PYTHON" "$@" -else - export ENV_WRAPPER_FIRST_ARG="$1" - shift - exec "$@" -fi diff --git a/tests/data/stubs/stub-python.py b/tests/data/stubs/stub-python.py new file mode 100755 index 00000000..d9239c2b --- /dev/null +++ b/tests/data/stubs/stub-python.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import json +import os +import subprocess +import sys + +os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv) +os.environ['THIS_IS_STUB_PYTHON'] = '1' + +if sys.argv[1].startswith('-'): + os.execvp(sys.executable, [sys.executable] + sys.argv[1:]) +else: + os.environ['STUB_PYTHON_FIRST_ARG'] = sys.argv.pop(1) + os.execvp(sys.executable, sys.argv[1:]) diff --git a/tests/local_test.py b/tests/local_test.py index 5a620e52..fe2bd149 100644 --- a/tests/local_test.py +++ b/tests/local_test.py @@ -5,11 +5,8 @@ import sys import unittest2 import mitogen -import mitogen.ssh -import mitogen.utils import testlib -import plain_old_module def get_sys_executable(): @@ -20,43 +17,37 @@ def get_os_environ(): return dict(os.environ) -class LocalTest(testlib.RouterMixin, testlib.TestCase): - stream_class = mitogen.ssh.Stream +class ConstructionTest(testlib.RouterMixin, testlib.TestCase): + stub_python_path = testlib.data_path('stubs/stub-python.py') def test_stream_name(self): context = self.router.local() pid = context.call(os.getpid) self.assertEquals('local.%d' % (pid,), context.name) - -class PythonPathTest(testlib.RouterMixin, testlib.TestCase): - stream_class = mitogen.ssh.Stream - - def test_inherited(self): + def test_python_path_inherited(self): context = self.router.local() self.assertEquals(sys.executable, context.call(get_sys_executable)) - def test_string(self): - os.environ['PYTHON'] = sys.executable + def test_python_path_string(self): context = self.router.local( - python_path=testlib.data_path('env_wrapper.sh'), + python_path=self.stub_python_path, ) - self.assertEquals(sys.executable, context.call(get_sys_executable)) env = context.call(get_os_environ) - self.assertEquals('1', env['EXECUTED_VIA_ENV_WRAPPER']) + self.assertEquals('1', env['THIS_IS_STUB_PYTHON']) - def test_list(self): + def test_python_path_list(self): context = self.router.local( python_path=[ - testlib.data_path('env_wrapper.sh'), + self.stub_python_path, "magic_first_arg", sys.executable ] ) self.assertEquals(sys.executable, context.call(get_sys_executable)) env = context.call(get_os_environ) - self.assertEquals('magic_first_arg', env['ENV_WRAPPER_FIRST_ARG']) - self.assertEquals('1', env['EXECUTED_VIA_ENV_WRAPPER']) + self.assertEquals('magic_first_arg', env['STUB_PYTHON_FIRST_ARG']) + self.assertEquals('1', env['THIS_IS_STUB_PYTHON']) if __name__ == '__main__': From fccf42414010a8fa5855f417a7764a41c2fecc37 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 6 Nov 2018 12:48:24 +0000 Subject: [PATCH 3/6] tests: Pythonize print_env.sh. --- .../regression/issue_122__environment_difference.yml | 2 +- tests/ansible/regression/scripts/print_env.py | 6 ++++++ tests/ansible/regression/scripts/print_env.sh | 2 -- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 tests/ansible/regression/scripts/print_env.py delete mode 100644 tests/ansible/regression/scripts/print_env.sh diff --git a/tests/ansible/regression/issue_122__environment_difference.yml b/tests/ansible/regression/issue_122__environment_difference.yml index bf9df861..b020cc5d 100644 --- a/tests/ansible/regression/issue_122__environment_difference.yml +++ b/tests/ansible/regression/issue_122__environment_difference.yml @@ -9,6 +9,6 @@ hosts: test-targets tasks: - - script: scripts/print_env.sh + - script: scripts/print_env.py register: env - debug: msg={{env}} diff --git a/tests/ansible/regression/scripts/print_env.py b/tests/ansible/regression/scripts/print_env.py new file mode 100644 index 00000000..50a2504e --- /dev/null +++ b/tests/ansible/regression/scripts/print_env.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import os +import pprint + +pprint.pprint(dict(os.environ)) diff --git a/tests/ansible/regression/scripts/print_env.sh b/tests/ansible/regression/scripts/print_env.sh deleted file mode 100644 index c03c9936..00000000 --- a/tests/ansible/regression/scripts/print_env.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -set From 43306fff81fed6cbadd5ab3162a5c7b670a7db9c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 6 Nov 2018 14:39:47 +0000 Subject: [PATCH 4/6] docs: drop sphinx-autobuild, avoids 10 deps (-16%) sphinx-autobuild==0.6.0 (from -r docs/docs-requirements.txt (line 2)) livereload>=2.3.0 (from sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) pathtools>=0.1.2 (from sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) tornado>=3.2 (from sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) argh>=0.24.1 (from sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) watchdog>=0.7.1 (from sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) port-for==0.3.1 (from sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) backports.ssl_match_hostname (from tornado>=3.2->sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) singledispatch (from tornado>=3.2->sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) backports_abc>=0.4 (from tornado>=3.2->sphinx-autobuild==0.6.0->-r docs/docs-requirements.txt (line 2)) --- docs/Makefile | 2 +- docs/docs-requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index bc394d34..8edc7fd5 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,7 +2,7 @@ # default: - sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + sphinx-autobuild . build/html/ # You can set these variables from the command line. SPHINXOPTS = diff --git a/docs/docs-requirements.txt b/docs/docs-requirements.txt index f0bddf36..a93c2140 100644 --- a/docs/docs-requirements.txt +++ b/docs/docs-requirements.txt @@ -1,4 +1,3 @@ Sphinx==1.7.1 -sphinx-autobuild==0.6.0 # Last version to support Python 2.6 sphinxcontrib-programoutput==0.11 alabaster==0.7.10 From b60a6d0f3b397401deab9d257655b6519bace792 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 6 Nov 2018 15:11:13 +0000 Subject: [PATCH 5/6] Split dev_requirements.txt up according to test mode. --- dev_requirements.txt | 28 +++++++------------ ...docs-requirements.txt => requirements.txt} | 0 tests/ansible/requirements.txt | 2 ++ tests/requirements.txt | 15 ++++++++++ 4 files changed, 27 insertions(+), 18 deletions(-) rename docs/{docs-requirements.txt => requirements.txt} (100%) create mode 100644 tests/requirements.txt diff --git a/dev_requirements.txt b/dev_requirements.txt index eb83f8be..2dc0d171 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,19 +1,11 @@ --r docs/docs-requirements.txt +# This file is no longer used by CI jobs, it's mostly for interactive use. +# Instead CI jobs grab the relevant sub-requirement. + +# mitogen_tests +-r tests/requirements.txt + +# ansible_tests -r tests/ansible/requirements.txt -psutil==5.4.8 -coverage==4.5.1 -Django==1.6.11 # Last version supporting 2.6. -mock==2.0.0 -pytz==2018.5 -cffi==1.11.2 # Random pin to try and fix pyparser==2.18 not having effect -pycparser==2.18 # Last version supporting 2.6. -faulthandler==3.1; python_version < '3.3' # used by testlib -pytest-catchlog==1.2.2 -pytest==3.1.2 -PyYAML==3.11; python_version < '2.7' -PyYAML==3.12; python_version >= '2.7' -timeoutcontext==1.2.0 -unittest2==1.1.0 -# Fix InsecurePlatformWarning while creating py26 tox environment -# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings -urllib3[secure]; python_version < '2.7.9' + +# readthedocs +-r docs/requirements.txt diff --git a/docs/docs-requirements.txt b/docs/requirements.txt similarity index 100% rename from docs/docs-requirements.txt rename to docs/requirements.txt diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index 551af999..74477c22 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -2,3 +2,5 @@ ansible; python_version >= '2.7' ansible<2.7; python_version < '2.7' paramiko==2.3.2 # Last 2.6-compat version. google-api-python-client==1.6.5 +PyYAML==3.11; python_version < '2.7' +PyYAML==3.12; python_version >= '2.7' diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..327f563a --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,15 @@ +psutil==5.4.8 +coverage==4.5.1 +Django==1.6.11 # Last version supporting 2.6. +mock==2.0.0 +pytz==2018.5 +cffi==1.11.2 # Random pin to try and fix pyparser==2.18 not having effect +pycparser==2.18 # Last version supporting 2.6. +faulthandler==3.1; python_version < '3.3' # used by testlib +pytest-catchlog==1.2.2 +pytest==3.1.2 +timeoutcontext==1.2.0 +unittest2==1.1.0 +# Fix InsecurePlatformWarning while creating py26 tox environment +# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings +urllib3[secure]; python_version < '2.7.9' From 2a6dbb038f4485e268ee80808c5780232e828fb8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 6 Nov 2018 13:47:09 +0000 Subject: [PATCH 6/6] Pythonize, split out and parallelize all jobs. --- .ci/ansible_install.py | 21 ++++++++ .ci/ansible_tests.py | 51 ++++++++------------ .ci/azure-pipelines.yml | 11 ++--- .ci/ci_lib.py | 69 ++++++++++++++++++++++++--- .ci/debops_common_install.py | 18 +++++++ .ci/debops_common_tests.py | 78 ++++++++++++++++++++++++++++++ .ci/debops_common_tests.sh | 90 ----------------------------------- .ci/mitogen_install.py | 15 ++++++ .ci/mitogen_tests.py | 14 ++++++ .ci/mitogen_tests.sh | 5 -- .ci/prep_azure.py | 40 +++++----------- .travis.yml | 17 +------ run_tests | 2 +- tests/show_docker_hostname.py | 9 ---- 14 files changed, 248 insertions(+), 192 deletions(-) create mode 100755 .ci/ansible_install.py create mode 100755 .ci/debops_common_install.py create mode 100755 .ci/debops_common_tests.py delete mode 100755 .ci/debops_common_tests.sh create mode 100755 .ci/mitogen_install.py create mode 100755 .ci/mitogen_tests.py delete mode 100755 .ci/mitogen_tests.sh delete mode 100644 tests/show_docker_hostname.py diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py new file mode 100755 index 00000000..167a9cb1 --- /dev/null +++ b/.ci/ansible_install.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +import ci_lib + +batches = [ + [ + # Must be installed separately, as PyNACL indirect requirement causes + # newer version to be installed if done in a single pip run. + 'pip install "pycparser<2.19"', + 'pip install ' + '-r tests/requirements.txt ' + '-r tests/ansible/requirements.txt', + ] +] + +batches.extend( + ['docker pull mitogen/%s-test' % (distro,)] + for distro in ci_lib.DISTROS +) + +ci_lib.run_batches(batches) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index d7730ba6..bae95902 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -8,53 +8,42 @@ import ci_lib from ci_lib import run -BASE_PORT = 2201 TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') +with ci_lib.Fold('unit_tests'): + os.environ['SKIP_MITOGEN'] = '1' + ci_lib.run('./run_tests -v') + + with ci_lib.Fold('docker_setup'): - for i, distro in enumerate(ci_lib.DISTROS): - try: - run("docker rm -f target-%s", distro) - except: pass - - run(""" - docker run - --rm - --detach - --publish 0.0.0.0:%s:22/tcp - --hostname=target-%s - --name=target-%s - mitogen/%s-test - """, BASE_PORT + i, distro, distro, distro) + containers = ci_lib.make_containers() + ci_lib.start_containers(containers) with ci_lib.Fold('job_setup'): - os.chdir(TESTS_DIR) - os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) - - run("pip install -qr requirements.txt") # tests/ansible/requirements # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) + os.chdir(TESTS_DIR) + os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) + run("mkdir %s", HOSTS_DIR) run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR) - docker_hostname = ci_lib.get_docker_hostname() with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp: fp.write('[test-targets]\n') - for i, distro in enumerate(ci_lib.DISTROS): - fp.write("target-%s " - "ansible_host=%s " - "ansible_port=%s " - "ansible_user=mitogen__has_sudo_nopw " - "ansible_password=has_sudo_nopw_password" - "\n" % ( - distro, - docker_hostname, - BASE_PORT + i, - )) + fp.writelines( + "%(name)s " + "ansible_host=%(hostname)s " + "ansible_port=%(port)s " + "ansible_user=mitogen__has_sudo_nopw " + "ansible_password=has_sudo_nopw_password" + "\n" + % container + for container in containers + ) # Build the binaries. # run("make -C %s", TESTS_DIR) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 2a22bdad..fbbb9640 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -76,11 +76,8 @@ jobs: - script: .ci/prep_azure.py displayName: "Install requirements." - - script: | - export TRAVIS_BUILD_DIR=`pwd` - if [ -f ".ci/$(MODE)_tests.sh" ]; then - .ci/$(MODE)_tests.sh; - else - .ci/$(MODE)_tests.py; - fi + - script: .ci/$(MODE)_install.py + displayName: "Install requirements." + + - script: .ci/$(MODE)_tests.py displayName: Run tests. diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index d7d6e09b..77cc30a2 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -4,12 +4,17 @@ from __future__ import print_function import atexit import os -import subprocess -import sys import shlex import shutil +import subprocess +import sys import tempfile +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + # # check_output() monkeypatch cutpasted from testlib.py @@ -60,13 +65,26 @@ def _argv(s, *args): def run(s, *args, **kwargs): - argv = _argv(s, *args) + argv = ['/usr/bin/time', '--'] + _argv(s, *args) print('Running: %s' % (argv,)) ret = subprocess.check_call(argv, **kwargs) print('Finished running: %s' % (argv,)) return ret +def run_batches(batches): + combine = lambda batch: 'set -x; ' + (' && '.join( + '( %s; )' % (cmd,) + for cmd in batch + )) + + procs = [ + subprocess.Popen(combine(batch), shell=True) + for batch in batches + ] + assert [proc.wait() for proc in procs] == [0] * len(procs) + + def get_output(s, *args, **kwargs): argv = _argv(s, *args) print('Running: %s' % (argv,)) @@ -103,7 +121,10 @@ os.environ.setdefault('ANSIBLE_STRATEGY', os.environ.get('STRATEGY', 'mitogen_linear')) ANSIBLE_VERSION = os.environ.get('VER', '2.6.2') GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +DISTRO = os.environ.get('DISTRO', 'debian') DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() +TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) +BASE_PORT = 2200 TMP = TempDir().path os.environ['PYTHONDONTWRITEBYTECODE'] = 'x' @@ -113,10 +134,44 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( ) def get_docker_hostname(): - return subprocess.check_output([ - sys.executable, - os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'), - ]).decode().strip() + url = os.environ.get('DOCKER_HOST') + if url in (None, 'http+docker://localunixsocket'): + return 'localhost' + + parsed = urlparse.urlparse(url) + return parsed.netloc.partition(':')[0] + + +def make_containers(): + docker_hostname = get_docker_hostname() + return [ + { + "distro": distro, + "name": "target-%s-%s" % (distro, i), + "hostname": docker_hostname, + "port": BASE_PORT + i, + } + for i, distro in enumerate(DISTROS, 1) + ] + + +def start_containers(containers): + run_batches([ + [ + "docker rm -f %(name)s || true" % container, + "docker run " + "--rm " + "--detach " + "--publish 0.0.0.0:%(port)s:22/tcp " + "--hostname=%(name)s " + "--name=%(name)s " + "mitogen/%(distro)s-test " + % container + ] + for container in containers + ]) + return containers + # SSH passes these through to the container when run interactively, causing # stdout to get messed up with libc warnings. diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py new file mode 100755 index 00000000..8830eaf6 --- /dev/null +++ b/.ci/debops_common_install.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +import ci_lib + +# Naturally DebOps only supports Debian. +ci_lib.DISTROS = ['debian'] + +ci_lib.run_batches([ + [ + # Must be installed separately, as PyNACL indirect requirement causes + # newer version to be installed if done in a single pip run. + 'pip install "pycparser<2.19"', + 'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION, + ], + [ + 'docker pull mitogen/debian-test', + ], +]) diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py new file mode 100755 index 00000000..04fbb938 --- /dev/null +++ b/.ci/debops_common_tests.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +from __future__ import print_function +import os + +import ci_lib + + +# DebOps only supports Debian. +ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT + +project_dir = os.path.join(ci_lib.TMP, 'project') +key_file = os.path.join( + ci_lib.GIT_ROOT, + 'tests/data/docker/mitogen__has_sudo_pubkey.key', +) +vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' +inventory_path = 'ansible/inventory/hosts' +docker_hostname = ci_lib.get_docker_hostname() + + +with ci_lib.Fold('docker_setup'): + containers = ci_lib.make_containers() + ci_lib.start_containers(containers) + + +with ci_lib.Fold('job_setup'): + ci_lib.run('debops-init %s', project_dir) + os.chdir(project_dir) + + with open('.debops.cfg', 'w') as fp: + fp.write( + "[ansible defaults]\n" + "strategy_plugins = %s/ansible_mitogen/plugins/strategy\n" + "strategy = mitogen_linear\n" + % (ci_lib.GIT_ROOT,) + ) + + ci_lib.run('chmod go= %s', key_file) + with open(vars_path, 'w') as fp: + fp.write( + "ansible_python_interpreter: /usr/bin/python2.7\n" + "\n" + "ansible_user: mitogen__has_sudo_pubkey\n" + "ansible_become_pass: has_sudo_pubkey_password\n" + "ansible_ssh_private_key_file: %s\n" + "\n" + # Speed up slow DH generation. + "dhparam__bits: ['128', '64']\n" + % (key_file,) + ) + + with open(inventory_path, 'a') as fp: + fp.writelines( + '%(name)s ' + 'ansible_host=%(hostname)s ' + 'ansible_port=%(port)d ' + '\n' + % container + for container in containers + ) + + print() + print(' echo --- ansible/inventory/hosts: ---') + ci_lib.run('cat ansible/inventory/hosts') + print('---') + print() + + # Now we have real host key checking, we need to turn it off + os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False' + + +with ci_lib.Fold('first_run'): + ci_lib.run('debops common') + + +with ci_lib.Fold('second_run'): + ci_lib.run('debops common') diff --git a/.ci/debops_common_tests.sh b/.ci/debops_common_tests.sh deleted file mode 100755 index 753d1c11..00000000 --- a/.ci/debops_common_tests.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -ex -# Run some invocations of DebOps. - -TMPDIR="/tmp/debops-$$" -TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" -TARGET_COUNT="${TARGET_COUNT:-2}" -ANSIBLE_VERSION="${VER:-2.6.1}" -DISTRO=debian # Naturally DebOps only supports Debian. - -export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}" - -function on_exit() -{ - echo travis_fold:start:cleanup - [ "$KEEP" ] || { - rm -rf "$TMPDIR" || true - for i in $(seq $TARGET_COUNT) - do - docker kill target$i || true - done - } - echo travis_fold:end:cleanup -} - -trap on_exit EXIT -mkdir "$TMPDIR" - - -echo travis_fold:start:job_setup -pip install -qqqU debops==0.7.2 ansible==${ANSIBLE_VERSION} |cat -debops-init "$TMPDIR/project" -cd "$TMPDIR/project" - -cat > .debops.cfg <<-EOF -[ansible defaults] -strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy -strategy = mitogen_linear -EOF - -chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key - -cat > ansible/inventory/group_vars/debops_all_hosts.yml <<-EOF -ansible_python_interpreter: /usr/bin/python2.7 - -ansible_user: mitogen__has_sudo_pubkey -ansible_become_pass: has_sudo_pubkey_password -ansible_ssh_private_key_file: ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key - -# Speed up slow DH generation. -dhparam__bits: ["128", "64"] -EOF - -DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)" - -for i in $(seq $TARGET_COUNT) -do - port=$((2200 + $i)) - docker run \ - --rm \ - --detach \ - --publish 0.0.0.0:$port:22/tcp \ - --name=target$i \ - mitogen/${DISTRO}-test - - echo \ - target$i \ - ansible_host=$DOCKER_HOSTNAME \ - ansible_port=$port \ - >> ansible/inventory/hosts -done - -echo -echo --- ansible/inventory/hosts: ---- -cat ansible/inventory/hosts -echo --- - -# Now we have real host key checking, we need to turn it off. :) -export ANSIBLE_HOST_KEY_CHECKING=False - -echo travis_fold:end:job_setup - - -echo travis_fold:start:first_run -/usr/bin/time debops common "$@" -echo travis_fold:end:first_run - - -echo travis_fold:start:second_run -/usr/bin/time debops common "$@" -echo travis_fold:end:second_run diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py new file mode 100755 index 00000000..4cc06c04 --- /dev/null +++ b/.ci/mitogen_install.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import ci_lib + +batches = [ + [ + 'pip install "pycparser<2.19"', + 'pip install -r tests/requirements.txt', + ], + [ + 'docker pull mitogen/%s-test' % (ci_lib.DISTRO,), + ] +] + +ci_lib.run_batches(batches) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py new file mode 100755 index 00000000..4ba796c2 --- /dev/null +++ b/.ci/mitogen_tests.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# Run the Mitogen tests. + +import os + +import ci_lib + +os.environ.update({ + 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, + 'MITOGEN_LOG_LEVEL': 'debug', + 'SKIP_ANSIBLE': '1', +}) + +ci_lib.run('./run_tests -v') diff --git a/.ci/mitogen_tests.sh b/.ci/mitogen_tests.sh deleted file mode 100755 index 33ee16ba..00000000 --- a/.ci/mitogen_tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -ex -# Run the Mitogen tests. - -MITOGEN_TEST_DISTRO="${DISTRO:-debian}" -MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests -v diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index 164e04e3..85c19947 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -1,36 +1,22 @@ #!/usr/bin/env python -# Run preparation steps in parallel. -import subprocess import ci_lib -subprocess.check_call( +batches = [] +batches.append([ 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', - shell=True, -) + 'sudo add-apt-repository ppa:deadsnakes/ppa', + 'sudo apt-get update', + 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', +]) -procs = [ - subprocess.Popen( - 'pip install -r dev_requirements.txt 2>&1 | cat', - shell=True, - ), - subprocess.Popen( - """ - sudo add-apt-repository ppa:deadsnakes/ppa && \ - ( sudo apt-get update 2>&1 | cat ) && \ - sudo apt-get -y install \ - python2.6 python2.6-dev libsasl2-dev libldap2-dev 2>&1 | cat - """, - shell=True, - ) -] +batches.append([ + 'pip install -r dev_requirements.txt', +]) -procs += [ - subprocess.Popen( - 'docker pull mitogen/%s-test 2>&1 | cat' % (distro,), - shell=True - ) +batches.extend( + ['docker pull mitogen/%s-test' % (distro,)] for distro in ci_lib.DISTROS -] +) -assert [proc.wait() for proc in procs] == [0] * len(procs) +ci_lib.run_batches(batches) diff --git a/.travis.yml b/.travis.yml index 8e4a8d51..a0d1f924 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,4 @@ sudo: required -addons: - apt: - update: true notifications: email: false @@ -14,20 +11,10 @@ cache: - /home/travis/virtualenv install: -# |cat to disable progress bar. -- pip install -r dev_requirements.txt |cat +- .ci/${MODE}_install.py script: -- | - if [ -f "${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh" ]; then - ${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.sh; - else - ${TRAVIS_BUILD_DIR}/.ci/${MODE}_tests.py; - fi - - -services: - - docker +- .ci/${MODE}_tests.py # To avoid matrix explosion, just test against oldest->newest and diff --git a/run_tests b/run_tests index 65bf1fef..8de99ace 100755 --- a/run_tests +++ b/run_tests @@ -1,4 +1,4 @@ -#/usr/bin/env bash +#!/usr/bin/env bash echo '----- ulimits -----' ulimit -a diff --git a/tests/show_docker_hostname.py b/tests/show_docker_hostname.py deleted file mode 100644 index 995c744b..00000000 --- a/tests/show_docker_hostname.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -""" -For use by the Travis scripts, just print out the hostname of the Docker -daemon from the environment. -""" - -import testlib -print(testlib.get_docker_host())