From 0aa4c9d8fc9d2ffe972b65afa79a979b4e6c2b28 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 14 Feb 2019 11:17:52 +0000 Subject: [PATCH] issue #542: .ci: move some tests to Azure and enable Mac job. --- .ci/azure-pipelines-steps.yml | 20 +++++ .ci/azure-pipelines.yml | 108 ++++++++++++++------------- .ci/ci_lib.py | 12 +++ .ci/mitogen_install.py | 8 +- .ci/mitogen_tests.py | 3 + .ci/prep_azure.py | 38 ++++++---- .travis.yml | 8 +- tests/ansible/tests/affinity_test.py | 4 + tests/file_service_test.py | 9 ++- tests/testlib.py | 2 + 10 files changed, 136 insertions(+), 76 deletions(-) create mode 100644 .ci/azure-pipelines-steps.yml diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml new file mode 100644 index 00000000..a377d795 --- /dev/null +++ b/.ci/azure-pipelines-steps.yml @@ -0,0 +1,20 @@ + +parameters: + name: '' + pool: '' + sign: false + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + +- script: .ci/prep_azure.py + displayName: "Install requirements." + +- script: .ci/$(MODE)_install.py + displayName: "Install requirements." + +- script: .ci/$(MODE)_tests.py + displayName: Run tests. diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index fbbb9640..dc5f7162 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -5,79 +5,85 @@ jobs: -- job: 'MitogenTests' +- job: Mac + steps: + - template: azure-pipelines-steps.yml pool: - vmImage: 'Ubuntu 16.04' + vmImage: macOS-10.13 strategy: matrix: - Mitogen27Debian_27: + Mito27_27: python.version: '2.7' MODE: mitogen - DISTRO: debian - MitogenPy27CentOS6_26: + +- job: Linux + pool: + vmImage: "Ubuntu 16.04" + steps: + - template: azure-pipelines-steps.yml + strategy: + matrix: + # + # Confirmed working + # + Mito27Debian_27: python.version: '2.7' MODE: mitogen - DISTRO: centos6 + DISTRO: debian - #Py26CentOS7: + #MitoPy27CentOS6_26: #python.version: '2.7' #MODE: mitogen #DISTRO: centos6 - Mitogen36CentOS6_26: + Mito36CentOS6_26: python.version: '3.6' MODE: mitogen DISTRO: centos6 - DebOps_2460_27_27: - python.version: '2.7' - MODE: debops_common - VER: 2.4.6.0 - - DebOps_262_36_27: - python.version: '3.6' - MODE: debops_common - VER: 2.6.2 - - Ansible_2460_26: - python.version: '2.7' - MODE: ansible - VER: 2.4.6.0 + # + # + # - Ansible_262_26: - python.version: '2.7' - MODE: ansible - VER: 2.6.2 + #Py26CentOS7: + #python.version: '2.7' + #MODE: mitogen + #DISTRO: centos6 - Ansible_2460_36: - python.version: '3.6' - MODE: ansible - VER: 2.4.6.0 + #DebOps_2460_27_27: + #python.version: '2.7' + #MODE: debops_common + #VER: 2.4.6.0 - Ansible_262_36: - python.version: '3.6' - MODE: ansible - VER: 2.6.2 + #DebOps_262_36_27: + #python.version: '3.6' + #MODE: debops_common + #VER: 2.6.2 - Vanilla_262_27: - python.version: '2.7' - MODE: ansible - VER: 2.6.2 - DISTROS: debian - STRATEGY: linear + #Ansible_2460_26: + #python.version: '2.7' + #MODE: ansible + #VER: 2.4.6.0 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' + #Ansible_262_26: + #python.version: '2.7' + #MODE: ansible + #VER: 2.6.2 - - script: .ci/prep_azure.py - displayName: "Install requirements." + #Ansible_2460_36: + #python.version: '3.6' + #MODE: ansible + #VER: 2.4.6.0 - - script: .ci/$(MODE)_install.py - displayName: "Install requirements." + #Ansible_262_36: + #python.version: '3.6' + #MODE: ansible + #VER: 2.6.2 - - script: .ci/$(MODE)_tests.py - displayName: Run tests. + #Vanilla_262_27: + #python.version: '2.7' + #MODE: ansible + #VER: 2.6.2 + #DISTROS: debian + #STRATEGY: linear diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 10e9d11e..d4f32f55 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -43,6 +43,18 @@ if not hasattr(subprocess, 'check_output'): subprocess.check_output = subprocess__check_output +# ------------------ + +def have_apt(): + proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True) + return proc.wait() == 0 + + +def have_docker(): + proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True) + return proc.wait() == 0 + + # ----------------- # Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars. diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index 10813b55..72bc75e3 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -6,10 +6,12 @@ batches = [ [ 'pip install "pycparser<2.19" "idna<2.7"', 'pip install -r tests/requirements.txt', - ], - [ - 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ] ] +if ci_lib.have_docker(): + batches.append([ + 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), + ]) + ci_lib.run_batches(batches) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 4ba796c2..36928ac9 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -11,4 +11,7 @@ os.environ.update({ 'SKIP_ANSIBLE': '1', }) +if not ci_lib.have_docker(): + os.environ['SKIP_DOCKER_TESTS'] = '1' + ci_lib.run('./run_tests -v') diff --git a/.ci/prep_azure.py b/.ci/prep_azure.py index 10126df2..5199a87e 100755 --- a/.ci/prep_azure.py +++ b/.ci/prep_azure.py @@ -1,22 +1,30 @@ #!/usr/bin/env python +import os +import sys + import ci_lib batches = [] -batches.append([ - 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', - 'sudo add-apt-repository ppa:deadsnakes/ppa', - 'sudo apt-get update', - 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', -]) - -batches.append([ - 'pip install -r dev_requirements.txt', -]) - -batches.extend( - ['docker pull %s' % (ci_lib.image_for_distro(distro),)] - for distro in ci_lib.DISTROS -) + +if ci_lib.have_apt(): + batches.append([ + 'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync', + 'sudo add-apt-repository ppa:deadsnakes/ppa', + 'sudo apt-get update', + 'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev', + ]) + + +#batches.append([ + #'pip install -r dev_requirements.txt', +#]) + +if ci_lib.have_docker(): + batches.extend( + ['docker pull %s' % (ci_lib.image_for_distro(distro),)] + for distro in ci_lib.DISTROS + ) + ci_lib.run_batches(batches) diff --git a/.travis.yml b/.travis.yml index aee14c00..921ad12b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,7 @@ matrix: # 2.4 -> 2.4 - language: c env: MODE=mitogen_py24 DISTRO=centos5 - # 2.7 -> 2.7 - - python: "2.7" - env: MODE=mitogen DISTRO=debian + # 2.7 -> 2.7 -- moved to Azure # 2.7 -> 2.6 #- python: "2.7" #env: MODE=mitogen DISTRO=centos6 @@ -39,9 +37,7 @@ matrix: # 2.6 -> 3.5 - python: "2.6" env: MODE=mitogen DISTRO=debian-py3 - # 3.6 -> 2.6 - - python: "3.6" - env: MODE=mitogen DISTRO=centos6 + # 3.6 -> 2.6 -- moved to Azure # Debops tests. # 2.4.6.0; 2.7 -> 2.7 diff --git a/tests/ansible/tests/affinity_test.py b/tests/ansible/tests/affinity_test.py index 8fa8cdb6..102608d4 100644 --- a/tests/ansible/tests/affinity_test.py +++ b/tests/ansible/tests/affinity_test.py @@ -17,6 +17,10 @@ class NullFixedPolicy(ansible_mitogen.affinity.FixedPolicy): self.mask = mask +@unittest2.skipIf( + reason='Linux only', + condition=(not os.uname()[0] == 'Linux') +) class FixedPolicyTest(testlib.TestCase): klass = NullFixedPolicy diff --git a/tests/file_service_test.py b/tests/file_service_test.py index 135d8e14..b9034bb1 100644 --- a/tests/file_service_test.py +++ b/tests/file_service_test.py @@ -1,4 +1,6 @@ +import sys + import unittest2 import mitogen.service @@ -32,10 +34,15 @@ class FetchTest(testlib.RouterMixin, testlib.TestCase): expect = service.unregistered_msg % ('/etc/shadow',) self.assertTrue(expect in e.args[0]) + if sys.platform == 'darwin': + ROOT_GROUP = 'wheel' + else: + ROOT_GROUP = 'root' + def _validate_response(self, resp): self.assertTrue(isinstance(resp, dict)) self.assertEquals('root', resp['owner']) - self.assertEquals('root', resp['group']) + self.assertEquals(self.ROOT_GROUP, resp['group']) self.assertTrue(isinstance(resp['mode'], int)) self.assertTrue(isinstance(resp['mtime'], float)) self.assertTrue(isinstance(resp['atime'], float)) diff --git a/tests/testlib.py b/tests/testlib.py index ef401a78..75061b26 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -450,6 +450,8 @@ class DockerMixin(RouterMixin): @classmethod def setUpClass(cls): super(DockerMixin, cls).setUpClass() + if os.environ.get('SKIP_DOCKER_TESTS'): + raise unittest2.SkipTest('SKIP_DOCKER_TESTS is set') cls.dockerized_ssh = DockerizedSshDaemon() cls.dockerized_ssh.wait_for_sshd()