archive
/
mitogen
mirror of https://github.com/mitogen-hq/mitogen.git
1
0
Fork 0
Code Releases Activity

ci: One Tox environment per Azure job

Browse Source
pull/872/head
Alex Willmer 3 years ago
parent 0af057e365
commit a080d7f134
11 changed files with 339 additions and 87 deletions
Show Stats Download Patch File Download Diff File

13
.ci/azure-pipelines-steps.yml
Unescape Escape View File

@ -10,7 +10,18 @@ steps:
inputs:
versionSpec: '$(python.version)'
- script: python -mpip install tox
- task: Cache@2
displayName: Cache pip packages
inputs:
key: python | "$(Agent.OS)" | tox.ini | tests/requirements.txt | tests/ansible/requirements.txt
restoreKeys: |
python | "$(Agent.OS)" | tox.ini | tests/requirements.txt
python | "$(Agent.OS)" | tox.ini
python | "$(Agent.OS)"
python
path: $(PIP_CACHE_DIR)
- script: python -mpip --no-python-version-warning --disable-pip-version-check install tox==3.24.4
displayName: Install tooling
- script: tox -e "$(tox.env)"

278
.ci/azure-pipelines.yml
Unescape Escape View File

@ -5,8 +5,9 @@
# User defined variables are also injected as environment variables
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#environment-variables
#variables:
variables:
#ANSIBLE_VERBOSITY: 3
PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
jobs:
@ -20,22 +21,78 @@ jobs:
vmImage: macOS-10.15
strategy:
matrix:
Mito27_27:
Py27Localhost2.10:
python.version: '2.7'
tox.env: py27-mode_localhost-ansible2.10
Py27Localhost3:
python.version: '2.7'
tox.env: py27-mode_localhost-ansible3
Py27Localhost4:
python.version: '2.7'
tox.env: py27-mode_localhost-ansible4
Py27Mitogen:
python.version: '2.7'
tox.env: py27-mode_mitogen
# TODO: test python3, python3 tests are broken
Local_Py27:
Py27Vanilla2.10:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '2.7'
tox.env: py27-mode_localhost-ansible{2.10,3,4}
# NOTE: this hangs when ran in Ubuntu 18.04
Vanilla_Py27:
tox.env: py27-mode_localhost-ansible2.10-strategy_linear
Py27Vanilla3:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '2.7'
tox.env: py27-mode_localhost-ansible{2.10,3,4}
STRATEGY: linear
tox.env: py27-mode_localhost-ansible3-strategy_linear
Py27Vanilla4:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '2.7'
tox.env: py27-mode_localhost-ansible4-strategy_linear
Py36Localhost2.10:
python.version: '3.6'
tox.env: py36-mode_localhost-ansible2.10
Py36Localhost3:
python.version: '3.6'
tox.env: py36-mode_localhost-ansible3
Py36Localhost4:
python.version: '3.6'
tox.env: py36-mode_localhost-ansible4
Py36Mitogen:
python.version: '3.6'
tox.env: py36-mode_mitogen
Py36Vanilla2.10:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.6'
tox.env: py36-mode_localhost-ansible2.10-strategy_linear
Py36Vanilla3:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.6'
tox.env: py36-mode_localhost-ansible3-strategy_linear
Py36Vanilla4:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.6'
tox.env: py36-mode_localhost-ansible4-strategy_linear
Py39Localhost2.10:
python.version: '3.9'
tox.env: py39-mode_localhost-ansible2.10
Py39Localhost3:
python.version: '3.9'
tox.env: py39-mode_localhost-ansible3
Py39Localhost4:
python.version: '3.9'
tox.env: py39-mode_localhost-ansible4
Py39Mitogen:
python.version: '3.9'
tox.env: py39-mode_mitogen
Py39Vanilla2.10:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.9'
tox.env: py39-mode_localhost-ansible2.10-strategy_linear
Py39Vanilla3:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.9'
tox.env: py39-mode_localhost-ansible3-strategy_linear
Py39Vanilla4:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.9'
tox.env: py39-mode_localhost-ansible4-strategy_linear
- job: Linux
pool:
@ -45,63 +102,204 @@ jobs:
- template: azure-pipelines-steps.yml
strategy:
matrix:
Mito_Py27:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
Mito_Py36:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
Mito_Py39:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
#DebOps_2460_27_27:
#python.version: '2.7'
#MODE: debops_common
#VER: 2.4.6.0
#DebOps_262_36_27:
#python.version: '3.6'
#MODE: debops_common
#VER: 2.6.2
#Ansible_2460_26:
#python.version: '2.7'
#MODE: ansible
#VER: 2.4.6.0
#Ansible_262_26:
#python.version: '2.7'
#MODE: ansible
#VER: 2.6.2
#Ansible_2460_36:
#python.version: '3.6'
#MODE: ansible
#VER: 2.4.6.0
#Ansible_262_36:
#python.version: '3.6'
#MODE: ansible
#VER: 2.6.2
#Vanilla_262_27:
#python.version: '2.7'
#MODE: ansible
#VER: 2.6.2
#DISTROS: debian
#STRATEGY: linear
Ansible_Py27:
#ANSIBLE_STRATEGY: linear
Py27Ansible2.10:
python.version: '2.7'
tox.env: py27-mode_ansible-ansible{2.10,3,4}
Ansible_Py36:
tox.env: py27-mode_ansible-ansible2.10
Py27Ansible3:
python.version: '2.7'
tox.env: py27-mode_ansible-ansible3
Py27Ansible4:
python.version: '2.7'
tox.env: py27-mode_ansible-ansible4
Py27Localhost2.10:
python.version: '2.7'
tox.env: py27-mode_localhost-ansible2.10
Py27Localhost3:
python.version: '2.7'
tox.env: py27-mode_localhost-ansible3
Py27Localhost4:
python.version: '2.7'
tox.env: py27-mode_localhost-ansible4
Py27MitogenCentos6:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_centos6
Py27MitogenCentos7:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_centos7
Py27MitogenCentos8:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_centos8
Py27MitogenDebian10:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_debian10
Py27MitogenDebian11:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_debian11
Py27MitogenDebian9:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_debian9
Py27MitogenUbuntu1604:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_ubuntu1604
Py27MitogenUbuntu1804:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_ubuntu1804
Py27MitogenUbuntu2004:
python.version: '2.7'
tox.env: py27-mode_mitogen-distro_ubuntu2004
Py27Vanilla2.10:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '2.7'
tox.env: py27-mode_localhost-ansible2.10-strategy_linear
Py27Vanilla3:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '2.7'
tox.env: py27-mode_localhost-ansible3-strategy_linear
Py27Vanilla4:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '2.7'
tox.env: py27-mode_localhost-ansible4-strategy_linear
Py36Ansible2.10:
python.version: '3.6'
tox.env: py36-mode_ansible-ansible{2.10,3,4}
Ansible_Py39:
tox.env: py36-mode_ansible-ansible2.10
Py36Ansible3:
python.version: '3.6'
tox.env: py36-mode_ansible-ansible3
Py36Ansible4:
python.version: '3.6'
tox.env: py36-mode_ansible-ansible4
Py36Localhost2.10:
python.version: '3.6'
tox.env: py36-mode_localhost-ansible2.10
Py36Localhost3:
python.version: '3.6'
tox.env: py36-mode_localhost-ansible3
Py36Localhost4:
python.version: '3.6'
tox.env: py36-mode_localhost-ansible4
Py36MitogenCentos6:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_centos6
Py36MitogenCentos7:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_centos7
Py36MitogenCentos8:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_centos8
Py36MitogenDebian10:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_debian10
Py36MitogenDebian11:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_debian11
Py36MitogenDebian9:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_debian9
Py36MitogenUbuntu1604:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_ubuntu1604
Py36MitogenUbuntu1804:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_ubuntu1804
Py36MitogenUbuntu2004:
python.version: '3.6'
tox.env: py36-mode_mitogen-distro_ubuntu2004
Py36Vanilla2.10:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.6'
tox.env: py36-mode_localhost-ansible2.10-strategy_linear
Py36Vanilla3:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.6'
tox.env: py36-mode_localhost-ansible3-strategy_linear
Py36Vanilla4:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.6'
tox.env: py36-mode_localhost-ansible4-strategy_linear
Py39Ansible2.10:
python.version: '3.9'
tox.env: py39-mode_ansible-ansible2.10
Py39Ansible3:
python.version: '3.9'
tox.env: py39-mode_ansible-ansible3
Py39Ansible4:
python.version: '3.9'
tox.env: py39-mode_ansible-ansible4
Py39Localhost2.10:
python.version: '3.9'
tox.env: py39-mode_localhost-ansible2.10
Py39Localhost3:
python.version: '3.9'
tox.env: py39-mode_localhost-ansible3
Py39Localhost4:
python.version: '3.9'
tox.env: py39-mode_localhost-ansible4
Py39MitogenCentos6:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_centos6
Py39MitogenCentos7:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_centos7
Py39MitogenCentos8:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_centos8
Py39MitogenDebian10:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_debian10
Py39MitogenDebian11:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_debian11
Py39MitogenDebian9:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_debian9
Py39MitogenUbuntu1604:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_ubuntu1604
Py39MitogenUbuntu1804:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_ubuntu1804
Py39MitogenUbuntu2004:
python.version: '3.9'
tox.env: py39-mode_mitogen-distro_ubuntu2004
Py39Vanilla2.10:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.9'
tox.env: py39-mode_localhost-ansible2.10-strategy_linear
Py39Vanilla3:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.9'
tox.env: py39-mode_localhost-ansible3-strategy_linear
Py39Vanilla4:
ANSIBLE_SKIP_TAGS: resource_intensive
python.version: '3.9'
tox.env: py39-mode_ansible-ansible{2.10,3,4}
tox.env: py39-mode_localhost-ansible4-strategy_linear

39
.ci/azure_matrix
Unescape Escape View File

@ -0,0 +1,39 @@
#!/usr/bin/env python3
import sys
import yaml
def matrix():
for py in '2.7 3.6 3.9'.split():
pynodot = py.replace('.', '')
for mode in ['ansible', 'localhost']:
for ansible in '2.10 3 4'.split():
yield {
"name": f"Py{pynodot}{mode.title()}{ansible}",
"vars": {
"python.version": py,
"tox.env": f"py{pynodot}-mode_{mode}-ansible{ansible}",
},
}
if mode in {'localhost'}:
yield {
"name": f"Py{pynodot}Vanilla{ansible}",
"vars": {
"ANSIBLE_SKIP_TAGS": "resource_intensive",
"python.version": py,
"tox.env": f"py{pynodot}-mode_{mode}-ansible{ansible}-strategy_linear",
},
}
for mode in ['mitogen']:
for distro in 'centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004'.split(','):
yield {
"name": f"Py{pynodot}{mode.title()}{distro.title()}",
"vars": {
"python.version": py,
"tox.env": f"py{pynodot}-mode_{mode}-distro_{distro}",
},
}
d = {x['name']: x['vars'] for x in matrix()}
yaml.dump(d, stream=sys.stdout)

2
.ci/ci_lib.py
Unescape Escape View File

@ -193,8 +193,6 @@ class Fold(object):
def __exit__(self, _1, _2, _3): pass
os.environ.setdefault('ANSIBLE_STRATEGY',
os.environ.get('STRATEGY', 'mitogen_linear'))
GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
# Used only when MODE=mitogen
DISTRO = os.environ.get('DISTRO', 'debian9')

32
.ci/localhost_ansible_tests.py
Unescape Escape View File

@ -2,6 +2,7 @@
# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
import os
import subprocess
import sys
import ci_lib
@ -24,29 +25,34 @@ with ci_lib.Fold('job_setup'):
# NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n",
# there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
if not ci_lib.exists_in_path('sshpass'):
os.system("curl -O -L https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \
subprocess.check_call(
"curl -O -L https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \
tar xvf sshpass-1.05.tar.gz && \
cd sshpass-1.05 && \
./configure && \
sudo make install")
sudo make install",
shell=True,
)
with ci_lib.Fold('machine_prep'):
# generate a new ssh key for localhost ssh
if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")):
os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
subprocess.check_call("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa", shell=True)
subprocess.check_call("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys", shell=True)
os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
#ci_lib.dump_file(os.path.expanduser('~/.ssh/authorized_keys'))
#ci_lib.dump_file('/etc/ssh/sshd_config')
#subprocess.check_call('ssh -vvv -i ~/.ssh/id_rsa -o BatchMode=no -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o PreferredAuthentications=publickey -o IdentitiesOnly=yes -o StrictHostKeyChecking=no localhost whoami', shell=True)
# also generate it for the sudo user
if os.system("sudo [ -f /var/root/.ssh/id_rsa ]") != 0:
os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
# run chmod through sudo since it's owned by root
os.system('sudo chmod 700 /var/root/.ssh')
os.system('sudo chmod 600 /var/root/.ssh/authorized_keys')
if os.system("sudo [ -f ~root/.ssh/id_rsa ]") != 0:
subprocess.check_call("sudo ssh-keygen -P '' -m pem -f ~root/.ssh/id_rsa", shell=True)
subprocess.check_call("sudo cat ~root/.ssh/id_rsa.pub | sudo tee -a ~root/.ssh/authorized_keys", shell=True)
subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True)
subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True)
#subprocess.check_call('sudo ssh -vvv -i ~root/.ssh/id_rsa -o BatchMode=no -o UserKnownHostsFile=/dev/null -o PasswordAuthentication=no -o PreferredAuthentications=publickey -o IdentitiesOnly=yes -o StrictHostKeyChecking=no localhost whoami', shell=True)
if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
os.chdir(IMAGE_PREP_DIR)

1
tests/ansible/integration/become/sudo_nonexistent.yml
Unescape Escape View File

@ -25,6 +25,7 @@
'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg)
or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)
or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')
or 'Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user (rc: 1, err: chmod: Invalid file mode: A+user:slartibartfast:rx:allow\n})' in out.msg
fail_msg: out={{out}}
when:
# https://github.com/ansible/ansible/pull/70785

47
tests/ansible/regression/issue_655__wait_for_connection_error.yml
Unescape Escape View File

@ -13,36 +13,33 @@
- name: set up test container and run tests inside it
block:
- name: install deps
block:
- name: install docker
shell: |
# NOTE: for tracking purposes: https://github.com/docker/for-mac/issues/2359
# using docker for mac CI workaround: https://github.com/drud/ddev/pull/1748/files#diff-19288f650af2dabdf1dcc5b354d1f245
DOCKER_URL=https://download.docker.com/mac/stable/31259/Docker.dmg &&
curl -O -sSL $DOCKER_URL &&
open -W Docker.dmg && cp -r /Volumes/Docker/Docker.app /Applications
sudo /Applications/Docker.app/Contents/MacOS/Docker --quit-after-install --unattended &&
ln -s /Applications/Docker.app/Contents/Resources/bin/docker /usr/local/bin/docker &&
nohup /Applications/Docker.app/Contents/MacOS/Docker --unattended &
# wait 2 min for docker to come up
counter=0 &&
while ! /usr/local/bin/docker ps 2>/dev/null ; do
if [ $counter -lt 24 ]; then
let counter=counter+1
else
exit 1
fi
sleep 5
done
homebrew:
name:
- podman
state: present
- name: start machine
command:
cmd: "{{ item.cmd }}"
loop:
- cmd: podman machine init
- cmd: podman machine start
- cmd: sleep 10 # FIXME Find a way to wait until the machine is available
- cmd: podman info
register: podman_machine
- debug:
var: podman_machine
# python bindings (docker_container) aren't working on this host, so gonna shell out
- name: create docker container
shell: /usr/local/bin/docker run --name testMitogen -d --rm centos:8 bash -c "sleep infinity & wait"
- name: create container
command:
cmd: podman run --name=testMitogen --detach --rm centos:8 bash -c "sleep infinity & wait"
- name: add container to inventory
add_host:
name: testMitogen
ansible_connection: docker
ansible_connection: podman
ansible_user: root
changed_when: false
environment:
@ -82,6 +79,6 @@
PATH: /usr/local/bin/:{{ ansible_env.PATH }}
- name: remove test container
shell: /usr/local/bin/docker stop testMitogen
shell: podman stop testMitogen
tags:
- issue_655

2
tests/image_prep/_user_accounts.yml
Unescape Escape View File

@ -108,7 +108,7 @@
when: ansible_system == 'Linux' and out.stat.exists
service:
name: accounts-daemon
restarted: true
state: restarted
- name: Readonly homedir for one account
shell: "chown -R root: ~mitogen__readonly_homedir"

2
tests/requirements.txt
Unescape Escape View File

@ -1,8 +1,8 @@
psutil==5.4.8
coverage==4.5.1
Django==1.6.11 # Last version supporting 2.6.
faulthandler==3.2; python_version < '3.3'
mock==2.0.0
psutil==5.8.0
pytz==2018.5
cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect
pycparser==2.18 # Last version supporting 2.6.

5
tests/testlib.py
Unescape Escape View File

@ -11,6 +11,7 @@ import threading
import time
import traceback
import psutil
import unittest2
import mitogen.core
@ -67,7 +68,6 @@ def get_fd_count():
"""
Return the number of FDs open by this process.
"""
import psutil
return psutil.Process().num_fds()
@ -386,7 +386,8 @@ class TestCase(unittest2.TestCase):
print('')
print('Children of unit test process:')
os.system('ps uww --ppid ' + str(os.getpid()))
child_pids = ','.join(str(p.pid) for p in psutil.Process().children())
subprocess.check_call(['ps', '-fww', '-p', child_pids])
assert 0, "%s leaked still-running subprocesses." % (self,)
def tearDown(self):

5
tox.ini
Unescape Escape View File

@ -65,7 +65,7 @@ deps =
ansible3: ansible==3.4.0
ansible4: ansible==4.8.0
install_command =
python -m pip --no-python-version-warning install {opts} {packages}
python -mpip --no-python-version-warning --disable-pip-version-check install {opts} {packages}
commands_pre =
mode_ansible: {toxinidir}/.ci/ansible_install.py
mode_debops_common: {toxinidir}/.ci/debops_common_install.py
@ -83,6 +83,7 @@ passenv =
AWS_SECRET_ACCESS_KEY
HOME
setenv =
ANSIBLE_STRATEGY = mitogen_linear
NOCOVERAGE_ERASE = 1
NOCOVERAGE_REPORT = 1
# Only applicable to MODE=mitogen
@ -113,7 +114,7 @@ setenv =
mode_ansible: MODE=ansible
mode_debops_common: MODE=debops_common
mode_mitogen: MODE=mitogen
strategy_linear: STRATEGY=linear
strategy_linear: ANSIBLE_STRATEGY=linear
whitelist_externals =
docker
docker-credential-secretservice

Write Preview
Loading…
Cancel
Save