archive
/
mitogen
mirror of https://github.com/mitogen-hq/mitogen.git
1
0
Fork 0
Code Releases Activity

Merge pull request #1247 from moreati/issue-1118-ci_lib

Browse Source 
CI: ci_lib cleanup
pull/1081/merge
Alex Willmer 11 months ago committed by GitHub
parent 26507481b1 a376daa04d
commit 0388cd5c0f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 33 additions and 54 deletions
Show Stats Download Patch File Download Diff File

29
.ci/ansible_tests.py
Unescape Escape View File

@ -12,9 +12,8 @@ import jinja2
import ci_lib
TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible')
TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts')
def pause_if_interactive():
@ -40,13 +39,13 @@ with ci_lib.Fold('docker_setup'):
with ci_lib.Fold('job_setup'):
os.chdir(TESTS_DIR)
os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 8))
os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
ci_lib.run("mkdir %s", HOSTS_DIR)
for path in glob.glob(TESTS_DIR + '/hosts/*'):
os.mkdir(TMP_HOSTS_DIR)
for path in glob.glob(os.path.join(ci_lib.ANSIBLE_TESTS_HOSTS_DIR, '*')):
if not path.endswith('default.hosts'):
ci_lib.run("ln -s %s %s", path, HOSTS_DIR)
os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path)))
distros = collections.defaultdict(list)
families = collections.defaultdict(list)
@ -55,12 +54,14 @@ with ci_lib.Fold('job_setup'):
families[container['family']].append(container['name'])
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
loader=jinja2.FileSystemLoader(
searchpath=ci_lib.ANSIBLE_TESTS_TEMPLATES_DIR,
),
lstrip_blocks=True, # Remove spaces and tabs from before a block
trim_blocks=True, # Remove first newline after a block
)
inventory_template = jinja_env.get_template('test-targets.j2')
inventory_path = os.path.join(HOSTS_DIR, 'target')
inventory_path = os.path.join(TMP_HOSTS_DIR, 'test-targets.ini')
with open(inventory_path, 'w') as fp:
fp.write(inventory_template.render(
@ -71,16 +72,12 @@ with ci_lib.Fold('job_setup'):
ci_lib.dump_file(inventory_path)
if not ci_lib.exists_in_path('sshpass'):
ci_lib.run("sudo apt-get update")
ci_lib.run("sudo apt-get install -y sshpass")
with ci_lib.Fold('ansible'):
playbook = os.environ.get('PLAYBOOK', 'all.yml')
try:
ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s',
playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
playbook, TMP_HOSTS_DIR, ' '.join(sys.argv[1:]),
)
except:
pause_if_interactive()
raise

34
.ci/ci_lib.py
Unescape Escape View File

@ -28,14 +28,20 @@ os.chdir(
)
GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
ANSIBLE_TESTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible')
ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts')
ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates')
DISTRO_SPECS = os.environ.get(
'MITOGEN_TEST_DISTRO_SPECS',
'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
)
IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep')
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key')
_print = print
@ -56,19 +62,11 @@ def _have_cmd(args):
if exc.errno == errno.ENOENT:
return False
raise
except subprocess.CallProcessError:
except subprocess.CalledProcessError:
return False
return True
def have_apt():
return _have_cmd(['apt', '--help'])
def have_brew():
return _have_cmd(['brew', 'help'])
def have_docker():
return _have_cmd(['docker', 'info'])
@ -185,8 +183,8 @@ def exists_in_path(progname):
class TempDir(object):
def __init__(self):
self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib')
def __init__(self, prefix='mitogen_ci_lib'):
self.path = tempfile.mkdtemp(prefix=prefix)
atexit.register(self.destroy)
def destroy(self, rmtree=shutil.rmtree):
@ -199,20 +197,6 @@ class Fold(object):
def __exit__(self, _1, _2, _3): pass
GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
TMP = TempDir().path
# We copy this out of the way to avoid random stuff modifying perms in the Git
# tree (like git pull).
src_key_file = os.path.join(GIT_ROOT,
'tests/data/docker/mitogen__has_sudo_pubkey.key')
key_file = os.path.join(TMP,
'mitogen__has_sudo_pubkey.key')
shutil.copyfile(src_key_file, key_file)
os.chmod(key_file, int('0600', 8))
os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
os.environ['PYTHONPATH'] = '%s:%s' % (
os.environ.get('PYTHONPATH', ''),

6
.ci/debops_common_tests.py
Unescape Escape View File

@ -6,7 +6,8 @@ import sys
import ci_lib
project_dir = os.path.join(ci_lib.TMP, 'project')
TMP = ci_lib.TempDir(prefix='mitogen_ci_debops')
project_dir = os.path.join(TMP.path, 'project')
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts'
docker_hostname = ci_lib.get_docker_hostname()
@ -22,6 +23,7 @@ with ci_lib.Fold('docker_setup'):
with ci_lib.Fold('job_setup'):
os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
ci_lib.run('debops-init %s', project_dir)
os.chdir(project_dir)
@ -45,7 +47,7 @@ with ci_lib.Fold('job_setup'):
"\n"
# Speed up slow DH generation.
"dhparam__bits: ['128', '64']\n"
% (ci_lib.key_file,)
% (ci_lib.TESTS_SSH_PRIVATE_KEY_FILE,)
)
with open(inventory_path, 'a') as fp:

14
.ci/localhost_ansible_tests.py
Unescape Escape View File

@ -10,19 +10,13 @@ import sys
import ci_lib
TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep')
HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts')
KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key')
with ci_lib.Fold('unit_tests'):
os.environ['SKIP_MITOGEN'] = '1'
ci_lib.run('./run_tests -v')
with ci_lib.Fold('job_setup'):
os.chmod(KEY_PATH, int('0600', 8))
os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
# NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n",
# there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
if not ci_lib.exists_in_path('sshpass'):
@ -51,11 +45,11 @@ with ci_lib.Fold('machine_prep'):
subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True)
subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True)
os.chdir(IMAGE_PREP_DIR)
os.chdir(ci_lib.IMAGE_PREP_DIR)
ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml")
if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
os.chdir(IMAGE_PREP_DIR)
os.chdir(ci_lib.IMAGE_PREP_DIR)
ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
cmd = ';'.join([
@ -80,7 +74,7 @@ with ci_lib.Fold('machine_prep'):
with ci_lib.Fold('ansible'):
os.chdir(TESTS_DIR)
os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
playbook = os.environ.get('PLAYBOOK', 'all.yml')
ci_lib.run('./run_ansible_playbook.py %s %s',
playbook, ' '.join(sys.argv[1:]))

2
.github/workflows/tests.yml vendored
Unescape Escape View File

@ -96,7 +96,7 @@ jobs:
set -o errexit -o nounset -o pipefail
sudo apt-get update
sudo apt-get install -y python2-dev python3-pip virtualenv
sudo apt-get install -y python2-dev python3-pip sshpass virtualenv
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail

2
docs/changelog.rst
Unescape Escape View File

@ -24,6 +24,8 @@ In progress (unreleased)
* :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes
* :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6
* :gh:issue:`1118` CI: Statically specify test usernames and group names
* :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir
* :gh:issue:`1118` CI: Don't share temporary directory between test groupings
v0.3.22 (2025-02-04)

Write Preview
Loading…
Cancel
Save