archive
/
mitogen
mirror of https://github.com/mitogen-hq/mitogen.git
1
0
Fork 0
Code Releases Activity

ci: Extract container registry location into variables

Browse Source 
Preperation for migrating from Azure DevOps with Amazon Elastic Container
Registry (AWS ECR), to GitHub Actions with GitHub Container Registry (GHCR).

DebOps tests are not currently being run, the updates to .ci/debops*.py are
best effort only.
pull/1128/head
Alex Willmer 2 months ago
parent b926795973
commit 8b92e09655
9 changed files with 62 additions and 37 deletions
Show Stats Download Patch File Download Diff File

1
.ci/README.md
Unescape Escape View File

@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
### Environment Variables
* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
* `DISTROS`: the `ansible_` tests can run against multiple targets

2
.ci/ansible_tests.py
Unescape Escape View File

@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting)
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers()
containers = ci_lib.container_specs(ci_lib.DISTROS)
ci_lib.start_containers(containers)

48
.ci/ci_lib.py
Unescape Escape View File

@ -27,6 +27,13 @@ os.chdir(
)
)
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
_print = print
def print(*args, **kwargs):
file = kwargs.get('file', sys.stdout)
@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
DISTRO = os.environ.get('DISTRO', 'debian9')
# Used only when MODE=ansible
DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
BASE_PORT = 2200
TMP = TempDir().path
@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
def get_docker_hostname():
"""Return the hostname where the docker daemon is running.
"""
# Duplicated in testlib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -225,27 +231,34 @@ def get_docker_hostname():
return parsed.netloc.partition(':')[0]
def make_containers(name_prefix='', port_offset=0):
def container_specs(
distros,
base_port=2200,
image_template=IMAGE_TEMPLATE,
name_template='target-%(distro)s-%(index)d',
):
"""
>>> import pprint
>>> BASE_PORT=2200; DISTROS=['debian11', 'centos6']
>>> pprint.pprint(make_containers())
>>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
[{'distro': 'debian11',
'family': 'debian',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
'index': 1,
'name': 'target-debian11-1',
'port': 2201,
'python_path': '/usr/bin/python'},
'python_path': '/usr/bin/python3'},
{'distro': 'centos6',
'family': 'centos',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
'index': 2,
'name': 'target-centos6-2',
'port': 2202,
'python_path': '/usr/bin/python'}]
"""
docker_hostname = get_docker_hostname()
# Code duplicated in testlib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
i = 1
lst = []
for distro in DISTROS:
for distro in distros:
# Code duplicated in testlib.py, both should be updated together
d = distro_pattern.match(distro).groupdict(default=None)
distro = d['distro']
family = d['family']
image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
if d['py'] == 'py3':
if d.pop('py') == 'py3':
python_path = '/usr/bin/python3'
else:
python_path = '/usr/bin/python'
if d['count']:
count = int(count)
else:
count = 1
count = int(d.pop('count') or '1', 10)
for x in range(count):
lst.append({
"distro": distro, "family": family, "image": image,
"name": name_prefix + ("target-%s-%s" % (distro, i)),
d['index'] = i
d.update({
'image': image_template % d,
'name': name_template % d,
"hostname": docker_hostname,
"port": BASE_PORT + i + port_offset,
'port': base_port + i,
"python_path": python_path,
})
lst.append(d)
i += 1
return lst

3
.ci/debops_common_install.py
Unescape Escape View File

@ -2,9 +2,6 @@
import ci_lib
# Naturally DebOps only supports Debian.
ci_lib.DISTROS = ['debian']
ci_lib.run_batches([
[
'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',

9
.ci/debops_common_tests.py
Unescape Escape View File

@ -6,9 +6,6 @@ import sys
import ci_lib
# DebOps only supports Debian.
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
project_dir = os.path.join(ci_lib.TMP, 'project')
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts'
@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname()
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
containers = ci_lib.container_specs(
['debian*2'],
base_port=2700,
name_template='debops-target-%(distro)s-%(index)d',
)
ci_lib.start_containers(containers)

2
tests/image_prep/_container_finalize.yml
Unescape Escape View File

@ -9,7 +9,7 @@
--change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]'
{{ inventory_hostname }}
public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
{{ container_image_name }}
delegate_to: localhost
- name: Stop containers

3
tests/image_prep/group_vars/all.yml
Unescape Escape View File

@ -4,6 +4,9 @@ common_packages:
- strace
- sudo
container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test"
container_registry: public.ecr.aws/n5z0e8q9
sudo_group:
MacOSX: admin
Debian: sudo

30
tests/testlib.py
Unescape Escape View File

@ -51,6 +51,12 @@ except NameError:
LOG = logging.getLogger(__name__)
DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
TESTS_DIR = os.path.join(os.path.dirname(__file__))
ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib')
ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
@ -509,6 +515,7 @@ class TestCase(unittest.TestCase):
def get_docker_host():
# Duplicated in ci_lib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -549,19 +556,23 @@ class DockerizedSshDaemon(object):
]
subprocess.check_output(args)
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
if '-' in mitogen_test_distro:
distro, _py3 = mitogen_test_distro.split('-')
else:
distro = mitogen_test_distro
_py3 = None
def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
# Code duplicated in ci_lib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
(?:\*(?P<count>[0-9]+))?
''',
re.VERBOSE,
)
d = distro_pattern.match(distro).groupdict(default=None)
if _py3 == 'py3':
if d.pop('py') == 'py3':
self.python_path = '/usr/bin/python3'
else:
self.python_path = '/usr/bin/python'
self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
self.image = image_template % d
self.start_container()
self.host = self.get_host()
self.port = self.get_port(self.container_name)
@ -601,6 +612,9 @@ class DockerizedSshDaemon(object):
class BrokerMixin(object):
broker_class = mitogen.master.Broker
# Flag for tests that shutdown the broker themself
# e.g. unix_test.ListenerTest
broker_shutdown = False
def setUp(self):

1
tox.ini
Unescape Escape View File

@ -98,6 +98,7 @@ commands =
passenv =
ANSIBLE_*
HOME
MITOGEN_*
# Azure DevOps, TF_BUILD is set to 'True' when running in a build task
# https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables
TF_BUILD

Write Preview
Loading…
Cancel
Save