ci: Extract container registry location into variables

Preperation for migrating from Azure DevOps with Amazon Elastic Container
Registry (AWS ECR), to GitHub Actions with GitHub Container Registry (GHCR).

DebOps tests are not currently being run, the updates to .ci/debops*.py are
best effort only.

.ci/README.md

@@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
 * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
   name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
 * `DISTROS`: the `ansible_` tests can run against multiple targets

.ci/ansible_tests.py

@@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting)
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers()
+    containers = ci_lib.container_specs(ci_lib.DISTROS)
     ci_lib.start_containers(containers)
 
 

.ci/ci_lib.py

@@ -27,6 +27,13 @@ os.chdir(
     )
 )
 
+
+IMAGE_TEMPLATE = os.environ.get(
+    'MITOGEN_TEST_IMAGE_TEMPLATE',
+    'public.ecr.aws/n5z0e8q9/%(distro)s-test',
+)
+
+
 _print = print
 def print(*args, **kwargs):
     file = kwargs.get('file', sys.stdout)
@@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 DISTRO = os.environ.get('DISTRO', 'debian9')
 # Used only when MODE=ansible
 DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
-TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
-BASE_PORT = 2200
 TMP = TempDir().path
 
 
@@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
 def get_docker_hostname():
     """Return the hostname where the docker daemon is running.
     """
+    # Duplicated in testlib
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -225,27 +231,34 @@ def get_docker_hostname():
     return parsed.netloc.partition(':')[0]
 
 
-def make_containers(name_prefix='', port_offset=0):
+def container_specs(
+        distros,
+        base_port=2200,
+        image_template=IMAGE_TEMPLATE,
+        name_template='target-%(distro)s-%(index)d',
+):
     """
     >>> import pprint
-    >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6']
+    >>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
-    >>> pprint.pprint(make_containers())
     [{'distro': 'debian11',
       'family': 'debian',
       'hostname': 'localhost',
       'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
+      'index': 1,
       'name': 'target-debian11-1',
       'port': 2201,
-      'python_path': '/usr/bin/python'},
+      'python_path': '/usr/bin/python3'},
      {'distro': 'centos6',
       'family': 'centos',
       'hostname': 'localhost',
       'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
+      'index': 2,
       'name': 'target-centos6-2',
       'port': 2202,
       'python_path': '/usr/bin/python'}]
     """
     docker_hostname = get_docker_hostname()
+    # Code duplicated in testlib.py, both should be updated together
     distro_pattern = re.compile(r'''
         (?P<distro>(?P<family>[a-z]+)[0-9]+)
         (?:-(?P<py>py3))?
@@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
     i = 1
     lst = []
 
-    for distro in DISTROS:
+    for distro in distros:
+        # Code duplicated in testlib.py, both should be updated together
         d = distro_pattern.match(distro).groupdict(default=None)
-        distro = d['distro']
-        family = d['family']
-        image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
 
-        if d['py'] == 'py3':
+        if d.pop('py') == 'py3':
             python_path = '/usr/bin/python3'
         else:
             python_path = '/usr/bin/python'
 
-        if d['count']:
+        count = int(d.pop('count') or '1', 10)
-            count = int(count)
-        else:
-            count = 1
 
         for x in range(count):
-            lst.append({
+            d['index'] = i
-                "distro": distro, "family": family, "image": image,
+            d.update({
-                "name": name_prefix + ("target-%s-%s" % (distro, i)),
+                'image': image_template % d,
+                'name': name_template % d,
                 "hostname": docker_hostname,
-                "port": BASE_PORT + i + port_offset,
+                'port': base_port + i,
                 "python_path": python_path,
             })
+            lst.append(d)
             i += 1
 
     return lst

.ci/debops_common_install.py

@@ -2,9 +2,6 @@
 
 import ci_lib
 
-# Naturally DebOps only supports Debian.
-ci_lib.DISTROS = ['debian']
-
 ci_lib.run_batches([
     [
         'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',

.ci/debops_common_tests.py

@@ -6,9 +6,6 @@ import sys
 import ci_lib
 
 
-# DebOps only supports Debian.
-ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
-
 project_dir = os.path.join(ci_lib.TMP, 'project')
 vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
 inventory_path = 'ansible/inventory/hosts'
@@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname()
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
+    containers = ci_lib.container_specs(
+        ['debian*2'],
+        base_port=2700,
+        name_template='debops-target-%(distro)s-%(index)d',
+    )
     ci_lib.start_containers(containers)
 
 

tests/image_prep/_container_finalize.yml

@@ -9,7 +9,7 @@
         --change 'EXPOSE 22'
         --change 'CMD ["/usr/sbin/sshd", "-D"]'
         {{ inventory_hostname }}
-        public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
+        {{ container_image_name }}
       delegate_to: localhost
 
     - name: Stop containers

tests/image_prep/group_vars/all.yml

@@ -4,6 +4,9 @@ common_packages:
   - strace
   - sudo
 
+container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test"
+container_registry: public.ecr.aws/n5z0e8q9
+
 sudo_group:
   MacOSX: admin
   Debian: sudo

tests/testlib.py

@@ -51,6 +51,12 @@ except NameError:
 
 LOG = logging.getLogger(__name__)
 
+DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
+IMAGE_TEMPLATE = os.environ.get(
+    'MITOGEN_TEST_IMAGE_TEMPLATE',
+    'public.ecr.aws/n5z0e8q9/%(distro)s-test',
+)
+
 TESTS_DIR =                     os.path.join(os.path.dirname(__file__))
 ANSIBLE_LIB_DIR =               os.path.join(TESTS_DIR, 'ansible', 'lib')
 ANSIBLE_MODULE_UTILS_DIR =      os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
@@ -509,6 +515,7 @@ class TestCase(unittest.TestCase):
 
 
 def get_docker_host():
+    # Duplicated in ci_lib
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -549,19 +556,23 @@ class DockerizedSshDaemon(object):
         ]
         subprocess.check_output(args)
 
-    def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
+    def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
-        if '-'  in mitogen_test_distro:
+        # Code duplicated in ci_lib.py, both should be updated together
-            distro, _py3 = mitogen_test_distro.split('-')
+        distro_pattern = re.compile(r'''
-        else:
+            (?P<distro>(?P<family>[a-z]+)[0-9]+)
-            distro = mitogen_test_distro
+            (?:-(?P<py>py3))?
-            _py3 = None
+            (?:\*(?P<count>[0-9]+))?
+            ''',
+            re.VERBOSE,
+        )
+        d = distro_pattern.match(distro).groupdict(default=None)
 
-        if _py3 == 'py3':
+        if d.pop('py') == 'py3':
             self.python_path = '/usr/bin/python3'
         else:
             self.python_path = '/usr/bin/python'
 
-        self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
+        self.image = image_template % d
         self.start_container()
         self.host = self.get_host()
         self.port = self.get_port(self.container_name)
@@ -601,6 +612,9 @@ class DockerizedSshDaemon(object):
 
 class BrokerMixin(object):
     broker_class = mitogen.master.Broker
+
+    # Flag for tests that shutdown the broker themself
+    # e.g. unix_test.ListenerTest
     broker_shutdown = False
 
     def setUp(self):

tox.ini

@@ -98,6 +98,7 @@ commands =
 passenv =
     ANSIBLE_*
     HOME
+    MITOGEN_*
     # Azure DevOps, TF_BUILD is set to 'True' when running in a build task
     # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables
     TF_BUILD