diff --git a/.ci/README.md b/.ci/README.md index 9a5e8898..17b7d2dd 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,14 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This - name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` -* `DISTROS`: the `ansible_` tests can run against multiple targets - simultaneously, which speeds things up. This is a space-separated list of - DISTRO names, but additionally, supports: +* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run + the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines + the Linux distribution, target Python interepreter & number of instances. + Only distributions with a pre-built Linux container image can be used. * `debian-py3`: when generating Ansible inventory file, set `ansible_python_interpreter` to `python3`, i.e. run a test where the target interpreter is Python 3. * `debian*16`: generate 16 Docker containers running Debian. Also works with -py3. +* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name, + and hence the container registry used for test targets. diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 3ec48dfd..62dfa8f5 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting) with ci_lib.Fold('docker_setup'): - containers = ci_lib.container_specs(ci_lib.DISTROS) + containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split()) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index dfe49b97..afb62e02 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -28,6 +28,10 @@ os.chdir( ) +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', @@ -196,10 +200,6 @@ class Fold(object): GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -# Used only when MODE=mitogen -DISTRO = os.environ.get('DISTRO', 'debian9') -# Used only when MODE=ansible -DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() TMP = TempDir().path diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index c36bad26..502a9abc 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -3,8 +3,6 @@ from __future__ import print_function -import getpass -import io import os import subprocess import sys @@ -57,18 +55,6 @@ with ci_lib.Fold('machine_prep'): os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") - # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022 - # and os.environ['USER'] is not populated on Azure macOS runners. - os.chdir(HOSTS_DIR) - with io.open('default.hosts', 'r+', encoding='utf-8') as f: - user = getpass.getuser() - content = f.read() - content = content.replace("{{ lookup('pipe', 'whoami') }}", user) - f.seek(0) - f.write(content) - f.truncate() - ci_lib.dump_file('default.hosts') - cmd = ';'.join([ 'from __future__ import print_function', 'import os, sys', diff --git a/.ci/mitogen_py24_tests.py b/.ci/mitogen_py24_tests.py index 228e79bd..96b144eb 100755 --- a/.ci/mitogen_py24_tests.py +++ b/.ci/mitogen_py24_tests.py @@ -8,8 +8,6 @@ import ci_lib os.environ.update({ 'NOCOVERAGE': '1', 'UNIT2': '/usr/local/python2.4.6/bin/unit2', - - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 4de94b4c..47aa2444 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -6,7 +6,6 @@ import os import ci_lib os.environ.update({ - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f3f31d82..cc20f04a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,80 +67,14 @@ jobs: python_version: '3.13' tox_env: py313-mode_ansible-ansible10-strategy_linear - - name: Mito_27_centos6 - tox_env: py27-mode_mitogen-distro_centos6 - - name: Mito_27_centos7 - tox_env: py27-mode_mitogen-distro_centos7 - - name: Mito_27_centos8 - tox_env: py27-mode_mitogen-distro_centos8 - - name: Mito_27_debian9 - tox_env: py27-mode_mitogen-distro_debian9 - - name: Mito_27_debian10 - tox_env: py27-mode_mitogen-distro_debian10 - - name: Mito_27_debian11 - tox_env: py27-mode_mitogen-distro_debian11 - - name: Mito_27_ubuntu1604 - tox_env: py27-mode_mitogen-distro_ubuntu1604 - - name: Mito_27_ubuntu1804 - tox_env: py27-mode_mitogen-distro_ubuntu1804 - - name: Mito_27_ubuntu2004 - tox_env: py27-mode_mitogen-distro_ubuntu2004 - - - name: Mito_36_centos6 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos6 - - name: Mito_36_centos7 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos7 - - name: Mito_36_centos8 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos8 - - name: Mito_36_debian9 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian9 - - name: Mito_36_debian10 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian10 - - name: Mito_36_debian11 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian11 - - name: Mito_36_ubuntu1604 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu1604 - - name: Mito_36_ubuntu1804 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu1804 - - name: Mito_36_ubuntu2004 + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_36 python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu2004 - - - name: Mito_313_centos6 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos6 - - name: Mito_313_centos7 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos7 - - name: Mito_313_centos8 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos8 - - name: Mito_313_debian9 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian9 - - name: Mito_313_debian10 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian10 - - name: Mito_313_debian11 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian11 - - name: Mito_313_ubuntu1604 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu1604 - - name: Mito_313_ubuntu1804 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu1804 - - name: Mito_313_ubuntu2004 + tox_env: py36-mode_mitogen + - name: Mito_313 python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu2004 + tox_env: py313-mode_mitogen steps: - uses: actions/checkout@v4 diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a715b2b0..4c1df1bd 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -814,7 +814,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.context = dct['context'] self.chain = CallChain(self, self.context, pipelined=True) - if self._play_context.become: + if self.become: self.login_context = dct['via'] else: self.login_context = self.context @@ -926,7 +926,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.close() inventory_name, stack = self._build_stack() - if self._play_context.become: + if self.become: stack = stack[:-1] worker_model = ansible_mitogen.process.get_worker_model() diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 1b6512e8..3953eb52 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -294,7 +294,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): if not path.startswith('~'): # /home/foo -> /home/foo return path - if sudoable or not self._play_context.become: + if sudoable or not self._connection.become: if path == '~': # ~ -> /home/dmw return self._connection.homedir diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index a9f67209..708c2897 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -417,6 +417,10 @@ class PlayContextSpec(Spec): # used to run interpreter discovery self._action = connection._action + def _become_option(self, name): + plugin = self._connection.become + return plugin.get_option(name, self._task_vars, self._play_context) + def _connection_option(self, name): try: return self._connection.get_option(name, hostvars=self._task_vars) @@ -437,13 +441,13 @@ class PlayContextSpec(Spec): return self._connection_option('remote_user') def become(self): - return self._play_context.become + return self._connection.become def become_method(self): return self._play_context.become_method def become_user(self): - return self._play_context.become_user + return self._become_option('become_user') def become_pass(self): # become_pass is owned/provided by the active become plugin. However diff --git a/docs/changelog.rst b/docs/changelog.rst index ea45a735..88a2bd41 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,14 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +v0.3.14 (2024-10-16) +-------------------- + +* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker + SSH tests +* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become username. + + v0.3.13 (2024-10-09) -------------------- diff --git a/docs/contributors.rst b/docs/contributors.rst index 69dc1e76..ad35f91c 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -134,6 +134,7 @@ sponsorship and outstanding future-thinking of its early adopters.
  • luto
  • Mayeu a.k.a Matthieu Maury
  • Michael D'Silva
    • +
  • mordek
  • @nathanhruby
  • Orion Poplawski
  • Philippe Kueck
    • diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 48baa7c8..f710a550 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 13) +__version__ = (0, 3, 14) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/tests/README.md b/tests/README.md index 35a8775c..06bf7ad7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,11 +30,19 @@ and run the tests there. 1. Run ``test`` -# Selecting a target distribution +# Selecting target distributions -Docker target images exist for testing against CentOS and Debian, with the -default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in -the environment. +Linux container images for testing are available at + +- https://github.com/orgs/mitogen-hq/packages +- https://public.ecr.aws/n5z0e8q9 + +The images used are determined by two environment variables + +- `MITOGEN_TEST_DISTRO_SPECS` +- `MITOGEN_TEST_IMAGE_TEMPLATE` + +Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py` # User Accounts diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 609cd9f8..ef05803e 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -4,8 +4,7 @@ # When running the tests outside CI, make a single 'target' host which is the # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. -# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022 -# and os.environ['USER'] is not populated on Azure macOS runners. +# os.environ['USER'] is an empty string on GitHub Actions macOS runners. target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] @@ -26,6 +25,20 @@ tt-bare [tt_targets_bare:vars] ansible_host=localhost +[tt_become_bare] +tt-become-bare + +[tt_become_bare:vars] +ansible_host=localhost +ansible_user="{{ lookup('pipe', 'whoami') }}" + +[tt_become_by_inv] +tt-become-user ansible_become=true ansible_become_user="{{ 'root' | trim }}" + +[tt_become_by_inv:vars] +ansible_host=localhost +ansible_user="{{ lookup('pipe', 'whoami') }}" + [tt_targets_inventory] tt-password ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/integration/become/all.yml b/tests/ansible/integration/become/all.yml index c9c331dd..1b507e16 100644 --- a/tests/ansible/integration/become/all.yml +++ b/tests/ansible/integration/become/all.yml @@ -5,3 +5,7 @@ - import_playbook: sudo_nopassword.yml - import_playbook: sudo_password.yml - import_playbook: sudo_requiretty.yml +- import_playbook: templated_by_inv.yml +- import_playbook: templated_by_play_keywords.yml +- import_playbook: templated_by_play_vars.yml +- import_playbook: templated_by_task_keywords.yml diff --git a/tests/ansible/integration/become/templated_by_inv.yml b/tests/ansible/integration/become/templated_by_inv.yml new file mode 100644 index 00000000..98b68f05 --- /dev/null +++ b/tests/ansible/integration/become/templated_by_inv.yml @@ -0,0 +1,14 @@ +- name: integration/become/templated_by_inv.yml + hosts: tt_become_by_inv + gather_facts: false + tasks: + - meta: reset_connection + - name: Templated become in inventory + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_inv_whoami + failed_when: + - become_templated_by_inv_whoami is failed + or become_templated_by_inv_whoami.stdout != 'root' diff --git a/tests/ansible/integration/become/templated_by_play_keywords.yml b/tests/ansible/integration/become/templated_by_play_keywords.yml new file mode 100644 index 00000000..e588c18f --- /dev/null +++ b/tests/ansible/integration/become/templated_by_play_keywords.yml @@ -0,0 +1,16 @@ +- name: integration/become/templated_by_play_keywords.yml + hosts: tt_become_bare + gather_facts: false + become: true + become_user: "{{ 'root' | trim }}" + tasks: + - meta: reset_connection + - name: Templated become by play keywords + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_keywords_whoami + failed_when: + - become_templated_by_play_keywords_whoami is failed + or become_templated_by_play_keywords_whoami.stdout != 'root' diff --git a/tests/ansible/integration/become/templated_by_play_vars.yml b/tests/ansible/integration/become/templated_by_play_vars.yml new file mode 100644 index 00000000..5618f7cc --- /dev/null +++ b/tests/ansible/integration/become/templated_by_play_vars.yml @@ -0,0 +1,16 @@ +- name: integration/become/templated_by_play_vars.yml + hosts: tt_become_bare + gather_facts: false + vars: + ansible_become: true + ansible_become_user: "{{ 'root' | trim }}" + tasks: + - name: Templated become by play vars + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_vars_whoami + failed_when: + - become_templated_by_play_vars_whoami is failed + or become_templated_by_play_vars_whoami.stdout != 'root' diff --git a/tests/ansible/integration/become/templated_by_task_keywords.yml b/tests/ansible/integration/become/templated_by_task_keywords.yml new file mode 100644 index 00000000..52fda111 --- /dev/null +++ b/tests/ansible/integration/become/templated_by_task_keywords.yml @@ -0,0 +1,27 @@ +- name: integration/become/templated_by_task_keywords.yml + hosts: tt_become_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + become: true + become_user: "{{ 'root' | trim }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Templated become by task keywords, with delegate_to + become: true + become_user: "{{ 'root' | trim }}" + delegate_to: "{{ groups.tt_become_bare[0] }}" + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_task_with_delegate_to_whoami + failed_when: + - become_templated_by_task_with_delegate_to_whoami is failed + or become_templated_by_task_with_delegate_to_whoami.stdout != 'root' diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index ab400051..eada992a 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -3,5 +3,7 @@ - import_playbook: password.yml - import_playbook: timeouts.yml - import_playbook: templated_by_inv.yml +- import_playbook: templated_by_play_keyword.yml - import_playbook: templated_by_play_taskvar.yml +- import_playbook: templated_by_task_keyword.yml - import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/templated_by_play_keyword.yml b/tests/ansible/integration/ssh/templated_by_play_keyword.yml new file mode 100644 index 00000000..e66cc5f3 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_play_keyword.yml @@ -0,0 +1,11 @@ +- name: integration/ssh/templated_by_play_keyword.yml + hosts: tt_targets_bare + gather_facts: false + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - meta: reset_connection + - name: Templated variables in play keywords + ping: diff --git a/tests/ansible/integration/ssh/templated_by_task_keyword.yml b/tests/ansible/integration/ssh/templated_by_task_keyword.yml new file mode 100644 index 00000000..df956af5 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_task_keyword.yml @@ -0,0 +1,24 @@ +- name: integration/ssh/templated_by_task_keyword.yml + hosts: tt_targets_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Templated by task keywords, with delegate_to + delegate_to: "{{ groups.tt_targets_bare[0] }}" + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ping: diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index 0fdef20b..47f2ccd4 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -48,6 +48,26 @@ ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} +[tt_become_bare] +tt-become-bare + +[tt_become_bare:vars] +ansible_host={{ tt.hostname }} +ansible_password=has_sudo_nopw_password +ansible_port={{ tt.port }} +ansible_python_interpreter={{ tt.python_path }} +ansible_user=mitogen__has_sudo_nopw + +[tt_become_by_inv] +tt-become-user ansible_become=true ansible_become_user="{{ '{{' }} 'root' | trim {{ '}}' }}" + +[tt_become_by_inv:vars] +ansible_host={{ tt.hostname }} +ansible_password=has_sudo_nopw_password +ansible_port={{ tt.port }} +ansible_python_interpreter={{ tt.python_path }} +ansible_user=mitogen__has_sudo_nopw + [tt_targets_inventory] tt-password ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} ansible_user=mitogen__has_sudo_nopw tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/fakessh_test.py b/tests/fakessh_test.py index 52321495..2ad722df 100644 --- a/tests/fakessh_test.py +++ b/tests/fakessh_test.py @@ -7,8 +7,8 @@ import mitogen.fakessh import testlib +@unittest.skip('broken') class RsyncTest(testlib.DockerMixin, testlib.TestCase): - @unittest.skip('broken') def test_rsync_from_master(self): context = self.docker_ssh_any() @@ -24,7 +24,6 @@ class RsyncTest(testlib.DockerMixin, testlib.TestCase): self.assertTrue(context.call(os.path.exists, '/tmp/data')) self.assertTrue(context.call(os.path.exists, '/tmp/data/simple_pkg/a.py')) - @unittest.skip('broken') def test_rsync_between_direct_children(self): # master -> SSH -> mitogen__has_sudo_pubkey -> rsync(.ssh) -> master -> # mitogen__has_sudo -> rsync diff --git a/tests/ssh_test.py b/tests/ssh_test.py index ce7dce96..e8e0e1eb 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -37,7 +37,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(3, context.call(plain_old_module.add, 1, 2)) -class SshTest(testlib.DockerMixin, testlib.TestCase): +class SshMixin(testlib.DockerMixin): def test_debug_decoding(self): # ensure filter_debug_logs() decodes the logged string. capture = testlib.LogCapturer() @@ -68,7 +68,7 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): password='has_sudo_password', ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) self.assertEqual(name, context.name) @@ -176,7 +176,18 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): fp.close() -class BannerTest(testlib.DockerMixin, testlib.TestCase): +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SshTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SshMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass + + +class BannerMixin(testlib.DockerMixin): # Verify the ability to disambiguate random spam appearing in the SSHd's # login banner from a legitimate password prompt. def test_verbose_enabled(self): @@ -186,13 +197,24 @@ class BannerTest(testlib.DockerMixin, testlib.TestCase): ssh_debug_level=3, ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) self.assertEqual(name, context.name) context.shutdown(wait=True) +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'BannerTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (BannerMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass + + class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase): def test_classic_prompt(self): self.assertRaises(mitogen.ssh.PasswordError, diff --git a/tests/su_test.py b/tests/su_test.py index 234c509b..3750454c 100644 --- a/tests/su_test.py +++ b/tests/su_test.py @@ -23,7 +23,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(argv[2], '-c') -class SuTest(testlib.DockerMixin, testlib.TestCase): +class SuMixin(testlib.DockerMixin): stub_su_path = testlib.data_path('stubs/stub-su.py') def test_slow_auth_failure(self): @@ -64,3 +64,14 @@ class SuTest(testlib.DockerMixin, testlib.TestCase): ) context = self.router.su(via=ssh, password='rootpassword') self.assertEqual(0, context.call(os.getuid)) + + +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SuTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SuMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass diff --git a/tests/testlib.py b/tests/testlib.py index 76743e82..05779dc0 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,7 +51,10 @@ except NameError: LOG = logging.getLogger(__name__) -DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', @@ -555,8 +558,9 @@ class DockerizedSshDaemon(object): self.image, ] subprocess.check_output(args) + self.port = self.get_port(self.container_name) - def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE): # Code duplicated in ci_lib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) @@ -565,7 +569,10 @@ class DockerizedSshDaemon(object): ''', re.VERBOSE, ) - d = distro_pattern.match(distro).groupdict(default=None) + d = distro_pattern.match(distro_spec).groupdict(default=None) + + self.distro = d['distro'] + self.family = d['family'] if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' @@ -573,15 +580,10 @@ class DockerizedSshDaemon(object): self.python_path = '/usr/bin/python' self.image = image_template % d - self.start_container() - self.host = self.get_host() - self.port = self.get_port(self.container_name) - - def get_host(self): - return get_docker_host() + self.host = get_docker_host() def wait_for_sshd(self): - wait_for_port(self.get_host(), self.port, pattern='OpenSSH') + wait_for_port(self.host, self.port, pattern='OpenSSH') def check_processes(self): # Get Accounting name (ucomm) & command line (args) of each process @@ -651,12 +653,10 @@ class DockerMixin(RouterMixin): if os.environ.get('SKIP_DOCKER_TESTS'): raise unittest.SkipTest('SKIP_DOCKER_TESTS is set') - # we want to be able to override test distro for some tests that need a different container spun up - daemon_args = {} - if hasattr(cls, 'mitogen_test_distro'): - daemon_args['mitogen_test_distro'] = cls.mitogen_test_distro - - cls.dockerized_ssh = DockerizedSshDaemon(**daemon_args) + # cls.dockerized_ssh is injected by dynamically generating TestCase + # subclasses. + # TODO Bite the bullet, switch to e.g. pytest + cls.dockerized_ssh.start_container() cls.dockerized_ssh.wait_for_sshd() @classmethod diff --git a/tox.ini b/tox.ini index 6a0eb180..5afa7eb1 100644 --- a/tox.ini +++ b/tox.ini @@ -56,9 +56,7 @@ envlist = py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, py{313}-mode_ansible-ansible{6,7,8,9,10}, - py{27,36,313}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,313}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,313}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{27,36,313}-mode_mitogen, report, [testenv] @@ -105,39 +103,27 @@ setenv = NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 PIP_CONSTRAINT={toxinidir}/tests/constraints.txt - # Only applicable to MODE=mitogen - distro_centos5: DISTRO=centos5 - distro_centos6: DISTRO=centos6 - distro_centos7: DISTRO=centos7 - distro_centos8: DISTRO=centos8 - distro_debian9: DISTRO=debian9 - distro_debian10: DISTRO=debian10 - distro_debian11: DISTRO=debian11 - distro_ubuntu1604: DISTRO=ubuntu1604 - distro_ubuntu1804: DISTRO=ubuntu1804 - distro_ubuntu2004: DISTRO=ubuntu2004 - # Note the plural, only applicable to MODE=ansible # Ansible 6 - 8 (ansible-core 2.13 - 2.15) require Python 2.7 or >= 3.5 on targets - ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible6: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible7: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible8: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets - ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + ansible9: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets - ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3 - distros_centos: DISTROS=centos6 centos7 centos8 - distros_centos5: DISTROS=centos5 - distros_centos6: DISTROS=centos6 - distros_centos7: DISTROS=centos7 - distros_centos8: DISTROS=centos8 - distros_debian: DISTROS=debian9 debian10 debian11 - distros_debian9: DISTROS=debian9 - distros_debian10: DISTROS=debian10 - distros_debian11: DISTROS=debian11 - distros_ubuntu: DISTROS=ubuntu1604 ubuntu1804 ubuntu2004 - distros_ubuntu1604: DISTROS=ubuntu1604 - distros_ubuntu1804: DISTROS=ubuntu1804 - distros_ubuntu2004: DISTROS=ubuntu2004 + ansible10: MITOGEN_TEST_DISTRO_SPECS=debian10-py3 debian11-py3 ubuntu2004-py3 + distros_centos: MITOGEN_TEST_DISTRO_SPECS=centos6 centos7 centos8 + distros_centos5: MITOGEN_TEST_DISTRO_SPECS=centos5 + distros_centos6: MITOGEN_TEST_DISTRO_SPECS=centos6 + distros_centos7: MITOGEN_TEST_DISTRO_SPECS=centos7 + distros_centos8: MITOGEN_TEST_DISTRO_SPECS=centos8 + distros_debian: MITOGEN_TEST_DISTRO_SPECS=debian9 debian10 debian11 + distros_debian9: MITOGEN_TEST_DISTRO_SPECS=debian9 + distros_debian10: MITOGEN_TEST_DISTRO_SPECS=debian10 + distros_debian11: MITOGEN_TEST_DISTRO_SPECS=debian11 + distros_ubuntu: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 ubuntu1804 ubuntu2004 + distros_ubuntu1604: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 + distros_ubuntu1804: MITOGEN_TEST_DISTRO_SPECS=ubuntu1804 + distros_ubuntu2004: MITOGEN_TEST_DISTRO_SPECS=ubuntu2004 mode_ansible: MODE=ansible mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive mode_ansible: ANSIBLE_CALLBACK_WHITELIST=profile_tasks