From 5d6a185242ca93ab45e27bd296ad2dcca45b74f8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 13:16:42 +0100 Subject: [PATCH 1/8] tests: Templated "remote_user" provided as Ansible playbook keyword The password is provided as a variable because there is no corresponding keyword. I get the impression that keywords are considered a legacy mechanism, so most (new) options are only overridable by variables. The port is proved as a variable for now, to test remote_name in isolation. --- tests/ansible/integration/ssh/all.yml | 1 + .../integration/ssh/templated_by_play_keyword.yml | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/ansible/integration/ssh/templated_by_play_keyword.yml diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index ab400051..ce964c3e 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -3,5 +3,6 @@ - import_playbook: password.yml - import_playbook: timeouts.yml - import_playbook: templated_by_inv.yml +- import_playbook: templated_by_play_keyword.yml - import_playbook: templated_by_play_taskvar.yml - import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/templated_by_play_keyword.yml b/tests/ansible/integration/ssh/templated_by_play_keyword.yml new file mode 100644 index 00000000..e66cc5f3 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_play_keyword.yml @@ -0,0 +1,11 @@ +- name: integration/ssh/templated_by_play_keyword.yml + hosts: tt_targets_bare + gather_facts: false + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - meta: reset_connection + - name: Templated variables in play keywords + ping: From 5e816be12c8ecafd1c07fa323a2ac75b05c008cc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Oct 2024 14:16:17 +0100 Subject: [PATCH 2/8] tests: Templated connection keywords with delegated_to --- tests/ansible/integration/ssh/all.yml | 1 + .../ssh/templated_by_task_keyword.yml | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/ansible/integration/ssh/templated_by_task_keyword.yml diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index ce964c3e..eada992a 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -5,4 +5,5 @@ - import_playbook: templated_by_inv.yml - import_playbook: templated_by_play_keyword.yml - import_playbook: templated_by_play_taskvar.yml +- import_playbook: templated_by_task_keyword.yml - import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/templated_by_task_keyword.yml b/tests/ansible/integration/ssh/templated_by_task_keyword.yml new file mode 100644 index 00000000..df956af5 --- /dev/null +++ b/tests/ansible/integration/ssh/templated_by_task_keyword.yml @@ -0,0 +1,24 @@ +- name: integration/ssh/templated_by_task_keyword.yml + hosts: tt_targets_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Templated by task keywords, with delegate_to + delegate_to: "{{ groups.tt_targets_bare[0] }}" + remote_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" + vars: + ansible_password: has_sudo_nopw_password + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ping: From 9859e44ee841fe7655d8bc9a2b74580be32a4810 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 10 Oct 2024 15:23:49 +0100 Subject: [PATCH 3/8] tests: Standardise on DockerizedSshDaemon.host & .port --- tests/ssh_test.py | 4 ++-- tests/testlib.py | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/ssh_test.py b/tests/ssh_test.py index ce7dce96..f8fad55a 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -68,7 +68,7 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): password='has_sudo_password', ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) self.assertEqual(name, context.name) @@ -186,7 +186,7 @@ class BannerTest(testlib.DockerMixin, testlib.TestCase): ssh_debug_level=3, ) name = 'ssh.%s:%s' % ( - self.dockerized_ssh.get_host(), + self.dockerized_ssh.host, self.dockerized_ssh.port, ) self.assertEqual(name, context.name) diff --git a/tests/testlib.py b/tests/testlib.py index 76743e82..0fd9c7d7 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -574,14 +574,11 @@ class DockerizedSshDaemon(object): self.image = image_template % d self.start_container() - self.host = self.get_host() + self.host = get_docker_host() self.port = self.get_port(self.container_name) - def get_host(self): - return get_docker_host() - def wait_for_sshd(self): - wait_for_port(self.get_host(), self.port, pattern='OpenSSH') + wait_for_port(self.host, self.port, pattern='OpenSSH') def check_processes(self): # Get Accounting name (ucomm) & command line (args) of each process From 28e08ef94c5a57ee672511e06ed594d49284644f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 10 Oct 2024 18:33:44 +0100 Subject: [PATCH 4/8] ci: Reduce number of Jobs by parameterizing Mitogen Docker SSH tests This reduces the number of jobs from 48 to 24. The Mitogen part of the test suite has been parameterized on the Linux container targets to be run against. Both the Ansible tests & Mitogen tests now use the same source of truth to control which targets to use: environment variable MITOGEN_TEST_DISTRO_SPECS. This replaces the two mutually exclusive env vars DISTRO and DISTROS. I've also removed vestgial traces of an unused env var MITOGEN_TEST_DISTRO. Parameterization adapted from https://eli.thegreenplace.net/2014/04/02/dynamically-generating-python-test-cases refs #1058, #1059 --- .ci/README.md | 11 +++--- .ci/ansible_tests.py | 2 +- .ci/ci_lib.py | 8 ++-- .ci/mitogen_py24_tests.py | 2 - .ci/mitogen_tests.py | 1 - .github/workflows/tests.yml | 78 +++---------------------------------- docs/changelog.rst | 2 + tests/README.md | 16 ++++++-- tests/fakessh_test.py | 3 +- tests/ssh_test.py | 26 ++++++++++++- tests/su_test.py | 13 ++++++- tests/testlib.py | 25 ++++++------ tox.ini | 52 +++++++++---------------- 13 files changed, 101 insertions(+), 138 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 9a5e8898..17b7d2dd 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,14 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This - name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` -* `DISTROS`: the `ansible_` tests can run against multiple targets - simultaneously, which speeds things up. This is a space-separated list of - DISTRO names, but additionally, supports: +* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run + the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines + the Linux distribution, target Python interepreter & number of instances. + Only distributions with a pre-built Linux container image can be used. * `debian-py3`: when generating Ansible inventory file, set `ansible_python_interpreter` to `python3`, i.e. run a test where the target interpreter is Python 3. * `debian*16`: generate 16 Docker containers running Debian. Also works with -py3. +* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name, + and hence the container registry used for test targets. diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 3ec48dfd..62dfa8f5 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting) with ci_lib.Fold('docker_setup'): - containers = ci_lib.container_specs(ci_lib.DISTROS) + containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split()) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index dfe49b97..afb62e02 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -28,6 +28,10 @@ os.chdir( ) +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', @@ -196,10 +200,6 @@ class Fold(object): GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -# Used only when MODE=mitogen -DISTRO = os.environ.get('DISTRO', 'debian9') -# Used only when MODE=ansible -DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() TMP = TempDir().path diff --git a/.ci/mitogen_py24_tests.py b/.ci/mitogen_py24_tests.py index 228e79bd..96b144eb 100755 --- a/.ci/mitogen_py24_tests.py +++ b/.ci/mitogen_py24_tests.py @@ -8,8 +8,6 @@ import ci_lib os.environ.update({ 'NOCOVERAGE': '1', 'UNIT2': '/usr/local/python2.4.6/bin/unit2', - - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py index 4de94b4c..47aa2444 100755 --- a/.ci/mitogen_tests.py +++ b/.ci/mitogen_tests.py @@ -6,7 +6,6 @@ import os import ci_lib os.environ.update({ - 'MITOGEN_TEST_DISTRO': ci_lib.DISTRO, 'MITOGEN_LOG_LEVEL': 'debug', 'SKIP_ANSIBLE': '1', }) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f3f31d82..cc20f04a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,80 +67,14 @@ jobs: python_version: '3.13' tox_env: py313-mode_ansible-ansible10-strategy_linear - - name: Mito_27_centos6 - tox_env: py27-mode_mitogen-distro_centos6 - - name: Mito_27_centos7 - tox_env: py27-mode_mitogen-distro_centos7 - - name: Mito_27_centos8 - tox_env: py27-mode_mitogen-distro_centos8 - - name: Mito_27_debian9 - tox_env: py27-mode_mitogen-distro_debian9 - - name: Mito_27_debian10 - tox_env: py27-mode_mitogen-distro_debian10 - - name: Mito_27_debian11 - tox_env: py27-mode_mitogen-distro_debian11 - - name: Mito_27_ubuntu1604 - tox_env: py27-mode_mitogen-distro_ubuntu1604 - - name: Mito_27_ubuntu1804 - tox_env: py27-mode_mitogen-distro_ubuntu1804 - - name: Mito_27_ubuntu2004 - tox_env: py27-mode_mitogen-distro_ubuntu2004 - - - name: Mito_36_centos6 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos6 - - name: Mito_36_centos7 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos7 - - name: Mito_36_centos8 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_centos8 - - name: Mito_36_debian9 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian9 - - name: Mito_36_debian10 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian10 - - name: Mito_36_debian11 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_debian11 - - name: Mito_36_ubuntu1604 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu1604 - - name: Mito_36_ubuntu1804 - python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu1804 - - name: Mito_36_ubuntu2004 + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_36 python_version: '3.6' - tox_env: py36-mode_mitogen-distro_ubuntu2004 - - - name: Mito_313_centos6 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos6 - - name: Mito_313_centos7 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos7 - - name: Mito_313_centos8 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_centos8 - - name: Mito_313_debian9 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian9 - - name: Mito_313_debian10 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian10 - - name: Mito_313_debian11 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_debian11 - - name: Mito_313_ubuntu1604 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu1604 - - name: Mito_313_ubuntu1804 - python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu1804 - - name: Mito_313_ubuntu2004 + tox_env: py36-mode_mitogen + - name: Mito_313 python_version: '3.13' - tox_env: py313-mode_mitogen-distro_ubuntu2004 + tox_env: py313-mode_mitogen steps: - uses: actions/checkout@v4 diff --git a/docs/changelog.rst b/docs/changelog.rst index 092fed26..1fad75a5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ +* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker + SSH tests v0.3.13 (2024-10-09) diff --git a/tests/README.md b/tests/README.md index 35a8775c..06bf7ad7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,11 +30,19 @@ and run the tests there. 1. Run ``test`` -# Selecting a target distribution +# Selecting target distributions -Docker target images exist for testing against CentOS and Debian, with the -default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in -the environment. +Linux container images for testing are available at + +- https://github.com/orgs/mitogen-hq/packages +- https://public.ecr.aws/n5z0e8q9 + +The images used are determined by two environment variables + +- `MITOGEN_TEST_DISTRO_SPECS` +- `MITOGEN_TEST_IMAGE_TEMPLATE` + +Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py` # User Accounts diff --git a/tests/fakessh_test.py b/tests/fakessh_test.py index 52321495..2ad722df 100644 --- a/tests/fakessh_test.py +++ b/tests/fakessh_test.py @@ -7,8 +7,8 @@ import mitogen.fakessh import testlib +@unittest.skip('broken') class RsyncTest(testlib.DockerMixin, testlib.TestCase): - @unittest.skip('broken') def test_rsync_from_master(self): context = self.docker_ssh_any() @@ -24,7 +24,6 @@ class RsyncTest(testlib.DockerMixin, testlib.TestCase): self.assertTrue(context.call(os.path.exists, '/tmp/data')) self.assertTrue(context.call(os.path.exists, '/tmp/data/simple_pkg/a.py')) - @unittest.skip('broken') def test_rsync_between_direct_children(self): # master -> SSH -> mitogen__has_sudo_pubkey -> rsync(.ssh) -> master -> # mitogen__has_sudo -> rsync diff --git a/tests/ssh_test.py b/tests/ssh_test.py index f8fad55a..e8e0e1eb 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -37,7 +37,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(3, context.call(plain_old_module.add, 1, 2)) -class SshTest(testlib.DockerMixin, testlib.TestCase): +class SshMixin(testlib.DockerMixin): def test_debug_decoding(self): # ensure filter_debug_logs() decodes the logged string. capture = testlib.LogCapturer() @@ -176,7 +176,18 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): fp.close() -class BannerTest(testlib.DockerMixin, testlib.TestCase): +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SshTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SshMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass + + +class BannerMixin(testlib.DockerMixin): # Verify the ability to disambiguate random spam appearing in the SSHd's # login banner from a legitimate password prompt. def test_verbose_enabled(self): @@ -193,6 +204,17 @@ class BannerTest(testlib.DockerMixin, testlib.TestCase): context.shutdown(wait=True) +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'BannerTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (BannerMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass + + class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase): def test_classic_prompt(self): self.assertRaises(mitogen.ssh.PasswordError, diff --git a/tests/su_test.py b/tests/su_test.py index 234c509b..3750454c 100644 --- a/tests/su_test.py +++ b/tests/su_test.py @@ -23,7 +23,7 @@ class ConstructorTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(argv[2], '-c') -class SuTest(testlib.DockerMixin, testlib.TestCase): +class SuMixin(testlib.DockerMixin): stub_su_path = testlib.data_path('stubs/stub-su.py') def test_slow_auth_failure(self): @@ -64,3 +64,14 @@ class SuTest(testlib.DockerMixin, testlib.TestCase): ) context = self.router.su(via=ssh, password='rootpassword') self.assertEqual(0, context.call(os.getuid)) + + +for distro_spec in testlib.DISTRO_SPECS.split(): + dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec) + klass_name = 'SuTest%s' % (dockerized_ssh.distro.capitalize(),) + klass = type( + klass_name, + (SuMixin, testlib.TestCase), + {'dockerized_ssh': dockerized_ssh}, + ) + globals()[klass_name] = klass diff --git a/tests/testlib.py b/tests/testlib.py index 0fd9c7d7..05779dc0 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,7 +51,10 @@ except NameError: LOG = logging.getLogger(__name__) -DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +DISTRO_SPECS = os.environ.get( + 'MITOGEN_TEST_DISTRO_SPECS', + 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', +) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', @@ -555,8 +558,9 @@ class DockerizedSshDaemon(object): self.image, ] subprocess.check_output(args) + self.port = self.get_port(self.container_name) - def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE): # Code duplicated in ci_lib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) @@ -565,7 +569,10 @@ class DockerizedSshDaemon(object): ''', re.VERBOSE, ) - d = distro_pattern.match(distro).groupdict(default=None) + d = distro_pattern.match(distro_spec).groupdict(default=None) + + self.distro = d['distro'] + self.family = d['family'] if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' @@ -573,9 +580,7 @@ class DockerizedSshDaemon(object): self.python_path = '/usr/bin/python' self.image = image_template % d - self.start_container() self.host = get_docker_host() - self.port = self.get_port(self.container_name) def wait_for_sshd(self): wait_for_port(self.host, self.port, pattern='OpenSSH') @@ -648,12 +653,10 @@ class DockerMixin(RouterMixin): if os.environ.get('SKIP_DOCKER_TESTS'): raise unittest.SkipTest('SKIP_DOCKER_TESTS is set') - # we want to be able to override test distro for some tests that need a different container spun up - daemon_args = {} - if hasattr(cls, 'mitogen_test_distro'): - daemon_args['mitogen_test_distro'] = cls.mitogen_test_distro - - cls.dockerized_ssh = DockerizedSshDaemon(**daemon_args) + # cls.dockerized_ssh is injected by dynamically generating TestCase + # subclasses. + # TODO Bite the bullet, switch to e.g. pytest + cls.dockerized_ssh.start_container() cls.dockerized_ssh.wait_for_sshd() @classmethod diff --git a/tox.ini b/tox.ini index 6a0eb180..5afa7eb1 100644 --- a/tox.ini +++ b/tox.ini @@ -56,9 +56,7 @@ envlist = py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, py{313}-mode_ansible-ansible{6,7,8,9,10}, - py{27,36,313}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,313}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,313}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{27,36,313}-mode_mitogen, report, [testenv] @@ -105,39 +103,27 @@ setenv = NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 PIP_CONSTRAINT={toxinidir}/tests/constraints.txt - # Only applicable to MODE=mitogen - distro_centos5: DISTRO=centos5 - distro_centos6: DISTRO=centos6 - distro_centos7: DISTRO=centos7 - distro_centos8: DISTRO=centos8 - distro_debian9: DISTRO=debian9 - distro_debian10: DISTRO=debian10 - distro_debian11: DISTRO=debian11 - distro_ubuntu1604: DISTRO=ubuntu1604 - distro_ubuntu1804: DISTRO=ubuntu1804 - distro_ubuntu2004: DISTRO=ubuntu2004 - # Note the plural, only applicable to MODE=ansible # Ansible 6 - 8 (ansible-core 2.13 - 2.15) require Python 2.7 or >= 3.5 on targets - ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible6: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible7: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ansible8: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets - ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + ansible9: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets - ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3 - distros_centos: DISTROS=centos6 centos7 centos8 - distros_centos5: DISTROS=centos5 - distros_centos6: DISTROS=centos6 - distros_centos7: DISTROS=centos7 - distros_centos8: DISTROS=centos8 - distros_debian: DISTROS=debian9 debian10 debian11 - distros_debian9: DISTROS=debian9 - distros_debian10: DISTROS=debian10 - distros_debian11: DISTROS=debian11 - distros_ubuntu: DISTROS=ubuntu1604 ubuntu1804 ubuntu2004 - distros_ubuntu1604: DISTROS=ubuntu1604 - distros_ubuntu1804: DISTROS=ubuntu1804 - distros_ubuntu2004: DISTROS=ubuntu2004 + ansible10: MITOGEN_TEST_DISTRO_SPECS=debian10-py3 debian11-py3 ubuntu2004-py3 + distros_centos: MITOGEN_TEST_DISTRO_SPECS=centos6 centos7 centos8 + distros_centos5: MITOGEN_TEST_DISTRO_SPECS=centos5 + distros_centos6: MITOGEN_TEST_DISTRO_SPECS=centos6 + distros_centos7: MITOGEN_TEST_DISTRO_SPECS=centos7 + distros_centos8: MITOGEN_TEST_DISTRO_SPECS=centos8 + distros_debian: MITOGEN_TEST_DISTRO_SPECS=debian9 debian10 debian11 + distros_debian9: MITOGEN_TEST_DISTRO_SPECS=debian9 + distros_debian10: MITOGEN_TEST_DISTRO_SPECS=debian10 + distros_debian11: MITOGEN_TEST_DISTRO_SPECS=debian11 + distros_ubuntu: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 ubuntu1804 ubuntu2004 + distros_ubuntu1604: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 + distros_ubuntu1804: MITOGEN_TEST_DISTRO_SPECS=ubuntu1804 + distros_ubuntu2004: MITOGEN_TEST_DISTRO_SPECS=ubuntu2004 mode_ansible: MODE=ansible mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive mode_ansible: ANSIBLE_CALLBACK_WHITELIST=profile_tasks From e9bddf0c039bef8c52fd0dd40f8cbde5a111e5b4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 8 Oct 2024 17:43:30 +0100 Subject: [PATCH 5/8] CI: Use templated ansible_user for localhost Ansible tests refs #1022, #1116 --- .ci/localhost_ansible_tests.py | 14 -------------- tests/ansible/hosts/default.hosts | 3 +-- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index c36bad26..502a9abc 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -3,8 +3,6 @@ from __future__ import print_function -import getpass -import io import os import subprocess import sys @@ -57,18 +55,6 @@ with ci_lib.Fold('machine_prep'): os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") - # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022 - # and os.environ['USER'] is not populated on Azure macOS runners. - os.chdir(HOSTS_DIR) - with io.open('default.hosts', 'r+', encoding='utf-8') as f: - user = getpass.getuser() - content = f.read() - content = content.replace("{{ lookup('pipe', 'whoami') }}", user) - f.seek(0) - f.write(content) - f.truncate() - ci_lib.dump_file('default.hosts') - cmd = ';'.join([ 'from __future__ import print_function', 'import os, sys', diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 609cd9f8..bebe90df 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -4,8 +4,7 @@ # When running the tests outside CI, make a single 'target' host which is the # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. -# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022 -# and os.environ['USER'] is not populated on Azure macOS runners. +# os.environ['USER'] is an empty string on GitHub Actions macOS runners. target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] From bf6607e27e01d22c1a45b4f5c7ecbdc589eac732 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 11 Oct 2024 15:50:09 +0100 Subject: [PATCH 6/8] ansible_mitogen: Support templated become_user This reads the become username from the `become_user` attribute of the play context, to the `"become_user"` option of the loaded become plugin. This has been supported by vanilla Ansible since Ansible 2.10 (ansible-base 2.10). To support this I've also switched from using the `play_context.become` (a bool), to `connection.become` (an instance of the appropriate) become plugin. New tests have been added, modelled on those for templated connection parameters (see #1147, #1153, #1159). See - https://github.com/ansible/ansible/commit/480b106d6535978ae6ecab68b40942ca4fa914a0 refs #1083 Co-authored-by: mordek --- ansible_mitogen/connection.py | 4 +-- ansible_mitogen/mixins.py | 2 +- ansible_mitogen/transport_config.py | 8 ++++-- docs/changelog.rst | 1 + docs/contributors.rst | 1 + tests/ansible/hosts/default.hosts | 14 ++++++++++ tests/ansible/integration/become/all.yml | 4 +++ .../integration/become/templated_by_inv.yml | 14 ++++++++++ .../become/templated_by_play_keywords.yml | 16 +++++++++++ .../become/templated_by_play_vars.yml | 16 +++++++++++ .../become/templated_by_task_keywords.yml | 27 +++++++++++++++++++ tests/ansible/templates/test-targets.j2 | 20 ++++++++++++++ 12 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 tests/ansible/integration/become/templated_by_inv.yml create mode 100644 tests/ansible/integration/become/templated_by_play_keywords.yml create mode 100644 tests/ansible/integration/become/templated_by_play_vars.yml create mode 100644 tests/ansible/integration/become/templated_by_task_keywords.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a715b2b0..4c1df1bd 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -814,7 +814,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.context = dct['context'] self.chain = CallChain(self, self.context, pipelined=True) - if self._play_context.become: + if self.become: self.login_context = dct['via'] else: self.login_context = self.context @@ -926,7 +926,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.close() inventory_name, stack = self._build_stack() - if self._play_context.become: + if self.become: stack = stack[:-1] worker_model = ansible_mitogen.process.get_worker_model() diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 1b6512e8..3953eb52 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -294,7 +294,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): if not path.startswith('~'): # /home/foo -> /home/foo return path - if sudoable or not self._play_context.become: + if sudoable or not self._connection.become: if path == '~': # ~ -> /home/dmw return self._connection.homedir diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index a9f67209..708c2897 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -417,6 +417,10 @@ class PlayContextSpec(Spec): # used to run interpreter discovery self._action = connection._action + def _become_option(self, name): + plugin = self._connection.become + return plugin.get_option(name, self._task_vars, self._play_context) + def _connection_option(self, name): try: return self._connection.get_option(name, hostvars=self._task_vars) @@ -437,13 +441,13 @@ class PlayContextSpec(Spec): return self._connection_option('remote_user') def become(self): - return self._play_context.become + return self._connection.become def become_method(self): return self._play_context.become_method def become_user(self): - return self._play_context.become_user + return self._become_option('become_user') def become_pass(self): # become_pass is owned/provided by the active become plugin. However diff --git a/docs/changelog.rst b/docs/changelog.rst index 1fad75a5..8ab68d97 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ In progress (unreleased) * :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker SSH tests +* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become username. v0.3.13 (2024-10-09) diff --git a/docs/contributors.rst b/docs/contributors.rst index 69dc1e76..ad35f91c 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -134,6 +134,7 @@ sponsorship and outstanding future-thinking of its early adopters.
  • luto
  • Mayeu a.k.a Matthieu Maury
  • Michael D'Silva
    • +
  • mordek
  • @nathanhruby
  • Orion Poplawski
  • Philippe Kueck
    • diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index bebe90df..ef05803e 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -25,6 +25,20 @@ tt-bare [tt_targets_bare:vars] ansible_host=localhost +[tt_become_bare] +tt-become-bare + +[tt_become_bare:vars] +ansible_host=localhost +ansible_user="{{ lookup('pipe', 'whoami') }}" + +[tt_become_by_inv] +tt-become-user ansible_become=true ansible_become_user="{{ 'root' | trim }}" + +[tt_become_by_inv:vars] +ansible_host=localhost +ansible_user="{{ lookup('pipe', 'whoami') }}" + [tt_targets_inventory] tt-password ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/integration/become/all.yml b/tests/ansible/integration/become/all.yml index c9c331dd..1b507e16 100644 --- a/tests/ansible/integration/become/all.yml +++ b/tests/ansible/integration/become/all.yml @@ -5,3 +5,7 @@ - import_playbook: sudo_nopassword.yml - import_playbook: sudo_password.yml - import_playbook: sudo_requiretty.yml +- import_playbook: templated_by_inv.yml +- import_playbook: templated_by_play_keywords.yml +- import_playbook: templated_by_play_vars.yml +- import_playbook: templated_by_task_keywords.yml diff --git a/tests/ansible/integration/become/templated_by_inv.yml b/tests/ansible/integration/become/templated_by_inv.yml new file mode 100644 index 00000000..98b68f05 --- /dev/null +++ b/tests/ansible/integration/become/templated_by_inv.yml @@ -0,0 +1,14 @@ +- name: integration/become/templated_by_inv.yml + hosts: tt_become_by_inv + gather_facts: false + tasks: + - meta: reset_connection + - name: Templated become in inventory + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_inv_whoami + failed_when: + - become_templated_by_inv_whoami is failed + or become_templated_by_inv_whoami.stdout != 'root' diff --git a/tests/ansible/integration/become/templated_by_play_keywords.yml b/tests/ansible/integration/become/templated_by_play_keywords.yml new file mode 100644 index 00000000..e588c18f --- /dev/null +++ b/tests/ansible/integration/become/templated_by_play_keywords.yml @@ -0,0 +1,16 @@ +- name: integration/become/templated_by_play_keywords.yml + hosts: tt_become_bare + gather_facts: false + become: true + become_user: "{{ 'root' | trim }}" + tasks: + - meta: reset_connection + - name: Templated become by play keywords + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_keywords_whoami + failed_when: + - become_templated_by_play_keywords_whoami is failed + or become_templated_by_play_keywords_whoami.stdout != 'root' diff --git a/tests/ansible/integration/become/templated_by_play_vars.yml b/tests/ansible/integration/become/templated_by_play_vars.yml new file mode 100644 index 00000000..5618f7cc --- /dev/null +++ b/tests/ansible/integration/become/templated_by_play_vars.yml @@ -0,0 +1,16 @@ +- name: integration/become/templated_by_play_vars.yml + hosts: tt_become_bare + gather_facts: false + vars: + ansible_become: true + ansible_become_user: "{{ 'root' | trim }}" + tasks: + - name: Templated become by play vars + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_play_vars_whoami + failed_when: + - become_templated_by_play_vars_whoami is failed + or become_templated_by_play_vars_whoami.stdout != 'root' diff --git a/tests/ansible/integration/become/templated_by_task_keywords.yml b/tests/ansible/integration/become/templated_by_task_keywords.yml new file mode 100644 index 00000000..52fda111 --- /dev/null +++ b/tests/ansible/integration/become/templated_by_task_keywords.yml @@ -0,0 +1,27 @@ +- name: integration/become/templated_by_task_keywords.yml + hosts: tt_become_bare + gather_facts: false + # FIXME Resetting the connection shouldn't require credentials + # https://github.com/mitogen-hq/mitogen/issues/1132 + become: true + become_user: "{{ 'root' | trim }}" + tasks: + - name: Reset connection to target that will be delegate_to + meta: reset_connection + +- name: Test connection template by task keywords, with delegate_to + hosts: test-targets[0] + gather_facts: false + tasks: + - name: Templated become by task keywords, with delegate_to + become: true + become_user: "{{ 'root' | trim }}" + delegate_to: "{{ groups.tt_become_bare[0] }}" + command: + cmd: whoami + changed_when: false + check_mode: false + register: become_templated_by_task_with_delegate_to_whoami + failed_when: + - become_templated_by_task_with_delegate_to_whoami is failed + or become_templated_by_task_with_delegate_to_whoami.stdout != 'root' diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index 0fdef20b..47f2ccd4 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -48,6 +48,26 @@ ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} +[tt_become_bare] +tt-become-bare + +[tt_become_bare:vars] +ansible_host={{ tt.hostname }} +ansible_password=has_sudo_nopw_password +ansible_port={{ tt.port }} +ansible_python_interpreter={{ tt.python_path }} +ansible_user=mitogen__has_sudo_nopw + +[tt_become_by_inv] +tt-become-user ansible_become=true ansible_become_user="{{ '{{' }} 'root' | trim {{ '}}' }}" + +[tt_become_by_inv:vars] +ansible_host={{ tt.hostname }} +ansible_password=has_sudo_nopw_password +ansible_port={{ tt.port }} +ansible_python_interpreter={{ tt.python_path }} +ansible_user=mitogen__has_sudo_nopw + [tt_targets_inventory] tt-password ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} ansible_user=mitogen__has_sudo_nopw tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw From c4ca015266555594a39cccfff09e169f01612c0d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 16 Oct 2024 15:00:59 +0100 Subject: [PATCH 7/8] Prepare v0.3.14 --- docs/changelog.rst | 4 ++-- mitogen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ab68d97..cec8f331 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -In progress (unreleased) ------------------------- +v0.3.14 (2024-10-16) +-------------------- * :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker SSH tests diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 36c448d9..f710a550 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 14, 'dev') +__version__ = (0, 3, 14) #: This is :data:`False` in slave contexts. Previously it was used to prevent From d35ca3e4afaf7e93d85df2d22250b0e78ee2b1cc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 16 Oct 2024 15:02:30 +0100 Subject: [PATCH 8/8] Begin 0.3.15.dev --- docs/changelog.rst | 5 +++++ mitogen/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cec8f331..3a1d6ff2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +In progress (unreleased) +------------------------ + + + v0.3.14 (2024-10-16) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index f710a550..fd5590f2 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 14) +__version__ = (0, 3, 15, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent