diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 4a7bedae..4bf86bca 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -12,9 +12,8 @@ import jinja2 import ci_lib -TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates') -TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') -HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') +TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible') +TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts') def pause_if_interactive(): @@ -40,13 +39,13 @@ with ci_lib.Fold('docker_setup'): with ci_lib.Fold('job_setup'): - os.chdir(TESTS_DIR) - os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 8)) + os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) + os.chdir(ci_lib.ANSIBLE_TESTS_DIR) - ci_lib.run("mkdir %s", HOSTS_DIR) - for path in glob.glob(TESTS_DIR + '/hosts/*'): + os.mkdir(TMP_HOSTS_DIR) + for path in glob.glob(os.path.join(ci_lib.ANSIBLE_TESTS_HOSTS_DIR, '*')): if not path.endswith('default.hosts'): - ci_lib.run("ln -s %s %s", path, HOSTS_DIR) + os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path))) distros = collections.defaultdict(list) families = collections.defaultdict(list) @@ -55,12 +54,14 @@ with ci_lib.Fold('job_setup'): families[container['family']].append(container['name']) jinja_env = jinja2.Environment( - loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR), + loader=jinja2.FileSystemLoader( + searchpath=ci_lib.ANSIBLE_TESTS_TEMPLATES_DIR, + ), lstrip_blocks=True, # Remove spaces and tabs from before a block trim_blocks=True, # Remove first newline after a block ) inventory_template = jinja_env.get_template('test-targets.j2') - inventory_path = os.path.join(HOSTS_DIR, 'target') + inventory_path = os.path.join(TMP_HOSTS_DIR, 'test-targets.ini') with open(inventory_path, 'w') as fp: fp.write(inventory_template.render( @@ -71,16 +72,12 @@ with ci_lib.Fold('job_setup'): ci_lib.dump_file(inventory_path) - if not ci_lib.exists_in_path('sshpass'): - ci_lib.run("sudo apt-get update") - ci_lib.run("sudo apt-get install -y sshpass") - - with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') try: ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s', - playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) + playbook, TMP_HOSTS_DIR, ' '.join(sys.argv[1:]), + ) except: pause_if_interactive() raise diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index afb62e02..8193a34c 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -28,14 +28,20 @@ os.chdir( ) +GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +ANSIBLE_TESTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible') +ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts') +ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates') DISTRO_SPECS = os.environ.get( 'MITOGEN_TEST_DISTRO_SPECS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', ) +IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep') IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'public.ecr.aws/n5z0e8q9/%(distro)s-test', + 'ghcr.io/mitogen-hq/%(distro)s-test:2021', ) +TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key') _print = print @@ -48,6 +54,7 @@ def print(*args, **kwargs): def _have_cmd(args): + # Code duplicated in testlib.py try: subprocess.run( args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -56,19 +63,11 @@ def _have_cmd(args): if exc.errno == errno.ENOENT: return False raise - except subprocess.CallProcessError: + except subprocess.CalledProcessError: return False return True -def have_apt(): - return _have_cmd(['apt', '--help']) - - -def have_brew(): - return _have_cmd(['brew', 'help']) - - def have_docker(): return _have_cmd(['docker', 'info']) @@ -185,8 +184,8 @@ def exists_in_path(progname): class TempDir(object): - def __init__(self): - self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib') + def __init__(self, prefix='mitogen_ci_lib'): + self.path = tempfile.mkdtemp(prefix=prefix) atexit.register(self.destroy) def destroy(self, rmtree=shutil.rmtree): @@ -199,20 +198,6 @@ class Fold(object): def __exit__(self, _1, _2, _3): pass -GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -TMP = TempDir().path - - -# We copy this out of the way to avoid random stuff modifying perms in the Git -# tree (like git pull). -src_key_file = os.path.join(GIT_ROOT, - 'tests/data/docker/mitogen__has_sudo_pubkey.key') -key_file = os.path.join(TMP, - 'mitogen__has_sudo_pubkey.key') -shutil.copyfile(src_key_file, key_file) -os.chmod(key_file, int('0600', 8)) - - os.environ['PYTHONDONTWRITEBYTECODE'] = 'x' os.environ['PYTHONPATH'] = '%s:%s' % ( os.environ.get('PYTHONPATH', ''), @@ -243,7 +228,7 @@ def container_specs( [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', - 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'image': 'ghcr.io/mitogen-hq/debian11-test:2021', 'index': 1, 'name': 'target-debian11-1', 'port': 2201, @@ -251,7 +236,7 @@ def container_specs( {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', - 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'image': 'ghcr.io/mitogen-hq/centos6-test:2021', 'index': 2, 'name': 'target-centos6-2', 'port': 2202, diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index b065486f..6fd50124 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,7 +6,8 @@ import sys import ci_lib -project_dir = os.path.join(ci_lib.TMP, 'project') +TMP = ci_lib.TempDir(prefix='mitogen_ci_debops') +project_dir = os.path.join(TMP.path, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' docker_hostname = ci_lib.get_docker_hostname() @@ -22,6 +23,7 @@ with ci_lib.Fold('docker_setup'): with ci_lib.Fold('job_setup'): + os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) ci_lib.run('debops-init %s', project_dir) os.chdir(project_dir) @@ -45,7 +47,7 @@ with ci_lib.Fold('job_setup'): "\n" # Speed up slow DH generation. "dhparam__bits: ['128', '64']\n" - % (ci_lib.key_file,) + % (ci_lib.TESTS_SSH_PRIVATE_KEY_FILE,) ) with open(inventory_path, 'a') as fp: diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index e4b8329b..359dc195 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -10,19 +10,13 @@ import sys import ci_lib -TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') -IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep') -HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts') -KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key') - - with ci_lib.Fold('unit_tests'): os.environ['SKIP_MITOGEN'] = '1' ci_lib.run('./run_tests -v') with ci_lib.Fold('job_setup'): - os.chmod(KEY_PATH, int('0600', 8)) + os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", # there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually if not ci_lib.exists_in_path('sshpass'): @@ -51,11 +45,11 @@ with ci_lib.Fold('machine_prep'): subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True) subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True) - os.chdir(IMAGE_PREP_DIR) + os.chdir(ci_lib.IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml") if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': - os.chdir(IMAGE_PREP_DIR) + os.chdir(ci_lib.IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") cmd = ';'.join([ @@ -80,7 +74,7 @@ with ci_lib.Fold('machine_prep'): with ci_lib.Fold('ansible'): - os.chdir(TESTS_DIR) + os.chdir(ci_lib.ANSIBLE_TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') ci_lib.run('./run_ansible_playbook.py %s %s', playbook, ' '.join(sys.argv[1:])) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a275a00..2524c2e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,23 +2,23 @@ name: Tests +# env: +# ANSIBLE_VERBOSITY: 3 +# MITOGEN_LOG_LEVEL: DEBUG + on: pull_request: push: branches-ignore: - docs-master -env: - #ANSIBLE_VERBOSITY: 3 - #MITOGEN_LOG_LEVEL: DEBUG - MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" - # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners # https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support jobs: - linux: - # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md - runs-on: ubuntu-20.04 + u2204: + name: u2204 ${{ matrix.tox_env }} + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -36,6 +36,115 @@ jobs: python_version: '3.6' tox_env: py36-mode_ansible-ansible4 + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_36 + python_version: '3.6' + tox_env: py36-mode_mitogen + + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install build deps + run: | + set -o errexit -o nounset -o pipefail + + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + sudo apt-get update + + if [[ $PYTHON == "python2.7" ]]; then + sudo apt install -y python2-dev sshpass virtualenv + elif [[ $PYTHON == "python3.6" ]]; then + sudo apt install -y gcc-10 make libbz2-dev liblzma-dev libreadline-dev libsqlite3-dev libssl-dev sshpass virtualenv zlib1g-dev + curl --fail --silent --show-error --location https://pyenv.run | bash + CC=gcc-10 ~/.pyenv/bin/pyenv install --force 3.6 + else + echo 1>&2 "Python interpreter $PYTHON not available" + exit 1 + fi + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + elif [[ $PYTHON == "python3.6" ]]; then + PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6" + fi + + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python3.6" ]]; then + PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6" + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + u2404: + name: u2404 ${{ matrix.tox_env }} + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md + runs-on: ubuntu-24.04 + + strategy: + fail-fast: false + matrix: + include: - name: Ans_311_210 python_version: '3.11' tox_env: py311-mode_ansible-ansible2.10 @@ -71,11 +180,6 @@ jobs: python_version: '3.13' tox_env: py313-mode_ansible-ansible11-strategy_linear - - name: Mito_27 - tox_env: py27-mode_mitogen - - name: Mito_36 - python_version: '3.6' - tox_env: py36-mode_mitogen - name: Mito_313 python_version: '3.13' tox_env: py313-mode_mitogen @@ -96,7 +200,7 @@ jobs: set -o errexit -o nounset -o pipefail sudo apt-get update - sudo apt-get install -y python2-dev python3-pip virtualenv + sudo apt-get install -y sshpass virtualenv - name: Show Python versions run: | set -o errexit -o nounset -o pipefail @@ -130,18 +234,7 @@ jobs: exit 1 fi - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - "$PYTHON" -m ensurepip --user --altinstall --no-default-pip - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - elif [[ $PYTHON == "python2.7" ]]; then - curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" - "$PYTHON" get-pip.py --user --no-python-version-warning - # Avoid Python 2.x pip masking system pip - rm -f ~/.local/bin/{easy_install,pip,wheel} - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - else - "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - fi + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - name: Run tests env: GITHUB_ACTOR: ${{ github.actor }} @@ -160,6 +253,7 @@ jobs: "$PYTHON" -m tox -e "${{ matrix.tox_env }}" macos: + name: macos ${{ matrix.tox_env }} # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md runs-on: macos-13 timeout-minutes: 15 @@ -221,20 +315,7 @@ jobs: exit 1 fi - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - # GitHub macOS 12 images: python2.7 is installed, but not on $PATH - PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" - "$PYTHON" -m ensurepip --user --altinstall --no-default-pip - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - elif [[ $PYTHON == "python2.7" ]]; then - curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" - "$PYTHON" get-pip.py --user --no-python-version-warning - # Avoid Python 2.x pip masking system pip - rm -f ~/.local/bin/{easy_install,pip,wheel} - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - else - "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - fi + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - name: Run tests env: GITHUB_ACTOR: ${{ github.actor }} @@ -250,18 +331,14 @@ jobs: exit 1 fi - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - # GitHub macOS 12 images: python2.7 is installed, but not on $PATH - PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" - fi - "$PYTHON" -m tox -e "${{ matrix.tox_env }}" # https://github.com/marketplace/actions/alls-green check: if: always() needs: - - linux + - u2204 + - u2404 - macos runs-on: ubuntu-latest steps: diff --git a/docs/changelog.rst b/docs/changelog.rst index ba40784f..f9226f2f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,18 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +v0.3.23 (2025-04-28) +-------------------- + +* :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes +* :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6 +* :gh:issue:`1118` CI: Statically specify test usernames and group names +* :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir +* :gh:issue:`1118` CI: Don't share temporary directory between test groupings +* :gh:issue:`1256` CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04 +* :gh:issue:`1263` packaging: Fix InvalidVersion in release versions + + v0.3.22 (2025-02-04) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 61fce888..88994957 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 22) +__version__ = (0, 3, 23) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/parent.py b/mitogen/parent.py index f301a42c..5b397d9f 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -224,8 +224,16 @@ def flags(names): Return the result of ORing a set of (space separated) :py:mod:`termios` module constants together. """ - return sum(getattr(termios, name, 0) - for name in names.split()) + i = 0 + skipped = [] + for name in names.split(): + try: + i |= getattr(termios, name) + except AttributeError: + skipped.append(name) + if skipped: + LOG.debug('Skipped termios attributes: %s', ', '.join(skipped)) + return i def cfmakeraw(tflags): @@ -234,12 +242,9 @@ def cfmakeraw(tflags): modified in a manner similar to the `cfmakeraw()` C library function, but additionally disabling local echo. """ - # BSD: github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162 - # Linux: github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20 iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tflags iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK ' - 'ISTRIP INLCR ICRNL IXON IGNPAR') - iflag &= ~flags('IGNBRK BRKINT PARMRK') + 'ISTRIP INLCR ICRNL IXON IGNPAR IGNBRK') oflag &= ~flags('OPOST') lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG ' 'IEXTEN NOFLSH TOSTOP PENDIN') diff --git a/setup.py b/setup.py index 9cb30427..1639d383 100644 --- a/setup.py +++ b/setup.py @@ -26,20 +26,23 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import ast import os +import re from setuptools import find_packages, setup def grep_version(): path = os.path.join(os.path.dirname(__file__), 'mitogen/__init__.py') + version_pattern = re.compile( + r"__version__ = \((\d+), (\d+), (\d+)(?:, '(dev)')?\)", + ) with open(path) as fp: - for line in fp: - if line.startswith('__version__'): - _, _, s = line.partition('=') - parts = ast.literal_eval(s.strip()) - return '.'.join(str(part) for part in parts) + match = version_pattern.search(fp.read()) + if match is None: + raise ValueError('Could not find __version__ string in %s', path) + # E.g. '0.1.2', '0.1.3dev' + return '.'.join(str(part) for part in match.groups() if part) def long_description(): diff --git a/tests/ansible/bench/loop-100-copies.yml b/tests/ansible/bench/loop-100-copies.yml index 2a8e7d6f..4fe5276e 100644 --- a/tests/ansible/bench/loop-100-copies.yml +++ b/tests/ansible/bench/loop-100-copies.yml @@ -4,6 +4,8 @@ - name: Create file tree connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: > mkdir -p /tmp/filetree.in; for i in `seq -f /tmp/filetree.in/%g 1 100`; do echo $RANDOM > $i; done; diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index 44e660fc..fa70b89d 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -27,4 +27,3 @@ become_unpriv_available: >- -}} pkg_mgr_python_interpreter: python -pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml index c90dd5f4..5ab4577d 100644 --- a/tests/ansible/hosts/group_vars/centos8.yml +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -1,7 +1,7 @@ --- pkg_mgr_python_interpreter: /usr/libexec/platform-python -pkg_repos_overrides: +package_manager_repos: - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo content: | [appstream] diff --git a/tests/ansible/hosts/group_vars/debian11.yml b/tests/ansible/hosts/group_vars/debian11.yml new file mode 100644 index 00000000..9f62f43c --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian11.yml @@ -0,0 +1,5 @@ +package_manager_keys: + - src: debian-archive-bullseye-automatic.gpg # Debian 11 + dest: /etc/apt/trusted.gpg.d/debian-archive-bullseye-automatic.gpg + - src: debian-archive-bookworm-automatic.gpg # Debian 12 + dest: /etc/apt/trusted.gpg.d/debian-archive-bookworm-automatic.gpg diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml index 4b180b13..e08b1ed2 100644 --- a/tests/ansible/hosts/group_vars/debian9.yml +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -1,4 +1,4 @@ -pkg_repos_overrides: +package_manager_repos: - dest: /etc/apt/sources.list content: | deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/integration/connection/disconnect_during_module.yml b/tests/ansible/integration/connection/disconnect_during_module.yml index 3846d037..d72062e9 100644 --- a/tests/ansible/integration/connection/disconnect_during_module.yml +++ b/tests/ansible/integration/connection/disconnect_during_module.yml @@ -10,6 +10,8 @@ - name: Run _disconnect_during_module.yml delegate_to: localhost + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | diff --git a/tests/ansible/integration/context_service/reconnection.yml b/tests/ansible/integration/context_service/reconnection.yml index 6fb605a2..b0b4a735 100644 --- a/tests/ansible/integration/context_service/reconnection.yml +++ b/tests/ansible/integration/context_service/reconnection.yml @@ -19,6 +19,8 @@ bash -c "( sleep 3; kill -9 {{ssh_account_env.pid}}; ) & disown" - connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: sleep 3 - wait_for_connection: diff --git a/tests/ansible/integration/local/cwd_preserved.yml b/tests/ansible/integration/local/cwd_preserved.yml index 0b1ddef4..f64ebd79 100644 --- a/tests/ansible/integration/local/cwd_preserved.yml +++ b/tests/ansible/integration/local/cwd_preserved.yml @@ -8,11 +8,15 @@ hosts: test-targets tasks: - name: Get local cwd + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" connection: local command: pwd register: pwd - connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" stat: path: "{{pwd.stdout}}/cwd_preserved.yml" register: stat diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml index 61b4d03d..88415f68 100644 --- a/tests/ansible/integration/playbook_semantics/delegate_to.yml +++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml @@ -30,6 +30,8 @@ # connection:local, no sudo # - name: "connection:local, no sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" copy: dest: "{{ local_path }}" content: "Hello, world." @@ -43,6 +45,8 @@ fail_msg: "{{ lookup('file', local_path) }}" - name: "connection:local, no sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: "{{ local_path }}" state: absent @@ -84,6 +88,8 @@ # connection:local, sudo # - name: "connection:local, sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: | whoami > "{{ local_path }}" args: @@ -102,6 +108,8 @@ - requires_local_sudo - name: "connection:local, sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: "{{ local_path }}" state: absent diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index eb6720d3..21747494 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -7,7 +7,10 @@ result['sockets'] = glob.glob('/tmp/mitogen_unix*.sock') register: socks - - shell: > + - name: Run whoami locally in an ansible subprocess + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + shell: >- ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" @@ -15,6 +18,7 @@ {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} + -e ansible_python_interpreter="{{ ansible_playbook_python }}" test-targets args: chdir: ../.. diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index e074c524..8e732477 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -6,6 +6,8 @@ connection: local environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index ec5aed05..fc581a3b 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -15,6 +15,8 @@ environment: ANSIBLE_SSH_TIMEOUT: 10 ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 51783881..5cb5b489 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -6,6 +6,7 @@ hosts: test-targets[0] connection: local vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" # ControlMaster has the effect of caching the previous auth to the same # account, so disable it. Can't disable with ControlMaster no since that # already appears on command line, so override ControlPath with junk. diff --git a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml index 4220ed4c..7f8a3d2d 100644 --- a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml @@ -4,7 +4,10 @@ tasks: - connection: local environment: + ANSIBLE_PYTHON_INTERPRETER: "{{ ansible_playbook_python }}" ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} @@ -18,6 +21,8 @@ - connection: local environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index 4ee00e39..8fe061d1 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -12,6 +12,7 @@ - custom_python_detect_environment: vars: ansible_connection: kubectl + ansible_python_interpreter: "{{ ansible_playbook_python }}" mitogen_kubectl_path: stub-kubectl.py register: out diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index eba808fc..97d8e598 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -8,6 +8,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxc + ansible_python_interpreter: "{{ ansible_playbook_python }}" mitogen_lxc_attach_path: stub-lxc-attach.py register: out diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index c1825d73..71108df9 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -8,6 +8,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxd + ansible_python_interpreter: "{{ ansible_playbook_python }}" mitogen_lxc_path: stub-lxc.py register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index c1bf991a..52c0ee3d 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -9,6 +9,7 @@ vars: ansible_connection: mitogen_doas ansible_doas_exe: stub-doas.py + ansible_python_interpreter: "{{ ansible_playbook_python }}" ansible_user: someuser register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index 3634dd21..624cb538 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -8,6 +8,7 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_sudo + ansible_python_interpreter: "{{ ansible_playbook_python }}" ansible_user: root ansible_become_exe: stub-sudo.py ansible_become_flags: -H --type=sometype --role=somerole diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index 58744a52..709f385c 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -7,6 +7,8 @@ gather_facts: false any_errors_fatal: false connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" tasks: - include_tasks: ../_mitogen_only.yml - include_tasks: _end_play_if_not_sudo_linux.yml @@ -19,7 +21,7 @@ -i localhost, -c setns -e mitogen_kind=lxc - -e ansible_python_interpreter=python + -e ansible_python_interpreter="{{ ansible_playbook_python }}" -e mitogen_lxc_info_path={{git_basedir}}/tests/data/stubs/stub-lxc-info.py -m shell -a "echo hi" diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index dd9fbe63..627b29f9 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -7,6 +7,8 @@ gather_facts: false any_errors_fatal: false connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" tasks: - include_tasks: ../_mitogen_only.yml - include_tasks: _end_play_if_not_sudo_linux.yml @@ -19,7 +21,7 @@ -i localhost, -c setns -e mitogen_kind=lxd - -e ansible_python_interpreter=python + -e ansible_python_interpreter="{{ ansible_playbook_python }}" -e mitogen_lxc_path={{git_basedir}}/tests/data/stubs/stub-lxc.py -m shell -a "echo hi" diff --git a/tests/ansible/regression/issue_1087__template_streamerror.yml b/tests/ansible/regression/issue_1087__template_streamerror.yml index fa950ea4..ffb565ce 100644 --- a/tests/ansible/regression/issue_1087__template_streamerror.yml +++ b/tests/ansible/regression/issue_1087__template_streamerror.yml @@ -17,6 +17,8 @@ - name: Test template does not cause StreamError delegate_to: localhost run_once: true + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index 163440a5..78864545 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -9,6 +9,8 @@ - name: Create file tree connection: local run_once: true + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: > mkdir /tmp/filetree.in; seq -f /tmp/filetree.in/%g 1 1000 | xargs touch; @@ -37,6 +39,8 @@ - name: Cleanup local file tree connection: local run_once: true + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: /tmp/filetree.in state: absent diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index aafcfad4..a4d4b7f1 100644 --- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -8,6 +8,8 @@ connection: local tasks: - name: Create /tmp/issue_152_interpreter.sh + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" copy: dest: /tmp/issue_152_interpreter.sh mode: u+x @@ -28,6 +30,8 @@ out={{ out }} - name: Cleanup /tmp/issue_152_interpreter.sh + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: /tmp/issue_152_interpreter.sh state: absent diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index ef573276..d482c41c 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -10,7 +10,7 @@ vars: ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers - tasks: + pre_tasks: # The package management modules require using the same Python version # as the target's package manager libraries. This is sometimes in conflict # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not @@ -19,31 +19,13 @@ when: - ansible_version.full is version('2.17', '>=', strict=True) + roles: + - role: package_manager + + tasks: - name: Gather facts manually setup: - - name: Switch to archived package repositories - copy: - dest: "{{ item.dest }}" - content: "{{ item.content }}" - mode: u=rw,go=r - loop: "{{ pkg_repos_overrides }}" - loop_control: - label: "{{ item.dest }}" - - - name: Add signing keys - copy: - src: "{{ item.src }}" - dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}" - mode: u=rw,go=r - loop: - - src: debian-archive-bullseye-automatic.gpg # Debian 11 - - src: debian-archive-bookworm-automatic.gpg # Debian 12 - when: - # Ideally this would check for Debian 11, but distribution_major_version - # is unpopulated sometimes. - - ansible_facts.distribution == "Debian" - - name: Update package index apt: update_cache: true diff --git a/tests/ansible/regression/issue_952__ask_become_pass.yml b/tests/ansible/regression/issue_952__ask_become_pass.yml index a0b92ff2..17163a94 100644 --- a/tests/ansible/regression/issue_952__ask_become_pass.yml +++ b/tests/ansible/regression/issue_952__ask_become_pass.yml @@ -6,6 +6,8 @@ tasks: - name: Test --ask-become-pass delegate_to: localhost + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" expect: diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 2972adda..4aa3b46d 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -1,5 +1,5 @@ - -- hosts: all +- name: Bootstrap containers + hosts: all strategy: linear gather_facts: false tasks: @@ -15,7 +15,8 @@ fi when: bootstrap_packages | length -- hosts: all +- name: Setup containers + hosts: all strategy: mitogen_free # Resource limitation, my laptop freezes doing every container concurrently serial: 4 @@ -30,7 +31,9 @@ - ansible_facts.virtualization_type != "docker" roles: + - role: package_manager - role: sshd + - role: sshd_container tasks: - name: Ensure requisite apt packages are installed diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index ad5a4ef5..bee89084 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -3,48 +3,53 @@ # # WARNING: this creates non-privilged accounts with pre-set passwords! # - -- import_playbook: ../ansible/setup/report_controller.yml - -- hosts: all +- name: Mitogen test users and groups + hosts: all gather_facts: true strategy: mitogen_free become: true vars: distro: "{{ansible_distribution}}" special_users: - - has_sudo - - has_sudo_nopw - - has_sudo_pubkey - - pw_required - - readonly_homedir - - require_tty - - require_tty_pw_required - - permdenied - - slow_user - - webapp - - sudo1 - - sudo2 - - sudo3 - - sudo4 + - name: mitogen__has_sudo + - name: mitogen__has_sudo_nopw + - name: mitogen__has_sudo_pubkey + - name: mitogen__pw_required + - name: mitogen__readonly_homedir + - name: mitogen__require_tty + - name: mitogen__require_tty_pw_required + - name: mitogen__permdenied + - name: mitogen__slow_user + - name: mitogen__webapp + - name: mitogen__sudo1 + - name: mitogen__sudo2 + - name: mitogen__sudo3 + - name: mitogen__sudo4 user_groups: - has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] - has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] - has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] - sudo1: ['mitogen__group', 'mitogen__sudo_nopw'] - sudo2: ['mitogen__group', '{{sudo_group[distro]}}'] - sudo3: ['mitogen__group', '{{sudo_group[distro]}}'] - sudo4: ['mitogen__group', '{{sudo_group[distro]}}'] - - normal_users: "{{ - lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) - }}" + mitogen__has_sudo: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__has_sudo_pubkey: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] + mitogen__sudo1: ['mitogen__group', 'mitogen__sudo_nopw'] + mitogen__sudo2: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__sudo3: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__sudo4: ['mitogen__group', '{{ sudo_group[distro] }}'] + + normal_users: + - name: mitogen__user1 + - name: mitogen__user2 + - name: mitogen__user3 + - name: mitogen__user4 + - name: mitogen__user5 all_users: "{{ special_users + normal_users }}" + + mitogen_test_groups: + - name: mitogen__group + - name: mitogen__sudo_nopw tasks: - name: Disable non-localhost SSH for Mitogen users when: false @@ -56,43 +61,34 @@ - name: Create Mitogen test groups group: - name: "mitogen__{{item}}" - with_items: - - group - - sudo_nopw + name: "{{ item.name }}" + with_items: "{{ mitogen_test_groups }}" - name: Create user accounts + vars: + password: "{{ item.name | replace('mitogen__', '') }}_password" block: - user: - name: "mitogen__{{item}}" + name: "{{ item.name }}" shell: /bin/bash - groups: "{{user_groups[item]|default(['mitogen__group'])}}" - password: "{{ (item + '_password') | password_hash('sha256') }}" + groups: "{{ user_groups[item.name] | default(['mitogen__group']) }}" + password: "{{ password | password_hash('sha256') }}" with_items: "{{all_users}}" when: ansible_system != 'Darwin' - user: - name: "mitogen__{{item}}" + name: "{{ item.name }}" shell: /bin/bash group: staff groups: | {{ ['com.apple.access_ssh'] + - (user_groups[item] | default(['mitogen__group'])) + (user_groups[item.name] | default(['mitogen__group'])) }} - password: "{{item}}_password" + hidden: true + password: "{{ password }}" with_items: "{{all_users}}" when: ansible_system == 'Darwin' - - name: Hide users from login window (Darwin). - when: ansible_system == 'Darwin' - with_items: "{{all_users}}" - osx_defaults: - array_add: true - domain: /Library/Preferences/com.apple.loginwindow - type: array - key: HiddenUsersList - value: ['mitogen_{{item}}'] - - name: Check if AccountsService is used stat: path: /var/lib/AccountsService/users @@ -102,7 +98,7 @@ when: ansible_system == 'Linux' and out.stat.exists with_items: "{{all_users}}" copy: - dest: /var/lib/AccountsService/users/mitogen__{{item}} + dest: /var/lib/AccountsService/users/{{ item.name }} mode: u=rw,go= content: | [User] @@ -115,7 +111,11 @@ state: restarted - name: Readonly homedir for one account - shell: "chown -R root: ~mitogen__readonly_homedir" + file: + path: ~mitogen__readonly_homedir + owner: root + recurse: true + state: directory - name: Slow bash profile for one account copy: @@ -154,42 +154,29 @@ owner: mitogen__has_sudo_pubkey group: mitogen__group - - name: Require a TTY for two accounts - lineinfile: - path: /etc/sudoers - line: "{{item}}" - with_items: - - Defaults>mitogen__pw_required targetpw - - Defaults>mitogen__require_tty requiretty - - Defaults>mitogen__require_tty_pw_required requiretty,targetpw - - - name: Require password for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}:ALL) ALL" - validate: '/usr/sbin/visudo -cf %s' - with_items: - - mitogen__pw_required - - mitogen__require_tty_pw_required - when: - - ansible_virtualization_type != "docker" - - - name: Allow passwordless sudo for require_tty/readonly_homedir - lineinfile: + - name: Configure sudoers defaults + blockinfile: path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}:ALL) NOPASSWD:ALL" + marker: "# {mark} Mitogen test defaults" + block: | + Defaults>mitogen__pw_required targetpw + Defaults>mitogen__require_tty requiretty + Defaults>mitogen__require_tty_pw_required requiretty,targetpw validate: '/usr/sbin/visudo -cf %s' - with_items: - - mitogen__require_tty - - mitogen__readonly_homedir - when: - - ansible_virtualization_type != "docker" - - name: Allow passwordless for many accounts - lineinfile: + - name: Configure sudoers users + blockinfile: path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}:ALL) NOPASSWD:ALL" + marker: "# {mark} Mitogen test users" + block: | + # User Host(s) = (runas user:runas group) Command(s) + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__pw_required:ALL) ALL + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__require_tty_pw_required:ALL) ALL + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__require_tty:ALL) NOPASSWD:ALL + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__readonly_homedir:ALL) NOPASSWD:ALL + {% for runas_user in normal_users %} + {{ lookup('pipe', 'whoami') }} ALL = ({{ runas_user.name }}:ALL) NOPASSWD:ALL + {% endfor %} validate: '/usr/sbin/visudo -cf %s' - with_items: "{{normal_users}}" when: - ansible_virtualization_type != "docker" diff --git a/tests/image_prep/macos_localhost.yml b/tests/image_prep/macos_localhost.yml index c046a2bc..b93bc780 100644 --- a/tests/image_prep/macos_localhost.yml +++ b/tests/image_prep/macos_localhost.yml @@ -5,3 +5,4 @@ become: true roles: - role: sshd + - role: sshd_macos diff --git a/tests/image_prep/roles/package_manager/defaults/main.yml b/tests/image_prep/roles/package_manager/defaults/main.yml new file mode 100644 index 00000000..c384b19a --- /dev/null +++ b/tests/image_prep/roles/package_manager/defaults/main.yml @@ -0,0 +1,2 @@ +package_manager_keys: [] +package_manager_repos: [] diff --git a/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg b/tests/image_prep/roles/package_manager/files/debian-archive-bookworm-automatic.gpg similarity index 100% rename from tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg rename to tests/image_prep/roles/package_manager/files/debian-archive-bookworm-automatic.gpg diff --git a/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg b/tests/image_prep/roles/package_manager/files/debian-archive-bullseye-automatic.gpg similarity index 100% rename from tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg rename to tests/image_prep/roles/package_manager/files/debian-archive-bullseye-automatic.gpg diff --git a/tests/image_prep/roles/package_manager/tasks/main.yml b/tests/image_prep/roles/package_manager/tasks/main.yml new file mode 100644 index 00000000..f0bc8ad8 --- /dev/null +++ b/tests/image_prep/roles/package_manager/tasks/main.yml @@ -0,0 +1,13 @@ +- name: Add signing keys + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: u=rw,go=r + with_items: "{{ package_manager_keys }}" + +- name: Configure package repositories + copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: u=rw,go=r + with_items: "{{ package_manager_repos }}" diff --git a/tests/image_prep/roles/sshd/tasks/main.yml b/tests/image_prep/roles/sshd/tasks/main.yml index 837c7d15..8f3e7e03 100644 --- a/tests/image_prep/roles/sshd/tasks/main.yml +++ b/tests/image_prep/roles/sshd/tasks/main.yml @@ -9,23 +9,12 @@ path: "{{ sshd_config_file }}" line: "{{ item.line }}" regexp: "{{ item.regexp }}" - loop: + with_items: - line: Banner /etc/ssh/banner.txt regexp: '^#? *Banner.*' - line: MaxAuthTries {{ sshd_config__max_auth_tries }} regexp: '^#? *MaxAuthTries.*' - line: PermitRootLogin yes regexp: '.*PermitRootLogin.*' - loop_control: - label: "{{ item.line }}" - register: configure_sshd_result - -- name: Restart sshd - shell: | - launchctl unload /System/Library/LaunchDaemons/ssh.plist - wait 5 - launchctl load -w /System/Library/LaunchDaemons/ssh.plist - changed_when: true - when: - - ansible_facts.distribution == "MacOSX" - - configure_sshd_result is changed + notify: + - Restart sshd # Handler in platform specific role diff --git a/tests/image_prep/roles/sshd_container/handlers/main.yml b/tests/image_prep/roles/sshd_container/handlers/main.yml new file mode 100644 index 00000000..cc7b9166 --- /dev/null +++ b/tests/image_prep/roles/sshd_container/handlers/main.yml @@ -0,0 +1,2 @@ +- name: Restart sshd + meta: noop diff --git a/tests/image_prep/roles/sshd_macos/handlers/main.yml b/tests/image_prep/roles/sshd_macos/handlers/main.yml new file mode 100644 index 00000000..6d4a7e21 --- /dev/null +++ b/tests/image_prep/roles/sshd_macos/handlers/main.yml @@ -0,0 +1,6 @@ +- name: Restart sshd + shell: | + launchctl unload /System/Library/LaunchDaemons/ssh.plist + wait 5 + launchctl load -w /System/Library/LaunchDaemons/ssh.plist + changed_when: true diff --git a/tests/testlib.py b/tests/testlib.py index 05779dc0..a0e27d95 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -57,7 +57,7 @@ DISTRO_SPECS = os.environ.get( ) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'public.ecr.aws/n5z0e8q9/%(distro)s-test', + 'ghcr.io/mitogen-hq/%(distro)s-test:2021', ) TESTS_DIR = os.path.join(os.path.dirname(__file__)) @@ -155,6 +155,29 @@ def data_path(suffix): return path +def _have_cmd(args): + # Code duplicated in ci_lib.py + try: + subprocess.run( + args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + except OSError as exc: + if exc.errno == errno.ENOENT: + return False + raise + except subprocess.CalledProcessError: + return False + return True + + +def have_python2(): + return _have_cmd(['python2']) + + +def have_python3(): + return _have_cmd(['python3']) + + def retry(fn, on, max_attempts, delay): for i in range(max_attempts): try: diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index ab9f4e19..4a529dc7 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -1,4 +1,3 @@ -import os import unittest import mitogen.core @@ -11,8 +10,8 @@ import simple_pkg.ping # There should be 100 tests in this file. @unittest.skipIf( - os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21, - "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)", + not testlib.have_python2() or not testlib.have_python3(), + "Python 2/3 compatibility tests require both versions on the controller", ) class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: diff --git a/tox.ini b/tox.ini index bd2d65e9..ddb8c88d 100644 --- a/tox.ini +++ b/tox.ini @@ -100,6 +100,7 @@ passenv = HOME MITOGEN_* setenv = + ANSIBLE_ROLES_PATH = {toxinidir}/tests/image_prep/roles # See also azure-pipelines.yml ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1