Merge pull request #886 from moreati/prepare-0.3.1

Prepare 0.3.1

.ci/README.md

@@ -28,8 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `VER`: Ansible version the `_install` script should install. Default changes
-  over time.
 * `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
 * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
   name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`

.ci/ansible_install.py

@@ -3,25 +3,9 @@
 import ci_lib
 
 batches = [
-    [
-        # Must be installed separately, as PyNACL indirect requirement causes
-        # newer version to be installed if done in a single pip run.
-        # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml
-        'pip install "pycparser<2.19" "idna<2.7"',
-        'pip install '
-            '-r tests/requirements.txt '
-            '-r tests/ansible/requirements.txt',
-        # encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error
-        'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
-    ],
     [
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
     ]
 ]
 
-batches[-1].extend([
-    'docker pull %s' % (ci_lib.image_for_distro(distro),)
-    for distro in ci_lib.DISTROS
-])
-
 ci_lib.run_batches(batches)

.ci/ansible_tests.py

@@ -70,7 +70,7 @@ with ci_lib.Fold('job_setup'):
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     try:
-        run('./run_ansible_playbook.py %s -i "%s" -vvv %s',
+        run('./run_ansible_playbook.py %s -i "%s" %s',
             playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
     except:
         pause_if_interactive()

.ci/azure-pipelines-steps.yml

@@ -5,20 +5,18 @@ parameters:
   sign: false
 
 steps:
-- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
+- task: UsePythonVersion@0
-  displayName: "Run prep_azure.py"
+  displayName: Install python
+  inputs:
+    versionSpec: '$(python.version)'
+  condition: ne(variables['python.version'], '')
 
-- script: |
+- script: python -mpip install tox
-    echo "##vso[task.prependpath]/tmp/venv/bin"
+  displayName: Install tooling
 
-  displayName: activate venv
+- script: python -mtox -e "$(tox.env)"
-
+  displayName: "Run tests"
-- script: .ci/$(MODE)_install.py
-  displayName: "Run $(MODE)_install.py"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
     AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
     AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
-
-- script: .ci/$(MODE)_tests.py
-  displayName: "Run $(MODE)_tests.py"

.ci/azure-pipelines.yml

@@ -3,75 +3,192 @@
 # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
 # https://docs.microsoft.com/azure/devops/pipelines/languages/python
 
-jobs:
+# User defined variables are also injected as environment variables
+# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#environment-variables
+#variables:
+  #ANSIBLE_VERBOSITY: 3
 
-- job: Mac
+jobs:
+- job: Mac1015
   # vanilla Ansible is really slow
   timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md
     vmImage: macOS-10.15
   strategy:
     matrix:
-      Mito27_27:
+      Mito_27:
         python.version: '2.7'
-        MODE: mitogen
+        tox.env: py27-mode_mitogen
+      Mito_36:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen
+      Mito_39:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen
+
       # TODO: test python3, python3 tests are broken
-      Ans210_27:
+      Loc_27_210:
         python.version: '2.7'
-        MODE: localhost_ansible
+        tox.env: py27-mode_localhost-ansible2.10
-        VER: 2.10.0
+      Loc_27_3:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible3
+      Loc_27_4:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible4
 
       # NOTE: this hangs when ran in Ubuntu 18.04
-      Vanilla_210_27:
+      Van_27_210:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible2.10
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_3:
         python.version: '2.7'
-        MODE: localhost_ansible
+        tox.env: py27-mode_localhost-ansible3
-        VER: 2.10.0
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_4:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible4
         STRATEGY: linear
         ANSIBLE_SKIP_TAGS: resource_intensive
 
+- job: Mac11
+  # vanilla Ansible is really slow
+  timeoutInMinutes: 120
+  steps:
+  - template: azure-pipelines-steps.yml
+  pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/macos/
+    vmImage: macOS-11
+  strategy:
+    matrix:
+      Mito_27:
+        tox.env: py27-mode_mitogen
+      Mito_37:
+        python.version: '3.7'
+        tox.env: py37-mode_mitogen
+      Mito_39:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen
+
+      # TODO: test python3, python3 tests are broken
+      Loc_27_210:
+        tox.env: py27-mode_localhost-ansible2.10
+      Loc_27_3:
+        tox.env: py27-mode_localhost-ansible3
+      Loc_27_4:
+        tox.env: py27-mode_localhost-ansible4
+
+      # NOTE: this hangs when ran in Ubuntu 18.04
+      Van_27_210:
+        tox.env: py27-mode_localhost-ansible2.10
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_3:
+        tox.env: py27-mode_localhost-ansible3
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_4:
+        tox.env: py27-mode_localhost-ansible4
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
 
 - job: Linux
   pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
     vmImage: "Ubuntu 18.04"
   steps:
   - template: azure-pipelines-steps.yml
   strategy:
     matrix:
-      #
+      Mito_27_centos6:
-      # Confirmed working
-      #
-      Mito27Debian_27:
         python.version: '2.7'
-        MODE: mitogen
+        tox.env: py27-mode_mitogen-distro_centos6
-        DISTRO: debian9
+      Mito_27_centos7:
-
+        python.version: '2.7'
-      #MitoPy27CentOS6_26:
+        tox.env: py27-mode_mitogen-distro_centos7
-        #python.version: '2.7'
+      Mito_27_centos8:
-        #MODE: mitogen
+        python.version: '2.7'
-        #DISTRO: centos6
+        tox.env: py27-mode_mitogen-distro_centos8
+      Mito_27_debian9:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_debian9
+      Mito_27_debian10:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_debian10
+      Mito_27_debian11:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_debian11
+      Mito_27_ubuntu1604:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_ubuntu1604
+      Mito_27_ubuntu1804:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_ubuntu1804
+      Mito_27_ubuntu2004:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_ubuntu2004
 
-      Mito36CentOS6_26:
+      Mito_36_centos6:
         python.version: '3.6'
-        MODE: mitogen
+        tox.env: py36-mode_mitogen-distro_centos6
-        DISTRO: centos6
+      Mito_36_centos7:
-
+        python.version: '3.6'
-      Mito37Debian_27:
+        tox.env: py36-mode_mitogen-distro_centos7
-        python.version: '3.7'
+      Mito_36_centos8:
-        MODE: mitogen
+        python.version: '3.6'
-        DISTRO: debian9
+        tox.env: py36-mode_mitogen-distro_centos8
+      Mito_36_debian9:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_debian9
+      Mito_36_debian10:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_debian10
+      Mito_36_debian11:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_debian11
+      Mito_36_ubuntu1604:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_ubuntu1604
+      Mito_36_ubuntu1804:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_ubuntu1804
+      Mito_36_ubuntu2004:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_ubuntu2004
 
-      Mito39Debian_27:
+      Mito_39_centos6:
         python.version: '3.9'
-        MODE: mitogen
+        tox.env: py39-mode_mitogen-distro_centos6
-        DISTRO: debian9
+      Mito_39_centos7:
-        VER: 2.10.0
+        python.version: '3.9'
-
+        tox.env: py39-mode_mitogen-distro_centos7
-      #Py26CentOS7:
+      Mito_39_centos8:
-        #python.version: '2.7'
+        python.version: '3.9'
-        #MODE: mitogen
+        tox.env: py39-mode_mitogen-distro_centos8
-        #DISTRO: centos6
+      Mito_39_debian9:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_debian9
+      Mito_39_debian10:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_debian10
+      Mito_39_debian11:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_debian11
+      Mito_39_ubuntu1604:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_ubuntu1604
+      Mito_39_ubuntu1804:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_ubuntu1804
+      Mito_39_ubuntu2004:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_ubuntu2004
 
       #DebOps_2460_27_27:
         #python.version: '2.7'
@@ -110,17 +227,35 @@ jobs:
         #DISTROS: debian
         #STRATEGY: linear
 
-      Ansible_210_27:
+      Ans_27_210:
         python.version: '2.7'
-        MODE: ansible
+        tox.env: py27-mode_ansible-ansible2.10
-        VER: 2.10.0
+      Ans_27_3:
+        python.version: '2.7'
+        tox.env: py27-mode_ansible-ansible3
+      Ans_27_4:
+        python.version: '2.7'
+        tox.env: py27-mode_ansible-ansible4
 
-      Ansible_210_35:
+      Ans_36_210:
-        python.version: '3.5'
+        python.version: '3.6'
-        MODE: ansible
+        tox.env: py36-mode_ansible-ansible2.10
-        VER: 2.10.0
+      Ans_36_3:
+        python.version: '3.6'
+        tox.env: py36-mode_ansible-ansible3
+      Ans_36_4:
+        python.version: '3.6'
+        tox.env: py36-mode_ansible-ansible4
 
-      Ansible_210_39:
+      Ans_39_210:
+        python.version: '3.9'
+        tox.env: py39-mode_ansible-ansible2.10
+      Ans_39_3:
+        python.version: '3.9'
+        tox.env: py39-mode_ansible-ansible3
+      Ans_39_4:
+        python.version: '3.9'
+        tox.env: py39-mode_ansible-ansible4
+      Ans_39_5:
         python.version: '3.9'
-        MODE: ansible
+        tox.env: py39-mode_ansible-ansible5
-        VER: 2.10.0

.ci/ci_lib.py

@@ -22,6 +22,14 @@ os.chdir(
     )
 )
 
+_print = print
+def print(*args, **kwargs):
+    file = kwargs.get('file', sys.stdout)
+    flush = kwargs.pop('flush', False)
+    _print(*args, **kwargs)
+    if flush:
+        file.flush()
+
 
 #
 # check_output() monkeypatch cutpasted from testlib.py
@@ -59,12 +67,6 @@ def have_docker():
     return proc.wait() == 0
 
 
-# -----------------
-
-# Force line buffering on stdout.
-sys.stdout = os.fdopen(1, 'w', 1)
-
-
 def _argv(s, *args):
     """Interpolate a command line using *args, return an argv style list.
 
@@ -77,24 +79,22 @@ def _argv(s, *args):
 
 
 def run(s, *args, **kwargs):
-    """ Run a command, with arguments, and print timing information
+    """ Run a command, with arguments
 
     >>> rc = run('echo "%s %s"', 'foo', 'bar')
-    Running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    Running: ['echo', 'foo bar']
     foo bar
-    0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k
+    Finished running: ['echo', 'foo bar']
-    0inputs+0outputs (0major+71minor)pagefaults 0swaps
-    Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar']
     >>> rc
     0
     """
-    argv = ['/usr/bin/time', '--'] + _argv(s, *args)
+    argv = _argv(s, *args)
-    print('Running: %s' % (argv,))
+    print('Running: %s' % (argv,), flush=True)
     try:
         ret = subprocess.check_call(argv, **kwargs)
-        print('Finished running: %s' % (argv,))
+        print('Finished running: %s' % (argv,), flush=True)
     except Exception:
-        print('Exception occurred while running: %s' % (argv,))
+        print('Exception occurred while running: %s' % (argv,), file=sys.stderr, flush=True)
         raise
 
     return ret
@@ -161,13 +161,13 @@ def get_output(s, *args, **kwargs):
     'foo bar\n'
     """
     argv = _argv(s, *args)
-    print('Running: %s' % (argv,))
+    print('Running: %s' % (argv,), flush=True)
     return subprocess.check_output(argv, **kwargs)
 
 
 def exists_in_path(progname):
     """
-    Return True if proganme exists in $PATH.
+    Return True if progname exists in $PATH.
 
     >>> exists_in_path('echo')
     True
@@ -195,13 +195,11 @@ class Fold(object):
 
 os.environ.setdefault('ANSIBLE_STRATEGY',
     os.environ.get('STRATEGY', 'mitogen_linear'))
-# Ignoreed when MODE=mitogen
-ANSIBLE_VERSION = os.environ.get('VER', '2.6.2')
 GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 # Used only when MODE=mitogen
-DISTRO = os.environ.get('DISTRO', 'debian')
+DISTRO = os.environ.get('DISTRO', 'debian9')
 # Used only when MODE=ansible
-DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split()
+DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
 TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
 BASE_PORT = 2200
 TMP = TempDir().path
@@ -254,11 +252,13 @@ def make_containers(name_prefix='', port_offset=0):
     >>> pprint.pprint(make_containers())
     [{'distro': 'debian',
       'hostname': 'localhost',
+      'image': 'public.ecr.aws/n5z0e8q9/debian-test',
       'name': 'target-debian-1',
       'port': 2201,
       'python_path': '/usr/bin/python'},
      {'distro': 'centos6',
       'hostname': 'localhost',
+      'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
       'name': 'target-centos6-2',
       'port': 2202,
       'python_path': '/usr/bin/python'}]
@@ -280,6 +280,7 @@ def make_containers(name_prefix='', port_offset=0):
         for x in range(count):
             lst.append({
                 "distro": firstbit(distro),
+                "image": image_for_distro(distro),
                 "name": name_prefix + ("target-%s-%s" % (distro, i)),
                 "hostname": docker_hostname,
                 "port": BASE_PORT + i + port_offset,
@@ -358,7 +359,7 @@ def start_containers(containers):
                 "--publish 0.0.0.0:%(port)s:22/tcp "
                 "--hostname=%(name)s "
                 "--name=%(name)s "
-                "mitogen/%(distro)s-test "
+                "%(image)s"
             % container
         ]
         for container in containers
@@ -373,12 +374,10 @@ def start_containers(containers):
 def verify_procs(hostname, old, new):
     oldpids = set(pid for pid, _ in old)
     if any(pid not in oldpids for pid, _ in new):
-        print('%r had stray processes running:' % (hostname,))
+        print('%r had stray processes running:' % (hostname,), file=sys.stderr, flush=True)
         for pid, line in new:
             if pid not in oldpids:
-                print('New process:', line)
+                print('New process:', line, flush=True)
-
-        print()
         return False
 
     return True
@@ -402,13 +401,10 @@ def check_stray_processes(old, containers=None):
 
 
 def dump_file(path):
-    print()
+    print('--- %s ---' % (path,), flush=True)
-    print('--- %s ---' % (path,))
-    print()
     with open(path, 'r') as fp:
-        print(fp.read().rstrip())
+        print(fp.read().rstrip(), flush=True)
-    print('---')
+    print('---', flush=True)
-    print()
 
 
 # SSH passes these through to the container when run interactively, causing

.ci/debops_common_install.py

@@ -7,14 +7,10 @@ ci_lib.DISTROS = ['debian']
 
 ci_lib.run_batches([
     [
-        # Must be installed separately, as PyNACL indirect requirement causes
+        'pip install -qqq "debops[ansible]==2.1.2"',
-        # newer version to be installed if done in a single pip run.
-        'pip install "pycparser<2.19"',
-        'pip install -qqq "debops[ansible]==2.1.2" "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION),
     ],
     [
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-        'docker pull %s' % (ci_lib.image_for_distro('debian'),),
     ],
 ])
 

.ci/debops_common_tests.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
 import os
-import shutil
 import sys
 
 import ci_lib
@@ -60,11 +58,7 @@ with ci_lib.Fold('job_setup'):
             for container in containers
         )
 
-    print()
+    ci_lib.dump_file('ansible/inventory/hosts')
-    print(' echo --- ansible/inventory/hosts: ---')
-    ci_lib.run('cat ansible/inventory/hosts')
-    print('---')
-    print()
 
     # Now we have real host key checking, we need to turn it off
     os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'

.ci/localhost_ansible_install.py

@@ -3,17 +3,6 @@
 import ci_lib
 
 batches = [
-    [
-        # Must be installed separately, as PyNACL indirect requirement causes
-        # newer version to be installed if done in a single pip run.
-        # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml
-        # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-        'pip install "pycparser<2.19" "idna<2.7" virtualenv',
-        'pip install '
-            '-r tests/requirements.txt '
-            '-r tests/ansible/requirements.txt',
-        'pip install -q "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
-    ]
 ]
 
 ci_lib.run_batches(batches)

.ci/localhost_ansible_tests.py

@@ -33,24 +33,28 @@ with ci_lib.Fold('job_setup'):
 
 with ci_lib.Fold('machine_prep'):
     # generate a new ssh key for localhost ssh
-    os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
+    if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")):
-    os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+        os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
+        os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+
     # also generate it for the sudo user
-    os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
+    if os.system("sudo [ -f /var/root/.ssh/id_rsa ]") != 0:
-    os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+        os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
+        os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+
     os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
     os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
     # run chmod through sudo since it's owned by root
-    os.system('sudo chmod 600 /var/root/.ssh')
+    os.system('sudo chmod 700 /var/root/.ssh')
     os.system('sudo chmod 600 /var/root/.ssh/authorized_keys')
 
     if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
         os.chdir(IMAGE_PREP_DIR)
-        run("ansible-playbook -c local -i localhost, _user_accounts.yml -vvv")
+        run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
 
 with ci_lib.Fold('ansible'):
     os.chdir(TESTS_DIR)
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
-    run('./run_ansible_playbook.py %s -l target %s -vvv',
+    run('./run_ansible_playbook.py %s -l target %s',
         playbook, ' '.join(sys.argv[1:]))

.ci/mitogen_install.py

@@ -3,16 +3,11 @@
 import ci_lib
 
 batches = [
-    [
-        'pip install "pycparser<2.19" "idna<2.7"',
-        'pip install -r tests/requirements.txt',
-    ]
 ]
 
 if ci_lib.have_docker():
     batches.append([
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ])
 
 

.ci/mitogen_py24_install.py

@@ -5,7 +5,6 @@ import ci_lib
 batches = [
     [
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-        'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ],
     [
         'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',

.ci/prep_azure.py

@@ -1,95 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import sys
-
-import ci_lib
-
-batches = []
-
-if 0 and os.uname()[0] == 'Linux':
-    batches += [
-        [
-            "sudo chown `whoami`: ~",
-            "chmod u=rwx,g=rx,o= ~",
-
-            "sudo mkdir /var/run/sshd",
-            "sudo /etc/init.d/ssh start",
-
-            "mkdir -p ~/.ssh",
-            "chmod u=rwx,go= ~/.ssh",
-
-            "ssh-keyscan -H localhost >> ~/.ssh/known_hosts",
-            "chmod u=rw,go= ~/.ssh/known_hosts",
-
-            "cat tests/data/docker/mitogen__has_sudo_pubkey.key > ~/.ssh/id_rsa",
-            "chmod u=rw,go= ~/.ssh/id_rsa",
-
-            "cat tests/data/docker/mitogen__has_sudo_pubkey.key.pub > ~/.ssh/authorized_keys",
-            "chmod u=rw,go=r ~/.ssh/authorized_keys",
-        ]
-    ]
-
-# setup venv, need all python commands in 1 list to be subprocessed at the same time
-venv_steps = []
-
-need_to_fix_psycopg2 = False
-
-is_python3 = os.environ['PYTHONVERSION'].startswith('3')
-
-# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
-# broken symlinks, incorrect permissions and missing codecs. So we use the
-# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
-# stuff into. The virtualenv can probably be removed again, but this was a
-# hard-fought battle and for now I am tired of this crap.
-if ci_lib.have_apt():
-    venv_steps.extend([
-        'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
-        'sudo add-apt-repository ppa:deadsnakes/ppa',
-        'sudo apt-get update',
-        'sudo apt-get -y install '
-            'python{pv} '
-            'python{pv}-dev '
-            'libsasl2-dev '
-            'libldap2-dev '
-            .format(pv=os.environ['PYTHONVERSION']),
-        'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}'
-        .format(pv=os.environ['PYTHONVERSION'])
-    ])
-    if is_python3:
-        venv_steps.append('sudo apt-get -y install python{pv}-venv'.format(pv=os.environ['PYTHONVERSION']))
-# TODO: somehow `Mito36CentOS6_26` has both brew and apt installed https://dev.azure.com/dw-mitogen/Mitogen/_build/results?buildId=1031&view=logs&j=7bdbcdc6-3d3e-568d-ccf8-9ddca1a9623a&t=73d379b6-4eea-540f-c97e-046a2f620483
-elif is_python3 and ci_lib.have_brew():
-    # Mac's System Integrity Protection prevents symlinking /usr/bin
-    # and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
-    # so we'll use /usr/local/bin/python for everything
-    # /usr/local/bin/python2.7 already exists!
-    need_to_fix_psycopg2 = True
-    venv_steps.append(
-        'brew install python@{pv} postgresql'
-        .format(pv=os.environ['PYTHONVERSION'])
-    )
-
-# need wheel before building virtualenv because of bdist_wheel and setuptools deps
-venv_steps.append('/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools'.format(pv=os.environ['PYTHONVERSION']))
-
-if os.environ['PYTHONVERSION'].startswith('2'):
-    venv_steps.extend([
-        '/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(pv=os.environ['PYTHONVERSION']),
-        '/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(pv=os.environ['PYTHONVERSION'])
-    ])
-else:
-    venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(pv=os.environ['PYTHONVERSION']))
-# fixes https://stackoverflow.com/questions/59595649/can-not-install-psycopg2-on-macos-catalina https://github.com/Azure/azure-cli/issues/12854#issuecomment-619213863
-if need_to_fix_psycopg2:
-    venv_steps.append('/tmp/venv/bin/pip3 install psycopg2==2.8.5 psycopg2-binary')
-
-venv_steps.extend([
-    # pbr is a transitive setup_requires of hdrhistogram. If it's not already
-    # installed then setuptools attempts to use easy_install, which fails.
-    '/tmp/venv/bin/pip install pbr==5.6.0',
-])
-
-batches.append(venv_steps)
-
-ci_lib.run_batches(batches)

ansible_mitogen/loaders.py

@@ -31,7 +31,8 @@ Stable names for PluginLoader instances across Ansible versions.
 """
 
 from __future__ import absolute_import
-import distutils.version
+
+import ansible_mitogen.utils
 
 __all__ = [
     'action_loader',
@@ -42,10 +43,9 @@ __all__ = [
     'strategy_loader',
 ]
 
-import ansible
 
 ANSIBLE_VERSION_MIN = (2, 10)
-ANSIBLE_VERSION_MAX = (2, 10)
+ANSIBLE_VERSION_MAX = (2, 12)
 
 NEW_VERSION_MSG = (
     "Your Ansible version (%s) is too recent. The most recent version\n"
@@ -68,10 +68,7 @@ def assert_supported_release():
     Throw AnsibleError with a descriptive message in case of being loaded into
     an unsupported Ansible release.
     """
-    v = ansible.__version__
+    v = ansible_mitogen.utils.ansible_version
-    if not isinstance(v, tuple):
-        v = tuple(distutils.version.LooseVersion(v).version)
-
     if v[:2] < ANSIBLE_VERSION_MIN:
         raise ansible.errors.AnsibleError(
             OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN)
@@ -79,7 +76,7 @@ def assert_supported_release():
 
     if v[:2] > ANSIBLE_VERSION_MAX:
         raise ansible.errors.AnsibleError(
-            NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX)
+            NEW_VERSION_MSG % (v, ANSIBLE_VERSION_MAX)
         )
 
 

ansible_mitogen/mixins.py

@@ -53,6 +53,8 @@ import mitogen.utils
 import ansible_mitogen.connection
 import ansible_mitogen.planner
 import ansible_mitogen.target
+import ansible_mitogen.utils
+
 from ansible.module_utils._text import to_text
 
 try:
@@ -226,7 +228,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         with a pipelined call to :func:`ansible_mitogen.target.prune_tree`.
         """
         LOG.debug('_remove_tmp_path(%r)', tmp_path)
-        if tmp_path is None and ansible.__version__ > '2.6':
+        if tmp_path is None and ansible_mitogen.utils.ansible_version[:2] >= (2, 6):
             tmp_path = self._connection._shell.tmpdir  # 06f73ad578d
         if tmp_path is not None:
             self._connection.get_chain().call_no_reply(
@@ -335,7 +337,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
     def _set_temp_file_args(self, module_args, wrap_async):
         # Ansible>2.5 module_utils reuses the action's temporary directory if
         # one exists. Older versions error if this key is present.
-        if ansible.__version__ > '2.5':
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 5):
             if wrap_async:
                 # Sharing is not possible with async tasks, as in that case,
                 # the directory must outlive the action plug-in.
@@ -346,7 +348,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         # If _ansible_tmpdir is unset, Ansible>2.6 module_utils will use
         # _ansible_remote_tmp as the location to create the module's temporary
         # directory. Older versions error if this key is present.
-        if ansible.__version__ > '2.6':
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 6):
             module_args['_ansible_remote_tmp'] = (
                 self._connection.get_good_temp_dir()
             )
@@ -393,7 +395,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
             )
         )
 
-        if tmp and ansible.__version__ < '2.5' and delete_remote_tmp:
+        if tmp and delete_remote_tmp and ansible_mitogen.utils.ansible_version[:2] < (2, 5):
             # Built-in actions expected tmpdir to be cleaned up automatically
             # on _execute_module().
             self._remove_tmp_path(tmp)

ansible_mitogen/plugins/connection/mitogen_kubectl.py

@@ -45,17 +45,11 @@ import ansible_mitogen.connection
 import ansible_mitogen.loaders
 
 
-_class = ansible_mitogen.loaders.connection_loader__get(
+_get_result = ansible_mitogen.loaders.connection_loader__get(
     'kubectl',
     class_only=True,
 )
 
-if _class:
-    kubectl = sys.modules[_class.__module__]
-    del _class
-else:
-    kubectl = None
-
 
 class Connection(ansible_mitogen.connection.Connection):
     transport = 'kubectl'
@@ -66,13 +60,19 @@ class Connection(ansible_mitogen.connection.Connection):
     )
 
     def __init__(self, *args, **kwargs):
-        if kubectl is None:
+        if not _get_result:
             raise AnsibleConnectionFailure(self.not_supported_msg)
         super(Connection, self).__init__(*args, **kwargs)
 
     def get_extra_args(self):
+        try:
+            # Ansible < 2.10, _get_result is the connection class
+            connection_options = _get_result.connection_options
+        except AttributeError:
+            # Ansible >= 2.10, _get_result is a get_with_context_result
+            connection_options = _get_result.object.connection_options
         parameters = []
-        for key, option in iteritems(kubectl.CONNECTION_OPTIONS):
+        for key, option in iteritems(connection_options):
             if self.get_task_var('ansible_' + key) is not None:
                 parameters += [ option, self.get_task_var('ansible_' + key) ]
 

ansible_mitogen/target.py

@@ -144,7 +144,7 @@ def subprocess__Popen__close_fds(self, but):
 
 if (
     sys.platform.startswith(u'linux') and
-    sys.version < u'3.0' and
+    sys.version_info < (3,) and
     hasattr(subprocess.Popen, u'_close_fds') and
     not mitogen.is_master
 ):

ansible_mitogen/transport_config.py

@@ -451,7 +451,7 @@ class PlayContextSpec(Spec):
         return self._play_context.private_key_file
 
     def ssh_executable(self):
-        return self._play_context.ssh_executable
+        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
 
     def timeout(self):
         return self._play_context.timeout
@@ -467,9 +467,9 @@ class PlayContextSpec(Spec):
         return [
             mitogen.core.to_text(term)
             for s in (
-                getattr(self._play_context, 'ssh_args', ''),
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                getattr(self._play_context, 'ssh_common_args', ''),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                getattr(self._play_context, 'ssh_extra_args', '')
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
             )
             for term in ansible.utils.shlex.shlex_split(s or '')
         ]
@@ -679,10 +679,7 @@ class MitogenViaSpec(Spec):
         )
 
     def ssh_executable(self):
-        return (
+        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
-            self._host_vars.get('ansible_ssh_executable') or
-            C.ANSIBLE_SSH_EXECUTABLE
-        )
 
     def timeout(self):
         # TODO: must come from PlayContext too.
@@ -699,22 +696,9 @@ class MitogenViaSpec(Spec):
         return [
             mitogen.core.to_text(term)
             for s in (
-                (
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                    self._host_vars.get('ansible_ssh_args') or
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
-                    getattr(C, 'ANSIBLE_SSH_ARGS', None) or
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
-                    os.environ.get('ANSIBLE_SSH_ARGS')
-                    # TODO: ini entry. older versions.
-                ),
-                (
-                    self._host_vars.get('ansible_ssh_common_args') or
-                    os.environ.get('ANSIBLE_SSH_COMMON_ARGS')
-                    # TODO: ini entry.
-                ),
-                (
-                    self._host_vars.get('ansible_ssh_extra_args') or
-                    os.environ.get('ANSIBLE_SSH_EXTRA_ARGS')
-                    # TODO: ini entry.
-                ),
             )
             for term in ansible.utils.shlex.shlex_split(s)
             if s

ansible_mitogen/utils.py

@@ -0,0 +1,13 @@
+from __future__ import absolute_import
+
+import distutils.version
+
+import ansible
+
+__all__ = [
+    'ansible_version',
+]
+
+ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version)
+del distutils
+del ansible

docs/ansible_detailed.rst

@@ -145,9 +145,12 @@ Testimonials
 Noteworthy Differences
 ----------------------
 
-* Ansible 2.3-2.9 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify
+* Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6.
-  your installation is running one of these versions by checking ``ansible
+  Mitogen 0.3.1+ supports
-  --version`` output.
+    - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.9
+    - Ansible 5; with Python 3.8-3.9
+  Verify your installation is running one of these versions by checking
+  ``ansible --version`` output.
 
 * The ``raw`` action executes as a regular Mitogen connection, which requires
   Python on the target, precluding its use for installing Python. This will be

docs/api.rst

@@ -95,7 +95,7 @@ Connection Methods
 
     :param str container:
         The name of the Buildah container to connect to.
-    :param str doas_path:
+    :param str buildah_path:
         Filename or complete path to the ``buildah`` binary. ``PATH`` will be
         searched if given as a filename. Defaults to ``buildah``.
     :param str username:
@@ -367,6 +367,20 @@ Connection Methods
         Filename or complete path to the ``lxc`` binary. ``PATH`` will be
         searched if given as a filename. Defaults to ``lxc``.
 
+.. currentmodule:: mitogen.parent
+.. method:: Router.podman (container=None, podman_path=None, username=None, \**kwargs)
+
+    Construct a context on the local machine over a ``podman`` invocation.
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str container:
+        The name of the Podman container to connect to.
+    :param str podman_path:
+        Filename or complete path to the ``podman`` binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``podman``.
+    :param str username:
+        Username to use, defaults to unset.
+
 .. method:: Router.setns (container, kind, username=None, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs)
 
     Construct a context in the style of :meth:`local`, but change the

docs/changelog.rst

@@ -17,6 +17,22 @@ Release Notes
 To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/dw/mitogen/>`_.
 
+
+v0.3.1 (2021-01-12)
+-------------------
+
+* :gh:issue:`874` Support for Ansible 5 (ansible-core 2.12)
+* :gh:issue:`774` Fix bootstrap failures on macOS 11.x and 12.x, involving Python 2.7 wrapper
+* :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11)
+* :gh:issue:`869` Continuous Integration tests are now run with Tox
+* :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04
+* :gh:issue:`860` Add initial support for podman connection (w/o Ansible support yet)
+* :gh:issue:`873` `python -c ...` first stage no longer uses :py:mod:`platform`` to detect the macOS release
+* :gh:issue:`876` `python -c ...` first stage no longer contains tab characters, to reduce size
+* :gh:issue:`878` Continuous Integration tests now correctly perform comparisons of 2 digit versions
+* :gh:issue:`878` Kubectl connector fixed with Ansible 2.10 and above
+
+
 v0.3.0 (2021-10-28)
 -------------------
 

docs/howitworks.rst

@@ -813,7 +813,7 @@ executes under the runtime importer lock, ensuring :py:keyword:`import`
 statements executing in local threads are serialized.
 
 .. note::
-   
+
     In Python 2, :py:exc:`ImportError` is raised when :py:keyword:`import` is
     attempted while the runtime import lock is held by another thread,
     therefore imports must be serialized by only attempting them from the main

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 0)
+__version__ = (0, 3, 1)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/core.py

@@ -1254,6 +1254,7 @@ class Importer(object):
         'minify',
         'os_fork',
         'parent',
+        'podman',
         'select',
         'service',
         'setns',

mitogen/master.py

@@ -108,7 +108,7 @@ def _stdlib_paths():
     ]
     prefixes = (getattr(sys, a, None) for a in attr_candidates)
     version = 'python%s.%s' % sys.version_info[0:2]
-    s = set(os.path.abspath(os.path.join(p, 'lib', version))
+    s = set(os.path.realpath(os.path.join(p, 'lib', version))
             for p in prefixes if p is not None)
 
     # When running 'unit2 tests/module_finder_test.py' in a Py2 venv on Ubuntu

mitogen/parent.py

@@ -42,7 +42,6 @@ import heapq
 import inspect
 import logging
 import os
-import platform
 import re
 import signal
 import socket
@@ -1410,9 +1409,15 @@ class Connection(object):
     #     their respective values.
     #   * CONTEXT_NAME must be prefixed with the name of the Python binary in
     #     order to allow virtualenvs to detect their install prefix.
-    #   * For Darwin, OS X installs a craptacular argv0-introspecting Python
+    #   * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version
-    #     version switcher as /usr/bin/python. Override attempts to call it
+    #     switcher as /usr/bin/python, which introspects argv0. To workaround
-    #     with an explicit call to python2.7
+    #     it we redirect attempts to call /usr/bin/python with an explicit
+    #     call to /usr/bin/python2.7. macOS 10.15 (Darwin 19) removed it.
+    #   * macOS 11.x (Darwin 20, Big Sur) and macOS 12.x (Darwin 21, Montery)
+    #     do something slightly different. The Python executable is patched to
+    #     perform an extra execvp(). I don't fully understand the details, but
+    #     setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it.
+    #   * macOS 13.x (Darwin 22?) may remove python 2.x entirely.
     #
     # Locals:
     #   R: read side of interpreter stdin.
@@ -1435,11 +1440,8 @@ class Connection(object):
             os.close(r)
             os.close(W)
             os.close(w)
-            # this doesn't apply anymore to Mac OSX 10.15+ (Darwin 19+), new interpreter looks like this:
+            if os.uname()[0]=='Darwin'and os.uname()[2][:2]<'19'and sys.executable=='/usr/bin/python':sys.executable='/usr/bin/python2.7'
-            # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
+            if os.uname()[0]=='Darwin'and os.uname()[2][:2]in'2021'and sys.version[:3]=='2.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1'
-            if sys.platform == 'darwin' and sys.executable == '/usr/bin/python' and \
-                    int(platform.release()[:2]) < 19:
-                sys.executable += sys.version[:3]
             os.environ['ARGV0']=sys.executable
             os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
         os.write(1,'MITO000\n'.encode())
@@ -1469,7 +1471,7 @@ class Connection(object):
     def get_boot_command(self):
         source = inspect.getsource(self._first_stage)
         source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:]))
-        source = source.replace('    ', '\t')
+        source = source.replace('    ', ' ')
         source = source.replace('CONTEXT_NAME', self.options.remote_name)
         preamble_compressed = self.get_preamble()
         source = source.replace('PREAMBLE_COMPRESSED_LEN',
@@ -2505,6 +2507,9 @@ class Router(mitogen.core.Router):
     def ssh(self, **kwargs):
         return self.connect(u'ssh', **kwargs)
 
+    def podman(self, **kwargs):
+        return self.connect(u'podman', **kwargs)
+
 
 class Reaper(object):
     """

mitogen/podman.py

@@ -0,0 +1,74 @@
+# Copyright 2019, David Wilson
+# Copyright 2021, Mitogen contributors
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Options(mitogen.parent.Options):
+    container = None
+    username = None
+    podman_path = 'podman'
+
+    def __init__(self, container=None, podman_path=None, username=None,
+                 **kwargs):
+        super(Options, self).__init__(**kwargs)
+        assert container is not None
+        self.container = container
+        if podman_path:
+            self.podman_path = podman_path
+        if username:
+            self.username = username
+
+
+class Connection(mitogen.parent.Connection):
+    options_class = Options
+    child_is_immediate_subprocess = False
+
+    # TODO: better way of capturing errors such as "No such container."
+    create_child_args = {
+        'merge_stdio': True
+    }
+
+    def _get_name(self):
+        return u'podman.' + self.options.container
+
+    def get_boot_command(self):
+        args = [self.options.podman_path, 'exec']
+        if self.options.username:
+            args += ['--user=' + self.options.username]
+        args += ["--interactive", "--", self.options.container]
+        return args + super(Connection, self).get_boot_command()

mitogen/profiler.py

@@ -90,7 +90,7 @@ def merge_stats(outpath, inpaths):
                     break
                 time.sleep(0.2)
 
-    stats.dump_stats(outpath)
+    pstats.dump_stats(outpath)
 
 
 def generate_stats(outpath, tmpdir):

preamble_size.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 """
 Print the size of a typical SSH command line and the bootstrap code sent to new
 contexts.

setup.py

@@ -26,6 +26,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import ast
 import os
 
 from setuptools import find_packages, setup
@@ -37,7 +38,8 @@ def grep_version():
         for line in fp:
             if line.startswith('__version__'):
                 _, _, s = line.partition('=')
-                return '%i.%i.%i' % eval(s)
+                parts = ast.literal_eval(s.strip())
+                return '.'.join(str(part) for part in parts)
 
 
 def long_description():
@@ -62,6 +64,7 @@ setup(
     zip_safe = False,
     classifiers = [
         'Environment :: Console',
+        'Frameworks :: Ansible',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: BSD License',
         'Operating System :: MacOS :: MacOS X',

tests/ansible/all.yml

@@ -1,4 +1,6 @@
-- include: setup/all.yml
+- import_playbook: setup/all.yml
-- include: regression/all.yml
+  tags: setup
-- include: integration/all.yml
+- import_playbook: regression/all.yml
-
+  tags: regression
+- import_playbook: integration/all.yml
+  tags: integration

tests/ansible/ansible.cfg

@@ -5,7 +5,11 @@ strategy_plugins = ../../ansible_mitogen/plugins/strategy
 inventory_plugins = lib/inventory
 action_plugins = lib/action
 callback_plugins = lib/callback
-stdout_callback = nice_stdout
+stdout_callback = yaml
+stdout_whitelist =
+    profile_roles,
+    timer,
+    yaml
 vars_plugins = lib/vars
 library = lib/modules
 filter_plugins = lib/filters
@@ -31,6 +35,9 @@ timeout = 10
 # On Travis, paramiko check fails due to host key checking enabled.
 host_key_checking = False
 
+[callback_profile_tasks]
+task_output_limit = 10
+
 [ssh_connection]
 ssh_args = -o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s
 pipelining = True

tests/ansible/integration/action/all.yml

@@ -1,10 +1,10 @@
-- include: copy.yml
+- import_playbook: copy.yml
-- include: fixup_perms2__copy.yml
+- import_playbook: fixup_perms2__copy.yml
-- include: low_level_execute_command.yml
+- import_playbook: low_level_execute_command.yml
-- include: make_tmp_path.yml
+- import_playbook: make_tmp_path.yml
-- include: make_tmp_path__double.yml
+- import_playbook: make_tmp_path__double.yml
-- include: remote_expand_user.yml
+- import_playbook: remote_expand_user.yml
-- include: remote_file_exists.yml
+- import_playbook: remote_file_exists.yml
-- include: remove_tmp_path.yml
+- import_playbook: remove_tmp_path.yml
-- include: synchronize.yml
+- import_playbook: synchronize.yml
-- include: transfer_data.yml
+- import_playbook: transfer_data.yml

tests/ansible/integration/action/copy.yml

@@ -82,3 +82,5 @@
       - /tmp/copy-large-inline-file.out
 
     # end of cleaning out files (again)
+  tags:
+    - copy

tests/ansible/integration/action/fixup_perms2__copy.yml

@@ -114,3 +114,5 @@
       - /tmp/copy-with-mode.out
 
     # end of cleaning out files
+  tags:
+    - fixup_perms2__copy

tests/ansible/integration/action/low_level_execute_command.yml

@@ -41,3 +41,5 @@
             ["root"],
           )
         fail_msg: raw={{raw}}
+  tags:
+    - low_level_execute_command

tests/ansible/integration/action/make_tmp_path.yml

@@ -160,3 +160,5 @@
         - out.module_path.startswith(good_temp_path2)
         - out.module_tmpdir.startswith(good_temp_path2)
         fail_msg: out={{out}}
+  tags:
+    - make_tmp_path

tests/ansible/integration/action/make_tmp_path__double.yml

@@ -18,3 +18,5 @@
       script: |
         assert not self._remote_file_exists("{{ out.t1 }}")
         assert not self._remote_file_exists("{{ out.t2 }}")
+  tags:
+    - make_tmp_path_double

tests/ansible/integration/action/remote_expand_user.yml

@@ -110,3 +110,5 @@
     - assert:
         that: out.result == '$HOME/foo'
         fail_msg: out={{out}}
+  tags:
+    - remote_expand_user

tests/ansible/integration/action/remote_file_exists.yml

@@ -35,4 +35,5 @@
     - file:
         path: /tmp/does-exist
         state: absent
-
+  tags:
+    - remote_file_exists

tests/ansible/integration/action/remove_tmp_path.yml

@@ -37,3 +37,5 @@
     - file:
         path: /tmp/remove_tmp_path_test
         state: absent
+  tags:
+    - remove_tmp_path

tests/ansible/integration/action/synchronize.yml

@@ -72,3 +72,5 @@
     #   - /tmp/synchronize-action-key
     #   - /tmp/sync-test
     #   - /tmp/sync-test.out
+  tags:
+    - synchronize

tests/ansible/integration/action/transfer_data.yml

@@ -46,3 +46,5 @@
     - file:
         path: /tmp/transfer-data
         state: absent
+  tags:
+    - transfer_data

tests/ansible/integration/all.yml

@@ -3,21 +3,39 @@
 # This playbook imports all tests that are known to work at present.
 #
 
-- include: action/all.yml
+- import_playbook: action/all.yml
-- include: async/all.yml
+  tags: action
-- include: become/all.yml
+- import_playbook: async/all.yml
-- include: connection/all.yml
+  tags: async
-- include: connection_delegation/all.yml
+- import_playbook: become/all.yml
-- include: connection_loader/all.yml
+  tags: become
-- include: context_service/all.yml
+- import_playbook: connection/all.yml
-- include: glibc_caches/all.yml
+  tags: connection
-- include: interpreter_discovery/all.yml
+- import_playbook: connection_delegation/all.yml
-- include: local/all.yml
+  tags: connection_delegation
-- include: module_utils/all.yml
+- import_playbook: connection_loader/all.yml
-- include: playbook_semantics/all.yml
+  tags: connection_loader
-- include: process/all.yml
+- import_playbook: context_service/all.yml
-- include: runner/all.yml
+  tags: context_service
-- include: ssh/all.yml
+- import_playbook: glibc_caches/all.yml
-- include: strategy/all.yml
+  tags: glibc_caches
-- include: stub_connections/all.yml
+- import_playbook: interpreter_discovery/all.yml
-- include: transport_config/all.yml
+  tags: interpreter_discovery
+- import_playbook: local/all.yml
+  tags: local
+- import_playbook: module_utils/all.yml
+  tags: module_utils
+- import_playbook: playbook_semantics/all.yml
+  tags: playbook_semantics
+- import_playbook: process/all.yml
+  tags: process
+- import_playbook: runner/all.yml
+  tags: runner
+- import_playbook: ssh/all.yml
+  tags: ssh
+- import_playbook: strategy/all.yml
+  tags: strategy
+- import_playbook: stub_connections/all.yml
+  tags: stub_connections
+- import_playbook: transport_config/all.yml
+  tags: transport_config

tests/ansible/integration/async/all.yml

@@ -1,9 +1,9 @@
-- include: multiple_items_loop.yml
+- import_playbook: multiple_items_loop.yml
-- include: result_binary_producing_json.yml
+- import_playbook: result_binary_producing_json.yml
-- include: result_binary_producing_junk.yml
+- import_playbook: result_binary_producing_junk.yml
-- include: result_shell_echo_hi.yml
+- import_playbook: result_shell_echo_hi.yml
-- include: runner_new_process.yml
+- import_playbook: runner_new_process.yml
-- include: runner_one_job.yml
+- import_playbook: runner_one_job.yml
-- include: runner_timeout_then_polling.yml
+- import_playbook: runner_timeout_then_polling.yml
-- include: runner_two_simultaneous_jobs.yml
+- import_playbook: runner_two_simultaneous_jobs.yml
-- include: runner_with_polling_and_timeout.yml
+- import_playbook: runner_with_polling_and_timeout.yml

tests/ansible/integration/async/multiple_items_loop.yml

@@ -35,3 +35,5 @@
       - out.results[1].rc == 0
       - out.results[1].delta > '0:00:05'
       fail_msg: out={{out}}
+  tags:
+    - multiple_items_loop

tests/ansible/integration/async/result_binary_producing_json.yml

@@ -55,3 +55,5 @@
       fail_msg: async_out={{async_out}}
     vars:
       async_out: "{{result.content|b64decode|from_json}}"
+  tags:
+    - result_binary_producing_json

tests/ansible/integration/async/result_binary_producing_junk.yml

@@ -42,3 +42,5 @@
       fail_msg: async_out={{async_out}}
     vars:
       async_out: "{{result.content|b64decode|from_json}}"
+  tags:
+    - result_binary_producing_junk

tests/ansible/integration/async/result_shell_echo_hi.yml

@@ -30,7 +30,10 @@
         - async_out.invocation.module_args.creates == None
         - async_out.invocation.module_args.executable == None
         - async_out.invocation.module_args.removes == None
-        - async_out.invocation.module_args.warn == True
+        # In Ansible 4 (ansible-core 2.11) the warn parameter is deprecated and defaults to false.
+        # It's scheduled for removal in ansible-core 2.13.
+        - (ansible_version.full is version("2.11", "<", strict=True) and async_out.invocation.module_args.warn == True)
+          or (ansible_version.full is version("2.11", ">=", strict=True) and async_out.invocation.module_args.warn == False)
         - async_out.rc == 0
         - async_out.start.startswith("20")
         - async_out.stderr == "there"
@@ -43,6 +46,9 @@
       that:
         - async_out.invocation.module_args.stdin == None
       fail_msg: async_out={{async_out}}
-    when: ansible_version.full > '2.4'
+    when:
+      - ansible_version.full is version('2.4', '>=', strict=True)
     vars:
       async_out: "{{result.content|b64decode|from_json}}"
+  tags:
+    - result_shell_echo_hi

tests/ansible/integration/async/runner_new_process.yml

@@ -55,3 +55,5 @@
       - async_result1.pid != async_result2.pid
       fail_msg: async_result1={{async_result1}} async_result2={{async_result2}}
     when: is_mitogen
+  tags:
+    - runner_new_process

tests/ansible/integration/async/runner_one_job.yml

@@ -41,9 +41,9 @@
       - result1.changed == True
       # ansible/b72e989e1837ccad8dcdc926c43ccbc4d8cdfe44
       - |
-        (ansible_version.full is version('2.8', ">=") and
+        (ansible_version.full is version('2.8', ">=", strict=True) and
          result1.cmd == "echo alldone;\nsleep 1;\n") or
-        (ansible_version.full is version('2.8', '<') and
+        (ansible_version.full is version('2.8', '<', strict=True) and
          result1.cmd == "echo alldone;\n sleep 1;")
       - result1.delta|length == 14
       - result1.start|length == 26
@@ -58,10 +58,14 @@
       - result1.stdout == "alldone"
       - result1.stdout_lines == ["alldone"]
       fail_msg: result1={{result1}}
-    when: ansible_version.full is version('2.8', '>')  # ansible#51393
+    when:
+      - ansible_version.full is version('2.8', '>', strict=True)  # ansible#51393
 
   - assert:
       that:
       - result1.failed == False
       fail_msg: result1={{result1}}
-    when: ansible_version.full is version('2.4', '>')
+    when:
+      - ansible_version.full is version('2.4', '>', strict=True)
+  tags:
+    - runner_one_job

tests/ansible/integration/async/runner_timeout_then_polling.yml

@@ -33,3 +33,5 @@
         - result.msg == "Job reached maximum time limit of 1 seconds."
       fail_msg: result={{result}}
     when: is_mitogen
+  tags:
+    - runner_timeout_then_polling

tests/ansible/integration/async/runner_two_simultaneous_jobs.yml

@@ -62,4 +62,7 @@
       that:
         - result2.stdout == 'im_alive'
       fail_msg: result2={{result2}}
-    when: ansible_version.full > '2.8'  # ansible#51393
+    when:
+      - ansible_version.full is version('2.8', '>=', strict=True)  # ansible#51393
+  tags:
+    - runner_two_simultaneous_jobs

tests/ansible/integration/async/runner_with_polling_and_timeout.yml

@@ -23,4 +23,5 @@
           job1.msg == "async task did not complete within the requested time - 1s" or
           job1.msg == "Job reached maximum time limit of 1 seconds."
       fail_msg: job1={{job1}}
-
+  tags:
+    - runner_with_polling_and_timeout

tests/ansible/integration/become/all.yml

@@ -1,7 +1,7 @@
 
-- include: su_password.yml
+- import_playbook: su_password.yml
-- include: sudo_flags_failure.yml
+- import_playbook: sudo_flags_failure.yml
-- include: sudo_nonexistent.yml
+- import_playbook: sudo_nonexistent.yml
-- include: sudo_nopassword.yml
+- import_playbook: sudo_nopassword.yml
-- include: sudo_password.yml
+- import_playbook: sudo_password.yml
-- include: sudo_requiretty.yml
+- import_playbook: sudo_requiretty.yml

tests/ansible/integration/become/su_password.yml

@@ -59,3 +59,6 @@
           - out.stdout == 'mitogen__user1'
         fail_msg: out={{out}}
       when: is_mitogen
+  tags:
+    - su
+    - su_password

tests/ansible/integration/become/sudo_flags_failure.yml

@@ -15,10 +15,13 @@
       assert:
         that:
           - out.failed
-          - |
+          - >-
-            ('sudo: no such option: --derps' in out.msg) or
+            'sudo: no such option: --derps' in out.msg
-            ("sudo: invalid option -- '-'" in out.module_stderr) or
+            or out.module_stdout is match("sudo: invalid option -- '-'")
-            ("sudo: unrecognized option `--derps'" in out.module_stderr) or
+            or out.module_stderr is match("sudo: invalid option -- '-'")
-            ("sudo: unrecognized option `--derps'" in out.module_stdout) or
+            or out.module_stdout is match("sudo: unrecognized option [`']--derps'")
-            ("sudo: unrecognized option '--derps'" in out.module_stderr)
+            or out.module_stderr is match("sudo: unrecognized option [`']--derps'")
         fail_msg: out={{out}}
+  tags:
+    - sudo
+    - sudo_flags_failure

tests/ansible/integration/become/sudo_nonexistent.yml

@@ -22,8 +22,8 @@
           # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
           # removed user/group error messages, as defence against CVE-2019-14287.
           - >-
-            ('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg))
+            'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg)
-            or ('chown: slartibartfast: illegal user name' in out.module_stderr | default(out.msg))
+            or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)
             or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')
         fail_msg: out={{out}}
       when:
@@ -31,3 +31,6 @@
         - ansible_facts.distribution not in ["MacOSX"]
           or ansible_version.full is version("2.11", ">=", strict=True)
           or is_mitogen
+  tags:
+    - sudo
+    - sudo_nonexistent

tests/ansible/integration/become/sudo_nopassword.yml

@@ -1,6 +1,6 @@
 # Verify passwordless sudo behaviour in various cases.
 
-- name: integration/become/sudo_basic.yml
+- name: integration/become/sudo_nopassword.yml
   hosts: test-targets
   any_errors_fatal: true
   tasks:
@@ -24,3 +24,6 @@
         that:
           - out.stdout == 'root'
         fail_msg: out={{out}}
+  tags:
+    - sudo
+    - sudo_nopassword

tests/ansible/integration/become/sudo_password.yml

@@ -70,3 +70,6 @@
     # - assert:
     #     that:
     #       - out.stdout == 'mitogen__pw_required'
+  tags:
+    - sudo
+    - sudo_password

tests/ansible/integration/become/sudo_requiretty.yml

@@ -35,3 +35,6 @@
     #     that:
     #       - out.stdout == 'mitogen__require_tty_pw_required'
     #   when: is_mitogen
+  tags:
+    - sudo
+    - sudo_requiretty

tests/ansible/integration/connection/all.yml

@@ -1,11 +1,11 @@
 ---
 
-- include: become_same_user.yml
+- import_playbook: become_same_user.yml
-- include: disconnect_during_module.yml
+- import_playbook: disconnect_during_module.yml
-- include: disconnect_resets_connection.yml
+- import_playbook: disconnect_resets_connection.yml
-- include: exec_command.yml
+- import_playbook: exec_command.yml
-- include: home_dir.yml
+- import_playbook: home_dir.yml
-- include: put_large_file.yml
+- import_playbook: put_large_file.yml
-- include: put_small_file.yml
+- import_playbook: put_small_file.yml
-- include: reset.yml
+- import_playbook: reset.yml
-- include: reset_become.yml
+- import_playbook: reset_become.yml

tests/ansible/integration/connection/become_same_user.yml

@@ -39,3 +39,5 @@
       - out.result|length == 2  # no sudo
       fail_msg: out={{out}}
     when: is_mitogen
+  tags:
+    - become_same_user

tests/ansible/integration/connection/disconnect_during_module.yml

@@ -26,3 +26,6 @@
       - out.rc == 4
       - "'Mitogen was disconnected from the remote environment while a call was in-progress.' in out.stdout"
       fail_msg: out={{out}}
+  tags:
+    - disconnect
+    - disconnect_during_module

tests/ansible/integration/connection/disconnect_resets_connection.yml

@@ -45,3 +45,6 @@
         except AnsibleConnectionFailure:
           e = sys.exc_info()[1]
           assert str(e).startswith('Mitogen was disconnected')
+  tags:
+    - disconnect
+    - disconnect_resets_connection

tests/ansible/integration/connection/exec_command.yml

@@ -18,3 +18,5 @@
       - out.result[1].decode() == "hello, world\r\n"
       - out.result[2].decode().startswith("Shared connection to ")
       fail_msg: out={{out}}
+  tags:
+    - exec_command

tests/ansible/integration/connection/home_dir.yml

@@ -37,3 +37,5 @@
             "homedir from facts": "{{root_facts.ansible_facts.ansible_user_dir}}"
           }
       when: is_mitogen
+  tags:
+    - home_dir

tests/ansible/integration/connection/put_large_file.yml

@@ -10,3 +10,6 @@
     file_size: 512
   tasks:
   - include: _put_file.yml
+  tags:
+    - put_file
+    - put_large_file

tests/ansible/integration/connection/put_small_file.yml

@@ -10,3 +10,6 @@
     file_size: 123
   tasks:
   - include: _put_file.yml
+  tags:
+    - put_file
+    - put_small_file

tests/ansible/integration/connection/reset.yml

@@ -10,10 +10,12 @@
       when: not is_mitogen
 
     - debug: msg="reset.yml skipped on Ansible<2.5.6"
-      when: ansible_version.full < '2.5.6'
+      when:
+        - ansible_version.full is version('2.5.6', '<', strict=True)
 
     - meta: end_play
-      when: ansible_version.full < '2.5.6'
+      when:
+        - ansible_version.full is version('2.5.6', '<', strict=True)
 
     - custom_python_detect_environment:
       register: out
@@ -44,3 +46,5 @@
         # sudo PID has changed.
         - out_become.ppid != out_become2.ppid
         fail_msg: out={{out}} out2={{out2}} out_become={{out_become}} out_become2={{out_become2}}
+  tags:
+    - reset

tests/ansible/integration/connection/reset_become.yml

@@ -6,10 +6,12 @@
   gather_facts: false
   tasks:
   - debug: msg="reset_become.yml skipped on Ansible<2.5.6"
-    when: ansible_version.full < '2.5.6'
+    when:
+      - ansible_version.full is version('2.5.6', '<', strict=True)
 
   - meta: end_play
-    when: ansible_version.full < '2.5.6'
+    when:
+      - ansible_version.full is version('2.5.6', '<', strict=True)
 
   - name: save pid of the become acct
     custom_python_detect_environment:
@@ -49,3 +51,5 @@
       that:
       - login_acct.pid != new_login_acct.pid
       fail_msg: login_acct={{login_acct}} new_login_acct={{new_login_acct}}
+  tags:
+    - reset_become

tests/ansible/integration/connection_delegation/all.yml

@@ -1,5 +1,5 @@
-- include: delegate_to_template.yml
+- import_playbook: delegate_to_template.yml
-- include: local_action.yml
+- import_playbook: local_action.yml
-- include: osa_container_standalone.yml
+- import_playbook: osa_container_standalone.yml
-- include: osa_delegate_to_self.yml
+- import_playbook: osa_delegate_to_self.yml
-- include: stack_construction.yml
+- import_playbook: stack_construction.yml

tests/ansible/integration/connection_delegation/delegate_to_template.yml

@@ -20,7 +20,8 @@
       when: not is_mitogen
 
     - meta: end_play
-      when: ansible_version.full < '2.4'
+      when:
+        - ansible_version.full is version('2.4', '<', strict=True)
 
     - mitogen_get_stack:
       delegate_to: "{{ physical_host }}"
@@ -90,3 +91,5 @@
             'method': 'ssh',
           }
         ]
+  tags:
+    - delegate_to_template

tests/ansible/integration/connection_delegation/local_action.yml

@@ -33,3 +33,5 @@
             'method': 'sudo',
           }
         ]
+  tags:
+    - local_action

tests/ansible/integration/connection_delegation/osa_container_standalone.yml

@@ -27,3 +27,6 @@
           'method': 'setns',
         },
       ]
+  tags:
+    - osa
+    - osa_container_standalone

tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml

@@ -30,3 +30,6 @@
           'method': 'setns',
         },
       ]
+  tags:
+    - osa
+    - osa_delegate_to_self

tests/ansible/integration/connection_delegation/stack_construction.yml

@@ -25,6 +25,8 @@
     # used later for local_action test.
     - local_action: custom_python_detect_environment
       register: local_env
+  tags:
+    - stack_construction
 
 
 - hosts: cd-normal
@@ -50,6 +52,8 @@
             "method": "doas",
           }
         ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-normal
@@ -94,6 +98,8 @@
                   'method': 'ssh',
               },
           ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-alias
@@ -137,6 +143,8 @@
                   'method': 'ssh',
               },
           ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-normal-normal
@@ -191,6 +199,8 @@
                   'method': 'ssh',
               },
           ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-normal-alias
@@ -264,6 +274,8 @@
                   'method': 'ssh',
               },
           ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-newuser-normal-normal
@@ -318,6 +330,8 @@
                   'method': 'ssh',
               },
           ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-newuser-normal-normal
@@ -362,6 +376,8 @@
                   'method': 'ssh',
               },
           ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-newuser-normal-normal
@@ -381,6 +397,8 @@
             'method': 'local',
           },
         ]
+  tags:
+    - stack_construction
 
 
 - hosts: cd-newuser-doas-normal
@@ -416,3 +434,5 @@
             'method': 'doas',
           },
       ]
+  tags:
+    - stack_construction

tests/ansible/integration/connection_loader/all.yml

@@ -1,3 +1,3 @@
-- include: local_blemished.yml
+- import_playbook: local_blemished.yml
-- include: paramiko_unblemished.yml
+- import_playbook: paramiko_unblemished.yml
-- include: ssh_blemished.yml
+- import_playbook: ssh_blemished.yml

tests/ansible/integration/connection_loader/local_blemished.yml

@@ -13,3 +13,6 @@
     - assert:
         that: (not not out.mitogen_loaded) == (not not is_mitogen)
         fail_msg: out={{out}}
+  tags:
+    - local
+    - local_blemished

tests/ansible/integration/connection_loader/paramiko_unblemished.yml

@@ -17,3 +17,6 @@
             that: not out.mitogen_loaded
             fail_msg: out={{out}}
       when: False
+  tags:
+    - paramiko
+    - paramiko_unblemished

tests/ansible/integration/connection_loader/ssh_blemished.yml

@@ -13,3 +13,6 @@
     - assert:
         that: (not not out.mitogen_loaded) == (not not is_mitogen)
         fail_msg: out={{out}}
+  tags:
+    - ssh
+    - ssh_blemished

tests/ansible/integration/context_service/all.yml

@@ -1,4 +1,4 @@
-- include: disconnect_cleanup.yml
+- import_playbook: disconnect_cleanup.yml
-- include: lru_one_target.yml
+- import_playbook: lru_one_target.yml
-- include: reconnection.yml
+- import_playbook: reconnection.yml
-- include: remote_name.yml
+- import_playbook: remote_name.yml

tests/ansible/integration/context_service/disconnect_cleanup.yml

@@ -9,7 +9,8 @@
     when: not is_mitogen
 
   - meta: end_play
-    when: ansible_version.full < '2.5.6'
+    when:
+      - ansible_version.full is version('2.5.6', '<', strict=True)
 
   # Start with a clean slate.
   - mitogen_shutdown_all:
@@ -48,3 +49,5 @@
 
   # - assert:
   #     that: out.dump|length == play_hosts|length # just the ssh account
+  tags:
+    - disconnect_cleanup

tests/ansible/integration/context_service/lru_one_target.yml

@@ -40,3 +40,5 @@
     #     that:
     #     - first_run.results[-1].pid != second_run.results[-1].pid
     #   when: is_mitogen
+  tags:
+    - lru_one_target

tests/ansible/integration/context_service/reconnection.yml

@@ -32,3 +32,5 @@
         that:
           - old_become_env.pid != new_become_env.pid
         fail_msg: old_become_env={{old_become_env}} new_become_env={{new_become_env}}
+  tags:
+    - reconnection

tests/ansible/integration/context_service/remote_name.yml

@@ -30,4 +30,5 @@
       that:
       - out.stdout is match('.*python([0-9.]+)?\(mitogen:ansible\)')
       fail_msg: out={{out}}
-
+  tags:
+    - remote_name

tests/ansible/integration/glibc_caches/all.yml

@@ -1,2 +1,2 @@
 
-- include: resolv_conf.yml
+- import_playbook: resolv_conf.yml

tests/ansible/integration/glibc_caches/resolv_conf.yml

@@ -8,36 +8,35 @@
   vars:
     ansible_become_pass: has_sudo_pubkey_password
   tasks:
-
     - mitogen_test_gethostbyname:
         name: www.google.com
       register: out
-      when: |
+      when:
-        ansible_virtualization_type == "docker" and
+        - ansible_facts.virtualization_type == "docker"
-        ansible_python_version > "2.5"
+        - ansible_facts.python.version_info[:2] >= [2, 5]
 
     - shell: cp /etc/resolv.conf /tmp/resolv.conf
-      when: |
+      when:
-        ansible_virtualization_type == "docker" and
+        - ansible_facts.virtualization_type == "docker"
-        ansible_python_version > "2.5"
+        - ansible_facts.python.version_info[:2] >= [2, 5]
 
     - shell: echo > /etc/resolv.conf
-      when: |
+      when:
-        ansible_virtualization_type == "docker" and
+        - ansible_facts.virtualization_type == "docker"
-        ansible_python_version > "2.5"
+        - ansible_facts.python.version_info[:2] >= [2, 5]
 
     - mitogen_test_gethostbyname:
         name: www.google.com
       register: out
       ignore_errors: true
-      when: |
+      when:
-        ansible_virtualization_type == "docker" and
+        - ansible_facts.virtualization_type == "docker"
-        ansible_python_version > "2.5"
+        - ansible_facts.python.version_info[:2] >= [2, 5]
 
     - shell: cat /tmp/resolv.conf > /etc/resolv.conf
-      when: |
+      when:
-        ansible_virtualization_type == "docker" and
+        - ansible_facts.virtualization_type == "docker"
-        ansible_python_version > "2.5"
+        - ansible_facts.python.version_info[:2] >= [2, 5]
 
     - assert:
         that:
@@ -45,6 +44,8 @@
         - '"Name or service not known" in out.msg or
            "Temporary failure in name resolution" in out.msg'
         fail_msg: out={{out}}
-      when: |
+      when:
-        ansible_virtualization_type == "docker" and
+        - ansible_facts.virtualization_type == "docker"
-        ansible_python_version > "2.5"
+        - ansible_facts.python.version_info[:2] >= [2, 5]
+  tags:
+    - resolv_conf

tests/ansible/integration/interpreter_discovery/all.yml

@@ -1,2 +1,2 @@
-- include: complex_args.yml
+- import_playbook: complex_args.yml
-- include: ansible_2_8_tests.yml
+- import_playbook: ansible_2_8_tests.yml

tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml

@@ -63,15 +63,33 @@
             ping:
             register: legacy
 
-          - name: check for dep warning (only on platforms where auto result is not /usr/bin/python and legacy is)
+          - name: check for dep warning (only on platforms where auto result is not /usr/bin/python and legacy is) for ansible 2.8-2.11
+            # from ansible 2.12 on this changed
+            #  - https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_5.html#python-interpreter-discovery
+            #  - https://docs.ansible.com/ansible/latest/reference_appendices/interpreter_discovery.html
+            # default discovery method is now auto and will default to python3
+            # and the message changed from a deprecation warning to a real warning that can not be suppressed by
+            # using deprecation_warnings=False
             assert:
               that:
               - legacy.deprecations | default([]) | length > 0
               fail_msg: legacy={{legacy}}
             # only check for a dep warning if legacy returned /usr/bin/python and auto didn't
-            when: legacy.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and
+            when:
-                  auto_out.ansible_facts.discovered_interpreter_python != '/usr/bin/python'
+              - legacy.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
+              - auto_out.ansible_facts.discovered_interpreter_python != '/usr/bin/python'
+              - ansible_version.full is version_compare('2.12.0', '<', strict=True)
 
+          - name: check for warning (only on platforms where auto result is not /usr/bin/python and legacy is) from ansible 2.12 on
+            assert:
+              that:
+              - legacy.warnings | default([]) | length > 0
+              fail_msg: legacy={{legacy}}
+            # only check for a warning if legacy returned /usr/bin/python and auto didn't
+            when:
+              - legacy.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
+              -  auto_out.ansible_facts.discovered_interpreter_python != '/usr/bin/python'
+              -  ansible_version.full is version_compare('2.12.0', '>=', strict=True)
 
         - name: test that auto_silent never warns and got the same answer as auto
           block:
@@ -132,27 +150,45 @@
               that:
               - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3'
               fail_msg: auto_out={{auto_out}}
-            when: distro == 'fedora' and distro_version is version('23', '>=')
+            when:
+              - distro == 'fedora'
+              - distro_version is version('23.0', '>=', strict=True)
+
+          - name: rhel < 8 assertions
+            assert:
+              that:
+                - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
+              fail_msg: auto_out={{auto_out}}
+            when:
+              - distro in ('redhat', 'centos')
+              - distro_version is version('8.0', '<', strict=true)
+
+          - name: rhel 8+ assertions
+            assert:
+              that:
+                - auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python'
+              fail_msg: auto_out={{auto_out}}
+            when:
+              - distro in ('redhat', 'centos')
+              - distro_version is version('8.0', '>=', strict=true)
 
-          - name: rhel assertions
+          - name: ubuntu < 16.04 assertions
             assert:
               that:
-              # rhel 6/7
+                - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
-              - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and distro_version is version('8','<')) or distro_version is version('8','>=')
-              # rhel 8+
-              - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python' and distro_version is version('8','>=')) or distro_version is version('8','<')
               fail_msg: auto_out={{auto_out}}
-            when: distro in ('redhat', 'centos')
+            when:
+              - distro == 'ubuntu'
+              - distro_version is version('16.04', '<', strict=true)
 
-          - name: ubuntu assertions
+          - name: ubuntu 16.04+ assertions
             assert:
               that:
-              # ubuntu < 16
+                - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3'
-              - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' and distro_version is version('16.04','<')) or distro_version is version('16.04','>=')
-              # ubuntu >= 16
-              - (auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' and distro_version is version('16.04','>=')) or distro_version is version('16.04','<')
               fail_msg: auto_out={{auto_out}}
-            when: distro == 'ubuntu'
+            when:
+              - distro == 'ubuntu'
+              - distro_version is version('16.04', '>=', strict=True)
 
           - name: mac assertions
             assert:
@@ -163,4 +199,7 @@
 
           always:
           - meta: clear_facts
-      when: ansible_version.full is version_compare('2.8.0', '>=')
+      when:
+        - ansible_version.full is version_compare('2.8.0', '>=', strict=True)
+  tags:
+    - ansible_2_8_tests

tests/ansible/integration/interpreter_discovery/complex_args.yml

@@ -54,3 +54,5 @@
       environment:
         https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
         no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
+  tags:
+    - complex_args

tests/ansible/integration/local/all.yml

@@ -1,4 +1,4 @@
 
-- include: cwd_preserved.yml
+- import_playbook: cwd_preserved.yml
-- include: env_preserved.yml
+- import_playbook: env_preserved.yml
 

tests/ansible/integration/local/cwd_preserved.yml

@@ -20,4 +20,5 @@
   - assert:
       that: stat.stat.exists
       fail_msg: stat={{stat}}
-
+  tags:
+    - cwd_prseserved

tests/ansible/integration/local/env_preserved.yml

@@ -6,3 +6,5 @@
   connection: local
   tasks:
     - shell: "env | grep EVIL_VARS_PLUGIN"
+  tags:
+    - env_preserved

tests/ansible/integration/module_utils/adjacent_to_playbook.yml

@@ -14,4 +14,5 @@
       - out.external1_path == "ansible/integration/module_utils/module_utils/external1.py"
       - out.external2_path == "ansible/lib/module_utils/external2.py"
       fail_msg: out={{out}}
-
+  tags:
+    - adjacent_to_playbook

tests/ansible/integration/module_utils/adjacent_to_role.yml

@@ -6,3 +6,5 @@
   any_errors_fatal: true
   roles:
   - modrole
+  tags:
+    - adjacent_to_playbook

tests/ansible/integration/module_utils/all.yml

@@ -1,6 +1,6 @@
 
-#- include: from_config_path.yml
+#- import_playbook: from_config_path.yml
-#- include: from_config_path_pkg.yml
+#- import_playbook: from_config_path_pkg.yml
-#- include: adjacent_to_playbook.yml
+#- import_playbook: adjacent_to_playbook.yml
-- include: adjacent_to_role.yml
+- import_playbook: adjacent_to_role.yml
-#- include: overrides_builtin.yml
+#- import_playbook: overrides_builtin.yml

tests/ansible/integration/module_utils/from_config_path.yml

@@ -13,4 +13,5 @@
       - out.external1_path == "ansible/lib/module_utils/external1.py"
       - out.external2_path == "ansible/lib/module_utils/external2.py"
       fail_msg: out={{out}}
-
+  tags:
+    - from_config_path

tests/ansible/integration/module_utils/from_config_path_pkg.yml

@@ -12,4 +12,5 @@
       that:
       - out.extmod_path == "ansible/lib/module_utils/externalpkg/extmod.py"
       fail_msg: out={{out}}
-
+  tags:
+    - from_config_path