ci: Upgrade VM Images to macOS 11 and Ubuntu 20.04

.ci/azure-pipelines-steps.yml

@@ -1,9 +1,3 @@
-
-parameters:
-  name: ''
-  pool: ''
-  sign: false
-
 steps:
 - task: UsePythonVersion@0
   displayName: Install python
@@ -11,6 +5,12 @@ steps:
     versionSpec: '$(python.version)'
   condition: ne(variables['python.version'], '')
 
+- script: |
+    sudo apt-get update
+    sudo apt-get install -y python2-dev
+  displayName: Install build deps
+  condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux'))
+
 - script: python -mpip install tox
   displayName: Install tooling
 

.ci/azure-pipelines.yml

@@ -9,61 +9,18 @@
   #ANSIBLE_VERBOSITY: 3
 
 jobs:
-- job: Mac1015
+- job: mac11
   # vanilla Ansible is really slow
   timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
-    # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md
+    # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md
-    vmImage: macOS-10.15
-  strategy:
-    matrix:
-      Mito_27:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen
-      Mito_36:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen
-      Mito_310:
-        python.version: '3.10'
-        tox.env: py310-mode_mitogen
-
-      # TODO: test python3, python3 tests are broken
-      Loc_27_210:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
-      Loc_27_4:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible4
-
-      # NOTE: this hangs when ran in Ubuntu 18.04
-      Van_27_210:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_4:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible4
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-
-- job: Mac11
-  # vanilla Ansible is really slow
-  timeoutInMinutes: 120
-  steps:
-  - template: azure-pipelines-steps.yml
-  pool:
-    # https://github.com/actions/virtual-environments/blob/main/images/macos/
     vmImage: macOS-11
   strategy:
     matrix:
       Mito_27:
         tox.env: py27-mode_mitogen
-      Mito_37:
-        python.version: '3.7'
-        tox.env: py37-mode_mitogen
       Mito_310:
         python.version: '3.10'
         tox.env: py310-mode_mitogen
@@ -86,38 +43,29 @@ jobs:
 
 - job: Linux
   pool:
-    # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
+    # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md
-    vmImage: "Ubuntu 18.04"
+    vmImage: ubuntu-20.04
   steps:
   - template: azure-pipelines-steps.yml
   strategy:
     matrix:
       Mito_27_centos6:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_centos6
       Mito_27_centos7:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_centos7
       Mito_27_centos8:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_centos8
       Mito_27_debian9:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_debian9
       Mito_27_debian10:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_debian10
       Mito_27_debian11:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_debian11
       Mito_27_ubuntu1604:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_ubuntu1604
       Mito_27_ubuntu1804:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_ubuntu1804
       Mito_27_ubuntu2004:
-        python.version: '2.7'
         tox.env: py27-mode_mitogen-distro_ubuntu2004
 
       Mito_36_centos6:
@@ -177,10 +125,8 @@ jobs:
         tox.env: py310-mode_mitogen-distro_ubuntu2004
 
       Ans_27_210:
-        python.version: '2.7'
         tox.env: py27-mode_ansible-ansible2.10
       Ans_27_4:
-        python.version: '2.7'
         tox.env: py27-mode_ansible-ansible4
 
       Ans_36_210:

tests/ansible/hosts/group_vars/all.yml

@@ -1,2 +1,2 @@
 ---
-pkg_mgr_python_interpreter: /usr/bin/python
+pkg_mgr_python_interpreter: python

tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils import external3

tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 import json
 import ansible.module_utils.basic

tests/ansible/lib/modules/custom_python_detect_environment.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible new-style Python module. I return details about the Python
 # interpreter I run within.
 

tests/ansible/lib/modules/custom_python_external_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I expect the quote from modules2/module_utils/joker.py.
 
 from ansible.module_utils.basic import AnsibleModule

tests/ansible/lib/modules/custom_python_external_pkg.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.externalpkg import extmod

tests/ansible/lib/modules/custom_python_json_args_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible Python JSONARGS module. I should receive an encoding string.
 
 json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"""

tests/ansible/lib/modules/custom_python_leaky_class_vars.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible new-style Python module. I leak state from each invocation
 # into a class variable and a global variable.
 

tests/ansible/lib/modules/custom_python_modify_environ.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible new-style Python module. I modify the process environment and
 # don't clean up after myself.
 

tests/ansible/lib/modules/custom_python_new_style_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible new-style Python module. I should receive an encoding string.
 
 import sys

tests/ansible/lib/modules/custom_python_os_getcwd.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the
 # process environment.
 

tests/ansible/lib/modules/custom_python_prehistoric_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # issue #555: I'm a module that cutpastes an old hack.
 
 from ansible.module_utils.basic import AnsibleModule

tests/ansible/lib/modules/custom_python_run_script.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible new-style Python module. I run the script provided in the
 # parameter.
 

tests/ansible/lib/modules/custom_python_uses_distro.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # issue #590: I am an Ansible new-style Python module that tries to use
 # ansible.module_utils.distro.
 

tests/ansible/lib/modules/custom_python_want_json_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file.
 
 import sys

tests/ansible/lib/modules/mitogen_test_gethostbyname.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # I am a module that indirectly depends on glibc cached /etc/resolv.conf state.
 

tests/ansible/lib/modules/test_echo_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>

tests/ansible/tests/connection_test.py

@@ -41,7 +41,7 @@ class ConnectionMixin(MuxProcessMixin):
         conn = self.klass(play_context, new_stdin=False)
         # conn functions don't fetch ActionModuleMixin objs from _get_task_vars()
         # through the usual walk-the-stack approach so we'll not run interpreter discovery here
-        conn._action = mock.MagicMock(_possible_python_interpreter='/usr/bin/python')
+        conn._action = mock.MagicMock(_possible_python_interpreter=testlib.base_executable())
         conn.on_action_run(
             task_vars={},
             delegate_to_hostname=None,

tests/data/minimize_samples/hashbang.py

@@ -1,3 +1,3 @@
-#/usr/bin/python -c
+#/usr/bin/env python -c
 # coding: utf-8
 # comment

tests/data/minimize_samples/hashbang_min.py

@@ -1,3 +1,3 @@
-#/usr/bin/python -c
+#/usr/bin/env python -c
 # coding: utf-8
 

tests/testlib.py

@@ -1,15 +1,22 @@
 import errno
+import io
 import logging
 import os
 import random
 import re
 import socket
+import stat
 import sys
 import threading
 import time
 import traceback
 import unittest
 
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
 import psutil
 import subprocess32 as subprocess
 
@@ -62,6 +69,67 @@ if faulthandler is not None:
 mitogen.core.LOG.propagate = True
 
 
+def base_executable(executable=None):
+    '''Return the path of the Python executable used to create the virtualenv.
+    '''
+    # https://docs.python.org/3/library/venv.html
+    # https://github.com/pypa/virtualenv/blob/main/src/virtualenv/discovery/py_info.py
+    # https://virtualenv.pypa.io/en/16.7.9/reference.html#compatibility-with-the-stdlib-venv-module
+    if executable is None:
+        executable = sys.executable
+
+    if not executable:
+        raise ValueError
+
+    try:
+        base_executable = sys._base_executable
+    except AttributeError:
+        base_executable = None
+
+    if base_executable and base_executable != executable:
+        return 'be', base_executable
+
+    # Python 2.x only has sys.base_prefix if running outside a virtualenv.
+    try:
+        sys.base_prefix
+    except AttributeError:
+        # Python 2.x outside a virtualenv
+        return executable
+
+    # Python 3.3+ has sys.base_prefix. In a virtualenv it differs to sys.prefix.
+    if sys.base_prefix == sys.prefix:
+        return executable
+
+    while executable.startswith(sys.prefix) and stat.S_ISLNK(os.lstat(executable).st_mode):
+        dirname = os.path.dirname(executable)
+        target = os.path.join(dirname, os.readlink(executable))
+        executable = os.path.abspath(os.path.normpath(target))
+        print(executable)
+
+    if executable.startswith(sys.base_prefix):
+        return executable
+
+    # Virtualenvs record details in pyvenv.cfg
+    parser = configparser.RawConfigParser()
+    with io.open(os.path.join(sys.prefix, 'pyvenv.cfg'), encoding='utf-8') as f:
+        content = u'[virtualenv]\n' + f.read()
+    try:
+        parser.read_string(content)
+    except AttributeError:
+        parser.readfp(io.StringIO(content))
+
+    # virtualenv style pyvenv.cfg includes the base executable.
+    # venv style pyvenv.cfg doesn't.
+    try:
+        return parser.get(u'virtualenv', u'base-executable')
+    except configparser.NoOptionError:
+        pass
+
+    basename = os.path.basename(executable)
+    home = parser.get(u'virtualenv', u'home')
+    return os.path.join(home, basename)
+
+
 def data_path(suffix):
     path = os.path.join(DATA_DIR, suffix)
     if path.endswith('.key'):