diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 50f1a16b..f4570744 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -1,9 +1,3 @@ - -parameters: - name: '' - pool: '' - sign: false - steps: - task: UsePythonVersion@0 displayName: Install python @@ -11,6 +5,12 @@ steps: versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') +- script: | + sudo apt-get update + sudo apt-get install -y python2-dev + displayName: Install build deps + condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) + - script: python -mpip install tox displayName: Install tooling diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 98f4805d..4513ae9f 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -9,61 +9,18 @@ #ANSIBLE_VERBOSITY: 3 jobs: -- job: Mac1015 +- job: mac11 # vanilla Ansible is really slow timeoutInMinutes: 120 steps: - template: azure-pipelines-steps.yml pool: - # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md - vmImage: macOS-10.15 - strategy: - matrix: - Mito_27: - python.version: '2.7' - tox.env: py27-mode_mitogen - Mito_36: - python.version: '3.6' - tox.env: py36-mode_mitogen - Mito_310: - python.version: '3.10' - tox.env: py310-mode_mitogen - - # TODO: test python3, python3 tests are broken - Loc_27_210: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible2.10 - Loc_27_4: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible4 - - # NOTE: this hangs when ran in Ubuntu 18.04 - Van_27_210: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible2.10 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_4: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible4 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - -- job: Mac11 - # vanilla Ansible is really slow - timeoutInMinutes: 120 - steps: - - template: azure-pipelines-steps.yml - pool: - # https://github.com/actions/virtual-environments/blob/main/images/macos/ + # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md vmImage: macOS-11 strategy: matrix: Mito_27: tox.env: py27-mode_mitogen - Mito_37: - python.version: '3.7' - tox.env: py37-mode_mitogen Mito_310: python.version: '3.10' tox.env: py310-mode_mitogen @@ -86,38 +43,29 @@ jobs: - job: Linux pool: - # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md - vmImage: "Ubuntu 18.04" + # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md + vmImage: ubuntu-20.04 steps: - template: azure-pipelines-steps.yml strategy: matrix: Mito_27_centos6: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_centos6 Mito_27_centos7: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_centos7 Mito_27_centos8: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_centos8 Mito_27_debian9: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_debian9 Mito_27_debian10: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_debian10 Mito_27_debian11: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_debian11 Mito_27_ubuntu1604: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_ubuntu1604 Mito_27_ubuntu1804: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_ubuntu1804 Mito_27_ubuntu2004: - python.version: '2.7' tox.env: py27-mode_mitogen-distro_ubuntu2004 Mito_36_centos6: @@ -177,10 +125,8 @@ jobs: tox.env: py310-mode_mitogen-distro_ubuntu2004 Ans_27_210: - python.version: '2.7' tox.env: py27-mode_ansible-ansible2.10 Ans_27_4: - python.version: '2.7' tox.env: py27-mode_ansible-ansible4 Ans_36_210: diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index 08c4495d..cea46113 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -1,2 +1,2 @@ --- -pkg_mgr_python_interpreter: /usr/bin/python +pkg_mgr_python_interpreter: python diff --git a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py index 78f2d71d..faf8a9a5 100644 --- a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py +++ b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils import external3 diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py index 688fc45a..4fc4e04c 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import json import ansible.module_utils.basic diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 65f660a8..c7a222e7 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I return details about the Python # interpreter I run within. diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py index 507e53dd..ae1b78cb 100644 --- a/tests/ansible/lib/modules/custom_python_external_module.py +++ b/tests/ansible/lib/modules/custom_python_external_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I expect the quote from modules2/module_utils/joker.py. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py index 95bd0c7b..be9acb24 100644 --- a/tests/ansible/lib/modules/custom_python_external_pkg.py +++ b/tests/ansible/lib/modules/custom_python_external_pkg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.externalpkg import extmod diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 846037ec..61640579 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py index 1d342329..255e3729 100644 --- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py +++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I leak state from each invocation # into a class variable and a global variable. diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py index 9767f855..51b74526 100644 --- a/tests/ansible/lib/modules/custom_python_modify_environ.py +++ b/tests/ansible/lib/modules/custom_python_modify_environ.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I modify the process environment and # don't clean up after myself. diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index f9c176c1..1e7270cd 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I should receive an encoding string. import sys diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index 7fe3fd1b..b4e20afb 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py index 7ce15208..0cf9774d 100644 --- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py +++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # issue #555: I'm a module that cutpastes an old hack. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py index 4a6243d0..d6a839ae 100644 --- a/tests/ansible/lib/modules/custom_python_run_script.py +++ b/tests/ansible/lib/modules/custom_python_run_script.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible new-style Python module. I run the script provided in the # parameter. diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py index 6b3a356b..03f3b6aa 100644 --- a/tests/ansible/lib/modules/custom_python_uses_distro.py +++ b/tests/ansible/lib/modules/custom_python_uses_distro.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # issue #590: I am an Ansible new-style Python module that tries to use # ansible.module_utils.distro. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index d9ea7113..4a8dbd7e 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import sys diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py index 289e9662..1b80a48b 100644 --- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py +++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am a module that indirectly depends on glibc cached /etc/resolv.conf state. diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 37ab655c..1f71e879 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- # (c) 2012, Michael DeHaan diff --git a/tests/ansible/tests/connection_test.py b/tests/ansible/tests/connection_test.py index 8a831a5e..0ade6994 100644 --- a/tests/ansible/tests/connection_test.py +++ b/tests/ansible/tests/connection_test.py @@ -41,7 +41,7 @@ class ConnectionMixin(MuxProcessMixin): conn = self.klass(play_context, new_stdin=False) # conn functions don't fetch ActionModuleMixin objs from _get_task_vars() # through the usual walk-the-stack approach so we'll not run interpreter discovery here - conn._action = mock.MagicMock(_possible_python_interpreter='/usr/bin/python') + conn._action = mock.MagicMock(_possible_python_interpreter=testlib.base_executable()) conn.on_action_run( task_vars={}, delegate_to_hostname=None, diff --git a/tests/data/minimize_samples/hashbang.py b/tests/data/minimize_samples/hashbang.py index 0245c48f..1d358671 100644 --- a/tests/data/minimize_samples/hashbang.py +++ b/tests/data/minimize_samples/hashbang.py @@ -1,3 +1,3 @@ -#/usr/bin/python -c +#/usr/bin/env python -c # coding: utf-8 # comment diff --git a/tests/data/minimize_samples/hashbang_min.py b/tests/data/minimize_samples/hashbang_min.py index 5a74f481..19836fa5 100644 --- a/tests/data/minimize_samples/hashbang_min.py +++ b/tests/data/minimize_samples/hashbang_min.py @@ -1,3 +1,3 @@ -#/usr/bin/python -c +#/usr/bin/env python -c # coding: utf-8 diff --git a/tests/testlib.py b/tests/testlib.py index 1e27be9a..83629c3d 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,15 +1,22 @@ import errno +import io import logging import os import random import re import socket +import stat import sys import threading import time import traceback import unittest +try: + import configparser +except ImportError: + import ConfigParser as configparser + import psutil import subprocess32 as subprocess @@ -62,6 +69,67 @@ if faulthandler is not None: mitogen.core.LOG.propagate = True +def base_executable(executable=None): + '''Return the path of the Python executable used to create the virtualenv. + ''' + # https://docs.python.org/3/library/venv.html + # https://github.com/pypa/virtualenv/blob/main/src/virtualenv/discovery/py_info.py + # https://virtualenv.pypa.io/en/16.7.9/reference.html#compatibility-with-the-stdlib-venv-module + if executable is None: + executable = sys.executable + + if not executable: + raise ValueError + + try: + base_executable = sys._base_executable + except AttributeError: + base_executable = None + + if base_executable and base_executable != executable: + return 'be', base_executable + + # Python 2.x only has sys.base_prefix if running outside a virtualenv. + try: + sys.base_prefix + except AttributeError: + # Python 2.x outside a virtualenv + return executable + + # Python 3.3+ has sys.base_prefix. In a virtualenv it differs to sys.prefix. + if sys.base_prefix == sys.prefix: + return executable + + while executable.startswith(sys.prefix) and stat.S_ISLNK(os.lstat(executable).st_mode): + dirname = os.path.dirname(executable) + target = os.path.join(dirname, os.readlink(executable)) + executable = os.path.abspath(os.path.normpath(target)) + print(executable) + + if executable.startswith(sys.base_prefix): + return executable + + # Virtualenvs record details in pyvenv.cfg + parser = configparser.RawConfigParser() + with io.open(os.path.join(sys.prefix, 'pyvenv.cfg'), encoding='utf-8') as f: + content = u'[virtualenv]\n' + f.read() + try: + parser.read_string(content) + except AttributeError: + parser.readfp(io.StringIO(content)) + + # virtualenv style pyvenv.cfg includes the base executable. + # venv style pyvenv.cfg doesn't. + try: + return parser.get(u'virtualenv', u'base-executable') + except configparser.NoOptionError: + pass + + basename = os.path.basename(executable) + home = parser.get(u'virtualenv', u'home') + return os.path.join(home, basename) + + def data_path(suffix): path = os.path.join(DATA_DIR, suffix) if path.endswith('.key'):