From 455fd2bcdfb5b127dce5e7a39a6350371f903f1f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 2 Jul 2023 09:28:21 +0100 Subject: [PATCH 01/32] Bump version --- docs/changelog.rst | 4 ++++ mitogen/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7fc23df5..8c447756 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,10 @@ Release Notes To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +Unreleased +---------- + + v0.3.4 (2023-07-02) ------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 8b32cbcc..db4e8b3e 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 4) +__version__ = (0, 3, 5, 'dev0') #: This is :data:`False` in slave contexts. Previously it was used to prevent From 270c3a25de989e3889687c9efcd637fa4a5fdf1e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 5 Jul 2023 23:00:50 +0100 Subject: [PATCH 02/32] tests: Support Ubuntu 22.04 as test suite runner (controller) To do so the test suite allows a weak cryptographic alogorithm (SHA1) to be used, principally for CentOS 6 targets. This can be removed if/when support for older (legacy) targets is dropped. Only the test suite enables this known weak alogorithm. Mitogen as-shipped doesn't enable or disable algorithms. --- tests/ansible/ansible.cfg | 18 +++- .../delegate_to_template.yml | 28 +++--- .../stack_construction.yml | 98 ++++++++----------- .../process/unix_socket_cleanup.yml | 2 +- tests/ansible/integration/ssh/variables.yml | 16 +-- tests/ssh_test.py | 8 +- tests/testlib.py | 31 +++++- tox.ini | 29 +++--- 8 files changed, 124 insertions(+), 106 deletions(-) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index ce4511f6..c34dd219 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -48,5 +48,21 @@ host_pattern_mismatch = error task_output_limit = 10 [ssh_connection] -ssh_args = -o UserKnownHostsFile=/dev/null -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s +# https://www.openssh.com/legacy.html +# ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. +# Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). +# Duplicated cases in +# - tests/ansible/ansible.cfg +# - tests/ansible/integration/connection_delegation/delegate_to_template.yml +# - tests/ansible/integration/connection_delegation/stack_construction.yml +# - tests/ansible/integration/process/unix_socket_cleanup.yml +# - tests/ansible/integration/ssh/variables.yml +# - tests/testlib.py +ssh_args = + -o ControlMaster=auto + -o ControlPersist=60s + -o ForwardAgent=yes + -o HostKeyAlgorithms=+ssh-rsa + -o PubkeyAcceptedKeyTypes=+ssh-rsa + -o UserKnownHostsFile=/dev/null pipelining = True diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 36de1657..3776a7db 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -44,14 +44,12 @@ 'python_path': ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -74,14 +72,12 @@ 'python_path': ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 380b3198..279a7b2b 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -81,14 +81,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -126,14 +124,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -182,14 +178,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -227,14 +221,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -257,14 +249,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -313,14 +303,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', @@ -359,14 +347,12 @@ "python_path": ["/usr/bin/python"], 'remote_name': null, 'ssh_args': [ - '-o', - 'UserKnownHostsFile=/dev/null', - '-o', - 'ForwardAgent=yes', - '-o', - 'ControlMaster=auto', - '-o', - 'ControlPersist=60s', + -o, ControlMaster=auto, + -o, ControlPersist=60s, + -o, ForwardAgent=yes, + -o, HostKeyAlgorithms=+ssh-rsa, + -o, PubkeyAcceptedKeyTypes=+ssh-rsa, + -o, UserKnownHostsFile=/dev/null, ], 'ssh_debug_level': null, 'ssh_path': 'ssh', diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index 625ad7b3..b5d40b7d 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -9,7 +9,7 @@ - shell: > ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -c local -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 8e912b2e..6041d304 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -17,7 +17,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -34,7 +34,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -59,7 +59,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -76,7 +76,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -101,7 +101,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -118,7 +118,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -148,7 +148,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -165,7 +165,7 @@ shell: > ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" diff --git a/tests/ssh_test.py b/tests/ssh_test.py index 429fdb57..3149fcbc 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -134,12 +134,13 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): def test_enforce_unknown_host_key(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: e = self.assertRaises(mitogen.ssh.HostKeyError, lambda: self.docker_ssh( username='mitogen__has_sudo_pubkey', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) ) @@ -149,11 +150,12 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): def test_accept_enforce_host_keys(self): fp = tempfile.NamedTemporaryFile() + ssh_args = self.docker_ssh_default_kwargs.get('ssh_args', []) try: context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='accept', ) context.shutdown(wait=True) @@ -166,7 +168,7 @@ class SshTest(testlib.DockerMixin, testlib.TestCase): context = self.docker_ssh( username='mitogen__has_sudo', password='has_sudo_password', - ssh_args=['-o', 'UserKnownHostsFile ' + fp.name], + ssh_args=ssh_args + ['-o', 'UserKnownHostsFile %s' % fp.name], check_host_keys='enforce', ) context.shutdown(wait=True) diff --git a/tests/testlib.py b/tests/testlib.py index 83629c3d..f47d4cbc 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -631,12 +631,33 @@ class DockerMixin(RouterMixin): cls.dockerized_ssh.close() super(DockerMixin, cls).tearDownClass() + @property + def docker_ssh_default_kwargs(self): + return { + 'hostname': self.dockerized_ssh.host, + 'port': self.dockerized_ssh.port, + 'check_host_keys': 'ignore', + 'ssh_debug_level': 3, + # https://www.openssh.com/legacy.html + # ssh-rsa uses SHA1. Least worst available with CentOS 7 sshd. + # Rejected by default in newer ssh clients (e.g. Ubuntu 22.04). + # Duplicated cases in + # - tests/ansible/ansible.cfg + # - tests/ansible/integration/connection_delegation/delegate_to_template.yml + # - tests/ansible/integration/connection_delegation/stack_construction.yml + # - tests/ansible/integration/process/unix_socket_cleanup.yml + # - tests/ansible/integration/ssh/variables.yml + # - tests/testlib.py + 'ssh_args': [ + '-o', 'HostKeyAlgorithms +ssh-rsa', + '-o', 'PubkeyAcceptedKeyTypes +ssh-rsa', + ], + 'python_path': self.dockerized_ssh.python_path, + } + def docker_ssh(self, **kwargs): - kwargs.setdefault('hostname', self.dockerized_ssh.host) - kwargs.setdefault('port', self.dockerized_ssh.port) - kwargs.setdefault('check_host_keys', 'ignore') - kwargs.setdefault('ssh_debug_level', 3) - kwargs.setdefault('python_path', self.dockerized_ssh.python_path) + for k, v in self.docker_ssh_default_kwargs.items(): + kwargs.setdefault(k, v) return self.router.ssh(**kwargs) def docker_ssh_any(self, **kwargs): diff --git a/tox.ini b/tox.ini index da710933..13017f81 100644 --- a/tox.ini +++ b/tox.ini @@ -1,23 +1,20 @@ -# This file is a local convenience. It is not a substitute for the full CI -# suite, and does not cover the full range of Python versions for Mitogen. - -# I use this on Ubuntu 20.04, with the following additions +# This configuration drives both CI and local development. +# I use this locally on Ubuntu 22.04, with the following additions # # sudo add-apt-repository ppa:deadsnakes/ppa # sudo apt update -# sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass +# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox -# Last version to support each python version -# -# Python tox virt'env pip A cntllr A target coverage -# ========== ======== ======== ======== ======== ======== ======== -# python2.4 1.4 1.8 1.1 2.3? -# python2.5 1.6.1 1.9.1 1.3.1 ??? -# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 2.13 4.5.4 -# python2.7 20.3 2.11 -# python3.5 2.11 -# python3.6 2.11 -# python3.7 2.11 +# Py tox virtualenv pip A cntrllr A target Jinja2 coverage psutil pytest +# ==== ======== ========== ======== ========= ========= ========== ======== ======== ========= +# 2.4 <= 1.4 <= 1.8 <= 1.1 2.3? <= 3.7.1 <= 2.1.3 +# 2.5 <= 1.6.1 <= 1.9.1 <= 1.3.1 ??? <= 3.7.1 <= 2.1.3 <= 2.8.7 +# 2.6 <= 2.9.1 <= 15.2.0 <= 9.0.3 <= 2.6.20 <= 2.13 <= 2.10.3 <= 4.5.4 <= 5.9.0 <= 3.2.5 +# 2.7 <= 3.28 <= 20.3? <= 20 <= 2.11 <= 2.11.3 <= 5.6 <= 4.6.11 +# 3.5 <= 3.28 <= 20.15 <= 20 <= 2.11 <= 2.13 <= 2.11.3 <= 5.6 <= 6.1.0 +# 3.6 <= 3.28 <= 20.16 <= 21 <= 2.11 <= 3.0.3 <= 6.2 <= 7.0.1 +# 3.7 <= 2.12 +# 3.8 <= 2.12 # Ansible Dependency # ================== ====================== From 6258365df61aef6c4df498179b0eb39278c13f0d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Aug 2023 14:27:49 +0100 Subject: [PATCH 03/32] tests: Handle square bracket IPv6 in `docker port` output Fixes ``` ====================================================================== ERROR: setUpClass (ssh_test.BannerTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/vsts/work/1/s/tests/testlib.py", line 625, in setUpClass cls.dockerized_ssh = DockerizedSshDaemon(**daemon_args) File "/home/vsts/work/1/s/tests/testlib.py", line 553, in __init__ self.start_container() File "/home/vsts/work/1/s/tests/testlib.py", line 533, in start_container self._get_container_port() File "/home/vsts/work/1/s/tests/testlib.py", line 510, in _get_container_port self.port = int(bport) ValueError: invalid literal for int() with base 10: ':]:32770' ``` --- tests/testlib.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/tests/testlib.py b/tests/testlib.py index f47d4cbc..242f211b 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -499,19 +499,18 @@ def get_docker_host(): class DockerizedSshDaemon(object): - def _get_container_port(self): - s = subprocess.check_output(['docker', 'port', self.container_name]) - for line in s.decode().splitlines(): - m = self.PORT_RE.match(line) - if not m: - continue - dport, proto, _, bport = m.groups() - if dport == '22' and proto == 'tcp': - self.port = int(bport) + PORT_RE = re.compile( + # e.g. 0.0.0.0:32771, :::32771, [::]:32771' + r'(?P[0-9.]+|::|\[[a-f0-9:.]+\]):(?P[0-9]+)', + ) - self.host = self.get_host() - if self.port is None: + @classmethod + def get_port(cls, container): + s = subprocess.check_output(['docker', 'port', container, '22/tcp']) + m = cls.PORT_RE.search(s.decode()) + if not m: raise ValueError('could not find SSH port in: %r' % (s,)) + return int(m.group('port')) def start_container(self): try: @@ -530,7 +529,6 @@ class DockerizedSshDaemon(object): self.image, ] subprocess.check_output(args) - self._get_container_port() def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): if '-' in mitogen_test_distro: @@ -545,12 +543,9 @@ class DockerizedSshDaemon(object): self.python_path = '/usr/bin/python' self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - - # 22/tcp -> 0.0.0.0:32771 - self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)') - self.port = None - self.start_container() + self.host = self.get_host() + self.port = self.get_port(self.container_name) def get_host(self): return get_docker_host() From 98d110ed1647cb66df42ca5afdeb019203581dda Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 9 Aug 2023 15:27:41 +0100 Subject: [PATCH 04/32] tests: Bump Django ModuleFinder test cases Preperation for Python 3.11 support --- .../webproject/modules_expected_py2x.json | 186 +++++++++++++ .../modules_expected_py3x-legacy.json | 209 ++++++++++++++ .../webproject/modules_expected_py3x-new.json | 209 ++++++++++++++ tests/module_finder_test.py | 259 +++++------------- tests/requirements.txt | 4 +- tox.ini | 20 +- 6 files changed, 679 insertions(+), 208 deletions(-) create mode 100644 tests/data/importer/webproject/modules_expected_py2x.json create mode 100644 tests/data/importer/webproject/modules_expected_py3x-legacy.json create mode 100644 tests/data/importer/webproject/modules_expected_py3x-new.json diff --git a/tests/data/importer/webproject/modules_expected_py2x.json b/tests/data/importer/webproject/modules_expected_py2x.json new file mode 100644 index 00000000..4bf0bd39 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py2x.json @@ -0,0 +1,186 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.utils", + "django.utils._os", + "django.utils.deprecation", + "django.utils.encoding", + "django.utils.functional", + "django.utils.inspect", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.six", + "django.utils.version" + ], + "django.db.models": [ + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.checks", + "django.core.checks.caches", + "django.core.checks.compatibility", + "django.core.checks.compatibility.django_1_10", + "django.core.checks.compatibility.django_1_8_0", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.urls", + "django.core.checks.utils", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.deletion", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.base", + "django.db.models.functions.datetime", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.dispatch.weakref_backports", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.decorators", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.html", + "django.utils.html_parser", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.lru_cache", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.safestring", + "django.utils.six", + "django.utils.text", + "django.utils.timezone", + "django.utils.translation", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-legacy.json b/tests/data/importer/webproject/modules_expected_py3x-legacy.json new file mode 100644 index 00000000..06e94e62 --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-legacy.json @@ -0,0 +1,209 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" + ] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.compatibility", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo" + ] + } +} diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json new file mode 100644 index 00000000..5e9086df --- /dev/null +++ b/tests/data/importer/webproject/modules_expected_py3x-new.json @@ -0,0 +1,209 @@ +{ + "find_related_imports": { + "django.db": [ + "django", + "django.core", + "django.core.signals", + "django.db.utils", + "django.utils.connection" + ], + "django.db.models": [ + "django", + "django.core.exceptions", + "django.db", + "django.db.models", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals" +] + }, + "find_related": { + "django.db": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.conf", + "django.conf.global_settings", + "django.core", + "django.core.exceptions", + "django.core.signals", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.utils", + "django.utils.connection", + "django.utils.deprecation", + "django.utils.functional", + "django.utils.hashable", + "django.utils.inspect", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.regex_helper", + "django.utils.version" + ], + "django.db.models": [ + "asgiref", + "asgiref.current_thread_executor", + "asgiref.local", + "asgiref.sync", + "django", + "django.apps", + "django.apps.config", + "django.apps.registry", + "django.conf", + "django.conf.global_settings", + "django.conf.locale", + "django.core", + "django.core.cache", + "django.core.cache.backends", + "django.core.cache.backends.base", + "django.core.cache.backends.filebased", + "django.core.checks", + "django.core.checks.async_checks", + "django.core.checks.caches", + "django.core.checks.database", + "django.core.checks.messages", + "django.core.checks.model_checks", + "django.core.checks.registry", + "django.core.checks.security", + "django.core.checks.security.base", + "django.core.checks.security.csrf", + "django.core.checks.security.sessions", + "django.core.checks.templates", + "django.core.checks.translation", + "django.core.checks.urls", + "django.core.exceptions", + "django.core.files", + "django.core.files.base", + "django.core.files.images", + "django.core.files.locks", + "django.core.files.move", + "django.core.files.storage", + "django.core.files.utils", + "django.core.signals", + "django.core.validators", + "django.db", + "django.db.backends", + "django.db.backends.utils", + "django.db.models.aggregates", + "django.db.models.base", + "django.db.models.constants", + "django.db.models.constraints", + "django.db.models.deletion", + "django.db.models.enums", + "django.db.models.expressions", + "django.db.models.fields", + "django.db.models.fields.files", + "django.db.models.fields.json", + "django.db.models.fields.mixins", + "django.db.models.fields.proxy", + "django.db.models.fields.related", + "django.db.models.fields.related_descriptors", + "django.db.models.fields.related_lookups", + "django.db.models.fields.reverse_related", + "django.db.models.functions", + "django.db.models.functions.comparison", + "django.db.models.functions.datetime", + "django.db.models.functions.math", + "django.db.models.functions.mixins", + "django.db.models.functions.text", + "django.db.models.functions.window", + "django.db.models.indexes", + "django.db.models.lookups", + "django.db.models.manager", + "django.db.models.options", + "django.db.models.query", + "django.db.models.query_utils", + "django.db.models.signals", + "django.db.models.sql", + "django.db.models.sql.constants", + "django.db.models.sql.datastructures", + "django.db.models.sql.query", + "django.db.models.sql.subqueries", + "django.db.models.sql.where", + "django.db.models.utils", + "django.db.transaction", + "django.db.utils", + "django.dispatch", + "django.dispatch.dispatcher", + "django.forms", + "django.forms.boundfield", + "django.forms.fields", + "django.forms.forms", + "django.forms.formsets", + "django.forms.models", + "django.forms.renderers", + "django.forms.utils", + "django.forms.widgets", + "django.template", + "django.template.backends", + "django.template.backends.base", + "django.template.backends.django", + "django.template.base", + "django.template.context", + "django.template.engine", + "django.template.exceptions", + "django.template.library", + "django.template.loader", + "django.template.utils", + "django.templatetags", + "django.templatetags.static", + "django.utils", + "django.utils._os", + "django.utils.autoreload", + "django.utils.connection", + "django.utils.crypto", + "django.utils.datastructures", + "django.utils.dateformat", + "django.utils.dateparse", + "django.utils.dates", + "django.utils.datetime_safe", + "django.utils.deconstruct", + "django.utils.deprecation", + "django.utils.duration", + "django.utils.encoding", + "django.utils.formats", + "django.utils.functional", + "django.utils.hashable", + "django.utils.html", + "django.utils.http", + "django.utils.inspect", + "django.utils.ipv6", + "django.utils.itercompat", + "django.utils.module_loading", + "django.utils.numberformat", + "django.utils.regex_helper", + "django.utils.safestring", + "django.utils.text", + "django.utils.timezone", + "django.utils.topological_sort", + "django.utils.translation", + "django.utils.translation.trans_real", + "django.utils.tree", + "django.utils.version", + "pytz", + "pytz.exceptions", + "pytz.lazy", + "pytz.tzfile", + "pytz.tzinfo", + "zipimport" + ] + } +} + \ No newline at end of file diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index ff18bbc5..02b8b886 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -1,4 +1,5 @@ import inspect +import json import os import sys import unittest @@ -296,199 +297,65 @@ class FindRelatedTest(testlib.TestCase): self.assertEqual(set(related), self.SIMPLE_EXPECT) -if sys.version_info > (2, 6): - class DjangoMixin(object): - WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') - - # TODO: rip out Django and replace with a static tree of weird imports - # that don't depend on .. Django! The hack below is because the version - # of Django we need to test against 2.6 doesn't actually run on 3.6. - # But we don't care, we just need to be able to import it. - # - # File "django/utils/html_parser.py", line 12, in - # AttributeError: module 'html.parser' has no attribute - # 'HTMLParseError' - # - from django.utils.six.moves import html_parser as _html_parser - _html_parser.HTMLParseError = Exception - - @classmethod - def setUpClass(cls): - super(DjangoMixin, cls).setUpClass() - sys.path.append(cls.WEBPROJECT_PATH) - os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.WEBPROJECT_PATH) - del os.environ['DJANGO_SETTINGS_MODULE'] - super(DjangoMixin, cls).tearDownClass() - - - class FindRelatedImportsTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - - def call(self, fullname): - return self.klass().find_related_imports(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEqual(related, [ - 'django', - 'django.core', - 'django.core.signals', - 'django.db.utils', - 'django.utils.functional', - ]) - - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.maxDiff=None - self.assertEqual(related, [ - u'django', - u'django.core.exceptions', - u'django.db', - u'django.db.models', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.query', - u'django.db.models.signals', - ]) - - - class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): - klass = mitogen.master.ModuleFinder - maxDiff = None - - def call(self, fullname): - return self.klass().find_related(fullname) - - def test_django_db(self): - import django.db - related = self.call('django.db') - self.assertEqual(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.signals', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.utils', - u'django.utils._os', - u'django.utils.encoding', - u'django.utils.functional', - u'django.utils.importlib', - u'django.utils.module_loading', - u'django.utils.six', - ]) - - @unittest.skipIf( - condition=(sys.version_info >= (3, 0)), - reason='broken due to ancient vendored six.py' - ) - def test_django_db_models(self): - import django.db.models - related = self.call('django.db.models') - self.assertEqual(related, [ - u'django', - u'django.conf', - u'django.conf.global_settings', - u'django.core', - u'django.core.exceptions', - u'django.core.files', - u'django.core.files.base', - u'django.core.files.images', - u'django.core.files.locks', - u'django.core.files.move', - u'django.core.files.storage', - u'django.core.files.utils', - u'django.core.signals', - u'django.core.validators', - u'django.db', - u'django.db.backends', - u'django.db.backends.signals', - u'django.db.backends.util', - u'django.db.models.aggregates', - u'django.db.models.base', - u'django.db.models.constants', - u'django.db.models.deletion', - u'django.db.models.expressions', - u'django.db.models.fields', - u'django.db.models.fields.files', - u'django.db.models.fields.proxy', - u'django.db.models.fields.related', - u'django.db.models.fields.subclassing', - u'django.db.models.loading', - u'django.db.models.manager', - u'django.db.models.options', - u'django.db.models.query', - u'django.db.models.query_utils', - u'django.db.models.related', - u'django.db.models.signals', - u'django.db.models.sql', - u'django.db.models.sql.aggregates', - u'django.db.models.sql.constants', - u'django.db.models.sql.datastructures', - u'django.db.models.sql.expressions', - u'django.db.models.sql.query', - u'django.db.models.sql.subqueries', - u'django.db.models.sql.where', - u'django.db.transaction', - u'django.db.utils', - u'django.dispatch', - u'django.dispatch.dispatcher', - u'django.dispatch.saferef', - u'django.forms', - u'django.forms.fields', - u'django.forms.forms', - u'django.forms.formsets', - u'django.forms.models', - u'django.forms.util', - u'django.forms.widgets', - u'django.utils', - u'django.utils._os', - u'django.utils.crypto', - u'django.utils.datastructures', - u'django.utils.dateformat', - u'django.utils.dateparse', - u'django.utils.dates', - u'django.utils.datetime_safe', - u'django.utils.decorators', - u'django.utils.deprecation', - u'django.utils.encoding', - u'django.utils.formats', - u'django.utils.functional', - u'django.utils.html', - u'django.utils.html_parser', - u'django.utils.importlib', - u'django.utils.ipv6', - u'django.utils.itercompat', - u'django.utils.module_loading', - u'django.utils.numberformat', - u'django.utils.safestring', - u'django.utils.six', - u'django.utils.text', - u'django.utils.timezone', - u'django.utils.translation', - u'django.utils.tree', - u'django.utils.tzinfo', - u'pytz', - u'pytz.exceptions', - u'pytz.lazy', - u'pytz.tzfile', - u'pytz.tzinfo', - ]) +class DjangoMixin(object): + WEBPROJECT_PATH = os.path.join(testlib.MODS_DIR, 'webproject') + + @classmethod + def modules_expected_path(cls): + if sys.version_info[0:2] < (3, 0): + modules_expected_filename = 'modules_expected_py2x.json' + elif sys.version_info[0:2] <= (3, 6): + modules_expected_filename = 'modules_expected_py3x-legacy.json' + elif sys.version_info[0:2] >= (3, 10): + modules_expected_filename = 'modules_expected_py3x-new.json' + return os.path.join(cls.WEBPROJECT_PATH, modules_expected_filename) + + @classmethod + def setUpClass(cls): + super(DjangoMixin, cls).setUpClass() + sys.path.append(cls.WEBPROJECT_PATH) + os.environ['DJANGO_SETTINGS_MODULE'] = 'webproject.settings' + with open(cls.modules_expected_path(), 'rb') as f: + cls.MODULES_EXPECTED = json.load(f) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.WEBPROJECT_PATH) + del os.environ['DJANGO_SETTINGS_MODULE'] + super(DjangoMixin, cls).tearDownClass() + + +class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db') + expected = self.MODULES_EXPECTED['find_related']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related('django.db.models') + expected = self.MODULES_EXPECTED['find_related']['django.db.models'] + self.assertEqual(related, expected) + + +class DjangoFindRelatedImportsTest(DjangoMixin, testlib.TestCase): + maxDiff = None + + def test_django_db(self): + import django.db + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db'] + self.assertEqual(related, expected) + + def test_django_db_models(self): + import django.db.models + module_finder = mitogen.master.ModuleFinder() + related = module_finder.find_related_imports('django.db.models') + expected = self.MODULES_EXPECTED['find_related_imports']['django.db.models'] + self.assertEqual(related, expected) diff --git a/tests/requirements.txt b/tests/requirements.txt index 7301bee1..df755585 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,9 @@ psutil==5.4.8 coverage==5.5; python_version < '3.7' coverage==6.4.4; python_version >= '3.7' -Django==1.6.11 # Last version supporting 2.6. +Django==1.11.29; python_version < '3.0' +Django==3.2.20; python_version >= '3.6' mock==2.0.0 -pytz==2018.5 cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect pycparser==2.18 # Last version supporting 2.6. pytest-catchlog==1.2.2 diff --git a/tox.ini b/tox.ini index 13017f81..db30a2de 100644 --- a/tox.ini +++ b/tox.ini @@ -5,16 +5,16 @@ # sudo apt update # sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox -# Py tox virtualenv pip A cntrllr A target Jinja2 coverage psutil pytest -# ==== ======== ========== ======== ========= ========= ========== ======== ======== ========= -# 2.4 <= 1.4 <= 1.8 <= 1.1 2.3? <= 3.7.1 <= 2.1.3 -# 2.5 <= 1.6.1 <= 1.9.1 <= 1.3.1 ??? <= 3.7.1 <= 2.1.3 <= 2.8.7 -# 2.6 <= 2.9.1 <= 15.2.0 <= 9.0.3 <= 2.6.20 <= 2.13 <= 2.10.3 <= 4.5.4 <= 5.9.0 <= 3.2.5 -# 2.7 <= 3.28 <= 20.3? <= 20 <= 2.11 <= 2.11.3 <= 5.6 <= 4.6.11 -# 3.5 <= 3.28 <= 20.15 <= 20 <= 2.11 <= 2.13 <= 2.11.3 <= 5.6 <= 6.1.0 -# 3.6 <= 3.28 <= 20.16 <= 21 <= 2.11 <= 3.0.3 <= 6.2 <= 7.0.1 -# 3.7 <= 2.12 -# 3.8 <= 2.12 +# Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv +# ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== +# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 +# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 +# 2.6 <= 2.6.20 <= 2.13 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 +# 2.7 <= 2.11 <= 5.6 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.3? +# 3.5 <= 2.11 <= 2.13 <= 5.6 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15 +# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 5.9.5 <= 7.0.1 <= 3.28 <= 20.16 +# 3.7 <= 2.12 <= 3.2.20 +# 3.8 <= 2.12 # Ansible Dependency # ================== ====================== From 49c54386b33458760c69a82fc85d3bec2eb034a2 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 10 Aug 2023 09:31:32 +0100 Subject: [PATCH 05/32] tests: Only use subprocess32 package on Python 2.x This is how the package documentation recommends and it's less likely to interfere with new features in stdlib subprocess module. --- .ci/ci_lib.py | 5 ++++- tests/requirements.txt | 2 +- tests/testlib.py | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 47a8ddf3..3e716385 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -10,7 +10,10 @@ import shutil import sys import tempfile -import subprocess32 as subprocess +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess try: import urlparse diff --git a/tests/requirements.txt b/tests/requirements.txt index df755585..9e5e8c83 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,7 +8,7 @@ cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect pycparser==2.18 # Last version supporting 2.6. pytest-catchlog==1.2.2 pytest==3.1.2 -subprocess32==3.5.4 +subprocess32==3.5.4; python_version < '3.0' timeoutcontext==1.2.0 # Fix InsecurePlatformWarning while creating py26 tox environment # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings diff --git a/tests/testlib.py b/tests/testlib.py index 242f211b..a8db4a2e 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -18,7 +18,10 @@ except ImportError: import ConfigParser as configparser import psutil -import subprocess32 as subprocess +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess import mitogen.core import mitogen.fork From 4089e875a98dc0cdaebbd070733199ba01ab9f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nerijus=20Bali=C5=ABnas?= Date: Tue, 21 Feb 2023 13:04:28 +0200 Subject: [PATCH 06/32] Add Python 3.11 support Co-authored-by: Alex Willmer --- .ci/azure-pipelines.yml | 90 +++++++++---------- docs/ansible_detailed.rst | 4 +- docs/changelog.rst | 2 + setup.py | 1 + tests/ansible/lib/action/assert_equal.py | 7 +- .../webproject/modules_expected_py3x-new.json | 3 +- tests/requirements.txt | 5 +- tests/testlib.py | 2 +- tox.ini | 9 +- 9 files changed, 65 insertions(+), 58 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 4513ae9f..f672240d 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -21,9 +21,9 @@ jobs: matrix: Mito_27: tox.env: py27-mode_mitogen - Mito_310: - python.version: '3.10' - tox.env: py310-mode_mitogen + Mito_311: + python.version: '3.11' + tox.env: py311-mode_mitogen # TODO: test python3, python3 tests are broken Loc_27_210: @@ -96,33 +96,33 @@ jobs: python.version: '3.6' tox.env: py36-mode_mitogen-distro_ubuntu2004 - Mito_310_centos6: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos6 - Mito_310_centos7: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos7 - Mito_310_centos8: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_centos8 - Mito_310_debian9: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian9 - Mito_310_debian10: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian10 - Mito_310_debian11: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_debian11 - Mito_310_ubuntu1604: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu1604 - Mito_310_ubuntu1804: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu1804 - Mito_310_ubuntu2004: - python.version: '3.10' - tox.env: py310-mode_mitogen-distro_ubuntu2004 + Mito_311_centos6: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_centos6 + Mito_311_centos7: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_centos7 + Mito_311_centos8: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_centos8 + Mito_311_debian9: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_debian9 + Mito_311_debian10: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_debian10 + Mito_311_debian11: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_debian11 + Mito_311_ubuntu1604: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_ubuntu1604 + Mito_311_ubuntu1804: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_ubuntu1804 + Mito_311_ubuntu2004: + python.version: '3.11' + tox.env: py311-mode_mitogen-distro_ubuntu2004 Ans_27_210: tox.env: py27-mode_ansible-ansible2.10 @@ -136,18 +136,18 @@ jobs: python.version: '3.6' tox.env: py36-mode_ansible-ansible4 - Ans_310_210: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible2.10 - Ans_310_3: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible3 - Ans_310_4: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible4 - Ans_310_5: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible5 - Ans_310_6: - python.version: '3.10' - tox.env: py310-mode_ansible-ansible6 + Ans_311_210: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible2.10 + Ans_311_3: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible3 + Ans_311_4: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible4 + Ans_311_5: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible5 + Ans_311_6: + python.version: '3.11' + tox.env: py311-mode_ansible-ansible6 diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 449771b8..64fcd636 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -147,8 +147,8 @@ Noteworthy Differences * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. Mitogen 0.3.1+ supports - - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.10 - - Ansible 5 and 6; with Python 3.8-3.10 + - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 + - Ansible 5 and 6; with Python 3.8-3.11 Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index 8c447756..d4060177 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,8 @@ To avail of fixes in an unreleased version, please download a ZIP file Unreleased ---------- +* :gh:issue:`987` Support Python 3.11 + v0.3.4 (2023-07-02) ------------------- diff --git a/setup.py b/setup.py index 9d2ff36f..4d7fadfc 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ setup( 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/ansible/lib/action/assert_equal.py b/tests/ansible/lib/action/assert_equal.py index 5923f4b5..72264cf6 100644 --- a/tests/ansible/lib/action/assert_equal.py +++ b/tests/ansible/lib/action/assert_equal.py @@ -16,7 +16,12 @@ from ansible.plugins.action import ActionBase TEMPLATE_KWARGS = {} -_argspec = inspect.getargspec(ansible.template.Templar.template) +try: + # inspect.getfullargspec() Added: 3.0 + _argspec = inspect.getfullargspec(ansible.template.Templar.template) +except AttributeError: + # inspect.getargspec() Added: 2.1 Deprecated: 3.0 Removed: 3.11 + _argspec = inspect.getargspec(ansible.template.Templar.template) if 'bare_deprecated' in _argspec.args: TEMPLATE_KWARGS['bare_deprecated'] = False diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json index 5e9086df..dcbcc785 100644 --- a/tests/data/importer/webproject/modules_expected_py3x-new.json +++ b/tests/data/importer/webproject/modules_expected_py3x-new.json @@ -201,8 +201,7 @@ "pytz.exceptions", "pytz.lazy", "pytz.tzfile", - "pytz.tzinfo", - "zipimport" + "pytz.tzinfo" ] } } diff --git a/tests/requirements.txt b/tests/requirements.txt index 9e5e8c83..1e5d2a1d 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,11 +1,10 @@ -psutil==5.4.8 +cffi==1.15.1 coverage==5.5; python_version < '3.7' coverage==6.4.4; python_version >= '3.7' Django==1.11.29; python_version < '3.0' Django==3.2.20; python_version >= '3.6' mock==2.0.0 -cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect -pycparser==2.18 # Last version supporting 2.6. +psutil==5.9.5 pytest-catchlog==1.2.2 pytest==3.1.2 subprocess32==3.5.4; python_version < '3.0' diff --git a/tests/testlib.py b/tests/testlib.py index a8db4a2e..ec0a7443 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -90,7 +90,7 @@ def base_executable(executable=None): base_executable = None if base_executable and base_executable != executable: - return 'be', base_executable + return base_executable # Python 2.x only has sys.base_prefix if running outside a virtualenv. try: diff --git a/tox.ini b/tox.ini index db30a2de..5bfb7fe9 100644 --- a/tox.ini +++ b/tox.ini @@ -34,10 +34,10 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, - py{310}-mode_ansible-ansible{2.10,3,4,5,6}, - py{27,36,310}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,310}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,310}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{311}-mode_ansible-ansible{2.10,3,4,5,6}, + py{27,36,311}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,311}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,311}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, [testenv] @@ -50,6 +50,7 @@ basepython = py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt From 6aa4fd3573cf4a081017c19c4cc726fbc27d8853 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 15 Aug 2023 14:16:11 +0100 Subject: [PATCH 07/32] docs: Fix generation of static website Bare minimum syntax errors and requirements constraints to work with Netlify hosting. --- docs/ansible_detailed.rst | 2 ++ docs/changelog.rst | 2 +- docs/conf.py | 3 +-- docs/requirements.txt | 3 +++ mitogen/service.py | 10 ++++++---- tox.ini | 4 +++- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 64fcd636..a95575ee 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -147,8 +147,10 @@ Noteworthy Differences * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. Mitogen 0.3.1+ supports + - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 - Ansible 5 and 6; with Python 3.8-3.11 + Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index d4060177..dde66319 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -92,7 +92,7 @@ v0.2.10 (2021-11-24) * :gh:issue:`756` ssh connections with `check_host_keys='accept'` would timeout, when using recent OpenSSH client versions. * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in - :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` + :py:meth:`ansible.plugins.callback.CallbackBase.v2_runner_on_start` * :gh:issue:`775` Test with Python 3.9 * :gh:issue:`775` Add msvcrt to the default module deny list diff --git a/docs/conf.py b/docs/conf.py index 54e3a5c7..80607694 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,9 @@ -import os import sys sys.path.append('..') sys.path.append('.') import mitogen -VERSION = '%s.%s.%s' % mitogen.__version__ +VERSION = '.'.join(str(part) for part in mitogen.__version__) author = u'Network Genomics' copyright = u'2021, the Mitogen authors' diff --git a/docs/requirements.txt b/docs/requirements.txt index 3c4674fd..a2894c82 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ +docutils<0.18 +Jinja2<3 +MarkupSafe<2.1 Sphinx==2.1.2; python_version > '3.0' sphinxcontrib-programoutput==0.14; python_version > '3.0' alabaster==0.7.10; python_version > '3.0' diff --git a/mitogen/service.py b/mitogen/service.py index 0e5f6419..7fde9013 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -752,10 +752,12 @@ class PushFileService(Service): One size fits all method to ensure a target context has been preloaded with a set of small files and Python modules. - overridden_sources: optional dict containing source code to override path's source code - extra_sys_paths: loads additional sys paths for use in finding modules; beneficial - in situations like loading Ansible Collections because source code - dependencies come from different file paths than where the source lives + :param dict overridden_sources: + Optional dict containing source code to override path's source code + :param extra_sys_paths: + Loads additional sys paths for use in finding modules; beneficial + in situations like loading Ansible Collections because source code + dependencies come from different file paths than where the source lives """ for path in paths: overridden_source = None diff --git a/tox.ini b/tox.ini index 5bfb7fe9..e4c26c39 100644 --- a/tox.ini +++ b/tox.ini @@ -139,7 +139,9 @@ whitelist_externals = echo [testenv:docs] -basepython = python3 +basepython = python3.8 changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +deps = + -r docs/requirements.txt From 63457b4866b5017b9ec4cd664d73aecad0bac5e7 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 15 Aug 2023 14:22:50 +0100 Subject: [PATCH 08/32] docs: Update external URLs (e.g. dw/mitogen -> mitogen-hq/mitogen) Found with sphinx-build -b linkcheck. Not all flagged URLs have been changed, e.g. Ansible plugins, deleted Github users. --- docs/_templates/github.html | 2 +- docs/ansible_detailed.rst | 8 ++++---- docs/changelog.rst | 11 +++++------ docs/conf.py | 16 ++++++++-------- docs/index.rst | 2 +- mitogen/parent.py | 2 +- 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/docs/_templates/github.html b/docs/_templates/github.html index bb2b5ee5..e6ed304a 100644 --- a/docs/_templates/github.html +++ b/docs/_templates/github.html @@ -1,4 +1,4 @@


-Star +Star

diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index a95575ee..5679537e 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -18,7 +18,7 @@ The extension is considered stable and real-world use is encouraged. .. _Ansible: https://www.ansible.com/ -.. _Bug reports: https://goo.gl/yLKZiJ +.. _Bug reports: https://github.com/mitogen-hq/mitogen/issues/new/choose Overview @@ -1293,7 +1293,7 @@ Sample Profiles --------------- The summaries below may be reproduced using data and scripts maintained in the -`pcaps branch `_. Traces were +`pcaps branch `_. Traces were recorded using Ansible 2.5.14. @@ -1302,7 +1302,7 @@ Trivial Loop: Local Host This demonstrates Mitogen vs. SSH pipelining to the local machine running `bench/loop-100-items.yml -`_, +`_, executing a simple command 100 times. Most Ansible controller overhead is isolated, characterizing just module executor and connection layer performance. Mitogen requires **63x less bandwidth and 5.9x less time**. @@ -1330,7 +1330,7 @@ File Transfer: UK to France ~~~~~~~~~~~~~~~~~~~~~~~~~~~ `This playbook -`_ +`_ was used to compare file transfer performance over a ~26 ms link. It uses the ``with_filetree`` loop syntax to copy a directory of 1,000 0-byte files to the target. diff --git a/docs/changelog.rst b/docs/changelog.rst index dde66319..ef26a047 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,7 +15,7 @@ Release Notes To avail of fixes in an unreleased version, please download a ZIP file -`directly from GitHub `_. +`directly from GitHub `_. Unreleased ---------- @@ -71,7 +71,7 @@ v0.3.0 (2021-11-24) ------------------- This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+. -`See here for details `_. +`See here for details `_. * :gh:issue:`827` NewStylePlanner: detect `ansible_collections` imports * :gh:issue:`770` better check for supported Ansible version @@ -178,7 +178,7 @@ Mitogen for Ansible :linux:man7:`unix` sockets across privilege domains. * :gh:issue:`467`: an incompatibility running Mitogen under `Molecule - `_ was resolved. + `_ was resolved. * :gh:issue:`547`, :gh:issue:`598`: fix a deadlock during initialization of connections, ``async`` tasks, tasks using custom :mod:`module_utils`, @@ -1230,9 +1230,8 @@ Core Library parameter may specify an argument vector prefix rather than a string program path. -* :gh:issue:`300`: the broker could crash on - OS X during shutdown due to scheduled `kqueue - `_ filter changes for +* :gh:issue:`300`: the broker could crash on OS X during shutdown due to + scheduled :freebsd:man2:`kqueue` filter changes for descriptors that were closed before the IO loop resumes. As a temporary workaround, kqueue's bulk change feature is not used. diff --git a/docs/conf.py b/docs/conf.py index 80607694..8350c79e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,15 +43,15 @@ version = VERSION domainrefs = { 'gh:commit': { 'text': '%s', - 'url': 'https://github.com/dw/mitogen/commit/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/commit/%s', }, 'gh:issue': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/issues/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/issues/%s', }, 'gh:pull': { 'text': '#%s', - 'url': 'https://github.com/dw/mitogen/pull/%s', + 'url': 'https://github.com/mitogen-hq/mitogen/pull/%s', }, 'ans:mod': { 'text': '%s module', @@ -63,23 +63,23 @@ domainrefs = { }, 'freebsd:man2': { 'text': '%s(2)', - 'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s', + 'url': 'https://man.freebsd.org/cgi/man.cgi?query=%s', }, 'linux:man1': { 'text': '%s(1)', - 'url': 'http://man7.org/linux/man-pages/man1/%s.1.html', + 'url': 'https://man7.org/linux/man-pages/man1/%s.1.html', }, 'linux:man2': { 'text': '%s(2)', - 'url': 'http://man7.org/linux/man-pages/man2/%s.2.html', + 'url': 'https://man7.org/linux/man-pages/man2/%s.2.html', }, 'linux:man3': { 'text': '%s(3)', - 'url': 'http://man7.org/linux/man-pages/man3/%s.3.html', + 'url': 'https://man7.org/linux/man-pages/man3/%s.3.html', }, 'linux:man7': { 'text': '%s(7)', - 'url': 'http://man7.org/linux/man-pages/man7/%s.7.html', + 'url': 'https://man7.org/linux/man-pages/man7/%s.7.html', }, } diff --git a/docs/index.rst b/docs/index.rst index d33cf29f..32083db0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,7 +26,7 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or `Fabric`_ can be built, and while the API is quite friendly and comparable to `Fabric`_, ultimately it is not intended for direct use by consumer software. -.. _Salt: https://docs.saltstack.com/en/latest/ +.. _Salt: https://docs.saltproject.io/en/latest/ .. _Ansible: https://docs.ansible.com/ .. _Fabric: https://www.fabfile.org/ diff --git a/mitogen/parent.py b/mitogen/parent.py index 32aa3cb6..59ee1685 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1027,7 +1027,7 @@ class KqueuePoller(mitogen.core.Poller): class EpollPoller(mitogen.core.Poller): """ - Poller based on the Linux :linux:man2:`epoll` interface. + Poller based on the Linux :linux:man7:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') _repr = 'EpollPoller()' From 3f105d516995fb9c20de4e0ebedcf4f045b992e9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 5 Oct 2023 13:26:21 +0100 Subject: [PATCH 09/32] ci: Authenticate UsePythonVersion requests to Github This should address the warning in Azure Pipelines > You should provide GitHub token if you want to download a python release. > Otherwise you may hit the GitHub anonymous download limit. The token is provided from a secret variable in the pipeline. --- .ci/azure-pipelines-steps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index b7b094e3..ed516d72 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -9,6 +9,7 @@ steps: - task: UsePythonVersion@0 displayName: Install python inputs: + githubToken: '$(GITHUB_PYVER_TOKEN)' versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') From e580258071d947964ef652ff99f908398f47b1b7 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 14 Nov 2023 14:05:25 +0000 Subject: [PATCH 10/32] docs: Bypass networkgenomics.com/try/ -> PyPI redirect refs #1028 --- docs/conf.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 8350c79e..5b930ab4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -83,10 +83,16 @@ domainrefs = { }, } +# > ## Official guidance +# > Query PyPI’s JSON API to determine where to download files from. +# > ## Predictable URLs +# > You can use our conveyor service to fetch this file, which exists for +# > cases where using the API is impractical or impossible. +# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls rst_epilog = """ .. |mitogen_version| replace:: %(VERSION)s -.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz `__ +.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz `__ """ % locals() From b7188c1cadaa2027f8e2be7581716563d45b42a1 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 14 Nov 2023 14:09:21 +0000 Subject: [PATCH 11/32] docs: Decouple website download version from package version This prevents unreleased versions appearing on the website (e.g. 0.3.5.dev0), but introduces the risk of forgetting to update the website after a release. A better fix requires deeper design/workflow thought. refs #1028 --- docs/conf.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5b930ab4..3a7fc002 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,8 @@ import sys -sys.path.append('..') sys.path.append('.') -import mitogen -VERSION = '.'.join(str(part) for part in mitogen.__version__) + +VERSION = '0.3.4' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' From f9a6748154f0ae48eadc4e4f1e2c4f732acf29f7 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 14 Nov 2023 14:45:28 +0000 Subject: [PATCH 12/32] ci: Fix Python 2.7 builds on macOS 11 With current macOS 11 runner images (20231216.1) the `python` on `$PATH` is Python 3.12 and setuptools isn't installed by default. E.g. ``` python -mtox -e "py27-mode_localhost-ansible4" ========================== Starting Command Output =========================== /bin/bash --noprofile --norc /Users/runner/work/_temp/93a29c4c-f606-45e4-8dbd-a4a5f51b8730.sh GLOB sdist-make: /Users/runner/work/1/s/setup.py ERROR: invocation failed (exit code 1), logfile: /Users/runner/work/1/s/.tox/log/GLOB-0.log ================================== log start =================================== Traceback (most recent call last): File "/Users/runner/work/1/s/setup.py", line 32, in from setuptools import find_packages, setup ModuleNotFoundError: No module named 'setuptools' ``` Installing setuptools under Python 3.12 chooses package versions incompatible with Python 2.7. Additionally Mitogen isn't yet compatible with Python 3.12 (#1033), so tests that call a local context with `python` fail. --- .ci/azure-pipelines-steps.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index ed516d72..88d219b8 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -13,21 +13,33 @@ steps: versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') -- script: | - type python - python --version - displayName: Show python version - - script: | sudo apt-get update sudo apt-get install -y python2-dev python3-pip virtualenv displayName: Install build deps condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) -- script: python -mpip install "tox<4.0" +- script: | + type python && python --version + type python2 && python2 --version + type python3 && python3 --version + displayName: Show python versions + +- script: | + if [[ $(uname) == "Darwin" ]]; then + python2 -m ensurepip --user --altinstall --no-default-pip + python2 -m pip install --user "tox<4.0" + else + python -m pip install "tox<4.0" + fi displayName: Install tooling -- script: python -mtox -e "$(tox.env)" +- script: | + if [[ $(uname) == "Darwin" ]]; then + python2 -m tox -e "$(tox.env)" + else + python -m tox -e "$(tox.env)" + fi displayName: "Run tests" env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) From fc3e788cb4d3c70e374e25c15320c1c20107265c Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 30 Jan 2024 14:06:34 +0000 Subject: [PATCH 13/32] non functional: Add comments about imp module removal in Python 3.12 --- ansible_mitogen/module_finder.py | 7 +++++++ ansible_mitogen/runner.py | 1 + mitogen/core.py | 13 ++++++++++++- mitogen/master.py | 7 +++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index cec465c1..19dcef7f 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -40,6 +40,13 @@ import mitogen.master PREFIX = 'ansible.module_utils.' +# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`. +# name Unqualified name of the module. +# path Filesystem path of the module. +# kind One of the constants in `imp`, as returned in `imp.find_module()` +# parent `ansible_mitogen.module_finder.Module` of parent package (if any). +# +# FIXME Python 3.12 removed `imp`, leaving no constants for `Module.kind`. Module = collections.namedtuple('Module', 'name path kind parent') diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index c4cb71ff..6986ddcd 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -521,6 +521,7 @@ class ModuleUtilsImporter(object): path, is_pkg = self._by_fullname[fullname] source = ansible_mitogen.target.get_small_file(self._context, path) code = compile(source, path, 'exec', 0, 1) + # FIXME Python 3.12 removed `imp` mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = "master:%s" % (path,) mod.__loader__ = self diff --git a/mitogen/core.py b/mitogen/core.py index bee722e6..707e901a 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1360,12 +1360,20 @@ class Importer(object): if fullname == '__main__': raise ModuleNotFoundError() + # For a module inside a package (e.g. pkg_a.mod_b) use the search path + # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']). parent, _, modname = str_rpartition(fullname, '.') if parent: path = sys.modules[parent].__path__ else: path = None + # For a top-level module search builtin modules, frozen modules, + # system specific locations (e.g. Windows registry, site-packages). + # Otherwise use search path of the parent package. + # Works for both stdlib modules & third-party modules. + # If the search is unsuccessful then raises ImportError. + # FIXME Python 3.12 removed `imp`. fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() @@ -1377,8 +1385,9 @@ class Importer(object): Implements importlib.abc.MetaPathFinder.find_module(). Deprecrated in Python 3.4+, replaced by find_spec(). Raises ImportWarning in Python 3.10+. + Removed in Python 3.12. - fullname A (fully qualified?) module name, e.g. "os.path". + fullname Fully qualified module name, e.g. "os.path". path __path__ of parent packge. None for a top level module. """ if hasattr(_tls, 'running'): @@ -1521,6 +1530,7 @@ class Importer(object): raise ModuleNotFoundError(self.absent_msg % (fullname,)) pkg_present = ret[1] + # FIXME Python 3.12 removed `imp` mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = self.get_filename(fullname) mod.__loader__ = self @@ -3921,6 +3931,7 @@ class ExternalContext(object): def _setup_package(self): global mitogen + # FIXME Python 3.12 removed `imp` mitogen = imp.new_module('mitogen') mitogen.__package__ = 'mitogen' mitogen.__path__ = [] diff --git a/mitogen/master.py b/mitogen/master.py index 4fb535f0..8cd1d27c 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -123,6 +123,10 @@ def is_stdlib_name(modname): Return :data:`True` if `modname` appears to come from the standard library. """ # `imp.is_builtin()` isn't a documented as part of Python's stdlib API. + # Returns 1 if modname names a module that is "builtin" to the the Python + # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces'). + # FIXME Python 3.12 removed `imp`, but `_imp.is_builtin()` remains. + # `sys.builtin_module_names` (Python 2.2+) may be an alternative. # # """ # Main is a little special - imp.is_builtin("__main__") will return False, @@ -759,6 +763,7 @@ class ParentEnumerationMethod(FinderMethod): def _find_one_component(self, modname, search_path): try: #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) + # FIXME The imp module was removed in Python 3.12. return imp.find_module(modname, search_path) except ImportError: e = sys.exc_info()[1] @@ -787,11 +792,13 @@ class ParentEnumerationMethod(FinderMethod): # Still more components to descent. Result must be a package if fp: fp.close() + # FIXME The imp module was removed in Python 3.12. if kind != imp.PKG_DIRECTORY: LOG.debug('%r: %r appears to be child of non-package %r', self, fullname, path) return None search_path = [path] + # FIXME The imp module was removed in Python 3.12. elif kind == imp.PKG_DIRECTORY: return self._found_package(fullname, path) else: From 9a9dd66ba0b09dde3742025502da32d9406f117e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 1 Feb 2024 15:52:18 +0000 Subject: [PATCH 14/32] Ignore Ansible retry files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index aa75f691..7297d720 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ venvs/** *.pyc *.pyd *.pyo +*.retry MANIFEST build/ dist/ From bde7f062b9faf6b06c6529c7cae95702d7c8e5e3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 13:53:38 +0000 Subject: [PATCH 15/32] tests: Fix Ansible module shebangs With https://github.com/ansible/ansible/pull/76677 Ansible fixed shebang substitution for Ansible modules and tightened up what shebang is allowed. Changing these fixes the tests using them with vanilla Ansible. https://docs.ansible.com/ansible/latest/dev_guide/testing/sanity/shebang.html --- tests/ansible/lib/modules/custom_python_detect_environment.py | 2 +- tests/ansible/lib/modules/custom_python_external_module.py | 2 +- tests/ansible/lib/modules/custom_python_external_pkg.py | 2 +- tests/ansible/lib/modules/custom_python_json_args_module.py | 2 +- tests/ansible/lib/modules/custom_python_leaky_class_vars.py | 2 +- tests/ansible/lib/modules/custom_python_modify_environ.py | 2 +- tests/ansible/lib/modules/custom_python_new_style_module.py | 2 +- tests/ansible/lib/modules/custom_python_os_getcwd.py | 2 +- tests/ansible/lib/modules/custom_python_prehistoric_module.py | 3 ++- tests/ansible/lib/modules/custom_python_run_script.py | 2 +- tests/ansible/lib/modules/custom_python_uses_distro.py | 2 +- tests/ansible/lib/modules/custom_python_want_json_module.py | 2 +- tests/ansible/lib/modules/mitogen_test_gethostbyname.py | 2 +- tests/ansible/lib/modules/test_echo_module.py | 2 +- 14 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index c7a222e7..65f660a8 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I return details about the Python # interpreter I run within. diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py index ae1b78cb..507e53dd 100644 --- a/tests/ansible/lib/modules/custom_python_external_module.py +++ b/tests/ansible/lib/modules/custom_python_external_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I expect the quote from modules2/module_utils/joker.py. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py index be9acb24..95bd0c7b 100644 --- a/tests/ansible/lib/modules/custom_python_external_pkg.py +++ b/tests/ansible/lib/modules/custom_python_external_pkg.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.externalpkg import extmod diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 61640579..846037ec 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py index 255e3729..1d342329 100644 --- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py +++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I leak state from each invocation # into a class variable and a global variable. diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py index 51b74526..9767f855 100644 --- a/tests/ansible/lib/modules/custom_python_modify_environ.py +++ b/tests/ansible/lib/modules/custom_python_modify_environ.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I modify the process environment and # don't clean up after myself. diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index 1e7270cd..f9c176c1 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I should receive an encoding string. import sys diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index c5e264ae..d465ac9e 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py index 0cf9774d..2268cbe6 100644 --- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py +++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/python + # issue #555: I'm a module that cutpastes an old hack. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py index d6a839ae..4a6243d0 100644 --- a/tests/ansible/lib/modules/custom_python_run_script.py +++ b/tests/ansible/lib/modules/custom_python_run_script.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I run the script provided in the # parameter. diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py index 03f3b6aa..6b3a356b 100644 --- a/tests/ansible/lib/modules/custom_python_uses_distro.py +++ b/tests/ansible/lib/modules/custom_python_uses_distro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # issue #590: I am an Ansible new-style Python module that tries to use # ansible.module_utils.distro. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index f5e33862..23eeeb55 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py index 1b80a48b..289e9662 100644 --- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py +++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am a module that indirectly depends on glibc cached /etc/resolv.conf state. diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 1f71e879..37ab655c 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # -*- coding: utf-8 -*- # (c) 2012, Michael DeHaan From 8b574f234dd60ba3f3e23e3030e194e951919633 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 14:02:12 +0000 Subject: [PATCH 16/32] tests: Report Ansible controller parameters before image prep & user creation --- tests/ansible/setup/all.yml | 3 ++- tests/ansible/setup/report_controller.yml | 17 +++++++++++++++++ .../setup/{report.yml => report_targets.yml} | 7 +------ tests/image_prep/_user_accounts.yml | 2 ++ 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 tests/ansible/setup/report_controller.yml rename tests/ansible/setup/{report.yml => report_targets.yml} (73%) diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml index 2ca6b97c..8903494d 100644 --- a/tests/ansible/setup/all.yml +++ b/tests/ansible/setup/all.yml @@ -1 +1,2 @@ -- import_playbook: report.yml +- import_playbook: report_controller.yml +- import_playbook: report_targets.yml diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml new file mode 100644 index 00000000..d0d5cc15 --- /dev/null +++ b/tests/ansible/setup/report_controller.yml @@ -0,0 +1,17 @@ +- name: Report controller parameters + hosts: localhost + gather_facts: false + tasks: + - debug: + msg: + - ${ANSIBLE_STRATEGY}: "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('') }}" + - ${USER}: "{{ lookup('env', 'USER') | default('') }}" + - $(groups): "{{ lookup('pipe', 'groups') }}" + - $(pwd): "{{ lookup('pipe', 'pwd') }}" + - $(whoami): "{{ lookup('pipe', 'whoami') }}" + - ansible_run_tags: "{{ ansible_run_tags | default('') }}" + - ansible_playbook_python: "{{ ansible_playbook_python | default('') }}" + - ansible_skip_tags: "{{ ansible_skip_tags | default('') }}" + - ansible_version.full: "{{ ansible_version.full | default('') }}" + - is_mitogen: "{{ is_mitogen | default('') }}" + - playbook_dir: "{{ playbook_dir | default('') }}" diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report_targets.yml similarity index 73% rename from tests/ansible/setup/report.yml rename to tests/ansible/setup/report_targets.yml index 450e4fb0..5aa67124 100644 --- a/tests/ansible/setup/report.yml +++ b/tests/ansible/setup/report_targets.yml @@ -1,4 +1,4 @@ -- name: Report runtime settings +- name: Report target facts hosts: localhost:test-targets gather_facts: true tasks: @@ -13,8 +13,3 @@ - debug: {var: ansible_facts.osversion} - debug: {var: ansible_facts.python} - debug: {var: ansible_facts.system} - - debug: {var: ansible_forks} - - debug: {var: ansible_run_tags} - - debug: {var: ansible_skip_tags} - - debug: {var: ansible_version.full} - - debug: {var: is_mitogen} diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 6224b61a..9472d099 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -4,6 +4,8 @@ # WARNING: this creates non-privilged accounts with pre-set passwords! # +- import_playbook: ../ansible/setup/report_controller.yml + - hosts: all gather_facts: true strategy: mitogen_free From a6c89751f90fdf1f581385193b182bfb68575e4a Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 14:04:28 +0000 Subject: [PATCH 17/32] tests: Cleanup ansible-lint errors & warnings in user creation playbook Task " Install slow profile for one account" removed because it duplicates earlier work. --- tests/image_prep/_user_accounts.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 9472d099..0b6d5e61 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -39,12 +39,12 @@ normal_users: "{{ lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) - }}" + }}" all_users: "{{ special_users + normal_users - }}" + }}" tasks: - name: Disable non-localhost SSH for Mitogen users when: false @@ -102,6 +102,7 @@ with_items: "{{all_users}}" copy: dest: /var/lib/AccountsService/users/mitogen__{{item}} + mode: u=rw,go= content: | [User] SystemAccount=true @@ -110,7 +111,7 @@ when: ansible_system == 'Linux' and out.stat.exists service: name: accounts-daemon - restarted: true + state: restarted - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" @@ -119,6 +120,9 @@ copy: dest: ~mitogen__slow_user/.{{item}} src: ../data/docker/mitogen__slow_user.profile + owner: mitogen__slow_user + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -127,6 +131,9 @@ copy: dest: ~mitogen__permdenied/.{{item}} src: ../data/docker/mitogen__permdenied.profile + owner: mitogen__permdenied + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -138,20 +145,13 @@ state: directory mode: go= owner: mitogen__has_sudo_pubkey + group: mitogen__group - copy: dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys src: ../data/docker/mitogen__has_sudo_pubkey.key.pub mode: go= owner: mitogen__has_sudo_pubkey - - - name: Install slow profile for one account - block: - - copy: - dest: ~mitogen__slow_user/.profile - src: ../data/docker/mitogen__slow_user.profile - - copy: - dest: ~mitogen__slow_user/.bashrc - src: ../data/docker/mitogen__slow_user.profile + group: mitogen__group - name: Require a TTY for two accounts lineinfile: From 591152bef0beb0b4df483ac553aa782ebb669a51 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 14:15:47 +0000 Subject: [PATCH 18/32] tests: Avoid intermittant 2 hour timeout in new style Ansible module tests This has been lurking for years, raising it's head at unpredictable times. This change doesn't fix it, but it should make it a lot less mysterious. --- ...m_python_new_style_missing_interpreter.yml | 5 ++ .../runner/custom_python_new_style_module.yml | 3 +- ...om_python_new_style_missing_interpreter.py | 49 +++++++++++++++++++ .../modules/custom_python_new_style_module.py | 49 +++++++++++++++++++ 4 files changed, 105 insertions(+), 1 deletion(-) diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index 0c620dac..0d7cf1b6 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -2,6 +2,11 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets tasks: + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. + - meta: end_play + when: not is_mitogen + - custom_python_new_style_missing_interpreter: foo: true with_sequence: start=0 end={{end|default(1)}} diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index e2384f81..8435b158 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -1,7 +1,8 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets tasks: - # without Mitogen Ansible 2.10 hangs on this play + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. - meta: end_play when: not is_mitogen diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 2e0ef0da..728685f4 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -1,6 +1,20 @@ # I am an Ansible new-style Python module, but I lack an interpreter. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980 # NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10 @@ -8,11 +22,46 @@ import sys # from ansible.module_utils. # import ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index f9c176c1..c84d241a 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -1,16 +1,65 @@ #!/usr/bin/python # I am an Ansible new-style Python module. I should receive an encoding string. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # This is the magic marker Ansible looks for: # from ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") From adfd4e17f3f378c859cfb49feebcd8a721f62bb0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 14:21:16 +0000 Subject: [PATCH 19/32] tests: Declare inventory file types to Visual Studio Code and Vim Works with the VS Code modeline extension. Enables syntax highlighting. --- tests/ansible/hosts/become_same_user.hosts | 2 ++ tests/ansible/hosts/connection_delegation.hosts | 1 + tests/ansible/hosts/default.hosts | 5 ++++- tests/ansible/hosts/k3.hosts | 1 + tests/ansible/hosts/localhost.hosts | 1 + tests/ansible/hosts/transport_config.hosts | 3 +++ 6 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts index a18b90d2..ac744ed7 100644 --- a/tests/ansible/hosts/become_same_user.hosts +++ b/tests/ansible/hosts/become_same_user.hosts @@ -1,3 +1,5 @@ +# code: language=ini +# vim: syntax=dosini # become_same_user.yml bsu-joe ansible_user=joe diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts index a22bd5df..4ae861b0 100644 --- a/tests/ansible/hosts/connection_delegation.hosts +++ b/tests/ansible/hosts/connection_delegation.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # Connection delegation scenarios. It's impossible to connect to them, but their would-be diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index d40c3dd0..1bec0014 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -1,9 +1,12 @@ +# code: language=ini # vim: syntax=dosini # When running the tests outside CI, make a single 'target' host which is the # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. -target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" +# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022 +# and os.environ['USER'] is not populated on Azure macOS runners. +target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] target diff --git a/tests/ansible/hosts/k3.hosts b/tests/ansible/hosts/k3.hosts index 34e1ff95..b210164b 100644 --- a/tests/ansible/hosts/k3.hosts +++ b/tests/ansible/hosts/k3.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # Used for manual testing. diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts index 41af412e..e42221e7 100644 --- a/tests/ansible/hosts/localhost.hosts +++ b/tests/ansible/hosts/localhost.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # issue #511, #536: we must not define an explicit localhost, as some diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 7d7b526a..32f5f61b 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -1,3 +1,6 @@ +# code: language=ini +# vim: syntax=dosini + # integration/transport_config # Hosts with twiddled configs that need to be checked somehow. From 2839954559403ffb18d310449a179abea4b892c6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 14:43:52 +0000 Subject: [PATCH 20/32] tests: Account for /tmp symlink in virtualenv test on macOS --- .../lib/modules/custom_python_detect_environment.py | 5 +++++ .../regression/issue_152__virtualenv_python_fails.yml | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 65f660a8..d2ceaf0a 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -25,6 +25,11 @@ except NameError: def main(): module = AnsibleModule(argument_spec={}) module.exit_json( + fs={ + '/tmp': { + 'resolved': os.path.realpath('/tmp'), + }, + }, python={ 'version': { 'full': '%i.%i.%i' % sys.version_info[:3], diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index f4c47aba..610eaf33 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -23,9 +23,16 @@ when: - lout.python.version.full is version('2.7', '>=', strict=True) - - assert: + - name: Check virtualenv was used + # On macOS runners a symlink /tmp -> /private/tmp has been seen + vars: + requested_executable: /tmp/issue_152_virtualenv/bin/python + expected_executables: + - "{{ requested_executable }}" + - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}" + assert: that: - - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python" + - out.sys_executable in expected_executables fail_msg: out={{out}} when: - lout.python.version.full is version('2.7', '>=', strict=True) From a6a5c5bb97388bac586aea8007064144d888d87b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 14:48:19 +0000 Subject: [PATCH 21/32] tests: Clarify status/purpose of Python 2.x era Ansible Module workaround --- .../integration/runner/custom_python_prehistoric_module.yml | 6 ++++++ .../ansible/lib/modules/custom_python_prehistoric_module.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml index f2a3eefd..ebe34cc8 100644 --- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml +++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml @@ -1,3 +1,7 @@ +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, if any - reload() is a Python 2.x builtin function. # issue #555 - name: integration/runner/custom_python_prehistoric_module.yml @@ -5,9 +9,11 @@ tasks: - custom_python_prehistoric_module: register: out + when: is_mitogen - assert: that: out.ok fail_msg: out={{out}} + when: is_mitogen tags: - custom_python_prehistoric_module diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py index 2268cbe6..61397488 100644 --- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py +++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py @@ -1,5 +1,9 @@ #!/usr/bin/python +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, reload() is a Python 2.x builtin function. # issue #555: I'm a module that cutpastes an old hack. from ansible.module_utils.basic import AnsibleModule From c2ad52e54eaeef48b00fc4ba7cc4f5eed22f6482 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 15:04:44 +0000 Subject: [PATCH 22/32] tests: Fix tests using get_url across Python versions Using https:// requires certificate store management and additional parameter passing that changed across Ansible and Python versions. Using http:// allows the same tests to be used across wider spans of Python version on the controller, and Python verison on the targets. Python 3.12 on a target + get_uri needs Ansible >= 8 (ansible-core >= 2.15). Python 3.12 removed deprecated httplib.HTTPSConnection() arguments. https://github.com/ansible/ansible/pull/80751 --- .../interpreter_discovery/complex_args.yml | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index 38d10124..6ffff5f4 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -4,6 +4,10 @@ - name: integration/interpreter_discovery/complex_args.yml hosts: test-targets gather_facts: true + environment: + http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}" + https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" + no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" tasks: - name: create temp file to source file: @@ -21,28 +25,24 @@ # special_python: source /tmp/fake && python - name: set python using sourced file set_fact: - special_python: source /tmp/fake || true && python + # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of). + special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}" - name: run get_url with specially-sourced python get_url: - url: https://google.com + # Plain http for wider Ansible & Python version compatibility + url: http://httpbin.org/get dest: "/tmp/" mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no vars: ansible_python_interpreter: "{{ special_python }}" - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" - name: run get_url with specially-sourced python including jinja get_url: - url: https://google.com + # Plain http for wider Ansible & Python version compatibility + url: http://httpbin.org/get dest: "/tmp/" mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no vars: ansible_python_interpreter: > {% if "1" == "1" %} @@ -50,8 +50,5 @@ {% else %} python {% endif %} - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" tags: - complex_args From e2f4d9275c0d3627a2c2616710d291dbb344e359 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 15:40:14 +0000 Subject: [PATCH 23/32] tests: Fix ansible_python_interpreter & discovered_interpreter_python tests on macOS Should account for fiddling in mitogen.parent.Connection._first_stage() and symlinks. I won't be surprised if it breaks again soon and often. --- mitogen/parent.py | 6 ++- .../ansible_2_8_tests.yml | 14 +++++-- tests/ansible/lib/modules/test_echo_module.py | 41 +++++++++++++++---- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 59ee1685..1045ddc3 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1409,6 +1409,9 @@ class Connection(object): # their respective values. # * CONTEXT_NAME must be prefixed with the name of the Python binary in # order to allow virtualenvs to detect their install prefix. + # + # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible + # module test_echo_module, used by the integration tests. # * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version # switcher as /usr/bin/python, which introspects argv0. To workaround # it we redirect attempts to call /usr/bin/python with an explicit @@ -1417,7 +1420,8 @@ class Connection(object): # do something slightly different. The Python executable is patched to # perform an extra execvp(). I don't fully understand the details, but # setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it. - # * macOS 13.x (Darwin 22?) may remove python 2.x entirely. + # * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python. + # https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python # # Locals: # R: read side of interpreter stdin. diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index ea828aa8..5b695c42 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -37,6 +37,7 @@ vars: ansible_python_interpreter: auto test_echo_module: + facts_copy: "{{ ansible_facts }}" register: echoout # can't test this assertion: @@ -44,11 +45,15 @@ # because Mitogen's ansible_python_interpreter is a connection-layer configurable that # "must be extracted during each task execution to form the complete connection-layer configuration". # Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered - - assert: + - name: assert discovered python matches invoked python + assert: that: - auto_out.ansible_facts.discovered_interpreter_python is defined - - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python - fail_msg: auto_out={{auto_out}} echoout={{echoout}} + - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen + - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved + fail_msg: + - "auto_out: {{ auto_out }}" + - "echoout: {{ echoout }}" - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result @@ -128,7 +133,8 @@ - name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter block: - test_echo_module: - facts: + facts_copy: "{{ ansible_facts }}" + facts_to_override: ansible_discovered_interpreter_bogus: from module discovered_interpreter_bogus: from_module ansible_bogus_interpreter: from_module diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 37ab655c..d44b85ab 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -9,28 +9,51 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +import os import platform import sys from ansible.module_utils.basic import AnsibleModule def main(): - result = dict(changed=False) - module = AnsibleModule(argument_spec=dict( - facts=dict(type=dict, default={}) + facts_copy=dict(type=dict, default={}), + facts_to_override=dict(type=dict, default={}) )) - result['ansible_facts'] = module.params['facts'] # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info - if sys.platform == 'darwin' and sys.executable != '/usr/bin/python': - if int(platform.release()[:2]) < 19: - sys.executable = sys.executable[:-3] - else: + # NB This must be synced with mitogen.parent.Connection.get_boot_command() + if sys.modules.get('mitogen') and sys.platform == 'darwin': + darwin_major = int(platform.release().partition('.')[0]) + if darwin_major < 19 and sys.executable == '/usr/bin/python2.7': + sys.executable = '/usr/bin/python' + if darwin_major in (20, 21) and sys.version_info[:2] == (2, 7): # only for tests to check version of running interpreter -- Mac 10.15+ changed python2 # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python sys.executable = "/usr/bin/python" - result['running_python_interpreter'] = sys.executable + + facts_copy = module.params['facts_copy'] + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + result = { + 'changed': False, + 'ansible_facts': module.params['facts_to_override'], + 'discovered_and_running_samefile': os.path.samefile( + os.path.realpath(discovered_interpreter_python), + os.path.realpath(sys.executable), + ), + 'discovered_python': { + 'as_seen': discovered_interpreter_python, + 'resolved': os.path.realpath(discovered_interpreter_python), + }, + 'running_python': { + 'sys': { + 'executable': { + 'as_seen': sys.executable, + 'resolved': os.path.realpath(sys.executable), + }, + }, + }, + } module.exit_json(**result) From 2973d906700f81b033de5c9629b62b85349b5601 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 16:02:18 +0000 Subject: [PATCH 24/32] tests: Enable su tests under vanilla Ansible >= 2.11 cwd_show was useful when debugging these tests, worth keeping around. --- tests/ansible/files/cwd_show | 22 ++++++++++ .../integration/become/su_password.yml | 41 +++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) create mode 100755 tests/ansible/files/cwd_show diff --git a/tests/ansible/files/cwd_show b/tests/ansible/files/cwd_show new file mode 100755 index 00000000..42ef3194 --- /dev/null +++ b/tests/ansible/files/cwd_show @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Show permissions and identities that impact the current working directory. +# On macOS libc cwd() can return EACCES after su or sudo. +# See also +# - https://github.com/ansible/ansible/pull/7078 +# - https://github.com/python/cpython/issues/115911 + +set -o errexit +set -o nounset +set -o pipefail + +whoami +groups +pwd + +d=$(pwd) +while [[ "$d" != "/" && -n "$d" ]]; do + ls -ld "$d" + d=$(dirname "$d") +done +ls -ld / diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index bd6a0aee..52d420db 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -1,5 +1,4 @@ # Verify passwordful su behaviour - # Ansible can't handle this on OS X. I don't care why. - name: integration/become/su_password.yml hosts: test-targets @@ -44,20 +43,54 @@ fail_msg: out={{out}} when: is_mitogen - - name: Ensure password su succeeds. + - name: Ensure password su with chdir succeeds shell: whoami + args: + chdir: ~mitogen__user1 become: true become_user: mitogen__user1 register: out vars: ansible_become_pass: user1_password - when: is_mitogen + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} - when: is_mitogen + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + + - name: Ensure password su without chdir succeeds + shell: whoami + become: true + become_user: mitogen__user1 + register: out + vars: + ansible_become_pass: user1_password + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + + - assert: + that: + - out.stdout == 'mitogen__user1' + fail_msg: out={{out}} + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + tags: - su - su_password From 1031551dd99ee0a2f848861c95041085a0ba7c2a Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 16:09:48 +0000 Subject: [PATCH 25/32] tests: Clarify transport config tests optimisation & correct value The ini inventory parser doesn't support comments after a value, so the value parsed was "python3000 # Not expected to exist". --- tests/ansible/hosts/transport_config.hosts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 32f5f61b..1c1c2e10 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -20,11 +20,12 @@ tc_remote_user tc_transport [transport_config_undiscover:vars] -# If python interpreter path is unset, Ansible tries to connect & discover it. -# That causes approx 10 seconds timeout per task - there's no host to connect to. +# If ansible_*_interpreter isn't set Ansible tries to connect & discover it. +# If that target doesn't exist we must wait $timeout seconds for each attempt. +# Setting a known (invalid) interpreter skips discovery & the many timeouts. # This optimisation should not be relied in any test. # Note: tc-python-path-* are intentionally not included. -ansible_python_interpreter = python3000 # Not expected to exist +ansible_python_interpreter = python3000 [tc_transport] tc-transport-unset From 3a31a7d886fa7018ecf71530dd7484f1830db4c7 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Mar 2024 16:28:17 +0000 Subject: [PATCH 26/32] mitogen: Workaround CPython importlib PermissionError when cwd is unreadable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On macOS when using a become plugin as an unprivileged user, to another unprivileged user it is likely that the current working directory can't be read. In this case os.cwd() raises PermissionError. On versions of Python currently in the wild (March 2024, CPython <= 3.13) if any non-builtin or non-frozen module (e.g. zlib, base64) is imported then `importlib._bootstrap_external.PathFinder._path_importer_cache()` attempts to call os.cwd() without catching PermissionError. The previous comment about needing an extra .encode() appears to be wrong, atleast for Python 3.x >= 3.6. Command size increased by 54 bytes, bootstrap by 804 bytes. Changed from codecs module to binascii & zlib because they're extensions, and importing them triggers fewer supporting imports (e.g. encodings module). Before ``` ✗ ./preamble_size.py SSH command size: 705 Bootstrap (mitogen.core) size: 17078 (16.68KiB) Original Minimized Compressed mitogen.parent 97884 95.6KiB 50515 49.3KiB 51.6% 12727 12.4KiB 13.0% mitogen.fork 8436 8.2KiB 4130 4.0KiB 49.0% 1648 1.6KiB 19.5% mitogen.ssh 10892 10.6KiB 6952 6.8KiB 63.8% 2113 2.1KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41699 40.7KiB 22477 22.0KiB 53.9% 5885 5.7KiB 14.1% mitogen.fakessh 15577 15.2KiB 7989 7.8KiB 51.3% 2623 2.6KiB 16.8% mitogen.master 51398 50.2KiB 25715 25.1KiB 50.0% 6886 6.7KiB 13.4% ``` After ``` ✗ ./preamble_size.py SSH command size: 759 Bootstrap (mitogen.core) size: 17882 (17.46KiB) Original Minimized Compressed mitogen.parent 98173 95.9KiB 50571 49.4KiB 51.5% 12747 12.4KiB 13.0% mitogen.fork 8436 8.2KiB 4130 4.0KiB 49.0% 1648 1.6KiB 19.5% mitogen.ssh 10892 10.6KiB 6952 6.8KiB 63.8% 2113 2.1KiB 19.4% mitogen.sudo 12089 11.8KiB 5924 5.8KiB 49.0% 2249 2.2KiB 18.6% mitogen.select 12325 12.0KiB 2929 2.9KiB 23.8% 964 0.9KiB 7.8% mitogen.service 41699 40.7KiB 22477 22.0KiB 53.9% 5885 5.7KiB 14.1% mitogen.fakessh 15577 15.2KiB 7989 7.8KiB 51.3% 2623 2.6KiB 16.8% mitogen.master 56116 54.8KiB 29427 28.7KiB 52.4% 7627 7.4KiB 13.6% ``` Fixes #885 Refs https://github.com/python/cpython/issues/115911 --- docs/changelog.rst | 2 ++ mitogen/core.py | 29 ++++++++++++++++++++++++++++- mitogen/parent.py | 21 +++++++++++---------- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ef26a047..32829ef6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ Unreleased ---------- * :gh:issue:`987` Support Python 3.11 +* :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when + becoming an unprivileged user with Python 3.x v0.3.4 (2023-07-02) diff --git a/mitogen/core.py b/mitogen/core.py index 707e901a..6a3f3da7 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -34,6 +34,34 @@ non-essential code in order to reduce its size, since it is also serves as the bootstrap implementation sent to every new slave context. """ +import sys +try: + import _frozen_importlib_external +except ImportError: + pass +else: + class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder): + """ + Meta path finder for sys.path and package __path__ attributes. + + Patched for https://github.com/python/cpython/issues/115911. + """ + @classmethod + def _path_importer_cache(cls, path): + if path == '': + try: + path = _frozen_importlib_external._os.getcwd() + except (FileNotFoundError, PermissionError): + return None + return super()._path_importer_cache(path) + + if sys.version_info[:2] <= (3, 12): + for i, mpf in enumerate(sys.meta_path): + if mpf is _frozen_importlib_external.PathFinder: + sys.meta_path[i] = MonkeyPatchedPathFinder + del i, mpf + + import binascii import collections import encodings.latin_1 @@ -49,7 +77,6 @@ import pstats import signal import socket import struct -import sys import syslog import threading import time diff --git a/mitogen/parent.py b/mitogen/parent.py index 1045ddc3..29bcf66d 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive connection. """ -import codecs +import binascii import errno import fcntl import getpass @@ -1405,6 +1405,7 @@ class Connection(object): # file descriptor 0 as 100, creates a pipe, then execs a new interpreter # with a custom argv. # * Optimized for minimum byte count after minification & compression. + # The script preamble_size.py measures this. # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with # their respective values. # * CONTEXT_NAME must be prefixed with the name of the Python binary in @@ -1449,7 +1450,7 @@ class Connection(object): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') os.write(1,'MITO000\n'.encode()) - C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') + C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN)) fp=os.fdopen(W,'wb',0) fp.write(C) fp.close() @@ -1481,16 +1482,16 @@ class Connection(object): source = source.replace('PREAMBLE_COMPRESSED_LEN', str(len(preamble_compressed))) compressed = zlib.compress(source.encode(), 9) - encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) - # We can't use bytes.decode() in 3.x since it was restricted to always - # return unicode, so codecs.decode() is used instead. In 3.x - # codecs.decode() requires a bytes object. Since we must be compatible - # with 2.4 (no bytes literal), an extra .encode() either returns the - # same str (2.x) or an equivalent bytes (3.x). + encoded = binascii.b2a_base64(compressed).replace(b('\n'), b('')) + + # Just enough to decode, decompress, and exec the first stage. + # Priorities: wider compatibility, faster startup, shorter length. + # `import os` here, instead of stage 1, to save a few bytes. + # `sys.path=...` for https://github.com/python/cpython/issues/115911. return self.get_python_argv() + [ '-c', - 'import codecs,os,sys;_=codecs.decode;' - 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),) + 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;' + 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),), ] def get_econtext_config(self): From 5ad3d14ceb758de7928897a66b235247aee6e1ff Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 17 Mar 2024 14:55:15 +0000 Subject: [PATCH 27/32] mitogen: Support PEP 451 ModuleSpec API, required for Python 3.12 importlib.machinery.ModuleSpec and find_spec() were introduced in Python 3.4 under PEP 451. They replace the find_module() API of PEP 302, which was deprecated from Python 3.4. They were removed in Python 3.12 along with the imp module. This change adds support for the PEP 451 APIs. Mitogen should no longer import imp on Python versions that support ModuleSpec. Tests have been added to cover the new APIs. CI jobs have been added to cover Python 3.x on macOS. Refs #1033 Co-authored-by: Witold Baryluk --- .ci/azure-pipelines.yml | 18 +- .ci/localhost_ansible_tests.py | 36 ++++ ansible_mitogen/module_finder.py | 124 +++++++++++++- ansible_mitogen/runner.py | 92 ++++++++++- docs/changelog.rst | 2 + docs/contributors.rst | 1 + mitogen/core.py | 156 ++++++++++++++++-- mitogen/master.py | 123 +++++++++++++- .../delegate_to_template.yml | 4 +- .../ansible_2_8_tests.yml | 7 - .../ansible/lib/modules/module_finder_test.py | 12 ++ tests/ansible/lib/modules/test_echo_module.py | 1 + tests/ansible/tests/module_finder_test.py | 80 +++++++++ tests/importer_test.py | 126 +++++++++++++- tests/module_finder_test.py | 14 +- tests/testlib.py | 9 +- tox.ini | 27 ++- 17 files changed, 766 insertions(+), 66 deletions(-) create mode 100644 tests/ansible/lib/modules/module_finder_test.py create mode 100644 tests/ansible/tests/module_finder_test.py diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index f672240d..0bedaa03 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -25,21 +25,17 @@ jobs: python.version: '3.11' tox.env: py311-mode_mitogen - # TODO: test python3, python3 tests are broken Loc_27_210: tox.env: py27-mode_localhost-ansible2.10 - Loc_27_4: - tox.env: py27-mode_localhost-ansible4 + Loc_311_6: + python.version: '3.11' + tox.env: py311-mode_localhost-ansible6 - # NOTE: this hangs when ran in Ubuntu 18.04 Van_27_210: - tox.env: py27-mode_localhost-ansible2.10 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_4: - tox.env: py27-mode_localhost-ansible4 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive + tox.env: py27-mode_localhost-ansible2.10-strategy_linear + Van_311_6: + python.version: '3.11' + tox.env: py311-mode_localhost-ansible6-strategy_linear - job: Linux pool: diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 69d67cd1..c50ef220 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen +from __future__ import print_function + +import getpass +import io import os import subprocess import sys @@ -53,6 +57,38 @@ with ci_lib.Fold('machine_prep'): os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") + # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022 + # and os.environ['USER'] is not populated on Azure macOS runners. + os.chdir(HOSTS_DIR) + with io.open('default.hosts', 'r+', encoding='utf-8') as f: + user = getpass.getuser() + content = f.read() + content = content.replace("{{ lookup('pipe', 'whoami') }}", user) + f.seek(0) + f.write(content) + f.truncate() + ci_lib.dump_file('default.hosts') + + cmd = ';'.join([ + 'from __future__ import print_function', + 'import os, sys', + 'print(sys.executable, os.path.realpath(sys.executable))', + ]) + for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']: + print(interpreter) + try: + subprocess.call([interpreter, '-c', cmd]) + except OSError as exc: + print(exc) + + print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1') + environ = os.environ.copy() + environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1' + try: + subprocess.call([interpreter, '-c', cmd], env=environ) + except OSError as exc: + print(exc) + with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index 19dcef7f..a1870833 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -31,12 +31,23 @@ from __future__ import unicode_literals __metaclass__ = type import collections -import imp +import logging import os +import re +import sys + +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp import mitogen.master +LOG = logging.getLogger(__name__) PREFIX = 'ansible.module_utils.' @@ -45,8 +56,6 @@ PREFIX = 'ansible.module_utils.' # path Filesystem path of the module. # kind One of the constants in `imp`, as returned in `imp.find_module()` # parent `ansible_mitogen.module_finder.Module` of parent package (if any). -# -# FIXME Python 3.12 removed `imp`, leaving no constants for `Module.kind`. Module = collections.namedtuple('Module', 'name path kind parent') @@ -126,14 +135,121 @@ def find_relative(parent, name, path=()): def scan_fromlist(code): + """Return an iterator of (level, name) for explicit imports in a code + object. + + Not all names identify a module. `from os import name, path` generates + `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string. + + >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n' + >>> code = compile(src, '', 'exec') + >>> list(scan_fromlist(code)) + [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')] + """ for level, modname_s, fromlist in mitogen.master.scan_code_imports(code): for name in fromlist: - yield level, '%s.%s' % (modname_s, name) + yield level, str('%s.%s' % (modname_s, name)) if not fromlist: yield level, modname_s +def walk_imports(code, prefix=None): + """Return an iterator of names for implicit parent imports & explicit + imports in a code object. + + If a prefix is provided, then only children of that prefix are included. + Not all names identify a module. `from os import name, path` generates + `'os', 'os.name', 'os.path'`, but `os.name` is usually a string. + + >>> source = 'import a; import b; import b.c; from b.d import e, f\\n' + >>> code = compile(source, '', 'exec') + >>> list(walk_imports(code)) + ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'] + >>> list(walk_imports(code, prefix='b')) + ['b.c', 'b.d', 'b.d.e', 'b.d.f'] + """ + if prefix is None: + prefix = '' + pattern = re.compile(r'(^|\.)(\w+)') + start = len(prefix) + for _, name, fromlist in mitogen.master.scan_code_imports(code): + if not name.startswith(prefix): + continue + for match in pattern.finditer(name, start): + yield name[:match.end()] + for leaf in fromlist: + yield str('%s.%s' % (name, leaf)) + + def scan(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + """Return a list of (name, path, is_package) for ansible.module_utils + imports used by an Ansible module. + """ + log = LOG.getChild('scan') + log.debug('%r, %r, %r', module_name, module_path, search_path) + + if sys.version_info >= (3, 4): + result = _scan_importlib_find_spec( + module_name, module_path, search_path, + ) + log.debug('_scan_importlib_find_spec %r', result) + else: + result = _scan_imp_find_module(module_name, module_path, search_path) + log.debug('_scan_imp_find_module %r', result) + return result + + +def _scan_importlib_find_spec(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + module = importlib.machinery.ModuleSpec( + module_name, loader=None, origin=module_path, + ) + prefix = importlib.machinery.ModuleSpec( + PREFIX.rstrip('.'), loader=None, + ) + prefix.submodule_search_locations = search_path + queue = collections.deque([module]) + specs = {prefix.name: prefix} + while queue: + spec = queue.popleft() + if spec.origin is None: + continue + try: + with open(spec.origin, 'rb') as f: + code = compile(f.read(), spec.name, 'exec') + except Exception as exc: + raise ValueError((exc, module, spec, specs)) + + for name in walk_imports(code, prefix.name): + if name in specs: + continue + + parent_name = name.rpartition('.')[0] + parent = specs[parent_name] + if parent is None or not parent.submodule_search_locations: + specs[name] = None + continue + + child = importlib.util._find_spec( + name, parent.submodule_search_locations, + ) + if child is None or child.origin is None: + specs[name] = None + continue + + specs[name] = child + queue.append(child) + + del specs[prefix.name] + return sorted( + (spec.name, spec.origin, spec.submodule_search_locations is not None) + for spec in specs.values() if spec is not None + ) + + +def _scan_imp_find_module(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] module = Module(module_name, module_path, imp.PY_SOURCE, None) stack = [module] seen = set() diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 6986ddcd..8da1b670 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import atexit -import imp import json import os import re @@ -64,6 +63,14 @@ except ImportError: # Python 2.4 ctypes = None +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + try: # Cannot use cStringIO as it does not support Unicode. from StringIO import StringIO @@ -514,14 +521,74 @@ class ModuleUtilsImporter(object): sys.modules.pop(fullname, None) def find_module(self, fullname, path=None): + """ + Return a loader for the module with fullname, if we will load it. + + Implements importlib.abc.MetaPathFinder.find_module(). + Deprecrated in Python 3.4+, replaced by find_spec(). + Raises ImportWarning in Python 3.10+. Removed in Python 3.12. + """ if fullname in self._by_fullname: return self + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`. + + Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+. + """ + if fullname.endswith('.'): + return None + + try: + module_path, is_package = self._by_fullname[fullname] + except KeyError: + LOG.debug('Skipping %s: not present', fullname) + return None + + LOG.debug('Handling %s', fullname) + origin = 'master:%s' % (module_path,) + return importlib.machinery.ModuleSpec( + fullname, loader=self, origin=origin, is_package=is_package, + ) + + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + spec = module.__spec__ + path, _ = self._by_fullname[spec.name] + source = ansible_mitogen.target.get_small_file(self._context, path) + code = compile(source, path, 'exec', 0, 1) + exec(code, module.__dict__) + self._loaded.add(spec.name) + def load_module(self, fullname): + """ + Return the loaded module specified by fullname. + + Implements PEP 302 importlib.abc.Loader.load_module(). + Deprecated in Python 3.4+, replaced by create_module() & exec_module(). + """ path, is_pkg = self._by_fullname[fullname] source = ansible_mitogen.target.get_small_file(self._context, path) code = compile(source, path, 'exec', 0, 1) - # FIXME Python 3.12 removed `imp` mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = "master:%s" % (path,) mod.__loader__ = self @@ -819,12 +886,17 @@ class NewStyleRunner(ScriptRunner): synchronization mechanism by importing everything the module will need prior to detaching. """ + # I think "custom" means "found in custom module_utils search path", + # e.g. playbook relative dir, ~/.ansible/..., Ansible collection. for fullname, _, _ in self.module_map['custom']: mitogen.core.import_module(fullname) + + # I think "builtin" means "part of ansible/ansible-base/ansible-core", + # as opposed to Python builtin modules such as sys. for fullname in self.module_map['builtin']: try: mitogen.core.import_module(fullname) - except ImportError: + except ImportError as exc: # #590: Ansible 2.8 module_utils.distro is a package that # replaces itself in sys.modules with a non-package during # import. Prior to replacement, it is a real package containing @@ -835,8 +907,18 @@ class NewStyleRunner(ScriptRunner): # loop progresses to the next entry and attempts to preload # 'distro._distro', the import mechanism will fail. So here we # silently ignore any failure for it. - if fullname != 'ansible.module_utils.distro._distro': - raise + if fullname == 'ansible.module_utils.distro._distro': + continue + + # ansible.module_utils.compat.selinux raises ImportError if it + # can't load libselinux.so. The importer would usually catch + # this & skip selinux operations. We don't care about selinux, + # we're using import to get a copy of the module. + if (fullname == 'ansible.module_utils.compat.selinux' + and exc.msg == 'unable to load libselinux.so'): + continue + + raise def _setup_excepthook(self): """ diff --git a/docs/changelog.rst b/docs/changelog.rst index 32829ef6..940e4900 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,8 @@ Unreleased * :gh:issue:`987` Support Python 3.11 * :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when becoming an unprivileged user with Python 3.x +* :gh:issue:`1033` Support `PEP 451 , + required by Python 3.12 v0.3.4 (2023-07-02) diff --git a/docs/contributors.rst b/docs/contributors.rst index 584c4cd4..61a9eb1b 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters.
  • randy — desperate for automation
  • Michael & Vicky Twomey-Lee
  • Wesley Moore
    • +
  • Witold Baryluk
    • diff --git a/mitogen/core.py b/mitogen/core.py index 6a3f3da7..cd02012f 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -81,13 +81,18 @@ import syslog import threading import time import traceback +import types import warnings import weakref import zlib -# Python >3.7 deprecated the imp module. -warnings.filterwarnings('ignore', message='the imp module is deprecated') -import imp +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp # Absolute imports for <2.5. select = __import__('select') @@ -1380,6 +1385,19 @@ class Importer(object): def __repr__(self): return 'Importer' + @staticmethod + def _loader_from_module(module, default=None): + """Return the loader for a module object.""" + try: + return module.__spec__.loader + except AttributeError: + pass + try: + return module.__loader__ + except AttributeError: + pass + return default + def builtin_find_module(self, fullname): # imp.find_module() will always succeed for __main__, because it is a # built-in module. That means it exists on a special linked list deep @@ -1400,7 +1418,6 @@ class Importer(object): # Otherwise use search path of the parent package. # Works for both stdlib modules & third-party modules. # If the search is unsuccessful then raises ImportError. - # FIXME Python 3.12 removed `imp`. fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() @@ -1424,14 +1441,13 @@ class Importer(object): try: #_v and self._log.debug('Python requested %r', fullname) fullname = to_text(fullname) - pkgname, dot, _ = str_rpartition(fullname, '.') + pkgname, _, suffix = str_rpartition(fullname, '.') pkg = sys.modules.get(pkgname) if pkgname and getattr(pkg, '__loader__', None) is not self: self._log.debug('%s is submodule of a locally loaded package', fullname) return None - suffix = fullname[len(pkgname+dot):] if pkgname and suffix not in self._present.get(pkgname, ()): self._log.debug('%s has no submodule %s', pkgname, suffix) return None @@ -1451,6 +1467,66 @@ class Importer(object): finally: del _tls.running + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`, allowing other finders to try. + + fullname Fully qualified name of the module (e.g. foo.bar.baz) + path Path entries to search. None for a top-level module. + target Existing module to be reloaded (if any). + + Implements importlib.abc.MetaPathFinder.find_spec() + Python 3.4+. + """ + # Presence of _tls.running indicates we've re-invoked importlib. + # Abort early to prevent infinite recursion. See below. + if hasattr(_tls, 'running'): + return None + + log = self._log.getChild('find_spec') + + if fullname.endswith('.'): + return None + + pkgname, _, modname = fullname.rpartition('.') + if pkgname and modname not in self._present.get(pkgname, ()): + log.debug('Skipping %s. Parent %s has no submodule %s', + fullname, pkgname, modname) + return None + + pkg = sys.modules.get(pkgname) + pkg_loader = self._loader_from_module(pkg) + if pkgname and pkg_loader is not self: + log.debug('Skipping %s. Parent %s was loaded by %r', + fullname, pkgname, pkg_loader) + return None + + # #114: whitelisted prefixes override any system-installed package. + if self.whitelist != ['']: + if any(s and fullname.startswith(s) for s in self.whitelist): + log.debug('Handling %s. It is whitelisted', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + if fullname == '__main__': + log.debug('Handling %s. A special case', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + # Re-invoke the import machinery to allow other finders to try. + # Set a guard, so we don't infinitely recurse. See top of this method. + _tls.running = True + try: + spec = importlib.util._find_spec(fullname, path, target) + finally: + del _tls.running + + if spec: + log.debug('Skipping %s. Available as %r', fullname, spec) + return spec + + log.debug('Handling %s. Unavailable locally', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + blacklisted_msg = ( '%r is present in the Mitogen importer blacklist, therefore this ' 'context will not attempt to request it from the master, as the ' @@ -1537,6 +1613,64 @@ class Importer(object): if present: callback() + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + self._log.debug('Creating module for %r', spec) + + # FIXME Should this be done in find_spec()? Can it? + self._refuse_imports(spec.name) + + # FIXME "create_module() should properly handle the case where it is + # called more than once for the same spec/module." -- PEP-451 + event = threading.Event() + self._request_module(spec.name, callback=event.set) + event.wait() + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[spec.name] + + if path is None: + raise ImportError(self.absent_msg % (spec.name)) + + spec.origin = self.get_filename(spec.name) + if pkg_present is not None: + # TODO Namespace packages + spec.submodule_search_locations = [] + self._present[spec.name] = pkg_present + + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + name = module.__spec__.name + origin = module.__spec__.origin + self._log.debug('Executing %s from %s', name, origin) + source = self.get_source(name) + try: + # Compile the source into a code object. Don't add any __future__ + # flags and don't inherit any from this module. + # FIXME Should probably be exposed as get_code() + code = compile(source, origin, 'exec', flags=0, dont_inherit=True) + except SyntaxError: + # FIXME Why is this LOG, rather than self._log? + LOG.exception('while importing %r', name) + raise + + exec(code, module.__dict__) + def load_module(self, fullname): """ Return the loaded module specified by fullname. @@ -1552,12 +1686,11 @@ class Importer(object): self._request_module(fullname, event.set) event.wait() - ret = self._cache[fullname] - if ret[2] is None: + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[fullname] + if path is None: raise ModuleNotFoundError(self.absent_msg % (fullname,)) - pkg_present = ret[1] - # FIXME Python 3.12 removed `imp` mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = self.get_filename(fullname) mod.__loader__ = self @@ -3958,8 +4091,7 @@ class ExternalContext(object): def _setup_package(self): global mitogen - # FIXME Python 3.12 removed `imp` - mitogen = imp.new_module('mitogen') + mitogen = types.ModuleType('mitogen') mitogen.__package__ = 'mitogen' mitogen.__path__ = [] mitogen.__loader__ = self.importer diff --git a/mitogen/master.py b/mitogen/master.py index 8cd1d27c..b1e0a1de 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -37,7 +37,6 @@ contexts. import dis import errno -import imp import inspect import itertools import logging @@ -50,6 +49,16 @@ import threading import types import zlib +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util + from _imp import is_builtin as _is_builtin +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + from imp import is_builtin as _is_builtin + try: import sysconfig except ImportError: @@ -122,18 +131,16 @@ def is_stdlib_name(modname): """ Return :data:`True` if `modname` appears to come from the standard library. """ - # `imp.is_builtin()` isn't a documented as part of Python's stdlib API. + # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib. # Returns 1 if modname names a module that is "builtin" to the the Python # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces'). - # FIXME Python 3.12 removed `imp`, but `_imp.is_builtin()` remains. - # `sys.builtin_module_names` (Python 2.2+) may be an alternative. # # """ # Main is a little special - imp.is_builtin("__main__") will return False, # but BuiltinImporter is still the most appropriate initial setting for # its __loader__ attribute. # """ -- comment in CPython pylifecycle.c:add_main_module() - if imp.is_builtin(modname) != 0: + if _is_builtin(modname) != 0: return True module = sys.modules.get(modname) @@ -464,6 +471,9 @@ class FinderMethod(object): name according to the running Python interpreter. You'd think this was a simple task, right? Naive young fellow, welcome to the real world. """ + def __init__(self): + self.log = LOG.getChild(self.__class__.__name__) + def __repr__(self): return '%s()' % (type(self).__name__,) @@ -645,7 +655,7 @@ class SysModulesMethod(FinderMethod): return path, source, is_pkg -class ParentEnumerationMethod(FinderMethod): +class ParentImpEnumerationMethod(FinderMethod): """ Attempt to fetch source code by examining the module's (hopefully less insane) parent package, and if no insane parents exist, simply use @@ -775,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod): """ See implementation for a description of how this works. """ + if sys.version_info >= (3, 4): + return None + #if fullname not in sys.modules: # Don't attempt this unless a module really exists in sys.modules, # else we could return junk. @@ -792,19 +805,110 @@ class ParentEnumerationMethod(FinderMethod): # Still more components to descent. Result must be a package if fp: fp.close() - # FIXME The imp module was removed in Python 3.12. if kind != imp.PKG_DIRECTORY: LOG.debug('%r: %r appears to be child of non-package %r', self, fullname, path) return None search_path = [path] - # FIXME The imp module was removed in Python 3.12. elif kind == imp.PKG_DIRECTORY: return self._found_package(fullname, path) else: return self._found_module(fullname, path, fp) +class ParentSpecEnumerationMethod(ParentImpEnumerationMethod): + def _find_parent_spec(self, fullname): + #history = [] + debug = self.log.debug + children = [] + for parent_name, child_name in self._iter_parents(fullname): + children.insert(0, child_name) + if not parent_name: + debug('abandoning %r, reached top-level', fullname) + return None, children + + try: + parent = sys.modules[parent_name] + except KeyError: + debug('skipping %r, not in sys.modules', parent_name) + continue + + try: + spec = parent.__spec__ + except AttributeError: + debug('skipping %r: %r.__spec__ is absent', + parent_name, parent) + continue + + if not spec: + debug('skipping %r: %r.__spec__=%r', + parent_name, parent, spec) + continue + + if spec.name != parent_name: + debug('skipping %r: %r.__spec__.name=%r does not match', + parent_name, parent, spec.name) + continue + + if not spec.submodule_search_locations: + debug('skipping %r: %r.__spec__.submodule_search_locations=%r', + parent_name, parent, spec.submodule_search_locations) + continue + + return spec, children + + raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + def find(self, fullname): + # Returns absolute path, ParentImpEnumerationMethod returns relative + # >>> spec_pem.find('six_brokenpkg._six')[::2] + # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False) + + if sys.version_info < (3, 4): + return None + + fullname = to_text(fullname) + spec, children = self._find_parent_spec(fullname) + for child_name in children: + if spec: + name = '%s.%s' % (spec.name, child_name) + submodule_search_locations = spec.submodule_search_locations + else: + name = child_name + submodule_search_locations = None + spec = importlib.util._find_spec(name, submodule_search_locations) + if spec is None: + self.log.debug('%r spec unavailable from %s', fullname, spec) + return None + + is_package = spec.submodule_search_locations is not None + if name != fullname: + if not is_package: + self.log.debug('%r appears to be child of non-package %r', + fullname, spec) + return None + continue + + if not spec.has_location: + self.log.debug('%r.origin cannot be read as a file', spec) + return None + + if os.path.splitext(spec.origin)[1] != '.py': + self.log.debug('%r.origin does not contain Python source code', + spec) + return None + + # FIXME This should use loader.get_source() + with open(spec.origin, 'rb') as f: + source = f.read() + + return spec.origin, source, is_package + + raise ValueError('%s.find(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + class ModuleFinder(object): """ Given the name of a loaded module, make a best-effort attempt at finding @@ -845,7 +949,8 @@ class ModuleFinder(object): DefectivePython3xMainMethod(), PkgutilMethod(), SysModulesMethod(), - ParentEnumerationMethod(), + ParentSpecEnumerationMethod(), + ParentImpEnumerationMethod(), ] def get_module_source(self, fullname): diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 3776a7db..7d33a161 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -41,7 +41,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -69,7 +69,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 5b695c42..ce2b3d3a 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -195,13 +195,6 @@ - distro == 'ubuntu' - distro_version is version('16.04', '>=', strict=True) - - name: mac assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: os_family == 'Darwin' - always: - meta: clear_facts when: diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py new file mode 100644 index 00000000..41cf1c1c --- /dev/null +++ b/tests/ansible/lib/modules/module_finder_test.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import os +import sys + +import ansible.module_utils.external1 + +from ansible.module_utils.externalpkg.extmod import path as epem_path + +def main(): + pass diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index d44b85ab..9d1d99b1 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -51,6 +51,7 @@ def main(): 'as_seen': sys.executable, 'resolved': os.path.realpath(sys.executable), }, + 'platform': sys.platform, }, }, } diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py new file mode 100644 index 00000000..79e8fdbd --- /dev/null +++ b/tests/ansible/tests/module_finder_test.py @@ -0,0 +1,80 @@ +import os.path +import sys +import textwrap +import unittest + +import ansible_mitogen.module_finder + +import testlib + + +class ScanFromListTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b.c; from d.e import f; from g import h, i + ''') + code = compile(source, '', 'exec') + self.assertEqual( + list(ansible_mitogen.module_finder.scan_fromlist(code)), + [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')], + ) + + +class WalkImportsTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b; import b.c; from b.d import e, f + ''') + code = compile(source, '', 'exec') + + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code)), + ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'], + ) + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')), + ['b.c', 'b.d', 'b.d.e', 'b.d.f'], + ) + + +class ScanTest(testlib.TestCase): + module_name = 'ansible_module_module_finder_test__this_should_not_matter' + module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py') + search_path = ( + 'does_not_exist/module_utils', + testlib.ANSIBLE_MODULE_UTILS_DIR, + ) + + @staticmethod + def relpath(path): + return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR) + + @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable') + def test_importlib_find_spec(self): + scan = ansible_mitogen.module_finder._scan_importlib_find_spec + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) + + @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred') + def test_imp_find_module(self): + scan = ansible_mitogen.module_finder._scan_imp_find_module + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) diff --git a/tests/importer_test.py b/tests/importer_test.py index e48c02a4..e86af8af 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -2,6 +2,7 @@ import sys import threading import types import zlib +import unittest import mock @@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin): super(ImporterMixin, self).tearDown() +class InvalidNameTest(ImporterMixin, testlib.TestCase): + modname = 'trailingdot.' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_invalid(self): + self.set_get_module_response(self.response) + self.assertEqual(self.importer.find_spec(self.modname, path=None), None) + + +class MissingModuleTest(ImporterMixin, testlib.TestCase): + modname = 'missing' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') + def test_load_module_missing(self): + self.set_get_module_response(self.response) + self.assertRaises(ImportError, self.importer.load_module, self.modname) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_missing(self): + """ + Importer should optimistically offer itself as a module loader + when there are no disqualifying criteria. + """ + import importlib.machinery + self.set_get_module_response(self.response) + spec = self.importer.find_spec(self.modname, path=None) + self.assertIsInstance(spec, importlib.machinery.ModuleSpec) + self.assertEqual(spec.name, self.modname) + self.assertEqual(spec.loader, self.importer) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_create_module_missing(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + self.assertRaises(ImportError, self.importer.create_module, spec) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModuleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): # 0:fullname 1:pkg_present 2:path 3:compressed 4:related response = (modname, None, path, data, []) - def test_no_such_module(self): - self.set_get_module_response( - # 0:fullname 1:pkg_present 2:path 3:compressed 4:related - (self.modname, None, None, None, None) - ) - self.assertRaises(ImportError, - lambda: self.importer.load_module(self.modname)) - def test_module_added_to_sys_modules(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) @@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): self.assertIsNone(mod.__package__) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ModuleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'fake_module' + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.__package__, 'mypkg') +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class SubmoduleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'mypkg.fake_module' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'mypkg.fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModulePackageTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("func = lambda: 1\n\n")) path = 'fake_pkg/__init__.py' @@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.func.__module__, self.modname) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class PackageSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("func = lambda: 1\n\n")) + path = 'fake_pkg/__init__.py' + modname = 'fake_pkg' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, [], path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_pkg') + #self.assertFalse(hasattr(mod, '__file__')) + + def test_get_filename(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + filename = self.importer.get_filename(self.modname) + self.assertEqual('master:fake_pkg/__init__.py', filename) + + def test_get_source(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + source = self.importer.get_source(self.modname) + self.assertEqual(source, + mitogen.core.to_text(zlib.decompress(self.data))) + + class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase): def initdir(self, caplog): self.caplog = caplog diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 02b8b886..67e937ed 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -140,9 +140,7 @@ class SysModulesMethodTest(testlib.TestCase): self.assertIsNone(tup) -class GetModuleViaParentEnumerationTest(testlib.TestCase): - klass = mitogen.master.ParentEnumerationMethod - +class ParentEnumerationMixin(object): def call(self, fullname): return self.klass().find(fullname) @@ -232,6 +230,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): self.assertEqual(is_pkg, False) +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4') +class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentImpEnumerationMethod + + +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentSpecEnumerationMethod + + class ResolveRelPathTest(testlib.TestCase): klass = mitogen.master.ModuleFinder diff --git a/tests/testlib.py b/tests/testlib.py index ec0a7443..8c40e7ff 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -50,8 +50,13 @@ except NameError: LOG = logging.getLogger(__name__) -DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') -MODS_DIR = os.path.join(DATA_DIR, 'importer') + +TESTS_DIR = os.path.join(os.path.dirname(__file__)) +ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') +ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') +ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules') +DATA_DIR = os.path.join(TESTS_DIR, 'data') +MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer') sys.path.append(DATA_DIR) sys.path.append(MODS_DIR) diff --git a/tox.ini b/tox.ini index e4c26c39..ae4f35c9 100644 --- a/tox.ini +++ b/tox.ini @@ -3,18 +3,34 @@ # # sudo add-apt-repository ppa:deadsnakes/ppa # sudo apt update -# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox +# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox # Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== # 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 # 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 -# 2.6 <= 2.6.20 <= 2.13 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 +# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 # 2.7 <= 2.11 <= 5.6 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.3? # 3.5 <= 2.11 <= 2.13 <= 5.6 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15 # 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 5.9.5 <= 7.0.1 <= 3.28 <= 20.16 # 3.7 <= 2.12 <= 3.2.20 # 3.8 <= 2.12 +# 3.9 <= 2.15 +# 3.10 +# 3.11 +# 3.12 >= 2.13¹ +# +# Notes +# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13). +# Python 3.12 removed support for find_module(), replaced by find_spec(). +# In Ansible <= 4.x ansible.module_utils.six lacks find_spec(). +# https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3 +# In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder +# lacks find_spec(). https://github.com/ansible/ansible/pull/76225 +# +# Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15). +# Python 3.12 removed deprecated httplib.HTTPSConnection() arguments. +# https://github.com/ansible/ansible/pull/80751 # Ansible Dependency # ================== ====================== @@ -24,6 +40,9 @@ # ansible == 4.* ansible-core ~= 2.11.0 # ansible == 5.* ansible-core ~= 2.12.0 # ansible == 6.* ansible-core ~= 2.13.0 +# ansible == 7.x ansible-core ~= 2.14.0 +# ansible == 8.x ansible-core ~= 2.15.0 +# ansible == 9.x ansible-core ~= 2.16.0 # pip --no-python-version-warning # pip --disable-pip-version-check @@ -51,6 +70,7 @@ basepython = py39: python3.9 py310: python3.10 py311: python3.11 + py312: python3.12 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt @@ -79,7 +99,6 @@ passenv = HOME setenv = # See also azure-pipelines.yml - ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 @@ -111,7 +130,9 @@ setenv = distros_ubuntu1804: DISTROS=ubuntu1804 distros_ubuntu2004: DISTROS=ubuntu2004 mode_ansible: MODE=ansible + mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive mode_debops_common: MODE=debops_common + mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive mode_mitogen: MODE=mitogen strategy_linear: ANSIBLE_STRATEGY=linear whitelist_externals = From 92c00d913ee57e2d0e965a996c3bb74c569ca0cb Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 17 Mar 2024 14:56:13 +0000 Subject: [PATCH 28/32] tests: Skip "discovered python matches invoked" on macOS 11/Python 2.7/Vanilla --- .ci/azure-pipelines-steps.yml | 21 ++++++++++++++++--- .../ansible_2_8_tests.yml | 9 ++++++++ tests/ansible/lib/modules/test_echo_module.py | 15 ++++++++++--- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 88d219b8..5d516eb0 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -2,6 +2,7 @@ # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool # `{script: ...}` is shorthand for `{task: CmdLine@, inputs: {script: ...}}`. +# The shell is bash. # https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines @@ -20,9 +21,23 @@ steps: condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) - script: | - type python && python --version - type python2 && python2 --version - type python3 && python3 --version + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi displayName: Show python versions - script: | diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index ce2b3d3a..6ac9bada 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -54,6 +54,15 @@ fail_msg: - "auto_out: {{ auto_out }}" - "echoout: {{ echoout }}" + when: + # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always + # reports the same path. I can't reach via symlinks. + # >>> sys.executable + # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python + - is_mitogen + or echoout.running_python.sys.version_info.major != 2 + or not (echoout.running_python.sys.platform == "darwin" + and echoout.running_python.platform.release.major == 20) - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 9d1d99b1..d6a5fb9e 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -23,11 +23,11 @@ def main(): # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info # NB This must be synced with mitogen.parent.Connection.get_boot_command() + platform_release_major = int(platform.release().partition('.')[0]) if sys.modules.get('mitogen') and sys.platform == 'darwin': - darwin_major = int(platform.release().partition('.')[0]) - if darwin_major < 19 and sys.executable == '/usr/bin/python2.7': + if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7': sys.executable = '/usr/bin/python' - if darwin_major in (20, 21) and sys.version_info[:2] == (2, 7): + if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7): # only for tests to check version of running interpreter -- Mac 10.15+ changed python2 # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python sys.executable = "/usr/bin/python" @@ -46,12 +46,21 @@ def main(): 'resolved': os.path.realpath(discovered_interpreter_python), }, 'running_python': { + 'platform': { + 'release': { + 'major': platform_release_major, + }, + }, 'sys': { 'executable': { 'as_seen': sys.executable, 'resolved': os.path.realpath(sys.executable), }, 'platform': sys.platform, + 'version_info': { + 'major': sys.version_info[0], + 'minor': sys.version_info[1], + }, }, }, } From fe8a3a71fcba652cc80d05641f11cba487f94a43 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 6 Mar 2024 11:07:45 +0000 Subject: [PATCH 29/32] ansible_mitogen: Remove use of distutils, which was removed in Python 3.12 --- ansible_mitogen/utils.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/utils.py b/ansible_mitogen/utils.py index 25c5a692..a01b261d 100644 --- a/ansible_mitogen/utils.py +++ b/ansible_mitogen/utils.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -import distutils.version +import re import ansible @@ -9,6 +9,21 @@ __all__ = [ 'ansible_version', ] -ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version) -del distutils + +def _parse(v_string): + # Adapted from distutils.version.LooseVersion.parse() + component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) + for component in component_re.split(v_string): + if not component or component == '.': + continue + try: + yield int(component) + except ValueError: + yield component + + +ansible_version = tuple(_parse(ansible.__version__)) + +del _parse +del re del ansible From 123efa751074f5d7e922efcb73e7a0673cc92b09 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 6 Mar 2024 11:51:12 +0000 Subject: [PATCH 30/32] mitogen: Support Python 3.12 Most of the necessary changes were made in recent PEP 451 commits. This bumps the CI jobs, and declares the support. Test dependendancies are bumped to latest supportted/available versions. refs #1033 --- .ci/azure-pipelines-steps.yml | 50 ++++++++++++++++++---- .ci/azure-pipelines.yml | 78 +++++++++++++++++------------------ docs/ansible_detailed.rst | 3 +- docs/changelog.rst | 1 + setup.py | 1 + tests/requirements-tox.txt | 4 ++ tests/requirements.txt | 31 ++++++++++---- tox.ini | 38 ++++++++++++----- 8 files changed, 140 insertions(+), 66 deletions(-) create mode 100644 tests/requirements-tox.txt diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 5d516eb0..919b992b 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -15,12 +15,20 @@ steps: condition: ne(variables['python.version'], '') - script: | + set -o errexit + set -o nounset + set -o pipefail + sudo apt-get update sudo apt-get install -y python2-dev python3-pip virtualenv displayName: Install build deps condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) - script: | + set -o errexit + set -o nounset + set -o pipefail + # macOS builders lack a realpath command type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version @@ -41,20 +49,46 @@ steps: displayName: Show python versions - script: | - if [[ $(uname) == "Darwin" ]]; then - python2 -m ensurepip --user --altinstall --no-default-pip - python2 -m pip install --user "tox<4.0" + set -o errexit + set -o nounset + set -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" else - python -m pip install "tox<4.0" + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" fi displayName: Install tooling - script: | - if [[ $(uname) == "Darwin" ]]; then - python2 -m tox -e "$(tox.env)" - else - python -m tox -e "$(tox.env)" + set -o errexit + set -o nounset + set -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 fi + + "$PYTHON" -m tox -e "$(tox.env)" displayName: "Run tests" env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 0bedaa03..3630f83e 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -21,21 +21,21 @@ jobs: matrix: Mito_27: tox.env: py27-mode_mitogen - Mito_311: - python.version: '3.11' - tox.env: py311-mode_mitogen + Mito_312: + python.version: '3.12' + tox.env: py312-mode_mitogen Loc_27_210: tox.env: py27-mode_localhost-ansible2.10 - Loc_311_6: - python.version: '3.11' - tox.env: py311-mode_localhost-ansible6 + Loc_312_6: + python.version: '3.12' + tox.env: py312-mode_localhost-ansible6 Van_27_210: tox.env: py27-mode_localhost-ansible2.10-strategy_linear - Van_311_6: - python.version: '3.11' - tox.env: py311-mode_localhost-ansible6-strategy_linear + Van_312_6: + python.version: '3.12' + tox.env: py312-mode_localhost-ansible6-strategy_linear - job: Linux pool: @@ -92,33 +92,33 @@ jobs: python.version: '3.6' tox.env: py36-mode_mitogen-distro_ubuntu2004 - Mito_311_centos6: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_centos6 - Mito_311_centos7: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_centos7 - Mito_311_centos8: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_centos8 - Mito_311_debian9: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_debian9 - Mito_311_debian10: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_debian10 - Mito_311_debian11: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_debian11 - Mito_311_ubuntu1604: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_ubuntu1604 - Mito_311_ubuntu1804: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_ubuntu1804 - Mito_311_ubuntu2004: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_ubuntu2004 + Mito_312_centos6: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos6 + Mito_312_centos7: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos7 + Mito_312_centos8: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos8 + Mito_312_debian9: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian9 + Mito_312_debian10: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian10 + Mito_312_debian11: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian11 + Mito_312_ubuntu1604: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu1604 + Mito_312_ubuntu1804: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu1804 + Mito_312_ubuntu2004: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu2004 Ans_27_210: tox.env: py27-mode_ansible-ansible2.10 @@ -144,6 +144,6 @@ jobs: Ans_311_5: python.version: '3.11' tox.env: py311-mode_ansible-ansible5 - Ans_311_6: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible6 + Ans_312_6: + python.version: '3.12' + tox.env: py312-mode_ansible-ansible6 diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 5679537e..cb83aa71 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -149,7 +149,8 @@ Noteworthy Differences Mitogen 0.3.1+ supports - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 - - Ansible 5 and 6; with Python 3.8-3.11 + - Ansible 5; with Python 3.8-3.11 + - Ansible 6; with Python 3.8-3.12 Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index 940e4900..126095fd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Unreleased becoming an unprivileged user with Python 3.x * :gh:issue:`1033` Support `PEP 451 , required by Python 3.12 +* :gh:issue:`1033` Support Python 3.12 v0.3.4 (2023-07-02) diff --git a/setup.py b/setup.py index 4d7fadfc..b17dab9d 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,7 @@ setup( 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/requirements-tox.txt b/tests/requirements-tox.txt new file mode 100644 index 00000000..bc7f7c2a --- /dev/null +++ b/tests/requirements-tox.txt @@ -0,0 +1,4 @@ +tox==3.28; python_version == '2.7' +tox==3.28; python_version == '3.6' +tox==4.8.0; python_version == '3.7' +tox>=4.13.0,~=4.0; python_version >= '3.8' diff --git a/tests/requirements.txt b/tests/requirements.txt index 1e5d2a1d..6d87d177 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,12 +1,24 @@ -cffi==1.15.1 -coverage==5.5; python_version < '3.7' -coverage==6.4.4; python_version >= '3.7' +cffi==1.15.1; python_version < '3.8' +cffi==1.16; python_version >= '3.8' + +coverage==5.5; python_version == '2.7' +coverage==6.2; python_version == '3.6' +coverage==7.2.7; python_version == '3.7' +coverage==7.4.3; python_version >= '3.8' + Django==1.11.29; python_version < '3.0' Django==3.2.20; python_version >= '3.6' -mock==2.0.0 -psutil==5.9.5 -pytest-catchlog==1.2.2 -pytest==3.1.2 + +mock==3.0.5; python_version == '2.7' +mock==5.1.0; python_version >= '3.6' + +psutil==5.9.8 + +pytest==4.6.11; python_version == '2.7' +pytest==7.0.1; python_version == '3.6' +pytest==7.4.4; python_version == '3.7' +pytest==8.0.2; python_version >= '3.8' + subprocess32==3.5.4; python_version < '3.0' timeoutcontext==1.2.0 # Fix InsecurePlatformWarning while creating py26 tox environment @@ -15,4 +27,7 @@ urllib3[secure]==1.23; python_version < '2.7' urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9' # Last idna compatible with Python 2.6 was idna 2.7. idna==2.7; python_version < '2.7' -virtualenv==20.10.0 + +virtualenv==20.15.1; python_version == '2.7' +virtualenv==20.17.1; python_version == '3.6' +virtualenv==20.25.1; python_version >= '3.7' diff --git a/tox.ini b/tox.ini index ae4f35c9..4a1772a0 100644 --- a/tox.ini +++ b/tox.ini @@ -10,10 +10,10 @@ # 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 # 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 # 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 -# 2.7 <= 2.11 <= 5.6 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.3? -# 3.5 <= 2.11 <= 2.13 <= 5.6 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15 -# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 5.9.5 <= 7.0.1 <= 3.28 <= 20.16 -# 3.7 <= 2.12 <= 3.2.20 +# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² +# 3.5 <= 2.11 <= 2.13 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² +# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² +# 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0 # 3.8 <= 2.12 # 3.9 <= 2.15 # 3.10 @@ -31,6 +31,9 @@ # Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15). # Python 3.12 removed deprecated httplib.HTTPSConnection() arguments. # https://github.com/ansible/ansible/pull/80751 +# +# 2. Higher virtualenv versions cannot run under this Python version. They can +# still generate virtual environments for it. # Ansible Dependency # ================== ====================== @@ -53,10 +56,11 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, - py{311}-mode_ansible-ansible{2.10,3,4,5,6}, - py{27,36,311}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,311}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,311}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{311}-mode_ansible-ansible{2.10,3,4,5}, + py{312}-mode_ansible-ansible{6}, + py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, [testenv] @@ -77,8 +81,8 @@ deps = ansible2.10: ansible==2.10.7 ansible3: ansible==3.4.0 ansible4: ansible==4.10.0 - ansible5: ansible==5.8.0 - ansible6: ansible==6.0.0 + ansible5: ansible~=5.0 + ansible6: ansible~=6.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -135,7 +139,21 @@ setenv = mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive mode_mitogen: MODE=mitogen strategy_linear: ANSIBLE_STRATEGY=linear +allowlist_externals = + # Added: Tox 3.18: Tox 4.0+ + *_install.py + *_tests.py + aws + docker + docker-credential-secretservice + echo + gpg2 + pass whitelist_externals = + # Deprecated: Tox 3.18+; Removed: Tox 4.0 + *_install.py + *_tests.py + aws docker docker-credential-secretservice echo From e97ab2f5975ab8c02b278ce69a42982cbfb7648d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 17 Mar 2024 15:57:12 +0000 Subject: [PATCH 31/32] Prepare v0.3.5 --- docs/changelog.rst | 4 ++-- docs/conf.py | 2 +- mitogen/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 126095fd..ead09d14 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,8 +17,8 @@ Release Notes To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -Unreleased ----------- +v0.3.5 (2024-03-17) +------------------- * :gh:issue:`987` Support Python 3.11 * :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when diff --git a/docs/conf.py b/docs/conf.py index 3a7fc002..fb9974cb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,7 +2,7 @@ import sys sys.path.append('.') -VERSION = '0.3.4' +VERSION = '0.3.5' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' diff --git a/mitogen/__init__.py b/mitogen/__init__.py index db4e8b3e..d030b966 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 5, 'dev0') +__version__ = (0, 3, 5) #: This is :data:`False` in slave contexts. Previously it was used to prevent From 1572da1563033f036b7886da793c7fe3d97d624b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Mar 2024 08:34:15 +0000 Subject: [PATCH 32/32] docs: Correct PEP 451 hyperlink (cherry picked from commit 50efa53f8fcb81ddd8f25c642efa0375e7538182) --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7a6badc3..1c7441c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,7 +23,7 @@ v0.3.5 (2024-03-17) * :gh:issue:`987` Support Python 3.11 * :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when becoming an unprivileged user with Python 3.x -* :gh:issue:`1033` Support `PEP 451 , +* :gh:issue:`1033` Support `PEP 451 `_, required by Python 3.12 * :gh:issue:`1033` Support Python 3.12