WIP

.github/workflows/tests.yml

@@ -24,58 +24,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Ans_27_210
-            tox_env: py27-mode_ansible-ansible2.10
           - name: Ans_27_4
             tox_env: py27-mode_ansible-ansible4
 
-          - name: Ans_36_210
-            python_version: '3.6'
-            tox_env: py36-mode_ansible-ansible2.10
           - name: Ans_36_4
             python_version: '3.6'
             tox_env: py36-mode_ansible-ansible4
-
-          - name: Ans_311_210
-            python_version: '3.11'
-            tox_env: py311-mode_ansible-ansible2.10
-          - name: Ans_311_3
-            python_version: '3.11'
-            tox_env: py311-mode_ansible-ansible3
-          - name: Ans_311_4
-            python_version: '3.11'
-            tox_env: py311-mode_ansible-ansible4
-          - name: Ans_311_5
-            python_version: '3.11'
-            tox_env: py311-mode_ansible-ansible5
-          - name: Ans_313_6
-            python_version: '3.13'
-            tox_env: py313-mode_ansible-ansible6
-          - name: Ans_313_7
-            python_version: '3.13'
-            tox_env: py313-mode_ansible-ansible7
-          - name: Ans_313_8
-            python_version: '3.13'
-            tox_env: py313-mode_ansible-ansible8
-          - name: Ans_313_9
-            python_version: '3.13'
-            tox_env: py313-mode_ansible-ansible9
-          - name: Ans_313_10
-            python_version: '3.13'
-            tox_env: py313-mode_ansible-ansible10
-          - name: Van_313_10
-            python_version: '3.13'
-            tox_env: py313-mode_ansible-ansible10-strategy_linear
-
-          - name: Mito_27
-            tox_env: py27-mode_mitogen
-          - name: Mito_36
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen
-          - name: Mito_313
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen
-
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -164,24 +118,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Mito_27
-            tox_env: py27-mode_mitogen
           - name: Mito_313
             python_version: '3.13'
             tox_env: py313-mode_mitogen
 
-          - name: Loc_27_210
-            tox_env: py27-mode_localhost-ansible2.10
-          - name: Loc_313_10
-            python_version: '3.13'
-            tox_env: py313-mode_localhost-ansible10
-
-          - name: Van_27_210
-            tox_env: py27-mode_localhost-ansible2.10-strategy_linear
-          - name: Van_313_10
-            python_version: '3.13'
-            tox_env: py313-mode_localhost-ansible10-strategy_linear
-
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

ansible_mitogen/runner.py

@@ -49,9 +49,11 @@ import shlex
 import shutil
 import sys
 import tempfile
+import textwrap
 import traceback
 import types
 
+from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.six.moves import shlex_quote
 
 import mitogen.core
@@ -75,6 +77,7 @@ except ImportError:
 # Prevent accidental import of an Ansible module from hanging on stdin read.
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
+import ansible.module_utils.common.respawn
 
 # For tasks that modify /etc/resolv.conf, non-Debian derivative glibcs cache
 # resolv.conf at startup and never implicitly reload it. Cope with that via an
@@ -622,6 +625,20 @@ class TemporaryArgv(object):
         sys.argv[:] = self.original
 
 
+def _create_payload():
+    smuggled_args = getattr(ansible.module_utils.basic, '_ANSIBLE_ARGS')
+    if not smuggled_args:
+        raise Exception('unable to access ansible.module_utils.basic._ANSIBLE_ARGS (not launched by AnsiballZ?)')
+
+    assert 0, '%r, %r, spec %r' % (
+        sys.modules['__main__'],
+        ' '.join('%s=%r' % (s, getattr(sys.modules['__main__'], s)) for s in sorted(dir(sys.modules['__main__']))),
+        ' '.join('%s=%r' % (s, getattr(sys.modules['__main__'].__spec__, s)) for s in sorted(dir(sys.modules['__main__'].__spec__))),
+    )
+    module_fqn = sys.modules['__main__']._module_fqn
+    modlib_path = sys.modules['__main__']._modlib_path
+
+
 class NewStyleStdio(object):
     """
     Patch ansible.module_utils.basic argument globals.
@@ -640,11 +657,15 @@ class NewStyleStdio(object):
         self.original_get_path = getattr(ansible.module_utils.basic,
                                         'get_module_path', None)
         ansible.module_utils.basic.get_module_path = self._get_path
+        #self.original_create_payload = getattr(ansible.module_utils.common.respawn,
+        #                                '_create_payload', None)
+        #ansible.module_utils.common.respawn._create_payload = _create_payload
 
     def _get_path(self):
         return self.temp_dir
 
     def revert(self):
+        #ansible.module_utils.common.respawn._create_payload = self.original_create_payload
         ansible.module_utils.basic.get_module_path = self.original_get_path
         sys.stdout = self.original_stdout
         sys.stderr = self.original_stderr
@@ -978,10 +999,7 @@ class NewStyleRunner(ScriptRunner):
                 True,                   # dont_inherit
             ))
 
-    if mitogen.core.PY3:
+    main_module_name = '__main__'
-        main_module_name = '__main__'
-    else:
-        main_module_name = b'__main__'
 
     def _handle_magic_exception(self, mod, exc):
         """
@@ -1032,6 +1050,10 @@ class NewStyleRunner(ScriptRunner):
             'ansible_module_' + os.path.basename(self.path),
         )
 
+        # CREDIT https://github.com/phemmer
+        # https://github.com/mitogen-hq/mitogen/issues/849#issuecomment-988207114
+        sys.modules['__main__']._module_fqn = self.py_module_name
+        sys.modules['__main__']._modlib_path = self.get_temp_dir()
         code = self._get_code()
         rc = 2
         try:

ansible_mitogen/target.py

@@ -60,8 +60,10 @@ import mitogen.service
 # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
 # we must setup a fake "__main__" before that module is ever imported. The
 # str() is to cast Unicode to bytes on Python 2.6.
-if not sys.modules.get(str('__main__')):
+if '__main__' not in sys.modules:
-    sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
+    sys.modules['__main__'] = types.ModuleType(
+        '__main__', doc='ansible_mitogen.target __main__ module substitute',
+    )
 
 import ansible.module_utils.json_utils
 

tests/ansible/all.yml

@@ -1,6 +1,4 @@
 - import_playbook: setup/all.yml
   tags: setup
-- import_playbook: regression/all.yml
-  tags: regression
 - import_playbook: integration/all.yml
   tags: integration

tests/ansible/hosts/group_vars/all.yml

@@ -1,3 +1,4 @@
 ---
 pkg_mgr_python_interpreter: python
+pkg_repos_gpg_keys: []
 pkg_repos_overrides: []

tests/ansible/hosts/group_vars/debian11.yml

@@ -0,0 +1,3 @@
+pkg_repos_gpg_keys:
+  - src: debian-archive-bullseye-automatic.gpg  # Debian 11
+  - src: debian-archive-bookworm-automatic.gpg  # Debian 12

tests/ansible/integration/all.yml

@@ -2,40 +2,4 @@
 #
 # This playbook imports all tests that are known to work at present.
 #
-
+- import_playbook: package_managers/all.yml
-- import_playbook: action/all.yml
-  tags: action
-- import_playbook: async/all.yml
-  tags: async
-- import_playbook: become/all.yml
-  tags: become
-- import_playbook: connection/all.yml
-  tags: connection
-- import_playbook: connection_delegation/all.yml
-  tags: connection_delegation
-- import_playbook: connection_loader/all.yml
-  tags: connection_loader
-- import_playbook: context_service/all.yml
-  tags: context_service
-- import_playbook: glibc_caches/all.yml
-  tags: glibc_caches
-- import_playbook: interpreter_discovery/all.yml
-  tags: interpreter_discovery
-- import_playbook: local/all.yml
-  tags: local
-- import_playbook: module_utils/all.yml
-  tags: module_utils
-- import_playbook: playbook_semantics/all.yml
-  tags: playbook_semantics
-- import_playbook: process/all.yml
-  tags: process
-- import_playbook: runner/all.yml
-  tags: runner
-- import_playbook: ssh/all.yml
-  tags: ssh
-- import_playbook: strategy/all.yml
-  tags: strategy
-- import_playbook: stub_connections/all.yml
-  tags: stub_connections
-- import_playbook: transport_config/all.yml
-  tags: transport_config

tests/ansible/integration/package_managers/all.yml

@@ -0,0 +1,69 @@
+- name: integration/package_managers/all.yml
+  hosts: test-targets
+  gather_facts: true
+  # Most OS package managers need root. Homebrew refuses to run as root. This
+  # approximates "pkg_mgr == homebrew" for the Linux & macOS runners in CI.
+  become: "{{ inventory_hostname in (groups.linux | default([])) }}"
+  vars:
+    ansible_python_interpreter: auto
+    package: rsync  # Chosen to exist in all tested distros/package managers
+  tasks:
+    - name: Switch to archived package repositories
+      copy:
+        dest: "{{ item.dest }}"
+        content: "{{ item.content }}"
+        mode: u=rw,go=r
+      loop: "{{ pkg_repos_overrides }}"
+      loop_control:
+        label: "{{ item.dest }}"
+
+    - name: Add signing keys
+      copy:
+        src: "{{ item.src }}"
+        dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}"
+        mode: u=rw,go=r
+      loop: "{{ pkg_repos_gpg_keys }}"
+
+    - name: Update package index
+      apt:
+        update_cache: true
+      when:
+        - ansible_facts.pkg_mgr in ["apt"]
+
+    - name: Test package module 1st call
+      package:
+        name: "{{ package }}"
+        state: present
+
+    - name: Test package module 2nd call
+      package:
+        name: "{{ package }}"
+        state: present
+
+    - name: Test dnf module 1st call
+      dnf:
+        name: "{{ package }}"
+        state: present
+      when:
+        - ansible_facts.pkg_mgr == 'dnf'
+
+    - name: Test dnf module 2nd call
+      dnf:
+        name: "{{ package }}"
+        state: present
+      when:
+        - ansible_facts.pkg_mgr == 'dnf'
+
+    - name: Test apt module 1st call
+      apt:
+        name: "{{ package }}"
+        state: present
+      when:
+        - ansible_facts.pkg_mgr == 'apt'
+
+    - name: Test apt module 2nd call
+      apt:
+        name: "{{ package }}"
+        state: present
+      when:
+        - ansible_facts.pkg_mgr == 'apt'

tests/ansible/regression/issue_776__load_plugins_called_twice.yml

@@ -60,7 +60,7 @@
         name: "{{ package }}"
         state: present
 
-    - name: Test dnf module 2nd call
+    - name: Test dnf module 1st call
       dnf:
         name: "{{ package }}"
         state: present