ansible_mitogen: alpha datatag handling & CI for Ansible 12 (ansible-core 2.19)

refs #1258

.github/workflows/tests.yml

@@ -177,10 +177,16 @@ jobs:
           - name: Ans_313_11
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible11
+          - name: Ans_313_12
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible12
 
           - name: Van_313_11
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible11-strategy_linear
+          - name: Van_313_12
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible12-strategy_linear
 
           - name: Mito_313
             python_version: '3.13'
@@ -273,6 +279,12 @@ jobs:
           - name: Van_313_11
             tox_env: py313-mode_localhost-ansible11-strategy_linear
 
+          - name: Loc_313_12
+            tox_env: py313-mode_localhost-ansible12
+
+          - name: Van_313_12
+            tox_env: py313-mode_localhost-ansible12-strategy_linear
+
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

ansible_mitogen/connection.py

@@ -767,7 +767,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
                               C.BECOME_ALLOW_SAME_USER):
             stack += (CONNECTION_METHOD[spec.become_method()](spec),)
 
-        return stack
+        return ansible_mitogen.utils.unsafe.cast(stack)
 
     def _build_stack(self):
         """

ansible_mitogen/mixins.py

@@ -402,15 +402,17 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
             if not self._mitogen_rediscovered_interpreter:
                 result['ansible_facts'][self._discovered_interpreter_key] = self._discovered_interpreter
 
-        if self._discovery_warnings:
+        discovery_warnings = getattr(self, '_discovery_warnings', [])
+        if discovery_warnings:
             if result.get('warnings') is None:
                 result['warnings'] = []
-            result['warnings'].extend(self._discovery_warnings)
+            result['warnings'].extend(discovery_warnings)
 
-        if self._discovery_deprecation_warnings:
+        discovery_deprecation_warnings = getattr(self, '_discovery_deprecation_warnings', [])
+        if discovery_deprecation_warnings:
             if result.get('deprecations') is None:
                 result['deprecations'] = []
-            result['deprecations'].extend(self._discovery_deprecation_warnings)
+            result['deprecations'].extend(discovery_deprecation_warnings)
 
         return ansible.utils.unsafe_proxy.wrap_var(result)
 
@@ -429,7 +431,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
                     "stderr": "stderr data"
                 }
         """
-        data = self._parse_returned_data(result)
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+            data = self._parse_returned_data(result, profile='legacy')
+        else:
+            data = self._parse_returned_data(result)
 
         # Cutpasted from the base implementation.
         if 'stdout' in data and 'stdout_lines' not in data:

ansible_mitogen/planner.py

@@ -170,6 +170,7 @@ class Planner(object):
         """
         binding = self._inv.connection.get_binding()
 
+        kwargs = ansible_mitogen.utils.unsafe.cast(kwargs)
         new = dict((mitogen.core.UnicodeType(k), kwargs[k])
                    for k in kwargs)
         new.setdefault('good_temp_dir',
@@ -204,7 +205,7 @@ class BinaryPlanner(Planner):
             module=self._inv.module_name,
             path=self._inv.module_path,
             json_args=json.dumps(self._inv.module_args),
-            env=self._inv.env,
+            env=ansible_mitogen.utils.unsafe.cast(self._inv.env),
             **kwargs
         )
 
@@ -546,7 +547,7 @@ def _invoke_async_task(invocation, planner):
         call_recv = context.call_async(
             ansible_mitogen.target.run_module_async,
             job_id=job_id,
-            timeout_secs=invocation.timeout_secs,
+            timeout_secs=ansible_mitogen.utils.unsafe.cast(invocation.timeout_secs),
             started_sender=started_recv.to_sender(),
             kwargs=planner.get_kwargs(),
         )

ansible_mitogen/runner.py

@@ -73,6 +73,7 @@ except ImportError:
     from io import StringIO
 
 # Prevent accidental import of an Ansible module from hanging on stdin read.
+# FIXME Should probably be b'{}' or None. Ansible 2.19 has bytes | None = None.
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
 
@@ -635,6 +636,7 @@ class NewStyleStdio(object):
         sys.stderr = StringIO()
         encoded = json.dumps({'ANSIBLE_MODULE_ARGS': args})
         ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded)
+        ansible.module_utils.basic._ANSIBLE_PROFILE = 'legacy'
         sys.stdin = StringIO(mitogen.core.to_text(encoded))
 
         self.original_get_path = getattr(ansible.module_utils.basic,
@@ -649,7 +651,9 @@ class NewStyleStdio(object):
         sys.stdout = self.original_stdout
         sys.stderr = self.original_stderr
         sys.stdin = self.original_stdin
+        # FIXME Should probably be b'{}' or None. Ansible 2.19 has bytes | None = None.
         ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
+        ansible.module_utils.basic._ANSIBLE_PROFILE = None
 
 
 class ProgramRunner(Runner):

ansible_mitogen/services.py

@@ -57,6 +57,7 @@ import mitogen.service
 import ansible_mitogen.loaders
 import ansible_mitogen.module_finder
 import ansible_mitogen.target
+import ansible_mitogen.utils
 import ansible_mitogen.utils.unsafe
 
 
@@ -338,7 +339,12 @@ class ContextService(mitogen.service.Service):
         'ansible_mitogen.target',
         'mitogen.fork',
         'mitogen.service',
-    )
+    ) + ((
+        'ansible.module_utils._internal._json._profiles._module_legacy_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+        'ansible.module_utils._internal._json._profiles._module_modern_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+    ) if ansible_mitogen.utils.ansible_version[:2] >= (2, 19) else ())
 
     def _send_module_forwards(self, context):
         if hasattr(self.router.responder, 'forward_modules'):

ansible_mitogen/utils/unsafe.py

@@ -16,8 +16,11 @@ __all__ = [
 
 def _cast_to_dict(obj): return {cast(k): cast(v) for k, v in obj.items()}
 def _cast_to_list(obj): return [cast(v) for v in obj]
+def _cast_to_set(obj): return set(cast(v) for v in obj)
+def _cast_to_tuple(obj): return tuple(cast(v) for v in obj)
 def _cast_unsafe(obj): return obj._strip_unsafe()
 def _passthrough(obj): return obj
+def _untag(obj): return obj._native_copy()
 
 
 # A dispatch table to cast objects based on their exact type.
@@ -26,30 +29,64 @@ _CAST_DISPATCH = {
     bytes: bytes,
     dict: _cast_to_dict,
     list: _cast_to_list,
-    tuple: _cast_to_list,
     mitogen.core.UnicodeType: mitogen.core.UnicodeType,
 }
 _CAST_DISPATCH.update({t: _passthrough for t in mitogen.utils.PASSTHROUGH})
 
-if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+_CAST_SUBTYPES = [
+    dict,
+    list,
+]
+
+if hasattr(ansible.utils.unsafe_proxy, 'TrustedAsTemplate'):
+    import datetime
+    import ansible.module_utils._internal._datatag
     _CAST_DISPATCH.update({
+        set: _cast_to_set,
+        tuple: _cast_to_tuple,
+        ansible.module_utils._internal._datatag._AnsibleTaggedBytes: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDate: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDateTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDict: _cast_to_dict,
+        ansible.module_utils._internal._datatag._AnsibleTaggedFloat: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedInt: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedList: _cast_to_list,
+        ansible.module_utils._internal._datatag._AnsibleTaggedSet: _cast_to_set,
+        ansible.module_utils._internal._datatag._AnsibleTaggedStr: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTuple: _cast_to_tuple,
         ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
         ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
+        datetime.date: _passthrough,
+        datetime.datetime: _passthrough,
+        datetime.time: _passthrough,
     })
+    _CAST_SUBTYPES.extend([
+        set,
+        tuple,
+    ])
 elif hasattr(ansible.utils.unsafe_proxy.AnsibleUnsafeText, '_strip_unsafe'):
     _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
         ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: _cast_unsafe,
         ansible.utils.unsafe_proxy.AnsibleUnsafeText: _cast_unsafe,
         ansible.utils.unsafe_proxy.NativeJinjaUnsafeText: _cast_unsafe,
     })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
 elif ansible_mitogen.utils.ansible_version[:2] <= (2, 16):
     _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
         ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
         ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
     })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
 else:
     mitogen_ver = '.'.join(str(v) for v in mitogen.__version__)
-    raise ImportError("Mitogen %s can't unwrap Ansible %s AnsibleUnsafe objects"
+    raise ImportError("Mitogen %s can't cast Ansible %s objects"
                       % (mitogen_ver, ansible.__version__))
 
 
@@ -78,7 +115,9 @@ def cast(obj):
         return unwrapper(obj)
 
     # Slow path: obj is some unknown subclass
-    if isinstance(obj, dict): return _cast_to_dict(obj)
-    if isinstance(obj, (list, tuple)): return _cast_to_list(obj)
+    for typ_ in _CAST_SUBTYPES:
+        if isinstance(obj, typ_):
+            unwrapper = _CAST_DISPATCH[typ_]
+            return unwrapper(obj)
 
     return mitogen.utils.cast(obj)

docs/changelog.rst

@@ -22,6 +22,10 @@ In progress (unreleased)
 ------------------------
 
 * :gh:issue:`1258` Initial Ansible 12 (ansible-core 2.19) support
+* :gh:issue:`1258` :mod:`ansible_mitogen`: Initial Ansible datatag support
+  (:gh:anspull:`84621`)
+* :gh:issue:`1258` :mod:`ansible_mitogen`: Ansible 12 (ansible-core 2.19) test
+  jobs
 
 
 v0.3.24 (2025-05-29)

docs/conf.py

@@ -64,6 +64,15 @@ domainrefs = {
         'text': '#%s',
         'url': 'https://github.com/mitogen-hq/mitogen/pull/%s',
     },
+    'gh:ansissue': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/issues/%s',
+    },
+    'gh:anspull': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/pull/%s',
+    },
+
     'ans:mod': {
         'text': '%s module',
         'url': 'https://docs.ansible.com/ansible/latest/modules/%s_module.html',

tests/ansible/integration/action/transfer_data.yml

@@ -1,7 +1,14 @@
 
-- name: integration/action/transfer_data.yml
+- name: integration/action/transfer_data.yml, json
   hosts: test-targets
   tasks:
+    - meta: end_play
+      when:
+        # Ansible >= 12 (ansible-core >= 2.19) only allows bytes|str through
+        # `ansible.plugins.action.ActionBase._transfer_data()`.
+        - ansible_version.full is version('2.18.999', '>', strict=True)
+        - not is_mitogen
+
     - name: Cleanup transfer data
       file:
         path: /tmp/transfer-data
@@ -15,26 +22,41 @@
           data: {
             "I am JSON": true
           }
+
     - name: Slurp JSON transfer data
       slurp:
         src: /tmp/transfer-data
       register: out
+
     - assert:
         that: |
           out.content|b64decode == '{"I am JSON": true}'
         fail_msg: |
           out={{ out }}
 
+    - name: Cleanup transfer data
+      file:
+        path: /tmp/transfer-data
+        state: absent
+  tags:
+    - transfer_data
+
+
+- name: integration/action/transfer_data.yml, text
+  hosts: test-targets
+  tasks:
     - name: Create text transfer data
       action_passthrough:
         method: _transfer_data
         kwargs:
           remote_path: /tmp/transfer-data
           data: "I am text."
+
     - name: Slurp text transfer data
       slurp:
         src: /tmp/transfer-data
       register: out
+
     - assert:
         that:
           out.content|b64decode == 'I am text.'

tests/ansible/integration/runner/crashy_new_style_module.yml

@@ -18,10 +18,17 @@
         - not out.changed
         - out is failed
         # https://github.com/ansible/ansible/commit/62d8c8fde6a76d9c567ded381e9b34dad69afcd6
-        - out.msg is match(msg_pattern)
-        - (out.module_stdout == "" and out.module_stderr is search(tb_pattern))
-          or
-          (out.module_stdout is search(tb_pattern) and out.module_stderr is match("Shared connection to localhost closed."))
+        - |
+          out.msg is match(msg_pattern)
+          or out.msg in (
+             "Task failed: Module failed: name 'kaboom' is not defined",
+             'Module result deserialization failed: No start of json char found',
+          )
+        # - out.exception is undefined
+        #   or out.exception | default('') is match(tb_pattern)
+        #   or out.module_stderr is search(tb_pattern)
+        # - out.module_stdout == ''
+        # - out.module_stderr is search(tb_pattern)
         fail_msg: |
           out={{ out }}
   tags:

tests/ansible/integration/runner/custom_binary_producing_junk.yml

@@ -30,6 +30,7 @@
         - out.failed
         - out.results[0].failed
         - out.results[0].msg.startswith('MODULE FAILURE')
+          or out.results[0].msg.startswith('Module result deserialization failed')
         - out.results[0].rc == 0
         fail_msg: |
           out={{ out }}

tests/ansible/integration/runner/custom_binary_single_null.yml

@@ -15,7 +15,9 @@
       that:
       - "out.failed"
       - "out.results[0].failed"
-      - "out.results[0].msg.startswith('MODULE FAILURE')"
+      - |
+        out.results[0].msg.startswith('MODULE FAILURE')
+        or out.results[0].msg == 'Module result deserialization failed: No start of json char found'
       # On Ubuntu 16.04 /bin/sh is dash 0.5.8. It treats custom_binary_single_null
       # as a valid executable. There's no error message, and rc == 0.
       - |

tests/ansible/integration/runner/missing_module.yml

@@ -26,6 +26,7 @@
     - assert:
         that: |
           'The module missing_module was not found in configured module paths' in out.stdout
+          or "Cannot resolve 'missing_module' to an action or module" in out.stdout
         fail_msg: |
           out={{ out }}
   tags:

tests/ansible/integration/ssh/timeouts.yml

@@ -43,6 +43,7 @@
           '"unreachable": true' in out.stdout
         - |
           '"msg": "Connection timed out."' in out.stdout
+          or '"msg": "Task failed: Connection timed out."' in out.stdout
         fail_msg: |
           out={{ out }}
   tags:

tests/ansible/integration/transport_config/password.yml

@@ -8,9 +8,14 @@
   tasks:
   - include_tasks: ../_mitogen_only.yml
   - {mitogen_get_stack: {}, register: out}
+  - assert_equal:
+      left: out.result[0].kwargs.password
+      right: null
+    when: ansible_version.full is version('2.18.999', '>=', strict=True)
   - assert_equal:
       left: out.result[0].kwargs.password
       right: ""  # actually null, but assert_equal limitation
+    when: ansible_version.full is version('2.18.999', '<', strict=True)
   tags:
     - mitogen_only
 
@@ -23,9 +28,14 @@
   - assert_equal:
       left: out.result[0].kwargs.password
       right: "ansi-ssh-pass"
+  - assert_equal:
+      left: out.result[1].kwargs.password
+      right: null
+    when: ansible_version.full is version('2.18.999', '>=', strict=True)
   - assert_equal:
       left: out.result[1].kwargs.password
       right: ""
+    when: ansible_version.full is version('2.18.999', '<', strict=True)
   tags:
     - mitogen_only
 
@@ -48,9 +58,14 @@
   tasks:
   - include_tasks: ../_mitogen_only.yml
   - {mitogen_get_stack: {}, register: out}
+  - assert_equal:
+      left: out.result[0].kwargs.password
+      right: null
+    when: ansible_version.full is version('2.18.999', '>=', strict=True)
   - assert_equal:
       left: out.result[0].kwargs.password
       right: ""
+    when: ansible_version.full is version('2.18.999', '<', strict=True)
   - assert_equal:
       left: out.result[1].kwargs.password
       right: "ansi-ssh-pass"
@@ -76,9 +91,14 @@
   tasks:
   - include_tasks: ../_mitogen_only.yml
   - {mitogen_get_stack: {}, register: out}
+  - assert_equal:
+      left: out.result[0].kwargs.password
+      right: null
+    when: ansible_version.full is version('2.18.999', '>=', strict=True)
   - assert_equal:
       left: out.result[0].kwargs.password
       right: ""
+    when: ansible_version.full is version('2.18.999', '<', strict=True)
   - assert_equal:
       left: out.result[1].kwargs.password
       right: "ansi-pass"
@@ -104,9 +124,14 @@
   tasks:
   - include_tasks: ../_mitogen_only.yml
   - {mitogen_get_stack: {}, register: out}
+  - assert_equal:
+      left: out.result[0].kwargs.password
+      right: null
+    when: ansible_version.full is version('2.18.999', '>=', strict=True)
   - assert_equal:
       left: out.result[0].kwargs.password
       right: ""
+    when: ansible_version.full is version('2.18.999', '<', strict=True)
   - assert_equal:
       left: out.result[1].kwargs.password
       right: "c.b.a"

tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml

@@ -24,7 +24,8 @@
       assert:
         that:
           - env.cwd == ansible_user_dir
-          - (not env.mitogen_loaded) or (env.python_path.count("") == 1)
+          - not env.mitogen_loaded
+            or (env.python_path | select('eq', '') | length == 1)
         fail_msg: |
           ansible_user_dir={{ ansible_user_dir }}
           env={{ env }}

tests/ansible/regression/issue_766__get_with_context.yml

@@ -28,6 +28,13 @@
         - ansible_version.full is version('2.11', '>=', strict=True)
         - ansible_version.full is version('2.12', '<', strict=True)
 
+    - meta: end_play
+      when:
+        # TASK [Get running configuration and state data ]
+        # Error: : Task failed: ActionBase._parse_returned_data() missing 1 required positional argument: 'profile'
+        # https://github.com/ansible-collections/ansible.netcommon/issues/698#issuecomment-2910082548
+        - ansible_version.full is version('2.18.999', '>=', strict=True)
+
     - block:
         - name: Start container
           command:

tests/ansible/tests/utils_unsafe_test.py

@@ -4,6 +4,7 @@ from ansible.utils.unsafe_proxy import AnsibleUnsafeBytes
 from ansible.utils.unsafe_proxy import AnsibleUnsafeText
 from ansible.utils.unsafe_proxy import wrap_var
 
+import ansible_mitogen.utils
 import ansible_mitogen.utils.unsafe
 
 import mitogen.core
@@ -17,7 +18,7 @@ class Text(mitogen.core.UnicodeType): pass
 class Tuple(tuple): pass
 
 
-class CastTest(unittest.TestCase):
+class CastMixin(unittest.TestCase):
     def assertIsType(self, obj, cls, msg=None):
         self.assertIs(type(obj), cls, msg)
 
@@ -29,6 +30,8 @@ class CastTest(unittest.TestCase):
         self.assertEqual(cast(obj), expected)
         self.assertIsType(cast(obj), type(expected))
 
+
+class CastKnownTest(CastMixin):
     def test_ansible_unsafe(self):
         self.assertCasts(AnsibleUnsafeBytes(b'abc'), b'abc')
         self.assertCasts(AnsibleUnsafeText(u'abc'), u'abc')
@@ -47,14 +50,12 @@ class CastTest(unittest.TestCase):
         self.assertCasts(wrap_var({}), {})
         self.assertCasts(wrap_var([]), [])
         self.assertCasts(wrap_var(u''), u'')
-        self.assertCasts(wrap_var(()), [])
 
     def test_subtypes_roundtrip(self):
         self.assertCasts(wrap_var(Bytes()), b'')
         self.assertCasts(wrap_var(Dict()), {})
         self.assertCasts(wrap_var(List()), [])
         self.assertCasts(wrap_var(Text()), u'')
-        self.assertCasts(wrap_var(Tuple()), [])
 
     def test_subtype_nested_dict(self):
         obj = Dict(foo=Dict(bar=u'abc'))
@@ -75,18 +76,59 @@ class CastTest(unittest.TestCase):
         self.assertIsType(unwrapped[0], list)
         self.assertIsType(unwrapped[0][0], mitogen.core.UnicodeType)
 
-    def test_subtype_roundtrip_tuple(self):
-        # wrap_var() preserves sequence types, cast() does not (for now)
+
+@unittest.skipIf(
+    ansible_mitogen.utils.ansible_version[:2] <= (2, 18),
+    'Ansible <= 11 (ansible-core >= 2.18) does not send/receive sets',
+)
+class CastSetTest(CastMixin):
+    def test_set(self):
+        self.assertCasts(wrap_var(set()), set())
+
+    def test_set_subclass(self):
+        self.assertCasts(wrap_var(Set()), set())
+
+
+class CastTupleTest(CastMixin):
+    def test_tuple(self):
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+            expected = ()
+        else:
+            expected = []
+        self.assertCasts(wrap_var(Tuple()), expected)
+
+    def test_tuple_subclass(self):
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+            expected = ()
+        else:
+            expected = []
+        self.assertCasts(wrap_var(()), expected)
+
+    def test_tuple_subclass_with_contents(self):
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+            expected = ((u'abc',),)
+        else:
+            expected = [[u'abc']]
+
         obj = Tuple([Tuple([u'abc'])])
         wrapped = wrap_var(obj)
         unwrapped = ansible_mitogen.utils.unsafe.cast(wrapped)
-        self.assertEqual(unwrapped, [[u'abc']])
-        self.assertIsType(unwrapped, list)
-        self.assertIsType(unwrapped[0], list)
+        self.assertEqual(unwrapped, expected)
+        self.assertIsType(unwrapped, type(expected))
+        self.assertIsType(unwrapped[0], type(expected[0]))
         self.assertIsType(unwrapped[0][0], mitogen.core.UnicodeType)
 
-    def test_unknown_types_raise(self):
+
+class CastUknownTypeTest(unittest.TestCase):
+    @unittest.skipIf(
+        ansible_mitogen.utils.ansible_version[:2] >= (2, 19),
+        'Ansible >= 12 (ansible-core >= 2.19) uses/preserves sets',
+    )
+    def test_set_raises(self):
         cast = ansible_mitogen.utils.unsafe.cast
         self.assertRaises(TypeError, cast, set())
         self.assertRaises(TypeError, cast, Set())
+
+    def test_complex_raises(self):
+        cast = ansible_mitogen.utils.unsafe.cast
         self.assertRaises(TypeError, cast, 4j)