Merge pull request #1127 from moreati/import-cleanups

Consolidate backward compatibility imports and polyfills

ansible_mitogen/affinity.py

@@ -83,7 +83,6 @@ import multiprocessing
 import os
 import struct
 
-import mitogen.core
 import mitogen.parent
 
 
@@ -265,7 +264,7 @@ class LinuxPolicy(FixedPolicy):
         for x in range(16):
             chunks.append(struct.pack('<Q', mask & shiftmask))
             mask >>= 64
-        return mitogen.core.b('').join(chunks)
+        return b''.join(chunks)
 
     def _get_thread_ids(self):
         try:

ansible_mitogen/logging.py

@@ -32,15 +32,13 @@ __metaclass__ = type
 import logging
 import os
 
+import ansible.utils.display
+
 import mitogen.core
 import mitogen.utils
 
-try:
-    from __main__ import display
-except ImportError:
-    import ansible.utils.display
-    display = ansible.utils.display.Display()
 
+display = ansible.utils.display.Display()
 
 #: The process name set via :func:`set_process_name`.
 _process_name = None

ansible_mitogen/mixins.py

@@ -35,18 +35,16 @@ import pwd
 import random
 import traceback
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-from ansible.module_utils._text import to_bytes
-from ansible.parsing.utils.jsonify import jsonify
-
 import ansible
 import ansible.constants
 import ansible.plugins
 import ansible.plugins.action
+import ansible.utils.unsafe_proxy
+import ansible.vars.clean
+
+from ansible.module_utils.common.text.converters import to_bytes, to_text
+from ansible.module_utils.six.moves import shlex_quote
+from ansible.parsing.utils.jsonify import jsonify
 
 import mitogen.core
 import mitogen.select
@@ -57,24 +55,6 @@ import ansible_mitogen.target
 import ansible_mitogen.utils
 import ansible_mitogen.utils.unsafe
 
-from ansible.module_utils._text import to_text
-
-try:
-    from ansible.utils.unsafe_proxy import wrap_var
-except ImportError:
-    from ansible.vars.unsafe_proxy import wrap_var
-
-try:
-    # ansible 2.8 moved remove_internal_keys to the clean module
-    from ansible.vars.clean import remove_internal_keys
-except ImportError:
-    try:
-        from ansible.vars.manager import remove_internal_keys
-    except ImportError:
-        # ansible 2.3.3 has remove_internal_keys as a protected func on the action class
-        # we'll fallback to calling self._remove_internal_keys in this case
-        remove_internal_keys = lambda a: "Not found"
-
 
 LOG = logging.getLogger(__name__)
 
@@ -413,10 +393,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
             self._remove_tmp_path(tmp)
 
         # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
-        # handle ansible 2.3.3 that has remove_internal_keys in a different place
-        check = remove_internal_keys(result)
-        if check == 'Not found':
-            self._remove_internal_keys(result)
+        ansible.vars.clean.remove_internal_keys(result)
 
         # taken from _execute_module of ansible 2.8.6
         # propagate interpreter discovery results back to the controller
@@ -440,7 +417,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
                 result['deprecations'] = []
             result['deprecations'].extend(self._discovery_deprecation_warnings)
 
-        return wrap_var(result)
+        return ansible.utils.unsafe_proxy.wrap_var(result)
 
     def _postprocess_response(self, result):
         """

ansible_mitogen/planner.py

@@ -477,7 +477,7 @@ def read_file(path):
     finally:
         os.close(fd)
 
-    return mitogen.core.b('').join(bits)
+    return b''.join(bits)
 
 
 def _propagate_deps(invocation, planner, context):

ansible_mitogen/plugins/connection/mitogen_local.py

@@ -42,13 +42,7 @@ except ImportError:
 import ansible_mitogen.connection
 import ansible_mitogen.process
 
-
-if sys.version_info > (3,):
-    viewkeys = dict.keys
-elif sys.version_info > (2, 7):
-    viewkeys = dict.viewkeys
-else:
-    viewkeys = lambda dct: set(dct)
+viewkeys = getattr(dict, 'viewkeys', dict.keys)
 
 
 def dict_diff(old, new):

ansible_mitogen/process.py

@@ -61,10 +61,9 @@ import mitogen.utils
 import ansible
 import ansible.constants as C
 import ansible.errors
+
 import ansible_mitogen.logging
 import ansible_mitogen.services
-
-from mitogen.core import b
 import ansible_mitogen.affinity
 
 
@@ -639,7 +638,7 @@ class MuxProcess(object):
 
         try:
             # Let the parent know our listening socket is ready.
-            mitogen.core.io_op(self.model.child_sock.send, b('1'))
+            mitogen.core.io_op(self.model.child_sock.send, b'1')
             # Block until the socket is closed, which happens on parent exit.
             mitogen.core.io_op(self.model.child_sock.recv, 1)
         finally:

ansible_mitogen/runner.py

@@ -40,7 +40,9 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import atexit
+import ctypes
 import json
+import logging
 import os
 import re
 import shlex
@@ -50,19 +52,12 @@ import tempfile
 import traceback
 import types
 
+from ansible.module_utils.six.moves import shlex_quote
+
 import mitogen.core
 import ansible_mitogen.target  # TODO: circular import
-from mitogen.core import b
-from mitogen.core import bytes_partition
-from mitogen.core import str_rpartition
 from mitogen.core import to_text
 
-try:
-    import ctypes
-except ImportError:
-    # Python 2.4
-    ctypes = None
-
 try:
     # Python >= 3.4, PEP 451 ModuleSpec API
     import importlib.machinery
@@ -77,15 +72,6 @@ try:
 except ImportError:
     from io import StringIO
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
-
 # Prevent accidental import of an Ansible module from hanging on stdin read.
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
@@ -95,7 +81,6 @@ ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
 # explicit call to res_init() on each task invocation. BSD-alikes export it
 # directly, Linux #defines it as "__res_init".
 libc__res_init = None
-if ctypes:
 libc = ctypes.CDLL(None)
 for symbol in 'res_init', '__res_init':
     try:
@@ -103,7 +88,6 @@ if ctypes:
     except AttributeError:
         pass
 
-iteritems = getattr(dict, 'iteritems', dict.items)
 LOG = logging.getLogger(__name__)
 
 
@@ -217,13 +201,13 @@ class EnvironmentFileWatcher(object):
         for line in fp:
             # '   #export foo=some var  ' -> ['#export', 'foo=some var  ']
             bits = shlex_split_b(line)
-            if (not bits) or bits[0].startswith(b('#')):
+            if (not bits) or bits[0].startswith(b'#'):
                 continue
 
-            if bits[0] == b('export'):
+            if bits[0] == b'export':
                 bits.pop(0)
 
-            key, sep, value = bytes_partition(b(' ').join(bits), b('='))
+            key, sep, value = b' '.join(bits).partition(b'=')
             if key and sep:
                 yield key, value
 
@@ -596,7 +580,7 @@ class ModuleUtilsImporter(object):
             mod.__path__ = []
             mod.__package__ = str(fullname)
         else:
-            mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
+            mod.__package__ = str(to_text(fullname).rpartition('.')[0])
         exec(code, mod.__dict__)
         self._loaded.add(fullname)
         return mod
@@ -611,7 +595,7 @@ class TemporaryEnvironment(object):
     def __init__(self, env=None):
         self.original = dict(os.environ)
         self.env = env or {}
-        for key, value in iteritems(self.env):
+        for key, value in mitogen.core.iteritems(self.env):
             key = mitogen.core.to_text(key)
             value = mitogen.core.to_text(value)
             if value is None:
@@ -819,7 +803,7 @@ class ScriptRunner(ProgramRunner):
         self.interpreter_fragment = interpreter_fragment
         self.is_python = is_python
 
-    b_ENCODING_STRING = b('# -*- coding: utf-8 -*-')
+    b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
 
     def _get_program(self):
         return self._rewrite_source(
@@ -852,13 +836,13 @@ class ScriptRunner(ProgramRunner):
         # While Ansible rewrites the #! using ansible_*_interpreter, it is
         # never actually used to execute the script, instead it is a shell
         # fragment consumed by shell/__init__.py::build_module_command().
-        new = [b('#!') + utf8(self.interpreter_fragment)]
+        new = [b'#!' + utf8(self.interpreter_fragment)]
         if self.is_python:
             new.append(self.b_ENCODING_STRING)
 
-        _, _, rest = bytes_partition(s, b('\n'))
+        _, _, rest = s.partition(b'\n')
         new.append(rest)
-        return b('\n').join(new)
+        return b'\n'.join(new)
 
 
 class NewStyleRunner(ScriptRunner):
@@ -971,8 +955,7 @@ class NewStyleRunner(ScriptRunner):
     # change the default encoding. This hack was removed from Ansible long ago,
     # but not before permeating into many third party modules.
     PREHISTORIC_HACK_RE = re.compile(
-        b(r'reload\s*\(\s*sys\s*\)\s*'
-          r'sys\s*\.\s*setdefaultencoding\([^)]+\)')
+        br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)',
     )
 
     def _setup_program(self):
@@ -980,7 +963,7 @@ class NewStyleRunner(ScriptRunner):
             context=self.service_context,
             path=self.path,
         )
-        self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source)
+        self.source = self.PREHISTORIC_HACK_RE.sub(b'', source)
 
     def _get_code(self):
         try:
@@ -998,7 +981,7 @@ class NewStyleRunner(ScriptRunner):
     if mitogen.core.PY3:
         main_module_name = '__main__'
     else:
-        main_module_name = b('__main__')
+        main_module_name = b'__main__'
 
     def _handle_magic_exception(self, mod, exc):
         """
@@ -1030,7 +1013,7 @@ class NewStyleRunner(ScriptRunner):
         approximation of the original package hierarchy, so that relative
         imports function correctly.
         """
-        pkg, sep, modname = str_rpartition(self.py_module_name, '.')
+        pkg, sep, _ = self.py_module_name.rpartition('.')
         if not sep:
             return None
         if mitogen.core.PY3:
@@ -1073,7 +1056,7 @@ class NewStyleRunner(ScriptRunner):
 
 
 class JsonArgsRunner(ScriptRunner):
-    JSON_ARGS = b('<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>')
+    JSON_ARGS = b'<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>'
 
     def _get_args_contents(self):
         return json.dumps(self.args).encode()

ansible_mitogen/services.py

@@ -50,6 +50,8 @@ import threading
 
 import ansible.constants
 
+from ansible.module_utils.six import reraise
+
 import mitogen.core
 import mitogen.service
 import ansible_mitogen.loaders
@@ -66,20 +68,6 @@ LOG = logging.getLogger(__name__)
 ansible_mitogen.loaders.shell_loader.get('sh')
 
 
-if sys.version_info[0] == 3:
-    def reraise(tp, value, tb):
-        if value is None:
-            value = tp()
-        if value.__traceback__ is not tb:
-            raise value.with_traceback(tb)
-        raise value
-else:
-    exec(
-        "def reraise(tp, value, tb=None):\n"
-        "    raise tp, value, tb\n"
-     )
-
-
 def _get_candidate_temp_dirs():
     try:
         # >=2.5

ansible_mitogen/target.py

@@ -39,6 +39,7 @@ __metaclass__ = type
 import errno
 import grp
 import json
+import logging
 import operator
 import os
 import pwd
@@ -51,26 +52,9 @@ import tempfile
 import traceback
 import types
 
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
 import mitogen.core
 import mitogen.parent
 import mitogen.service
-from mitogen.core import b
-
-try:
-    reduce
-except NameError:
-    # Python 3.x.
-    from functools import reduce
-
-try:
-    BaseException
-except NameError:
-    # Python 2.4
-    BaseException = Exception
-
 
 # Ansible since PR #41749 inserts "import __main__" into
 # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
@@ -80,6 +64,9 @@ if not sys.modules.get(str('__main__')):
     sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
 
 import ansible.module_utils.json_utils
+
+from ansible.module_utils.six.moves import reduce
+
 import ansible_mitogen.runner
 
 
@@ -615,8 +602,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
     stdout, stderr = proc.communicate(in_data)
 
     if emulate_tty:
-        stdout = stdout.replace(b('\n'), b('\r\n'))
-    return proc.returncode, stdout, stderr or b('')
+        stdout = stdout.replace(b'\n', b'\r\n')
+    return proc.returncode, stdout, stderr or b''
 
 
 def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
@@ -746,9 +733,7 @@ def set_file_mode(path, spec, fd=None):
     """
     Update the permissions of a file using the same syntax as chmod(1).
     """
-    if isinstance(spec, int):
-        new_mode = spec
-    elif not mitogen.core.PY3 and isinstance(spec, long):
+    if isinstance(spec, mitogen.core.integer_types):
         new_mode = spec
     elif spec.isdigit():
         new_mode = int(spec, 8)

ansible_mitogen/transport_config.py

@@ -65,21 +65,12 @@ import abc
 import os
 import ansible.utils.shlex
 import ansible.constants as C
+import ansible.executor.interpreter_discovery
+import ansible.utils.unsafe_proxy
 
 from ansible.module_utils.six import with_metaclass
 from ansible.module_utils.parsing.convert_bool import boolean
 
-# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary
-try:
-    from ansible.executor.interpreter_discovery import discover_interpreter
-except ImportError:
-    discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python'
-
-try:
-    from ansible.utils.unsafe_proxy import AnsibleUnsafeText
-except ImportError:
-    from ansible.vars.unsafe_proxy import AnsibleUnsafeText
-
 import mitogen.core
 
 
@@ -115,12 +106,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             action._finding_python_interpreter = True
             # fake pipelining so discover_interpreter can be happy
             action._connection.has_pipelining = True
-            s = AnsibleUnsafeText(discover_interpreter(
+            s = ansible.executor.interpreter_discovery.discover_interpreter(
                 action=action,
                 interpreter_name=interpreter_name,
                 discovery_mode=s,
-                task_vars=task_vars))
-
+                task_vars=task_vars,
+            )
+            s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s)
             # cache discovered interpreter
             task_vars['ansible_facts'][discovered_interpreter_config] = s
             action._connection.has_pipelining = False

docs/changelog.rst

@@ -21,6 +21,14 @@ To avail of fixes in an unreleased version, please download a ZIP file
 Unreleased
 ----------
 
+* :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility
+  fallbacks and polyfills into :mod:`mitogen.core`
+* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility
+  fallbacks for Python 2.4 & 2.5.
+* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible
+  releases before 2.10
+* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3
+  compatibility
 
 
 v0.3.10 (2024-09-20)

mitogen/core.py

@@ -102,21 +102,6 @@ try:
 except ImportError:
     cProfile = None
 
-try:
-    import thread
-except ImportError:
-    import threading as thread
-
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 try:
     BaseException
 except NameError:
@@ -169,31 +154,35 @@ STUB_CALL_SERVICE = 111
 #:    :meth:`mitogen.core.Router.add_handler` callbacks to clean up.
 IS_DEAD = 999
 
-try:
-    BaseException
-except NameError:
-    BaseException = Exception
-
 PY24 = sys.version_info < (2, 5)
 PY3 = sys.version_info > (3,)
 if PY3:
+    import pickle
+    import _thread as thread
+    from io import BytesIO
     b = str.encode
     BytesType = bytes
     UnicodeType = str
     FsPathTypes = (str,)
     BufferType = lambda buf, start: memoryview(buf)[start:]
-    long = int
+    integer_types = (int,)
+    iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values
 else:
+    import cPickle as pickle
+    import thread
+    from cStringIO import StringIO as BytesIO
     b = str
     BytesType = str
     FsPathTypes = (str, unicode)
     BufferType = buffer
     UnicodeType = unicode
+    integer_types = (int, long)
+    iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues
 
 AnyTextType = (BytesType, UnicodeType)
 
 try:
-    next
+    next = next
 except NameError:
     next = lambda it: it.next()
 
@@ -400,12 +389,19 @@ now = getattr(time, 'monotonic', time.time)
 
 # Python 2.4
 try:
-    any
+    all, any = all, any
 except NameError:
+    def all(it):
+        for elem in it:
+            if not elem:
+                return False
+        return True
+
     def any(it):
         for elem in it:
             if elem:
                 return True
+        return False
 
 
 def _partition(s, sep, find):
@@ -1065,8 +1061,8 @@ class Sender(object):
 
 def _unpickle_sender(router, context_id, dst_handle):
     if not (isinstance(router, Router) and
-            isinstance(context_id, (int, long)) and context_id >= 0 and
-            isinstance(dst_handle, (int, long)) and dst_handle > 0):
+            isinstance(context_id, integer_types) and context_id >= 0 and
+            isinstance(dst_handle, integer_types) and dst_handle > 0):
         raise TypeError('cannot unpickle Sender: bad input or missing router')
     return Sender(Context(router, context_id), dst_handle)
 
@@ -2508,7 +2504,7 @@ class Context(object):
 
 
 def _unpickle_context(context_id, name, router=None):
-    if not (isinstance(context_id, (int, long)) and context_id >= 0 and (
+    if not (isinstance(context_id, integer_types) and context_id >= 0 and (
         (name is None) or
         (isinstance(name, UnicodeType) and len(name) < 100))
     ):

mitogen/master.py

@@ -74,9 +74,11 @@ import mitogen.core
 import mitogen.minify
 import mitogen.parent
 
+from mitogen.core import any
 from mitogen.core import b
 from mitogen.core import IOLOG
 from mitogen.core import LOG
+from mitogen.core import next
 from mitogen.core import str_partition
 from mitogen.core import str_rpartition
 from mitogen.core import to_text
@@ -84,17 +86,6 @@ from mitogen.core import to_text
 imap = getattr(itertools, 'imap', map)
 izip = getattr(itertools, 'izip', zip)
 
-try:
-    any
-except NameError:
-    from mitogen.core import any
-
-try:
-    next
-except NameError:
-    from mitogen.core import next
-
-
 RLOG = logging.getLogger('mitogen.ctx')
 
 

mitogen/parent.py

@@ -56,15 +56,13 @@ import zlib
 # Absolute imports for <2.5.
 select = __import__('select')
 
-try:
-    import thread
-except ImportError:
-    import threading as thread
-
 import mitogen.core
 from mitogen.core import b
 from mitogen.core import bytes_partition
 from mitogen.core import IOLOG
+from mitogen.core import itervalues
+from mitogen.core import next
+from mitogen.core import thread
 
 
 LOG = logging.getLogger(__name__)
@@ -80,15 +78,6 @@ except IOError:
     SELINUX_ENABLED = False
 
 
-try:
-    next
-except NameError:
-    # Python 2.4/2.5
-    from mitogen.core import next
-
-
-itervalues = getattr(dict, 'itervalues', dict.values)
-
 if mitogen.core.PY3:
     xrange = range
     closure_attr = '__closure__'

mitogen/service.py

@@ -39,18 +39,10 @@ import threading
 
 import mitogen.core
 import mitogen.select
+from mitogen.core import all
 from mitogen.core import b
 from mitogen.core import str_rpartition
 
-try:
-    all
-except NameError:
-    def all(it):
-        for elem in it:
-            if not elem:
-                return False
-        return True
-
 
 LOG = logging.getLogger(__name__)
 

mitogen/ssh.py

@@ -43,11 +43,6 @@ except ImportError:
 import mitogen.parent
 from mitogen.core import b
 
-try:
-    any
-except NameError:
-    from mitogen.core import any
-
 
 LOG = logging.getLogger(__name__)
 

mitogen/su.py

@@ -34,11 +34,6 @@ import re
 import mitogen.core
 import mitogen.parent
 
-try:
-    any
-except NameError:
-    from mitogen.core import any
-
 
 LOG = logging.getLogger(__name__)
 

mitogen/utils.py

@@ -37,13 +37,7 @@ import sys
 import mitogen.core
 import mitogen.master
 
-
-iteritems = getattr(dict, 'iteritems', dict.items)
-
-if mitogen.core.PY3:
-    iteritems = dict.items
-else:
-    iteritems = dict.iteritems
+from mitogen.core import iteritems
 
 
 def setup_gil():

tests/ansible/lib/modules/custom_python_detect_environment.py

@@ -11,17 +11,6 @@ import socket
 import sys
 
 
-try:
-    all
-except NameError:
-    # Python 2.4
-    def all(it):
-        for elem in it:
-            if not elem:
-                return False
-        return True
-
-
 def main():
     module = AnsibleModule(argument_spec={})
     module.exit_json(

tests/connection_test.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import signal
 import sys
 import tempfile
 import threading

tests/iter_split_test.py

@@ -2,11 +2,7 @@ import unittest
 
 import mitogen.core
 
-try:
-    next
-except NameError:
-    def next(it):
-        return it.next()
+from mitogen.core import next
 
 
 class IterSplitTest(unittest.TestCase):

tests/lxc_test.py

@@ -3,9 +3,6 @@ import os
 import mitogen.lxc
 import mitogen.parent
 
-try:
-    any
-except NameError:
 from mitogen.core import any
 
 import testlib

tests/poller_test.py

@@ -8,14 +8,10 @@ import unittest
 import mitogen.core
 import mitogen.parent
 
-import testlib
-
-try:
-    next
-except NameError:
-    # Python 2.4
 from mitogen.core import next
 
+import testlib
+
 
 class SockMixin(object):
     def tearDown(self):

tox.ini

@@ -48,6 +48,9 @@
 # ansible == 9.x     ansible-core ~= 2.16.0
 # ansible == 10.x    ansible-core ~= 2.17.0
 
+# See also
+# - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix
+
 [tox]
 envlist =
     init,