WIP

ansible_mitogen/affinity.py

@@ -83,7 +83,6 @@ import multiprocessing
 import os
 import struct
 
-import mitogen.core
 import mitogen.parent
 
 
@@ -265,7 +264,7 @@ class LinuxPolicy(FixedPolicy):
         for x in range(16):
             chunks.append(struct.pack('<Q', mask & shiftmask))
             mask >>= 64
-        return mitogen.core.b('').join(chunks)
+        return b''.join(chunks)
 
     def _get_thread_ids(self):
         try:

ansible_mitogen/logging.py

@@ -32,15 +32,13 @@ __metaclass__ = type
 import logging
 import os
 
+import ansible.utils.display
+
 import mitogen.core
 import mitogen.utils
 
-try:
-    from __main__ import display
-except ImportError:
-    import ansible.utils.display
-    display = ansible.utils.display.Display()
 
+display = ansible.utils.display.Display()
 
 #: The process name set via :func:`set_process_name`.
 _process_name = None

ansible_mitogen/mixins.py

@@ -35,18 +35,16 @@ import pwd
 import random
 import traceback
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-from ansible.module_utils._text import to_bytes
-from ansible.parsing.utils.jsonify import jsonify
-
 import ansible
 import ansible.constants
 import ansible.plugins
 import ansible.plugins.action
+import ansible.utils.unsafe_proxy
+import ansible.vars.clean
+
+from ansible.module_utils.common.text.converters import to_bytes, to_text
+from ansible.module_utils.six.moves import shlex_quote
+from ansible.parsing.utils.jsonify import jsonify
 
 import mitogen.core
 import mitogen.select
@@ -57,24 +55,6 @@ import ansible_mitogen.target
 import ansible_mitogen.utils
 import ansible_mitogen.utils.unsafe
 
-from ansible.module_utils._text import to_text
-
-try:
-    from ansible.utils.unsafe_proxy import wrap_var
-except ImportError:
-    from ansible.vars.unsafe_proxy import wrap_var
-
-try:
-    # ansible 2.8 moved remove_internal_keys to the clean module
-    from ansible.vars.clean import remove_internal_keys
-except ImportError:
-    try:
-        from ansible.vars.manager import remove_internal_keys
-    except ImportError:
-        # ansible 2.3.3 has remove_internal_keys as a protected func on the action class
-        # we'll fallback to calling self._remove_internal_keys in this case
-        remove_internal_keys = lambda a: "Not found"
-
 
 LOG = logging.getLogger(__name__)
 
@@ -413,10 +393,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
             self._remove_tmp_path(tmp)
 
         # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
-        # handle ansible 2.3.3 that has remove_internal_keys in a different place
+        ansible.vars.clean.remove_internal_keys(result)
-        check = remove_internal_keys(result)
-        if check == 'Not found':
-            self._remove_internal_keys(result)
 
         # taken from _execute_module of ansible 2.8.6
         # propagate interpreter discovery results back to the controller
@@ -440,7 +417,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
                 result['deprecations'] = []
             result['deprecations'].extend(self._discovery_deprecation_warnings)
 
-        return wrap_var(result)
+        return ansible.utils.unsafe_proxy.wrap_var(result)
 
     def _postprocess_response(self, result):
         """

ansible_mitogen/planner.py

@@ -477,7 +477,7 @@ def read_file(path):
     finally:
         os.close(fd)
 
-    return mitogen.core.b('').join(bits)
+    return b''.join(bits)
 
 
 def _propagate_deps(invocation, planner, context):

ansible_mitogen/plugins/connection/mitogen_local.py

@@ -42,13 +42,7 @@ except ImportError:
 import ansible_mitogen.connection
 import ansible_mitogen.process
 
-
+viewkeys = getattr(dict, 'viewkeys', dict.keys)
-if sys.version_info > (3,):
-    viewkeys = dict.keys
-elif sys.version_info > (2, 7):
-    viewkeys = dict.viewkeys
-else:
-    viewkeys = lambda dct: set(dct)
 
 
 def dict_diff(old, new):

ansible_mitogen/process.py

@@ -61,10 +61,9 @@ import mitogen.utils
 import ansible
 import ansible.constants as C
 import ansible.errors
+
 import ansible_mitogen.logging
 import ansible_mitogen.services
-
-from mitogen.core import b
 import ansible_mitogen.affinity
 
 
@@ -639,7 +638,7 @@ class MuxProcess(object):
 
         try:
             # Let the parent know our listening socket is ready.
-            mitogen.core.io_op(self.model.child_sock.send, b('1'))
+            mitogen.core.io_op(self.model.child_sock.send, b'1')
             # Block until the socket is closed, which happens on parent exit.
             mitogen.core.io_op(self.model.child_sock.recv, 1)
         finally:

ansible_mitogen/runner.py

@@ -40,7 +40,10 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import atexit
+import ctypes
 import json
+import io
+import logging
 import os
 import re
 import shlex
@@ -50,19 +53,12 @@ import tempfile
 import traceback
 import types
 
+from ansible.module_utils.six.moves import iteritems, shlex_quote
+
 import mitogen.core
 import ansible_mitogen.target  # TODO: circular import
-from mitogen.core import b
-from mitogen.core import bytes_partition
-from mitogen.core import str_rpartition
 from mitogen.core import to_text
 
-try:
-    import ctypes
-except ImportError:
-    # Python 2.4
-    ctypes = None
-
 try:
     # Python >= 3.4, PEP 451 ModuleSpec API
     import importlib.machinery
@@ -71,21 +67,6 @@ except ImportError:
     # Python < 3.4, PEP 302 Import Hooks
     import imp
 
-try:
-    # Cannot use cStringIO as it does not support Unicode.
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
-
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
-
 # Prevent accidental import of an Ansible module from hanging on stdin read.
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
@@ -103,7 +84,6 @@ if ctypes:
         except AttributeError:
             pass
 
-iteritems = getattr(dict, 'iteritems', dict.items)
 LOG = logging.getLogger(__name__)
 
 
@@ -217,13 +197,13 @@ class EnvironmentFileWatcher(object):
         for line in fp:
             # '   #export foo=some var  ' -> ['#export', 'foo=some var  ']
             bits = shlex_split_b(line)
-            if (not bits) or bits[0].startswith(b('#')):
+            if (not bits) or bits[0].startswith(b'#'):
                 continue
 
-            if bits[0] == b('export'):
+            if bits[0] == b'export':
                 bits.pop(0)
 
-            key, sep, value = bytes_partition(b(' ').join(bits), b('='))
+            key, sep, value = b' '.join(bits).partition(b'=')
             if key and sep:
                 yield key, value
 
@@ -596,7 +576,7 @@ class ModuleUtilsImporter(object):
             mod.__path__ = []
             mod.__package__ = str(fullname)
         else:
-            mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
+            mod.__package__ = str(to_text(fullname).rpartition('.')[0])
         exec(code, mod.__dict__)
         self._loaded.add(fullname)
         return mod
@@ -647,11 +627,11 @@ class NewStyleStdio(object):
         self.original_stdout = sys.stdout
         self.original_stderr = sys.stderr
         self.original_stdin = sys.stdin
-        sys.stdout = StringIO()
+        sys.stdout = io.StringIO()
-        sys.stderr = StringIO()
+        sys.stderr = io.StringIO()
         encoded = json.dumps({'ANSIBLE_MODULE_ARGS': args})
         ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded)
-        sys.stdin = StringIO(mitogen.core.to_text(encoded))
+        sys.stdin = io.StringIO(mitogen.core.to_text(encoded))
 
         self.original_get_path = getattr(ansible.module_utils.basic,
                                         'get_module_path', None)
@@ -819,7 +799,7 @@ class ScriptRunner(ProgramRunner):
         self.interpreter_fragment = interpreter_fragment
         self.is_python = is_python
 
-    b_ENCODING_STRING = b('# -*- coding: utf-8 -*-')
+    b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
 
     def _get_program(self):
         return self._rewrite_source(
@@ -852,13 +832,13 @@ class ScriptRunner(ProgramRunner):
         # While Ansible rewrites the #! using ansible_*_interpreter, it is
         # never actually used to execute the script, instead it is a shell
         # fragment consumed by shell/__init__.py::build_module_command().
-        new = [b('#!') + utf8(self.interpreter_fragment)]
+        new = [b'#!' + utf8(self.interpreter_fragment)]
         if self.is_python:
             new.append(self.b_ENCODING_STRING)
 
-        _, _, rest = bytes_partition(s, b('\n'))
+        _, _, rest = s.partition(b'\n')
         new.append(rest)
-        return b('\n').join(new)
+        return b'\n'.join(new)
 
 
 class NewStyleRunner(ScriptRunner):
@@ -971,8 +951,7 @@ class NewStyleRunner(ScriptRunner):
     # change the default encoding. This hack was removed from Ansible long ago,
     # but not before permeating into many third party modules.
     PREHISTORIC_HACK_RE = re.compile(
-        b(r'reload\s*\(\s*sys\s*\)\s*'
+        rb'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)',
-          r'sys\s*\.\s*setdefaultencoding\([^)]+\)')
     )
 
     def _setup_program(self):
@@ -980,7 +959,7 @@ class NewStyleRunner(ScriptRunner):
             context=self.service_context,
             path=self.path,
         )
-        self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source)
+        self.source = self.PREHISTORIC_HACK_RE.sub(b'', source)
 
     def _get_code(self):
         try:
@@ -998,7 +977,7 @@ class NewStyleRunner(ScriptRunner):
     if mitogen.core.PY3:
         main_module_name = '__main__'
     else:
-        main_module_name = b('__main__')
+        main_module_name = b'__main__'
 
     def _handle_magic_exception(self, mod, exc):
         """
@@ -1030,7 +1009,7 @@ class NewStyleRunner(ScriptRunner):
         approximation of the original package hierarchy, so that relative
         imports function correctly.
         """
-        pkg, sep, modname = str_rpartition(self.py_module_name, '.')
+        pkg, sep, _ = self.py_module_name.rpartition('.')
         if not sep:
             return None
         if mitogen.core.PY3:
@@ -1073,7 +1052,7 @@ class NewStyleRunner(ScriptRunner):
 
 
 class JsonArgsRunner(ScriptRunner):
-    JSON_ARGS = b('<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>')
+    JSON_ARGS = b'<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>'
 
     def _get_args_contents(self):
         return json.dumps(self.args).encode()

ansible_mitogen/target.py

@@ -39,6 +39,7 @@ __metaclass__ = type
 import errno
 import grp
 import json
+import logging
 import operator
 import os
 import pwd
@@ -51,13 +52,9 @@ import tempfile
 import traceback
 import types
 
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
 import mitogen.core
 import mitogen.parent
 import mitogen.service
-from mitogen.core import b
 
 try:
     reduce
@@ -65,12 +62,6 @@ except NameError:
     # Python 3.x.
     from functools import reduce
 
-try:
-    BaseException
-except NameError:
-    # Python 2.4
-    BaseException = Exception
-
 
 # Ansible since PR #41749 inserts "import __main__" into
 # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
@@ -615,8 +606,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
     stdout, stderr = proc.communicate(in_data)
 
     if emulate_tty:
-        stdout = stdout.replace(b('\n'), b('\r\n'))
+        stdout = stdout.replace(b'\n', b'\r\n')
-    return proc.returncode, stdout, stderr or b('')
+    return proc.returncode, stdout, stderr or b''
 
 
 def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
@@ -746,9 +737,7 @@ def set_file_mode(path, spec, fd=None):
     """
     Update the permissions of a file using the same syntax as chmod(1).
     """
-    if isinstance(spec, int):
+    if isinstance(spec, (int, mitogen.core.long)):
-        new_mode = spec
-    elif not mitogen.core.PY3 and isinstance(spec, long):
         new_mode = spec
     elif spec.isdigit():
         new_mode = int(spec, 8)

ansible_mitogen/transport_config.py

@@ -65,21 +65,12 @@ import abc
 import os
 import ansible.utils.shlex
 import ansible.constants as C
+import ansible.executor.interpreter_discovery
+import ansible.utils.unsafe_proxy
 
 from ansible.module_utils.six import with_metaclass
 from ansible.module_utils.parsing.convert_bool import boolean
 
-# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary
-try:
-    from ansible.executor.interpreter_discovery import discover_interpreter
-except ImportError:
-    discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python'
-
-try:
-    from ansible.utils.unsafe_proxy import AnsibleUnsafeText
-except ImportError:
-    from ansible.vars.unsafe_proxy import AnsibleUnsafeText
-
 import mitogen.core
 
 
@@ -115,12 +106,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             action._finding_python_interpreter = True
             # fake pipelining so discover_interpreter can be happy
             action._connection.has_pipelining = True
-            s = AnsibleUnsafeText(discover_interpreter(
+            s = ansible.executor.interpreter_discovery.discover_interpreter(
                 action=action,
                 interpreter_name=interpreter_name,
                 discovery_mode=s,
-                task_vars=task_vars))
+                task_vars=task_vars,
-
+            )
+            s = ansible.utils.unsafe_proxy.UnsafeProxy(s)
             # cache discovered interpreter
             task_vars['ansible_facts'][discovered_interpreter_config] = s
             action._connection.has_pipelining = False

examples/the_basics.py

@@ -18,7 +18,6 @@ import mitogen.service
 import mitogen.utils
 
 
-
 def get_file_contents(path):
     """
     Get the contents of a file.
@@ -79,7 +78,6 @@ def md5sum(path):
     return hashlib.md5(get_file_contents(path)).hexdigest()
 
 
-
 def work_on_machine(context):
     """
     Do stuff to a remote context.

mitogen/compat/__init__.py

@@ -0,0 +1,4 @@
+try:
+    from shlex import quote as shlex_quote
+except ImportError:
+    from pipes import quote as shlex_quote

mitogen/core.py

@@ -102,21 +102,11 @@ try:
 except ImportError:
     cProfile = None
 
-try:
-    import thread
-except ImportError:
-    import threading as thread
-
 try:
     import cPickle as pickle
 except ImportError:
     import pickle
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 try:
     BaseException
 except NameError:
@@ -169,31 +159,32 @@ STUB_CALL_SERVICE = 111
 #:    :meth:`mitogen.core.Router.add_handler` callbacks to clean up.
 IS_DEAD = 999
 
-try:
-    BaseException
-except NameError:
-    BaseException = Exception
-
 PY24 = sys.version_info < (2, 5)
 PY3 = sys.version_info > (3,)
 if PY3:
+    import _thread as thread
+    from io import BytesIO
     b = str.encode
     BytesType = bytes
     UnicodeType = str
     FsPathTypes = (str,)
     BufferType = lambda buf, start: memoryview(buf)[start:]
+    iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values
     long = int
 else:
+    import thread
+    from cStringIO import StringIO as BytesIO
     b = str
     BytesType = str
     FsPathTypes = (str, unicode)
     BufferType = buffer
     UnicodeType = unicode
+    iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues
 
 AnyTextType = (BytesType, UnicodeType)
 
 try:
-    next
+    next = next
 except NameError:
     next = lambda it: it.next()
 
@@ -313,14 +304,14 @@ class Kwargs(dict):
     """
     if PY3:
         def __init__(self, dct):
-            for k, v in dct.items():
+            for k, v in iteritems(dct):
                 if type(k) is bytes:
                     self[k.decode()] = v
                 else:
                     self[k] = v
     elif sys.version_info < (2, 6, 5):
         def __init__(self, dct):
-            for k, v in dct.iteritems():
+            for k, v in iteritems(dct):
                 if type(k) is unicode:
                     k, _ = encodings.utf_8.encode(k)
                 self[k] = v
@@ -400,12 +391,19 @@ now = getattr(time, 'monotonic', time.time)
 
 # Python 2.4
 try:
-    any
+    all, any = all, any
 except NameError:
+    def all(it):
+        for elem in it:
+            if not elem:
+                return False
+        return True
+
     def any(it):
         for elem in it:
             if elem:
                 return True
+        return False
 
 
 def _partition(s, sep, find):

mitogen/master.py

@@ -74,9 +74,11 @@ import mitogen.core
 import mitogen.minify
 import mitogen.parent
 
+from mitogen.core import any
 from mitogen.core import b
 from mitogen.core import IOLOG
 from mitogen.core import LOG
+from mitogen.core import next
 from mitogen.core import str_partition
 from mitogen.core import str_rpartition
 from mitogen.core import to_text
@@ -84,17 +86,6 @@ from mitogen.core import to_text
 imap = getattr(itertools, 'imap', map)
 izip = getattr(itertools, 'izip', zip)
 
-try:
-    any
-except NameError:
-    from mitogen.core import any
-
-try:
-    next
-except NameError:
-    from mitogen.core import next
-
-
 RLOG = logging.getLogger('mitogen.ctx')
 
 
@@ -1003,6 +994,7 @@ class ModuleFinder(object):
             fullname, _, _ = str_rpartition(to_text(fullname), u'.')
             yield fullname
 
+    def _
     def find_related_imports(self, fullname):
         """
         Return a list of non-stdlib modules that are directly imported by
@@ -1064,12 +1056,14 @@ class ModuleFinder(object):
             for which source code can be retrieved
         :type fullname: str
         """
+        #LOG.info('find_related(%r)', fullname)
         stack = [fullname]
         found = set()
 
         while stack:
             name = stack.pop(0)
             names = self.find_related_imports(name)
+            #LOG.info('find_related(%r) -> %r -> %r', fullname, name, names)
             stack.extend(set(names).difference(set(found).union(stack)))
             found.update(names)
 

mitogen/parent.py

@@ -56,16 +56,13 @@ import zlib
 # Absolute imports for <2.5.
 select = __import__('select')
 
-try:
-    import thread
-except ImportError:
-    import threading as thread
-
 import mitogen.core
 from mitogen.core import b
 from mitogen.core import bytes_partition
 from mitogen.core import IOLOG
-
+from mitogen.core import itervalues
+from mitogen.core import next
+from mitogen.core import thread
 
 LOG = logging.getLogger(__name__)
 
@@ -79,16 +76,6 @@ try:
 except IOError:
     SELINUX_ENABLED = False
 
-
-try:
-    next
-except NameError:
-    # Python 2.4/2.5
-    from mitogen.core import next
-
-
-itervalues = getattr(dict, 'itervalues', dict.values)
-
 if mitogen.core.PY3:
     xrange = range
     closure_attr = '__closure__'

mitogen/service.py

@@ -39,18 +39,10 @@ import threading
 
 import mitogen.core
 import mitogen.select
+from mitogen.core import all
 from mitogen.core import b
 from mitogen.core import str_rpartition
 
-try:
-    all
-except NameError:
-    def all(it):
-        for elem in it:
-            if not elem:
-                return False
-        return True
-
 
 LOG = logging.getLogger(__name__)
 

mitogen/ssh.py

@@ -35,19 +35,10 @@ Construct new children via the OpenSSH client.
 import logging
 import re
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
 import mitogen.parent
+from mitogen.compat import shlex_quote
 from mitogen.core import b
 
-try:
-    any
-except NameError:
-    from mitogen.core import any
-
 
 LOG = logging.getLogger(__name__)
 

mitogen/su.py

@@ -34,11 +34,6 @@ import re
 import mitogen.core
 import mitogen.parent
 
-try:
-    any
-except NameError:
-    from mitogen.core import any
-
 
 LOG = logging.getLogger(__name__)
 

mitogen/utils.py

@@ -37,13 +37,7 @@ import sys
 import mitogen.core
 import mitogen.master
 
-
+from mitogen.core import iteritems
-iteritems = getattr(dict, 'iteritems', dict.items)
-
-if mitogen.core.PY3:
-    iteritems = dict.items
-else:
-    iteritems = dict.iteritems
 
 
 def setup_gil():

tests/ansible/lib/modules/custom_python_detect_environment.py

@@ -11,17 +11,6 @@ import socket
 import sys
 
 
-try:
-    all
-except NameError:
-    # Python 2.4
-    def all(it):
-        for elem in it:
-            if not elem:
-                return False
-        return True
-
-
 def main():
     module = AnsibleModule(argument_spec={})
     module.exit_json(

tests/connection_test.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import signal
 import sys
 import tempfile
 import threading

tests/iter_split_test.py

@@ -2,11 +2,7 @@ import unittest
 
 import mitogen.core
 
-try:
+from mitogen.core import next
-    next
-except NameError:
-    def next(it):
-        return it.next()
 
 
 class IterSplitTest(unittest.TestCase):

tests/lxc_test.py

@@ -3,9 +3,6 @@ import os
 import mitogen.lxc
 import mitogen.parent
 
-try:
-    any
-except NameError:
 from mitogen.core import any
 
 import testlib

tests/poller_test.py

@@ -8,14 +8,10 @@ import unittest
 import mitogen.core
 import mitogen.parent
 
-import testlib
-
-try:
-    next
-except NameError:
-    # Python 2.4
 from mitogen.core import next
 
+import testlib
+
 
 class SockMixin(object):
     def tearDown(self):