From 8f940e2ccb7df4aafdd9ecf815fceca3aef40ec2 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 13:08:44 +0100
Subject: [PATCH 01/18] issue #590: teach importer to handle self-replacing
 modules

---
 docs/changelog.rst                             | 6 ++++++
 mitogen/core.py                                | 5 ++++-
 tests/data/simple_pkg/imports_replaces_self.py | 6 ++++++
 tests/data/simple_pkg/replaces_self.py         | 4 ++++
 tests/importer_test.py                         | 9 +++++++++
 5 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 tests/data/simple_pkg/imports_replaces_self.py
 create mode 100644 tests/data/simple_pkg/replaces_self.py

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 8e1ddc1a..4a3a5612 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -30,6 +30,12 @@ Enhancements
   <https://docs.ansible.com/ansible/latest/plugins/become.html>`_
   functionality, which will be addressed in a future release.
 
+Fixes
+^^^^^
+
+* `#590 <https://github.com/dw/mitogen/issues/590>`_: the importer can handle
+  modules that replace themselves in :mod:`sys.modules` during import.
+
 
 Thanks!
 ~~~~~~~
diff --git a/mitogen/core.py b/mitogen/core.py
index ff77bba9..0d88d7f0 100644
--- a/mitogen/core.py
+++ b/mitogen/core.py
@@ -1355,7 +1355,10 @@ class Importer(object):
             exec(code, vars(mod))
         else:
             exec('exec code in vars(mod)')
-        return mod
+
+        # #590: if a module replaces itself in sys.modules during import, below
+        # is necessary. This matches PyImport_ExecCodeModuleEx()
+        return sys.modules.get(fullname, mod)
 
     def get_filename(self, fullname):
         if fullname in self._cache:
diff --git a/tests/data/simple_pkg/imports_replaces_self.py b/tests/data/simple_pkg/imports_replaces_self.py
new file mode 100644
index 00000000..b1b43813
--- /dev/null
+++ b/tests/data/simple_pkg/imports_replaces_self.py
@@ -0,0 +1,6 @@
+# issue #590: this module imports a module that replaces itself in sys.modules
+# during initialization.
+import simple_pkg.replaces_self
+
+def subtract_one(n):
+    return simple_pkg.replaces_self.subtract_one(n)
diff --git a/tests/data/simple_pkg/replaces_self.py b/tests/data/simple_pkg/replaces_self.py
new file mode 100644
index 00000000..5d853ebf
--- /dev/null
+++ b/tests/data/simple_pkg/replaces_self.py
@@ -0,0 +1,4 @@
+# issue #590: this module replaces itself in sys.modules during initialization.
+import sys
+import simple_pkg.b
+sys.modules[__name__] = simple_pkg.b
diff --git a/tests/importer_test.py b/tests/importer_test.py
index fc6f4bd6..c796f7d0 100644
--- a/tests/importer_test.py
+++ b/tests/importer_test.py
@@ -12,6 +12,7 @@ import mitogen.utils
 from mitogen.core import b
 
 import testlib
+import simple_pkg.imports_replaces_self
 
 
 class ImporterMixin(testlib.RouterMixin):
@@ -214,5 +215,13 @@ class Python24LineCacheTest(testlib.TestCase):
     pass
 
 
+class SelfReplacingModuleTest(testlib.RouterMixin, testlib.TestCase):
+    # issue #590
+    def test_importer_handles_self_replacement(self):
+        c = self.router.local()
+        self.assertEquals(0,
+            c.call(simple_pkg.imports_replaces_self.subtract_one, 1))
+
+
 if __name__ == '__main__':
     unittest2.main()

From 7a5c436a39b66e197c964656e30733f7d4ace3b2 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 13:08:44 +0100
Subject: [PATCH 02/18] issue #590: Ansible test for module_utils.distro use.

---
 .../lib/modules/custom_python_uses_distro.py        | 13 +++++++++++++
 .../regression/issue_590__sys_modules_crap.yml      |  9 +++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 tests/ansible/lib/modules/custom_python_uses_distro.py
 create mode 100644 tests/ansible/regression/issue_590__sys_modules_crap.yml

diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py
new file mode 100644
index 00000000..c5ffbb84
--- /dev/null
+++ b/tests/ansible/lib/modules/custom_python_uses_distro.py
@@ -0,0 +1,13 @@
+#!/usr/bin/python
+# issue #590: I am an Ansible new-style Python module that tries to use
+# ansible.module_utils.distro.
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils import distro
+
+def main():
+    module = AnsibleModule(argument_spec={})
+    module.exit_json(info=distro.info())
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/ansible/regression/issue_590__sys_modules_crap.yml b/tests/ansible/regression/issue_590__sys_modules_crap.yml
new file mode 100644
index 00000000..83a9a286
--- /dev/null
+++ b/tests/ansible/regression/issue_590__sys_modules_crap.yml
@@ -0,0 +1,9 @@
+
+- hosts: test-targets
+  tasks:
+  - custom_python_uses_distro:
+    register: out
+
+  - assert:
+      that:
+      - "'id' in out.info"

From 875ff5c06023e56385d0e47bbfbf215329ea21a4 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 13:08:44 +0100
Subject: [PATCH 03/18] issue #590: refactor ModuleFinder and teach it a new
 special case.

Now it's possible to find both packages and modules when the
sys.modules[...] state for the package/module is junk. Previously only
modules were possible.

This also refactors things to make writing better tests for all these
cases much simpler.
---
 mitogen/master.py | 235 ++++++++++++++++++++++++++++------------------
 1 file changed, 146 insertions(+), 89 deletions(-)

diff --git a/mitogen/master.py b/mitogen/master.py
index 1396f4e1..7188ba14 100644
--- a/mitogen/master.py
+++ b/mitogen/master.py
@@ -36,6 +36,7 @@ contexts.
 """
 
 import dis
+import errno
 import imp
 import inspect
 import itertools
@@ -142,6 +143,41 @@ def get_child_modules(path):
     return [to_text(name) for _, name, _ in it]
 
 
+def _looks_like_script(path):
+    """
+    Return :data:`True` if the (possibly extensionless) file at `path`
+    resembles a Python script. For now we simply verify the file contains
+    ASCII text.
+    """
+    try:
+        fp = open(path, 'rb')
+    except IOError:
+        e = sys.exc_info()[1]
+        if e.args[0] == errno.EISDIR:
+            return False
+        raise
+
+    try:
+        sample = fp.read(512).decode('latin-1')
+        return not set(sample).difference(string.printable)
+    finally:
+        fp.close()
+
+
+def _py_filename(path):
+    if not path:
+        return None
+
+    if path[-4:] in ('.pyc', '.pyo'):
+        path = path.rstrip('co')
+
+    if path.endswith('.py'):
+        return path
+
+    if os.path.exists(path) and _looks_like_script(path):
+        return path
+
+
 def _get_core_source():
     """
     Master version of parent.get_core_source().
@@ -368,56 +404,22 @@ class LogForwarder(object):
         return 'LogForwarder(%r)' % (self._router,)
 
 
-class ModuleFinder(object):
-    """
-    Given the name of a loaded module, make a best-effort attempt at finding
-    related modules likely needed by a child context requesting the original
-    module.
-    """
-    def __init__(self):
-        #: Import machinery is expensive, keep :py:meth`:get_module_source`
-        #: results around.
-        self._found_cache = {}
-
-        #: Avoid repeated dependency scanning, which is expensive.
-        self._related_cache = {}
-
+class FinderMethod(object):
     def __repr__(self):
-        return 'ModuleFinder()'
+        return '%s()' % (type(self).__name__,)
 
-    def _looks_like_script(self, path):
-        """
-        Return :data:`True` if the (possibly extensionless) file at `path`
-        resembles a Python script. For now we simply verify the file contains
-        ASCII text.
-        """
-        fp = open(path, 'rb')
-        try:
-            sample = fp.read(512).decode('latin-1')
-            return not set(sample).difference(string.printable)
-        finally:
-            fp.close()
+    def find(self, fullname):
+        pass
 
-    def _py_filename(self, path):
-        if not path:
-            return None
 
-        if path[-4:] in ('.pyc', '.pyo'):
-            path = path.rstrip('co')
-
-        if path.endswith('.py'):
-            return path
-
-        if os.path.exists(path) and self._looks_like_script(path):
-            return path
-
-    def _get_main_module_defective_python_3x(self, fullname):
-        """
-        Recent versions of Python 3.x introduced an incomplete notion of
-        importer specs, and in doing so created permanent asymmetry in the
-        :mod:`pkgutil` interface handling for the `__main__` module. Therefore
-        we must handle `__main__` specially.
-        """
+class DefectivePython3xMainMethod(FinderMethod):
+    """
+    Recent versions of Python 3.x introduced an incomplete notion of
+    importer specs, and in doing so created permanent asymmetry in the
+    :mod:`pkgutil` interface handling for the `__main__` module. Therefore
+    we must handle `__main__` specially.
+    """
+    def find(self, fullname):
         if fullname != '__main__':
             return None
 
@@ -426,7 +428,7 @@ class ModuleFinder(object):
             return None
 
         path = getattr(mod, '__file__', None)
-        if not (os.path.exists(path) and self._looks_like_script(path)):
+        if not (os.path.exists(path) and _looks_like_script(path)):
             return None
 
         fp = open(path, 'rb')
@@ -437,11 +439,13 @@ class ModuleFinder(object):
 
         return path, source, False
 
-    def _get_module_via_pkgutil(self, fullname):
-        """
-        Attempt to fetch source code via pkgutil. In an ideal world, this would
-        be the only required implementation of get_module().
-        """
+
+class PkgutilMethod(FinderMethod):
+    """
+    Attempt to fetch source code via pkgutil. In an ideal world, this would
+    be the only required implementation of get_module().
+    """
+    def find(self, fullname):
         try:
             # Pre-'import spec' this returned None, in Python3.6 it raises
             # ImportError.
@@ -458,7 +462,7 @@ class ModuleFinder(object):
             return
 
         try:
-            path = self._py_filename(loader.get_filename(fullname))
+            path = _py_filename(loader.get_filename(fullname))
             source = loader.get_source(fullname)
             is_pkg = loader.is_package(fullname)
         except (AttributeError, ImportError):
@@ -484,19 +488,27 @@ class ModuleFinder(object):
 
         return path, source, is_pkg
 
-    def _get_module_via_sys_modules(self, fullname):
-        """
-        Attempt to fetch source code via sys.modules. This is specifically to
-        support __main__, but it may catch a few more cases.
-        """
+
+class SysModulesMethod(FinderMethod):
+    """
+    Attempt to fetch source code via sys.modules. This is specifically to
+    support __main__, but it may catch a few more cases.
+    """
+    def find(self, fullname):
         module = sys.modules.get(fullname)
         LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module)
+        if getattr(module, '__name__', None) != fullname:
+            LOG.debug('sys.modules[%r].__name__ does not match %r, assuming '
+                      'this is a hacky module alias and ignoring it',
+                      fullname, fullname)
+            return
+
         if not isinstance(module, types.ModuleType):
             LOG.debug('sys.modules[%r] absent or not a regular module',
                       fullname)
             return
 
-        path = self._py_filename(getattr(module, '__file__', ''))
+        path = _py_filename(getattr(module, '__file__', ''))
         if not path:
             return
 
@@ -517,12 +529,19 @@ class ModuleFinder(object):
 
         return path, source, is_pkg
 
-    def _get_module_via_parent_enumeration(self, fullname):
-        """
-        Attempt to fetch source code by examining the module's (hopefully less
-        insane) parent package. Required for older versions of
-        ansible.compat.six and plumbum.colors.
-        """
+
+class ParentEnumerationMethod(FinderMethod):
+    """
+    Attempt to fetch source code by examining the module's (hopefully less
+    insane) parent package. Required for older versions of
+    ansible.compat.six and plumbum.colors, and Ansible 2.8
+    ansible.module_utils.distro.
+
+    For cases like module_utils.distro, this must handle cases where a package
+    transmuted itself into a totally unrelated module during import and vice
+    versa.
+    """
+    def find(self, fullname):
         if fullname not in sys.modules:
             # Don't attempt this unless a module really exists in sys.modules,
             # else we could return junk.
@@ -531,30 +550,68 @@ class ModuleFinder(object):
         pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
         pkg = sys.modules.get(pkgname)
         if pkg is None or not hasattr(pkg, '__file__'):
+            LOG.debug('%r: %r is not a package or lacks __file__ attribute',
+                      self, pkgname)
             return
 
-        pkg_path = os.path.dirname(pkg.__file__)
+        pkg_path = [os.path.dirname(pkg.__file__)]
         try:
-            fp, path, ext = imp.find_module(modname, [pkg_path])
-            try:
-                path = self._py_filename(path)
-                if not path:
-                    fp.close()
-                    return
-
-                source = fp.read()
-            finally:
-                if fp:
-                    fp.close()
-
-            if isinstance(source, mitogen.core.UnicodeType):
-                # get_source() returns "string" according to PEP-302, which was
-                # reinterpreted for Python 3 to mean a Unicode string.
-                source = source.encode('utf-8')
-            return path, source, False
+            fp, path, (suffix, _, kind) = imp.find_module(modname, pkg_path)
         except ImportError:
             e = sys.exc_info()[1]
-            LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e)
+            LOG.debug('%r: imp.find_module(%r, %r) -> %s',
+                      self, modname, [pkg_path], e)
+            return None
+
+        if kind == imp.PKG_DIRECTORY:
+            return self._found_package(fullname, path)
+        else:
+            return self._found_module(fullname, path, fp)
+
+    def _found_package(self, fullname, path):
+        path = os.path.join(path, '__init__.py')
+        LOG.debug('%r: %r is PKG_DIRECTORY: %r', self, fullname, path)
+        return self._found_module(
+            fullname=fullname,
+            path=path,
+            fp=open(path, 'rb'),
+            is_pkg=True,
+        )
+
+    def _found_module(self, fullname, path, fp, is_pkg=False):
+        try:
+            path = _py_filename(path)
+            if not path:
+                return
+
+            source = fp.read()
+        finally:
+            if fp:
+                fp.close()
+
+        if isinstance(source, mitogen.core.UnicodeType):
+            # get_source() returns "string" according to PEP-302, which was
+            # reinterpreted for Python 3 to mean a Unicode string.
+            source = source.encode('utf-8')
+        return path, source, is_pkg
+
+
+class ModuleFinder(object):
+    """
+    Given the name of a loaded module, make a best-effort attempt at finding
+    related modules likely needed by a child context requesting the original
+    module.
+    """
+    def __init__(self):
+        #: Import machinery is expensive, keep :py:meth`:get_module_source`
+        #: results around.
+        self._found_cache = {}
+
+        #: Avoid repeated dependency scanning, which is expensive.
+        self._related_cache = {}
+
+    def __repr__(self):
+        return 'ModuleFinder()'
 
     def add_source_override(self, fullname, path, source, is_pkg):
         """
@@ -576,10 +633,10 @@ class ModuleFinder(object):
         self._found_cache[fullname] = (path, source, is_pkg)
 
     get_module_methods = [
-        _get_main_module_defective_python_3x,
-        _get_module_via_pkgutil,
-        _get_module_via_sys_modules,
-        _get_module_via_parent_enumeration,
+        DefectivePython3xMainMethod(),
+        PkgutilMethod(),
+        SysModulesMethod(),
+        ParentEnumerationMethod(),
     ]
 
     def get_module_source(self, fullname):
@@ -595,7 +652,7 @@ class ModuleFinder(object):
             return tup
 
         for method in self.get_module_methods:
-            tup = method(self, fullname)
+            tup = method.find(fullname)
             if tup:
                 #LOG.debug('%r returned %r', method, tup)
                 break

From c1db0d3858424ccaaab2e9dd0ee14fcbb4b91558 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 13:08:44 +0100
Subject: [PATCH 04/18] issue #590: move example modules to module_finder/,
 fix/add tests

---
 .../module_finder_testmod/__init__.py         |  0
 .../module_finder_testmod/empty_mod.py        |  0
 .../module_finder_testmod/regular_mod.py      |  0
 .../sibling_dep_mod_abs_import.py             |  0
 .../sibling_dep_mod_py2_import.py             |  0
 .../sibling_dep_mod_rel_import.py             |  0
 .../pkg_like_plumbum/__init__.py              |  0
 .../pkg_like_plumbum/colors.py                |  0
 .../simple_pkg/__init__.py                    |  0
 .../data/{ => module_finder}/simple_pkg/a.py  |  0
 .../data/{ => module_finder}/simple_pkg/b.py  |  0
 .../simple_pkg/imports_replaces_self.py       |  0
 .../{ => module_finder}/simple_pkg/ping.py    |  0
 .../simple_pkg/replaces_self.py               |  0
 .../six_brokenpkg/__init__.py                 |  0
 .../{ => module_finder}/six_brokenpkg/_six.py |  0
 .../{ => module_finder}/webproject/manage.py  |  0
 .../webproject/serve_django_app.py            |  0
 .../webproject/webapp/__init__.py             |  0
 .../webproject/webapp/admin.py                |  0
 .../webproject/webapp/apps.py                 |  0
 .../webproject/webapp/migrations/__init__.py  |  0
 .../webproject/webapp/models.py               |  0
 .../webproject/webapp/tests.py                |  0
 .../webproject/webapp/views.py                |  0
 .../webproject/webproject/__init__.py         |  0
 .../webproject/webproject/settings.py         |  0
 .../webproject/webproject/urls.py             |  0
 .../webproject/webproject/wsgi.py             |  0
 tests/module_finder_test.py                   | 54 ++++++++++++-------
 tests/testlib.py                              |  1 -
 31 files changed, 36 insertions(+), 19 deletions(-)
 rename tests/data/{ => module_finder}/module_finder_testmod/__init__.py (100%)
 rename tests/data/{ => module_finder}/module_finder_testmod/empty_mod.py (100%)
 rename tests/data/{ => module_finder}/module_finder_testmod/regular_mod.py (100%)
 rename tests/data/{ => module_finder}/module_finder_testmod/sibling_dep_mod_abs_import.py (100%)
 rename tests/data/{ => module_finder}/module_finder_testmod/sibling_dep_mod_py2_import.py (100%)
 rename tests/data/{ => module_finder}/module_finder_testmod/sibling_dep_mod_rel_import.py (100%)
 rename tests/data/{ => module_finder}/pkg_like_plumbum/__init__.py (100%)
 rename tests/data/{ => module_finder}/pkg_like_plumbum/colors.py (100%)
 rename tests/data/{ => module_finder}/simple_pkg/__init__.py (100%)
 rename tests/data/{ => module_finder}/simple_pkg/a.py (100%)
 rename tests/data/{ => module_finder}/simple_pkg/b.py (100%)
 rename tests/data/{ => module_finder}/simple_pkg/imports_replaces_self.py (100%)
 rename tests/data/{ => module_finder}/simple_pkg/ping.py (100%)
 rename tests/data/{ => module_finder}/simple_pkg/replaces_self.py (100%)
 rename tests/data/{ => module_finder}/six_brokenpkg/__init__.py (100%)
 rename tests/data/{ => module_finder}/six_brokenpkg/_six.py (100%)
 rename tests/data/{ => module_finder}/webproject/manage.py (100%)
 rename tests/data/{ => module_finder}/webproject/serve_django_app.py (100%)
 rename tests/data/{ => module_finder}/webproject/webapp/__init__.py (100%)
 rename tests/data/{ => module_finder}/webproject/webapp/admin.py (100%)
 rename tests/data/{ => module_finder}/webproject/webapp/apps.py (100%)
 rename tests/data/{ => module_finder}/webproject/webapp/migrations/__init__.py (100%)
 rename tests/data/{ => module_finder}/webproject/webapp/models.py (100%)
 rename tests/data/{ => module_finder}/webproject/webapp/tests.py (100%)
 rename tests/data/{ => module_finder}/webproject/webapp/views.py (100%)
 rename tests/data/{ => module_finder}/webproject/webproject/__init__.py (100%)
 rename tests/data/{ => module_finder}/webproject/webproject/settings.py (100%)
 rename tests/data/{ => module_finder}/webproject/webproject/urls.py (100%)
 rename tests/data/{ => module_finder}/webproject/webproject/wsgi.py (100%)

diff --git a/tests/data/module_finder_testmod/__init__.py b/tests/data/module_finder/module_finder_testmod/__init__.py
similarity index 100%
rename from tests/data/module_finder_testmod/__init__.py
rename to tests/data/module_finder/module_finder_testmod/__init__.py
diff --git a/tests/data/module_finder_testmod/empty_mod.py b/tests/data/module_finder/module_finder_testmod/empty_mod.py
similarity index 100%
rename from tests/data/module_finder_testmod/empty_mod.py
rename to tests/data/module_finder/module_finder_testmod/empty_mod.py
diff --git a/tests/data/module_finder_testmod/regular_mod.py b/tests/data/module_finder/module_finder_testmod/regular_mod.py
similarity index 100%
rename from tests/data/module_finder_testmod/regular_mod.py
rename to tests/data/module_finder/module_finder_testmod/regular_mod.py
diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py b/tests/data/module_finder/module_finder_testmod/sibling_dep_mod_abs_import.py
similarity index 100%
rename from tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py
rename to tests/data/module_finder/module_finder_testmod/sibling_dep_mod_abs_import.py
diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py b/tests/data/module_finder/module_finder_testmod/sibling_dep_mod_py2_import.py
similarity index 100%
rename from tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py
rename to tests/data/module_finder/module_finder_testmod/sibling_dep_mod_py2_import.py
diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py b/tests/data/module_finder/module_finder_testmod/sibling_dep_mod_rel_import.py
similarity index 100%
rename from tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py
rename to tests/data/module_finder/module_finder_testmod/sibling_dep_mod_rel_import.py
diff --git a/tests/data/pkg_like_plumbum/__init__.py b/tests/data/module_finder/pkg_like_plumbum/__init__.py
similarity index 100%
rename from tests/data/pkg_like_plumbum/__init__.py
rename to tests/data/module_finder/pkg_like_plumbum/__init__.py
diff --git a/tests/data/pkg_like_plumbum/colors.py b/tests/data/module_finder/pkg_like_plumbum/colors.py
similarity index 100%
rename from tests/data/pkg_like_plumbum/colors.py
rename to tests/data/module_finder/pkg_like_plumbum/colors.py
diff --git a/tests/data/simple_pkg/__init__.py b/tests/data/module_finder/simple_pkg/__init__.py
similarity index 100%
rename from tests/data/simple_pkg/__init__.py
rename to tests/data/module_finder/simple_pkg/__init__.py
diff --git a/tests/data/simple_pkg/a.py b/tests/data/module_finder/simple_pkg/a.py
similarity index 100%
rename from tests/data/simple_pkg/a.py
rename to tests/data/module_finder/simple_pkg/a.py
diff --git a/tests/data/simple_pkg/b.py b/tests/data/module_finder/simple_pkg/b.py
similarity index 100%
rename from tests/data/simple_pkg/b.py
rename to tests/data/module_finder/simple_pkg/b.py
diff --git a/tests/data/simple_pkg/imports_replaces_self.py b/tests/data/module_finder/simple_pkg/imports_replaces_self.py
similarity index 100%
rename from tests/data/simple_pkg/imports_replaces_self.py
rename to tests/data/module_finder/simple_pkg/imports_replaces_self.py
diff --git a/tests/data/simple_pkg/ping.py b/tests/data/module_finder/simple_pkg/ping.py
similarity index 100%
rename from tests/data/simple_pkg/ping.py
rename to tests/data/module_finder/simple_pkg/ping.py
diff --git a/tests/data/simple_pkg/replaces_self.py b/tests/data/module_finder/simple_pkg/replaces_self.py
similarity index 100%
rename from tests/data/simple_pkg/replaces_self.py
rename to tests/data/module_finder/simple_pkg/replaces_self.py
diff --git a/tests/data/six_brokenpkg/__init__.py b/tests/data/module_finder/six_brokenpkg/__init__.py
similarity index 100%
rename from tests/data/six_brokenpkg/__init__.py
rename to tests/data/module_finder/six_brokenpkg/__init__.py
diff --git a/tests/data/six_brokenpkg/_six.py b/tests/data/module_finder/six_brokenpkg/_six.py
similarity index 100%
rename from tests/data/six_brokenpkg/_six.py
rename to tests/data/module_finder/six_brokenpkg/_six.py
diff --git a/tests/data/webproject/manage.py b/tests/data/module_finder/webproject/manage.py
similarity index 100%
rename from tests/data/webproject/manage.py
rename to tests/data/module_finder/webproject/manage.py
diff --git a/tests/data/webproject/serve_django_app.py b/tests/data/module_finder/webproject/serve_django_app.py
similarity index 100%
rename from tests/data/webproject/serve_django_app.py
rename to tests/data/module_finder/webproject/serve_django_app.py
diff --git a/tests/data/webproject/webapp/__init__.py b/tests/data/module_finder/webproject/webapp/__init__.py
similarity index 100%
rename from tests/data/webproject/webapp/__init__.py
rename to tests/data/module_finder/webproject/webapp/__init__.py
diff --git a/tests/data/webproject/webapp/admin.py b/tests/data/module_finder/webproject/webapp/admin.py
similarity index 100%
rename from tests/data/webproject/webapp/admin.py
rename to tests/data/module_finder/webproject/webapp/admin.py
diff --git a/tests/data/webproject/webapp/apps.py b/tests/data/module_finder/webproject/webapp/apps.py
similarity index 100%
rename from tests/data/webproject/webapp/apps.py
rename to tests/data/module_finder/webproject/webapp/apps.py
diff --git a/tests/data/webproject/webapp/migrations/__init__.py b/tests/data/module_finder/webproject/webapp/migrations/__init__.py
similarity index 100%
rename from tests/data/webproject/webapp/migrations/__init__.py
rename to tests/data/module_finder/webproject/webapp/migrations/__init__.py
diff --git a/tests/data/webproject/webapp/models.py b/tests/data/module_finder/webproject/webapp/models.py
similarity index 100%
rename from tests/data/webproject/webapp/models.py
rename to tests/data/module_finder/webproject/webapp/models.py
diff --git a/tests/data/webproject/webapp/tests.py b/tests/data/module_finder/webproject/webapp/tests.py
similarity index 100%
rename from tests/data/webproject/webapp/tests.py
rename to tests/data/module_finder/webproject/webapp/tests.py
diff --git a/tests/data/webproject/webapp/views.py b/tests/data/module_finder/webproject/webapp/views.py
similarity index 100%
rename from tests/data/webproject/webapp/views.py
rename to tests/data/module_finder/webproject/webapp/views.py
diff --git a/tests/data/webproject/webproject/__init__.py b/tests/data/module_finder/webproject/webproject/__init__.py
similarity index 100%
rename from tests/data/webproject/webproject/__init__.py
rename to tests/data/module_finder/webproject/webproject/__init__.py
diff --git a/tests/data/webproject/webproject/settings.py b/tests/data/module_finder/webproject/webproject/settings.py
similarity index 100%
rename from tests/data/webproject/webproject/settings.py
rename to tests/data/module_finder/webproject/webproject/settings.py
diff --git a/tests/data/webproject/webproject/urls.py b/tests/data/module_finder/webproject/webproject/urls.py
similarity index 100%
rename from tests/data/webproject/webproject/urls.py
rename to tests/data/module_finder/webproject/webproject/urls.py
diff --git a/tests/data/webproject/webproject/wsgi.py b/tests/data/module_finder/webproject/webproject/wsgi.py
similarity index 100%
rename from tests/data/webproject/webproject/wsgi.py
rename to tests/data/module_finder/webproject/webproject/wsgi.py
diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index 409adc6d..93755aa0 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -8,6 +8,9 @@ import mitogen.master
 
 import testlib
 
+MODS_PATH = testlib.data_path('module_finder')
+sys.path.append(MODS_PATH)
+
 
 class ConstructorTest(testlib.TestCase):
     klass = mitogen.master.ModuleFinder
@@ -51,10 +54,10 @@ class IsStdlibNameTest(testlib.TestCase):
 
 
 class GetMainModuleDefectivePython3x(testlib.TestCase):
-    klass = mitogen.master.ModuleFinder
+    klass = mitogen.master.DefectivePython3xMainMethod
 
     def call(self, fullname):
-        return self.klass()._get_main_module_defective_python_3x(fullname)
+        return self.klass().find(fullname)
 
     def test_builtin(self):
         self.assertEquals(None, self.call('sys'))
@@ -77,23 +80,23 @@ class GetMainModuleDefectivePython3x(testlib.TestCase):
         self.assertFalse(is_pkg)
 
 
-class GetModuleViaPkgutilTest(testlib.TestCase):
-    klass = mitogen.master.ModuleFinder
+class PkgutilMethodTest(testlib.TestCase):
+    klass = mitogen.master.PkgutilMethod
 
     def call(self, fullname):
-        return self.klass()._get_module_via_pkgutil(fullname)
+        return self.klass().find(fullname)
 
     def test_empty_source_pkg(self):
         path, src, is_pkg = self.call('module_finder_testmod')
         self.assertEquals(path,
-            testlib.data_path('module_finder_testmod/__init__.py'))
+            os.path.join(MODS_PATH, 'module_finder_testmod/__init__.py'))
         self.assertEquals(mitogen.core.b(''), src)
         self.assertTrue(is_pkg)
 
     def test_empty_source_module(self):
         path, src, is_pkg = self.call('module_finder_testmod.empty_mod')
         self.assertEquals(path,
-            testlib.data_path('module_finder_testmod/empty_mod.py'))
+            os.path.join(MODS_PATH, 'module_finder_testmod/empty_mod.py'))
         self.assertEquals(mitogen.core.b(''), src)
         self.assertFalse(is_pkg)
 
@@ -101,17 +104,17 @@ class GetModuleViaPkgutilTest(testlib.TestCase):
         from module_finder_testmod import regular_mod
         path, src, is_pkg = self.call('module_finder_testmod.regular_mod')
         self.assertEquals(path,
-            testlib.data_path('module_finder_testmod/regular_mod.py'))
+            os.path.join(MODS_PATH, 'module_finder_testmod/regular_mod.py'))
         self.assertEquals(mitogen.core.to_text(src),
                           inspect.getsource(regular_mod))
         self.assertFalse(is_pkg)
 
 
-class GetModuleViaSysModulesTest(testlib.TestCase):
-    klass = mitogen.master.ModuleFinder
+class SysModulesMethodTest(testlib.TestCase):
+    klass = mitogen.master.SysModulesMethod
 
     def call(self, fullname):
-        return self.klass()._get_module_via_sys_modules(fullname)
+        return self.klass().find(fullname)
 
     def test_main(self):
         import __main__
@@ -133,10 +136,10 @@ class GetModuleViaSysModulesTest(testlib.TestCase):
 
 
 class GetModuleViaParentEnumerationTest(testlib.TestCase):
-    klass = mitogen.master.ModuleFinder
+    klass = mitogen.master.ParentEnumerationMethod
 
     def call(self, fullname):
-        return self.klass()._get_module_via_parent_enumeration(fullname)
+        return self.klass().find(fullname)
 
     def test_main_fails(self):
         import __main__
@@ -157,13 +160,28 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
         # plumbum has been eating too many rainbow-colored pills
         import pkg_like_plumbum.colors
         path, src, is_pkg = self.call('pkg_like_plumbum.colors')
-        self.assertEquals(path,
-            testlib.data_path('pkg_like_plumbum/colors.py'))
+        modpath = os.path.join(MODS_PATH, 'pkg_like_plumbum/colors.py')
+        self.assertEquals(path, modpath)
 
-        s = open(testlib.data_path('pkg_like_plumbum/colors.py'), 'rb').read()
-        self.assertEquals(src, s)
+        self.assertEquals(src, open(modpath, 'rb').read())
         self.assertFalse(is_pkg)
 
+    def test_ansible_module_utils_distro_succeeds(self):
+        # #590: a package that turns itself into a module.
+        import pkg_like_ansible.module_utils.distro as d
+        self.assertEquals(d.I_AM, "the module that replaced the package")
+        self.assertEquals(
+            sys.modules['pkg_like_ansible.module_utils.distro'].__name__,
+            'pkg_like_ansible.module_utils.distro._distro'
+        )
+
+        path, src, is_pkg = self.call('pkg_like_ansible.module_utils.distro')
+        modpath = os.path.join(MODS_PATH,
+            'pkg_like_ansible/module_utils/distro/__init__.py')
+        self.assertEquals(path, modpath)
+        self.assertEquals(src, open(modpath, 'rb').read())
+        self.assertEquals(is_pkg, True)
+
 
 class ResolveRelPathTest(testlib.TestCase):
     klass = mitogen.master.ModuleFinder
@@ -235,7 +253,7 @@ class FindRelatedTest(testlib.TestCase):
 
 if sys.version_info > (2, 6):
     class DjangoMixin(object):
-        WEBPROJECT_PATH = testlib.data_path('webproject')
+        WEBPROJECT_PATH = os.path.join(MODS_PATH, 'webproject')
 
         # TODO: rip out Django and replace with a static tree of weird imports
         # that don't depend on .. Django! The hack below is because the version
diff --git a/tests/testlib.py b/tests/testlib.py
index 37c3c654..2ee672be 100644
--- a/tests/testlib.py
+++ b/tests/testlib.py
@@ -41,7 +41,6 @@ except NameError:
 
 LOG = logging.getLogger(__name__)
 DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
-sys.path.append(DATA_DIR)
 
 if mitogen.is_master:
     mitogen.utils.log_to_file()

From 4f23f0bec10b8d311a06ef0adeaf8ebe2bd023e9 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 13:08:44 +0100
Subject: [PATCH 05/18] issue #590: update comment to indicate the hack is
 permanent

---
 ansible_mitogen/runner.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py
index a8dae8b1..8dbddadd 100644
--- a/ansible_mitogen/runner.py
+++ b/ansible_mitogen/runner.py
@@ -763,7 +763,16 @@ class NewStyleRunner(ScriptRunner):
             try:
                 mitogen.core.import_module(fullname)
             except ImportError:
-                # TODO: this is a huge hack to work around issue #590.
+                # #590: Ansible 2.8 module_utils.distro is a package that
+                # replaces itself in sys.modules with a non-package during
+                # import. Prior to replacement, it is a real package containing
+                # a '_distro' submodule which is used on 2.x. Given a 2.x
+                # controller and 3.x target, the import hook never needs to run
+                # again before this replacement occurs, and 'distro' is
+                # replaced with a module from the stdlib. In this case as this
+                # loop progresses to the next entry and attempts to preload
+                # 'distro._distro', the import mechanism will fail. So here we
+                # silently ignore any failure for it.
                 if fullname != 'ansible.module_utils.distro._distro':
                     raise
 

From 72ab917c89835930579de5f8fa2c0a6f3a3221ab Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 13:08:44 +0100
Subject: [PATCH 06/18] issue #590: add FinderMethod docstrings.

---
 mitogen/master.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/mitogen/master.py b/mitogen/master.py
index 7188ba14..1eed8c60 100644
--- a/mitogen/master.py
+++ b/mitogen/master.py
@@ -405,11 +405,26 @@ class LogForwarder(object):
 
 
 class FinderMethod(object):
+    """
+    Interface to a method for locating a Python module or package given its
+    name according to the running Python interpreter. You'd think this was a
+    simple task, right? Naive young fellow, welcome to the real world.
+    """
     def __repr__(self):
         return '%s()' % (type(self).__name__,)
 
     def find(self, fullname):
-        pass
+        """
+        Accept a canonical module name and return `(path, source, is_pkg)`
+        tuples, where:
+
+        * `path`: Unicode string containing path to source file.
+        * `source`: Bytestring containing source file's content.
+        * `is_pkg`: :data:`True` if `fullname` is a package.
+
+        :returns:
+            :data:`None` if not found, or tuple as described above.
+        """
 
 
 class DefectivePython3xMainMethod(FinderMethod):

From cf1e7129a744e2f4650b16a69ca8bd679d1b513e Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 13:08:44 +0100
Subject: [PATCH 07/18] issue #590: add dummy package for new test.

---
 tests/data/module_finder/pkg_like_ansible/__init__.py        | 0
 .../module_finder/pkg_like_ansible/module_utils/__init__.py  | 0
 .../pkg_like_ansible/module_utils/distro/__init__.py         | 5 +++++
 .../pkg_like_ansible/module_utils/distro/_distro.py          | 1 +
 4 files changed, 6 insertions(+)
 create mode 100644 tests/data/module_finder/pkg_like_ansible/__init__.py
 create mode 100644 tests/data/module_finder/pkg_like_ansible/module_utils/__init__.py
 create mode 100644 tests/data/module_finder/pkg_like_ansible/module_utils/distro/__init__.py
 create mode 100644 tests/data/module_finder/pkg_like_ansible/module_utils/distro/_distro.py

diff --git a/tests/data/module_finder/pkg_like_ansible/__init__.py b/tests/data/module_finder/pkg_like_ansible/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/data/module_finder/pkg_like_ansible/module_utils/__init__.py b/tests/data/module_finder/pkg_like_ansible/module_utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/data/module_finder/pkg_like_ansible/module_utils/distro/__init__.py b/tests/data/module_finder/pkg_like_ansible/module_utils/distro/__init__.py
new file mode 100644
index 00000000..3a149657
--- /dev/null
+++ b/tests/data/module_finder/pkg_like_ansible/module_utils/distro/__init__.py
@@ -0,0 +1,5 @@
+# #590: a package that turns itself into a module.
+I_AM = "the package that was replaced"
+import sys
+from pkg_like_ansible.module_utils.distro import _distro
+sys.modules[__name__] = _distro
diff --git a/tests/data/module_finder/pkg_like_ansible/module_utils/distro/_distro.py b/tests/data/module_finder/pkg_like_ansible/module_utils/distro/_distro.py
new file mode 100644
index 00000000..9f113fef
--- /dev/null
+++ b/tests/data/module_finder/pkg_like_ansible/module_utils/distro/_distro.py
@@ -0,0 +1 @@
+I_AM = "the module that replaced the package"

From 5eb10aacef96135ab1b09228ba0f617b0bff1899 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 18:31:21 +0100
Subject: [PATCH 08/18] master: fix _is_stdlib_path() failure on Ubuntu.

---
 mitogen/master.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/mitogen/master.py b/mitogen/master.py
index 1eed8c60..681b870b 100644
--- a/mitogen/master.py
+++ b/mitogen/master.py
@@ -46,8 +46,9 @@ import pkgutil
 import re
 import string
 import sys
-import time
+import sysconfig
 import threading
+import time
 import types
 import zlib
 
@@ -93,10 +94,15 @@ def _stdlib_paths():
         'real_prefix',  # virtualenv: only set inside a virtual environment.
         'base_prefix',  # venv: always set, equal to prefix if outside.
     ]
-    prefixes = (getattr(sys, a) for a in attr_candidates if hasattr(sys, a))
+    prefixes = (getattr(sys, a, None) for a in attr_candidates)
     version = 'python%s.%s' % sys.version_info[0:2]
-    return set(os.path.abspath(os.path.join(p, 'lib', version))
-               for p in prefixes)
+    s = set(os.path.abspath(os.path.join(p, 'lib', version))
+            for p in prefixes if p is not None)
+
+    # When running 'unit2 tests/module_finder_test.py' in a Py2 venv on Ubuntu
+    # 18.10, above is insufficient to catch the real directory.
+    s.add(sysconfig.get_config_var('DESTLIB'))
+    return s
 
 
 def is_stdlib_name(modname):
@@ -425,6 +431,7 @@ class FinderMethod(object):
         :returns:
             :data:`None` if not found, or tuple as described above.
         """
+        raise NotImplementedError()
 
 
 class DefectivePython3xMainMethod(FinderMethod):

From 2f68a5a6608dbb716964599496494586853181ea Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 18:55:42 +0100
Subject: [PATCH 09/18] module_finder_test: mask one more difference between
 unit2 vs. direct start

---
 tests/module_finder_test.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index 93755aa0..a77f1084 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -120,7 +120,13 @@ class SysModulesMethodTest(testlib.TestCase):
         import __main__
         path, src, is_pkg = self.call('__main__')
         self.assertEquals(path, __main__.__file__)
-        self.assertEquals(src, open(path, 'rb').read())
+
+        # linecache adds a line ending to the final line if one is missing.
+        actual_src = open(path, 'rb').read()
+        if actual_src[-1] != '\n':
+            actual_src += '\n'
+
+        self.assertEquals(src, actual_src)
         self.assertFalse(is_pkg)
 
     def test_dylib_fails(self):

From 2f29f3e8e638a48d242d40cfda2533988d2fbbea Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 18:56:56 +0100
Subject: [PATCH 10/18] tests: rearrange test modules again, they're used in
 multiple places

They're back on sys.path.
---
 .../module_finder_testmod/__init__.py            |  0
 .../module_finder_testmod/empty_mod.py           |  0
 .../module_finder_testmod/regular_mod.py         |  0
 .../sibling_dep_mod_abs_import.py                |  0
 .../sibling_dep_mod_py2_import.py                |  0
 .../sibling_dep_mod_rel_import.py                |  0
 .../pkg_like_ansible/__init__.py                 |  0
 .../pkg_like_ansible/module_utils/__init__.py    |  0
 .../module_utils/distro/__init__.py              |  0
 .../module_utils/distro/_distro.py               |  0
 .../pkg_like_plumbum/__init__.py                 |  0
 .../pkg_like_plumbum/colors.py                   |  0
 .../simple_pkg/__init__.py                       |  0
 .../{module_finder => importer}/simple_pkg/a.py  |  0
 .../{module_finder => importer}/simple_pkg/b.py  |  0
 .../simple_pkg/imports_replaces_self.py          |  0
 .../simple_pkg/ping.py                           |  0
 .../simple_pkg/replaces_self.py                  |  0
 .../six_brokenpkg/__init__.py                    |  0
 .../six_brokenpkg/_six.py                        |  0
 .../webproject/manage.py                         |  0
 .../webproject/serve_django_app.py               |  0
 .../webproject/webapp/__init__.py                |  0
 .../webproject/webapp/admin.py                   |  0
 .../webproject/webapp/apps.py                    |  0
 .../webproject/webapp/migrations/__init__.py     |  0
 .../webproject/webapp/models.py                  |  0
 .../webproject/webapp/tests.py                   |  0
 .../webproject/webapp/views.py                   |  0
 .../webproject/webproject/__init__.py            |  0
 .../webproject/webproject/settings.py            |  0
 .../webproject/webproject/urls.py                |  0
 .../webproject/webproject/wsgi.py                |  0
 tests/module_finder_test.py                      | 16 +++++++---------
 tests/testlib.py                                 |  5 +++++
 35 files changed, 12 insertions(+), 9 deletions(-)
 rename tests/data/{module_finder => importer}/module_finder_testmod/__init__.py (100%)
 rename tests/data/{module_finder => importer}/module_finder_testmod/empty_mod.py (100%)
 rename tests/data/{module_finder => importer}/module_finder_testmod/regular_mod.py (100%)
 rename tests/data/{module_finder => importer}/module_finder_testmod/sibling_dep_mod_abs_import.py (100%)
 rename tests/data/{module_finder => importer}/module_finder_testmod/sibling_dep_mod_py2_import.py (100%)
 rename tests/data/{module_finder => importer}/module_finder_testmod/sibling_dep_mod_rel_import.py (100%)
 rename tests/data/{module_finder => importer}/pkg_like_ansible/__init__.py (100%)
 rename tests/data/{module_finder => importer}/pkg_like_ansible/module_utils/__init__.py (100%)
 rename tests/data/{module_finder => importer}/pkg_like_ansible/module_utils/distro/__init__.py (100%)
 rename tests/data/{module_finder => importer}/pkg_like_ansible/module_utils/distro/_distro.py (100%)
 rename tests/data/{module_finder => importer}/pkg_like_plumbum/__init__.py (100%)
 rename tests/data/{module_finder => importer}/pkg_like_plumbum/colors.py (100%)
 rename tests/data/{module_finder => importer}/simple_pkg/__init__.py (100%)
 rename tests/data/{module_finder => importer}/simple_pkg/a.py (100%)
 rename tests/data/{module_finder => importer}/simple_pkg/b.py (100%)
 rename tests/data/{module_finder => importer}/simple_pkg/imports_replaces_self.py (100%)
 rename tests/data/{module_finder => importer}/simple_pkg/ping.py (100%)
 rename tests/data/{module_finder => importer}/simple_pkg/replaces_self.py (100%)
 rename tests/data/{module_finder => importer}/six_brokenpkg/__init__.py (100%)
 rename tests/data/{module_finder => importer}/six_brokenpkg/_six.py (100%)
 rename tests/data/{module_finder => importer}/webproject/manage.py (100%)
 rename tests/data/{module_finder => importer}/webproject/serve_django_app.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webapp/__init__.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webapp/admin.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webapp/apps.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webapp/migrations/__init__.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webapp/models.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webapp/tests.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webapp/views.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webproject/__init__.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webproject/settings.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webproject/urls.py (100%)
 rename tests/data/{module_finder => importer}/webproject/webproject/wsgi.py (100%)

diff --git a/tests/data/module_finder/module_finder_testmod/__init__.py b/tests/data/importer/module_finder_testmod/__init__.py
similarity index 100%
rename from tests/data/module_finder/module_finder_testmod/__init__.py
rename to tests/data/importer/module_finder_testmod/__init__.py
diff --git a/tests/data/module_finder/module_finder_testmod/empty_mod.py b/tests/data/importer/module_finder_testmod/empty_mod.py
similarity index 100%
rename from tests/data/module_finder/module_finder_testmod/empty_mod.py
rename to tests/data/importer/module_finder_testmod/empty_mod.py
diff --git a/tests/data/module_finder/module_finder_testmod/regular_mod.py b/tests/data/importer/module_finder_testmod/regular_mod.py
similarity index 100%
rename from tests/data/module_finder/module_finder_testmod/regular_mod.py
rename to tests/data/importer/module_finder_testmod/regular_mod.py
diff --git a/tests/data/module_finder/module_finder_testmod/sibling_dep_mod_abs_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py
similarity index 100%
rename from tests/data/module_finder/module_finder_testmod/sibling_dep_mod_abs_import.py
rename to tests/data/importer/module_finder_testmod/sibling_dep_mod_abs_import.py
diff --git a/tests/data/module_finder/module_finder_testmod/sibling_dep_mod_py2_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py
similarity index 100%
rename from tests/data/module_finder/module_finder_testmod/sibling_dep_mod_py2_import.py
rename to tests/data/importer/module_finder_testmod/sibling_dep_mod_py2_import.py
diff --git a/tests/data/module_finder/module_finder_testmod/sibling_dep_mod_rel_import.py b/tests/data/importer/module_finder_testmod/sibling_dep_mod_rel_import.py
similarity index 100%
rename from tests/data/module_finder/module_finder_testmod/sibling_dep_mod_rel_import.py
rename to tests/data/importer/module_finder_testmod/sibling_dep_mod_rel_import.py
diff --git a/tests/data/module_finder/pkg_like_ansible/__init__.py b/tests/data/importer/pkg_like_ansible/__init__.py
similarity index 100%
rename from tests/data/module_finder/pkg_like_ansible/__init__.py
rename to tests/data/importer/pkg_like_ansible/__init__.py
diff --git a/tests/data/module_finder/pkg_like_ansible/module_utils/__init__.py b/tests/data/importer/pkg_like_ansible/module_utils/__init__.py
similarity index 100%
rename from tests/data/module_finder/pkg_like_ansible/module_utils/__init__.py
rename to tests/data/importer/pkg_like_ansible/module_utils/__init__.py
diff --git a/tests/data/module_finder/pkg_like_ansible/module_utils/distro/__init__.py b/tests/data/importer/pkg_like_ansible/module_utils/distro/__init__.py
similarity index 100%
rename from tests/data/module_finder/pkg_like_ansible/module_utils/distro/__init__.py
rename to tests/data/importer/pkg_like_ansible/module_utils/distro/__init__.py
diff --git a/tests/data/module_finder/pkg_like_ansible/module_utils/distro/_distro.py b/tests/data/importer/pkg_like_ansible/module_utils/distro/_distro.py
similarity index 100%
rename from tests/data/module_finder/pkg_like_ansible/module_utils/distro/_distro.py
rename to tests/data/importer/pkg_like_ansible/module_utils/distro/_distro.py
diff --git a/tests/data/module_finder/pkg_like_plumbum/__init__.py b/tests/data/importer/pkg_like_plumbum/__init__.py
similarity index 100%
rename from tests/data/module_finder/pkg_like_plumbum/__init__.py
rename to tests/data/importer/pkg_like_plumbum/__init__.py
diff --git a/tests/data/module_finder/pkg_like_plumbum/colors.py b/tests/data/importer/pkg_like_plumbum/colors.py
similarity index 100%
rename from tests/data/module_finder/pkg_like_plumbum/colors.py
rename to tests/data/importer/pkg_like_plumbum/colors.py
diff --git a/tests/data/module_finder/simple_pkg/__init__.py b/tests/data/importer/simple_pkg/__init__.py
similarity index 100%
rename from tests/data/module_finder/simple_pkg/__init__.py
rename to tests/data/importer/simple_pkg/__init__.py
diff --git a/tests/data/module_finder/simple_pkg/a.py b/tests/data/importer/simple_pkg/a.py
similarity index 100%
rename from tests/data/module_finder/simple_pkg/a.py
rename to tests/data/importer/simple_pkg/a.py
diff --git a/tests/data/module_finder/simple_pkg/b.py b/tests/data/importer/simple_pkg/b.py
similarity index 100%
rename from tests/data/module_finder/simple_pkg/b.py
rename to tests/data/importer/simple_pkg/b.py
diff --git a/tests/data/module_finder/simple_pkg/imports_replaces_self.py b/tests/data/importer/simple_pkg/imports_replaces_self.py
similarity index 100%
rename from tests/data/module_finder/simple_pkg/imports_replaces_self.py
rename to tests/data/importer/simple_pkg/imports_replaces_self.py
diff --git a/tests/data/module_finder/simple_pkg/ping.py b/tests/data/importer/simple_pkg/ping.py
similarity index 100%
rename from tests/data/module_finder/simple_pkg/ping.py
rename to tests/data/importer/simple_pkg/ping.py
diff --git a/tests/data/module_finder/simple_pkg/replaces_self.py b/tests/data/importer/simple_pkg/replaces_self.py
similarity index 100%
rename from tests/data/module_finder/simple_pkg/replaces_self.py
rename to tests/data/importer/simple_pkg/replaces_self.py
diff --git a/tests/data/module_finder/six_brokenpkg/__init__.py b/tests/data/importer/six_brokenpkg/__init__.py
similarity index 100%
rename from tests/data/module_finder/six_brokenpkg/__init__.py
rename to tests/data/importer/six_brokenpkg/__init__.py
diff --git a/tests/data/module_finder/six_brokenpkg/_six.py b/tests/data/importer/six_brokenpkg/_six.py
similarity index 100%
rename from tests/data/module_finder/six_brokenpkg/_six.py
rename to tests/data/importer/six_brokenpkg/_six.py
diff --git a/tests/data/module_finder/webproject/manage.py b/tests/data/importer/webproject/manage.py
similarity index 100%
rename from tests/data/module_finder/webproject/manage.py
rename to tests/data/importer/webproject/manage.py
diff --git a/tests/data/module_finder/webproject/serve_django_app.py b/tests/data/importer/webproject/serve_django_app.py
similarity index 100%
rename from tests/data/module_finder/webproject/serve_django_app.py
rename to tests/data/importer/webproject/serve_django_app.py
diff --git a/tests/data/module_finder/webproject/webapp/__init__.py b/tests/data/importer/webproject/webapp/__init__.py
similarity index 100%
rename from tests/data/module_finder/webproject/webapp/__init__.py
rename to tests/data/importer/webproject/webapp/__init__.py
diff --git a/tests/data/module_finder/webproject/webapp/admin.py b/tests/data/importer/webproject/webapp/admin.py
similarity index 100%
rename from tests/data/module_finder/webproject/webapp/admin.py
rename to tests/data/importer/webproject/webapp/admin.py
diff --git a/tests/data/module_finder/webproject/webapp/apps.py b/tests/data/importer/webproject/webapp/apps.py
similarity index 100%
rename from tests/data/module_finder/webproject/webapp/apps.py
rename to tests/data/importer/webproject/webapp/apps.py
diff --git a/tests/data/module_finder/webproject/webapp/migrations/__init__.py b/tests/data/importer/webproject/webapp/migrations/__init__.py
similarity index 100%
rename from tests/data/module_finder/webproject/webapp/migrations/__init__.py
rename to tests/data/importer/webproject/webapp/migrations/__init__.py
diff --git a/tests/data/module_finder/webproject/webapp/models.py b/tests/data/importer/webproject/webapp/models.py
similarity index 100%
rename from tests/data/module_finder/webproject/webapp/models.py
rename to tests/data/importer/webproject/webapp/models.py
diff --git a/tests/data/module_finder/webproject/webapp/tests.py b/tests/data/importer/webproject/webapp/tests.py
similarity index 100%
rename from tests/data/module_finder/webproject/webapp/tests.py
rename to tests/data/importer/webproject/webapp/tests.py
diff --git a/tests/data/module_finder/webproject/webapp/views.py b/tests/data/importer/webproject/webapp/views.py
similarity index 100%
rename from tests/data/module_finder/webproject/webapp/views.py
rename to tests/data/importer/webproject/webapp/views.py
diff --git a/tests/data/module_finder/webproject/webproject/__init__.py b/tests/data/importer/webproject/webproject/__init__.py
similarity index 100%
rename from tests/data/module_finder/webproject/webproject/__init__.py
rename to tests/data/importer/webproject/webproject/__init__.py
diff --git a/tests/data/module_finder/webproject/webproject/settings.py b/tests/data/importer/webproject/webproject/settings.py
similarity index 100%
rename from tests/data/module_finder/webproject/webproject/settings.py
rename to tests/data/importer/webproject/webproject/settings.py
diff --git a/tests/data/module_finder/webproject/webproject/urls.py b/tests/data/importer/webproject/webproject/urls.py
similarity index 100%
rename from tests/data/module_finder/webproject/webproject/urls.py
rename to tests/data/importer/webproject/webproject/urls.py
diff --git a/tests/data/module_finder/webproject/webproject/wsgi.py b/tests/data/importer/webproject/webproject/wsgi.py
similarity index 100%
rename from tests/data/module_finder/webproject/webproject/wsgi.py
rename to tests/data/importer/webproject/webproject/wsgi.py
diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index a77f1084..d35fb382 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -7,9 +7,7 @@ import unittest2
 import mitogen.master
 
 import testlib
-
-MODS_PATH = testlib.data_path('module_finder')
-sys.path.append(MODS_PATH)
+from testlib import MODS_DIR
 
 
 class ConstructorTest(testlib.TestCase):
@@ -89,14 +87,14 @@ class PkgutilMethodTest(testlib.TestCase):
     def test_empty_source_pkg(self):
         path, src, is_pkg = self.call('module_finder_testmod')
         self.assertEquals(path,
-            os.path.join(MODS_PATH, 'module_finder_testmod/__init__.py'))
+            os.path.join(MODS_DIR, 'module_finder_testmod/__init__.py'))
         self.assertEquals(mitogen.core.b(''), src)
         self.assertTrue(is_pkg)
 
     def test_empty_source_module(self):
         path, src, is_pkg = self.call('module_finder_testmod.empty_mod')
         self.assertEquals(path,
-            os.path.join(MODS_PATH, 'module_finder_testmod/empty_mod.py'))
+            os.path.join(MODS_DIR, 'module_finder_testmod/empty_mod.py'))
         self.assertEquals(mitogen.core.b(''), src)
         self.assertFalse(is_pkg)
 
@@ -104,7 +102,7 @@ class PkgutilMethodTest(testlib.TestCase):
         from module_finder_testmod import regular_mod
         path, src, is_pkg = self.call('module_finder_testmod.regular_mod')
         self.assertEquals(path,
-            os.path.join(MODS_PATH, 'module_finder_testmod/regular_mod.py'))
+            os.path.join(MODS_DIR, 'module_finder_testmod/regular_mod.py'))
         self.assertEquals(mitogen.core.to_text(src),
                           inspect.getsource(regular_mod))
         self.assertFalse(is_pkg)
@@ -166,7 +164,7 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
         # plumbum has been eating too many rainbow-colored pills
         import pkg_like_plumbum.colors
         path, src, is_pkg = self.call('pkg_like_plumbum.colors')
-        modpath = os.path.join(MODS_PATH, 'pkg_like_plumbum/colors.py')
+        modpath = os.path.join(MODS_DIR, 'pkg_like_plumbum/colors.py')
         self.assertEquals(path, modpath)
 
         self.assertEquals(src, open(modpath, 'rb').read())
@@ -182,7 +180,7 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
         )
 
         path, src, is_pkg = self.call('pkg_like_ansible.module_utils.distro')
-        modpath = os.path.join(MODS_PATH,
+        modpath = os.path.join(MODS_DIR,
             'pkg_like_ansible/module_utils/distro/__init__.py')
         self.assertEquals(path, modpath)
         self.assertEquals(src, open(modpath, 'rb').read())
@@ -259,7 +257,7 @@ class FindRelatedTest(testlib.TestCase):
 
 if sys.version_info > (2, 6):
     class DjangoMixin(object):
-        WEBPROJECT_PATH = os.path.join(MODS_PATH, 'webproject')
+        WEBPROJECT_PATH = os.path.join(MODS_DIR, 'webproject')
 
         # TODO: rip out Django and replace with a static tree of weird imports
         # that don't depend on .. Django! The hack below is because the version
diff --git a/tests/testlib.py b/tests/testlib.py
index 2ee672be..04a48d84 100644
--- a/tests/testlib.py
+++ b/tests/testlib.py
@@ -41,6 +41,11 @@ except NameError:
 
 LOG = logging.getLogger(__name__)
 DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
+MODS_DIR = os.path.join(DATA_DIR, 'importer')
+
+sys.path.append(DATA_DIR)
+sys.path.append(MODS_DIR)
+
 
 if mitogen.is_master:
     mitogen.utils.log_to_file()

From ed8acb5153576dc11486f7cd5ab5208b584ca7b3 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 19:05:54 +0100
Subject: [PATCH 11/18] master: sysconfig did not exist until 2.7.

---
 mitogen/master.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/mitogen/master.py b/mitogen/master.py
index 681b870b..fb4f505b 100644
--- a/mitogen/master.py
+++ b/mitogen/master.py
@@ -46,12 +46,16 @@ import pkgutil
 import re
 import string
 import sys
-import sysconfig
 import threading
 import time
 import types
 import zlib
 
+try:
+    import sysconfig
+except ImportError:
+    sysconfig = None
+
 if not hasattr(pkgutil, 'find_loader'):
     # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has
     # been kept intentionally 2.3 compatible so we can reuse it.
@@ -101,7 +105,8 @@ def _stdlib_paths():
 
     # When running 'unit2 tests/module_finder_test.py' in a Py2 venv on Ubuntu
     # 18.10, above is insufficient to catch the real directory.
-    s.add(sysconfig.get_config_var('DESTLIB'))
+    if sysconfig is not None:
+        s.add(sysconfig.get_config_var('DESTLIB'))
     return s
 
 

From 50cdf63c27177c75b815af3e01337126f5bf801f Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 19:12:43 +0100
Subject: [PATCH 12/18] tests: Py3.x fix.

---
 tests/module_finder_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index d35fb382..3bc4088f 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -5,6 +5,7 @@ import sys
 import unittest2
 
 import mitogen.master
+from mitogen.core import b
 
 import testlib
 from testlib import MODS_DIR
@@ -122,7 +123,7 @@ class SysModulesMethodTest(testlib.TestCase):
         # linecache adds a line ending to the final line if one is missing.
         actual_src = open(path, 'rb').read()
         if actual_src[-1] != '\n':
-            actual_src += '\n'
+            actual_src += b('\n')
 
         self.assertEquals(src, actual_src)
         self.assertFalse(is_pkg)

From e94200aeb90a3120c62051c498133ea75d067528 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 19:13:23 +0100
Subject: [PATCH 13/18] issue #590: actually run Ansible test.

---
 tests/ansible/regression/all.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml
index 123d87d9..62606b00 100644
--- a/tests/ansible/regression/all.yml
+++ b/tests/ansible/regression/all.yml
@@ -8,3 +8,4 @@
 - include: issue_154__module_state_leaks.yml
 - include: issue_177__copy_module_failing.yml
 - include: issue_332_ansiblemoduleerror_first_occurrence.yml
+- include: issue_590__sys_modules_crap.yml

From 90105e20315d6fc9edbea2b640ed0c6543de03fa Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 19:12:43 +0100
Subject: [PATCH 14/18] tests: Py3.x fix.

---
 tests/module_finder_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index 3bc4088f..11062512 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -122,7 +122,7 @@ class SysModulesMethodTest(testlib.TestCase):
 
         # linecache adds a line ending to the final line if one is missing.
         actual_src = open(path, 'rb').read()
-        if actual_src[-1] != '\n':
+        if actual_src[-1] != b('\n'):
             actual_src += b('\n')
 
         self.assertEquals(src, actual_src)

From a1f2ec222d5b6904c2c73ad10bad1d5cbdae58ca Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 19:38:42 +0100
Subject: [PATCH 15/18] issue #590: fix test for <2.8 Ansibles.

---
 .../ansible/lib/modules/custom_python_uses_distro.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py
index c5ffbb84..1fc31b4e 100644
--- a/tests/ansible/lib/modules/custom_python_uses_distro.py
+++ b/tests/ansible/lib/modules/custom_python_uses_distro.py
@@ -2,12 +2,20 @@
 # issue #590: I am an Ansible new-style Python module that tries to use
 # ansible.module_utils.distro.
 
+import ansible
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils import distro
+
+if ansible.__version__ > '2.8':
+    from ansible.module_utils import distro
+else:
+    distro = None
 
 def main():
     module = AnsibleModule(argument_spec={})
-    module.exit_json(info=distro.info())
+    if ansible.__version__ > '2.8':
+        module.exit_json(info=distro.info())
+    else:
+        module.exit_json(info={'id': None})
 
 if __name__ == '__main__':
     main()

From f1287e6e9af7397cfd6aa5dc6d49eec7fe916c95 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 19:12:43 +0100
Subject: [PATCH 16/18] tests: Py3.x fix.

---
 tests/module_finder_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py
index 11062512..e61c768f 100644
--- a/tests/module_finder_test.py
+++ b/tests/module_finder_test.py
@@ -122,7 +122,7 @@ class SysModulesMethodTest(testlib.TestCase):
 
         # linecache adds a line ending to the final line if one is missing.
         actual_src = open(path, 'rb').read()
-        if actual_src[-1] != b('\n'):
+        if actual_src[-1:] != b('\n'):
             actual_src += b('\n')
 
         self.assertEquals(src, actual_src)

From c1d763f92976c45f4cbfcf91f229fe1134cdac3d Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 19:53:12 +0100
Subject: [PATCH 17/18] ci: Ansible 2.8 jobs aren't running against all host
 types.

---
 .ci/azure-pipelines.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml
index ef344bba..95f239ff 100644
--- a/.ci/azure-pipelines.yml
+++ b/.ci/azure-pipelines.yml
@@ -92,10 +92,8 @@ jobs:
         python.version: '2.7'
         MODE: ansible
         VER: 2.8.0
-        DISTROS: debian
 
       Ansible_280_35:
         python.version: '3.5'
         MODE: ansible
         VER: 2.8.0
-        DISTROS: debian

From fe7c3610ffa86a8fe1886ddf13bb59f3be290e04 Mon Sep 17 00:00:00 2001
From: David Wilson <dw@botanicus.net>
Date: Sun, 2 Jun 2019 20:52:18 +0100
Subject: [PATCH 18/18] issue #590: disable distro test on vanilla

---
 tests/ansible/regression/issue_590__sys_modules_crap.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/ansible/regression/issue_590__sys_modules_crap.yml b/tests/ansible/regression/issue_590__sys_modules_crap.yml
index 83a9a286..41130b68 100644
--- a/tests/ansible/regression/issue_590__sys_modules_crap.yml
+++ b/tests/ansible/regression/issue_590__sys_modules_crap.yml
@@ -1,6 +1,9 @@
 
 - hosts: test-targets
   tasks:
+  - meta: end_play
+    when: ansible_version.full < '2.8'
+
   - custom_python_uses_distro:
     register: out