Beginnings of module_finder_test

dev_requirements.txt

@@ -1,4 +1,5 @@
 -r docs/docs-requirements.txt
+ansible==2.3.1.0
 docker==2.5.1
 docker[tls]==2.5.1
 pytest-capturelog==0.7

mitogen/master.py

@@ -405,7 +405,7 @@ class ModuleFinder(object):
         """Attempt to fetch source code via pkgutil. In an ideal world, this
         would be the only required implementation of get_module()."""
         loader = pkgutil.find_loader(fullname)
-        LOG.debug('pkgutil.find_loader(%r) -> %r', fullname, loader)
+        LOG.debug('pkgutil._get_module_via_pkgutil(%r) -> %r', fullname, loader)
         if not loader:
             return
 
@@ -420,51 +420,54 @@ class ModuleFinder(object):
     def _get_module_via_sys_modules(self, fullname):
         """Attempt to fetch source code via sys.modules. This is specifically
         to support __main__, but it may catch a few more cases."""
-        if fullname not in sys.modules:
+        module = sys.modules.get(fullname)
-            LOG.debug('%r does not appear in sys.modules', fullname)
+        if not isinstance(module, types.ModuleType):
+            LOG.debug('sys.modules[%r] absent or not a regular module',
+                      fullname)
             return
 
-        if 'six.moves' in fullname:
+        modpath = getattr(module, '__file__', '')
-            # TODO: causes inspect.getsource() to explode.
-            return None, None, None
-
-        modpath = getattr(sys.modules[fullname], '__file__', '')
         if not modpath.rstrip('co').endswith('.py'):
             # Probably a native module.
-            return None, None, None
+            return
 
-        is_pkg = hasattr(sys.modules[fullname], '__path__')
+        is_pkg = hasattr(module, '__path__')
         try:
-            source = inspect.getsource(sys.modules[fullname])
+            source = inspect.getsource(module)
         except IOError:
             # Work around inspect.getsourcelines() bug.
             if not is_pkg:
                 raise
             source = '\n'
 
-        return (sys.modules[fullname].__file__.rstrip('co'),
+        return (module.__file__.rstrip('co'),
                 source,
-                hasattr(sys.modules[fullname], '__path__'))
+                hasattr(module, '__path__'))
 
-    def _get_module_via_parent_enumeration(self, fullname):
+    def _get_module_via_parent(self, fullname):
         """Attempt to fetch source code by examining the module's (hopefully
         less insane) parent package. Required for ansible.compat.six."""
+        # Need to find the ancient version of Ansible with the ancient
+        # non-package version of six that required this method to exist.
+        # Currently it doesn't seem to be needed at all, and it's broken for
+        # packages.
         pkgname, _, modname = fullname.rpartition('.')
         pkg = sys.modules.get(pkgname)
-        if pkg is None or not hasattr(pkg, '__file__'):
+        if not (isinstance(pkg, types.ModuleType) and hasattr(pkg, '__file__')):
             return
 
         pkg_path = os.path.dirname(pkg.__file__)
         try:
             fp, path, ext = imp.find_module(modname, [pkg_path])
-            LOG.error('%r', (fp, path, ext))
+            if ext and ext[-1] == imp.PKG_DIRECTORY:
+                assert 0, "TODO"
             return path, fp.read(), False
         except ImportError, e:
             LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e)
 
     get_module_methods = [_get_module_via_pkgutil,
                           _get_module_via_sys_modules,
-                          _get_module_via_parent_enumeration]
+                          _get_module_via_parent]
 
     def get_module_source(self, fullname):
         """Given the name of a loaded module `fullname`, attempt to find its
@@ -489,11 +492,11 @@ class ModuleFinder(object):
         """Given an ImportFrom AST node, guess the prefix that should be tacked
         on to an alias name to produce a canonical name. `fullname` is the name
         of the module in which the ImportFrom appears."""
-        if level == 0:
+        if level == 0 or not fullname:
             return ''
 
         bits = fullname.split('.')
-        if len(bits) < level:
+        if len(bits) <= level:
             # This would be an ImportError in real code.
             return ''
 
@@ -536,13 +539,15 @@ class ModuleFinder(object):
                 for name in namelist
             )
 
-        return self._related_cache.setdefault(fullname, [
+        return self._related_cache.setdefault(fullname, sorted(
-            name
+            set(
-            for name in maybe_names
+                name
-            if sys.modules.get(name) is not None
+                for name in maybe_names
-            and not self.is_stdlib_name(name)
+                if sys.modules.get(name) is not None
-            and 'six.moves' not in name  # TODO: crap
+                and not self.is_stdlib_name(name)
-        ])
+                and 'six.moves' not in name  # TODO: crap
+            )
+        ))
 
     def find_related(self, fullname):
         stack = [fullname]

tests/data/module_finder_testmod/regular_mod.py

@@ -0,0 +1,6 @@
+
+import sys
+
+
+def say_hi():
+    print 'hi'

tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py

@@ -0,0 +1,3 @@
+
+from __future__ import absolute_import
+from module_finder_testmod.regular_mod import say_hi

tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py

@@ -0,0 +1,2 @@
+
+from regular_mod import say_hi

tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py

@@ -0,0 +1 @@
+from .regular_mod import say_hi

tests/module_finder_test.py

@@ -2,26 +2,168 @@
 import unittest
 import mitogen.master
 
+import testlib
 
-class CompilerModuleTest(unittest.TestCase):
-    klass = mitogen.master.ModuleScanner
 
-    @classmethod
+class ConstructorTest(testlib.TestCase):
-    def setUpClass(cls):
+    klass = mitogen.master.ModuleFinder
-        super(CompilerModuleTest, cls).setUpClass()
-        #import compiler
-        #mitogen.master.ast = None
-        #mitogen.master.compiler = compiler
 
     def test_simple(self):
-        for x in range(100):
+        self.klass()
-            finder = self.klass()
+
-            from pprint import pprint
+
-            import time
+class ReprTest(testlib.TestCase):
-            t0 = time.time()
+    klass = mitogen.master.ModuleFinder
-            import mitogen.fakessh
+
-            pprint(finder.find_related('mitogen.fakessh'))
+    def test_simple(self):
-            print 1000 * (time.time() - t0)
+        self.assertEquals('ModuleFinder()', repr(self.klass()))
+
+
+class IsStdlibNameTest(testlib.TestCase):
+    klass = mitogen.master.ModuleFinder
+
+    def call(self, fullname):
+        return self.klass().is_stdlib_name(fullname)
+
+    def test_builtin(self):
+        import sys
+        self.assertTrue(self.call('sys'))
+
+    def test_stdlib_1(self):
+        import logging
+        self.assertTrue(self.call('logging'))
+
+    def test_stdlib_2(self):
+        # virtualenv only symlinks some paths to its local site-packages
+        # directory. Ensure both halves of the search path return the correct
+        # result.
+        import email
+        self.assertTrue(self.call('email'))
+
+    def test_mitogen_core(self):
+        import mitogen.core
+        self.assertFalse(self.call('mitogen.core'))
+
+    def test_mitogen_fakessh(self):
+        import mitogen.fakessh
+        self.assertFalse(self.call('mitogen.fakessh'))
+
+
+class GetModuleViaPkgutilTest(testlib.TestCase):
+    klass = mitogen.master.ModuleFinder
+
+    def call(self, fullname):
+        return self.klass()._get_module_via_pkgutil(fullname)
+
+    def test_empty_source_pkg(self):
+        path, src, is_pkg = self.call('module_finder_testmod')
+        self.assertEquals(path,
+            testlib.data_path('module_finder_testmod/__init__.py'))
+        self.assertEquals('', src)
+        self.assertTrue(is_pkg)
+
+    def test_empty_source_module(self):
+        path, src, is_pkg = self.call('module_finder_testmod.empty_mod')
+        self.assertEquals(path,
+            testlib.data_path('module_finder_testmod/empty_mod.py'))
+        self.assertEquals('', src)
+        self.assertFalse(is_pkg)
+
+    def test_regular_mod(self):
+        from module_finder_testmod import regular_mod
+        path, src, is_pkg = self.call('module_finder_testmod.regular_mod')
+        self.assertEquals(path,
+            testlib.data_path('module_finder_testmod/regular_mod.py'))
+        self.assertEquals(src, file(regular_mod.__file__).read())
+        self.assertFalse(is_pkg)
+
+
+class GetModuleViaSysModulesTest(testlib.TestCase):
+    klass = mitogen.master.ModuleFinder
+
+    def call(self, fullname):
+        return self.klass()._get_module_via_sys_modules(fullname)
+
+    def test_main(self):
+        import __main__
+        path, src, is_pkg = self.call('__main__')
+        self.assertEquals(path, __main__.__file__)
+        self.assertEquals(src, file(path).read())
+        self.assertFalse(is_pkg)
+
+    def test_dylib_fails(self):
+        # _socket comes from a .so
+        import _socket
+        tup = self.call('_socket')
+        self.assertEquals(None, tup)
+
+    def test_builtin_fails(self):
+        # sys is built-in
+        tup = self.call('sys')
+        self.assertEquals(None, tup)
+
+
+class GetModuleViaParentEnumerationTest(testlib.TestCase):
+    klass = mitogen.master.ModuleFinder
+
+    def call(self, fullname):
+        return self.klass()._get_module_via_parent(fullname)
+
+    def test_simple_module(self):
+        import email.utils
+        path, src, is_pkg = self.call('email.utils')
+        self.assertEquals(path, email.utils.__file__.rstrip('co'))
+        self.assertEquals(src, file(email.utils.__file__.rstrip('co')).read())
+        self.assertFalse(is_pkg)
+
+    def test_ansible_compat_six(self):
+        # See comment in _get_module_via_parent
+        raise unittest.SkipTest()
+        import ansible.compat.six
+        path, src, is_pkg = self.call('ansible.compat.six')
+        self.assertEquals(path, __main__.__file__)
+        self.assertEquals(src, file(path).read())
+        self.assertFalse(is_pkg)
+
+
+class ResolveRelPathTest(testlib.TestCase):
+    klass = mitogen.master.ModuleFinder
+
+    def call(self, fullname, level):
+        return self.klass().resolve_relpath(fullname, level)
+
+    def test_empty(self):
+        self.assertEquals('', self.call('', 0))
+        self.assertEquals('', self.call('', 1))
+        self.assertEquals('', self.call('', 2))
+
+    def test_absolute(self):
+        self.assertEquals('', self.call('email.utils', 0))
+
+    def test_rel1(self):
+        self.assertEquals('email.', self.call('email.utils', 1))
+
+    def test_rel2(self):
+        self.assertEquals('', self.call('email.utils', 2))
+
+    def test_rel_overflow(self):
+        self.assertEquals('', self.call('email.utils', 3))
+
+
+class FindRelatedImportsTest(testlib.TestCase):
+    klass = mitogen.master.ModuleFinder
+
+    def call(self, fullname):
+        return self.klass().find_related_imports(fullname)
+
+    def test_simple(self):
+        import mitogen.fakessh
+        related = self.call('mitogen.fakessh')
+        self.assertEquals(related, [
+            'mitogen',
+            'mitogen.core',
+            'mitogen.master',
+        ])
 
 
 if __name__ == '__main__':