From 15bf0f54e239424b6d67d33d24400f7111e29f72 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Oct 2017 13:54:01 +0530 Subject: [PATCH] Beginnings of module_finder_test --- dev_requirements.txt | 1 + mitogen/master.py | 57 +++--- tests/data/module_finder_testmod/__init__.py | 0 tests/data/module_finder_testmod/empty_mod.py | 0 .../data/module_finder_testmod/regular_mod.py | 6 + .../sibling_dep_mod_abs_import.py | 3 + .../sibling_dep_mod_py2_import.py | 2 + .../sibling_dep_mod_rel_import.py | 1 + tests/module_finder_test.py | 174 ++++++++++++++++-- 9 files changed, 202 insertions(+), 42 deletions(-) create mode 100644 tests/data/module_finder_testmod/__init__.py create mode 100644 tests/data/module_finder_testmod/empty_mod.py create mode 100644 tests/data/module_finder_testmod/regular_mod.py create mode 100644 tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py create mode 100644 tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py create mode 100644 tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py diff --git a/dev_requirements.txt b/dev_requirements.txt index aebcabf3..0afa6c1f 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,5 @@ -r docs/docs-requirements.txt +ansible==2.3.1.0 docker==2.5.1 docker[tls]==2.5.1 pytest-capturelog==0.7 diff --git a/mitogen/master.py b/mitogen/master.py index a4207a65..e31e7aa6 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -405,7 +405,7 @@ class ModuleFinder(object): """Attempt to fetch source code via pkgutil. In an ideal world, this would be the only required implementation of get_module().""" loader = pkgutil.find_loader(fullname) - LOG.debug('pkgutil.find_loader(%r) -> %r', fullname, loader) + LOG.debug('pkgutil._get_module_via_pkgutil(%r) -> %r', fullname, loader) if not loader: return @@ -420,51 +420,54 @@ class ModuleFinder(object): def _get_module_via_sys_modules(self, fullname): """Attempt to fetch source code via sys.modules. This is specifically to support __main__, but it may catch a few more cases.""" - if fullname not in sys.modules: - LOG.debug('%r does not appear in sys.modules', fullname) + module = sys.modules.get(fullname) + if not isinstance(module, types.ModuleType): + LOG.debug('sys.modules[%r] absent or not a regular module', + fullname) return - if 'six.moves' in fullname: - # TODO: causes inspect.getsource() to explode. - return None, None, None - - modpath = getattr(sys.modules[fullname], '__file__', '') + modpath = getattr(module, '__file__', '') if not modpath.rstrip('co').endswith('.py'): # Probably a native module. - return None, None, None + return - is_pkg = hasattr(sys.modules[fullname], '__path__') + is_pkg = hasattr(module, '__path__') try: - source = inspect.getsource(sys.modules[fullname]) + source = inspect.getsource(module) except IOError: # Work around inspect.getsourcelines() bug. if not is_pkg: raise source = '\n' - return (sys.modules[fullname].__file__.rstrip('co'), + return (module.__file__.rstrip('co'), source, - hasattr(sys.modules[fullname], '__path__')) + hasattr(module, '__path__')) - def _get_module_via_parent_enumeration(self, fullname): + def _get_module_via_parent(self, fullname): """Attempt to fetch source code by examining the module's (hopefully less insane) parent package. Required for ansible.compat.six.""" + # Need to find the ancient version of Ansible with the ancient + # non-package version of six that required this method to exist. + # Currently it doesn't seem to be needed at all, and it's broken for + # packages. pkgname, _, modname = fullname.rpartition('.') pkg = sys.modules.get(pkgname) - if pkg is None or not hasattr(pkg, '__file__'): + if not (isinstance(pkg, types.ModuleType) and hasattr(pkg, '__file__')): return pkg_path = os.path.dirname(pkg.__file__) try: fp, path, ext = imp.find_module(modname, [pkg_path]) - LOG.error('%r', (fp, path, ext)) + if ext and ext[-1] == imp.PKG_DIRECTORY: + assert 0, "TODO" return path, fp.read(), False except ImportError, e: LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e) get_module_methods = [_get_module_via_pkgutil, _get_module_via_sys_modules, - _get_module_via_parent_enumeration] + _get_module_via_parent] def get_module_source(self, fullname): """Given the name of a loaded module `fullname`, attempt to find its @@ -489,11 +492,11 @@ class ModuleFinder(object): """Given an ImportFrom AST node, guess the prefix that should be tacked on to an alias name to produce a canonical name. `fullname` is the name of the module in which the ImportFrom appears.""" - if level == 0: + if level == 0 or not fullname: return '' bits = fullname.split('.') - if len(bits) < level: + if len(bits) <= level: # This would be an ImportError in real code. return '' @@ -536,13 +539,15 @@ class ModuleFinder(object): for name in namelist ) - return self._related_cache.setdefault(fullname, [ - name - for name in maybe_names - if sys.modules.get(name) is not None - and not self.is_stdlib_name(name) - and 'six.moves' not in name # TODO: crap - ]) + return self._related_cache.setdefault(fullname, sorted( + set( + name + for name in maybe_names + if sys.modules.get(name) is not None + and not self.is_stdlib_name(name) + and 'six.moves' not in name # TODO: crap + ) + )) def find_related(self, fullname): stack = [fullname] diff --git a/tests/data/module_finder_testmod/__init__.py b/tests/data/module_finder_testmod/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/module_finder_testmod/empty_mod.py b/tests/data/module_finder_testmod/empty_mod.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/module_finder_testmod/regular_mod.py b/tests/data/module_finder_testmod/regular_mod.py new file mode 100644 index 00000000..865a115e --- /dev/null +++ b/tests/data/module_finder_testmod/regular_mod.py @@ -0,0 +1,6 @@ + +import sys + + +def say_hi(): + print 'hi' diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py b/tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py new file mode 100644 index 00000000..5e64ffd6 --- /dev/null +++ b/tests/data/module_finder_testmod/sibling_dep_mod_abs_import.py @@ -0,0 +1,3 @@ + +from __future__ import absolute_import +from module_finder_testmod.regular_mod import say_hi diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py b/tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py new file mode 100644 index 00000000..aaefff70 --- /dev/null +++ b/tests/data/module_finder_testmod/sibling_dep_mod_py2_import.py @@ -0,0 +1,2 @@ + +from regular_mod import say_hi diff --git a/tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py b/tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py new file mode 100644 index 00000000..bbcd17a4 --- /dev/null +++ b/tests/data/module_finder_testmod/sibling_dep_mod_rel_import.py @@ -0,0 +1 @@ +from .regular_mod import say_hi diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index cb55694c..6dff80c5 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -2,26 +2,168 @@ import unittest import mitogen.master +import testlib -class CompilerModuleTest(unittest.TestCase): - klass = mitogen.master.ModuleScanner - @classmethod - def setUpClass(cls): - super(CompilerModuleTest, cls).setUpClass() - #import compiler - #mitogen.master.ast = None - #mitogen.master.compiler = compiler +class ConstructorTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder def test_simple(self): - for x in range(100): - finder = self.klass() - from pprint import pprint - import time - t0 = time.time() - import mitogen.fakessh - pprint(finder.find_related('mitogen.fakessh')) - print 1000 * (time.time() - t0) + self.klass() + + +class ReprTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + + def test_simple(self): + self.assertEquals('ModuleFinder()', repr(self.klass())) + + +class IsStdlibNameTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + + def call(self, fullname): + return self.klass().is_stdlib_name(fullname) + + def test_builtin(self): + import sys + self.assertTrue(self.call('sys')) + + def test_stdlib_1(self): + import logging + self.assertTrue(self.call('logging')) + + def test_stdlib_2(self): + # virtualenv only symlinks some paths to its local site-packages + # directory. Ensure both halves of the search path return the correct + # result. + import email + self.assertTrue(self.call('email')) + + def test_mitogen_core(self): + import mitogen.core + self.assertFalse(self.call('mitogen.core')) + + def test_mitogen_fakessh(self): + import mitogen.fakessh + self.assertFalse(self.call('mitogen.fakessh')) + + +class GetModuleViaPkgutilTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + + def call(self, fullname): + return self.klass()._get_module_via_pkgutil(fullname) + + def test_empty_source_pkg(self): + path, src, is_pkg = self.call('module_finder_testmod') + self.assertEquals(path, + testlib.data_path('module_finder_testmod/__init__.py')) + self.assertEquals('', src) + self.assertTrue(is_pkg) + + def test_empty_source_module(self): + path, src, is_pkg = self.call('module_finder_testmod.empty_mod') + self.assertEquals(path, + testlib.data_path('module_finder_testmod/empty_mod.py')) + self.assertEquals('', src) + self.assertFalse(is_pkg) + + def test_regular_mod(self): + from module_finder_testmod import regular_mod + path, src, is_pkg = self.call('module_finder_testmod.regular_mod') + self.assertEquals(path, + testlib.data_path('module_finder_testmod/regular_mod.py')) + self.assertEquals(src, file(regular_mod.__file__).read()) + self.assertFalse(is_pkg) + + +class GetModuleViaSysModulesTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + + def call(self, fullname): + return self.klass()._get_module_via_sys_modules(fullname) + + def test_main(self): + import __main__ + path, src, is_pkg = self.call('__main__') + self.assertEquals(path, __main__.__file__) + self.assertEquals(src, file(path).read()) + self.assertFalse(is_pkg) + + def test_dylib_fails(self): + # _socket comes from a .so + import _socket + tup = self.call('_socket') + self.assertEquals(None, tup) + + def test_builtin_fails(self): + # sys is built-in + tup = self.call('sys') + self.assertEquals(None, tup) + + +class GetModuleViaParentEnumerationTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + + def call(self, fullname): + return self.klass()._get_module_via_parent(fullname) + + def test_simple_module(self): + import email.utils + path, src, is_pkg = self.call('email.utils') + self.assertEquals(path, email.utils.__file__.rstrip('co')) + self.assertEquals(src, file(email.utils.__file__.rstrip('co')).read()) + self.assertFalse(is_pkg) + + def test_ansible_compat_six(self): + # See comment in _get_module_via_parent + raise unittest.SkipTest() + import ansible.compat.six + path, src, is_pkg = self.call('ansible.compat.six') + self.assertEquals(path, __main__.__file__) + self.assertEquals(src, file(path).read()) + self.assertFalse(is_pkg) + + +class ResolveRelPathTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + + def call(self, fullname, level): + return self.klass().resolve_relpath(fullname, level) + + def test_empty(self): + self.assertEquals('', self.call('', 0)) + self.assertEquals('', self.call('', 1)) + self.assertEquals('', self.call('', 2)) + + def test_absolute(self): + self.assertEquals('', self.call('email.utils', 0)) + + def test_rel1(self): + self.assertEquals('email.', self.call('email.utils', 1)) + + def test_rel2(self): + self.assertEquals('', self.call('email.utils', 2)) + + def test_rel_overflow(self): + self.assertEquals('', self.call('email.utils', 3)) + + +class FindRelatedImportsTest(testlib.TestCase): + klass = mitogen.master.ModuleFinder + + def call(self, fullname): + return self.klass().find_related_imports(fullname) + + def test_simple(self): + import mitogen.fakessh + related = self.call('mitogen.fakessh') + self.assertEquals(related, [ + 'mitogen', + 'mitogen.core', + 'mitogen.master', + ]) if __name__ == '__main__':