From 0b421e0d3c0867938937b7eb37fed1e02e1fddc9 Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Tue, 25 Aug 2020 23:24:41 -0700 Subject: [PATCH] able to run docker_container installed via 'ansible-galaxy collection install community.general' --- ansible_mitogen/planner.py | 61 ++++--------------------- mitogen/master.py | 92 ++++++-------------------------------- mitogen/service.py | 13 +++--- 3 files changed, 30 insertions(+), 136 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 6ae5ef08..e77ba631 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -42,12 +42,9 @@ import logging import os import random -from ansible import context as ansible_context from ansible.executor import module_common -from ansible.galaxy.collection import ( - find_existing_collections, - validate_collection_path -) +from ansible.galaxy.collection import find_existing_collections +from ansible.utils.collection_loader import AnsibleCollectionConfig import ansible.errors import ansible.module_utils import ansible.release @@ -489,6 +486,7 @@ def _propagate_deps(invocation, planner, context): paths=planner.get_push_files(), # modules=planner.get_module_deps(), TODO overridden_sources=invocation._overridden_sources, + # needs to be a list because can't unpickle() a set() extra_sys_paths=list(invocation._extra_sys_paths) ) @@ -569,55 +567,17 @@ def _fix_py35(invocation, module_source): def _load_collections(invocation): """ - Special loader that ensures that `ansible_collections` exists as a module path for import + Special loader that ensures that `ansible_collections` exist as a module path for import + Goes through all collection path possibilities and stores paths to installed collections + Stores them on the current invocation to later be passed to the master service """ - # import epdb; epdb.set_trace() - # find_existing_collections() - # collection_path = validate_collection_path(path) - # collection_path = GalaxyCLI._resolve_path(path) - - # import epdb; epdb.set_trace() - from ansible.utils.collection_loader import AnsibleCollectionConfig - from ansible.cli.galaxy import _get_collection_widths, _display_header, _display_collection - from ansible.module_utils._text import to_bytes, to_native, to_text - import sys - - from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder - - # for path in AnsibleCollectionConfig.collection_paths: - # if os.path.isdir(path): - # collections = find_existing_collections(path, fallback_metadata=True) - - # fqcn_width, version_width = _get_collection_widths(collections) - # # _display_header(path, 'Collection', 'Version', fqcn_width, version_width) - - # # Sort collections by the namespace and name - # collections.sort(key=to_text) - # for collection in collections: - # _display_collection(collection, fqcn_width, version_width) - for path in AnsibleCollectionConfig.collection_paths: if os.path.isdir(path): - # import epdb; epdb.set_trace() collections = find_existing_collections(path, fallback_metadata=True) - # add the collection's parent path to sys.path - # additionally, handle __synthetic__ - # TODO: left off here. See ansible.utils.collection_loader; can't just add to path - # jjj for collection in collections: - # collection_path_parent = collection.b_path - # import epdb; epdb.set_trace() - # sys.path.insert(0, '/Users/me/.ansible/collections/ansible_collections') - # sys.path.insert(0, collection.b_path.decode('utf-8')) invocation._extra_sys_paths.add(collection.b_path.decode('utf-8')) - # import epdb; epdb.set_trace() - # handle '__synthetic__' created by ansible - # sys.modules['ansible_collections'].__file__ = sys.modules['ansible_collections'].__file__ + ".py" - # import epdb; epdb.set_trace() - # import epdb; epdb.set_trace() - # uuu - # finder = _AnsibleCollectionFinder(AnsibleCollectionConfig.collection_paths, True) + def invoke(invocation): """ @@ -630,7 +590,6 @@ def invoke(invocation): :raises ansible.errors.AnsibleError: Unrecognized/unsupported module type. """ - # import epdb; epdb.set_trace() path = ansible_mitogen.loaders.module_loader.find_plugin( invocation.module_name, '', @@ -641,13 +600,10 @@ def invoke(invocation): )) invocation.module_path = mitogen.core.to_text(path) - #jjj - # if 'ansible_collections' in invocation.module_path: - # import epdb; epdb.set_trace() if invocation.module_path not in _planner_by_path: if 'ansible_collections' in invocation.module_path: - # import epdb; epdb.set_trace() _load_collections(invocation) + module_source = invocation.get_module_source() _fix_py35(invocation, module_source) _planner_by_path[invocation.module_path] = _get_planner( @@ -663,7 +619,6 @@ def invoke(invocation): response = _invoke_isolated_task(invocation, planner) else: _propagate_deps(invocation, planner, invocation.connection.context) - # import epdb; epdb.set_trace() response = invocation.connection.get_chain().call( ansible_mitogen.target.run_module, kwargs=planner.get_kwargs(), diff --git a/mitogen/master.py b/mitogen/master.py index 00e3948e..e1a352e2 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -89,6 +89,14 @@ except NameError: RLOG = logging.getLogger('mitogen.ctx') +# there are some cases where modules are loaded in memory only, such as +# ansible collections, and the module "filename" is something like __synthetic__ +# which doesn't actually exist +SPECIAL_FILE_PATHS = [ + "__synthetic__" +] + + def _stdlib_paths(): """ Return a set of paths from which Python imports the standard library. @@ -155,26 +163,6 @@ def get_child_modules(path, fullname): :return: List of submodule name suffixes. """ - # jjj - # TODO: move this somehow to ansible_mitogen, if it's even possible - # ISSUE: not everything is being loaded via sys.modules in ansible when it comes to collections - # only `action` and `modules` show up in sys.modules[fullname] - # but sometimes you want things like `module_utils` - # if fullname.startswith("ansible_collections"): - # submodules = [] - # # import epdb; epdb.set_trace() - # # sys.modules[fullname].__path__ - # # for each in dir(sys.modules[fullname]): - # # if not each.startswith("__"): - # # submodules.append(to_text(each)) - # for each in os.listdir(sys.modules[fullname].__path__[0]): - # if not each.startswith("__"): - # submodules.append(to_text(each)) - # # jjj - # # hack: insert submodule on the path so it can be loaded - # # sys.path.insert(0, each) - # return submodules - # else: it = pkgutil.iter_modules([os.path.dirname(path)]) return [to_text(name) for _, name, _ in it] @@ -213,6 +201,11 @@ def _py_filename(path): if os.path.exists(path) and _looks_like_script(path): return path + basepath = os.path.basename(path) + for filename in SPECIAL_FILE_PATHS: + if basepath == filename: + return path + def _get_core_source(): """ @@ -521,21 +514,6 @@ class PkgutilMethod(FinderMethod): if not loader: return - # jjjj - # if fullname == "ansible_collections": - # import epdb; epdb.set_trace() - # jjj - # if fullname == "ansible_collections": - # import epdb; epdb.set_trace() - # ba = loader.load_module("ansible_collections.alikins") - # # if fullname == "ansible_collections": - # # # ansible named the fake __file__ for collections `__synthetic__` with no extension - # # module.__file__ = module.__file__ + ".py" - # # import epdb; epdb.set_trace() - # # # import epdb; epdb.set_trace() - # # # jjj - # doesn't work because .get_source doesn't exist. Collections loader is super complicated - # and doesn't offer an easy way to extract source code try: path = _py_filename(loader.get_filename(fullname)) source = loader.get_source(fullname) @@ -574,10 +552,7 @@ class SysModulesMethod(FinderMethod): Find `fullname` using its :data:`__file__` attribute. """ module = sys.modules.get(fullname) - # jjj - # this is hit - # if fullname.startswith("ansible_collections"): - # import epdb; epdb.set_trace() + if not isinstance(module, types.ModuleType): LOG.debug('%r: sys.modules[%r] absent or not a regular module', self, fullname) @@ -592,13 +567,6 @@ class SysModulesMethod(FinderMethod): fullname, alleged_name, module) return - # TODO: move to ansible_mitogen somehow if possible - # ansible names the fake __file__ for collections `__synthetic__` with no extension - # if fullname.startswith("ansible_collections"): - # print(fullname) - # module.__file__ = module.__file__ + ".py" - # # import epdb; epdb.set_trace() - # # jjj path = _py_filename(getattr(module, '__file__', '')) if not path: return @@ -705,24 +673,6 @@ class ParentEnumerationMethod(FinderMethod): return path, source, is_pkg def _find_one_component(self, modname, search_path): - """ - Creates an __init__.py if one doesn't exist in the search path dirs for ansible collections - This will help imp load packages like `.ansible/collections/ansible_collections/.....plugins/module_utils` - that don't get loaded from Ansible via sys.modules - Unfortunately this leaves __init__.py files around in collections that don't have them - TODO: delete these when Mitogen exits? - Tried to hack types.ModuleType instead but no luck - Appears imp loads modules old-style with a required __init__.py - TODO: can the __init__ stuff be moved to ansible_mitogen somewhere, really want it to be there instead - """ - # jjj - # for path in search_path: - # if "collections/ansible_collections" in path: - # init_file = os.path.join(path, modname, "__init__.py") - # if not os.path.isfile(init_file): - # with open(init_file, "w") as f: - # pass - try: #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) return imp.find_module(modname, search_path) @@ -820,7 +770,6 @@ class ModuleFinder(object): if tup: return tup - # jjj for method in self.get_module_methods: tup = method.find(fullname) if tup: @@ -1010,18 +959,6 @@ class ModuleResponder(object): minify_safe_re = re.compile(b(r'\s+#\s*!mitogen:\s*minify_safe')) def _build_tuple(self, fullname): - # tried to see if anything with collections was in the cache already - # no luck though - # jjj - # if fullname == "ansible_collections": - # import epdb; epdb.set_trace() - # for key, val in self._cache.items(): - # if "collection_inspect" in key: - # print(key, val) - # import epdb; epdb.set_trace() - # JJJJ - if fullname == "ansible_collections": - import epdb; epdb.set_trace() if fullname in self._cache: return self._cache[fullname] @@ -1052,7 +989,6 @@ class ModuleResponder(object): self.minify_secs += mitogen.core.now() - t0 if is_pkg: - # jjj pkg_present = get_child_modules(path, fullname) self._log.debug('%s is a package at %s with submodules %r', fullname, path, pkg_present) diff --git a/mitogen/service.py b/mitogen/service.py index 84a66518..0b27b3d6 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -766,11 +766,14 @@ class PushFileService(Service): # NOTE: could possibly be handled by the above TODO, but not sure how forward_modules works enough # to know for sure, so for now going to pass the sys paths themselves and have `propagate_to` # load them up in sys.path for later import - # NOOOO there should be no need for this because we aren't sending a file(s) to the child process - # jjj - if extra_sys_paths: - sys.path = extra_sys_paths + sys.path - # import epdb; epdb.set_trace() + # jjjj + # ensure we don't add to sys.path the same path we've already seen + for extra_path in extra_sys_paths: + # store extra paths in cached set for O(1) lookup + if extra_path not in self._extra_sys_paths: + # not sure if it matters but we could prepend to sys.path instead if we need to + sys.path.append(extra_path) + self._extra_sys_paths.add(extra_path) @expose(policy=AllowParents()) @arg_spec({