able to run docker_container installed via 'ansible-galaxy collection install community.general'

ansible_mitogen/planner.py

@@ -42,12 +42,9 @@ import logging
 import os
 import random
 
-from ansible import context as ansible_context
 from ansible.executor import module_common
-from ansible.galaxy.collection import (
-    find_existing_collections,
-    validate_collection_path
-)
+from ansible.galaxy.collection import find_existing_collections
+from ansible.utils.collection_loader import AnsibleCollectionConfig
 import ansible.errors
 import ansible.module_utils
 import ansible.release
@@ -489,6 +486,7 @@ def _propagate_deps(invocation, planner, context):
         paths=planner.get_push_files(),
         # modules=planner.get_module_deps(), TODO
         overridden_sources=invocation._overridden_sources,
+        # needs to be a list because can't unpickle() a set()
         extra_sys_paths=list(invocation._extra_sys_paths)
     )
 
@@ -569,55 +567,17 @@ def _fix_py35(invocation, module_source):
 
 def _load_collections(invocation):
     """
-    Special loader that ensures that `ansible_collections` exists as a module path for import
+    Special loader that ensures that `ansible_collections` exist as a module path for import
+    Goes through all collection path possibilities and stores paths to installed collections
+    Stores them on the current invocation to later be passed to the master service
     """
-    # import epdb; epdb.set_trace()
-    # find_existing_collections()
-    # collection_path = validate_collection_path(path)
-    # collection_path = GalaxyCLI._resolve_path(path)
-
-    # import epdb; epdb.set_trace()
-    from ansible.utils.collection_loader import AnsibleCollectionConfig
-    from ansible.cli.galaxy import _get_collection_widths, _display_header, _display_collection
-    from ansible.module_utils._text import to_bytes, to_native, to_text
-    import sys
-
-    from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
-
-    # for path in AnsibleCollectionConfig.collection_paths:
-    #     if os.path.isdir(path):
-    #         collections = find_existing_collections(path, fallback_metadata=True)
-
-    #         fqcn_width, version_width = _get_collection_widths(collections)
-    #         # _display_header(path, 'Collection', 'Version', fqcn_width, version_width)
-
-    #         # Sort collections by the namespace and name
-    #         collections.sort(key=to_text)
-    #         for collection in collections:
-    #             _display_collection(collection, fqcn_width, version_width)
-
     for path in AnsibleCollectionConfig.collection_paths:
         if os.path.isdir(path):
-            # import epdb; epdb.set_trace()
             collections = find_existing_collections(path, fallback_metadata=True)
 
-            # add the collection's parent path to sys.path
-            # additionally, handle __synthetic__
-            # TODO: left off here. See ansible.utils.collection_loader; can't just add to path
-            # jjj
             for collection in collections:
-                # collection_path_parent = collection.b_path
-                # import epdb; epdb.set_trace()
-                # sys.path.insert(0, '/Users/me/.ansible/collections/ansible_collections')
-                # sys.path.insert(0, collection.b_path.decode('utf-8'))
                 invocation._extra_sys_paths.add(collection.b_path.decode('utf-8'))
-    # import epdb; epdb.set_trace()
-    # handle '__synthetic__' created by ansible
-    # sys.modules['ansible_collections'].__file__ = sys.modules['ansible_collections'].__file__ + ".py"
-    # import epdb; epdb.set_trace()
-    # import epdb; epdb.set_trace()
-    # uuu
-    # finder = _AnsibleCollectionFinder(AnsibleCollectionConfig.collection_paths, True)
+
 
 def invoke(invocation):
     """
@@ -630,7 +590,6 @@ def invoke(invocation):
     :raises ansible.errors.AnsibleError:
         Unrecognized/unsupported module type.
     """
-    # import epdb; epdb.set_trace()
     path = ansible_mitogen.loaders.module_loader.find_plugin(
         invocation.module_name,
         '',
@@ -641,13 +600,10 @@ def invoke(invocation):
         ))
 
     invocation.module_path = mitogen.core.to_text(path)
-    #jjj
-    # if 'ansible_collections' in invocation.module_path:
-    #     import epdb; epdb.set_trace()
     if invocation.module_path not in _planner_by_path:
         if 'ansible_collections' in invocation.module_path:
-            # import epdb; epdb.set_trace()
             _load_collections(invocation)
+
         module_source = invocation.get_module_source()
         _fix_py35(invocation, module_source)
         _planner_by_path[invocation.module_path] = _get_planner(
@@ -663,7 +619,6 @@ def invoke(invocation):
         response = _invoke_isolated_task(invocation, planner)
     else:
         _propagate_deps(invocation, planner, invocation.connection.context)
-        # import epdb; epdb.set_trace()
         response = invocation.connection.get_chain().call(
             ansible_mitogen.target.run_module,
             kwargs=planner.get_kwargs(),

mitogen/master.py

@@ -89,6 +89,14 @@ except NameError:
 RLOG = logging.getLogger('mitogen.ctx')
 
 
+# there are some cases where modules are loaded in memory only, such as
+# ansible collections, and the module "filename" is something like __synthetic__
+# which doesn't actually exist
+SPECIAL_FILE_PATHS = [
+    "__synthetic__"
+]
+
+
 def _stdlib_paths():
     """
     Return a set of paths from which Python imports the standard library.
@@ -155,26 +163,6 @@ def get_child_modules(path, fullname):
     :return:
         List of submodule name suffixes.
     """
-    # jjj
-    # TODO: move this somehow to ansible_mitogen, if it's even possible
-    # ISSUE: not everything is being loaded via sys.modules in ansible when it comes to collections
-    # only `action` and `modules` show up in sys.modules[fullname]
-    # but sometimes you want things like `module_utils`
-    # if fullname.startswith("ansible_collections"):
-    #     submodules = []
-    #     # import epdb; epdb.set_trace()
-    #     # sys.modules[fullname].__path__
-    #     # for each in dir(sys.modules[fullname]):
-    #     #     if not each.startswith("__"):
-    #     #         submodules.append(to_text(each))
-    #     for each in os.listdir(sys.modules[fullname].__path__[0]):
-    #         if not each.startswith("__"):
-    #             submodules.append(to_text(each))
-    #             # jjj
-    #             # hack: insert submodule on the path so it can be loaded
-    #             # sys.path.insert(0, each)
-    #     return submodules
-    # else:
     it = pkgutil.iter_modules([os.path.dirname(path)])
     return [to_text(name) for _, name, _ in it]
 
@@ -213,6 +201,11 @@ def _py_filename(path):
     if os.path.exists(path) and _looks_like_script(path):
         return path
 
+    basepath = os.path.basename(path)
+    for filename in SPECIAL_FILE_PATHS:
+        if basepath == filename:
+            return path
+
 
 def _get_core_source():
     """
@@ -521,21 +514,6 @@ class PkgutilMethod(FinderMethod):
         if not loader:
             return
 
-        # jjjj
-        # if fullname == "ansible_collections":
-        #     import epdb; epdb.set_trace()
-        # jjj
-        # if fullname == "ansible_collections":
-        #     import epdb; epdb.set_trace()
-        #     ba = loader.load_module("ansible_collections.alikins")
-        # # if fullname == "ansible_collections":
-        # #     # ansible named the fake __file__ for collections `__synthetic__` with no extension
-        # #     module.__file__ = module.__file__ + ".py"
-        # #     import epdb; epdb.set_trace()
-        # #     # import epdb; epdb.set_trace()
-        # #     # jjj      
-        # doesn't work because .get_source doesn't exist. Collections loader is super complicated
-        # and doesn't offer an easy way to extract source code  
         try:
             path = _py_filename(loader.get_filename(fullname))
             source = loader.get_source(fullname)
@@ -574,10 +552,7 @@ class SysModulesMethod(FinderMethod):
         Find `fullname` using its :data:`__file__` attribute.
         """
         module = sys.modules.get(fullname)
-        # jjj
-        # this is hit
-        # if fullname.startswith("ansible_collections"):
-        #     import epdb; epdb.set_trace()
+
         if not isinstance(module, types.ModuleType):
             LOG.debug('%r: sys.modules[%r] absent or not a regular module',
                       self, fullname)
@@ -592,13 +567,6 @@ class SysModulesMethod(FinderMethod):
                       fullname, alleged_name, module)
             return
 
-        # TODO: move to ansible_mitogen somehow if possible
-        # ansible names the fake __file__ for collections `__synthetic__` with no extension
-        # if fullname.startswith("ansible_collections"):
-        #     print(fullname)
-        #     module.__file__ = module.__file__ + ".py"
-        #     # import epdb; epdb.set_trace()
-        #     # jjj
         path = _py_filename(getattr(module, '__file__', ''))
         if not path:
             return
@@ -705,24 +673,6 @@ class ParentEnumerationMethod(FinderMethod):
         return path, source, is_pkg
 
     def _find_one_component(self, modname, search_path):
-        """
-        Creates an __init__.py if one doesn't exist in the search path dirs for ansible collections
-        This will help imp load packages like `.ansible/collections/ansible_collections/.....plugins/module_utils`
-        that don't get loaded from Ansible via sys.modules
-        Unfortunately this leaves __init__.py files around in collections that don't have them
-        TODO: delete these when Mitogen exits?
-        Tried to hack types.ModuleType instead but no luck
-        Appears imp loads modules old-style with a required __init__.py
-        TODO: can the __init__ stuff be moved to ansible_mitogen somewhere, really want it to be there instead
-        """
-        # jjj
-        # for path in search_path:
-        #     if "collections/ansible_collections" in path:
-        #         init_file = os.path.join(path, modname, "__init__.py")
-        #         if not os.path.isfile(init_file):
-        #             with open(init_file, "w") as f:
-        #                 pass
-
         try:
             #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path)
             return imp.find_module(modname, search_path)
@@ -820,7 +770,6 @@ class ModuleFinder(object):
         if tup:
             return tup
 
-        # jjj
         for method in self.get_module_methods:
             tup = method.find(fullname)
             if tup:
@@ -1010,18 +959,6 @@ class ModuleResponder(object):
     minify_safe_re = re.compile(b(r'\s+#\s*!mitogen:\s*minify_safe'))
 
     def _build_tuple(self, fullname):
-        # tried to see if anything with collections was in the cache already
-        # no luck though
-        # jjj
-        # if fullname == "ansible_collections":
-        #     import epdb; epdb.set_trace()
-        #     for key, val in self._cache.items():
-        #         if "collection_inspect" in key:
-        #             print(key, val)
-        #     import epdb; epdb.set_trace()
-        # JJJJ
-        if fullname == "ansible_collections":
-            import epdb; epdb.set_trace()
         if fullname in self._cache:
             return self._cache[fullname]
 
@@ -1052,7 +989,6 @@ class ModuleResponder(object):
             self.minify_secs += mitogen.core.now() - t0
 
         if is_pkg:
-            # jjj
             pkg_present = get_child_modules(path, fullname)
             self._log.debug('%s is a package at %s with submodules %r',
                             fullname, path, pkg_present)

mitogen/service.py

@@ -766,11 +766,14 @@ class PushFileService(Service):
         # NOTE: could possibly be handled by the above TODO, but not sure how forward_modules works enough
         #       to know for sure, so for now going to pass the sys paths themselves and have `propagate_to`
         #       load them up in sys.path for later import
-        # NOOOO there should be no need for this because we aren't sending a file(s) to the child process
-        # jjj
-        if extra_sys_paths:
-            sys.path = extra_sys_paths + sys.path
-            # import epdb; epdb.set_trace()
+        # jjjj
+        # ensure we don't add to sys.path the same path we've already seen
+        for extra_path in extra_sys_paths:
+            # store extra paths in cached set for O(1) lookup
+            if extra_path not in self._extra_sys_paths:
+                # not sure if it matters but we could prepend to sys.path instead if we need to
+                sys.path.append(extra_path)
+                self._extra_sys_paths.add(extra_path)
 
     @expose(policy=AllowParents())
     @arg_spec({