able to send collections paths to master with very little change to core Mitogen, need to ensure imports work properly now though

ansible_mitogen/planner.py

@@ -104,6 +104,10 @@ class Invocation(object):
         #: Initially ``{}``, but set by :func:`invoke`. Optional source to send
         #: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors
         self._overridden_sources = {}
+        #: Initially ``set()``, but set by :func:`invoke`. Optional source paths to send
+        #: to :func:`propagate_paths_and_modules` to handle loading source dependencies from
+        #: places outside of the main source path, such as collections
+        self._extra_sys_paths = set()
 
     def get_module_source(self):
         if self._module_source is None:
@@ -483,8 +487,9 @@ def _propagate_deps(invocation, planner, context):
 
         context=context,
         paths=planner.get_push_files(),
-        modules=planner.get_module_deps(),
+        # modules=planner.get_module_deps(), TODO
-        overridden_sources=invocation._overridden_sources
+        overridden_sources=invocation._overridden_sources,
+        extra_sys_paths=list(invocation._extra_sys_paths)
     )
 
 
@@ -577,6 +582,8 @@ def _load_collections(invocation):
     from ansible.module_utils._text import to_bytes, to_native, to_text
     import sys
 
+    from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
+
     # for path in AnsibleCollectionConfig.collection_paths:
     #     if os.path.isdir(path):
     #         collections = find_existing_collections(path, fallback_metadata=True)
@@ -599,11 +606,18 @@ def _load_collections(invocation):
             # TODO: left off here. See ansible.utils.collection_loader; can't just add to path
             # jjj
             for collection in collections:
-                collection_path_parent = collection.b_path
+                # collection_path_parent = collection.b_path
                 # import epdb; epdb.set_trace()
                 # sys.path.insert(0, '/Users/me/.ansible/collections/ansible_collections')
-                sys.path.insert(0, collection.b_path.decode('utf-8'))
+                # sys.path.insert(0, collection.b_path.decode('utf-8'))
+                invocation._extra_sys_paths.add(collection.b_path.decode('utf-8'))
+    # import epdb; epdb.set_trace()
+    # handle '__synthetic__' created by ansible
+    # sys.modules['ansible_collections'].__file__ = sys.modules['ansible_collections'].__file__ + ".py"
+    # import epdb; epdb.set_trace()
     # import epdb; epdb.set_trace()
+    # uuu
+    # finder = _AnsibleCollectionFinder(AnsibleCollectionConfig.collection_paths, True)
 
 def invoke(invocation):
     """
@@ -616,6 +630,7 @@ def invoke(invocation):
     :raises ansible.errors.AnsibleError:
         Unrecognized/unsupported module type.
     """
+    # import epdb; epdb.set_trace()
     path = ansible_mitogen.loaders.module_loader.find_plugin(
         invocation.module_name,
         '',
@@ -631,6 +646,7 @@ def invoke(invocation):
     #     import epdb; epdb.set_trace()
     if invocation.module_path not in _planner_by_path:
         if 'ansible_collections' in invocation.module_path:
+            # import epdb; epdb.set_trace()
             _load_collections(invocation)
         module_source = invocation.get_module_source()
         _fix_py35(invocation, module_source)
@@ -647,6 +663,7 @@ def invoke(invocation):
         response = _invoke_isolated_task(invocation, planner)
     else:
         _propagate_deps(invocation, planner, invocation.connection.context)
+        # import epdb; epdb.set_trace()
         response = invocation.connection.get_chain().call(
             ansible_mitogen.target.run_module,
             kwargs=planner.get_kwargs(),

mitogen/master.py

@@ -521,6 +521,9 @@ class PkgutilMethod(FinderMethod):
         if not loader:
             return
 
+        # jjjj
+        # if fullname == "ansible_collections":
+        #     import epdb; epdb.set_trace()
         # jjj
         # if fullname == "ansible_collections":
         #     import epdb; epdb.set_trace()
@@ -817,6 +820,7 @@ class ModuleFinder(object):
         if tup:
             return tup
 
+        # jjj
         for method in self.get_module_methods:
             tup = method.find(fullname)
             if tup:
@@ -1015,6 +1019,9 @@ class ModuleResponder(object):
         #         if "collection_inspect" in key:
         #             print(key, val)
         #     import epdb; epdb.set_trace()
+        # JJJJ
+        if fullname == "ansible_collections":
+            import epdb; epdb.set_trace()
         if fullname in self._cache:
             return self._cache[fullname]
 

mitogen/service.py

@@ -691,6 +691,7 @@ class PushFileService(Service):
         super(PushFileService, self).__init__(**kwargs)
         self._lock = threading.Lock()
         self._cache = {}
+        self._extra_sys_paths = set()
         self._waiters = {}
         self._sent_by_stream = {}
 
@@ -744,14 +745,17 @@ class PushFileService(Service):
     @arg_spec({
         'context': mitogen.core.Context,
         'paths': list,
-        'modules': list,
+        # 'modules': list, TODO, modules was passed into this func but it's not used yet
     })
-    def propagate_paths_and_modules(self, context, paths, modules, overridden_sources=None):
+    def propagate_paths_and_modules(self, context, paths, overridden_sources=None, extra_sys_paths=None):
         """
         One size fits all method to ensure a target context has been preloaded
         with a set of small files and Python modules.
 
         overridden_sources: optional dict containing source code to override path's source code
+        extra_sys_paths:    loads additional sys paths for use in finding modules; beneficial
+                            in situations like loading Ansible Collections because source code
+                            dependencies come from different file paths than where the source lives
         """
         for path in paths:
             overridden_source = None
@@ -759,6 +763,14 @@ class PushFileService(Service):
                 overridden_source = overridden_sources[path]
             self.propagate_to(context, mitogen.core.to_text(path), overridden_source)
         # self.router.responder.forward_modules(context, modules) TODO
+        # NOTE: could possibly be handled by the above TODO, but not sure how forward_modules works enough
+        #       to know for sure, so for now going to pass the sys paths themselves and have `propagate_to`
+        #       load them up in sys.path for later import
+        # NOOOO there should be no need for this because we aren't sending a file(s) to the child process
+        # jjj
+        if extra_sys_paths:
+            sys.path = extra_sys_paths + sys.path
+            # import epdb; epdb.set_trace()
 
     @expose(policy=AllowParents())
     @arg_spec({