From 45797a0d34dab1c25539b5f48a990eb48575c04d Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Mon, 24 Aug 2020 21:25:59 -0700 Subject: [PATCH] able to send collections paths to master with very little change to core Mitogen, need to ensure imports work properly now though --- ansible_mitogen/planner.py | 25 +++++++++++++++++++++---- mitogen/master.py | 7 +++++++ mitogen/service.py | 18 +++++++++++++++--- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index cae28bb1..6ae5ef08 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -104,6 +104,10 @@ class Invocation(object): #: Initially ``{}``, but set by :func:`invoke`. Optional source to send #: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors self._overridden_sources = {} + #: Initially ``set()``, but set by :func:`invoke`. Optional source paths to send + #: to :func:`propagate_paths_and_modules` to handle loading source dependencies from + #: places outside of the main source path, such as collections + self._extra_sys_paths = set() def get_module_source(self): if self._module_source is None: @@ -483,8 +487,9 @@ def _propagate_deps(invocation, planner, context): context=context, paths=planner.get_push_files(), - modules=planner.get_module_deps(), - overridden_sources=invocation._overridden_sources + # modules=planner.get_module_deps(), TODO + overridden_sources=invocation._overridden_sources, + extra_sys_paths=list(invocation._extra_sys_paths) ) @@ -577,6 +582,8 @@ def _load_collections(invocation): from ansible.module_utils._text import to_bytes, to_native, to_text import sys + from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder + # for path in AnsibleCollectionConfig.collection_paths: # if os.path.isdir(path): # collections = find_existing_collections(path, fallback_metadata=True) @@ -599,11 +606,18 @@ def _load_collections(invocation): # TODO: left off here. See ansible.utils.collection_loader; can't just add to path # jjj for collection in collections: - collection_path_parent = collection.b_path + # collection_path_parent = collection.b_path # import epdb; epdb.set_trace() # sys.path.insert(0, '/Users/me/.ansible/collections/ansible_collections') - sys.path.insert(0, collection.b_path.decode('utf-8')) + # sys.path.insert(0, collection.b_path.decode('utf-8')) + invocation._extra_sys_paths.add(collection.b_path.decode('utf-8')) + # import epdb; epdb.set_trace() + # handle '__synthetic__' created by ansible + # sys.modules['ansible_collections'].__file__ = sys.modules['ansible_collections'].__file__ + ".py" + # import epdb; epdb.set_trace() # import epdb; epdb.set_trace() + # uuu + # finder = _AnsibleCollectionFinder(AnsibleCollectionConfig.collection_paths, True) def invoke(invocation): """ @@ -616,6 +630,7 @@ def invoke(invocation): :raises ansible.errors.AnsibleError: Unrecognized/unsupported module type. """ + # import epdb; epdb.set_trace() path = ansible_mitogen.loaders.module_loader.find_plugin( invocation.module_name, '', @@ -631,6 +646,7 @@ def invoke(invocation): # import epdb; epdb.set_trace() if invocation.module_path not in _planner_by_path: if 'ansible_collections' in invocation.module_path: + # import epdb; epdb.set_trace() _load_collections(invocation) module_source = invocation.get_module_source() _fix_py35(invocation, module_source) @@ -647,6 +663,7 @@ def invoke(invocation): response = _invoke_isolated_task(invocation, planner) else: _propagate_deps(invocation, planner, invocation.connection.context) + # import epdb; epdb.set_trace() response = invocation.connection.get_chain().call( ansible_mitogen.target.run_module, kwargs=planner.get_kwargs(), diff --git a/mitogen/master.py b/mitogen/master.py index 8fa089a7..00e3948e 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -521,6 +521,9 @@ class PkgutilMethod(FinderMethod): if not loader: return + # jjjj + # if fullname == "ansible_collections": + # import epdb; epdb.set_trace() # jjj # if fullname == "ansible_collections": # import epdb; epdb.set_trace() @@ -817,6 +820,7 @@ class ModuleFinder(object): if tup: return tup + # jjj for method in self.get_module_methods: tup = method.find(fullname) if tup: @@ -1015,6 +1019,9 @@ class ModuleResponder(object): # if "collection_inspect" in key: # print(key, val) # import epdb; epdb.set_trace() + # JJJJ + if fullname == "ansible_collections": + import epdb; epdb.set_trace() if fullname in self._cache: return self._cache[fullname] diff --git a/mitogen/service.py b/mitogen/service.py index 69376a80..84a66518 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -691,6 +691,7 @@ class PushFileService(Service): super(PushFileService, self).__init__(**kwargs) self._lock = threading.Lock() self._cache = {} + self._extra_sys_paths = set() self._waiters = {} self._sent_by_stream = {} @@ -744,21 +745,32 @@ class PushFileService(Service): @arg_spec({ 'context': mitogen.core.Context, 'paths': list, - 'modules': list, + # 'modules': list, TODO, modules was passed into this func but it's not used yet }) - def propagate_paths_and_modules(self, context, paths, modules, overridden_sources=None): + def propagate_paths_and_modules(self, context, paths, overridden_sources=None, extra_sys_paths=None): """ One size fits all method to ensure a target context has been preloaded with a set of small files and Python modules. overridden_sources: optional dict containing source code to override path's source code + extra_sys_paths: loads additional sys paths for use in finding modules; beneficial + in situations like loading Ansible Collections because source code + dependencies come from different file paths than where the source lives """ for path in paths: overridden_source = None if overridden_sources is not None and path in overridden_sources: overridden_source = overridden_sources[path] self.propagate_to(context, mitogen.core.to_text(path), overridden_source) - #self.router.responder.forward_modules(context, modules) TODO + # self.router.responder.forward_modules(context, modules) TODO + # NOTE: could possibly be handled by the above TODO, but not sure how forward_modules works enough + # to know for sure, so for now going to pass the sys paths themselves and have `propagate_to` + # load them up in sys.path for later import + # NOOOO there should be no need for this because we aren't sending a file(s) to the child process + # jjj + if extra_sys_paths: + sys.path = extra_sys_paths + sys.path + # import epdb; epdb.set_trace() @expose(policy=AllowParents()) @arg_spec({