fixed lookup search path (#16630)

* fixed lookup search path

added ansible_search_path var that contains the proper list and in order
removed roledir var which was only used by first_found, rest used role_path
added needle function for lookups that mirrors the action plugin one, now
both types of plugins use same pathing.

* added missing os import

* renamed as per feedback

* fixed missing rename in first_found

* also fixed first_found

* fixed import to match new error class

* fixed getattr ref

lib/ansible/executor/task_executor.py

@@ -83,12 +83,8 @@ class TaskExecutor:
         display.debug("in run()")
 
         try:
-            # lookup plugins need to know if this task is executing from
-            # a role, so that it can properly find files/templates/etc.
-            roledir = None
-            if self._task._role:
-                roledir = self._task._role._role_path
-            self._job_vars['roledir'] = roledir
+            # get search path for this task to pass to lookup plugins
+            self._job_vars['ansible_search_path'] = self._task.get_search_path()
 
             items = self._get_loop_items()
             if items is not None:
@@ -192,7 +188,18 @@ class TaskExecutor:
                     except AnsibleUndefinedVariable as e:
                         display.deprecated("Skipping task due to undefined Error, in the future this will be a fatal error.: %s" % to_bytes(e))
                         return None
-                items = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar).run(terms=loop_terms, variables=self._job_vars, wantlist=True)
+
+                # get lookup
+                mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar)
+
+                # give lookup task 'context' for subdir (mostly needed for first_found)
+                for subdir in ['tempalte', 'var', 'file']: #TODO: move this to constants?
+                    if subdir in self._task.name:
+                        break
+                setattr(mylookup,'_subdir', subdir + 's')
+
+                # run lookup
+                items = mylookup.run(terms=loop_terms, variables=self._job_vars, wantlist=True)
             else:
                 raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop)
 

lib/ansible/playbook/task.py

@@ -19,6 +19,8 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import os
+
 from ansible.compat.six import iteritems, string_types
 
 from ansible.errors import AnsibleError, AnsibleParserError
@@ -454,3 +456,24 @@ class Task(Base, Conditional, Taggable, Become):
 
     def _get_attr_loop_control(self):
         return self._attributes['loop_control']
+
+
+    def get_search_path(self):
+        '''
+        Return the list of paths you should search for files, in order.
+        This follows role/playbook dependency chain.
+        '''
+        path_stack = []
+
+        dep_chain =  self._block.get_dep_chain()
+        # inside role: add the dependency chain from current to dependant
+        if dep_chain:
+            path_stack.extend(reversed([x._role_path for x in dep_chain]))
+
+        # add path of task itself, unless it is already in the list
+        task_dir = os.path.dirname(self.get_path())
+        if task_dir not in path_stack:
+            path_stack.append(task_dir)
+
+        return path_stack
+

lib/ansible/plugins/action/__init__.py

@@ -831,18 +831,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
             to get back the first existing file found.
         '''
 
-        path_stack = []
-
-        dep_chain =  self._task._block.get_dep_chain()
-        # inside role: add the dependency chain
-        if dep_chain:
-            path_stack.extend(reversed([x._role_path for x in dep_chain]))
-
-
-        task_dir = os.path.dirname(self._task.get_path())
-        # include from diff directory: add it to file path
-        if not task_dir.endswith('tasks') and task_dir != self._loader.get_basedir():
-            path_stack.append(task_dir)
+        path_stack = self._task.get_search_path()
 
         result = self._loader.path_dwim_relative_stack(path_stack, dirname, needle)
 

lib/ansible/plugins/lookup/__init__.py

@@ -22,6 +22,7 @@ __metaclass__ = type
 from abc import ABCMeta, abstractmethod
 
 from ansible.compat.six import with_metaclass
+from ansible.errors import AnsibleFileNotFound
 
 try:
     from __main__ import display
@@ -101,3 +102,19 @@ class LookupBase(with_metaclass(ABCMeta, object)):
             result_string = to_unicode(result_string)
         """
         pass
+
+    def find_file_in_search_path(self, myvars, subdir, needle):
+        '''
+        Return a file (needle) in the task's expected search path.
+        '''
+
+        if 'ansible_search_path' in myvars:
+            paths = myvars['ansible_search_path']
+        else:
+            paths = self.get_basedir(myvars)
+
+        result = self._loader.path_dwim_relative_stack(paths, subdir, needle)
+        if result is None:
+            raise AnsibleFileNotFound("Unable to find '%s' in expected paths." % needle)
+
+        return result

lib/ansible/plugins/lookup/csvfile.py

@@ -72,8 +72,6 @@ class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):
 
-        basedir = self.get_basedir(variables)
-
         ret = []
 
         for term in terms:
@@ -100,7 +98,7 @@ class LookupModule(LookupBase):
             if paramvals['delimiter'] == 'TAB':
                 paramvals['delimiter'] = "\t"
 
-            lookupfile = self._loader.path_dwim_relative(basedir, 'files', paramvals['file'])
+            lookupfile = self.find_file_in_search_path(variables, 'files', paramvals['file'])
             var = self.read_csv(lookupfile, key, paramvals['delimiter'], paramvals['encoding'], paramvals['default'], paramvals['col'])
             if var is not None:
                 if type(var) is list:

lib/ansible/plugins/lookup/file.py

@@ -33,18 +33,11 @@ class LookupModule(LookupBase):
 
         ret = []
 
-        basedir = self.get_basedir(variables)
-
         for term in terms:
             display.debug("File lookup term: %s" % term)
 
-            # Special handling of the file lookup, used primarily when the
-            # lookup is done from a role. If the file isn't found in the
-            # basedir of the current file, use dwim_relative to look in the
-            # role/files/ directory, and finally the playbook directory
-            # itself (which will be relative to the current working dir)
-
-            lookupfile = self._loader.path_dwim_relative(basedir, 'files', term)
+            # Find the file in the expected search path
+            lookupfile = self.find_file_in_search_path(variables, 'files', term)
             display.vvvv("File lookup using %s as file" % lookupfile)
             try:
                 if lookupfile:

lib/ansible/plugins/lookup/fileglob.py

@@ -26,12 +26,10 @@ class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):
 
-        basedir = self.get_basedir(variables)
-
         ret = []
         for term in terms:
             term_file = os.path.basename(term)
-            dwimmed_path = self._loader.path_dwim_relative(basedir, 'files', os.path.dirname(term))
+            dwimmed_path = self.find_file_in_search_path(variables, 'files', os.path.dirname(term))
             globbed = glob.glob(os.path.join(dwimmed_path, term_file))
             ret.extend(g for g in globbed if os.path.isfile(g))
         return ret

lib/ansible/plugins/lookup/first_found.py

@@ -118,12 +118,11 @@ __metaclass__ = type
 #     - ../files/baz
 #    ignore_errors: true
 
-
 import os
 
 from jinja2.exceptions import UndefinedError
 
-from ansible.errors import AnsibleLookupError, AnsibleUndefinedVariable
+from ansible.errors import AnsibleFileNotFound, AnsibleLookupError, AnsibleUndefinedVariable
 from ansible.plugins.lookup import LookupBase
 from ansible.utils.boolean import boolean
 
@@ -131,7 +130,6 @@ class LookupModule(LookupBase):
 
     def run(self, terms, variables, **kwargs):
 
-        result = None
         anydict = False
         skip = False
 
@@ -173,28 +171,20 @@ class LookupModule(LookupBase):
         else:
             total_search = self._flatten(terms)
 
-        roledir = variables.get('roledir')
         for fn in total_search:
             try:
                 fn = self._templar.template(fn)
-            except (AnsibleUndefinedVariable, UndefinedError) as e:
+            except (AnsibleUndefinedVariable, UndefinedError):
                 continue
 
-            if os.path.isabs(fn) and os.path.exists(fn):
-                return [fn]
-            else:
-                if roledir is not None:
-                    # check the templates and vars directories too,if they exist
-                    for subdir in ('templates', 'vars', 'files'):
-                        path = self._loader.path_dwim_relative(roledir, subdir, fn)
-                        if os.path.exists(path):
-                            return [path]
-
-                # if none of the above were found, just check the
-                # current filename against the current dir
-                path = self._loader.path_dwim(fn)
-                if os.path.exists(path):
-                    return [path]
+             # get subdir if set by task executor, default to files otherwise
+            subdir = getattr(self, '_subdir', 'files')
+            path = None
+            try:
+                path = self.find_file_in_search_path(variables, subdir, fn)
+                return [path]
+            except AnsibleFileNotFound:
+                continue
         else:
             if skip:
                 return []

lib/ansible/plugins/lookup/ini.py

@@ -107,7 +107,7 @@ class LookupModule(LookupBase):
             except (ValueError, AssertionError) as e:
                 raise AnsibleError(e)
 
-            path = self._loader.path_dwim_relative(basedir, 'files', paramvals['file'])
+            path = self.find_file_in_search_path(variables, 'files', paramvals['file'])
             if paramvals['type'] == "properties":
                 var = self.read_properties(path, key, paramvals['default'], paramvals['re'])
             else:

lib/ansible/plugins/lookup/shelvefile.py

@@ -18,7 +18,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import shelve
-import os
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
@@ -43,9 +42,6 @@ class LookupModule(LookupBase):
         ret = []
 
         for term in terms:
-            playbook_path = None
-            relative_path = None
-
             paramvals = {"file": None, "key": None}
             params = term.split()
 
@@ -59,24 +55,18 @@ class LookupModule(LookupBase):
                 # In case "file" or "key" are not present
                 raise AnsibleError(e)
 
-            file = paramvals['file']
             key = paramvals['key']
-            basedir_path  = self._loader.path_dwim(file)
 
             # Search also in the role/files directory and in the playbook directory
-            if 'role_path' in variables:
-                relative_path = self._loader.path_dwim_relative(variables['role_path'], 'files', file)
-            if 'playbook_dir' in variables:
-                playbook_path = self._loader.path_dwim_relative(variables['playbook_dir'],'files', file)
-
-            for path in (basedir_path, relative_path, playbook_path):
-                if path and os.path.exists(path):
-                    res = self.read_shelve(path, key)
-                    if res is None:
-                        raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
-                    # Convert the value read to string
-                    ret.append(str(res))
-                    break
+            shelvefile = self.find_file_in_search_path(variables, 'files', paramvals['file'])
+
+            if shelvefile:
+                res = self.read_shelve(shelvefile, key)
+                if res is None:
+                    raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
+                # Convert the value read to string
+                ret.append(str(res))
+                break
             else:
                 raise AnsibleError("Could not locate shelve file in lookup: %s" % file)
 

lib/ansible/plugins/lookup/template.py

@@ -19,7 +19,6 @@ __metaclass__ = type
 
 import os
 
-from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.utils.unicode import to_unicode
@@ -36,26 +35,25 @@ class LookupModule(LookupBase):
     def run(self, terms, variables, **kwargs):
 
         convert_data_p = kwargs.get('convert_data', True)
-        basedir = self.get_basedir(variables)
-
         ret = []
 
         for term in terms:
             display.debug("File lookup term: %s" % term)
 
-            lookupfile = self._loader.path_dwim_relative(basedir, 'templates', term)
+            lookupfile = self.find_file_in_search_path(variables, 'templates', term)
             display.vvvv("File lookup using %s as file" % lookupfile)
-            if lookupfile and os.path.exists(lookupfile):
+            if lookupfile:
                 with open(lookupfile, 'r') as f:
                     template_data = to_unicode(f.read())
 
-                    searchpath = [self._loader._basedir, os.path.dirname(lookupfile)]
-                    if 'role_path' in variables:
-                        if C.DEFAULT_ROLES_PATH:
-                            searchpath[:0] = C.DEFAULT_ROLES_PATH
-                        searchpath.insert(1, variables['role_path'])
-
+                    # set jinja2 internal search path for includes
+                    if 'ansible_search_path' in variables:
+                        searchpath = variables['ansible_search_path']
+                    else:
+                        searchpath = [self._loader._basedir, os.path.dirname(lookupfile)]
                     self._templar.environment.loader.searchpath = searchpath
+
+                    # do the templating
                     res = self._templar.template(template_data, preserve_trailing_newlines=True,convert_data=convert_data_p)
                     ret.append(res)
             else: