allow vars_files to use dwim (vars/) and vaults (#75526)

* allow vars_files to use dwim (vars/) and vaults

*  also fixed bug with temp file cleanup, some pythons dont like it when you alter loop

Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com>

lib/ansible/parsing/dataloader.py

@@ -411,7 +411,7 @@ class DataLoader:
         Removes all temporary files that DataLoader has created
         NOTE: not thread safe, forks also need special handling see __init__ for details.
         """
-        for f in self._tempfiles:
+        for f in list(self._tempfiles):
             try:
                 self.cleanup_tmp_file(f)
             except Exception as e:

lib/ansible/playbook/base.py

@@ -7,6 +7,7 @@ __metaclass__ = type
 
 import itertools
 import operator
+import os
 
 from copy import copy as shallowcopy
 from functools import partial
@@ -378,7 +379,6 @@ class FieldAttributeBase(with_metaclass(BaseMeta, object)):
                     validated_defaults_dict[defaults_entry] = defaults
 
                 else:
-                    action_names = []
                     if len(defaults_entry.split('.')) < 3:
                         defaults_entry = 'ansible.legacy.' + defaults_entry
 
@@ -854,3 +854,42 @@ class Base(FieldAttributeBase):
 
     # used to hold sudo/su stuff
     DEPRECATED_ATTRIBUTES = []
+
+    def get_path(self):
+        ''' return the absolute path of the playbook object and its line number '''
+
+        path = ""
+        try:
+            path = "%s:%s" % (self._ds._data_source, self._ds._line_number)
+        except AttributeError:
+            try:
+                path = "%s:%s" % (self._parent._play._ds._data_source, self._parent._play._ds._line_number)
+            except AttributeError:
+                pass
+        return path
+
+    def get_dep_chain(self):
+
+        if hasattr(self, '_parent') and self._parent:
+            return self._parent.get_dep_chain()
+        else:
+            return None
+
+    def get_search_path(self):
+        '''
+        Return the list of paths you should search for files, in order.
+        This follows role/playbook dependency chain.
+        '''
+        path_stack = []
+
+        dep_chain = self.get_dep_chain()
+        # inside role: add the dependency chain from current to dependent
+        if dep_chain:
+            path_stack.extend(reversed([x._role_path for x in dep_chain if hasattr(x, '_role_path')]))
+
+        # add path of task itself, unless it is already in the list
+        task_dir = os.path.dirname(self.get_path())
+        if task_dir not in path_stack:
+            path_stack.append(task_dir)
+
+        return path_stack

lib/ansible/playbook/task.py

@@ -103,16 +103,6 @@ class Task(Base, Conditional, Taggable, CollectionSearch):
 
         super(Task, self).__init__()
 
-    def get_path(self):
-        ''' return the absolute path of the task with its line number '''
-
-        path = ""
-        if hasattr(self, '_ds') and hasattr(self._ds, '_data_source') and hasattr(self._ds, '_line_number'):
-            path = "%s:%s" % (self._ds._data_source, self._ds._line_number)
-        elif hasattr(self._parent._play, '_ds') and hasattr(self._parent._play._ds, '_data_source') and hasattr(self._parent._play._ds, '_line_number'):
-            path = "%s:%s" % (self._parent._play._ds._data_source, self._parent._play._ds._line_number)
-        return path
-
     def get_name(self, include_role_fqcn=True):
         ''' return the name of the task '''
 
@@ -493,31 +483,6 @@ class Task(Base, Conditional, Taggable, CollectionSearch):
 
         return value
 
-    def get_dep_chain(self):
-        if self._parent:
-            return self._parent.get_dep_chain()
-        else:
-            return None
-
-    def get_search_path(self):
-        '''
-        Return the list of paths you should search for files, in order.
-        This follows role/playbook dependency chain.
-        '''
-        path_stack = []
-
-        dep_chain = self.get_dep_chain()
-        # inside role: add the dependency chain from current to dependent
-        if dep_chain:
-            path_stack.extend(reversed([x._role_path for x in dep_chain]))
-
-        # add path of task itself, unless it is already in the list
-        task_dir = os.path.dirname(self.get_path())
-        if task_dir not in path_stack:
-            path_stack.append(task_dir)
-
-        return path_stack
-
     def all_parents_static(self):
         if self._parent:
             return self._parent.all_parents_static()

lib/ansible/vars/manager.py

@@ -355,7 +355,10 @@ class VariableManager:
                                     "a list of string types after template expansion" % vars_file
                                 )
                             try:
-                                data = preprocess_vars(self._loader.load_from_file(vars_file, unsafe=True))
+                                play_search_stack = play.get_search_path()
+                                found_file = real_file = self._loader.path_dwim_relative_stack(play_search_stack, 'vars', vars_file)
+                                decrypted_file = self._loader.get_real_file(found_file)
+                                data = preprocess_vars(self._loader.load_from_file(decrypted_file, unsafe=True))
                                 if data is not None:
                                     for item in data:
                                         all_vars = _combine_and_track(all_vars, item, "play vars_files from '%s'" % vars_file)