Use templar all the way down

Fixes bugs related to creating Templar() objects on the fly, where
the shared loader objects (serialized to TaskExecutor) aren't used
so information loaded into plugin loaders after forking is lost.

Fixes #11815

lib/ansible/executor/task_executor.py

@@ -29,7 +29,7 @@ from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.task import Task
-from ansible.plugins import lookup_loader, connection_loader, action_loader
+from ansible.plugins import connection_loader, action_loader
 from ansible.template import Templar
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.unicode import to_unicode
@@ -149,10 +149,14 @@ class TaskExecutor:
         # now we update them with the play context vars
         self._play_context.update_vars(vars_copy)
 
+        templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=vars_copy)
         items = None
-        if self._task.loop and self._task.loop in lookup_loader:
+        if self._task.loop:
-            loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, variables=vars_copy, loader=self._loader, fail_on_undefined=True)
+            if self._task.loop in self._shared_loader_obj.lookup_loader:
-            items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=loop_terms, variables=vars_copy)
+                loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=True)
+                items = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar).run(terms=loop_terms, variables=vars_copy)
+            else:
+                raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop)
 
         return items
 

lib/ansible/playbook/__init__.py

@@ -26,7 +26,7 @@ from ansible.parsing import DataLoader
 from ansible.playbook.attribute import Attribute, FieldAttribute
 from ansible.playbook.play import Play
 from ansible.playbook.playbook_include import PlaybookInclude
-from ansible.plugins import push_basedir
+from ansible.plugins import get_all_plugin_loaders
 
 
 __all__ = ['Playbook']
@@ -57,8 +57,12 @@ class Playbook:
         # set the loaders basedir
         self._loader.set_basedir(self._basedir)
 
-        # also add the basedir to the list of module directories
+        # dynamically load any plugins from the role directory
-        push_basedir(self._basedir)
+        for name, obj in get_all_plugin_loaders():
+            if obj.subdir:
+                plugin_path = os.path.join(self._basedir, obj.subdir)
+                if os.path.isdir(plugin_path):
+                    obj.add_directory(plugin_path)
 
         ds = self._loader.load_from_file(os.path.basename(file_name))
         if not isinstance(ds, list):

lib/ansible/playbook/role/__init__.py

@@ -37,7 +37,7 @@ from ansible.playbook.helpers import load_list_of_blocks
 from ansible.playbook.role.include import RoleInclude
 from ansible.playbook.role.metadata import RoleMetadata
 from ansible.playbook.taggable import Taggable
-from ansible.plugins import get_all_plugin_loaders, push_basedir
+from ansible.plugins import get_all_plugin_loaders
 from ansible.utils.vars import combine_vars
 
 
@@ -142,8 +142,6 @@ class Role(Base, Become, Conditional, Taggable):
         self._variable_manager = role_include.get_variable_manager()
         self._loader           = role_include.get_loader()
 
-        push_basedir(self._role_path)
-
         if parent_role:
             self.add_parent(parent_role)
 

lib/ansible/plugins/__init__.py

@@ -42,6 +42,11 @@ PATH_CACHE = {}
 PLUGIN_PATH_CACHE = {}
 _basedirs = []
 
+# FIXME: the _basedirs code may be dead, and no longer needed, as
+#        we now use add_directory for all plugin types here instead
+#        of relying on this global variable (which also causes problems
+#        with forked processes). See the Playbook() and Role() classes
+#        for how we now ue get_all_plugin_loaders() below.
 def push_basedir(basedir):
     # avoid pushing the same absolute dir more than once
     basedir = to_unicode(os.path.realpath(basedir))

lib/ansible/plugins/lookup/__init__.py

@@ -28,8 +28,9 @@ except ImportError:
 __all__ = ['LookupBase']
 
 class LookupBase:
-    def __init__(self, loader=None, **kwargs):
+    def __init__(self, loader=None, templar=None, **kwargs):
         self._loader = loader
+        self._templar = templar
         self._display = display
 
     def get_basedir(self, variables):

lib/ansible/plugins/lookup/cartesian.py

@@ -29,16 +29,16 @@ class LookupModule(LookupBase):
     [1, 2, 3], [a, b] -> [1, a], [1, b], [2, a], [2, b], [3, a], [3, b]
     """
 
-    def __lookup_variabless(self, terms, variables):
+    def __lookup_variables(self, terms):
         results = []
         for x in terms:
-            intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader)
+            intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader)
             results.append(intermediate)
         return results
 
     def run(self, terms, variables=None, **kwargs):
 
-        terms = self.__lookup_variabless(terms, variables)
+        terms = self.__lookup_variables(terms)
 
         my_list = terms[:]
         if len(my_list) == 0:

lib/ansible/plugins/lookup/first_found.py

@@ -125,7 +125,6 @@ from jinja2.exceptions import UndefinedError
 
 from ansible.errors import AnsibleLookupError, AnsibleUndefinedVariable
 from ansible.plugins.lookup import LookupBase
-from ansible.template import Templar
 from ansible.utils.boolean import boolean
 
 class LookupModule(LookupBase):
@@ -174,11 +173,10 @@ class LookupModule(LookupBase):
         else:
             total_search = terms
 
-        templar = Templar(loader=self._loader, variables=variables)
         roledir = variables.get('roledir')
         for fn in total_search:
             try:
-                fn = templar.template(fn)
+                fn = self._templar.template(fn)
             except (AnsibleUndefinedVariable, UndefinedError) as e:
                 continue
 

lib/ansible/plugins/lookup/flattened.py

@@ -46,7 +46,7 @@ class LookupModule(LookupBase):
 
             if isinstance(term, basestring):
                 # convert a variable to a list
-                term2 = listify_lookup_plugin_terms(term, variables, loader=self._loader)
+                term2 = listify_lookup_plugin_terms(term, templar=self._templar, loader=self._loader)
                 # but avoid converting a plain string to a list of one string
                 if term2 != [ term ]:
                     term = term2

lib/ansible/plugins/lookup/nested.py

@@ -31,7 +31,7 @@ class LookupModule(LookupBase):
         results = []
         for x in terms:
             try:
-                intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader, fail_on_undefined=True)
+                intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader, fail_on_undefined=True)
             except UndefinedError, e:
                 raise AnsibleUndefinedVariable("One of the nested variables was undefined. The error was: %s" % e)
             results.append(intermediate)

lib/ansible/plugins/lookup/sequence.py

@@ -22,7 +22,6 @@ from re import compile as re_compile, IGNORECASE
 from ansible.errors import *
 from ansible.parsing.splitter import parse_kv
 from ansible.plugins.lookup import LookupBase
-from ansible.template import Templar
 
 # shortcut format
 NUM = "(0?x?[0-9a-f]+)"
@@ -188,13 +187,11 @@ class LookupModule(LookupBase):
         if isinstance(terms, basestring):
             terms = [ terms ]
 
-        templar = Templar(loader=self._loader, variables=variables)
-
         for term in terms:
             try:
                 self.reset()  # clear out things for this iteration
 
-                term = templar.template(term)
+                term = self._templar.template(term)
                 try:
                     if not self.parse_simple_args(term):
                         self.parse_kv_args(parse_kv(term))

lib/ansible/plugins/lookup/subelements.py

@@ -33,8 +33,9 @@ class LookupModule(LookupBase):
             raise AnsibleError(
                 "subelements lookup expects a list of two or three items, "
                 + msg)
-        terms = listify_lookup_plugin_terms(terms, variables, loader=self._loader)
+
-        terms[0] = listify_lookup_plugin_terms(terms[0], variables, loader=self._loader)
+        terms = listify_lookup_plugin_terms(terms, templar=self._templar, loader=self._loader)
+        terms[0] = listify_lookup_plugin_terms(terms[0], templar=self._templar, loader=self._loader)
 
         # check lookup terms - check number of terms
         if not isinstance(terms, list) or not 2 <= len(terms) <= 3:

lib/ansible/plugins/lookup/template.py

@@ -21,7 +21,6 @@ import os
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.template import Templar
 
 class LookupModule(LookupBase):
 
@@ -34,8 +33,6 @@ class LookupModule(LookupBase):
 
         ret = []
 
-        templar = Templar(loader=self._loader, variables=variables)
-
         for term in terms:
             self._display.debug("File lookup term: %s" % term)
 
@@ -44,7 +41,7 @@ class LookupModule(LookupBase):
             if lookupfile and os.path.exists(lookupfile):
                 with open(lookupfile, 'r') as f:
                     template_data = f.read()
-                    res = templar.template(template_data, preserve_trailing_newlines=True)
+                    res = self._templar.template(template_data, preserve_trailing_newlines=True)
                     ret.append(res)
             else:
                 raise AnsibleError("the template file %s could not be found for the lookup" % term)

lib/ansible/plugins/lookup/together.py

@@ -31,16 +31,16 @@ class LookupModule(LookupBase):
     [1, 2], [3] -> [1, 3], [2, None]
     """
 
-    def __lookup_variabless(self, terms, variables):
+    def __lookup_variables(self, terms):
         results = []
         for x in terms:
-            intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader)
+            intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader)
             results.append(intermediate)
         return results
 
     def run(self, terms, variables=None, **kwargs):
 
-        terms = self.__lookup_variabless(terms, variables)
+        terms = self.__lookup_variables(terms)
 
         my_list = terms[:]
         if len(my_list) == 0:

lib/ansible/template/__init__.py

@@ -65,8 +65,6 @@ class Templar:
             self._basedir = './'
 
         if shared_loader_obj:
-            global _basedirs
-            _basedirs = shared_loader_obj.basedirs[:]
             self._filter_loader = getattr(shared_loader_obj, 'filter_loader')
             self._lookup_loader = getattr(shared_loader_obj, 'lookup_loader')
         else:
@@ -250,7 +248,7 @@ class Templar:
         return thing if thing is not None else ''
 
     def _lookup(self, name, *args, **kwargs):
-        instance = self._lookup_loader.get(name.lower(), loader=self._loader)
+        instance = self._lookup_loader.get(name.lower(), loader=self._loader, templar=self)
 
         if instance is not None:
             # safely catch run failures per #5059

lib/ansible/utils/listify.py

@@ -26,17 +26,16 @@ from ansible.template.safe_eval import safe_eval
 __all__ = ['listify_lookup_plugin_terms']
 
 #FIXME: probably just move this into lookup plugin base class
-def listify_lookup_plugin_terms(terms, variables, loader, fail_on_undefined=False):
+def listify_lookup_plugin_terms(terms, templar, loader, fail_on_undefined=False):
 
     if isinstance(terms, basestring):
         stripped = terms.strip()
-        templar = Templar(loader=loader, variables=variables)
-
         #FIXME: warn/deprecation on bare vars in with_ so we can eventually remove fail on undefined override
         terms = templar.template(terms, convert_bare=True, fail_on_undefined=fail_on_undefined)
-
         #TODO: check if this is needed as template should also return correct type already
-        terms = safe_eval(terms)
+        #terms = safe_eval(terms)
+    else:
+        terms = templar.template(terms, fail_on_undefined=fail_on_undefined)
 
     if isinstance(terms, basestring) or not isinstance(terms, Iterable):
         terms = [ terms ]