From 52666799641171a676f075a19f9ef733acacb870 Mon Sep 17 00:00:00 2001
From: James Cammarata <jimi@sngx.net>
Date: Tue, 4 Aug 2015 12:10:23 -0400
Subject: [PATCH] Use templar all the way down

Fixes bugs related to creating Templar() objects on the fly, where
the shared loader objects (serialized to TaskExecutor) aren't used
so information loaded into plugin loaders after forking is lost.

Fixes #11815
---
 lib/ansible/executor/task_executor.py     | 12 ++++++++----
 lib/ansible/playbook/__init__.py          | 10 +++++++---
 lib/ansible/playbook/role/__init__.py     |  4 +---
 lib/ansible/plugins/__init__.py           |  5 +++++
 lib/ansible/plugins/lookup/__init__.py    |  3 ++-
 lib/ansible/plugins/lookup/cartesian.py   |  6 +++---
 lib/ansible/plugins/lookup/first_found.py |  4 +---
 lib/ansible/plugins/lookup/flattened.py   |  2 +-
 lib/ansible/plugins/lookup/nested.py      |  2 +-
 lib/ansible/plugins/lookup/sequence.py    |  5 +----
 lib/ansible/plugins/lookup/subelements.py |  5 +++--
 lib/ansible/plugins/lookup/template.py    |  5 +----
 lib/ansible/plugins/lookup/together.py    |  6 +++---
 lib/ansible/template/__init__.py          |  4 +---
 lib/ansible/utils/listify.py              | 13 ++++++-------
 15 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py
index b317c72cb5a..297d8b2526f 100644
--- a/lib/ansible/executor/task_executor.py
+++ b/lib/ansible/executor/task_executor.py
@@ -29,7 +29,7 @@ from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.task import Task
-from ansible.plugins import lookup_loader, connection_loader, action_loader
+from ansible.plugins import connection_loader, action_loader
 from ansible.template import Templar
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.unicode import to_unicode
@@ -149,10 +149,14 @@ class TaskExecutor:
         # now we update them with the play context vars
         self._play_context.update_vars(vars_copy)
 
+        templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=vars_copy)
         items = None
-        if self._task.loop and self._task.loop in lookup_loader:
-            loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, variables=vars_copy, loader=self._loader, fail_on_undefined=True)
-            items = lookup_loader.get(self._task.loop, loader=self._loader).run(terms=loop_terms, variables=vars_copy)
+        if self._task.loop:
+            if self._task.loop in self._shared_loader_obj.lookup_loader:
+                loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=True)
+                items = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar).run(terms=loop_terms, variables=vars_copy)
+            else:
+                raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop)
 
         return items
 
diff --git a/lib/ansible/playbook/__init__.py b/lib/ansible/playbook/__init__.py
index 40e6638f239..6f7aee35ad9 100644
--- a/lib/ansible/playbook/__init__.py
+++ b/lib/ansible/playbook/__init__.py
@@ -26,7 +26,7 @@ from ansible.parsing import DataLoader
 from ansible.playbook.attribute import Attribute, FieldAttribute
 from ansible.playbook.play import Play
 from ansible.playbook.playbook_include import PlaybookInclude
-from ansible.plugins import push_basedir
+from ansible.plugins import get_all_plugin_loaders
 
 
 __all__ = ['Playbook']
@@ -57,8 +57,12 @@ class Playbook:
         # set the loaders basedir
         self._loader.set_basedir(self._basedir)
 
-        # also add the basedir to the list of module directories
-        push_basedir(self._basedir)
+        # dynamically load any plugins from the role directory
+        for name, obj in get_all_plugin_loaders():
+            if obj.subdir:
+                plugin_path = os.path.join(self._basedir, obj.subdir)
+                if os.path.isdir(plugin_path):
+                    obj.add_directory(plugin_path)
 
         ds = self._loader.load_from_file(os.path.basename(file_name))
         if not isinstance(ds, list):
diff --git a/lib/ansible/playbook/role/__init__.py b/lib/ansible/playbook/role/__init__.py
index 415ce12d1f8..f46014f60bc 100644
--- a/lib/ansible/playbook/role/__init__.py
+++ b/lib/ansible/playbook/role/__init__.py
@@ -37,7 +37,7 @@ from ansible.playbook.helpers import load_list_of_blocks
 from ansible.playbook.role.include import RoleInclude
 from ansible.playbook.role.metadata import RoleMetadata
 from ansible.playbook.taggable import Taggable
-from ansible.plugins import get_all_plugin_loaders, push_basedir
+from ansible.plugins import get_all_plugin_loaders
 from ansible.utils.vars import combine_vars
 
 
@@ -142,8 +142,6 @@ class Role(Base, Become, Conditional, Taggable):
         self._variable_manager = role_include.get_variable_manager()
         self._loader           = role_include.get_loader()
 
-        push_basedir(self._role_path)
-
         if parent_role:
             self.add_parent(parent_role)
 
diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py
index f8273fbcdad..4054c616335 100644
--- a/lib/ansible/plugins/__init__.py
+++ b/lib/ansible/plugins/__init__.py
@@ -42,6 +42,11 @@ PATH_CACHE = {}
 PLUGIN_PATH_CACHE = {}
 _basedirs = []
 
+# FIXME: the _basedirs code may be dead, and no longer needed, as
+#        we now use add_directory for all plugin types here instead
+#        of relying on this global variable (which also causes problems
+#        with forked processes). See the Playbook() and Role() classes
+#        for how we now ue get_all_plugin_loaders() below.
 def push_basedir(basedir):
     # avoid pushing the same absolute dir more than once
     basedir = to_unicode(os.path.realpath(basedir))
diff --git a/lib/ansible/plugins/lookup/__init__.py b/lib/ansible/plugins/lookup/__init__.py
index 9ce4add7010..5abca22ac7d 100644
--- a/lib/ansible/plugins/lookup/__init__.py
+++ b/lib/ansible/plugins/lookup/__init__.py
@@ -28,8 +28,9 @@ except ImportError:
 __all__ = ['LookupBase']
 
 class LookupBase:
-    def __init__(self, loader=None, **kwargs):
+    def __init__(self, loader=None, templar=None, **kwargs):
         self._loader = loader
+        self._templar = templar
         self._display = display
 
     def get_basedir(self, variables):
diff --git a/lib/ansible/plugins/lookup/cartesian.py b/lib/ansible/plugins/lookup/cartesian.py
index 7d8e08cb94d..9ae18587aeb 100644
--- a/lib/ansible/plugins/lookup/cartesian.py
+++ b/lib/ansible/plugins/lookup/cartesian.py
@@ -29,16 +29,16 @@ class LookupModule(LookupBase):
     [1, 2, 3], [a, b] -> [1, a], [1, b], [2, a], [2, b], [3, a], [3, b]
     """
 
-    def __lookup_variabless(self, terms, variables):
+    def __lookup_variables(self, terms):
         results = []
         for x in terms:
-            intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader)
+            intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader)
             results.append(intermediate)
         return results
 
     def run(self, terms, variables=None, **kwargs):
 
-        terms = self.__lookup_variabless(terms, variables)
+        terms = self.__lookup_variables(terms)
 
         my_list = terms[:]
         if len(my_list) == 0:
diff --git a/lib/ansible/plugins/lookup/first_found.py b/lib/ansible/plugins/lookup/first_found.py
index e9fe9a676a5..6e0aaee117a 100644
--- a/lib/ansible/plugins/lookup/first_found.py
+++ b/lib/ansible/plugins/lookup/first_found.py
@@ -125,7 +125,6 @@ from jinja2.exceptions import UndefinedError
 
 from ansible.errors import AnsibleLookupError, AnsibleUndefinedVariable
 from ansible.plugins.lookup import LookupBase
-from ansible.template import Templar
 from ansible.utils.boolean import boolean
 
 class LookupModule(LookupBase):
@@ -174,11 +173,10 @@ class LookupModule(LookupBase):
         else:
             total_search = terms
 
-        templar = Templar(loader=self._loader, variables=variables)
         roledir = variables.get('roledir')
         for fn in total_search:
             try:
-                fn = templar.template(fn)
+                fn = self._templar.template(fn)
             except (AnsibleUndefinedVariable, UndefinedError) as e:
                 continue
 
diff --git a/lib/ansible/plugins/lookup/flattened.py b/lib/ansible/plugins/lookup/flattened.py
index f0a8adaf5e6..7477db4b834 100644
--- a/lib/ansible/plugins/lookup/flattened.py
+++ b/lib/ansible/plugins/lookup/flattened.py
@@ -46,7 +46,7 @@ class LookupModule(LookupBase):
 
             if isinstance(term, basestring):
                 # convert a variable to a list
-                term2 = listify_lookup_plugin_terms(term, variables, loader=self._loader)
+                term2 = listify_lookup_plugin_terms(term, templar=self._templar, loader=self._loader)
                 # but avoid converting a plain string to a list of one string
                 if term2 != [ term ]:
                     term = term2
diff --git a/lib/ansible/plugins/lookup/nested.py b/lib/ansible/plugins/lookup/nested.py
index b4b4e5d9714..23938f6a19c 100644
--- a/lib/ansible/plugins/lookup/nested.py
+++ b/lib/ansible/plugins/lookup/nested.py
@@ -31,7 +31,7 @@ class LookupModule(LookupBase):
         results = []
         for x in terms:
             try:
-                intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader, fail_on_undefined=True)
+                intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader, fail_on_undefined=True)
             except UndefinedError, e:
                 raise AnsibleUndefinedVariable("One of the nested variables was undefined. The error was: %s" % e)
             results.append(intermediate)
diff --git a/lib/ansible/plugins/lookup/sequence.py b/lib/ansible/plugins/lookup/sequence.py
index 1e66626b68e..5cd87f4f524 100644
--- a/lib/ansible/plugins/lookup/sequence.py
+++ b/lib/ansible/plugins/lookup/sequence.py
@@ -22,7 +22,6 @@ from re import compile as re_compile, IGNORECASE
 from ansible.errors import *
 from ansible.parsing.splitter import parse_kv
 from ansible.plugins.lookup import LookupBase
-from ansible.template import Templar
 
 # shortcut format
 NUM = "(0?x?[0-9a-f]+)"
@@ -188,13 +187,11 @@ class LookupModule(LookupBase):
         if isinstance(terms, basestring):
             terms = [ terms ]
 
-        templar = Templar(loader=self._loader, variables=variables)
-
         for term in terms:
             try:
                 self.reset()  # clear out things for this iteration
 
-                term = templar.template(term)
+                term = self._templar.template(term)
                 try:
                     if not self.parse_simple_args(term):
                         self.parse_kv_args(parse_kv(term))
diff --git a/lib/ansible/plugins/lookup/subelements.py b/lib/ansible/plugins/lookup/subelements.py
index d8c2b1086e3..e014e382bab 100644
--- a/lib/ansible/plugins/lookup/subelements.py
+++ b/lib/ansible/plugins/lookup/subelements.py
@@ -33,8 +33,9 @@ class LookupModule(LookupBase):
             raise AnsibleError(
                 "subelements lookup expects a list of two or three items, "
                 + msg)
-        terms = listify_lookup_plugin_terms(terms, variables, loader=self._loader)
-        terms[0] = listify_lookup_plugin_terms(terms[0], variables, loader=self._loader)
+
+        terms = listify_lookup_plugin_terms(terms, templar=self._templar, loader=self._loader)
+        terms[0] = listify_lookup_plugin_terms(terms[0], templar=self._templar, loader=self._loader)
 
         # check lookup terms - check number of terms
         if not isinstance(terms, list) or not 2 <= len(terms) <= 3:
diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py
index e40247bbc3d..311904b280b 100644
--- a/lib/ansible/plugins/lookup/template.py
+++ b/lib/ansible/plugins/lookup/template.py
@@ -21,7 +21,6 @@ import os
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.template import Templar
 
 class LookupModule(LookupBase):
 
@@ -34,8 +33,6 @@ class LookupModule(LookupBase):
 
         ret = []
 
-        templar = Templar(loader=self._loader, variables=variables)
-
         for term in terms:
             self._display.debug("File lookup term: %s" % term)
 
@@ -44,7 +41,7 @@ class LookupModule(LookupBase):
             if lookupfile and os.path.exists(lookupfile):
                 with open(lookupfile, 'r') as f:
                     template_data = f.read()
-                    res = templar.template(template_data, preserve_trailing_newlines=True)
+                    res = self._templar.template(template_data, preserve_trailing_newlines=True)
                     ret.append(res)
             else:
                 raise AnsibleError("the template file %s could not be found for the lookup" % term)
diff --git a/lib/ansible/plugins/lookup/together.py b/lib/ansible/plugins/lookup/together.py
index 2f53121cc8b..42c98455070 100644
--- a/lib/ansible/plugins/lookup/together.py
+++ b/lib/ansible/plugins/lookup/together.py
@@ -31,16 +31,16 @@ class LookupModule(LookupBase):
     [1, 2], [3] -> [1, 3], [2, None]
     """
 
-    def __lookup_variabless(self, terms, variables):
+    def __lookup_variables(self, terms):
         results = []
         for x in terms:
-            intermediate = listify_lookup_plugin_terms(x, variables, loader=self._loader)
+            intermediate = listify_lookup_plugin_terms(x, templar=self._templar, loader=self._loader)
             results.append(intermediate)
         return results
 
     def run(self, terms, variables=None, **kwargs):
 
-        terms = self.__lookup_variabless(terms, variables)
+        terms = self.__lookup_variables(terms)
 
         my_list = terms[:]
         if len(my_list) == 0:
diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py
index 8d0a8730456..574359955db 100644
--- a/lib/ansible/template/__init__.py
+++ b/lib/ansible/template/__init__.py
@@ -65,8 +65,6 @@ class Templar:
             self._basedir = './'
 
         if shared_loader_obj:
-            global _basedirs
-            _basedirs = shared_loader_obj.basedirs[:]
             self._filter_loader = getattr(shared_loader_obj, 'filter_loader')
             self._lookup_loader = getattr(shared_loader_obj, 'lookup_loader')
         else:
@@ -250,7 +248,7 @@ class Templar:
         return thing if thing is not None else ''
 
     def _lookup(self, name, *args, **kwargs):
-        instance = self._lookup_loader.get(name.lower(), loader=self._loader)
+        instance = self._lookup_loader.get(name.lower(), loader=self._loader, templar=self)
 
         if instance is not None:
             # safely catch run failures per #5059
diff --git a/lib/ansible/utils/listify.py b/lib/ansible/utils/listify.py
index 95d48121339..237e131613f 100644
--- a/lib/ansible/utils/listify.py
+++ b/lib/ansible/utils/listify.py
@@ -26,19 +26,18 @@ from ansible.template.safe_eval import safe_eval
 __all__ = ['listify_lookup_plugin_terms']
 
 #FIXME: probably just move this into lookup plugin base class
-def listify_lookup_plugin_terms(terms, variables, loader, fail_on_undefined=False):
+def listify_lookup_plugin_terms(terms, templar, loader, fail_on_undefined=False):
 
     if isinstance(terms, basestring):
         stripped = terms.strip()
-        templar = Templar(loader=loader, variables=variables)
-
         #FIXME: warn/deprecation on bare vars in with_ so we can eventually remove fail on undefined override
         terms = templar.template(terms, convert_bare=True, fail_on_undefined=fail_on_undefined)
-
         #TODO: check if this is needed as template should also return correct type already
-        terms = safe_eval(terms)
+        #terms = safe_eval(terms)
+    else:
+        terms = templar.template(terms, fail_on_undefined=fail_on_undefined)
 
-        if isinstance(terms, basestring) or not isinstance(terms, Iterable):
-            terms = [ terms ]
+    if isinstance(terms, basestring) or not isinstance(terms, Iterable):
+        terms = [ terms ]
 
     return terms