Starting to move over lookups and fixing some bugs related to that

v2/ansible/executor/play_iterator.py

@@ -230,7 +230,7 @@ class PlayIterator:
         #        as args to __init__
         all_vars = inventory._variable_manager.get_vars(loader=inventory._loader, play=play)
         new_play = play.copy()
-        new_play.post_validate(all_vars, ignore_undefined=True)
+        new_play.post_validate(all_vars, fail_on_undefined=False)
 
         for host in inventory.get_hosts(new_play.hosts):
             if self._first_host is None:

v2/ansible/executor/playbook_executor.py

@@ -65,7 +65,7 @@ class PlaybookExecutor:
                     # on it without the templating changes affecting the original object.
                     all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
                     new_play = play.copy()
-                    new_play.post_validate(all_vars, ignore_undefined=True)
+                    new_play.post_validate(all_vars, fail_on_undefined=False)
 
                     result = True
                     for batch in self._get_serialized_batches(new_play):

v2/ansible/executor/task_executor.py

@@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from ansible import constants as C
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.executor.connection_info import ConnectionInformation
 from ansible.playbook.task import Task
 from ansible.plugins import lookup_loader, connection_loader, action_loader
@@ -55,6 +55,8 @@ class TaskExecutor:
         '''
 
         debug("in run()")
+
+        try:
             items = self._get_loop_items()
             if items:
                 if len(items) > 0:
@@ -71,6 +73,8 @@ class TaskExecutor:
             result = json.dumps(res)
             debug("done dumping result, returning")
             return result
+        except AnsibleError, e:
+            return dict(failed=True, msg=str(e))
 
     def _get_loop_items(self):
         '''
@@ -80,7 +84,7 @@ class TaskExecutor:
 
         items = None
         if self._task.loop and self._task.loop in lookup_loader:
-            items = lookup_loader.get(self._task.loop).run(self._task.loop_args)
+            items = lookup_loader.get(self._task.loop).run(terms=self._task.loop_args, variables=self._job_vars)
 
         return items
 
@@ -98,7 +102,28 @@ class TaskExecutor:
         #        than it is today?
 
         for item in items:
+            # make copies of the job vars and task so we can add the item to
+            # the variables and re-validate the task with the item variable
+            task_vars = self._job_vars.copy()
+            task_vars['item'] = item
+
+            try:
+                tmp_task = self._task.copy()
+                tmp_task.post_validate(task_vars)
+            except AnsibleParserError, e:
+                results.append(dict(failed=True, msg=str(e)))
+                continue
+
+            # now we swap the internal task with the re-validate copy, execute,
+            # and swap them back so we can do the next iteration cleanly
+            (self._task, tmp_task) = (tmp_task, self._task)
             res = self._execute()
+            (self._task, tmp_task) = (tmp_task, self._task)
+
+            # FIXME: we should be sending back a callback result for each item in the loop here
+
+            # now update the result with the item info, and append the result
+            # to the list of results
             res['item'] = item
             results.append(res)
 

v2/ansible/playbook/base.py

@@ -167,7 +167,7 @@ class Base:
 
         return new_me
 
-    def post_validate(self, all_vars=dict(), ignore_undefined=False):
+    def post_validate(self, all_vars=dict(), fail_on_undefined=True):
         '''
         we can't tell that everything is of the right type until we have
         all the variables.  Run basic types (from isa) as well as
@@ -178,7 +178,7 @@ class Base:
         if self._loader is not None:
             basedir = self._loader.get_basedir()
 
-        templar = Templar(basedir=basedir, variables=all_vars)
+        templar = Templar(basedir=basedir, variables=all_vars, fail_on_undefined=fail_on_undefined)
 
         for (name, attribute) in iteritems(self._get_base_attributes()):
 
@@ -195,7 +195,7 @@ class Base:
                 # run the post-validator if present
                 method = getattr(self, '_post_validate_%s' % name, None)
                 if method:
-                    method(self, attribute, value)
+                    value = method(attribute, value, all_vars, fail_on_undefined)
                 else:
                     # otherwise, just make sure the attribute is of the type it should be
                     if attribute.isa == 'string':
@@ -217,7 +217,7 @@ class Base:
                 #raise AnsibleParserError("the field '%s' has an invalid value, and could not be converted to an %s" % (name, attribute.isa), obj=self.get_ds())
                 raise AnsibleParserError("the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e))
             except UndefinedError:
-                if not ignore_undefined:
+                if fail_on_undefined:
                     raise AnsibleParserError("the field '%s' has an invalid value, which appears to include a variable that is undefined" % (name,))
 
     def serialize(self):

v2/ansible/playbook/task.py

@@ -34,6 +34,8 @@ from ansible.playbook.conditional import Conditional
 from ansible.playbook.role import Role
 from ansible.playbook.taggable import Taggable
 
+from ansible.utils.listify import listify_lookup_plugin_terms
+
 class Task(Base, Conditional, Taggable):
 
     """
@@ -186,18 +188,19 @@ class Task(Base, Conditional, Taggable):
 
         return new_ds
 
-    def post_validate(self, all_vars=dict(), ignore_undefined=False):
+    def post_validate(self, all_vars=dict(), fail_on_undefined=True):
         '''
         Override of base class post_validate, to also do final validation on
         the block to which this task belongs.
         '''
 
         if self._block:
-            self._block.post_validate(all_vars=all_vars, ignore_undefined=ignore_undefined)
-        #if self._role:
-        #    self._role.post_validate(all_vars=all_vars, ignore_undefined=ignore_undefined)
+            self._block.post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined)
+
+        super(Task, self).post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined)
 
-        super(Task, self).post_validate(all_vars=all_vars, ignore_undefined=ignore_undefined)
+    def _post_validate_loop_args(self, attr, value, all_vars, fail_on_undefined):
+        return listify_lookup_plugin_terms(value, all_vars)
 
     def get_vars(self):
         return self.serialize()

v2/ansible/plugins/action/synchronize.py

@@ -140,9 +140,7 @@ class ActionModule(ActionBase):
             else:
                 private_key = task_vars.get('ansible_ssh_private_key_file', self.runner.private_key_file)
 
-            private_key = template.template(self.runner.basedir, private_key, task_vars, fail_on_undefined=True)
-
-            if not private_key is None:
+            if private_key is not None:
                 private_key = os.path.expanduser(private_key)
                 
             # use the mode to define src and dest's url
@@ -172,7 +170,7 @@ class ActionModule(ActionBase):
         #    module_args = "CHECKMODE=True"
 
         # run the module and store the result
-        result = self.runner._execute_module('synchronize', tmp=tmpmodule_args, complex_args=options, task_vars=task_vars)
+        result = self.runner._execute_module('synchronize', module_args=, complex_args=options, task_vars=task_vars)
 
         return result
 

v2/ansible/plugins/lookup/__init__.py

@@ -19,3 +19,25 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+__all__ = ['LookupBase']
+
+class LookupBase:
+    def __init__(self, **kwargs):
+        pass
+
+    def _flatten(self, terms):
+        ret = []
+        for term in terms:
+            if isinstance(term, (list, tuple)):
+                ret.extend(term)
+            else:
+                ret.append(term)
+        return ret
+
+    def _combine(self, a, b):
+        results = []
+        for x in a:
+            for y in b:
+                results.append(self._flatten([x,y]))
+        return results
+

v2/ansible/plugins/lookup/items.py

@@ -15,30 +15,10 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-#from ansible.utils import safe_eval
-#import ansible.utils as utils
-#import ansible.errors as errors
+from ansible.plugins.lookup import LookupBase
 
-def flatten(terms):
-    ret = []
-    for term in terms:
-        if isinstance(term, list):
-            ret.extend(term)
-        else:
-            ret.append(term)
-    return ret
-
-class LookupModule(object):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, inject=None, **kwargs):
-        # FIXME: this function needs to be ported still, or something like it
-        #        where really the intention is just to template a bare variable
-        #        with the result being a list of terms
-        #terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
-
-        return flatten(terms)
+class LookupModule(LookupBase):
 
+    def run(self, terms, **kwargs):
+        return self._flatten(terms)
 

v2/ansible/plugins/lookup/nested.py

@@ -0,0 +1,49 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.errors import AnsibleError
+from ansible.plugins.lookup import LookupBase
+from ansible.utils.listify import listify_lookup_plugin_terms
+
+class LookupModule(LookupBase):
+
+    def __lookup_variabless(self, terms, variables):
+        results = []
+        for x in terms:
+            intermediate = listify_lookup_plugin_terms(x, variables)
+            results.append(intermediate)
+        return results
+
+    def run(self, terms, variables=None, **kwargs):
+
+        terms = self.__lookup_variabless(terms, variables)
+
+        my_list = terms[:]
+        my_list.reverse()
+        result = []
+        if len(my_list) == 0:
+            raise AnsibleError("with_nested requires at least one element in the nested list")
+        result = my_list.pop()
+        while len(my_list) > 0:
+            result2 = self._combine(result, my_list.pop())
+            result  = result2
+        new_result = []
+        for x in result:
+            new_result.append(self._flatten(x))
+        return new_result
+
+

v2/ansible/plugins/strategies/__init__.py

@@ -96,6 +96,12 @@ class StrategyBase:
         debug("done getting variables")
 
         debug("running post_validate() on the task")
+        if new_task.loop:
+            # if the task has a lookup loop specified, we do not error out
+            # on undefined variables yet, as fields may use {{item}} or some
+            # variant, which won't be defined until execution time
+            new_task.post_validate(task_vars, fail_on_undefined=False)
+        else:
             new_task.post_validate(task_vars)
         debug("done running post_validate() on the task")
 

v2/ansible/template/__init__.py

@@ -41,7 +41,7 @@ class Templar:
     The main class for templating, with the main entry-point of template().
     '''
 
-    def __init__(self, basedir=None, variables=dict()):
+    def __init__(self, basedir=None, variables=dict(), fail_on_undefined=C.DEFAULT_UNDEFINED_VAR_BEHAVIOR):
         self._basedir             = basedir
         self._filters             = None
         self._available_variables = variables
@@ -50,7 +50,7 @@ class Templar:
         # should result in fatal errors being raised
         self._fail_on_lookup_errors    = True
         self._fail_on_filter_errors    = True
-        self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR
+        self._fail_on_undefined_errors = fail_on_undefined
 
     def _count_newlines_from_end(self, in_str):
         '''

v2/ansible/utils/listify.py

@@ -0,0 +1,68 @@
+# (c) 2014 Michael DeHaan, <michael@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from six import iteritems, string_types
+
+import re
+
+from ansible.template import Templar
+from ansible.template.safe_eval import safe_eval
+
+__all__ = ['listify_lookup_plugin_terms']
+
+LOOKUP_REGEX = re.compile(r'lookup\s*\(')
+
+def listify_lookup_plugin_terms(terms, variables):
+
+    if isinstance(terms, basestring):
+        # someone did:
+        #    with_items: alist
+        # OR
+        #    with_items: {{ alist }}
+
+        stripped = terms.strip()
+        if not (stripped.startswith('{') or stripped.startswith('[')) and \
+           not stripped.startswith("/") and \
+           not stripped.startswith('set([') and \
+           not LOOKUP_REGEX.search(terms):
+            # if not already a list, get ready to evaluate with Jinja2
+            # not sure why the "/" is in above code :)
+            try:
+                templar = Templar(variables=variables)
+                new_terms = templar.template("{{ %s }}" % terms)
+                if isinstance(new_terms, basestring) and "{{" in new_terms:
+                    pass
+                else:
+                    terms = new_terms
+            except:
+                pass
+
+        if '{' in terms or '[' in terms:
+            # Jinja2 already evaluated a variable to a list.
+            # Jinja2-ified list needs to be converted back to a real type
+            # TODO: something a bit less heavy than eval
+            return safe_eval(terms)
+
+        if isinstance(terms, basestring):
+            terms = [ terms ]
+
+    return terms
+

v2/samples/with_items.yml

@@ -0,0 +1,11 @@
+- hosts: localhost
+  connection: local
+  vars:
+    my_list:
+    - a
+    - b
+    - c
+  gather_facts: no
+  tasks:
+  - debug: msg="item is {{item}}"
+    with_items: my_list

v2/samples/with_nested.yml

@@ -0,0 +1,13 @@
+- hosts: localhost
+  connection: local
+  gather_facts: no
+  vars:
+    users:
+    - foo
+    - bar
+    - bam
+  tasks:
+  - debug: msg="item.0={{ item[0] }} item.1={{ item[1] }}"
+    with_nested:
+    - users
+    - [ 'clientdb', 'employeedb', 'providerdb' ]