Merge branch 'static_includes' into devel

examples/ansible.cfg

@@ -54,6 +54,13 @@
 # enable additional callbacks
 #callback_whitelist = timer, mail
 
+# Determine whether includes in tasks and handlers are "static" by
+# default. As of 2.0, includes are dynamic by default. Setting these
+# values to True will make includes behave more like they did in the
+# 1.x versions.
+#task_includes_static = True
+#handler_includes_static = True
+
 # change this for alternative sudo implementations
 #sudo_exe = sudo
 

lib/ansible/cli/playbook.py

@@ -30,6 +30,7 @@ from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.playbook_executor import PlaybookExecutor
 from ansible.inventory import Inventory
 from ansible.parsing.dataloader import DataLoader
+from ansible.playbook.block import Block
 from ansible.playbook.play_context import PlayContext
 from ansible.utils.vars import load_extra_vars
 from ansible.vars import VariableManager
@@ -172,26 +173,34 @@ class PlaybookCLI(CLI):
                         if self.options.listtasks:
                             taskmsg = '    tasks:\n'
 
+                        def _process_block(b):
+                            taskmsg = ''
+                            for task in b.block:
+                                if isinstance(task, Block):
+                                    taskmsg += _process_block(task)
+                                else:
+                                    if task.action == 'meta':
+                                        continue
+
+                                    all_tags.update(task.tags)
+                                    if self.options.listtasks:
+                                        cur_tags = list(mytags.union(set(task.tags)))
+                                        cur_tags.sort()
+                                        if task.name:
+                                            taskmsg += "      %s" % task.get_name()
+                                        else:
+                                            taskmsg += "      %s" % task.action
+                                        taskmsg += "\tTAGS: [%s]\n" % ', '.join(cur_tags)
+
+                            return taskmsg
+
                         all_vars = variable_manager.get_vars(loader=loader, play=play)
                         play_context = PlayContext(play=play, options=self.options)
                         for block in play.compile():
                             block = block.filter_tagged_tasks(play_context, all_vars)
                             if not block.has_tasks():
                                 continue
-
+                            taskmsg += _process_block(block)
-                            for task in block.block:
-                                if task.action == 'meta':
-                                    continue
-
-                                all_tags.update(task.tags)
-                                if self.options.listtasks:
-                                    cur_tags = list(mytags.union(set(task.tags)))
-                                    cur_tags.sort()
-                                    if task.name:
-                                        taskmsg += "      %s" % task.get_name()
-                                    else:
-                                        taskmsg += "      %s" % task.action
-                                    taskmsg += "\tTAGS: [%s]\n" % ', '.join(cur_tags)
 
                         if self.options.listtags:
                             cur_tags = list(mytags.union(all_tags))

lib/ansible/constants.py

@@ -162,6 +162,10 @@ DEFAULT_FORCE_HANDLERS    = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_F
 DEFAULT_INVENTORY_IGNORE  = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True)
 DEFAULT_VAR_COMPRESSION_LEVEL = get_config(p, DEFAULTS, 'var_compression_level', 'ANSIBLE_VAR_COMPRESSION_LEVEL', 0, integer=True)
 
+# static includes
+DEFAULT_TASK_INCLUDES_STATIC    = get_config(p, DEFAULTS, 'task_includes_static', 'ANSIBLE_TASK_INCLUDES_STATIC', False, boolean=True)
+DEFAULT_HANDLER_INCLUDES_STATIC = get_config(p, DEFAULTS, 'handler_includes_static', 'ANSIBLE_HANDLER_INCLUDES_STATIC', False, boolean=True)
+
 # disclosure
 DEFAULT_NO_LOG           = get_config(p, DEFAULTS, 'no_log', 'ANSIBLE_NO_LOG', False, boolean=True)
 DEFAULT_NO_TARGET_SYSLOG   = get_config(p, DEFAULTS, 'no_target_syslog', 'ANSIBLE_NO_TARGET_SYSLOG', False, boolean=True)

lib/ansible/errors/__init__.py

@@ -44,7 +44,7 @@ class AnsibleError(Exception):
     which should be returned by the DataLoader() class.
     '''
 
-    def __init__(self, message="", obj=None, show_content=True):
+    def __init__(self, message="", obj=None, show_content=True, suppress_extended_error=False):
         # we import this here to prevent an import loop problem,
         # since the objects code also imports ansible.errors
         from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
@@ -53,8 +53,10 @@ class AnsibleError(Exception):
         self._show_content = show_content
         if obj and isinstance(obj, AnsibleBaseYAMLObject):
             extended_error = self._get_extended_error()
-            if extended_error:
+            if extended_error and not suppress_extended_error:
                 self.message = '%s\n\n%s' % (to_str(message), to_str(extended_error))
+            else:
+                self.message = '%s' % to_str(message)
         else:
             self.message = '%s' % to_str(message)
 

lib/ansible/executor/task_queue_manager.py

@@ -28,6 +28,7 @@ from ansible.errors import AnsibleError
 from ansible.executor.play_iterator import PlayIterator
 from ansible.executor.process.result import ResultProcess
 from ansible.executor.stats import AggregateStats
+from ansible.playbook.block import Block
 from ansible.playbook.play_context import PlayContext
 from ansible.plugins import callback_loader, strategy_loader, module_loader
 from ansible.template import Templar
@@ -118,11 +119,18 @@ class TaskQueueManager:
         for key in self._notified_handlers.keys():
             del self._notified_handlers[key]
 
-        # FIXME: there is a block compile helper for this...
+        def _process_block(b):
+            temp_list = []
+            for t in b.block:
+                if isinstance(t, Block):
+                    temp_list.extend(_process_block(t))
+                else:
+                    temp_list.append(t)
+            return temp_list
+
         handler_list = []
         for handler_block in handlers:
-            for handler in handler_block.block:
+            handler_list.extend(_process_block(handler_block))
-                handler_list.append(handler)
 
         # then initialize it with the handler names from the handler list
         for handler in handler_list:

lib/ansible/playbook/helpers.py

@@ -20,9 +20,17 @@ __metaclass__ = type
 
 import os
 
-from ansible.errors import AnsibleParserError
+from ansible import constants as C
+from ansible.compat.six import string_types
+from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleSequence
 
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
 
 def load_list_of_blocks(ds, play, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None):
     '''
@@ -72,16 +80,18 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h
     from ansible.playbook.block import Block
     from ansible.playbook.handler import Handler
     from ansible.playbook.task import Task
+    from ansible.playbook.task_include import TaskInclude
+    from ansible.template import Templar
 
     assert isinstance(ds, list)
 
     task_list = []
-    for task in ds:
+    for task_ds in ds:
-        assert isinstance(task, dict)
+        assert isinstance(task_ds, dict)
 
-        if 'block' in task:
+        if 'block' in task_ds:
             t = Block.load(
-                task,
+                task_ds,
                 play=play,
                 parent_block=block,
                 role=role,
@@ -90,13 +100,133 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h
                 variable_manager=variable_manager,
                 loader=loader,
             )
+            task_list.append(t)
         else:
-            if use_handlers:
+            if 'include' in task_ds:
-                t = Handler.load(task, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
+                t = TaskInclude.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
+
+                all_vars = variable_manager.get_vars(loader=loader, play=play, task=t)
+                templar = Templar(loader=loader, variables=all_vars)
+
+                # check to see if this include is static, which can be true if:
+                # 1. the user set the 'static' option to true
+                # 2. one of the appropriate config options was set
+                # 3. the included file name contains no variables, and has no loop
+                is_static = t.static or \
+                            C.DEFAULT_TASK_INCLUDES_STATIC or \
+                            (use_handlers and C.DEFAULT_HANDLER_INCLUDES_STATIC) or \
+                            not templar._contains_vars(t.args.get('_raw_params')) and t.loop is None
+
+                if is_static:
+                    if t.loop is not None:
+                        raise AnsibleParserError("You cannot use 'static' on an include with a loop", obj=task_ds)
+
+                    # FIXME: all of this code is very similar (if not identical) to that in 
+                    #        plugins/strategy/__init__.py, and should be unified to avoid
+                    #        patches only being applied to one or the other location
+                    if task_include:
+                        # handle relative includes by walking up the list of parent include
+                        # tasks and checking the relative result to see if it exists
+                        parent_include = task_include
+                        cumulative_path = None
+                        while parent_include is not None:
+                            parent_include_dir = templar.template(os.path.dirname(parent_include.args.get('_raw_params')))
+                            if cumulative_path is None:
+                                cumulative_path = parent_include_dir
+                            elif not os.path.isabs(cumulative_path):
+                                cumulative_path = os.path.join(parent_include_dir, cumulative_path)
+                            include_target = templar.template(t.args['_raw_params'])
+                            if t._role:
+                                new_basedir = os.path.join(t._role._role_path, 'tasks', cumulative_path)
+                                include_file = loader.path_dwim_relative(new_basedir, 'tasks', include_target)
+                            else:
+                                include_file = loader.path_dwim_relative(loader.get_basedir(), cumulative_path, include_target)
+
+                            if os.path.exists(include_file):
+                                break
+                            else:
+                                parent_include = parent_include._task_include
+                    else:
+                        try:
+                            include_target = templar.template(t.args['_raw_params'])
+                        except AnsibleUndefinedVariable as e:
+                            raise AnsibleParserError(
+                                      "Error when evaluating variable in include name: %s.\n\n" \
+                                      "When using static includes, ensure that any variables used in their names are defined in vars/vars_files\n" \
+                                      "or extra-vars passed in from the command line. Static includes cannot use variables from inventory\n" \
+                                      "sources like group or host vars." % t.args['_raw_params'],
+                                      obj=task_ds,
+                                      suppress_extended_error=True,
+                                  )
+                        if t._role:
+                            if use_handlers:
+                                include_file = loader.path_dwim_relative(t._role._role_path, 'handlers', include_target)
+                            else:
+                                include_file = loader.path_dwim_relative(t._role._role_path, 'tasks', include_target)
+                        else:
+                            include_file = loader.path_dwim(include_target)
+
+                    data = loader.load_from_file(include_file)
+                    if data is None:
+                        return []
+                    elif not isinstance(data, list):
+                        raise AnsibleError("included task files must contain a list of tasks", obj=data)
+
+                    included_blocks = load_list_of_blocks(
+                        data,
+                        play=play,
+                        parent_block=block,
+                        task_include=t,
+                        role=role,
+                        use_handlers=use_handlers,
+                        loader=loader,
+                        variable_manager=variable_manager,
+                    )
+
+                    # Remove the raw params field from the module args, so it won't show up
+                    # later when getting the vars for this task/childen
+                    t.args.pop('_raw_params', None)
+
+                    # pop tags out of the include args, if they were specified there, and assign
+                    # them to the include. If the include already had tags specified, we raise an
+                    # error so that users know not to specify them both ways
+                    tags = t.vars.pop('tags', [])
+                    if isinstance(tags, string_types):
+                        tags = tags.split(',')
+
+                    if len(tags) > 0:
+                        if len(t.tags) > 0:
+                            raise AnsibleParserError(
+                                "Include tasks should not specify tags in more than one way (both via args and directly on the task)." \
+                                " Mixing tag specify styles is prohibited for whole import hierarchy, not only for single import statement",
+                                obj=task_ds,
+                                suppress_extended_error=True,
+                            )
+                        display.deprecated("You should not specify tags in the include parameters. All tags should be specified using the task-level option")
+                    else:
+                        tags = t.tags[:]
+
+                    # now we extend the tags on each of the included blocks
+                    for b in included_blocks:
+                        b.tags = list(set(b.tags).union(tags))
+                    # END FIXME
+
+                    # FIXME: send callback here somehow...
+                    # FIXME: handlers shouldn't need this special handling, but do
+                    #        right now because they don't iterate blocks correctly
+                    if use_handlers:
+                        for b in included_blocks:
+                            task_list.extend(b.block)
+                    else:
+                        task_list.extend(included_blocks)
+                else:
+                    task_list.append(t)
+            elif use_handlers:
+                t = Handler.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
+                task_list.append(t)
             else:
-                t = Task.load(task, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
+                t = Task.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)
-
+                task_list.append(t)
-        task_list.append(t)
 
     return task_list
 

lib/ansible/playbook/included_file.py

@@ -84,6 +84,9 @@ class IncludedFile:
                         task_vars['item'] = include_variables['item'] = include_result['item']
 
                     if original_task:
+                        if original_task.static:
+                            continue
+
                         if original_task._task_include:
                             # handle relative includes by walking up the list of parent include
                             # tasks and checking the relative result to see if it exists

lib/ansible/playbook/role/__init__.py

@@ -176,16 +176,16 @@ class Role(Base, Become, Conditional, Taggable):
         task_data = self._load_role_yaml('tasks')
         if task_data:
             try:
-                self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader)
+                self._task_blocks = load_list_of_blocks(task_data, play=self._play, role=self, loader=self._loader, variable_manager=self._variable_manager)
             except AssertionError:
                 raise AnsibleParserError("The tasks/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=task_data)
 
         handler_data = self._load_role_yaml('handlers')
         if handler_data:
             try:
-                self._handler_blocks = load_list_of_blocks(handler_data, play=self._play, role=self, use_handlers=True, loader=self._loader)
+                self._handler_blocks = load_list_of_blocks(handler_data, play=self._play, role=self, use_handlers=True, loader=self._loader, variable_manager=self._variable_manager)
-            except:
+            except AssertionError:
-                raise AnsibleParserError("The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=task_data)
+                raise AnsibleParserError("The handlers/main.yml file for role '%s' must contain a list of tasks" % self._role_name , obj=handler_data)
 
         # vars and default vars are regular dictionaries
         self._role_vars  = self._load_role_yaml('vars')

lib/ansible/playbook/task_include.py

@@ -0,0 +1,72 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.playbook.attribute import FieldAttribute
+from ansible.playbook.task import Task
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+__all__ = ['TaskInclude']
+
+
+class TaskInclude(Task):
+
+    """
+    A task include is derived from a regular task to handle the special
+    circumstances related to the `- include: ...` task.
+    """
+
+    # =================================================================================
+    # ATTRIBUTES
+
+    _static = FieldAttribute(isa='bool', default=False)
+
+    @staticmethod
+    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
+        t = TaskInclude(block=block, role=role, task_include=task_include)
+        return t.load_data(data, variable_manager=variable_manager, loader=loader)
+
+    def get_vars(self):
+        '''
+        We override the parent Task() classes get_vars here because
+        we need to include the args of the include into the vars as
+        they are params to the included tasks.
+        '''
+        all_vars = dict()
+        if self._block:
+            all_vars.update(self._block.get_vars())
+        if self._task_include:
+            all_vars.update(self._task_include.get_vars())
+
+        all_vars.update(self.vars)
+        all_vars.update(self.args)
+
+        if 'tags' in all_vars:
+            del all_vars['tags']
+        if 'when' in all_vars:
+            del all_vars['when']
+
+        return all_vars
+