Dynamic role include (#17401)

* dynamic role_include

* more fixes for dynamic include roles

* set play yfrom iterator when dynamic

* changes from jimi-c

* avoid modules that break ad hoc

TODO: should really be a config

lib/ansible/cli/adhoc.py

@@ -155,6 +155,10 @@ class AdHocCLI(CLI):
                 err = err + ' (did you mean to run ansible-playbook?)'
             raise AnsibleOptionsError(err)
 
+        # Avoid modules that don't work with ad-hoc
+        if self.options.module_name in ('include', 'include_role'):
+            raise AnsibleOptionsError("'%s' is not a valid action for ad-hoc commands" % self.options.module_name)
+
         # dynamically load any plugins from the playbook directory
         for name, obj in get_all_plugin_loaders():
             if obj.subdir:

lib/ansible/executor/task_executor.py

@@ -403,7 +403,7 @@ class TaskExecutor:
                 return dict(changed=False, skipped=True, skip_reason='Conditional check failed', _ansible_no_log=self._play_context.no_log)
         except AnsibleError:
             # skip conditional exception in the case of includes as the vars needed might not be avaiable except in the included tasks or due to tags
-            if self._task.action != 'include':
+            if self._task.action in ['include', 'include_role']:
                 raise
 
         # if we ran into an error while setting up the PlayContext, raise it now
@@ -425,10 +425,10 @@ class TaskExecutor:
         # if this task is a IncludeRole, we just return now with a success code so the main thread can expand the task list for the given host
         elif self._task.action == 'include_role':
             include_variables = self._task.args.copy()
-            role = include_variables.pop('name')
+            role = templar.template(self._task._role_name)
             if not role:
                 return dict(failed=True, msg="No role was specified to include")
-            return dict(name=role, include_variables=include_variables)
+            return dict(include_role=role, include_variables=include_variables)
 
         # Now we do final validation on the task, which sets all fields to their final values.
         self._task.post_validate(templar=templar)

lib/ansible/playbook/base.py

@@ -108,13 +108,10 @@ class BaseMeta(type):
                     # its value from a parent object
                     method = "_get_attr_%s" % attr_name
                     if method in src_dict or method in dst_dict:
-                        #print("^ assigning generic_g_method to %s" % attr_name)
                         getter = partial(_generic_g_method, attr_name)
                     elif '_get_parent_attribute' in dst_dict and value.inherit:
-                        #print("^ assigning generic_g_parent to %s" % attr_name)
                         getter = partial(_generic_g_parent, attr_name)
                     else:
-                        #print("^ assigning generic_g to %s" % attr_name)
                         getter = partial(_generic_g, attr_name)
 
                     setter = partial(_generic_s, attr_name)
@@ -140,7 +137,6 @@ class BaseMeta(type):
 
         # now create the attributes based on the FieldAttributes
         # available, including from parent (and grandparent) objects
-        #print("creating class %s" % name)
         _create_attrs(dct, dct)
         _process_parents(parents, dct)
 
@@ -201,7 +197,6 @@ class Base(with_metaclass(BaseMeta, object)):
         if hasattr(self, '_parent') and self._parent:
             self._parent.dump_me(depth+2)
             dep_chain = self._parent.get_dep_chain()
-            #print("%s^ dep chain: %s" % (" "*(depth+2), dep_chain))
             if dep_chain:
                 for dep in dep_chain:
                     dep.dump_me(depth+2)

lib/ansible/playbook/helpers.py

@@ -22,7 +22,7 @@ import os
 
 from ansible import constants as C
 from ansible.compat.six import string_types
-from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound, AnsibleError
+from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound
 
 try:
     from __main__ import display
@@ -260,16 +260,41 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h
                     task_list.append(t)
 
             elif 'include_role' in task_ds:
-                task_list.extend(
-                    IncludeRole.load(
-                        task_ds,
-                        block=block,
-                        role=role,
-                        task_include=None,
-                        variable_manager=variable_manager,
-                        loader=loader
-                    )
-                )
+
+                ir = IncludeRole.load(
+                            task_ds,
+                            block=block,
+                            role=role,
+                            task_include=None,
+                            variable_manager=variable_manager,
+                            loader=loader
+                     )
+
+                #   1. the user has set the 'static' option to false or true
+                #   2. one of the appropriate config options was set
+                if ir.static is not None:
+                    is_static = ir.static
+                else:
+                    display.debug('Determine if include_role is static')
+                    # Check to see if this include is dynamic or static:
+                    all_vars = variable_manager.get_vars(loader=loader, play=play, task=ir)
+                    templar = Templar(loader=loader, variables=all_vars)
+                    needs_templating = False
+                    for param in ir.args:
+                        if templar._contains_vars(ir.args[param]):
+                            if not templar.templatable(ir.args[param]):
+                                needs_templating = True
+                                break
+                    is_static = C.DEFAULT_TASK_INCLUDES_STATIC or \
+                                (use_handlers and C.DEFAULT_HANDLER_INCLUDES_STATIC)  or \
+                                (not needs_templating and ir.all_parents_static() and not ir.loop)
+                    display.debug('Determined that if include_role static is %s' % str(is_static))
+                if is_static:
+                    # uses compiled list from object
+                    t = task_list.extend(ir.get_block_list(variable_manager=variable_manager, loader=loader))
+                else:
+                    # passes task object itself for latter generation of list
+                    t = task_list.append(ir)
             else:
                 if use_handlers:
                     t = Handler.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader)

lib/ansible/playbook/role_include.py

@@ -45,40 +45,73 @@ class IncludeRole(Task):
     # =================================================================================
     # ATTRIBUTES
 
-    _name   = FieldAttribute(isa='string', default=None)
-    _tasks_from = FieldAttribute(isa='string', default=None)
+    # private as this is a 'module options' vs a task property
+    _static = FieldAttribute(isa='bool', default=None, private=True)
+    _private = FieldAttribute(isa='bool', default=None, private=True)
 
-    # these should not be changeable?
-    _static = FieldAttribute(isa='bool', default=False)
-    _private = FieldAttribute(isa='bool', default=True)
+    def __init__(self, block=None, role=None, task_include=None):
 
-    @staticmethod
-    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
+        super(IncludeRole, self).__init__(block=block, role=role, task_include=task_include)
 
-        r = IncludeRole().load_data(data, variable_manager=variable_manager, loader=loader)
-        args = r.preprocess_data(data).get('args', dict())
+        self._role_name = None
+        self.statically_loaded = False
+        self._from_files = {}
+        self._parent_role = role
 
-        ri = RoleInclude.load(args.get('name'), play=block._play, variable_manager=variable_manager, loader=loader)
-        ri.vars.update(r.vars)
 
-        # build options for roles
-        from_files = {}
-        for key in ['tasks', 'vars', 'defaults']:
-            from_key = key + '_from'
-            if  args.get(from_key):
-                from_files[key] = basename(args.get(from_key))
+    def get_block_list(self, play=None, variable_manager=None, loader=None):
+
+        # only need play passed in when dynamic
+        if play is None:
+            myplay =  self._parent._play
+        else:
+            myplay = play
+
+        ri = RoleInclude.load(self._role_name, play=myplay, variable_manager=variable_manager, loader=loader)
+        ri.vars.update(self.vars)
+        #ri._role_params.update(self.args) # jimi-c cant we avoid this?
 
         #build role
-        actual_role = Role.load(ri, block._play, parent_role=role, from_files=from_files)
+        actual_role = Role.load(ri, myplay, parent_role=self._parent_role, from_files=self._from_files)
 
         # compile role
-        blocks = actual_role.compile(play=block._play)
+        blocks = actual_role.compile(play=myplay)
 
         # set parent to ensure proper inheritance
         for b in blocks:
-            b._parent = block
+            b._parent = self._parent
 
         # updated available handlers in play
-        block._play.handlers = block._play.handlers + actual_role.get_handler_blocks(play=block._play)
+        myplay.handlers = myplay.handlers + actual_role.get_handler_blocks(play=myplay)
 
         return blocks
+
+    @staticmethod
+    def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None):
+
+        ir = IncludeRole(block, role, task_include=task_include).load_data(data, variable_manager=variable_manager, loader=loader)
+
+        #TODO: use more automated list: for builtin in r.get_attributes(): #jimi-c: doing this to avoid using role_params and conflating include_role specific opts with other tasks
+        # set built in's
+        ir._role_name =  ir.args.get('name')
+        for builtin in ['static', 'private']:
+            if ir.args.get(builtin):
+                setattr(ir, builtin, ir.args.get(builtin))
+
+        # build options for roles
+        for key in ['tasks', 'vars', 'defaults']:
+            from_key = key + '_from'
+            if  ir.args.get(from_key):
+                ir._from_files[key] = basename(ir.args.get(from_key))
+
+        return ir.load_data(data, variable_manager=variable_manager, loader=loader)
+
+    def copy(self, exclude_parent=False, exclude_tasks=False):
+
+        new_me = super(IncludeRole, self).copy(exclude_parent=exclude_parent, exclude_tasks=exclude_tasks)
+        new_me.statically_loaded = self.statically_loaded
+        new_me._role_name = self._role_name
+        new_me._from_files = self._from_files.copy()
+        new_me._parent_role = self._parent_role
+
+        return new_me

lib/ansible/plugins/callback/default.py

@@ -62,7 +62,7 @@ class CallbackModule(CallbackBase):
 
         self._clean_results(result._result, result._task.action)
         delegated_vars = result._result.get('_ansible_delegated_vars', None)
-        if result._task.action == 'include':
+        if result._task.action in ('include', 'include_role'):
             return
         elif result._result.get('changed', False):
             if delegated_vars:
@@ -158,7 +158,7 @@ class CallbackModule(CallbackBase):
 
     def v2_runner_item_on_ok(self, result):
         delegated_vars = result._result.get('_ansible_delegated_vars', None)
-        if result._task.action == 'include':
+        if result._task.action in ('include', 'include_role'):
             return
         elif result._result.get('changed', False):
             msg = 'changed'

lib/ansible/plugins/strategy/__init__.py

@@ -376,7 +376,7 @@ class StrategyBase:
                         if self._diff:
                             self._tqm.send_callback('v2_on_file_diff', task_result)
 
-                    if original_task.action != 'include':
+                    if original_task.action in ['include', 'include_role']:
                         self._tqm._stats.increment('ok', original_host.name)
                         if 'changed' in task_result._result and task_result._result['changed']:
                             self._tqm._stats.increment('changed', original_host.name)
@@ -390,7 +390,7 @@ class StrategyBase:
 
                 # If this is a role task, mark the parent role as being run (if
                 # the task was ok or failed, but not skipped or unreachable)
-                if original_task._role is not None and role_ran and original_task.action != 'include_role':
+                if original_task._role is not None and role_ran: #TODO:  and original_task.action != 'include_role':?
                     # lookup the role in the ROLE_CACHE to make sure we're dealing
                     # with the correct object and mark it as executed
                     for (entry, role_obj) in iteritems(iterator._play.ROLE_CACHE[original_task._role._role_name]):

lib/ansible/plugins/strategy/linear.py

@@ -280,6 +280,36 @@ class StrategyModule(StrategyBase):
                 results += self._wait_on_pending_results(iterator)
                 host_results.extend(results)
 
+                all_role_blocks = []
+                for hr in results:
+                    # handle include_role
+                    if hr._task.action == 'include_role':
+                        loop_var = None
+                        if hr._task.loop:
+                            loop_var = 'item'
+                            if hr._task.loop_control:
+                                loop_var = hr._task.loop_control.loop_var or 'item'
+                            include_results = hr._result['results']
+                        else:
+                            include_results = [ hr._result ]
+
+                        for include_result in include_results:
+                            if 'skipped' in include_result and include_result['skipped'] or 'failed' in include_result and include_result['failed']:
+                                continue
+
+                            role_vars = include_result.get('include_variables', dict())
+                            if loop_var and loop_var in include_result:
+                                role_vars[loop_var] = include_result[loop_var]
+
+                            display.debug("generating all_blocks data for role")
+                            new_ir = hr._task.copy()
+                            new_ir.args.update(role_vars)
+                            all_role_blocks.extend(new_ir.get_block_list(play=iterator._play, variable_manager=self._variable_manager, loader=self._loader))
+
+                if len(all_role_blocks) > 0:
+                    for host in hosts_left:
+                        iterator.add_tasks(host, all_role_blocks)
+
                 try:
                     included_files = IncludedFile.process_include_results(
                         host_results,

lib/ansible/template/__init__.py

@@ -367,13 +367,25 @@ class Templar:
             else:
                 return variable
 
+    def templatable(self, data):
+        '''
+        returns True if the data can be templated w/o errors
+        '''
+        templatable = True
+        try:
+            self.template(data)
+        except:
+           templatable = False
+        return templatable
+
     def _contains_vars(self, data):
         '''
         returns True if the data contains a variable pattern
         '''
-        for marker in  [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]:
-            if marker in data:
-                return True
+        if isinstance(data, string_types):
+            for marker in [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]:
+                if marker in data:
+                    return True
         return False
 
     def _convert_bare_variable(self, variable, bare_deprecated):