From ff34f5548ddde266742bee2765f32af2828c5ee7 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 5 Sep 2016 20:07:58 -0400 Subject: [PATCH] Dynamic role include (#17401) * dynamic role_include * more fixes for dynamic include roles * set play yfrom iterator when dynamic * changes from jimi-c * avoid modules that break ad hoc TODO: should really be a config --- lib/ansible/cli/adhoc.py | 4 ++ lib/ansible/executor/task_executor.py | 6 +- lib/ansible/playbook/base.py | 5 -- lib/ansible/playbook/helpers.py | 47 +++++++++++---- lib/ansible/playbook/role_include.py | 75 +++++++++++++++++------- lib/ansible/plugins/callback/default.py | 4 +- lib/ansible/plugins/strategy/__init__.py | 4 +- lib/ansible/plugins/strategy/linear.py | 30 ++++++++++ lib/ansible/template/__init__.py | 18 +++++- 9 files changed, 146 insertions(+), 47 deletions(-) diff --git a/lib/ansible/cli/adhoc.py b/lib/ansible/cli/adhoc.py index 66dfa8afafd..cfaeca955b5 100644 --- a/lib/ansible/cli/adhoc.py +++ b/lib/ansible/cli/adhoc.py @@ -155,6 +155,10 @@ class AdHocCLI(CLI): err = err + ' (did you mean to run ansible-playbook?)' raise AnsibleOptionsError(err) + # Avoid modules that don't work with ad-hoc + if self.options.module_name in ('include', 'include_role'): + raise AnsibleOptionsError("'%s' is not a valid action for ad-hoc commands" % self.options.module_name) + # dynamically load any plugins from the playbook directory for name, obj in get_all_plugin_loaders(): if obj.subdir: diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 436f0381d34..a5bb0767613 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -403,7 +403,7 @@ class TaskExecutor: return dict(changed=False, skipped=True, skip_reason='Conditional check failed', _ansible_no_log=self._play_context.no_log) except AnsibleError: # skip conditional exception in the case of includes as the vars needed might not be avaiable except in the included tasks or due to tags - if self._task.action != 'include': + if self._task.action in ['include', 'include_role']: raise # if we ran into an error while setting up the PlayContext, raise it now @@ -425,10 +425,10 @@ class TaskExecutor: # if this task is a IncludeRole, we just return now with a success code so the main thread can expand the task list for the given host elif self._task.action == 'include_role': include_variables = self._task.args.copy() - role = include_variables.pop('name') + role = templar.template(self._task._role_name) if not role: return dict(failed=True, msg="No role was specified to include") - return dict(name=role, include_variables=include_variables) + return dict(include_role=role, include_variables=include_variables) # Now we do final validation on the task, which sets all fields to their final values. self._task.post_validate(templar=templar) diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py index d54ebafb245..60cda790706 100644 --- a/lib/ansible/playbook/base.py +++ b/lib/ansible/playbook/base.py @@ -108,13 +108,10 @@ class BaseMeta(type): # its value from a parent object method = "_get_attr_%s" % attr_name if method in src_dict or method in dst_dict: - #print("^ assigning generic_g_method to %s" % attr_name) getter = partial(_generic_g_method, attr_name) elif '_get_parent_attribute' in dst_dict and value.inherit: - #print("^ assigning generic_g_parent to %s" % attr_name) getter = partial(_generic_g_parent, attr_name) else: - #print("^ assigning generic_g to %s" % attr_name) getter = partial(_generic_g, attr_name) setter = partial(_generic_s, attr_name) @@ -140,7 +137,6 @@ class BaseMeta(type): # now create the attributes based on the FieldAttributes # available, including from parent (and grandparent) objects - #print("creating class %s" % name) _create_attrs(dct, dct) _process_parents(parents, dct) @@ -201,7 +197,6 @@ class Base(with_metaclass(BaseMeta, object)): if hasattr(self, '_parent') and self._parent: self._parent.dump_me(depth+2) dep_chain = self._parent.get_dep_chain() - #print("%s^ dep chain: %s" % (" "*(depth+2), dep_chain)) if dep_chain: for dep in dep_chain: dep.dump_me(depth+2) diff --git a/lib/ansible/playbook/helpers.py b/lib/ansible/playbook/helpers.py index 123e2d3f98f..ded247afc85 100644 --- a/lib/ansible/playbook/helpers.py +++ b/lib/ansible/playbook/helpers.py @@ -22,7 +22,7 @@ import os from ansible import constants as C from ansible.compat.six import string_types -from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound, AnsibleError +from ansible.errors import AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound try: from __main__ import display @@ -260,16 +260,41 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h task_list.append(t) elif 'include_role' in task_ds: - task_list.extend( - IncludeRole.load( - task_ds, - block=block, - role=role, - task_include=None, - variable_manager=variable_manager, - loader=loader - ) - ) + + ir = IncludeRole.load( + task_ds, + block=block, + role=role, + task_include=None, + variable_manager=variable_manager, + loader=loader + ) + + # 1. the user has set the 'static' option to false or true + # 2. one of the appropriate config options was set + if ir.static is not None: + is_static = ir.static + else: + display.debug('Determine if include_role is static') + # Check to see if this include is dynamic or static: + all_vars = variable_manager.get_vars(loader=loader, play=play, task=ir) + templar = Templar(loader=loader, variables=all_vars) + needs_templating = False + for param in ir.args: + if templar._contains_vars(ir.args[param]): + if not templar.templatable(ir.args[param]): + needs_templating = True + break + is_static = C.DEFAULT_TASK_INCLUDES_STATIC or \ + (use_handlers and C.DEFAULT_HANDLER_INCLUDES_STATIC) or \ + (not needs_templating and ir.all_parents_static() and not ir.loop) + display.debug('Determined that if include_role static is %s' % str(is_static)) + if is_static: + # uses compiled list from object + t = task_list.extend(ir.get_block_list(variable_manager=variable_manager, loader=loader)) + else: + # passes task object itself for latter generation of list + t = task_list.append(ir) else: if use_handlers: t = Handler.load(task_ds, block=block, role=role, task_include=task_include, variable_manager=variable_manager, loader=loader) diff --git a/lib/ansible/playbook/role_include.py b/lib/ansible/playbook/role_include.py index 7dc928bf7bc..65a046d8e96 100644 --- a/lib/ansible/playbook/role_include.py +++ b/lib/ansible/playbook/role_include.py @@ -45,40 +45,73 @@ class IncludeRole(Task): # ================================================================================= # ATTRIBUTES - _name = FieldAttribute(isa='string', default=None) - _tasks_from = FieldAttribute(isa='string', default=None) + # private as this is a 'module options' vs a task property + _static = FieldAttribute(isa='bool', default=None, private=True) + _private = FieldAttribute(isa='bool', default=None, private=True) - # these should not be changeable? - _static = FieldAttribute(isa='bool', default=False) - _private = FieldAttribute(isa='bool', default=True) + def __init__(self, block=None, role=None, task_include=None): - @staticmethod - def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None): + super(IncludeRole, self).__init__(block=block, role=role, task_include=task_include) - r = IncludeRole().load_data(data, variable_manager=variable_manager, loader=loader) - args = r.preprocess_data(data).get('args', dict()) + self._role_name = None + self.statically_loaded = False + self._from_files = {} + self._parent_role = role - ri = RoleInclude.load(args.get('name'), play=block._play, variable_manager=variable_manager, loader=loader) - ri.vars.update(r.vars) - # build options for roles - from_files = {} - for key in ['tasks', 'vars', 'defaults']: - from_key = key + '_from' - if args.get(from_key): - from_files[key] = basename(args.get(from_key)) + def get_block_list(self, play=None, variable_manager=None, loader=None): + + # only need play passed in when dynamic + if play is None: + myplay = self._parent._play + else: + myplay = play + + ri = RoleInclude.load(self._role_name, play=myplay, variable_manager=variable_manager, loader=loader) + ri.vars.update(self.vars) + #ri._role_params.update(self.args) # jimi-c cant we avoid this? #build role - actual_role = Role.load(ri, block._play, parent_role=role, from_files=from_files) + actual_role = Role.load(ri, myplay, parent_role=self._parent_role, from_files=self._from_files) # compile role - blocks = actual_role.compile(play=block._play) + blocks = actual_role.compile(play=myplay) # set parent to ensure proper inheritance for b in blocks: - b._parent = block + b._parent = self._parent # updated available handlers in play - block._play.handlers = block._play.handlers + actual_role.get_handler_blocks(play=block._play) + myplay.handlers = myplay.handlers + actual_role.get_handler_blocks(play=myplay) return blocks + + @staticmethod + def load(data, block=None, role=None, task_include=None, variable_manager=None, loader=None): + + ir = IncludeRole(block, role, task_include=task_include).load_data(data, variable_manager=variable_manager, loader=loader) + + #TODO: use more automated list: for builtin in r.get_attributes(): #jimi-c: doing this to avoid using role_params and conflating include_role specific opts with other tasks + # set built in's + ir._role_name = ir.args.get('name') + for builtin in ['static', 'private']: + if ir.args.get(builtin): + setattr(ir, builtin, ir.args.get(builtin)) + + # build options for roles + for key in ['tasks', 'vars', 'defaults']: + from_key = key + '_from' + if ir.args.get(from_key): + ir._from_files[key] = basename(ir.args.get(from_key)) + + return ir.load_data(data, variable_manager=variable_manager, loader=loader) + + def copy(self, exclude_parent=False, exclude_tasks=False): + + new_me = super(IncludeRole, self).copy(exclude_parent=exclude_parent, exclude_tasks=exclude_tasks) + new_me.statically_loaded = self.statically_loaded + new_me._role_name = self._role_name + new_me._from_files = self._from_files.copy() + new_me._parent_role = self._parent_role + + return new_me diff --git a/lib/ansible/plugins/callback/default.py b/lib/ansible/plugins/callback/default.py index 63ef5288fb1..7bd2ac6ae14 100644 --- a/lib/ansible/plugins/callback/default.py +++ b/lib/ansible/plugins/callback/default.py @@ -62,7 +62,7 @@ class CallbackModule(CallbackBase): self._clean_results(result._result, result._task.action) delegated_vars = result._result.get('_ansible_delegated_vars', None) - if result._task.action == 'include': + if result._task.action in ('include', 'include_role'): return elif result._result.get('changed', False): if delegated_vars: @@ -158,7 +158,7 @@ class CallbackModule(CallbackBase): def v2_runner_item_on_ok(self, result): delegated_vars = result._result.get('_ansible_delegated_vars', None) - if result._task.action == 'include': + if result._task.action in ('include', 'include_role'): return elif result._result.get('changed', False): msg = 'changed' diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index e4b9d1e9716..20f07668e6b 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -376,7 +376,7 @@ class StrategyBase: if self._diff: self._tqm.send_callback('v2_on_file_diff', task_result) - if original_task.action != 'include': + if original_task.action in ['include', 'include_role']: self._tqm._stats.increment('ok', original_host.name) if 'changed' in task_result._result and task_result._result['changed']: self._tqm._stats.increment('changed', original_host.name) @@ -390,7 +390,7 @@ class StrategyBase: # If this is a role task, mark the parent role as being run (if # the task was ok or failed, but not skipped or unreachable) - if original_task._role is not None and role_ran and original_task.action != 'include_role': + if original_task._role is not None and role_ran: #TODO: and original_task.action != 'include_role':? # lookup the role in the ROLE_CACHE to make sure we're dealing # with the correct object and mark it as executed for (entry, role_obj) in iteritems(iterator._play.ROLE_CACHE[original_task._role._role_name]): diff --git a/lib/ansible/plugins/strategy/linear.py b/lib/ansible/plugins/strategy/linear.py index 975dd51e0fe..50ba4a70eb4 100644 --- a/lib/ansible/plugins/strategy/linear.py +++ b/lib/ansible/plugins/strategy/linear.py @@ -280,6 +280,36 @@ class StrategyModule(StrategyBase): results += self._wait_on_pending_results(iterator) host_results.extend(results) + all_role_blocks = [] + for hr in results: + # handle include_role + if hr._task.action == 'include_role': + loop_var = None + if hr._task.loop: + loop_var = 'item' + if hr._task.loop_control: + loop_var = hr._task.loop_control.loop_var or 'item' + include_results = hr._result['results'] + else: + include_results = [ hr._result ] + + for include_result in include_results: + if 'skipped' in include_result and include_result['skipped'] or 'failed' in include_result and include_result['failed']: + continue + + role_vars = include_result.get('include_variables', dict()) + if loop_var and loop_var in include_result: + role_vars[loop_var] = include_result[loop_var] + + display.debug("generating all_blocks data for role") + new_ir = hr._task.copy() + new_ir.args.update(role_vars) + all_role_blocks.extend(new_ir.get_block_list(play=iterator._play, variable_manager=self._variable_manager, loader=self._loader)) + + if len(all_role_blocks) > 0: + for host in hosts_left: + iterator.add_tasks(host, all_role_blocks) + try: included_files = IncludedFile.process_include_results( host_results, diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index f55cab6eec4..05ec12f03d9 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -367,13 +367,25 @@ class Templar: else: return variable + def templatable(self, data): + ''' + returns True if the data can be templated w/o errors + ''' + templatable = True + try: + self.template(data) + except: + templatable = False + return templatable + def _contains_vars(self, data): ''' returns True if the data contains a variable pattern ''' - for marker in [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]: - if marker in data: - return True + if isinstance(data, string_types): + for marker in [self.environment.block_start_string, self.environment.variable_start_string, self.environment.comment_start_string]: + if marker in data: + return True return False def _convert_bare_variable(self, variable, bare_deprecated):