From c60c295ada57c5d82c05ad1c87b1e9af198f8391 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Mon, 19 Jan 2015 16:18:18 -0600 Subject: [PATCH] Adding more fixes for integration testing under v2 --- v2/ansible/executor/task_result.py | 5 +- v2/ansible/module_utils/database.py | 128 +++++++++++++++++++++++ v2/ansible/parsing/mod_args.py | 4 + v2/ansible/playbook/block.py | 75 +++++++++++-- v2/ansible/playbook/helpers.py | 2 +- v2/ansible/playbook/role/definition.py | 4 +- v2/ansible/playbook/task.py | 27 ++++- v2/ansible/playbook/task_include.py | 92 ++++++++++++++-- v2/ansible/plugins/lookup/first_found.py | 1 + v2/ansible/plugins/strategies/linear.py | 2 +- 10 files changed, 317 insertions(+), 23 deletions(-) create mode 100644 v2/ansible/module_utils/database.py diff --git a/v2/ansible/executor/task_result.py b/v2/ansible/executor/task_result.py index d911713651a..a0a0c6a5ddf 100644 --- a/v2/ansible/executor/task_result.py +++ b/v2/ansible/executor/task_result.py @@ -43,7 +43,10 @@ class TaskResult: return self._check_key('skipped') def is_failed(self): - return self._check_key('failed') or self._result.get('rc', 0) != 0 + if 'failed_when_result' in self._result: + return self._check_key('failed_when_result') + else: + return self._check_key('failed') or self._result.get('rc', 0) != 0 def is_unreachable(self): return self._check_key('unreachable') diff --git a/v2/ansible/module_utils/database.py b/v2/ansible/module_utils/database.py new file mode 100644 index 00000000000..0dd1990d3e7 --- /dev/null +++ b/v2/ansible/module_utils/database.py @@ -0,0 +1,128 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c) 2014, Toshio Kuratomi +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +class SQLParseError(Exception): + pass + +class UnclosedQuoteError(SQLParseError): + pass + +# maps a type of identifier to the maximum number of dot levels that are +# allowed to specifiy that identifier. For example, a database column can be +# specified by up to 4 levels: database.schema.table.column +_PG_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, schema=2, table=3, column=4, role=1) +_MYSQL_IDENTIFIER_TO_DOT_LEVEL = dict(database=1, table=2, column=3, role=1, vars=1) + +def _find_end_quote(identifier, quote_char): + accumulate = 0 + while True: + try: + quote = identifier.index(quote_char) + except ValueError: + raise UnclosedQuoteError + accumulate = accumulate + quote + try: + next_char = identifier[quote+1] + except IndexError: + return accumulate + if next_char == quote_char: + try: + identifier = identifier[quote+2:] + accumulate = accumulate + 2 + except IndexError: + raise UnclosedQuoteError + else: + return accumulate + + +def _identifier_parse(identifier, quote_char): + if not identifier: + raise SQLParseError('Identifier name unspecified or unquoted trailing dot') + + already_quoted = False + if identifier.startswith(quote_char): + already_quoted = True + try: + end_quote = _find_end_quote(identifier[1:], quote_char=quote_char) + 1 + except UnclosedQuoteError: + already_quoted = False + else: + if end_quote < len(identifier) - 1: + if identifier[end_quote+1] == '.': + dot = end_quote + 1 + first_identifier = identifier[:dot] + next_identifier = identifier[dot+1:] + further_identifiers = _identifier_parse(next_identifier, quote_char) + further_identifiers.insert(0, first_identifier) + else: + raise SQLParseError('User escaped identifiers must escape extra quotes') + else: + further_identifiers = [identifier] + + if not already_quoted: + try: + dot = identifier.index('.') + except ValueError: + identifier = identifier.replace(quote_char, quote_char*2) + identifier = ''.join((quote_char, identifier, quote_char)) + further_identifiers = [identifier] + else: + if dot == 0 or dot >= len(identifier) - 1: + identifier = identifier.replace(quote_char, quote_char*2) + identifier = ''.join((quote_char, identifier, quote_char)) + further_identifiers = [identifier] + else: + first_identifier = identifier[:dot] + next_identifier = identifier[dot+1:] + further_identifiers = _identifier_parse(next_identifier, quote_char) + first_identifier = first_identifier.replace(quote_char, quote_char*2) + first_identifier = ''.join((quote_char, first_identifier, quote_char)) + further_identifiers.insert(0, first_identifier) + + return further_identifiers + + +def pg_quote_identifier(identifier, id_type): + identifier_fragments = _identifier_parse(identifier, quote_char='"') + if len(identifier_fragments) > _PG_IDENTIFIER_TO_DOT_LEVEL[id_type]: + raise SQLParseError('PostgreSQL does not support %s with more than %i dots' % (id_type, _PG_IDENTIFIER_TO_DOT_LEVEL[id_type])) + return '.'.join(identifier_fragments) + +def mysql_quote_identifier(identifier, id_type): + identifier_fragments = _identifier_parse(identifier, quote_char='`') + if len(identifier_fragments) > _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type]: + raise SQLParseError('MySQL does not support %s with more than %i dots' % (id_type, _MYSQL_IDENTIFIER_TO_DOT_LEVEL[id_type])) + + special_cased_fragments = [] + for fragment in identifier_fragments: + if fragment == '`*`': + special_cased_fragments.append('*') + else: + special_cased_fragments.append(fragment) + + return '.'.join(special_cased_fragments) diff --git a/v2/ansible/parsing/mod_args.py b/v2/ansible/parsing/mod_args.py index 37ed350de54..eddc093ef31 100644 --- a/v2/ansible/parsing/mod_args.py +++ b/v2/ansible/parsing/mod_args.py @@ -267,6 +267,10 @@ class ModuleArgsParser: # if we didn't see any module in the task at all, it's not a task really if action is None: raise AnsibleParserError("no action detected in task", obj=self._task_ds) + # FIXME: disabled for now, as there are other places besides the shell/script modules where + # having variables as the sole param for the module is valid (include_vars, add_host, and group_by?) + #elif args.get('_raw_params', '') != '' and action not in ('command', 'shell', 'script', 'include_vars'): + # raise AnsibleParserError("this task has extra params, which is only allowed in the command, shell or script module.", obj=self._task_ds) # shell modules require special handling (action, args) = self._handle_shell_weirdness(action, args) diff --git a/v2/ansible/playbook/block.py b/v2/ansible/playbook/block.py index d30bde53804..1e62aa0c98f 100644 --- a/v2/ansible/playbook/block.py +++ b/v2/ansible/playbook/block.py @@ -45,10 +45,20 @@ class Block(Base, Conditional, Taggable): super(Block, self).__init__() - def get_variables(self): - # blocks do not (currently) store any variables directly, - # so we just return an empty dict here - return dict() + def get_vars(self): + ''' + Blocks do not store variables directly, however they may be a member + of a role or task include which does, so return those if present. + ''' + + all_vars = dict() + + if self._role: + all_vars.update(self._role.get_vars()) + if self._task_include: + all_vars.update(self._task_include.get_vars()) + + return all_vars @staticmethod def load(data, parent_block=None, role=None, task_include=None, use_handlers=False, variable_manager=None, loader=None): @@ -73,17 +83,49 @@ class Block(Base, Conditional, Taggable): return ds def _load_block(self, attr, ds): - return load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers) + return load_list_of_tasks( + ds, + block=self, + role=self._role, + task_include=self._task_include, + variable_manager=self._variable_manager, + loader=self._loader, + use_handlers=self._use_handlers, + ) def _load_rescue(self, attr, ds): - return load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers) + return load_list_of_tasks( + ds, + block=self, + role=self._role, + task_include=self._task_include, + variable_manager=self._variable_manager, + loader=self._loader, + use_handlers=self._use_handlers, + ) def _load_always(self, attr, ds): - return load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers) + return load_list_of_tasks( + ds, + block=self, + role=self._role, + task_include=self._task_include, + variable_manager=self._variable_manager, + loader=self._loader, + use_handlers=self._use_handlers, + ) # not currently used #def _load_otherwise(self, attr, ds): - # return self._load_list_of_tasks(ds, block=self, role=self._role, variable_manager=self._variable_manager, loader=self._loader, use_handlers=self._use_handlers) + # return load_list_of_tasks( + # ds, + # block=self, + # role=self._role, + # task_include=self._task_include, + # variable_manager=self._variable_manager, + # loader=self._loader, + # use_handlers=self._use_handlers, + # ) def compile(self): ''' @@ -125,6 +167,8 @@ class Block(Base, Conditional, Taggable): if self._role is not None: data['role'] = self._role.serialize() + if self._task_include is not None: + data['task_include'] = self._task_include.serialize() return data @@ -134,6 +178,8 @@ class Block(Base, Conditional, Taggable): serialize method ''' + from ansible.playbook.task_include import TaskInclude + # unpack the when attribute, which is the only one we want self.when = data.get('when') @@ -144,7 +190,17 @@ class Block(Base, Conditional, Taggable): r.deserialize(role_data) self._role = r + # if there was a serialized task include, unpack it too + ti_data = data.get('task_include') + if ti_data: + ti = TaskInclude() + ti.deserialize(ti_data) + self._task_include = ti + def evaluate_conditional(self, all_vars): + if self._task_include is not None: + if not self._task_include.evaluate_conditional(all_vars): + return False if self._parent_block is not None: if not self._parent_block.evaluate_conditional(all_vars): return False @@ -168,3 +224,6 @@ class Block(Base, Conditional, Taggable): elif self._role: self._role.set_loader(loader) + if self._task_include: + self._task_include.set_loader(loader) + diff --git a/v2/ansible/playbook/helpers.py b/v2/ansible/playbook/helpers.py index f4bbc8d5586..3b6d59d0190 100644 --- a/v2/ansible/playbook/helpers.py +++ b/v2/ansible/playbook/helpers.py @@ -30,7 +30,7 @@ def load_list_of_blocks(ds, parent_block=None, role=None, task_include=None, use return a list of Block() objects, where implicit blocks are created for each bare Task. ''' - + # we import here to prevent a circular dependency with imports from ansible.playbook.block import Block diff --git a/v2/ansible/playbook/role/definition.py b/v2/ansible/playbook/role/definition.py index 841d081af79..2f906aa056c 100644 --- a/v2/ansible/playbook/role/definition.py +++ b/v2/ansible/playbook/role/definition.py @@ -44,8 +44,8 @@ class RoleDefinition(Base, Conditional, Taggable): self._role_params = dict() super(RoleDefinition, self).__init__() - def __repr__(self): - return 'ROLEDEF: ' + self._attributes.get('role', '') + #def __repr__(self): + # return 'ROLEDEF: ' + self._attributes.get('role', '') @staticmethod def load(data, variable_manager=None, loader=None): diff --git a/v2/ansible/playbook/task.py b/v2/ansible/playbook/task.py index db5c017d673..df919615550 100644 --- a/v2/ansible/playbook/task.py +++ b/v2/ansible/playbook/task.py @@ -33,6 +33,7 @@ from ansible.playbook.block import Block from ansible.playbook.conditional import Conditional from ansible.playbook.role import Role from ansible.playbook.taggable import Taggable +from ansible.playbook.task_include import TaskInclude class Task(Base, Conditional, Taggable): @@ -189,16 +190,23 @@ class Task(Base, Conditional, Taggable): def post_validate(self, all_vars=dict(), fail_on_undefined=True): ''' Override of base class post_validate, to also do final validation on - the block to which this task belongs. + the block and task include (if any) to which this task belongs. ''' if self._block: self._block.post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) + if self._task_include: + self._task_include.post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) super(Task, self).post_validate(all_vars=all_vars, fail_on_undefined=fail_on_undefined) def get_vars(self): - all_vars = self.serialize() + all_vars = dict() + if self._task_include: + all_vars.update(self._task_include.get_vars()) + + all_vars.update(self.serialize()) + if 'tags' in all_vars: del all_vars['tags'] if 'when' in all_vars: @@ -242,6 +250,9 @@ class Task(Base, Conditional, Taggable): if self._role: data['role'] = self._role.serialize() + if self._task_include: + data['task_include'] = self._task_include.serialize() + return data def deserialize(self, data): @@ -261,6 +272,13 @@ class Task(Base, Conditional, Taggable): self._role = r del data['role'] + ti_data = data.get('task_include') + if ti_data: + ti = TaskInclude() + ti.deserialize(ti_data) + self._task_include = ti + del data['task_include'] + super(Task, self).deserialize(data) def evaluate_conditional(self, all_vars): @@ -271,6 +289,9 @@ class Task(Base, Conditional, Taggable): if self._block is not None: if not self._block.evaluate_conditional(all_vars): return False + if self._task_include is not None: + if not self._task_include.evaluate_conditional(all_vars): + return False return super(Task, self).evaluate_conditional(all_vars) def evaluate_tags(self, only_tags, skip_tags, all_vars): @@ -293,6 +314,8 @@ class Task(Base, Conditional, Taggable): if self._block: self._block.set_loader(loader) + if self._task_include: + self._task_include.set_loader(loader) for dep in self._dep_chain: dep.set_loader(loader) diff --git a/v2/ansible/playbook/task_include.py b/v2/ansible/playbook/task_include.py index e30a55857f6..c2bfb6fdaef 100644 --- a/v2/ansible/playbook/task_include.py +++ b/v2/ansible/playbook/task_include.py @@ -24,14 +24,16 @@ from ansible.parsing.splitter import split_args, parse_kv from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base +from ansible.playbook.conditional import Conditional from ansible.playbook.helpers import load_list_of_blocks, compile_block_list +from ansible.playbook.taggable import Taggable from ansible.plugins import lookup_loader __all__ = ['TaskInclude'] -class TaskInclude(Base): +class TaskInclude(Base, Conditional, Taggable): ''' A class used to wrap the use of `include: /some/other/file.yml` @@ -146,13 +148,13 @@ class TaskInclude(Base): raise AnsibleParsingError("included task files must contain a list of tasks", obj=ds) self._task_blocks = load_list_of_blocks( - data, - parent_block=self._block, - task_include=self, - role=self._role, - use_handlers=self._use_handlers, - loader=self._loader - ) + data, + parent_block=self._block, + task_include=self, + role=self._role, + use_handlers=self._use_handlers, + loader=self._loader + ) return ds def compile(self): @@ -164,3 +166,77 @@ class TaskInclude(Base): task_list.extend(compile_block_list(self._task_blocks)) return task_list + def get_vars(self): + ''' + Returns the vars for this task include, but also first merges in + those from any parent task include which may exist. + ''' + + all_vars = dict() + if self._task_include: + all_vars.update(self._task_include.get_vars()) + all_vars.update(self.vars) + return all_vars + + def serialize(self): + + data = super(TaskInclude, self).serialize() + + if self._block: + data['block'] = self._block.serialize() + + if self._role: + data['role'] = self._role.serialize() + + if self._task_include: + data['task_include'] = self._task_include.serialize() + + return data + + def deserialize(self, data): + + # import here to prevent circular importing issues + from ansible.playbook.block import Block + from ansible.playbook.role import Role + + block_data = data.get('block') + if block_data: + b = Block() + b.deserialize(block_data) + self._block = b + del data['block'] + + role_data = data.get('role') + if role_data: + r = Role() + r.deserialize(role_data) + self._role = r + del data['role'] + + ti_data = data.get('task_include') + if ti_data: + ti = TaskInclude() + ti.deserialize(ti_data) + self._task_include = ti + del data['task_include'] + + super(TaskInclude, self).deserialize(data) + + def evaluate_conditional(self, all_vars): + if self._task_include is not None: + if not self._task_include.evaluate_conditional(all_vars): + return False + if self._block is not None: + if not self._block.evaluate_conditional(all_vars): + return False + elif self._role is not None: + if not self._role.evaluate_conditional(all_vars): + return False + return super(TaskInclude, self).evaluate_conditional(all_vars) + + def set_loader(self, loader): + self._loader = loader + if self._block: + self._block.set_loader(loader) + elif self._task_include: + self._task_include.set_loader(loader) diff --git a/v2/ansible/plugins/lookup/first_found.py b/v2/ansible/plugins/lookup/first_found.py index 3b59a191cbf..ea43e13c4d8 100644 --- a/v2/ansible/plugins/lookup/first_found.py +++ b/v2/ansible/plugins/lookup/first_found.py @@ -121,6 +121,7 @@ import os from ansible.plugins.lookup import LookupBase from ansible.template import Templar +from ansible.utils.boolean import boolean class LookupModule(LookupBase): diff --git a/v2/ansible/plugins/strategies/linear.py b/v2/ansible/plugins/strategies/linear.py index 93c2c5a4b3b..b77381ce80c 100644 --- a/v2/ansible/plugins/strategies/linear.py +++ b/v2/ansible/plugins/strategies/linear.py @@ -70,7 +70,7 @@ class StrategyModule(StrategyBase): debug("'%s' skipped because role has already run" % task) continue - if not task.evaluate_tags(connection_info.only_tags, connection_info.skip_tags, task_vars): + if not task.evaluate_tags(connection_info.only_tags, connection_info.skip_tags, task_vars) and task.action != 'setup': debug("'%s' failed tag evaluation" % task) continue