From 24b536d7edb35da5b226da851250d862c80fd012 Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Tue, 13 Nov 2012 00:08:10 +0100 Subject: [PATCH 1/5] Add some comments to templating system --- lib/ansible/utils/template.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/ansible/utils/template.py b/lib/ansible/utils/template.py index b01010622fa..0077a838bd9 100644 --- a/lib/ansible/utils/template.py +++ b/lib/ansible/utils/template.py @@ -34,14 +34,22 @@ _LISTRE = re.compile(r"(\w+)\[(\d+)\]") def _varFindLimitSpace(vars, space, part, depth): + ''' limits the search space of space to part + + basically does space.get(part, None), but with + templating for part and a few more things + ''' - # TODO: comments - + # Previous part couldn't be found, nothing to limit to if space is None: return space + # A part with escaped .s in it is compounded by { and }, remove them if part[0] == '{' and part[-1] == '}': part = part[1:-1] + # Template part to resolve variables within (${var$var2}) part = varReplace(part, vars, depth=depth + 1) + + # Now find it if part in space: space = space[part] elif "[" in part: @@ -55,11 +63,30 @@ def _varFindLimitSpace(vars, space, part, depth): return None else: return None + return space def _varFind(text, vars, depth=0): + ''' Searches for a variable in text and finds its replacement in vars + + The variables can have two formats; + - simple, $ followed by alphanumerics and/or underscores + - complex, ${ followed by alphanumerics, underscores, periods, braces and brackets, ended by a } + + Examples: + - $variable: simple variable that will have vars['variable'] as its replacement + - ${variable.complex}: complex variable that will have vars['variable']['complex'] as its replacement + - $variable.complex: simple variable, identical to the first, .complex ignored + + Complex variables are broken into parts by separating on periods, except if enclosed in {}. + ${variable.{fully.qualified.domain}} would be parsed as two parts, variable and fully.qualified.domain, + whereas ${variable.fully.qualified.domain} would be parsed as four parts. - # TODO: comments + Returns a dict(replacement=, start=, + end=) + or None if no variable could be found in text. If replacement is None, it should be replaced with the + original data in the caller. + ''' start = text.find("$") if start == -1: @@ -80,6 +107,8 @@ def _varFind(text, vars, depth=0): is_complex = False brace_level = 0 end = var_start + # part_start is a tuple of where the current part started and its current brace_level + # brace_level is used to implement .-escaping part_start = (var_start, brace_level) space = vars while end < len(text) and ((is_complex and brace_level > 0) or not is_complex): From a2bb3a09d241d7b997f0400033ba29c2ec7de600 Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Tue, 13 Nov 2012 00:08:33 +0100 Subject: [PATCH 2/5] Completely ignore stray $ in input Previously, "a $ string $var" would not have gotten replaced at all. --- lib/ansible/utils/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/utils/template.py b/lib/ansible/utils/template.py index 0077a838bd9..f86d9b09f40 100644 --- a/lib/ansible/utils/template.py +++ b/lib/ansible/utils/template.py @@ -133,7 +133,7 @@ def _varFind(text, vars, depth=0): if text[var_end] != '}' or brace_level != 0: return None if var_end == part_start[0]: - return None + return {'replacement': '$', 'start': start, 'end': end} space = _varFindLimitSpace(vars, space, text[part_start[0]:var_end], depth) return {'replacement': space, 'start': start, 'end': end} From f91fa9f765b92a3587d091ade6dc5c9f3fe2e8c3 Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Tue, 13 Nov 2012 00:09:24 +0100 Subject: [PATCH 3/5] Fix lookup plugin test --- test/TestPlayBook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TestPlayBook.py b/test/TestPlayBook.py index 55e318cb66f..9ff679973d6 100644 --- a/test/TestPlayBook.py +++ b/test/TestPlayBook.py @@ -202,7 +202,7 @@ class TestPlaybook(unittest.TestCase): assert utils.jsonify(expected, format=True) == utils.jsonify(actual,format=True) - assert len(EVENTS) == 44 + assert len(EVENTS) == 60 def test_includes(self): pb = os.path.join(self.test_dir, 'playbook-includer.yml') From d2dce1d63f9a8c1d5281598d5afd60c0628c7ff1 Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Tue, 13 Nov 2012 02:03:16 +0100 Subject: [PATCH 4/5] Make lookup plugin replacements part of the main variable logic --- lib/ansible/utils/template.py | 116 ++++++++++++++++------------------ test/TestUtils.py | 52 +++++++-------- 2 files changed, 79 insertions(+), 89 deletions(-) diff --git a/lib/ansible/utils/template.py b/lib/ansible/utils/template.py index f86d9b09f40..119ede895c1 100644 --- a/lib/ansible/utils/template.py +++ b/lib/ansible/utils/template.py @@ -33,7 +33,7 @@ import pwd _LISTRE = re.compile(r"(\w+)\[(\d+)\]") -def _varFindLimitSpace(vars, space, part, depth): +def _varFindLimitSpace(basedir, vars, space, part, depth): ''' limits the search space of space to part basically does space.get(part, None), but with @@ -47,7 +47,7 @@ def _varFindLimitSpace(vars, space, part, depth): if part[0] == '{' and part[-1] == '}': part = part[1:-1] # Template part to resolve variables within (${var$var2}) - part = varReplace(part, vars, depth=depth + 1) + part = varReplace(basedir, part, vars, depth=depth + 1) # Now find it if part in space: @@ -66,7 +66,7 @@ def _varFindLimitSpace(vars, space, part, depth): return space -def _varFind(text, vars, depth=0): +def _varFind(basedir, text, vars, depth=0): ''' Searches for a variable in text and finds its replacement in vars The variables can have two formats; @@ -105,15 +105,28 @@ def _varFind(text, vars, depth=0): var_start += 1 else: is_complex = False - brace_level = 0 + brace_level = 1 + # is_lookup is true for $FILE(...) and friends + is_lookup = False + lookup_plugin_name = None end = var_start - # part_start is a tuple of where the current part started and its current brace_level - # brace_level is used to implement .-escaping - part_start = (var_start, brace_level) + # part_start is an index of where the current part started + part_start = var_start space = vars - while end < len(text) and ((is_complex and brace_level > 0) or not is_complex): + while end < len(text) and (((is_lookup or is_complex) and brace_level > 0) or (not is_complex and not is_lookup)): if text[end].isalnum() or text[end] == '_': pass + elif not is_complex and not is_lookup and text[end] == '(' and text[part_start:end].isupper(): + is_lookup = True + lookup_plugin_name = text[part_start:end] + part_start = end + 1 + elif is_lookup and text[end] == '(': + brace_level += 1 + elif is_lookup and text[end] == ')': + brace_level -= 1 + elif is_lookup: + # lookups are allowed arbitrary contents + pass elif is_complex and text[end] == '{': brace_level += 1 elif is_complex and text[end] == '}': @@ -121,23 +134,44 @@ def _varFind(text, vars, depth=0): elif is_complex and text[end] in ('$', '[', ']'): pass elif is_complex and text[end] == '.': - if brace_level == part_start[1]: - space = _varFindLimitSpace(vars, space, text[part_start[0]:end], depth) - part_start = (end + 1, brace_level) + if brace_level == 1: + space = _varFindLimitSpace(basedir, vars, space, text[part_start:end], depth) + part_start = end + 1 else: + # This breaks out of the loop on non-variable name characters break end += 1 var_end = end + # Handle "This has $ in it" + if var_end == part_start: + return {'replacement': None, 'start': start, 'end': end} + + # Handle lookup plugins + if is_lookup: + # When basedir is None, handle lookup plugins later + if basedir is None: + return {'replacement': None, 'start': start, 'end': end} + var_end -= 1 + from ansible import utils + args = text[part_start:var_end] + if lookup_plugin_name == 'LOOKUP': + lookup_plugin_name, args = args.split(",", 1) + args = args.strip() + instance = utils.plugins.lookup_loader.get(lookup_plugin_name.lower(), basedir=basedir) + if instance is not None: + replacement = instance.run(args, inject=vars) + else: + replacement = None + return {'replacement': replacement, 'start': start, 'end': end} + if is_complex: var_end -= 1 if text[var_end] != '}' or brace_level != 0: return None - if var_end == part_start[0]: - return {'replacement': '$', 'start': start, 'end': end} - space = _varFindLimitSpace(vars, space, text[part_start[0]:var_end], depth) + space = _varFindLimitSpace(basedir, vars, space, text[part_start:var_end], depth) return {'replacement': space, 'start': start, 'end': end} -def varReplace(raw, vars, depth=0, expand_lists=False): +def varReplace(basedir, raw, vars, depth=0, expand_lists=False): ''' Perform variable replacement of $variables in string raw using vars dictionary ''' # this code originally from yum @@ -147,7 +181,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False): done = [] # Completed chunks to return while raw: - m = _varFind(raw, vars, depth) + m = _varFind(basedir, raw, vars, depth) if not m: done.append(raw) break @@ -159,7 +193,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False): if expand_lists and isinstance(replacement, (list, tuple)): replacement = ",".join(replacement) if isinstance(replacement, (str, unicode)): - replacement = varReplace(replacement, vars, depth=depth+1, expand_lists=expand_lists) + replacement = varReplace(basedir, replacement, vars, depth=depth+1, expand_lists=expand_lists) if replacement is None: replacement = raw[m['start']:m['end']] @@ -170,53 +204,11 @@ def varReplace(raw, vars, depth=0, expand_lists=False): return ''.join(done) -_FILEPIPECRE = re.compile(r"\$(?P[A-Z]+)\(([^\)]*)\)") -def _varReplaceLookups(basedir, raw, vars): - from ansible import utils - done = [] # Completed chunks to return - - while raw: - m = _FILEPIPECRE.search(raw) - if not m: - done.append(raw) - break - - # Determine replacement value (if unknown lookup plugin then preserve - # original) - - replacement = m.group() - if m.group(1) == "FILE": - module_name = "file" - args = m.group(2) - elif m.group(1) == "PIPE": - module_name = "pipe" - args = m.group(2) - elif m.group(1) == "LOOKUP": - module_name, args = m.group(2).split(",", 1) - args = args.strip() - else: - module_name = m.group(1).lower() - args = m.group(2) - instance = utils.plugins.lookup_loader.get(module_name, basedir=basedir) - if instance is not None: - replacement = instance.run(args, inject=vars) - if not isinstance(replacement, basestring): - replacement = ",".join(replacement) - else: - replacement = m.group(0) - - start, end = m.span() - done.append(raw[:start]) # Keep stuff leading up to token - done.append(replacement.rstrip()) # Append replacement value - raw = raw[end:] # Continue with remainder of string - - return ''.join(done) - def template_ds(basedir, varname, vars): ''' templates a data structure by traversing it and substituting for other data structures ''' if isinstance(varname, basestring): - m = _varFind(varname, vars) + m = _varFind(basedir, varname, vars) if not m: return varname if m['start'] == 0 and m['end'] == len(varname): @@ -243,9 +235,7 @@ def template(basedir, text, vars, expand_lists=False): text = text.decode('utf-8') except UnicodeEncodeError: pass # already unicode - text = varReplace(unicode(text), vars, expand_lists=expand_lists) - if basedir is not None: - text = _varReplaceLookups(basedir, text, vars) + text = varReplace(basedir, unicode(text), vars, expand_lists=expand_lists) return text def template_from_file(basedir, path, vars): diff --git a/test/TestUtils.py b/test/TestUtils.py index b8a2bfa6efc..1cf6e61bf26 100644 --- a/test/TestUtils.py +++ b/test/TestUtils.py @@ -15,14 +15,14 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' def test_varReplace_trailing_dollar(self): template = '$what $who $' vars = dict(what='hello', who='world') - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world $' def test_varReplace_multiple(self): @@ -32,7 +32,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -42,7 +42,7 @@ class TestUtils(unittest.TestCase): 'whoVar': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) print res assert res == 'hello world' @@ -52,7 +52,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world!' @@ -62,7 +62,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -72,7 +72,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world}' @@ -82,7 +82,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -92,7 +92,7 @@ class TestUtils(unittest.TestCase): 'who': 'world', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world }' @@ -104,7 +104,7 @@ class TestUtils(unittest.TestCase): }, } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) print res assert res == template @@ -117,7 +117,7 @@ class TestUtils(unittest.TestCase): }, } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -130,7 +130,7 @@ class TestUtils(unittest.TestCase): 'what': 'hello', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello 2' @@ -140,7 +140,7 @@ class TestUtils(unittest.TestCase): 'who': u'wórld', } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == u'hello wórld' @@ -150,7 +150,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', 'world' ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -160,7 +160,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', 'world' ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -170,7 +170,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', 'world' ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -180,7 +180,7 @@ class TestUtils(unittest.TestCase): 'data': { 'no-one': 0, 'world': 1 } } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == template @@ -190,7 +190,7 @@ class TestUtils(unittest.TestCase): 'data': [ 'no-one', {'msg': [ 'world'] } ] } - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'hello world' @@ -201,7 +201,7 @@ class TestUtils(unittest.TestCase): } template = '${foo}${bar}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'foobar' def test_varReplace_escape_dot(self): @@ -214,7 +214,7 @@ class TestUtils(unittest.TestCase): } template = '${hostvars.{test.example.com}.foo}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'bar' def test_varReplace_list_join(self): @@ -227,7 +227,7 @@ class TestUtils(unittest.TestCase): } template = 'yum pkg=${list} state=installed' - res = ansible.utils.varReplace(template, vars, expand_lists=True) + res = ansible.utils.varReplace(None, template, vars, expand_lists=True) assert res == 'yum pkg=foo,bar,baz state=installed' def test_varReplace_escaped_var(self): @@ -235,7 +235,7 @@ class TestUtils(unittest.TestCase): 'foo': 'bar', } template = 'action \$foo' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'action $foo' def test_varReplace_var_part(self): @@ -246,7 +246,7 @@ class TestUtils(unittest.TestCase): 'key': 'bar', } template = 'test ${foo.$key}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'test result' def test_varReplace_var_partial_part(self): @@ -257,7 +257,7 @@ class TestUtils(unittest.TestCase): 'key': 'bar', } template = 'test ${foo.${key}baz}' - res = ansible.utils.varReplace(template, vars) + res = ansible.utils.varReplace(None, template, vars) assert res == 'test result' def test_template_varReplace_iterated(self): @@ -274,14 +274,14 @@ class TestUtils(unittest.TestCase): def test_varReplace_include(self): template = 'hello $FILE(world) $LOOKUP(file, world)' - res = ansible.utils.template("test", template, {}) + res = ansible.utils.template("test", template, {}, expand_lists=True) assert res == u'hello world world' def test_varReplace_include_script(self): template = 'hello $PIPE(echo world) $LOOKUP(pipe, echo world)' - res = ansible.utils.template("test", template, {}) + res = ansible.utils.template("test", template, {}, expand_lists=True) assert res == u'hello world world' From 25a8787e950dea58967738edc4d6821a0600d0db Mon Sep 17 00:00:00 2001 From: Daniel Hokka Zakrisson Date: Mon, 12 Nov 2012 15:25:01 +0100 Subject: [PATCH 5/5] Template all variables before returning them to Jinja2 --- lib/ansible/utils/template.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/ansible/utils/template.py b/lib/ansible/utils/template.py index 119ede895c1..db6e7fa754d 100644 --- a/lib/ansible/utils/template.py +++ b/lib/ansible/utils/template.py @@ -238,6 +238,18 @@ def template(basedir, text, vars, expand_lists=False): text = varReplace(basedir, unicode(text), vars, expand_lists=expand_lists) return text +class _jinja2_vars(object): + ''' helper class to template all variable content before jinja2 sees it ''' + def __init__(self, basedir, vars): + self.basedir = basedir + self.vars = vars + def __contains__(self, k): + return k in self.vars + def __getitem__(self, varname): + if varname not in self.vars: + raise KeyError("undefined variable: %s" % varname) + return template_ds(self.basedir, self.vars[varname], self.vars) + def template_from_file(basedir, path, vars): ''' run a file through the templating engine ''' @@ -276,7 +288,11 @@ def template_from_file(basedir, path, vars): vars['ansible_managed'] = time.strftime(managed_str, time.localtime(os.path.getmtime(realpath))) - res = t.render(vars) + # This line performs deep Jinja2 magic that uses the _jinja2_vars object for vars + # Ideally, this could use some API where setting shared=True and the object won't get + # passed through dict(o), but I have not found that yet. + res = jinja2.utils.concat(t.root_render_func(t.new_context(_jinja2_vars(basedir, vars), shared=True))) + if data.endswith('\n') and not res.endswith('\n'): res = res + '\n' return template(basedir, res, vars)