archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1607 from dhozac/j2-templated-vars

Browse Source 
Templating fixes and features
pull/1648/head
Michael DeHaan 12 years ago
parent df458fb3b5 25a8787e95
commit 96e5408046
3 changed files with 128 additions and 93 deletions
Show Stats Download Patch File Download Diff File

163
lib/ansible/utils/template.py
Unescape Escape View File

@ -33,15 +33,23 @@ import pwd
_LISTRE = re.compile(r"(\w+)\[(\d+)\]") _LISTRE = re.compile(r"(\w+)\[(\d+)\]")
def _varFindLimitSpace(vars, space, part, depth): def _varFindLimitSpace(basedir, vars, space, part, depth):
''' limits the search space of space to part
# TODO: comments basically does space.get(part, None), but with
templating for part and a few more things
'''
# Previous part couldn't be found, nothing to limit to
if space is None: if space is None:
return space return space
# A part with escaped .s in it is compounded by { and }, remove them
if part[0] == '{' and part[-1] == '}': if part[0] == '{' and part[-1] == '}':
part = part[1:-1] part = part[1:-1]
part = varReplace(part, vars, depth=depth + 1) # Template part to resolve variables within (${var$var2})
part = varReplace(basedir, part, vars, depth=depth + 1)
# Now find it
if part in space: if part in space:
space = space[part] space = space[part]
elif "[" in part: elif "[" in part:
@ -55,11 +63,30 @@ def _varFindLimitSpace(vars, space, part, depth):
return None return None
else: else:
return None return None
return space return space
def _varFind(text, vars, depth=0): def _varFind(basedir, text, vars, depth=0):
''' Searches for a variable in text and finds its replacement in vars
The variables can have two formats;
- simple, $ followed by alphanumerics and/or underscores
- complex, ${ followed by alphanumerics, underscores, periods, braces and brackets, ended by a }
# TODO: comments Examples:
- $variable: simple variable that will have vars['variable'] as its replacement
- ${variable.complex}: complex variable that will have vars['variable']['complex'] as its replacement
- $variable.complex: simple variable, identical to the first, .complex ignored
Complex variables are broken into parts by separating on periods, except if enclosed in {}.
${variable.{fully.qualified.domain}} would be parsed as two parts, variable and fully.qualified.domain,
whereas ${variable.fully.qualified.domain} would be parsed as four parts.
Returns a dict(replacement=<value in vars>, start=<index into text where the variable stated>,
end=<index into text where the variable ends>)
or None if no variable could be found in text. If replacement is None, it should be replaced with the
original data in the caller.
'''
start = text.find("$") start = text.find("$")
if start == -1: if start == -1:
@ -78,13 +105,28 @@ def _varFind(text, vars, depth=0):
var_start += 1 var_start += 1
else: else:
is_complex = False is_complex = False
brace_level = 0 brace_level = 1
# is_lookup is true for $FILE(...) and friends
is_lookup = False
lookup_plugin_name = None
end = var_start end = var_start
part_start = (var_start, brace_level) # part_start is an index of where the current part started
part_start = var_start
space = vars space = vars
while end < len(text) and ((is_complex and brace_level > 0) or not is_complex): while end < len(text) and (((is_lookup or is_complex) and brace_level > 0) or (not is_complex and not is_lookup)):
if text[end].isalnum() or text[end] == '_': if text[end].isalnum() or text[end] == '_':
pass pass
elif not is_complex and not is_lookup and text[end] == '(' and text[part_start:end].isupper():
is_lookup = True
lookup_plugin_name = text[part_start:end]
part_start = end + 1
elif is_lookup and text[end] == '(':
brace_level += 1
elif is_lookup and text[end] == ')':
brace_level -= 1
elif is_lookup:
# lookups are allowed arbitrary contents
pass
elif is_complex and text[end] == '{': elif is_complex and text[end] == '{':
brace_level += 1 brace_level += 1
elif is_complex and text[end] == '}': elif is_complex and text[end] == '}':
@ -92,23 +134,44 @@ def _varFind(text, vars, depth=0):
elif is_complex and text[end] in ('$', '[', ']'): elif is_complex and text[end] in ('$', '[', ']'):
pass pass
elif is_complex and text[end] == '.': elif is_complex and text[end] == '.':
if brace_level == part_start[1]: if brace_level == 1:
space = _varFindLimitSpace(vars, space, text[part_start[0]:end], depth) space = _varFindLimitSpace(basedir, vars, space, text[part_start:end], depth)
part_start = (end + 1, brace_level) part_start = end + 1
else: else:
# This breaks out of the loop on non-variable name characters
break break
end += 1 end += 1
var_end = end var_end = end
# Handle "This has $ in it"
if var_end == part_start:
return {'replacement': None, 'start': start, 'end': end}
# Handle lookup plugins
if is_lookup:
# When basedir is None, handle lookup plugins later
if basedir is None:
return {'replacement': None, 'start': start, 'end': end}
var_end -= 1
from ansible import utils
args = text[part_start:var_end]
if lookup_plugin_name == 'LOOKUP':
lookup_plugin_name, args = args.split(",", 1)
args = args.strip()
instance = utils.plugins.lookup_loader.get(lookup_plugin_name.lower(), basedir=basedir)
if instance is not None:
replacement = instance.run(args, inject=vars)
else:
replacement = None
return {'replacement': replacement, 'start': start, 'end': end}
if is_complex: if is_complex:
var_end -= 1 var_end -= 1
if text[var_end] != '}' or brace_level != 0: if text[var_end] != '}' or brace_level != 0:
return None return None
if var_end == part_start[0]: space = _varFindLimitSpace(basedir, vars, space, text[part_start:var_end], depth)
return None
space = _varFindLimitSpace(vars, space, text[part_start[0]:var_end], depth)
return {'replacement': space, 'start': start, 'end': end} return {'replacement': space, 'start': start, 'end': end}
def varReplace(raw, vars, depth=0, expand_lists=False): def varReplace(basedir, raw, vars, depth=0, expand_lists=False):
''' Perform variable replacement of $variables in string raw using vars dictionary ''' ''' Perform variable replacement of $variables in string raw using vars dictionary '''
# this code originally from yum # this code originally from yum
@ -118,7 +181,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False):
done = [] # Completed chunks to return done = [] # Completed chunks to return
while raw: while raw:
m = _varFind(raw, vars, depth) m = _varFind(basedir, raw, vars, depth)
if not m: if not m:
done.append(raw) done.append(raw)
break break
@ -130,7 +193,7 @@ def varReplace(raw, vars, depth=0, expand_lists=False):
if expand_lists and isinstance(replacement, (list, tuple)): if expand_lists and isinstance(replacement, (list, tuple)):
replacement = ",".join(replacement) replacement = ",".join(replacement)
if isinstance(replacement, (str, unicode)): if isinstance(replacement, (str, unicode)):
replacement = varReplace(replacement, vars, depth=depth+1, expand_lists=expand_lists) replacement = varReplace(basedir, replacement, vars, depth=depth+1, expand_lists=expand_lists)
if replacement is None: if replacement is None:
replacement = raw[m['start']:m['end']] replacement = raw[m['start']:m['end']]
@ -141,53 +204,11 @@ def varReplace(raw, vars, depth=0, expand_lists=False):
return ''.join(done) return ''.join(done)
_FILEPIPECRE = re.compile(r"\$(?P<special>[A-Z]+)\(([^\)]*)\)")
def _varReplaceLookups(basedir, raw, vars):
from ansible import utils
done = [] # Completed chunks to return
while raw:
m = _FILEPIPECRE.search(raw)
if not m:
done.append(raw)
break
# Determine replacement value (if unknown lookup plugin then preserve
# original)
replacement = m.group()
if m.group(1) == "FILE":
module_name = "file"
args = m.group(2)
elif m.group(1) == "PIPE":
module_name = "pipe"
args = m.group(2)
elif m.group(1) == "LOOKUP":
module_name, args = m.group(2).split(",", 1)
args = args.strip()
else:
module_name = m.group(1).lower()
args = m.group(2)
instance = utils.plugins.lookup_loader.get(module_name, basedir=basedir)
if instance is not None:
replacement = instance.run(args, inject=vars)
if not isinstance(replacement, basestring):
replacement = ",".join(replacement)
else:
replacement = m.group(0)
start, end = m.span()
done.append(raw[:start]) # Keep stuff leading up to token
done.append(replacement.rstrip()) # Append replacement value
raw = raw[end:] # Continue with remainder of string
return ''.join(done)
def template_ds(basedir, varname, vars): def template_ds(basedir, varname, vars):
''' templates a data structure by traversing it and substituting for other data structures ''' ''' templates a data structure by traversing it and substituting for other data structures '''
if isinstance(varname, basestring): if isinstance(varname, basestring):
m = _varFind(varname, vars) m = _varFind(basedir, varname, vars)
if not m: if not m:
return varname return varname
if m['start'] == 0 and m['end'] == len(varname): if m['start'] == 0 and m['end'] == len(varname):
@ -214,11 +235,21 @@ def template(basedir, text, vars, expand_lists=False):
text = text.decode('utf-8') text = text.decode('utf-8')
except UnicodeEncodeError: except UnicodeEncodeError:
pass # already unicode pass # already unicode
text = varReplace(unicode(text), vars, expand_lists=expand_lists) text = varReplace(basedir, unicode(text), vars, expand_lists=expand_lists)
if basedir is not None:
text = _varReplaceLookups(basedir, text, vars)
return text return text
class _jinja2_vars(object):
''' helper class to template all variable content before jinja2 sees it '''
def __init__(self, basedir, vars):
self.basedir = basedir
self.vars = vars
def __contains__(self, k):
return k in self.vars
def __getitem__(self, varname):
if varname not in self.vars:
raise KeyError("undefined variable: %s" % varname)
return template_ds(self.basedir, self.vars[varname], self.vars)
def template_from_file(basedir, path, vars): def template_from_file(basedir, path, vars):
''' run a file through the templating engine ''' ''' run a file through the templating engine '''
@ -257,7 +288,11 @@ def template_from_file(basedir, path, vars):
vars['ansible_managed'] = time.strftime(managed_str, vars['ansible_managed'] = time.strftime(managed_str,
time.localtime(os.path.getmtime(realpath))) time.localtime(os.path.getmtime(realpath)))
res = t.render(vars) # This line performs deep Jinja2 magic that uses the _jinja2_vars object for vars
# Ideally, this could use some API where setting shared=True and the object won't get
# passed through dict(o), but I have not found that yet.
res = jinja2.utils.concat(t.root_render_func(t.new_context(_jinja2_vars(basedir, vars), shared=True)))
if data.endswith('\n') and not res.endswith('\n'): if data.endswith('\n') and not res.endswith('\n'):
res = res + '\n' res = res + '\n'
return template(basedir, res, vars) return template(basedir, res, vars)

2
test/TestPlayBook.py
Unescape Escape View File

@ -202,7 +202,7 @@ class TestPlaybook(unittest.TestCase):
assert utils.jsonify(expected, format=True) == utils.jsonify(actual,format=True) assert utils.jsonify(expected, format=True) == utils.jsonify(actual,format=True)
assert len(EVENTS) == 44 assert len(EVENTS) == 60
def test_includes(self): def test_includes(self):
pb = os.path.join(self.test_dir, 'playbook-includer.yml') pb = os.path.join(self.test_dir, 'playbook-includer.yml')

52
test/TestUtils.py
Unescape Escape View File

@ -15,14 +15,14 @@ class TestUtils(unittest.TestCase):
'who': 'world', 'who': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world' assert res == 'hello world'
def test_varReplace_trailing_dollar(self): def test_varReplace_trailing_dollar(self):
template = '$what $who $' template = '$what $who $'
vars = dict(what='hello', who='world') vars = dict(what='hello', who='world')
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world $' assert res == 'hello world $'
def test_varReplace_multiple(self): def test_varReplace_multiple(self):
@ -32,7 +32,7 @@ class TestUtils(unittest.TestCase):
'who': 'world', 'who': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world' assert res == 'hello world'
@ -42,7 +42,7 @@ class TestUtils(unittest.TestCase):
'whoVar': 'world', 'whoVar': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
print res print res
assert res == 'hello world' assert res == 'hello world'
@ -52,7 +52,7 @@ class TestUtils(unittest.TestCase):
'who': 'world', 'who': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world!' assert res == 'hello world!'
@ -62,7 +62,7 @@ class TestUtils(unittest.TestCase):
'who': 'world', 'who': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world' assert res == 'hello world'
@ -72,7 +72,7 @@ class TestUtils(unittest.TestCase):
'who': 'world', 'who': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world}' assert res == 'hello world}'
@ -82,7 +82,7 @@ class TestUtils(unittest.TestCase):
'who': 'world', 'who': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == template assert res == template
@ -92,7 +92,7 @@ class TestUtils(unittest.TestCase):
'who': 'world', 'who': 'world',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world }' assert res == 'hello world }'
@ -104,7 +104,7 @@ class TestUtils(unittest.TestCase):
}, },
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
print res print res
assert res == template assert res == template
@ -117,7 +117,7 @@ class TestUtils(unittest.TestCase):
}, },
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world' assert res == 'hello world'
@ -130,7 +130,7 @@ class TestUtils(unittest.TestCase):
'what': 'hello', 'what': 'hello',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello 2' assert res == 'hello 2'
@ -140,7 +140,7 @@ class TestUtils(unittest.TestCase):
'who': u'wórld', 'who': u'wórld',
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == u'hello wórld' assert res == u'hello wórld'
@ -150,7 +150,7 @@ class TestUtils(unittest.TestCase):
'data': [ 'no-one', 'world' ] 'data': [ 'no-one', 'world' ]
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world' assert res == 'hello world'
@ -160,7 +160,7 @@ class TestUtils(unittest.TestCase):
'data': [ 'no-one', 'world' ] 'data': [ 'no-one', 'world' ]
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == template assert res == template
@ -170,7 +170,7 @@ class TestUtils(unittest.TestCase):
'data': [ 'no-one', 'world' ] 'data': [ 'no-one', 'world' ]
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == template assert res == template
@ -180,7 +180,7 @@ class TestUtils(unittest.TestCase):
'data': { 'no-one': 0, 'world': 1 } 'data': { 'no-one': 0, 'world': 1 }
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == template assert res == template
@ -190,7 +190,7 @@ class TestUtils(unittest.TestCase):
'data': [ 'no-one', {'msg': [ 'world'] } ] 'data': [ 'no-one', {'msg': [ 'world'] } ]
} }
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'hello world' assert res == 'hello world'
@ -201,7 +201,7 @@ class TestUtils(unittest.TestCase):
} }
template = '${foo}${bar}' template = '${foo}${bar}'
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'foobar' assert res == 'foobar'
def test_varReplace_escape_dot(self): def test_varReplace_escape_dot(self):
@ -214,7 +214,7 @@ class TestUtils(unittest.TestCase):
} }
template = '${hostvars.{test.example.com}.foo}' template = '${hostvars.{test.example.com}.foo}'
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'bar' assert res == 'bar'
def test_varReplace_list_join(self): def test_varReplace_list_join(self):
@ -227,7 +227,7 @@ class TestUtils(unittest.TestCase):
} }
template = 'yum pkg=${list} state=installed' template = 'yum pkg=${list} state=installed'
res = ansible.utils.varReplace(template, vars, expand_lists=True) res = ansible.utils.varReplace(None, template, vars, expand_lists=True)
assert res == 'yum pkg=foo,bar,baz state=installed' assert res == 'yum pkg=foo,bar,baz state=installed'
def test_varReplace_escaped_var(self): def test_varReplace_escaped_var(self):
@ -235,7 +235,7 @@ class TestUtils(unittest.TestCase):
'foo': 'bar', 'foo': 'bar',
} }
template = 'action \$foo' template = 'action \$foo'
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'action $foo' assert res == 'action $foo'
def test_varReplace_var_part(self): def test_varReplace_var_part(self):
@ -246,7 +246,7 @@ class TestUtils(unittest.TestCase):
'key': 'bar', 'key': 'bar',
} }
template = 'test ${foo.$key}' template = 'test ${foo.$key}'
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'test result' assert res == 'test result'
def test_varReplace_var_partial_part(self): def test_varReplace_var_partial_part(self):
@ -257,7 +257,7 @@ class TestUtils(unittest.TestCase):
'key': 'bar', 'key': 'bar',
} }
template = 'test ${foo.${key}baz}' template = 'test ${foo.${key}baz}'
res = ansible.utils.varReplace(template, vars) res = ansible.utils.varReplace(None, template, vars)
assert res == 'test result' assert res == 'test result'
def test_template_varReplace_iterated(self): def test_template_varReplace_iterated(self):
@ -274,14 +274,14 @@ class TestUtils(unittest.TestCase):
def test_varReplace_include(self): def test_varReplace_include(self):
template = 'hello $FILE(world) $LOOKUP(file, world)' template = 'hello $FILE(world) $LOOKUP(file, world)'
res = ansible.utils.template("test", template, {}) res = ansible.utils.template("test", template, {}, expand_lists=True)
assert res == u'hello world world' assert res == u'hello world world'
def test_varReplace_include_script(self): def test_varReplace_include_script(self):
template = 'hello $PIPE(echo world) $LOOKUP(pipe, echo world)' template = 'hello $PIPE(echo world) $LOOKUP(pipe, echo world)'
res = ansible.utils.template("test", template, {}) res = ansible.utils.template("test", template, {}, expand_lists=True)
assert res == u'hello world world' assert res == u'hello world world'

Write Preview
Loading…
Cancel
Save