From 347b0260c32b35c2632e75e3e654715044771099 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Thu, 25 Oct 2012 21:18:08 -0400 Subject: [PATCH] Merge varReplace cleanup and move code into new file 'template.py' so it's easier to keep template code all together. --- lib/ansible/template.py | 267 ++++++++++++++++++++++++++++++++++++++++ lib/ansible/utils.py | 215 +------------------------------- 2 files changed, 272 insertions(+), 210 deletions(-) create mode 100644 lib/ansible/template.py diff --git a/lib/ansible/template.py b/lib/ansible/template.py new file mode 100644 index 00000000000..f88ab2b18ed --- /dev/null +++ b/lib/ansible/template.py @@ -0,0 +1,267 @@ +# (c) 2012, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import sys +import os +import shlex +import re +import codecs +import jinja2 +import yaml +import operator +import json +from ansible import errors +from ansible import __version__ +import ansible.constants as C +import time +import StringIO +import imp +import glob +import subprocess +import datetime +import pwd + +# TODO: refactor this file + +_LISTRE = re.compile(r"(\w+)\[(\d+)\]") + + +def _varFindLimitSpace(space, part, depth): + + # TODO: comments + + if space is None: + return space + if part[0] == '{' and part[-1] == '}': + part = part[1:-1] + part = varReplace(part, vars, depth=depth + 1) + if part in space: + space = space[part] + elif "[" in part: + m = _LISTRE.search(part) + if not m: + return None + else: + try: + space = space[m.group(1)][int(m.group(2))] + except (KeyError, IndexError): + return None + else: + return None + return space + +def _varFind(text, vars, depth=0): + + # TODO: comments + + start = text.find("$") + if start == -1: + return None + # $ as last character + if start + 1 == len(text): + return None + # Escaped var + if start > 0 and text[start - 1] == '\\': + return {'replacement': '$', 'start': start - 1, 'end': start + 1} + + var_start = start + 1 + if text[var_start] == '{': + is_complex = True + brace_level = 1 + var_start += 1 + else: + is_complex = False + brace_level = 0 + end = var_start + path = [] + part_start = (var_start, brace_level) + space = vars + while end < len(text) and ((is_complex and brace_level > 0) or not is_complex): + if text[end].isalnum() or text[end] == '_': + pass + elif is_complex and text[end] == '{': + brace_level += 1 + elif is_complex and text[end] == '}': + brace_level -= 1 + elif is_complex and text[end] in ('$', '[', ']'): + pass + elif is_complex and text[end] == '.': + if brace_level == part_start[1]: + space = _varFindLimitSpace(space, text[part_start[0]:end], depth) + part_start = (end + 1, brace_level) + else: + break + end += 1 + var_end = end + if is_complex: + var_end -= 1 + if text[var_end] != '}' or brace_level != 0: + return None + if var_end == part_start[0]: + return None + space = _varFindLimitSpace(space, text[part_start[0]:var_end], depth) + return {'replacement': space, 'start': start, 'end': end} + +def varReplace(raw, vars, depth=0, expand_lists=False): + ''' Perform variable replacement of $variables in string raw using vars dictionary ''' + # this code originally from yum + + if (depth > 20): + raise errors.AnsibleError("template recursion depth exceeded") + + done = [] # Completed chunks to return + + while raw: + m = _varFind(raw, vars, depth) + if not m: + done.append(raw) + break + + # Determine replacement value (if unknown variable then preserve + # original) + + replacement = m['replacement'] + if expand_lists and isinstance(replacement, (list, tuple)): + replacement = ",".join(replacement) + if isinstance(replacement, (str, unicode)): + replacement = varReplace(replacement, vars, depth=depth+1, expand_lists=expand_lists) + if replacement is None: + replacement = raw[m['start']:m['end']] + + start, end = m['start'], m['end'] + done.append(raw[:start]) # Keep stuff leading up to token + done.append(unicode(replacement)) # Append replacement value + raw = raw[end:] # Continue with remainder of string + + return ''.join(done) + +_FILEPIPECRE = re.compile(r"\$(?PFILE|PIPE)\(([^\)]+)\)") +def _varReplaceFilesAndPipes(basedir, raw): + done = [] # Completed chunks to return + + while raw: + m = _FILEPIPECRE.search(raw) + if not m: + done.append(raw) + break + + # Determine replacement value (if unknown variable then preserve + # original) + + replacement = m.group() + if m.group(1) == "FILE": + from ansible import utils + path = utils.path_dwim(basedir, m.group(2)) + try: + f = open(path, "r") + replacement = f.read() + f.close() + except IOError: + raise errors.AnsibleError("$FILE(%s) failed" % path) + elif m.group(1) == "PIPE": + p = subprocess.Popen(m.group(2), shell=True, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode == 0: + replacement = stdout + else: + raise errors.AnsibleError("$PIPE(%s) returned %d" % (m.group(2), p.returncode)) + + start, end = m.span() + done.append(raw[:start]) # Keep stuff leading up to token + done.append(replacement.rstrip()) # Append replacement value + raw = raw[end:] # Continue with remainder of string + + return ''.join(done) + +def varReplaceWithItems(basedir, varname, vars): + ''' helper function used by with_items ''' + + if isinstance(varname, basestring): + m = _varFind(varname, vars) + if not m: + return varname + if m['start'] == 0 and m['end'] == len(varname): + try: + return varReplaceWithItems(basedir, m['replacement'], vars) + except VarNotFoundException: + return varname + else: + return template(basedir, varname, vars) + elif isinstance(varname, (list, tuple)): + return [varReplaceWithItems(basedir, v, vars) for v in varname] + elif isinstance(varname, dict): + d = {} + for (k, v) in varname.iteritems(): + d[k] = varReplaceWithItems(basedir, v, vars) + return d + else: + return varname + +def template(basedir, text, vars, expand_lists=False): + ''' run a text buffer through the templating engine until it no longer changes ''' + + prev_text = '' + try: + text = text.decode('utf-8') + except UnicodeEncodeError: + pass # already unicode + text = varReplace(unicode(text), vars, expand_lists=expand_lists) + text = _varReplaceFilesAndPipes(basedir, text) + return text + +def template_from_file(basedir, path, vars): + ''' run a file through the templating engine ''' + + from ansible import utils + realpath = utils.path_dwim(basedir, path) + loader=jinja2.FileSystemLoader([basedir,os.path.dirname(realpath)]) + environment = jinja2.Environment(loader=loader, trim_blocks=True) + environment.filters['to_json'] = json.dumps + environment.filters['from_json'] = json.loads + environment.filters['to_yaml'] = yaml.dump + environment.filters['from_yaml'] = yaml.load + try: + data = codecs.open(realpath, encoding="utf8").read() + except UnicodeDecodeError: + raise errors.AnsibleError("unable to process as utf-8: %s" % realpath) + except: + raise errors.AnsibleError("unable to read %s" % realpath) + t = environment.from_string(data) + vars = vars.copy() + try: + template_uid = pwd.getpwuid(os.stat(realpath).st_uid).pw_name + except: + template_uid = os.stat(realpath).st_uid + vars['template_host'] = os.uname()[1] + vars['template_path'] = realpath + vars['template_mtime'] = datetime.datetime.fromtimestamp(os.path.getmtime(realpath)) + vars['template_uid'] = template_uid + + managed_default = C.DEFAULT_MANAGED_STR + managed_str = managed_default.format( + host = vars['template_host'], + uid = vars['template_uid'], + file = vars['template_path'] + ) + vars['ansible_managed'] = time.strftime(managed_str, + time.localtime(os.path.getmtime(realpath))) + + res = t.render(vars) + if data.endswith('\n') and not res.endswith('\n'): + res = res + '\n' + return template(basedir, res, vars) + diff --git a/lib/ansible/utils.py b/lib/ansible/utils.py index 6c8501a6df6..92d1ef9c3dc 100644 --- a/lib/ansible/utils.py +++ b/lib/ansible/utils.py @@ -26,6 +26,7 @@ import optparse import operator from ansible import errors from ansible import __version__ +from ansible import template as ans_template import ansible.constants as C import time import StringIO @@ -35,7 +36,6 @@ import subprocess import stat import termios import tty -from multiprocessing import Manager import datetime import pwd @@ -218,229 +218,24 @@ def parse_json(raw_data): return { "failed" : True, "parsed" : False, "msg" : orig_data } return results -_LISTRE = re.compile(r"(\w+)\[(\d+)\]") - -def _varFindLimitSpace(space, part, depth): - if space is None: - return space - if part[0] == '{' and part[-1] == '}': - part = part[1:-1] - part = varReplace(part, vars, depth=depth + 1) - if part in space: - space = space[part] - elif "[" in part: - m = _LISTRE.search(part) - if not m: - return None - else: - try: - space = space[m.group(1)][int(m.group(2))] - except (KeyError, IndexError): - return None - else: - return None - return space - -def _varFind(text, vars, depth=0): - start = text.find("$") - if start == -1: - return None - # $ as last character - if start + 1 == len(text): - return None - # Escaped var - if start > 0 and text[start - 1] == '\\': - return {'replacement': '$', 'start': start - 1, 'end': start + 1} - - var_start = start + 1 - if text[var_start] == '{': - is_complex = True - brace_level = 1 - var_start += 1 - else: - is_complex = False - brace_level = 0 - end = var_start - path = [] - part_start = (var_start, brace_level) - space = vars - while end < len(text) and ((is_complex and brace_level > 0) or not is_complex): - if text[end].isalnum() or text[end] == '_': - pass - elif is_complex and text[end] == '{': - brace_level += 1 - elif is_complex and text[end] == '}': - brace_level -= 1 - elif is_complex and text[end] in ('$', '[', ']'): - pass - elif is_complex and text[end] == '.': - if brace_level == part_start[1]: - space = _varFindLimitSpace(space, text[part_start[0]:end], depth) - part_start = (end + 1, brace_level) - else: - break - end += 1 - var_end = end - if is_complex: - var_end -= 1 - if text[var_end] != '}' or brace_level != 0: - return None - if var_end == part_start[0]: - return None - space = _varFindLimitSpace(space, text[part_start[0]:var_end], depth) - return {'replacement': space, 'start': start, 'end': end} - def varReplace(raw, vars, depth=0, expand_lists=False): ''' Perform variable replacement of $variables in string raw using vars dictionary ''' - # this code originally from yum - - if (depth > 20): - raise errors.AnsibleError("template recursion depth exceeded") - - done = [] # Completed chunks to return - - while raw: - m = _varFind(raw, vars, depth) - if not m: - done.append(raw) - break - - # Determine replacement value (if unknown variable then preserve - # original) - - replacement = m['replacement'] - if expand_lists and isinstance(replacement, (list, tuple)): - replacement = ",".join(replacement) - if isinstance(replacement, (str, unicode)): - replacement = varReplace(replacement, vars, depth=depth+1, expand_lists=expand_lists) - if replacement is None: - replacement = raw[m['start']:m['end']] - - start, end = m['start'], m['end'] - done.append(raw[:start]) # Keep stuff leading up to token - done.append(unicode(replacement)) # Append replacement value - raw = raw[end:] # Continue with remainder of string - - return ''.join(done) - -_FILEPIPECRE = re.compile(r"\$(?PFILE|PIPE)\(([^\)]+)\)") -def varReplaceFilesAndPipes(basedir, raw): - done = [] # Completed chunks to return - - while raw: - m = _FILEPIPECRE.search(raw) - if not m: - done.append(raw) - break - - # Determine replacement value (if unknown variable then preserve - # original) - - replacement = m.group() - if m.group(1) == "FILE": - path = path_dwim(basedir, m.group(2)) - try: - f = open(path, "r") - replacement = f.read() - f.close() - except IOError: - raise errors.AnsibleError("$FILE(%s) failed" % path) - elif m.group(1) == "PIPE": - p = subprocess.Popen(m.group(2), shell=True, stdout=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if p.returncode == 0: - replacement = stdout - else: - raise errors.AnsibleError("$PIPE(%s) returned %d" % (m.group(2), p.returncode)) - - start, end = m.span() - done.append(raw[:start]) # Keep stuff leading up to token - done.append(replacement.rstrip()) # Append replacement value - raw = raw[end:] # Continue with remainder of string - - return ''.join(done) + return ans_template.varReplace(raw, vars, depth=depth, expand_lists=expand_lists) def varReplaceWithItems(basedir, varname, vars): ''' helper function used by with_items ''' - - if isinstance(varname, basestring): - m = _varFind(varname, vars) - if not m: - return varname - if m['start'] == 0 and m['end'] == len(varname): - try: - return varReplaceWithItems(basedir, m['replacement'], vars) - except VarNotFoundException: - return varname - else: - return template(basedir, varname, vars) - elif isinstance(varname, (list, tuple)): - return [varReplaceWithItems(basedir, v, vars) for v in varname] - elif isinstance(varname, dict): - d = {} - for (k, v) in varname.iteritems(): - d[k] = varReplaceWithItems(basedir, v, vars) - return d - else: - return varname - + return ans_template.varReplaceWithItems(basedir, varname, vars) def template(basedir, text, vars, expand_lists=False): ''' run a text buffer through the templating engine until it no longer changes ''' - - prev_text = '' - try: - text = text.decode('utf-8') - except UnicodeEncodeError: - pass # already unicode - text = varReplace(unicode(text), vars, expand_lists=expand_lists) - text = varReplaceFilesAndPipes(basedir, text) - return text + return ans_template.template(basedir, text, vars, expand_lists=expand_lists) def template_from_file(basedir, path, vars): ''' run a file through the templating engine ''' - - realpath = path_dwim(basedir, path) - loader=jinja2.FileSystemLoader([basedir,os.path.dirname(realpath)]) - environment = jinja2.Environment(loader=loader, trim_blocks=True) - environment.filters['to_json'] = json.dumps - environment.filters['from_json'] = json.loads - environment.filters['to_yaml'] = yaml.dump - environment.filters['from_yaml'] = yaml.load - try: - data = codecs.open(realpath, encoding="utf8").read() - except UnicodeDecodeError: - raise errors.AnsibleError("unable to process as utf-8: %s" % realpath) - except: - raise errors.AnsibleError("unable to read %s" % realpath) - t = environment.from_string(data) - vars = vars.copy() - try: - template_uid = pwd.getpwuid(os.stat(realpath).st_uid).pw_name - except: - template_uid = os.stat(realpath).st_uid - vars['template_host'] = os.uname()[1] - vars['template_path'] = realpath - vars['template_mtime'] = datetime.datetime.fromtimestamp(os.path.getmtime(realpath)) - vars['template_uid'] = template_uid - - managed_default = C.DEFAULT_MANAGED_STR - managed_str = managed_default.format( - host = vars['template_host'], - uid = vars['template_uid'], - file = vars['template_path'] - ) - vars['ansible_managed'] = time.strftime(managed_str, - time.localtime(os.path.getmtime(realpath))) - - res = t.render(vars) - if data.endswith('\n') and not res.endswith('\n'): - res = res + '\n' - return template(basedir, res, vars) + return ans_template.template_from_file(basedir, path, vars) def parse_yaml(data): ''' convert a yaml string to a data structure ''' - return yaml.load(data) def parse_yaml_from_file(path):