Merge branch 'devel_cache_for_do_template_call' of https://github.com/Yannig/ansible into Yannig-devel_cache_for_do_template_call

lib/ansible/inventory/__init__.py

@@ -37,6 +37,8 @@ from ansible.plugins import vars_loader
 from ansible.utils.vars import combine_vars
 from ansible.parsing.utils.addresses import parse_address
 
+HOSTS_PATTERNS_CACHE = {}
+
 try:
     from __main__ import display
 except ImportError:
@@ -163,6 +165,11 @@ class Inventory(object):
         or applied subsets
         """
 
+        # Check if pattern already computed
+        pattern_hash = str(pattern)
+        if pattern_hash in HOSTS_PATTERNS_CACHE:
+            return HOSTS_PATTERNS_CACHE[pattern_hash]
+
         patterns = Inventory.split_host_pattern(pattern)
         hosts = self._evaluate_patterns(patterns)
 
@@ -177,6 +184,7 @@ class Inventory(object):
             if self._restriction is not None:
                 hosts = [ h for h in hosts if h in self._restriction ]
 
+        HOSTS_PATTERNS_CACHE[pattern_hash] = hosts
         return hosts
 
     @classmethod

lib/ansible/template/__init__.py

@@ -39,6 +39,11 @@ from ansible.template.template import AnsibleJ2Template
 from ansible.template.vars import AnsibleJ2Vars
 from ansible.utils.debug import debug
 
+try:
+    from hashlib import sha1
+except ImportError:
+    from sha import sha as sha1
+
 from numbers import Number
 
 __all__ = ['Templar']
@@ -122,6 +127,7 @@ class Templar:
         self._filters             = None
         self._tests               = None
         self._available_variables = variables
+        self._cached_result       = {}
 
         if loader:
             self._basedir = loader.get_basedir()
@@ -254,19 +260,24 @@ class Templar:
         '''
         Sets the list of template variables this Templar instance will use
         to template things, so we don't have to pass them around between
-        internal methods.
+        internal methods. We also clear the template cache here, as the variables
+        are being changed.
         '''
 
         assert isinstance(variables, dict)
         self._available_variables = variables
+        self._cached_result       = {}
 
-    def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True):
+    def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, convert_data=True, static_vars = ['']):
         '''
         Templates (possibly recursively) any given data as input. If convert_bare is
         set to True, the given data will be wrapped as a jinja2 variable ('{{foo}}')
         before being sent through the template engine. 
         '''
 
+        if fail_on_undefined is None:
+            fail_on_undefined = self._fail_on_undefined_errors
+
         # Don't template unsafe variables, instead drop them back down to
         # their constituent type.
         if hasattr(variable, '__UNSAFE__'):
@@ -298,18 +309,26 @@ class Templar:
                             elif resolved_val is None:
                                 return C.DEFAULT_NULL_REPRESENTATION
 
-                    result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides)
+                    # Using a cache in order to prevent template calls with already templated variables
+                    variable_hash = sha1(text_type(variable).encode('utf-8'))
+                    options_hash  = sha1((text_type(preserve_trailing_newlines) + text_type(escape_backslashes) + text_type(fail_on_undefined) + text_type(overrides)).encode('utf-8'))
+                    sha1_hash = variable_hash.hexdigest() + options_hash.hexdigest()
+                    if sha1_hash in self._cached_result:
+                        result = self._cached_result[sha1_hash]
+                    else:
+                        result = self._do_template(variable, preserve_trailing_newlines=preserve_trailing_newlines, escape_backslashes=escape_backslashes, fail_on_undefined=fail_on_undefined, overrides=overrides)
+                        if convert_data:
+                            # if this looks like a dictionary or list, convert it to such using the safe_eval method
+                            if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
+                              result.startswith("[") or result in ("True", "False"):
+                                eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
+                                if eval_results[1] is None:
+                                    result = eval_results[0]
+                                else:
+                                    # FIXME: if the safe_eval raised an error, should we do something with it?
+                                    pass
+                        self._cached_result[sha1_hash] = result
 
-                    if convert_data:
-                        # if this looks like a dictionary or list, convert it to such using the safe_eval method
-                        if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \
-                           result.startswith("[") or result in ("True", "False"):
-                            eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True)
-                            if eval_results[1] is None:
-                                result = eval_results[0]
-                            else:
-                                # FIXME: if the safe_eval raised an error, should we do something with it?
-                                pass
 
                 #return self._clean_data(result)
                 return result
@@ -321,7 +340,10 @@ class Templar:
                 # we don't use iteritems() here to avoid problems if the underlying dict
                 # changes sizes due to the templating, which can happen with hostvars
                 for k in variable.keys():
-                    d[k] = self.template(variable[k], preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides)
+                    if k not in static_vars:
+                        d[k] = self.template(variable[k], preserve_trailing_newlines=preserve_trailing_newlines, fail_on_undefined=fail_on_undefined, overrides=overrides)
+                    else:
+                        d[k] = variable[k]
                 return d
             else:
                 return variable

lib/ansible/vars/hostvars.py

@@ -28,6 +28,18 @@ from ansible import constants as C
 from ansible.inventory.host import Host
 from ansible.template import Templar
 
+STATIC_VARS = [
+  'inventory_hostname', 'inventory_hostname_short',
+  'inventory_file', 'inventory_dir', 'playbook_dir',
+  'ansible_play_hosts', 'play_hosts', 'groups', 'ungrouped', 'group_names',
+  'ansible_version', 'omit', 'role_names'
+]
+
+try:
+    from hashlib import sha1
+except ImportError:
+    from sha import sha as sha1
+
 __all__ = ['HostVars']
 
 # Note -- this is a Mapping, not a MutableMapping
@@ -39,6 +51,7 @@ class HostVars(collections.Mapping):
         self._loader = loader
         self._play = play
         self._variable_manager = variable_manager
+        self._cached_result = {}
 
         hosts = inventory.get_hosts(ignore_limits_and_restrictions=True)
 
@@ -68,8 +81,16 @@ class HostVars(collections.Mapping):
 
         host = self._lookup.get(host_name)
         data = self._variable_manager.get_vars(loader=self._loader, host=host, play=self._play, include_hostvars=False)
-        templar = Templar(variables=data, loader=self._loader)
-        return templar.template(data, fail_on_undefined=False)
+
+        # Using cache in order to avoid template call
+        sha1_hash = sha1(str(data).encode('utf-8')).hexdigest()
+        if sha1_hash in self._cached_result:
+            result = self._cached_result[sha1_hash]
+        else:
+            templar = Templar(variables=data, loader=self._loader)
+            result = templar.template(data, fail_on_undefined=False, static_vars=STATIC_VARS)
+            self._cached_result[sha1_hash] = result
+        return result
 
     def __contains__(self, host_name):
         item = self.get(host_name)