[GCE] Caching support for inventory script. (#18093)

* [GCE] Caching support for inventory script.

The GCE inventory script now supports reading from a cache rather than making the request each time.  The format of the list and host output have not changed.

On script execution, the cache is checked to see if it older than 'cache_max_age', and if so, it is rebuilt (it can also be explicity rebuilt).

To support this functionality, the following have been added.

* Config file (gce.ini) changes: A new 'cache' section has been added to the config file, with 'cache_path' and 'cache_max_age' options to allow for configuration.  There are intelligent defaults in place if that section and options are not found in the configuration file.

* Command line argument: A new --refresh-cache argument has been added to force the cache to be rebuild.

* A CloudInventoryCache class, contained in the same file has been added.  As a seperate class, it allowed for testing (unit tests not included in this PR) and hopefully could be re-used in the future (it contains borrowed code from other inventory scripts)

* load_inventory_from_cache and do_api_calls_and_update_cache methods (, which were largely lifted from other inventory scripts, in a hope to promote consistency in the future) to determine if the cache is fresh and rebuild if necessary.

* A 'main' check, to support the script being imported and testable.

A new dictionary has been added to the list output, located at ['_meta']['stats'] that informs if the cache was used and how long it took to load the inventory (in 'cache_used' and 'inventory_load_time', respectively).

* fixed default value error; change cache time to 300

contrib/inventory/gce.ini

@@ -37,6 +37,7 @@
 # exist in your PYTHONPATH and be picked up automatically with an import
 # statement in the inventory script.  However, you can specify an absolute
 # path to the secrets.py file with 'libcloud_secrets' parameter.
+# This option will be deprecated in a future release.
 libcloud_secrets = 
 
 # If you are not going to use a 'secrets.py' file, you can set the necessary
@@ -58,3 +59,11 @@ gce_project_id =
 # The INVENTORY_IP_TYPE environment variable will override this value.
 inventory_ip_type =
 
+[cache]
+# directory in which cache should be created
+cache_path = ~/.ansible/tmp
+
+# The number of seconds a cache file is considered valid. After this many
+# seconds, a new API call will be made, and the cache file will be updated.
+# To disable the cache, set this value to 0
+cache_max_age = 300

contrib/inventory/gce.py

@@ -69,8 +69,8 @@ Examples:
   $ contrib/inventory/gce.py --host my_instance
 
 Author: Eric Johnson <erjohnso@google.com>
-Contributors: Matt Hite <mhite@hotmail.com>
-Version: 0.0.2
+Contributors: Matt Hite <mhite@hotmail.com>, Tom Melendez <supertom@google.com>
+Version: 0.0.3
 '''
 
 __requires__ = ['pycrypto>=2.6']
@@ -89,6 +89,9 @@ USER_AGENT_VERSION="v2"
 import sys
 import os
 import argparse
+
+from time import time
+
 import ConfigParser
 
 import logging
@@ -107,8 +110,57 @@ except:
     sys.exit("GCE inventory script requires libcloud >= 0.13")
 
 
+class CloudInventoryCache(object):
+    def __init__(self, cache_name='ansible-cloud-cache', cache_path='/tmp',
+                 cache_max_age=300):
+        cache_dir = os.path.expanduser(cache_path)
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+        self.cache_path_cache = os.path.join(cache_dir, cache_name)
+
+        self.cache_max_age = cache_max_age
+
+    def is_valid(self, max_age=None):
+        ''' Determines if the cache files have expired, or if it is still valid '''
+
+        if max_age is None:
+            max_age = self.cache_max_age
+
+        if os.path.isfile(self.cache_path_cache):
+            mod_time = os.path.getmtime(self.cache_path_cache)
+            current_time = time()
+            if (mod_time + max_age) > current_time:
+                return True
+
+        return False
+
+    def get_all_data_from_cache(self, filename=''):
+        ''' Reads the JSON inventory from the cache file. Returns Python dictionary. '''
+
+        data = ''
+        if not filename:
+            filename = self.cache_path_cache
+        with open(filename, 'r') as cache:
+            data = cache.read()
+        return json.loads(data)
+
+    def write_to_cache(self, data, filename=''):
+        ''' Writes data to file as JSON.  Returns True. '''
+        if not filename:
+            filename = self.cache_path_cache
+        json_data = json.dumps(data)
+        with open(filename, 'w') as cache:
+            cache.write(json_data)
+        return True
+
+
 class GceInventory(object):
     def __init__(self):
+        # Cache object
+        self.cache = None
+        # dictionary containing inventory read from disk
+        self.inventory = {}
+
         # Read settings and parse CLI arguments
         self.parse_cli_args()
         self.config = self.get_config()
@@ -117,22 +169,36 @@ class GceInventory(object):
         if self.ip_type:
             self.ip_type = self.ip_type.lower()
 
+        # Cache management
+        start_inventory_time = time()
+        cache_used = False
+        if self.args.refresh_cache or not self.cache.is_valid():
+            self.do_api_calls_update_cache()
+        else:
+            self.load_inventory_from_cache()
+            cache_used = True
+            self.inventory['_meta']['stats'] = {'use_cache': True}
+        self.inventory['_meta']['stats'] = {
+            'inventory_load_time': time() - start_inventory_time,
+            'cache_used': cache_used
+        }
+
         # Just display data for specific host
         if self.args.host:
-            print(self.json_format_dict(self.node_to_dict(
-                    self.get_instance(self.args.host)),
-                    pretty=self.args.pretty))
-            sys.exit(0)
-
-        zones = self.parse_env_zones()
-
-        # Otherwise, assume user wants all instances grouped
-        print(self.json_format_dict(self.group_instances(zones),
-            pretty=self.args.pretty))
+            print(self.json_format_dict(
+                self.inventory['_meta']['hostvars'][self.args.host],
+                pretty=self.args.pretty))
+        else:
+            # Otherwise, assume user wants all instances grouped
+            zones = self.parse_env_zones()
+            print(self.json_format_dict(self.inventory,
+                                        pretty=self.args.pretty))
         sys.exit(0)
 
     def get_config(self):
         """
+        Reads the settings from the gce.ini file.
+
         Populates a SafeConfigParser object with defaults and
         attempts to read an .ini-style configuration from the filename
         specified in GCE_INI_PATH. If the environment variable is
@@ -153,11 +219,15 @@ class GceInventory(object):
             'gce_project_id': '',
             'libcloud_secrets': '',
             'inventory_ip_type': '',
+            'cache_path': '~/.ansible/tmp',
+            'cache_max_age': '300'
         })
         if 'gce' not in config.sections():
             config.add_section('gce')
         if 'inventory' not in config.sections():
             config.add_section('inventory')
+        if 'cache' not in config.sections():
+            config.add_section('cache')
 
         config.read(gce_ini_path)
 
@@ -173,6 +243,14 @@ class GceInventory(object):
             if states:
                 self.instance_states = states.split(',')
 
+        # Caching
+        cache_path = config.get('cache', 'cache_path')
+        cache_max_age = config.getint('cache', 'cache_max_age')
+        # TOOD(supertom): support project-specific caches
+        cache_name = 'ansible-gce.cache'
+        self.cache = CloudInventoryCache(cache_path=cache_path,
+                                         cache_max_age=cache_max_age,
+                                         cache_name=cache_name)
         return config
 
     def get_inventory_options(self):
@@ -252,6 +330,9 @@ class GceInventory(object):
                            help='Get all information about an instance')
         parser.add_argument('--pretty', action='store_true', default=False,
                            help='Pretty format (default: False)')
+        parser.add_argument(
+            '--refresh-cache', action='store_true', default=False,
+            help='Force refresh of cache by making API requests (default: False - use cache files)')
         self.args = parser.parse_args()
 
 
@@ -290,12 +371,24 @@ class GceInventory(object):
             'ansible_ssh_host': ssh_host
         }
 
-    def get_instance(self, instance_name):
-        '''Gets details about a specific instance '''
+    def load_inventory_from_cache(self):
+        ''' Loads inventory from JSON on disk. '''
+
         try:
-            return self.driver.ex_get_node(instance_name)
+            self.inventory = self.cache.get_all_data_from_cache()
+            hosts = self.inventory['_meta']['hostvars']
         except Exception as e:
-            return None
+            print(
+                "Invalid inventory file %s.  Please rebuild with -refresh-cache option."
+                % (self.cache.cache_path_cache))
+            raise
+
+    def do_api_calls_update_cache(self):
+        ''' Do API calls and save data in cache. '''
+        zones = self.parse_env_zones()
+        data = self.group_instances(zones)
+        self.cache.write_to_cache(data)
+        self.inventory = data
 
     def group_instances(self, zones=None):
         '''Group all instances'''
@@ -369,6 +462,6 @@ class GceInventory(object):
         else:
             return json.dumps(data)
 
-
 # Run the script
-GceInventory()
+if __name__ == '__main__':
+    GceInventory()