Merge pull request #10946 from izhukov/inventory-DO-api-v2

Update DigitalOcean dynamic inventory to API v2

plugins/inventory/digital_ocean.ini

@@ -3,12 +3,11 @@
 
 [digital_ocean]
 
-# The module needs your DigitalOcean Client ID and API Key.
+# The module needs your DigitalOcean API Token.
-# These may also be specified on the command line via --client-id and --api-key
+# It may also be specified on the command line via --api-token
-# or via the environment variables DO_CLIENT_ID and DO_API_KEY
+# or via the environment variables DO_API_TOKEN or DO_API_KEY
 #
-#client_id = abcdefg123456
+#api_token = 123456abcdefg
-#api_key = 123456abcdefg
 
 
 # API calls to DigitalOcean may be slow. For this reason, we cache the results

plugins/inventory/digital_ocean.py

@@ -24,12 +24,12 @@ found.  You can force this script to use the cache with --force-cache.
 Configuration is read from `digital_ocean.ini`, then from environment variables,
 then and command-line arguments.
 
-Most notably, the DigitalOcean Client ID and API Key must be specified.  They
+Most notably, the DigitalOcean API Token must be specified. It can be specified
-can be specified in the INI file or with the following environment variables:
+in the INI file or with the following environment variables:
-    export DO_CLIENT_ID='DO123' DO_API_KEY='abc123'
+    export DO_API_TOKEN='abc123' or
+    export DO_API_KEY='abc123'
 
-Alternatively, they can be passed on the command-line with --client-id and
+Alternatively, it can be passed on the command-line with --api-token.
---api-key.
 
 If you specify DigitalOcean credentials in the INI file, a handy way to
 get them into your environment (e.g., to use the digital_ocean module)
@@ -43,25 +43,30 @@ The following groups are generated from --list:
  - image_ID
  - image_NAME
  - distro_NAME  (distribution NAME from image)
- - region_ID
  - region_NAME
- - size_ID
  - size_NAME
  - status_STATUS
 
 When run against a specific host, this script returns the following variables:
+ - do_backup_ids
  - do_created_at
- - do_distroy
+ - do_disk
+ - do_features - list
  - do_id
- - do_image
+ - do_image - object
- - do_image_id
  - do_ip_address
+ - do_kernel - object
+ - do_locked
+ - de_memory
  - do_name
- - do_region
+ - do_networks - object
- - do_region_id
+ - do_next_backup_window
- - do_size
+ - do_region - object
- - do_size_id
+ - do_size - object
+ - do_size_slug
+ - do_snapshot_ids - list
  - do_status
+ - do_vcpus
 
 -----
 ```
@@ -70,8 +75,9 @@ usage: digital_ocean.py [-h] [--list] [--host HOST] [--all]
                                  [--ssh-keys] [--domains] [--pretty]
                                  [--cache-path CACHE_PATH]
                                  [--cache-max_age CACHE_MAX_AGE]
-                                 [--refresh-cache] [--client-id CLIENT_ID]
+                                 [--force-cache]
-                                 [--api-key API_KEY]
+                                 [--refresh-cache]
+                                 [--api-token API_TOKEN]
 
 Produce an Ansible Inventory file based on DigitalOcean credentials
 
@@ -93,12 +99,11 @@ optional arguments:
                         Path to the cache files (default: .)
   --cache-max_age CACHE_MAX_AGE
                         Maximum age of the cached items (default: 0)
+  --force-cache         Only use data from the cache
   --refresh-cache       Force refresh of cache by making API requests to
                         DigitalOcean (default: False - use cache files)
-  --client-id CLIENT_ID, -c CLIENT_ID
+  --api-token API_TOKEN, -a API_TOKEN
-                        DigitalOcean Client ID
+                        DigitalOcean API Token
-  --api-key API_KEY, -a API_KEY
-                        DigitalOcean API Key
 ```
 
 '''
@@ -157,7 +162,6 @@ class DigitalOceanInventory(object):
         # DigitalOceanInventory data
         self.data = {}      # All DigitalOcean data
         self.inventory = {} # Ansible Inventory
-        self.index = {}     # Various indices of Droplet metadata
 
         # Define defaults
         self.cache_path = '.'
@@ -169,50 +173,62 @@ class DigitalOceanInventory(object):
         self.read_cli_args()
 
         # Verify credentials were set
-        if not hasattr(self, 'client_id') or not hasattr(self, 'api_key'):
+        if not hasattr(self, 'api_token'):
-            print '''Could not find values for DigitalOcean client_id and api_key.
+            print '''Could not find values for DigitalOcean api_token.
-They must be specified via either ini file, command line argument (--client-id and --api-key),
+They must be specified via either ini file, command line argument (--api-token),
-or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
+or environment variables (DO_API_TOKEN)'''
             sys.exit(-1)
 
         # env command, show DigitalOcean credentials
         if self.args.env:
-            print "DO_CLIENT_ID=%s DO_API_KEY=%s" % (self.client_id, self.api_key)
+            print "DO_API_TOKEN=%s" % self.api_token
             sys.exit(0)
 
         # Manage cache
         self.cache_filename = self.cache_path + "/ansible-digital_ocean.cache"
         self.cache_refreshed = False
 
-        if not self.args.force_cache and self.args.refresh_cache or not self.is_cache_valid():
+        if self.is_cache_valid:
-            self.load_all_data_from_digital_ocean()
-        else:
             self.load_from_cache()
             if len(self.data) == 0:
                 if self.args.force_cache:
                     print '''Cache is empty and --force-cache was specified'''
                     sys.exit(-1)
-                self.load_all_data_from_digital_ocean()
+
-            else:
+        self.manager = DoManager(None, self.api_token, api_version=2)
-                # We always get fresh droplets for --list, --host, --all, and --droplets
-                # unless --force-cache is specified
-                if not self.args.force_cache and (
-                   self.args.list or self.args.host or self.args.all or self.args.droplets):
-                    self.load_droplets_from_digital_ocean()
 
         # Pick the json_data to print based on the CLI command
-        if self.args.droplets:   json_data = { 'droplets': self.data['droplets'] }
+        if self.args.droplets:
-        elif self.args.regions:  json_data = { 'regions':  self.data['regions'] }
+            self.load_from_digital_ocean('droplets')
-        elif self.args.images:   json_data = { 'images':   self.data['images'] }
+            json_data = {'droplets': self.data['droplets']}
-        elif self.args.sizes:    json_data = { 'sizes':    self.data['sizes'] }
+        elif self.args.regions:
-        elif self.args.ssh_keys: json_data = { 'ssh_keys': self.data['ssh_keys'] }
+            self.load_from_digital_ocean('regions')
-        elif self.args.domains:  json_data = { 'domains':  self.data['domains'] }
+            json_data = {'regions': self.data['regions']}
-        elif self.args.all:      json_data = self.data
+        elif self.args.images:
-
+            self.load_from_digital_ocean('images')
-        elif self.args.host:     json_data = self.load_droplet_variables_for_host()
+            json_data = {'images': self.data['images']}
+        elif self.args.sizes:
+            self.load_from_digital_ocean('sizes')
+            json_data = {'sizes': self.data['sizes']}
+        elif self.args.ssh_keys:
+            self.load_from_digital_ocean('ssh_keys')
+            json_data = {'ssh_keys': self.data['ssh_keys']}
+        elif self.args.domains:
+            self.load_from_digital_ocean('domains')
+            json_data = {'domains': self.data['domains']}
+        elif self.args.all:
+            self.load_from_digital_ocean()
+            json_data = self.data
+        elif self.args.host:
+            json_data = self.load_droplet_variables_for_host()
         else:    # '--list' this is last to make it default
+            self.load_from_digital_ocean('droplets')
+            self.build_inventory()
             json_data = self.inventory
 
+        if self.cache_refreshed:
+            self.write_to_cache()
+
         if self.args.pretty:
             print json.dumps(json_data, sort_keys=True, indent=2)
         else:
@@ -230,10 +246,8 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
         config.read(os.path.dirname(os.path.realpath(__file__)) + '/digital_ocean.ini')
 
         # Credentials
-        if config.has_option('digital_ocean', 'client_id'):
+        if config.has_option('digital_ocean', 'api_token'):
-            self.client_id = config.get('digital_ocean', 'client_id')
+            self.api_token = config.get('digital_ocean', 'api_token')
-        if config.has_option('digital_ocean', 'api_key'):
-            self.api_key = config.get('digital_ocean', 'api_key')
 
         # Cache related
         if config.has_option('digital_ocean', 'cache_path'):
@@ -245,8 +259,10 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
     def read_environment(self):
         ''' Reads the settings from environment variables '''
         # Setup credentials
-        if os.getenv("DO_CLIENT_ID"): self.client_id = os.getenv("DO_CLIENT_ID")
+        if os.getenv("DO_API_TOKEN"):
-        if os.getenv("DO_API_KEY"):   self.api_key = os.getenv("DO_API_KEY")
+            self.api_token = os.getenv("DO_API_TOKEN")
+        if os.getenv("DO_API_KEY"):
+            self.api_token = os.getenv("DO_API_KEY")
 
 
     def read_cli_args(self):
@@ -269,22 +285,21 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
         parser.add_argument('--cache-path', action='store', help='Path to the cache files (default: .)')
         parser.add_argument('--cache-max_age', action='store', help='Maximum age of the cached items (default: 0)')
         parser.add_argument('--force-cache', action='store_true', default=False, help='Only use data from the cache')
-        parser.add_argument('--refresh-cache','-r', action='store_true', default=False, help='Force refresh of cache by making API requests to DigitalOcean (default: False - use cache files)')
+        parser.add_argument('--refresh-cache','-r', action='store_true', default=False,
+                            help='Force refresh of cache by making API requests to DigitalOcean (default: False - use cache files)')
 
-        parser.add_argument('--env','-e', action='store_true', help='Display DO_CLIENT_ID and DO_API_KEY')
+        parser.add_argument('--env','-e', action='store_true', help='Display DO_API_TOKEN')
-        parser.add_argument('--client-id','-c', action='store', help='DigitalOcean Client ID')
+        parser.add_argument('--api-token','-a', action='store', help='DigitalOcean API Token')
-        parser.add_argument('--api-key','-a', action='store', help='DigitalOcean API Key')
 
         self.args = parser.parse_args()
 
-        if self.args.client_id: self.client_id = self.args.client_id
+        if self.args.api_token:
-        if self.args.api_key: self.api_key = self.args.api_key
+            self.api_token = self.args.api_token
-        if self.args.cache_path: self.cache_path = self.args.cache_path
-        if self.args.cache_max_age: self.cache_max_age = self.args.cache_max_age
 
         # Make --list default if none of the other commands are specified
-        if (not self.args.droplets and not self.args.regions and not self.args.images and
+        if (not self.args.droplets and not self.args.regions and
-            not self.args.sizes and not self.args.ssh_keys and not self.args.domains and
+                not self.args.images and not self.args.sizes and
+                not self.args.ssh_keys and not self.args.domains and
                 not self.args.all and not self.args.host):
             self.args.list = True
 
@@ -293,46 +308,34 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
     # Data Management
     ###########################################################################
 
-    def load_all_data_from_digital_ocean(self):
+    def load_from_digital_ocean(self, resource=None):
-        ''' Use dopy to get all the information from DigitalOcean and save data in cache files '''
+        '''Get JSON from DigitalOcean API'''
-        manager  = DoManager(self.client_id, self.api_key)
+        if self.args.force_cache:
-
+            return
-        self.data = {}
+        # We always get fresh droplets
-        self.data['droplets'] = self.sanitize_list(manager.all_active_droplets())
+        if self.is_cache_valid() and not (resource=='droplets' or resource is None):
-        self.data['regions']  = self.sanitize_list(manager.all_regions())
+            return
-        self.data['images']   = self.sanitize_list(manager.all_images(filter=None))
+        if self.args.refresh_cache:
-        self.data['sizes']    = self.sanitize_list(manager.sizes())
+            resource=None
-        self.data['ssh_keys'] = self.sanitize_list(manager.all_ssh_keys())
+
-        self.data['domains']  = self.sanitize_list(manager.all_domains())
+        if resource == 'droplets' or resource is None:
-
+            self.data['droplets'] = self.manager.all_active_droplets()
-        self.index = {}
+            self.cache_refreshed = True
-        self.index['region_to_name']  = self.build_index(self.data['regions'], 'id', 'name')
+        if resource == 'regions' or resource is None:
-        self.index['size_to_name']    = self.build_index(self.data['sizes'], 'id', 'name')
+            self.data['regions'] = self.manager.all_regions()
-        self.index['image_to_name']   = self.build_index(self.data['images'], 'id', 'name')
+            self.cache_refreshed = True
-        self.index['image_to_distro'] = self.build_index(self.data['images'], 'id', 'distribution')
+        if resource == 'images' or resource is None:
-        self.index['host_to_droplet'] = self.build_index(self.data['droplets'], 'ip_address', 'id', False)
+            self.data['images'] = self.manager.all_images(filter=None)
-
+            self.cache_refreshed = True
-        self.build_inventory()
+        if resource == 'sizes' or resource is None:
-
+            self.data['sizes'] = self.manager.sizes()
-        self.write_to_cache()
+            self.cache_refreshed = True
-
+        if resource == 'ssh_keys' or resource is None:
-
+            self.data['ssh_keys'] = self.manager.all_ssh_keys()
-    def load_droplets_from_digital_ocean(self):
+            self.cache_refreshed = True
-        ''' Use dopy to get droplet information from DigitalOcean and save data in cache files '''
+        if resource == 'domains' or resource is None:
-        manager  = DoManager(self.client_id, self.api_key)
+            self.data['domains'] = self.manager.all_domains()
-        self.data['droplets'] = self.sanitize_list(manager.all_active_droplets())
+            self.cache_refreshed = True
-        self.index['host_to_droplet'] = self.build_index(self.data['droplets'], 'ip_address', 'id', False)
-        self.build_inventory()
-        self.write_to_cache()
-
-
-    def build_index(self, source_seq, key_from, key_to, use_slug=True):
-        dest_dict = {}
-        for item in source_seq:
-            name = (use_slug and item.has_key('slug')) and item['slug'] or item[key_to]
-            key = item[key_from]
-            dest_dict[key] = name
-        return dest_dict
 
 
     def build_inventory(self):
@@ -345,70 +348,34 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
 
             self.inventory[droplet['id']] = [dest]
             self.push(self.inventory, droplet['name'], dest)
-            self.push(self.inventory, 'region_'+droplet['region_id'], dest)
+            self.push(self.inventory, 'region_' + droplet['region']['slug'], dest)
-            self.push(self.inventory, 'image_' +droplet['image_id'], dest)
+            self.push(self.inventory, 'image_' + str(droplet['image']['id']), dest)
-            self.push(self.inventory, 'size_'  +droplet['size_id'], dest)
+            self.push(self.inventory, 'size_' + droplet['size']['slug'], dest)
-            self.push(self.inventory, 'status_'+droplet['status'], dest)
-
-            region_name = self.index['region_to_name'].get(droplet['region_id'])
-            if region_name:
-                self.push(self.inventory, 'region_'+region_name, dest)
 
-            size_name = self.index['size_to_name'].get(droplet['size_id'])
+            image_slug = droplet['image']['slug']
-            if size_name:
+            if image_slug:
-                self.push(self.inventory, 'size_'+size_name, dest)
+                self.push(self.inventory, 'image_' + self.to_safe(image_slug), dest)
-
+            else:
-            image_name = self.index['image_to_name'].get(droplet['image_id'])
+                image_name = droplet['image']['name']
                 if image_name:
-                self.push(self.inventory, 'image_'+image_name, dest)
+                    self.push(self.inventory, 'image_' + self.to_safe(image_name), dest)
 
-            distro_name = self.index['image_to_distro'].get(droplet['image_id'])
+            self.push(self.inventory, 'distro_' + self.to_safe(droplet['image']['distribution']), dest)
-            if distro_name:
+            self.push(self.inventory, 'status_' + droplet['status'], dest)
-                self.push(self.inventory, 'distro_'+distro_name, dest)
 
 
     def load_droplet_variables_for_host(self):
         '''Generate a JSON response to a --host call'''
-        host = self.to_safe(str(self.args.host))
+        host = int(self.args.host)
-
-        if not host in self.index['host_to_droplet']:
-            # try updating cache
-            if not self.args.force_cache:
-                self.load_all_data_from_digital_ocean()
-            if not host in self.index['host_to_droplet']:
-                # host might not exist anymore
-                return {}
 
-        droplet = None
+        droplet = self.manager.show_droplet(host)
-        if self.cache_refreshed:
-            for drop in self.data['droplets']:
-                if drop['ip_address'] == host:
-                    droplet = self.sanitize_dict(drop)
-                    break
-        else:
-            # Cache wasn't refreshed this run, so hit DigitalOcean API
-            manager = DoManager(self.client_id, self.api_key)
-            droplet_id = self.index['host_to_droplet'][host]
-            droplet = self.sanitize_dict(manager.show_droplet(droplet_id))
-       
-        if not droplet:
-            return {}
 
         # Put all the information in a 'do_' namespace
         info = {}
         for k, v in droplet.items():
             info['do_'+k] = v
 
-        # Generate user-friendly variables (i.e. not the ID's) 
+        return {'droplet': info}
-        if droplet.has_key('region_id'):
-            info['do_region'] = self.index['region_to_name'].get(droplet['region_id'])
-        if droplet.has_key('size_id'):
-            info['do_size'] = self.index['size_to_name'].get(droplet['size_id'])
-        if droplet.has_key('image_id'):
-            info['do_image']  = self.index['image_to_name'].get(droplet['image_id'])
-            info['do_distro'] = self.index['image_to_distro'].get(droplet['image_id'])
-
-        return info
 
 
 
@@ -428,19 +395,21 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
 
     def load_from_cache(self):
         ''' Reads the data from the cache file and assigns it to member variables as Python Objects'''
+        try:
             cache = open(self.cache_filename, 'r')
             json_data = cache.read()
             cache.close()
             data = json.loads(json_data)
+        except IOError:
+            data = {'data': {}, 'inventory': {}}
 
         self.data = data['data']
         self.inventory = data['inventory']
-        self.index = data['index']
 
 
     def write_to_cache(self):
         ''' Writes data in JSON format to a file '''
-        data = { 'data': self.data, 'index': self.index, 'inventory': self.inventory }
+        data = { 'data': self.data, 'inventory': self.inventory }
         json_data = json.dumps(data, sort_keys=True, indent=2)
 
         cache = open(self.cache_filename, 'w')
@@ -448,7 +417,6 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
         cache.close()
 
 
-
     ###########################################################################
     # Utilities
     ###########################################################################
@@ -456,7 +424,7 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
     def push(self, my_dict, key, element):
         ''' Pushed an element onto an array that may not have been defined in the dict '''
         if key in my_dict:
-            my_dict[key].append(element);
+            my_dict[key].append(element)
         else:
             my_dict[key] = [element]
 
@@ -466,21 +434,6 @@ or environment variables (DO_CLIENT_ID and DO_API_KEY)'''
         return re.sub("[^A-Za-z0-9\-\.]", "_", word)
 
 
-    def sanitize_dict(self, d):
-        new_dict = {}
-        for k, v in d.items():
-            if v != None:
-                new_dict[self.to_safe(str(k))] = self.to_safe(str(v))
-        return new_dict
-
-
-    def sanitize_list(self, seq):
-        new_seq = []
-        for d in seq:
-            new_seq.append(self.sanitize_dict(d))
-        return new_seq
-
-
 
 ###########################################################################
 # Run the script