Add cloudscale.ch API inventory plugin (#53517)

lib/ansible/plugins/inventory/cloudscale.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright: (c) 2018, Gaudenz Steinlin <gaudenz.steinlin@cloudscale.ch>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+name: cloudscale
+plugin_type: inventory
+author:
+  - Gaudenz Steinlin (@gaudenz)
+short_description: cloudscale.ch inventory source
+description:
+    - Get inventory hosts from cloudscale.ch API
+version_added: '2.8'
+extends_documentation_fragment:
+  - constructed
+options:
+    plugin:
+        description: |
+            Token that ensures this is a source file for the 'cloudscale'
+            plugin.
+        required: True
+        choices: ['cloudscale']
+    inventory_hostname:
+        description: |
+            What to register as the inventory hostname.
+            If set to 'uuid' the uuid of the server will be used and a
+            group will be created for the server name.
+            If set to 'name' the name of the server will be used unless
+            there are more than one server with the same name in which
+            case the 'uuid' logic will be used.
+        type: str
+        choices:
+            - name
+            - uuid
+        default: "name"
+    ansible_host:
+        description: |
+            Which IP address to register as the ansible_host. If the
+            requested value does not exist or this is set to 'none', no
+            ansible_host will be set.
+        type: str
+        choices:
+            - public_v4
+            - public_v6
+            - private
+            - none
+        default: public_v4
+    api_token:
+        description: cloudscale.ch API token
+        env:
+          - name: CLOUDSCALE_API_TOKEN
+        type: str
+    api_timeout:
+        description: Timeout in seconds for calls to the cloudscale.ch API.
+        default: 30
+        type: int
+'''
+
+EXAMPLES = r'''
+# cloudscale_inventory.yml file in YAML format
+# Example command line: ansible-inventory --list -i cloudscale_inventory.yml
+
+plugin: cloudscale
+'''
+
+from collections import defaultdict
+from json import loads
+
+from ansible.errors import AnsibleError
+from ansible.module_utils.cloudscale import API_URL
+from ansible.module_utils.urls import open_url
+from ansible.inventory.group import to_safe_group_name
+from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
+
+iface_type_map = {
+    'public_v4': ('public', 4),
+    'public_v6': ('public', 6),
+    'private': ('private', 4),
+    'none': (None, None),
+}
+
+
+class InventoryModule(BaseInventoryPlugin, Constructable):
+
+    NAME = 'cloudscale'
+
+    def _get_server_list(self):
+        # Get list of servers from cloudscale.ch API
+        response = open_url(
+            API_URL + '/servers',
+            headers={'Authorization': 'Bearer %s' % self._token}
+        )
+        return loads(response.read())
+
+    def verify_file(self, path):
+        '''
+            :param path: the path to the inventory config file
+            :return the contents of the config file
+        '''
+        if super(InventoryModule, self).verify_file(path):
+            if path.endswith(('cloudscale.yml', 'cloudscale.yaml')):
+                return True
+        self.display.debug(
+            "cloudscale inventory filename must end with 'cloudscale.yml' or 'cloudscale.yaml'"
+        )
+        return False
+
+    def parse(self, inventory, loader, path, cache=True):
+        super(InventoryModule, self).parse(inventory, loader, path)
+
+        self._read_config_data(path)
+
+        self._token = self.get_option('api_token')
+        if not self._token:
+            raise AnsibleError('Could not find an API token. Set the '
+                               'CLOUDSCALE_API_TOKEN environment variable.')
+
+        inventory_hostname = self.get_option('inventory_hostname')
+        if inventory_hostname not in ('name', 'uuid'):
+            raise AnsibleError('Invalid value for option inventory_hostname: %s'
+                               % inventory_hostname)
+
+        ansible_host = self.get_option('ansible_host')
+        if ansible_host not in iface_type_map:
+            raise AnsibleError('Invalid value for option ansible_host: %s'
+                               % ansible_host)
+
+        # Merge servers with the same name
+        firstpass = defaultdict(list)
+        for server in self._get_server_list():
+            firstpass[server['name']].append(server)
+
+        # Add servers to inventory
+        for name, servers in firstpass.items():
+            if len(servers) == 1 and inventory_hostname == 'name':
+                self.inventory.add_host(name)
+                servers[0]['inventory_hostname'] = name
+            else:
+                # Two servers with the same name exist, create a group
+                # with this name and add the servers by UUID
+                group_name = to_safe_group_name(name)
+                if group_name not in self.inventory.groups:
+                    self.inventory.add_group(group_name)
+                for server in servers:
+                    self.inventory.add_host(server['uuid'], group_name)
+                    server['inventory_hostname'] = server['uuid']
+
+            # Set variables
+            iface_type, iface_version = iface_type_map[ansible_host]
+            for server in servers:
+                hostname = server.pop('inventory_hostname')
+                if ansible_host != 'none':
+                    addresses = [address['address']
+                                 for interface in server['interfaces']
+                                 for address in interface['addresses']
+                                 if interface['type'] == iface_type
+                                 and address['version'] == iface_version]
+
+                    if len(addresses) > 0:
+                        self.inventory.set_variable(
+                            hostname,
+                            'ansible_host',
+                            addresses[0],
+                        )
+                self.inventory.set_variable(
+                    hostname,
+                    'cloudscale',
+                    server,
+                )
+
+                variables = self.inventory.hosts[hostname].get_vars()
+                # Set composed variables
+                self._set_composite_vars(
+                    self.get_option('compose'),
+                    variables,
+                    hostname,
+                    self.get_option('strict'),
+                )
+
+                # Add host to composed groups
+                self._add_host_to_composed_groups(
+                    self.get_option('groups'),
+                    variables,
+                    hostname,
+                    self.get_option('strict'),
+                )
+
+                # Add host to keyed groups
+                self._add_host_to_keyed_groups(
+                    self.get_option('keyed_groups'),
+                    variables,
+                    hostname,
+                    self.get_option('strict'),
+                )

test/integration/targets/cloudscale_common/defaults/main.yml

@@ -2,6 +2,9 @@
 # The image to use for test servers
 cloudscale_test_image: 'debian-9'
 
+# Alternate test image to use if a different image is required
+cloudscale_alt_test_image: 'ubuntu-18.04'
+
 # The flavor to use for test servers
 cloudscale_test_flavor: 'flex-2'
 

test/integration/targets/inventory_cloudscale/aliases

@@ -0,0 +1,3 @@
+cloud/cloudscale
+unsupported
+needs/target/cloudscale_common

test/integration/targets/inventory_cloudscale/filter_plugins/group_name.py

@@ -0,0 +1,14 @@
+from ansible.inventory.group import to_safe_group_name
+
+
+def safe_group_name(name):
+    return to_safe_group_name(name)
+
+
+class FilterModule(object):
+    filter_map = {
+        'safe_group_name': safe_group_name
+    }
+
+    def filters(self):
+        return self.filter_map

test/integration/targets/inventory_cloudscale/inventory-private.yml

@@ -0,0 +1,14 @@
+plugin: cloudscale
+ansible_host: private
+inventory_hostname: name
+groups:
+  ansible: inventory_hostname.startswith('ansible')
+  private_net: (cloudscale.interfaces | selectattr('type', 'equalto', 'private') | list | length) > 0
+keyed_groups:
+  - prefix: net
+    key: (cloudscale.interfaces.0.addresses.0.address + '/' + cloudscale.interfaces.0.addresses.0.prefix_length | string) | ipaddr('network')
+  - prefix: distro
+    key: cloudscale.image.operating_system
+compose:
+  flavor_image: cloudscale.flavor.slug + '_' + cloudscale.image.slug
+strict: False

test/integration/targets/inventory_cloudscale/inventory-public.yml

@@ -0,0 +1,14 @@
+plugin: cloudscale
+ansible_host: public_v4
+inventory_hostname: name
+groups:
+  ansible: inventory_hostname.startswith('ansible')
+  private_net: (cloudscale.interfaces | selectattr('type', 'equalto', 'private') | list | length) > 0
+keyed_groups:
+  - prefix: net
+    key: (cloudscale.interfaces.0.addresses.0.address + '/' + cloudscale.interfaces.0.addresses.0.prefix_length | string) | ipaddr('network')
+  - prefix: distro
+    key: cloudscale.image.operating_system
+compose:
+  flavor_image: cloudscale.flavor.slug + '_' + cloudscale.image.slug
+strict: False

test/integration/targets/inventory_cloudscale/inventory-uuid.yml

@@ -0,0 +1,14 @@
+plugin: cloudscale
+ansible_host: public_v4
+inventory_hostname: uuid
+groups:
+  ansible: cloudscale.name.startswith('ansible')
+  private_net: (cloudscale.interfaces | selectattr('type', 'equalto', 'private') | list | length) > 0
+keyed_groups:
+  - prefix: net
+    key: (cloudscale.interfaces.0.addresses.0.address + '/' + cloudscale.interfaces.0.addresses.0.prefix_length | string) | ipaddr('network')
+  - prefix: distro
+    key: cloudscale.image.operating_system
+compose:
+  flavor_image: cloudscale.flavor.slug + '_' + cloudscale.image.slug
+strict: False

test/integration/targets/inventory_cloudscale/playbooks/change-inventory-config.yml

@@ -0,0 +1,8 @@
+- name: Change inventory configuration to {{ inventory_config }}
+  file:
+    src: '{{ inventory_config }}'
+    dest: ../inventory_cloudscale.yml
+    state: link
+
+- name: Refresh inventory
+  meta: refresh_inventory

test/integration/targets/inventory_cloudscale/playbooks/cleanup.yml

@@ -0,0 +1,17 @@
+---
+- name: List all servers
+  uri:
+    url: 'https://api.cloudscale.ch/v1/servers'
+    headers:
+      Authorization: 'Bearer {{ lookup("env", "CLOUDSCALE_API_TOKEN") }}'
+    status_code: 200
+  register: server_list
+
+- name: Remove all servers created by this test run
+  cloudscale_server:
+    uuid: '{{ item.uuid }}'
+    state: 'absent'
+  when: cloudscale_resource_prefix in item.name
+  with_items: '{{ server_list.json }}'
+  loop_control:
+    label: '{{ item.name }} ({{ item.uuid }})'

test/integration/targets/inventory_cloudscale/playbooks/common-asserts.yml

@@ -0,0 +1,50 @@
+---
+- name: '{{ inventory }}: Verify basic inventory'
+  assert:
+    that:
+      - server_public[identifier] in hostvars
+      - server_private[identifier] in hostvars
+      - server_public_private[identifier] in hostvars
+      - server_unsafe_chars[identifier] in hostvars
+
+- name: '{{ inventory }}: Verify duplicate host names in inventory'
+  assert:
+    that:
+      - cloudscale_resource_prefix + '-duplicate' not in hostvars
+      - (cloudscale_resource_prefix + '-duplicate') | safe_group_name in groups
+
+- name: '{{ inventory }}: Verify constructed groups in inventory'
+  assert:
+    that:
+      # Test for the "ansible" group
+      - '"ansible" in groups'
+      - server_public[identifier] in groups.ansible
+      - server_private[identifier] in groups.ansible
+      - server_public_private[identifier] in groups.ansible
+      - server_unsafe_chars[identifier] in groups.ansible
+      - server_other_prefix[identifier] not in groups.ansible
+      # Tests for the "private_net" group
+      - '"private_net" in groups'
+      - server_public[identifier] not in groups["private_net"]
+      - server_private[identifier] in groups["private_net"]
+      - server_public_private[identifier] in groups["private_net"]
+      # Tests for "distro" keyed group
+      - '"distro_Debian" in groups'
+      - '"distro_Ubuntu" in groups'
+      - server_public[identifier] in groups.distro_Debian
+      - server_private[identifier] not in groups.distro_Debian
+      - server_public[identifier] not in groups.distro_Ubuntu
+      - server_private[identifier] in groups.distro_Ubuntu
+      # Test for flavor_image composed variable
+      - hostvars[server_public[identifier]].flavor_image == 'flex-2_debian-9'
+      - hostvars[server_private[identifier]].flavor_image == 'flex-2_ubuntu-18.04'
+
+- name: '{{ inventory }}: Verify cloudscale specific host variables'
+  assert:
+    that:
+      - hostvars[item.0[identifier]].cloudscale[item.1] == item.0[item.1]
+  with_nested:
+    - [ '{{ server_public }}', '{{ server_private }}', '{{ server_public_private }}' ]
+    - [ 'anti_affinity_with', 'flavor', 'href', 'image', 'interfaces', 'name', 'uuid', 'volumes' ]
+  loop_control:
+    label: '{{ item.0.name }} ({{ item.0.uuid }}): {{ item.1 }}'

test/integration/targets/inventory_cloudscale/playbooks/setup.yml

@@ -0,0 +1,74 @@
+---
+- name: Create server with public network only
+  cloudscale_server:
+    name: '{{ cloudscale_resource_prefix }}-inventory-public'
+    flavor: '{{ cloudscale_test_flavor }}'
+    image: '{{ cloudscale_test_image }}'
+    ssh_keys: '{{ cloudscale_test_ssh_key }}'
+    use_public_network: True
+    use_private_network: False
+  register: server_public
+
+- name: Create server with private network only
+  cloudscale_server:
+    name: '{{ cloudscale_resource_prefix }}-inventory-private'
+    flavor: '{{ cloudscale_test_flavor }}'
+    image: '{{ cloudscale_alt_test_image }}'
+    ssh_keys: '{{ cloudscale_test_ssh_key }}'
+    use_public_network: False
+    use_private_network: True
+  register: server_private
+
+- name: Create server with public and private network
+  cloudscale_server:
+    name: '{{ cloudscale_resource_prefix }}-inventory-public-private'
+    flavor: '{{ cloudscale_test_flavor }}'
+    image: '{{ cloudscale_test_image }}'
+    ssh_keys: '{{ cloudscale_test_ssh_key }}'
+    use_public_network: True
+    use_private_network: True
+  register: server_public_private
+
+- name: Create servers with duplicate names
+  # The cloudscale_server module does not allow creating two servers with the same
+  # name. To do this the uri module has to be used.
+  uri:
+    url: 'https://api.cloudscale.ch/v1/servers'
+    method: POST
+    headers:
+      Authorization: 'Bearer {{ lookup("env", "CLOUDSCALE_API_TOKEN") }}'
+    body:
+      name: '{{ cloudscale_resource_prefix }}-duplicate'
+      flavor: '{{ cloudscale_test_flavor }}'
+      image: '{{ cloudscale_test_image }}'
+      ssh_keys:
+        - '{{ cloudscale_test_ssh_key }}'
+    body_format: json
+    status_code: 201
+  register: duplicate
+  with_sequence: count=2
+
+- name: Create server with different prefix
+  cloudscale_server:
+    name: 'other-prefix-{{ cloudscale_resource_prefix }}-inventory'
+    flavor: '{{ cloudscale_test_flavor }}'
+    image: '{{ cloudscale_test_image }}'
+    ssh_keys: '{{ cloudscale_test_ssh_key }}'
+  register: server_other_prefix
+
+# The API does not allow creation of a server with a name containing
+# characters not allowed in DNS names. So create a server and rename
+# it afterwards (which is possible). The resaon for this restriction is
+# that on creation a PTR entry for the server is created.
+- name: Create server to be renamed with unsafe characters
+  cloudscale_server:
+    name: '{{ cloudscale_resource_prefix }}-unsafe-chars'
+    flavor: '{{ cloudscale_test_flavor }}'
+    image: '{{ cloudscale_test_image }}'
+    ssh_keys: '{{ cloudscale_test_ssh_key }}'
+  register: server_unsafe_chars
+- name: Rename server to contain unsafe characters
+  cloudscale_server:
+    uuid: '{{ server_unsafe_chars.uuid }}'
+    name: '{{ cloudscale_resource_prefix }}-snowmans-are-cool-☃!'
+  register: server_unsafe_chars

test/integration/targets/inventory_cloudscale/playbooks/test-inventory.yml

@@ -0,0 +1,68 @@
+---
+- name: Create servers and test cloudscale inventory plugin
+  hosts: localhost
+  gather_facts: False
+  roles:
+    - cloudscale_common
+  tasks:
+    - block:
+        - import_tasks: setup.yml
+
+        - import_tasks: change-inventory-config.yml
+          vars:
+            inventory_config: inventory-public.yml
+
+        - import_tasks: common-asserts.yml
+          vars:
+            identifier: 'name'
+            inventory: 'Public v4'
+
+        - name: Verify inventory with public IP
+          assert:
+            that:
+              # Test ansible_host setting
+              - server_public.interfaces.0.addresses.0.address
+                == hostvars[server_public.name].ansible_host
+              - server_public_private.interfaces.0.addresses.0.address
+                == hostvars[server_public_private.name].ansible_host
+              - '"ansible_host" not in hostvars[server_private.name]'
+
+        - import_tasks: change-inventory-config.yml
+          vars:
+            inventory_config: inventory-private.yml
+
+        - import_tasks: common-asserts.yml
+          vars:
+            identifier: 'name'
+            inventory: 'Private v4'
+
+        - name: Verify inventory with private IP
+          assert:
+            that:
+              # Test ansible_host setting
+              - '"ansible_host" not in hostvars[server_public.name]'
+              - server_private.interfaces.0.addresses.0.address
+                == hostvars[server_private.name].ansible_host
+              - server_public_private.interfaces.1.addresses.0.address
+                == hostvars[server_public_private.name].ansible_host
+
+        - import_tasks: change-inventory-config.yml
+          vars:
+            inventory_config: inventory-uuid.yml
+
+        - import_tasks: common-asserts.yml
+          vars:
+            identifier: 'uuid'
+            inventory: 'UUID'
+
+        - name: Verify inventory with UUID
+          assert:
+            that:
+              # Test server name groups
+              - groups[server_public.name | safe_group_name] == [server_public.uuid]
+              - groups[server_private.name | safe_group_name] == [server_private.uuid]
+              - groups[server_public_private.name | safe_group_name] == [server_public_private.uuid]
+              - groups[server_unsafe_chars.name | safe_group_name] == [server_unsafe_chars.uuid]
+
+      always:
+        - import_tasks: cleanup.yml

test/integration/targets/inventory_cloudscale/runme.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Exit on errors, exit when accessing unset variables and print all commands
+set -eux
+
+# Set the role path so that the cloudscale_common role is available
+export ANSIBLE_ROLES_PATH="../"
+
+# Set the filter plugin search path so that the safe_group_name filter is available
+export ANSIBLE_FILTER_PLUGINS="./filter_plugins"
+
+rm -f inventory.yml
+export ANSIBLE_INVENTORY="./inventory_cloudscale.yml"
+
+# Run without converting invalid characters in group names
+export ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS=never
+ansible-playbook playbooks/test-inventory.yml "$@"
+
+# Run with converting invalid characters in group names
+export ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS=always
+ansible-playbook playbooks/test-inventory.yml "$@"