Added a Docker Machine dynamic inventory plugin (#54946)

* Added my Docker Machine dynamic inventory plugin (from https://github.com/ximon18/ansible-docker-machine-inventory-plugin) to begin the process of proposing it for inclusion in Ansible core. There are no integration tests yet. The docker_swarm inventory plugin has such tests but has some concerning note in its 'aliases' file about disabling docker due to test instability and also I wouldn't know at his point how to get Docker Machine installed on the integration test platform.

lib/ansible/plugins/inventory/docker_machine.py

@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Ximon Eighteen <ximon.eighteen@gmail.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    name: docker_machine
+    plugin_type: inventory
+    author: Ximon Eighteen (@ximon18)
+    short_description: Docker Machine inventory source
+    requirements:
+        - L(Docker Machine,https://docs.docker.com/machine/)
+    extends_documentation_fragment:
+        - constructed
+    description:
+        - Get inventory hosts from Docker Machine.
+        - Uses a YAML configuration file that ends with docker_machine.(yml|yaml).
+        - The plugin sets standard host variables C(ansible_host), C(ansible_port), C(ansible_user) and C(ansible_ssh_private_key).
+        - The plugin stores the Docker Machine 'env' output variables in I(dm_) prefixed host variables.
+
+    options:
+        plugin:
+            description: token that ensures this is a source file for the C(docker_machine) plugin.
+            required: yes
+            choices: ['docker_machine']
+        daemon_env:
+            description:
+                - Whether docker daemon connection environment variables should be fetched, and how to behave if they cannot be fetched.
+                - With C(require) and C(require-silently), fetch them and skip any host for which they cannot be fetched.
+                  A warning will be issued for any skipped host if the choice is C(require).
+                - With C(optional) and C(optional-silently), fetch them and not skip hosts for which they cannot be fetched.
+                  A warning will be issued for hosts where they cannot be fetched if the choice is C(optional).
+                - With C(skip), do not attempt to fetch the docker daemon connection environment variables.
+                - If fetched successfully, the variables will be prefixed with I(dm_) and stored as host variables.
+            type: str
+            choices:
+                - require
+                - require-silently
+                - optional
+                - optional-silently
+                - skip
+            default: require
+        running_required:
+            description: when true, hosts which Docker Machine indicates are in a state other than C(running) will be skipped.
+            type: bool
+            default: yes
+        verbose_output:
+            description: when true, include all available nodes metadata (e.g. Image, Region, Size) as a JSON object named C(docker_machine_node_attributes).
+            type: bool
+            default: yes
+'''
+
+EXAMPLES = '''
+# Minimal example
+plugin: docker_machine
+
+# Example using constructed features to create a group per Docker Machine driver
+# (https://docs.docker.com/machine/drivers/), e.g.:
+#   $ docker-machine create --driver digitalocean ... mymachine
+#   $ ansible-inventory -i ./path/to/docker-machine.yml --host=mymachine
+#   {
+#     ...
+#     "digitalocean": {
+#       "hosts": [
+#           "mymachine"
+#       ]
+#     ...
+#   }
+strict: no
+keyed_groups:
+  - separator: ''
+    key: docker_machine_node_attributes.DriverName
+
+# Example grouping hosts by Digital Machine tag
+strict: no
+keyed_groups:
+  - prefix: tag
+    key: 'dm_tags'
+
+# Example using compose to override the default SSH behaviour of asking the user to accept the remote host key
+compose:
+  ansible_ssh_common_args: '"-o StrictHostKeyChecking=accept-new"'
+'''
+
+from ansible.errors import AnsibleError
+from ansible.module_utils._text import to_native
+from ansible.module_utils._text import to_text
+from ansible.module_utils.common.process import get_bin_path
+from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
+from ansible.utils.display import Display
+
+import json
+import re
+import subprocess
+
+display = Display()
+
+
+class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
+    ''' Host inventory parser for ansible using Docker machine as source. '''
+
+    NAME = 'docker_machine'
+
+    DOCKER_MACHINE_PATH = None
+
+    def _run_command(self, args):
+        if not self.DOCKER_MACHINE_PATH:
+            try:
+                self.DOCKER_MACHINE_PATH = get_bin_path('docker-machine', required=True)
+            except ValueError as e:
+                raise AnsibleError('Unable to locate the docker-machine binary.', orig_exc=e)
+
+        command = [self.DOCKER_MACHINE_PATH]
+        command.extend(args)
+        display.debug('Executing command {0}'.format(command))
+        try:
+            result = subprocess.check_output(command)
+        except subprocess.CalledProcessError as e:
+            display.warning('Exception {0} caught while executing command {1}, this was the original exception: {2}'.format(type(e).__name__, command, e))
+            raise e
+
+        return to_text(result).strip()
+
+    def _get_docker_daemon_variables(self, machine_name):
+        '''
+        Capture settings from Docker Machine that would be needed to connect to the remote Docker daemon installed on
+        the Docker Machine remote host. Note: passing '--shell=sh' is a workaround for 'Error: Unknown shell'.
+        '''
+        try:
+            env_lines = self._run_command(['env', '--shell=sh', machine_name]).splitlines()
+        except subprocess.CalledProcessError:
+            # This can happen when the machine is created but provisioning is incomplete
+            return []
+
+        # example output of docker-machine env --shell=sh:
+        #   export DOCKER_TLS_VERIFY="1"
+        #   export DOCKER_HOST="tcp://134.209.204.160:2376"
+        #   export DOCKER_CERT_PATH="/root/.docker/machine/machines/routinator"
+        #   export DOCKER_MACHINE_NAME="routinator"
+        #   # Run this command to configure your shell:
+        #   # eval $(docker-machine env --shell=bash routinator)
+
+        # capture any of the DOCKER_xxx variables that were output and create Ansible host vars
+        # with the same name and value but with a dm_ name prefix.
+        vars = []
+        for line in env_lines:
+            match = re.search('(DOCKER_[^=]+)="([^"]+)"', line)
+            if match:
+                env_var_name = match.group(1)
+                env_var_value = match.group(2)
+                vars.append((env_var_name, env_var_value))
+
+        return vars
+
+    def _get_machine_names(self):
+        # Filter out machines that are not in the Running state as we probably can't do anything useful actions
+        # with them.
+        ls_command = ['ls', '-q']
+        if self.get_option('running_required'):
+            ls_command.extend(['--filter', 'state=Running'])
+
+        try:
+            ls_lines = self._run_command(ls_command)
+        except subprocess.CalledProcessError:
+            return []
+
+        return ls_lines.splitlines()
+
+    def _inspect_docker_machine_host(self, node):
+        try:
+            inspect_lines = self._run_command(['inspect', self.node])
+        except subprocess.CalledProcessError:
+            return None
+
+        return json.loads(inspect_lines)
+
+    def _should_skip_host(self, machine_name, env_var_tuples, daemon_env):
+        if not env_var_tuples:
+            warning_prefix = 'Unable to fetch Docker daemon env vars from Docker Machine for host {0}'.format(machine_name)
+            if daemon_env in ('require', 'require-silently'):
+                if daemon_env == 'require':
+                    display.warning('{0}: host will be skipped'.format(warning_prefix))
+                return True
+            else:  # 'optional', 'optional-silently'
+                if daemon_env == 'optional':
+                    display.warning('{0}: host will lack dm_DOCKER_xxx variables'.format(warning_prefix))
+        return False
+
+    def _populate(self):
+        daemon_env = self.get_option('daemon_env')
+        try:
+            for self.node in self._get_machine_names():
+                self.node_attrs = self._inspect_docker_machine_host(self.node)
+                if not self.node_attrs:
+                    continue
+
+                machine_name = self.node_attrs['Driver']['MachineName']
+
+                # query `docker-machine env` to obtain remote Docker daemon connection settings in the form of commands
+                # that could be used to set environment variables to influence a local Docker client:
+                if daemon_env == 'skip':
+                    env_var_tuples = []
+                else:
+                    env_var_tuples = self._get_docker_daemon_variables(machine_name)
+                    if self._should_skip_host(machine_name, env_var_tuples, daemon_env):
+                        continue
+
+                # add an entry in the inventory for this host
+                self.inventory.add_host(machine_name)
+
+                # set standard Ansible remote host connection settings to details captured from `docker-machine`
+                # see: https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html
+                self.inventory.set_variable(machine_name, 'ansible_host', self.node_attrs['Driver']['IPAddress'])
+                self.inventory.set_variable(machine_name, 'ansible_port', self.node_attrs['Driver']['SSHPort'])
+                self.inventory.set_variable(machine_name, 'ansible_user', self.node_attrs['Driver']['SSHUser'])
+                self.inventory.set_variable(machine_name, 'ansible_ssh_private_key_file', self.node_attrs['Driver']['SSHKeyPath'])
+
+                # set variables based on Docker Machine tags
+                tags = self.node_attrs['Driver'].get('Tags') or ''
+                self.inventory.set_variable(machine_name, 'dm_tags', tags)
+
+                # set variables based on Docker Machine env variables
+                for kv in env_var_tuples:
+                    self.inventory.set_variable(machine_name, 'dm_{0}'.format(kv[0]), kv[1])
+
+                if self.get_option('verbose_output'):
+                    self.inventory.set_variable(machine_name, 'docker_machine_node_attributes', self.node_attrs)
+
+                # Use constructed if applicable
+                strict = self.get_option('strict')
+
+                # Composed variables
+                self._set_composite_vars(self.get_option('compose'), self.node_attrs, machine_name, strict=strict)
+
+                # Complex groups based on jinja2 conditionals, hosts that meet the conditional are added to group
+                self._add_host_to_composed_groups(self.get_option('groups'), self.node_attrs, machine_name, strict=strict)
+
+                # Create groups based on variable values and add the corresponding hosts to it
+                self._add_host_to_keyed_groups(self.get_option('keyed_groups'), self.node_attrs, machine_name, strict=strict)
+
+        except Exception as e:
+            raise AnsibleError('Unable to fetch hosts from Docker Machine, this was the original exception: %s' %
+                               to_native(e), orig_exc=e)
+
+    def verify_file(self, path):
+        """Return the possibility of a file being consumable by this plugin."""
+        return (
+            super(InventoryModule, self).verify_file(path) and
+            path.endswith((self.NAME + '.yaml', self.NAME + '.yml')))
+
+    def parse(self, inventory, loader, path, cache=True):
+        super(InventoryModule, self).parse(inventory, loader, path, cache)
+        self._read_config_data(path)
+        self._populate()

test/integration/targets/inventory_docker_machine/aliases

@@ -0,0 +1,8 @@
+shippable/posix/group2
+skip/osx
+skip/freebsd
+destructive
+skip/docker  # We need SSH access to the VM and need to be able to let
+             # docker-machine install docker in the VM. This won't work
+             # with tests running in docker containers.
+needs/root

test/integration/targets/inventory_docker_machine/docker-machine

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Mock Docker Machine wrapper for testing purposes
+
+[ "$MOCK_ERROR_IN" == "$1" ] && echo >&2 "Mock Docker Machine error" && exit 1
+case $1 in
+    env)        
+        cat <<'EOF'
+export DOCKER_TLS_VERIFY="1"
+export DOCKER_HOST="tcp://134.209.204.160:2376"
+export DOCKER_CERT_PATH="/root/.docker/machine/machines/routinator"
+export DOCKER_MACHINE_NAME="routinator"
+# Run this command to configure your shell:
+# eval $(docker-machine env --shell=bash routinator)
+EOF
+        ;;
+
+    *)
+        /usr/bin/docker-machine $*
+        ;;
+esac

test/integration/targets/inventory_docker_machine/inventory_1.docker_machine.yml

@@ -0,0 +1 @@
+plugin: docker_machine

test/integration/targets/inventory_docker_machine/inventory_2.docker_machine.yml

@@ -0,0 +1,2 @@
+plugin: docker_machine
+daemon_env: require

test/integration/targets/inventory_docker_machine/inventory_3.docker_machine.yml

@@ -0,0 +1,2 @@
+plugin: docker_machine
+daemon_env: optional

test/integration/targets/inventory_docker_machine/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - setup_docker

test/integration/targets/inventory_docker_machine/playbooks/pre-setup.yml

@@ -0,0 +1,18 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  tasks:
+    - name: Setup docker
+      include_role:
+        name: setup_docker
+
+    # There seems to be no better way to install docker-machine. At least I couldn't find any packages for RHEL7/8.
+    - name: Download docker-machine binary
+      vars:
+        docker_machine_version: "0.16.1"
+      get_url:
+        url: "https://github.com/docker/machine/releases/download/v{{ docker_machine_version }}/docker-machine-{{ ansible_system }}-{{ ansible_userspace_architecture }}"
+        dest: /tmp/docker-machine
+    - name: Install docker-machine binary
+      command: install /tmp/docker-machine /usr/bin/docker-machine
+      become: yes

test/integration/targets/inventory_docker_machine/playbooks/setup.yml

@@ -0,0 +1,11 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  tasks:
+    - name: Request Docker Machine to use this machine as a generic VM
+      command: "docker-machine --debug create \
+        --driver generic \
+        --generic-ip-address=localhost \
+        --generic-ssh-key {{ lookup('env', 'HOME') }}/.ssh/id_rsa \
+        --generic-ssh-user root \
+        vm"

test/integration/targets/inventory_docker_machine/playbooks/teardown.yml

@@ -0,0 +1,6 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  tasks:
+    - name: Request Docker Machine to remove this machine as a generic VM
+      command: "docker-machine rm vm -f"

test/integration/targets/inventory_docker_machine/playbooks/test_inventory_1.yml

@@ -0,0 +1,50 @@
+- hosts: 127.0.0.1
+  gather_facts: no
+  tasks:
+  - name: sanity check Docker Machine output
+    vars:
+      dm_ls_format: !unsafe '{{.Name}} | {{.DriverName}} | {{.State}} | {{.URL}} | {{.Error}}'
+      success_regex: "^vm | [^|]+ | Running | tcp://.+ |$"
+    command: docker-machine ls --format '{{ dm_ls_format }}'
+    register: result
+    failed_when: result.rc != 0 or result.stdout is not match(success_regex)
+
+  - name: verify Docker Machine ip
+    command: docker-machine ip vm
+    register: result
+    failed_when: result.rc != 0 or result.stdout != hostvars['vm'].ansible_host
+
+  - name: verify Docker Machine env
+    command: docker-machine env --shell=sh vm
+    register: result
+
+  - debug: var=result.stdout
+
+  - assert:
+      that:
+        - "'DOCKER_TLS_VERIFY=\"{{ hostvars['vm'].dm_DOCKER_TLS_VERIFY }}\"' in result.stdout"
+        - "'DOCKER_HOST=\"{{ hostvars['vm'].dm_DOCKER_HOST }}\"' in result.stdout"
+        - "'DOCKER_CERT_PATH=\"{{ hostvars['vm'].dm_DOCKER_CERT_PATH }}\"' in result.stdout"
+        - "'DOCKER_MACHINE_NAME=\"{{ hostvars['vm'].dm_DOCKER_MACHINE_NAME }}\"' in result.stdout"
+
+- hosts: vm
+  gather_facts: no
+  tasks:
+  - name: do something to verify that accept-new ssh setting was applied by the docker-machine inventory plugin
+    raw: uname -a
+    register: result
+
+  - debug: var=result.stdout
+
+- hosts: 127.0.0.1
+  gather_facts: no
+  environment:
+    DOCKER_CERT_PATH: "{{ hostvars['vm'].dm_DOCKER_CERT_PATH }}"
+    DOCKER_HOST: "{{ hostvars['vm'].dm_DOCKER_HOST }}"
+    DOCKER_MACHINE_NAME: "{{ hostvars['vm'].dm_DOCKER_MACHINE_NAME }}"
+    DOCKER_TLS_VERIFY: "{{ hostvars['vm'].dm_DOCKER_TLS_VERIFY }}"
+  tasks:
+  - name: run a Docker container on the target Docker Machine host to verify that Docker daemon connection settings from the docker-machine inventory plugin work as expected
+    docker_container:
+      name: test
+      image: hello-world:latest

test/integration/targets/inventory_docker_machine/runme.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(dirname "$0")
+
+echo "Who am I:    $(whoami)"
+echo "Home:        ${HOME}"
+echo "PWD:         $(pwd)"
+echo "Script dir:  ${SCRIPT_DIR}"
+
+# restrict Ansible just to our inventory plugin, to prevent inventory data being matched by the test but being provided
+# by some other dynamic inventory provider
+export ANSIBLE_INVENTORY_ENABLED=docker_machine
+
+[[ -n "$DEBUG" || -n "$ANSIBLE_DEBUG" ]] && set -x
+
+set -euo pipefail
+
+SAVED_PATH="$PATH"
+
+cleanup() {
+    PATH="${SAVED_PATH}"
+    echo "Cleanup"
+    ansible-playbook -i teardown.docker_machine.yml playbooks/teardown.yml
+    echo "Done"
+}
+
+trap cleanup INT TERM EXIT
+
+echo "Pre-setup (install docker, docker-machine)"
+ANSIBLE_ROLES_PATH=.. ansible-playbook playbooks/pre-setup.yml
+
+echo "Print docker-machine version"
+docker-machine --version
+
+echo "Check preconditions"
+# Host should NOT be known to Ansible before the test starts
+ansible-inventory -i inventory_1.docker_machine.yml --host vm >/dev/null && exit 1
+
+echo "Test that the docker_machine inventory plugin is being loaded"
+ANSIBLE_DEBUG=yes ansible-inventory -i inventory_1.docker_machine.yml --list | grep -F "Loading InventoryModule 'docker_machine'"
+
+echo "Setup"
+ansible-playbook playbooks/setup.yml
+
+echo "Test docker_machine inventory 1"
+ansible-playbook -i inventory_1.docker_machine.yml playbooks/test_inventory_1.yml
+
+echo "Activate Docker Machine mock"
+PATH=${SCRIPT_DIR}:$PATH
+
+echo "Test docker_machine inventory 2: daemon_env=require daemon env success=yes"
+ansible-inventory -i inventory_2.docker_machine.yml --list
+
+echo "Test docker_machine inventory 2: daemon_env=require daemon env success=no"
+export MOCK_ERROR_IN=env
+ansible-inventory -i inventory_2.docker_machine.yml --list
+unset MOCK_ERROR_IN
+
+echo "Test docker_machine inventory 3: daemon_env=optional daemon env success=yes"
+ansible-inventory -i inventory_3.docker_machine.yml --list
+
+echo "Test docker_machine inventory 3: daemon_env=optional daemon env success=no"
+export MOCK_ERROR_IN=env
+ansible-inventory -i inventory_2.docker_machine.yml --list
+unset MOCK_ERROR_IN
+
+echo "Deactivate Docker Machine mock"
+PATH="${SAVED_PATH}"

test/integration/targets/inventory_docker_machine/teardown.docker_machine.yml

@@ -0,0 +1,3 @@
+plugin: docker_machine
+daemon_env: skip
+running_required: no