diff --git a/lib/ansible/plugins/inventory/docker_machine.py b/lib/ansible/plugins/inventory/docker_machine.py new file mode 100644 index 00000000000..4b6007868ec --- /dev/null +++ b/lib/ansible/plugins/inventory/docker_machine.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Ximon Eighteen +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + name: docker_machine + plugin_type: inventory + author: Ximon Eighteen (@ximon18) + short_description: Docker Machine inventory source + requirements: + - L(Docker Machine,https://docs.docker.com/machine/) + extends_documentation_fragment: + - constructed + description: + - Get inventory hosts from Docker Machine. + - Uses a YAML configuration file that ends with docker_machine.(yml|yaml). + - The plugin sets standard host variables C(ansible_host), C(ansible_port), C(ansible_user) and C(ansible_ssh_private_key). + - The plugin stores the Docker Machine 'env' output variables in I(dm_) prefixed host variables. + + options: + plugin: + description: token that ensures this is a source file for the C(docker_machine) plugin. + required: yes + choices: ['docker_machine'] + daemon_env: + description: + - Whether docker daemon connection environment variables should be fetched, and how to behave if they cannot be fetched. + - With C(require) and C(require-silently), fetch them and skip any host for which they cannot be fetched. + A warning will be issued for any skipped host if the choice is C(require). + - With C(optional) and C(optional-silently), fetch them and not skip hosts for which they cannot be fetched. + A warning will be issued for hosts where they cannot be fetched if the choice is C(optional). + - With C(skip), do not attempt to fetch the docker daemon connection environment variables. + - If fetched successfully, the variables will be prefixed with I(dm_) and stored as host variables. + type: str + choices: + - require + - require-silently + - optional + - optional-silently + - skip + default: require + running_required: + description: when true, hosts which Docker Machine indicates are in a state other than C(running) will be skipped. + type: bool + default: yes + verbose_output: + description: when true, include all available nodes metadata (e.g. Image, Region, Size) as a JSON object named C(docker_machine_node_attributes). + type: bool + default: yes +''' + +EXAMPLES = ''' +# Minimal example +plugin: docker_machine + +# Example using constructed features to create a group per Docker Machine driver +# (https://docs.docker.com/machine/drivers/), e.g.: +# $ docker-machine create --driver digitalocean ... mymachine +# $ ansible-inventory -i ./path/to/docker-machine.yml --host=mymachine +# { +# ... +# "digitalocean": { +# "hosts": [ +# "mymachine" +# ] +# ... +# } +strict: no +keyed_groups: + - separator: '' + key: docker_machine_node_attributes.DriverName + +# Example grouping hosts by Digital Machine tag +strict: no +keyed_groups: + - prefix: tag + key: 'dm_tags' + +# Example using compose to override the default SSH behaviour of asking the user to accept the remote host key +compose: + ansible_ssh_common_args: '"-o StrictHostKeyChecking=accept-new"' +''' + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native +from ansible.module_utils._text import to_text +from ansible.module_utils.common.process import get_bin_path +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable +from ansible.utils.display import Display + +import json +import re +import subprocess + +display = Display() + + +class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): + ''' Host inventory parser for ansible using Docker machine as source. ''' + + NAME = 'docker_machine' + + DOCKER_MACHINE_PATH = None + + def _run_command(self, args): + if not self.DOCKER_MACHINE_PATH: + try: + self.DOCKER_MACHINE_PATH = get_bin_path('docker-machine', required=True) + except ValueError as e: + raise AnsibleError('Unable to locate the docker-machine binary.', orig_exc=e) + + command = [self.DOCKER_MACHINE_PATH] + command.extend(args) + display.debug('Executing command {0}'.format(command)) + try: + result = subprocess.check_output(command) + except subprocess.CalledProcessError as e: + display.warning('Exception {0} caught while executing command {1}, this was the original exception: {2}'.format(type(e).__name__, command, e)) + raise e + + return to_text(result).strip() + + def _get_docker_daemon_variables(self, machine_name): + ''' + Capture settings from Docker Machine that would be needed to connect to the remote Docker daemon installed on + the Docker Machine remote host. Note: passing '--shell=sh' is a workaround for 'Error: Unknown shell'. + ''' + try: + env_lines = self._run_command(['env', '--shell=sh', machine_name]).splitlines() + except subprocess.CalledProcessError: + # This can happen when the machine is created but provisioning is incomplete + return [] + + # example output of docker-machine env --shell=sh: + # export DOCKER_TLS_VERIFY="1" + # export DOCKER_HOST="tcp://134.209.204.160:2376" + # export DOCKER_CERT_PATH="/root/.docker/machine/machines/routinator" + # export DOCKER_MACHINE_NAME="routinator" + # # Run this command to configure your shell: + # # eval $(docker-machine env --shell=bash routinator) + + # capture any of the DOCKER_xxx variables that were output and create Ansible host vars + # with the same name and value but with a dm_ name prefix. + vars = [] + for line in env_lines: + match = re.search('(DOCKER_[^=]+)="([^"]+)"', line) + if match: + env_var_name = match.group(1) + env_var_value = match.group(2) + vars.append((env_var_name, env_var_value)) + + return vars + + def _get_machine_names(self): + # Filter out machines that are not in the Running state as we probably can't do anything useful actions + # with them. + ls_command = ['ls', '-q'] + if self.get_option('running_required'): + ls_command.extend(['--filter', 'state=Running']) + + try: + ls_lines = self._run_command(ls_command) + except subprocess.CalledProcessError: + return [] + + return ls_lines.splitlines() + + def _inspect_docker_machine_host(self, node): + try: + inspect_lines = self._run_command(['inspect', self.node]) + except subprocess.CalledProcessError: + return None + + return json.loads(inspect_lines) + + def _should_skip_host(self, machine_name, env_var_tuples, daemon_env): + if not env_var_tuples: + warning_prefix = 'Unable to fetch Docker daemon env vars from Docker Machine for host {0}'.format(machine_name) + if daemon_env in ('require', 'require-silently'): + if daemon_env == 'require': + display.warning('{0}: host will be skipped'.format(warning_prefix)) + return True + else: # 'optional', 'optional-silently' + if daemon_env == 'optional': + display.warning('{0}: host will lack dm_DOCKER_xxx variables'.format(warning_prefix)) + return False + + def _populate(self): + daemon_env = self.get_option('daemon_env') + try: + for self.node in self._get_machine_names(): + self.node_attrs = self._inspect_docker_machine_host(self.node) + if not self.node_attrs: + continue + + machine_name = self.node_attrs['Driver']['MachineName'] + + # query `docker-machine env` to obtain remote Docker daemon connection settings in the form of commands + # that could be used to set environment variables to influence a local Docker client: + if daemon_env == 'skip': + env_var_tuples = [] + else: + env_var_tuples = self._get_docker_daemon_variables(machine_name) + if self._should_skip_host(machine_name, env_var_tuples, daemon_env): + continue + + # add an entry in the inventory for this host + self.inventory.add_host(machine_name) + + # set standard Ansible remote host connection settings to details captured from `docker-machine` + # see: https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html + self.inventory.set_variable(machine_name, 'ansible_host', self.node_attrs['Driver']['IPAddress']) + self.inventory.set_variable(machine_name, 'ansible_port', self.node_attrs['Driver']['SSHPort']) + self.inventory.set_variable(machine_name, 'ansible_user', self.node_attrs['Driver']['SSHUser']) + self.inventory.set_variable(machine_name, 'ansible_ssh_private_key_file', self.node_attrs['Driver']['SSHKeyPath']) + + # set variables based on Docker Machine tags + tags = self.node_attrs['Driver'].get('Tags') or '' + self.inventory.set_variable(machine_name, 'dm_tags', tags) + + # set variables based on Docker Machine env variables + for kv in env_var_tuples: + self.inventory.set_variable(machine_name, 'dm_{0}'.format(kv[0]), kv[1]) + + if self.get_option('verbose_output'): + self.inventory.set_variable(machine_name, 'docker_machine_node_attributes', self.node_attrs) + + # Use constructed if applicable + strict = self.get_option('strict') + + # Composed variables + self._set_composite_vars(self.get_option('compose'), self.node_attrs, machine_name, strict=strict) + + # Complex groups based on jinja2 conditionals, hosts that meet the conditional are added to group + self._add_host_to_composed_groups(self.get_option('groups'), self.node_attrs, machine_name, strict=strict) + + # Create groups based on variable values and add the corresponding hosts to it + self._add_host_to_keyed_groups(self.get_option('keyed_groups'), self.node_attrs, machine_name, strict=strict) + + except Exception as e: + raise AnsibleError('Unable to fetch hosts from Docker Machine, this was the original exception: %s' % + to_native(e), orig_exc=e) + + def verify_file(self, path): + """Return the possibility of a file being consumable by this plugin.""" + return ( + super(InventoryModule, self).verify_file(path) and + path.endswith((self.NAME + '.yaml', self.NAME + '.yml'))) + + def parse(self, inventory, loader, path, cache=True): + super(InventoryModule, self).parse(inventory, loader, path, cache) + self._read_config_data(path) + self._populate() diff --git a/test/integration/targets/inventory_docker_machine/aliases b/test/integration/targets/inventory_docker_machine/aliases new file mode 100644 index 00000000000..3d5007f3970 --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/aliases @@ -0,0 +1,8 @@ +shippable/posix/group2 +skip/osx +skip/freebsd +destructive +skip/docker # We need SSH access to the VM and need to be able to let + # docker-machine install docker in the VM. This won't work + # with tests running in docker containers. +needs/root diff --git a/test/integration/targets/inventory_docker_machine/docker-machine b/test/integration/targets/inventory_docker_machine/docker-machine new file mode 100644 index 00000000000..be5d00c5919 --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/docker-machine @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Mock Docker Machine wrapper for testing purposes + +[ "$MOCK_ERROR_IN" == "$1" ] && echo >&2 "Mock Docker Machine error" && exit 1 +case $1 in + env) + cat <<'EOF' +export DOCKER_TLS_VERIFY="1" +export DOCKER_HOST="tcp://134.209.204.160:2376" +export DOCKER_CERT_PATH="/root/.docker/machine/machines/routinator" +export DOCKER_MACHINE_NAME="routinator" +# Run this command to configure your shell: +# eval $(docker-machine env --shell=bash routinator) +EOF + ;; + + *) + /usr/bin/docker-machine $* + ;; +esac diff --git a/test/integration/targets/inventory_docker_machine/inventory_1.docker_machine.yml b/test/integration/targets/inventory_docker_machine/inventory_1.docker_machine.yml new file mode 100644 index 00000000000..17d64816ac3 --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/inventory_1.docker_machine.yml @@ -0,0 +1 @@ +plugin: docker_machine diff --git a/test/integration/targets/inventory_docker_machine/inventory_2.docker_machine.yml b/test/integration/targets/inventory_docker_machine/inventory_2.docker_machine.yml new file mode 100644 index 00000000000..e6d5fd78a83 --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/inventory_2.docker_machine.yml @@ -0,0 +1,2 @@ +plugin: docker_machine +daemon_env: require \ No newline at end of file diff --git a/test/integration/targets/inventory_docker_machine/inventory_3.docker_machine.yml b/test/integration/targets/inventory_docker_machine/inventory_3.docker_machine.yml new file mode 100644 index 00000000000..c674bcddc8e --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/inventory_3.docker_machine.yml @@ -0,0 +1,2 @@ +plugin: docker_machine +daemon_env: optional \ No newline at end of file diff --git a/test/integration/targets/inventory_docker_machine/meta/main.yml b/test/integration/targets/inventory_docker_machine/meta/main.yml new file mode 100644 index 00000000000..07da8c6ddae --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - setup_docker diff --git a/test/integration/targets/inventory_docker_machine/playbooks/pre-setup.yml b/test/integration/targets/inventory_docker_machine/playbooks/pre-setup.yml new file mode 100644 index 00000000000..9f526fb44dc --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/playbooks/pre-setup.yml @@ -0,0 +1,18 @@ +--- +- hosts: 127.0.0.1 + connection: local + tasks: + - name: Setup docker + include_role: + name: setup_docker + + # There seems to be no better way to install docker-machine. At least I couldn't find any packages for RHEL7/8. + - name: Download docker-machine binary + vars: + docker_machine_version: "0.16.1" + get_url: + url: "https://github.com/docker/machine/releases/download/v{{ docker_machine_version }}/docker-machine-{{ ansible_system }}-{{ ansible_userspace_architecture }}" + dest: /tmp/docker-machine + - name: Install docker-machine binary + command: install /tmp/docker-machine /usr/bin/docker-machine + become: yes diff --git a/test/integration/targets/inventory_docker_machine/playbooks/setup.yml b/test/integration/targets/inventory_docker_machine/playbooks/setup.yml new file mode 100644 index 00000000000..78042b6297e --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/playbooks/setup.yml @@ -0,0 +1,11 @@ +--- +- hosts: 127.0.0.1 + connection: local + tasks: + - name: Request Docker Machine to use this machine as a generic VM + command: "docker-machine --debug create \ + --driver generic \ + --generic-ip-address=localhost \ + --generic-ssh-key {{ lookup('env', 'HOME') }}/.ssh/id_rsa \ + --generic-ssh-user root \ + vm" diff --git a/test/integration/targets/inventory_docker_machine/playbooks/teardown.yml b/test/integration/targets/inventory_docker_machine/playbooks/teardown.yml new file mode 100644 index 00000000000..b272c09472a --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/playbooks/teardown.yml @@ -0,0 +1,6 @@ +--- +- hosts: 127.0.0.1 + connection: local + tasks: + - name: Request Docker Machine to remove this machine as a generic VM + command: "docker-machine rm vm -f" diff --git a/test/integration/targets/inventory_docker_machine/playbooks/test_inventory_1.yml b/test/integration/targets/inventory_docker_machine/playbooks/test_inventory_1.yml new file mode 100644 index 00000000000..476262f1ab4 --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/playbooks/test_inventory_1.yml @@ -0,0 +1,50 @@ +- hosts: 127.0.0.1 + gather_facts: no + tasks: + - name: sanity check Docker Machine output + vars: + dm_ls_format: !unsafe '{{.Name}} | {{.DriverName}} | {{.State}} | {{.URL}} | {{.Error}}' + success_regex: "^vm | [^|]+ | Running | tcp://.+ |$" + command: docker-machine ls --format '{{ dm_ls_format }}' + register: result + failed_when: result.rc != 0 or result.stdout is not match(success_regex) + + - name: verify Docker Machine ip + command: docker-machine ip vm + register: result + failed_when: result.rc != 0 or result.stdout != hostvars['vm'].ansible_host + + - name: verify Docker Machine env + command: docker-machine env --shell=sh vm + register: result + + - debug: var=result.stdout + + - assert: + that: + - "'DOCKER_TLS_VERIFY=\"{{ hostvars['vm'].dm_DOCKER_TLS_VERIFY }}\"' in result.stdout" + - "'DOCKER_HOST=\"{{ hostvars['vm'].dm_DOCKER_HOST }}\"' in result.stdout" + - "'DOCKER_CERT_PATH=\"{{ hostvars['vm'].dm_DOCKER_CERT_PATH }}\"' in result.stdout" + - "'DOCKER_MACHINE_NAME=\"{{ hostvars['vm'].dm_DOCKER_MACHINE_NAME }}\"' in result.stdout" + +- hosts: vm + gather_facts: no + tasks: + - name: do something to verify that accept-new ssh setting was applied by the docker-machine inventory plugin + raw: uname -a + register: result + + - debug: var=result.stdout + +- hosts: 127.0.0.1 + gather_facts: no + environment: + DOCKER_CERT_PATH: "{{ hostvars['vm'].dm_DOCKER_CERT_PATH }}" + DOCKER_HOST: "{{ hostvars['vm'].dm_DOCKER_HOST }}" + DOCKER_MACHINE_NAME: "{{ hostvars['vm'].dm_DOCKER_MACHINE_NAME }}" + DOCKER_TLS_VERIFY: "{{ hostvars['vm'].dm_DOCKER_TLS_VERIFY }}" + tasks: + - name: run a Docker container on the target Docker Machine host to verify that Docker daemon connection settings from the docker-machine inventory plugin work as expected + docker_container: + name: test + image: hello-world:latest \ No newline at end of file diff --git a/test/integration/targets/inventory_docker_machine/runme.sh b/test/integration/targets/inventory_docker_machine/runme.sh new file mode 100755 index 00000000000..074e64fc25a --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/runme.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(dirname "$0") + +echo "Who am I: $(whoami)" +echo "Home: ${HOME}" +echo "PWD: $(pwd)" +echo "Script dir: ${SCRIPT_DIR}" + +# restrict Ansible just to our inventory plugin, to prevent inventory data being matched by the test but being provided +# by some other dynamic inventory provider +export ANSIBLE_INVENTORY_ENABLED=docker_machine + +[[ -n "$DEBUG" || -n "$ANSIBLE_DEBUG" ]] && set -x + +set -euo pipefail + +SAVED_PATH="$PATH" + +cleanup() { + PATH="${SAVED_PATH}" + echo "Cleanup" + ansible-playbook -i teardown.docker_machine.yml playbooks/teardown.yml + echo "Done" +} + +trap cleanup INT TERM EXIT + +echo "Pre-setup (install docker, docker-machine)" +ANSIBLE_ROLES_PATH=.. ansible-playbook playbooks/pre-setup.yml + +echo "Print docker-machine version" +docker-machine --version + +echo "Check preconditions" +# Host should NOT be known to Ansible before the test starts +ansible-inventory -i inventory_1.docker_machine.yml --host vm >/dev/null && exit 1 + +echo "Test that the docker_machine inventory plugin is being loaded" +ANSIBLE_DEBUG=yes ansible-inventory -i inventory_1.docker_machine.yml --list | grep -F "Loading InventoryModule 'docker_machine'" + +echo "Setup" +ansible-playbook playbooks/setup.yml + +echo "Test docker_machine inventory 1" +ansible-playbook -i inventory_1.docker_machine.yml playbooks/test_inventory_1.yml + +echo "Activate Docker Machine mock" +PATH=${SCRIPT_DIR}:$PATH + +echo "Test docker_machine inventory 2: daemon_env=require daemon env success=yes" +ansible-inventory -i inventory_2.docker_machine.yml --list + +echo "Test docker_machine inventory 2: daemon_env=require daemon env success=no" +export MOCK_ERROR_IN=env +ansible-inventory -i inventory_2.docker_machine.yml --list +unset MOCK_ERROR_IN + +echo "Test docker_machine inventory 3: daemon_env=optional daemon env success=yes" +ansible-inventory -i inventory_3.docker_machine.yml --list + +echo "Test docker_machine inventory 3: daemon_env=optional daemon env success=no" +export MOCK_ERROR_IN=env +ansible-inventory -i inventory_2.docker_machine.yml --list +unset MOCK_ERROR_IN + +echo "Deactivate Docker Machine mock" +PATH="${SAVED_PATH}" diff --git a/test/integration/targets/inventory_docker_machine/teardown.docker_machine.yml b/test/integration/targets/inventory_docker_machine/teardown.docker_machine.yml new file mode 100644 index 00000000000..6258853dd88 --- /dev/null +++ b/test/integration/targets/inventory_docker_machine/teardown.docker_machine.yml @@ -0,0 +1,3 @@ +plugin: docker_machine +daemon_env: skip +running_required: no