diff --git a/lib/ansible/plugins/callback/foreman.py b/lib/ansible/plugins/callback/foreman.py deleted file mode 100644 index 2b6a8a0c5e4..00000000000 --- a/lib/ansible/plugins/callback/foreman.py +++ /dev/null @@ -1,253 +0,0 @@ -# -*- coding: utf-8 -*- -# (c) 2015, 2016 Daniel Lobato -# (c) 2016 Guido Günther -# (c) 2017 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -DOCUMENTATION = ''' - callback: foreman - type: notification - short_description: Sends events to Foreman - description: - - This callback will report facts and task events to Foreman https://theforeman.org/ - - Before 2.4, if you wanted to use an ini configuration, the file must be placed in the same directory as this plugin and named foreman.ini - - In 2.4 and above you can just put it in the main Ansible configuration file. - version_added: "2.2" - requirements: - - whitelisting in configuration - - requests (python library) - options: - url: - description: URL to the Foreman server - env: - - name: FOREMAN_URL - required: True - default: http://localhost:3000 - ini: - - section: callback_foreman - key: url - client_cert: - description: X509 certificate to authenticate to Foreman if https is used - env: - - name: FOREMAN_SSL_CERT - default: /etc/foreman/client_cert.pem - ini: - - section: callback_foreman - key: ssl_cert - - section: callback_foreman - key: client_cert - aliases: [ ssl_cert ] - client_key: - description: the corresponding private key - env: - - name: FOREMAN_SSL_KEY - default: /etc/foreman/client_key.pem - ini: - - section: callback_foreman - key: ssl_key - - section: callback_foreman - key: client_key - aliases: [ ssl_key ] - verify_certs: - description: - - Toggle to decide whether to verify the Foreman certificate. - - It can be set to '1' to verify SSL certificates using the installed CAs or to a path pointing to a CA bundle. - - Set to '0' to disable certificate checking. - env: - - name: FOREMAN_SSL_VERIFY - default: 1 - ini: - - section: callback_foreman - key: verify_certs -''' - -import os -from datetime import datetime -from collections import defaultdict -import json -import time - -try: - import requests - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - -from ansible.module_utils._text import to_text -from ansible.plugins.callback import CallbackBase - - -class CallbackModule(CallbackBase): - CALLBACK_VERSION = 2.0 - CALLBACK_TYPE = 'notification' - CALLBACK_NAME = 'foreman' - CALLBACK_NEEDS_WHITELIST = True - - FOREMAN_HEADERS = { - "Content-Type": "application/json", - "Accept": "application/json" - } - TIME_FORMAT = "%Y-%m-%d %H:%M:%S %f" - - def __init__(self): - super(CallbackModule, self).__init__() - self.items = defaultdict(list) - self.start_time = int(time.time()) - - def set_options(self, task_keys=None, var_options=None, direct=None): - - super(CallbackModule, self).set_options(task_keys=task_keys, var_options=var_options, direct=direct) - - self.FOREMAN_URL = self.get_option('url') - self.FOREMAN_SSL_CERT = (self.get_option('client_cert'), self.get_option('client_key')) - self.FOREMAN_SSL_VERIFY = str(self.get_option('verify_certs')) - - self.ssl_verify = self._ssl_verify() - - if HAS_REQUESTS: - requests_major = int(requests.__version__.split('.')[0]) - if requests_major < 2: - self._disable_plugin('The `requests` python module is too old.') - else: - self._disable_plugin('The `requests` python module is not installed.') - - if self.FOREMAN_URL.startswith('https://'): - if not os.path.exists(self.FOREMAN_SSL_CERT[0]): - self._disable_plugin('FOREMAN_SSL_CERT %s not found.' % self.FOREMAN_SSL_CERT[0]) - - if not os.path.exists(self.FOREMAN_SSL_CERT[1]): - self._disable_plugin('FOREMAN_SSL_KEY %s not found.' % self.FOREMAN_SSL_CERT[1]) - - def _disable_plugin(self, msg): - self.disabled = True - if msg: - self._display.warning(msg + ' Disabling the Foreman callback plugin.') - else: - self._display.warning('Disabling the Foreman callback plugin.') - - def _ssl_verify(self): - if self.FOREMAN_SSL_VERIFY.lower() in ["1", "true", "on"]: - verify = True - elif self.FOREMAN_SSL_VERIFY.lower() in ["0", "false", "off"]: - requests.packages.urllib3.disable_warnings() - self._display.warning("SSL verification of %s disabled" % - self.FOREMAN_URL) - verify = False - else: # Set to a CA bundle: - verify = self.FOREMAN_SSL_VERIFY - return verify - - def send_facts(self, host, data): - """ - Sends facts to Foreman, to be parsed by foreman_ansible fact - parser. The default fact importer should import these facts - properly. - """ - data["_type"] = "ansible" - data["_timestamp"] = datetime.now().strftime(self.TIME_FORMAT) - facts = {"name": host, - "facts": data, - } - try: - r = requests.post(url=self.FOREMAN_URL + '/api/v2/hosts/facts', - data=json.dumps(facts), - headers=self.FOREMAN_HEADERS, - cert=self.FOREMAN_SSL_CERT, - verify=self.ssl_verify) - r.raise_for_status() - except requests.exceptions.RequestException as err: - print(to_text(err)) - - def _build_log(self, data): - logs = [] - for entry in data: - source, msg = entry - if 'failed' in msg: - level = 'err' - elif 'changed' in msg and msg['changed']: - level = 'notice' - else: - level = 'info' - logs.append({ - "log": { - 'sources': { - 'source': source - }, - 'messages': { - 'message': json.dumps(msg) - }, - 'level': level - } - }) - return logs - - def send_reports(self, stats): - """ - Send reports to Foreman to be parsed by its config report - importer. THe data is in a format that Foreman can handle - without writing another report importer. - """ - status = defaultdict(lambda: 0) - metrics = {} - - for host in stats.processed.keys(): - sum = stats.summarize(host) - status["applied"] = sum['changed'] - status["failed"] = sum['failures'] + sum['unreachable'] - status["skipped"] = sum['skipped'] - log = self._build_log(self.items[host]) - metrics["time"] = {"total": int(time.time()) - self.start_time} - now = datetime.now().strftime(self.TIME_FORMAT) - report = { - "config_report": { - "host": host, - "reported_at": now, - "metrics": metrics, - "status": status, - "logs": log, - } - } - try: - r = requests.post(url=self.FOREMAN_URL + '/api/v2/config_reports', - data=json.dumps(report), - headers=self.FOREMAN_HEADERS, - cert=self.FOREMAN_SSL_CERT, - verify=self.ssl_verify) - r.raise_for_status() - except requests.exceptions.RequestException as err: - print(to_text(err)) - self.items[host] = [] - - def append_result(self, result): - name = result._task.get_name() - host = result._host.get_name() - self.items[host].append((name, result._result)) - - # Ansible callback API - def v2_runner_on_failed(self, result, ignore_errors=False): - self.append_result(result) - - def v2_runner_on_unreachable(self, result): - self.append_result(result) - - def v2_runner_on_async_ok(self, result, jid): - self.append_result(result) - - def v2_runner_on_async_failed(self, result, jid): - self.append_result(result) - - def v2_playbook_on_stats(self, stats): - self.send_reports(stats) - - def v2_runner_on_ok(self, result): - res = result._result - module = result._task.action - - if module == 'setup' or 'ansible_facts' in res: - host = result._host.get_name() - self.send_facts(host, res) - else: - self.append_result(result) diff --git a/lib/ansible/plugins/inventory/foreman.py b/lib/ansible/plugins/inventory/foreman.py deleted file mode 100644 index 43073f81ad4..00000000000 --- a/lib/ansible/plugins/inventory/foreman.py +++ /dev/null @@ -1,295 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2016 Guido Günther , Daniel Lobato Garcia -# Copyright (c) 2018 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -DOCUMENTATION = ''' - name: foreman - plugin_type: inventory - short_description: foreman inventory source - version_added: "2.6" - requirements: - - requests >= 1.1 - description: - - Get inventory hosts from the foreman service. - - "Uses a configuration file as an inventory source, it must end in ``.foreman.yml`` or ``.foreman.yaml`` and has a ``plugin: foreman`` entry." - extends_documentation_fragment: - - inventory_cache - - constructed - options: - plugin: - description: the name of this plugin, it should always be set to 'foreman' for this plugin to recognize it as it's own. - required: True - choices: ['foreman'] - url: - description: url to foreman - default: 'http://localhost:3000' - env: - - name: FOREMAN_SERVER - version_added: "2.8" - user: - description: foreman authentication user - required: True - env: - - name: FOREMAN_USER - version_added: "2.8" - password: - description: foreman authentication password - required: True - env: - - name: FOREMAN_PASSWORD - version_added: "2.8" - validate_certs: - description: verify SSL certificate if using https - type: boolean - default: False - group_prefix: - description: prefix to apply to foreman groups - default: foreman_ - vars_prefix: - description: prefix to apply to host variables, does not include facts nor params - default: foreman_ - want_facts: - description: Toggle, if True the plugin will retrieve host facts from the server - type: boolean - default: False - want_params: - description: Toggle, if true the inventory will retrieve 'all_parameters' information as host vars - type: boolean - default: False - want_hostcollections: - description: Toggle, if true the plugin will create Ansible groups for host collections - type: boolean - default: False - version_added: '2.10' - want_ansible_ssh_host: - description: Toggle, if true the plugin will populate the ansible_ssh_host variable to explicitly specify the connection target - type: boolean - default: False - version_added: '2.10' - -''' - -EXAMPLES = ''' -# my.foreman.yml -plugin: foreman -url: http://localhost:2222 -user: ansible-tester -password: secure -validate_certs: False -''' - -from distutils.version import LooseVersion - -from ansible.errors import AnsibleError -from ansible.module_utils._text import to_bytes, to_native, to_text -from ansible.module_utils.common._collections_compat import MutableMapping -from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable, to_safe_group_name, Constructable - -# 3rd party imports -try: - import requests - if LooseVersion(requests.__version__) < LooseVersion('1.1.0'): - raise ImportError -except ImportError: - raise AnsibleError('This script requires python-requests 1.1 as a minimum version') - -from requests.auth import HTTPBasicAuth - - -class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable): - ''' Host inventory parser for ansible using foreman as source. ''' - - NAME = 'foreman' - - def __init__(self): - - super(InventoryModule, self).__init__() - - # from config - self.foreman_url = None - - self.session = None - self.cache_key = None - self.use_cache = None - - def verify_file(self, path): - - valid = False - if super(InventoryModule, self).verify_file(path): - if path.endswith(('foreman.yaml', 'foreman.yml')): - valid = True - else: - self.display.vvv('Skipping due to inventory source not ending in "foreman.yaml" nor "foreman.yml"') - return valid - - def _get_session(self): - if not self.session: - self.session = requests.session() - self.session.auth = HTTPBasicAuth(self.get_option('user'), to_bytes(self.get_option('password'))) - self.session.verify = self.get_option('validate_certs') - return self.session - - def _get_json(self, url, ignore_errors=None): - - if not self.use_cache or url not in self._cache.get(self.cache_key, {}): - - if self.cache_key not in self._cache: - self._cache[self.cache_key] = {url: ''} - - results = [] - s = self._get_session() - params = {'page': 1, 'per_page': 250} - while True: - ret = s.get(url, params=params) - if ignore_errors and ret.status_code in ignore_errors: - break - ret.raise_for_status() - json = ret.json() - - # process results - # FIXME: This assumes 'return type' matches a specific query, - # it will break if we expand the queries and they dont have different types - if 'results' not in json: - # /hosts/:id dos not have a 'results' key - results = json - break - elif isinstance(json['results'], MutableMapping): - # /facts are returned as dict in 'results' - results = json['results'] - break - else: - # /hosts 's 'results' is a list of all hosts, returned is paginated - results = results + json['results'] - - # check for end of paging - if len(results) >= json['subtotal']: - break - if len(json['results']) == 0: - self.display.warning("Did not make any progress during loop. expected %d got %d" % (json['subtotal'], len(results))) - break - - # get next page - params['page'] += 1 - - self._cache[self.cache_key][url] = results - - return self._cache[self.cache_key][url] - - def _get_hosts(self): - return self._get_json("%s/api/v2/hosts" % self.foreman_url) - - def _get_all_params_by_id(self, hid): - url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid) - ret = self._get_json(url, [404]) - if not ret or not isinstance(ret, MutableMapping) or not ret.get('all_parameters', False): - return {} - return ret.get('all_parameters') - - def _get_facts_by_id(self, hid): - url = "%s/api/v2/hosts/%s/facts" % (self.foreman_url, hid) - return self._get_json(url) - - def _get_host_data_by_id(self, hid): - url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid) - return self._get_json(url) - - def _get_facts(self, host): - """Fetch all host facts of the host""" - - ret = self._get_facts_by_id(host['id']) - if len(ret.values()) == 0: - facts = {} - elif len(ret.values()) == 1: - facts = list(ret.values())[0] - else: - raise ValueError("More than one set of facts returned for '%s'" % host) - return facts - - def _populate(self): - - for host in self._get_hosts(): - - if host.get('name'): - host_name = self.inventory.add_host(host['name']) - - # create directly mapped groups - group_name = host.get('hostgroup_title', host.get('hostgroup_name')) - if group_name: - group_name = to_safe_group_name('%s%s' % (self.get_option('group_prefix'), group_name.lower().replace(" ", ""))) - group_name = self.inventory.add_group(group_name) - self.inventory.add_child(group_name, host_name) - - # set host vars from host info - try: - for k, v in host.items(): - if k not in ('name', 'hostgroup_title', 'hostgroup_name'): - try: - self.inventory.set_variable(host_name, self.get_option('vars_prefix') + k, v) - except ValueError as e: - self.display.warning("Could not set host info hostvar for %s, skipping %s: %s" % (host, k, to_text(e))) - except ValueError as e: - self.display.warning("Could not get host info for %s, skipping: %s" % (host_name, to_text(e))) - - # set host vars from params - if self.get_option('want_params'): - for p in self._get_all_params_by_id(host['id']): - try: - self.inventory.set_variable(host_name, p['name'], p['value']) - except ValueError as e: - self.display.warning("Could not set hostvar %s to '%s' for the '%s' host, skipping: %s" % - (p['name'], to_native(p['value']), host, to_native(e))) - - # set host vars from facts - if self.get_option('want_facts'): - self.inventory.set_variable(host_name, 'foreman_facts', self._get_facts(host)) - - # create group for host collections - if self.get_option('want_hostcollections'): - host_data = self._get_host_data_by_id(host['id']) - hostcollections = host_data.get('host_collections') - if hostcollections: - # Create Ansible groups for host collections - for hostcollection in hostcollections: - try: - hostcollection_group = to_safe_group_name('%shostcollection_%s' % (self.get_option('group_prefix'), - hostcollection['name'].lower().replace(" ", ""))) - hostcollection_group = self.inventory.add_group(hostcollection_group) - self.inventory.add_child(hostcollection_group, host_name) - except ValueError as e: - self.display.warning("Could not create groups for host collections for %s, skipping: %s" % (host_name, to_text(e))) - - # put ansible_ssh_host as hostvar - if self.get_option('want_ansible_ssh_host'): - for key in ('ip', 'ipv4', 'ipv6'): - if host.get(key): - try: - self.inventory.set_variable(host_name, 'ansible_ssh_host', host[key]) - break - except ValueError as e: - self.display.warning("Could not set hostvar ansible_ssh_host to '%s' for the '%s' host, skipping: %s" % - (host[key], host_name, to_text(e))) - - strict = self.get_option('strict') - - hostvars = self.inventory.get_host(host_name).get_vars() - self._set_composite_vars(self.get_option('compose'), hostvars, host_name, strict) - self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host_name, strict) - self._add_host_to_keyed_groups(self.get_option('keyed_groups'), hostvars, host_name, strict) - - def parse(self, inventory, loader, path, cache=True): - - super(InventoryModule, self).parse(inventory, loader, path) - - # read config from file, this sets 'options' - self._read_config_data(path) - - # get connection host - self.foreman_url = self.get_option('url') - self.cache_key = self.get_cache_key(path) - self.use_cache = cache and self.get_option('cache') - - # actually populate inventory - self._populate() diff --git a/test/integration/targets/inventory_foreman/aliases b/test/integration/targets/inventory_foreman/aliases deleted file mode 100644 index 641e938ad23..00000000000 --- a/test/integration/targets/inventory_foreman/aliases +++ /dev/null @@ -1,3 +0,0 @@ -shippable/cloud/group1 -cloud/foreman -destructive diff --git a/test/integration/targets/inventory_foreman/ansible.cfg b/test/integration/targets/inventory_foreman/ansible.cfg deleted file mode 100644 index 63e24c4bd09..00000000000 --- a/test/integration/targets/inventory_foreman/ansible.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[defaults] -inventory = test-config.foreman.yaml - -[inventory] -enable_plugins = foreman diff --git a/test/integration/targets/inventory_foreman/inspect_cache.yml b/test/integration/targets/inventory_foreman/inspect_cache.yml deleted file mode 100644 index c91f4c38681..00000000000 --- a/test/integration/targets/inventory_foreman/inspect_cache.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- hosts: localhost - vars: - foreman_stub_host: "{{ lookup('env', 'FOREMAN_HOST') }}" - foreman_stub_port: "{{ lookup('env', 'FOREMAN_PORT') }}" - foreman_stub_api_path: /api/v2 - cached_hosts_key: "http://{{ foreman_stub_host }}:{{ foreman_stub_port }}{{ foreman_stub_api_path }}/hosts" - tasks: - - name: verify a cache file was created - find: - path: - - ./foreman_cache - register: matching_files - - - assert: - that: - - matching_files.matched == 1 - - name: read the cached inventory - set_fact: - contents: "{{ lookup('file', matching_files.files.0.path) }}" - - - name: extract all the host names - set_fact: - cached_hosts: "{{ contents[cached_hosts_key] | json_query('[*].name') }}" - - - assert: - that: - "'{{ item }}' in cached_hosts" - loop: - - "v6.example-780.com" - - "c4.j1.y5.example-487.com" diff --git a/test/integration/targets/inventory_foreman/runme.sh b/test/integration/targets/inventory_foreman/runme.sh deleted file mode 100755 index ba94a9360f5..00000000000 --- a/test/integration/targets/inventory_foreman/runme.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "$DEBUG" || -n "$ANSIBLE_DEBUG" ]] && set -x - -set -euo pipefail - -export ANSIBLE_INVENTORY -export ANSIBLE_PYTHON_INTERPRETER - -unset ANSIBLE_INVENTORY -unset ANSIBLE_PYTHON_INTERPRETER - -export ANSIBLE_CONFIG=ansible.cfg -export FOREMAN_HOST="${FOREMAN_HOST:-localhost}" -export FOREMAN_PORT="${FOREMAN_PORT:-8080}" -FOREMAN_CONFIG=test-config.foreman.yaml - -# Set inventory caching environment variables to populate a jsonfile cache -export ANSIBLE_INVENTORY_CACHE=True -export ANSIBLE_INVENTORY_CACHE_PLUGIN=jsonfile -export ANSIBLE_INVENTORY_CACHE_CONNECTION=./foreman_cache - -# flag for checking whether cleanup has already fired -_is_clean= - -function _cleanup() { - [[ -n "$_is_clean" ]] && return # don't double-clean - echo Cleanup: removing $FOREMAN_CONFIG... - rm -vf "$FOREMAN_CONFIG" - unset ANSIBLE_CONFIG - unset FOREMAN_HOST - unset FOREMAN_PORT - unset FOREMAN_CONFIG - _is_clean=1 -} -trap _cleanup INT TERM EXIT - -cat > "$FOREMAN_CONFIG" <- - Foreman host: {{ foreman_stub_host }} | - Foreman port: {{ foreman_stub_port }} | - API path: {{ foreman_stub_api_path }} | - Foreman API URL: {{ foreman_stub_api_uri }} - - - name: Wait for Foreman API stub to come up online - wait_for: - host: "{{ foreman_stub_host }}" - port: "{{ foreman_stub_port }}" - state: started - - # smoke test that flask app is serving - - name: Smoke test HTTP response from Foreman stub - uri: - url: "{{ foreman_stub_heartbeat_uri }}" - return_content: yes - register: heartbeat_resp - failed_when: > - heartbeat_resp.json.status != 'ok' or heartbeat_resp.json.response != 'pong' - - #### Testing start - - name: > - Check that there are 'foreman_pgagne_sats' and 'foreman_base' - groups present in inventory - assert: - that: > - '{{ item }}' in groups - with_items: - - foreman_pgagne_sats - - foreman_base - - - name: Check that host are in appropriate groups - assert: - that: > - '{{ item.key }}' in groups['{{ item.value }}'] - with_dict: - v6.example-780.com: foreman_base - c4.j1.y5.example-487.com: ungrouped - - - name: Check host UUIDs - assert: - that: > - hostvars['{{ item.key }}']['foreman_subscription_facet_attributes']['uuid'] == '{{ item.value }}' - with_dict: - v6.example-780.com: 2c72fa49-995a-4bbf-bda0-684c7048ad9f - c4.j1.y5.example-487.com: 0a494b6e-7e90-4ed2-8edc-43a41436a242 - #### Testing end