From 08e0f6ada5905ce53ec581a28d8bd4f5840bf732 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 5 Jan 2017 16:38:36 -0500 Subject: [PATCH] allow modules to set custom stats (#18946) can be per run or per host, also aggregate or not set_stats action plugin as reference implementation added doc stub display stats in calblack made custom stats showing configurable --- docsite/rst/intro_configuration.rst | 10 +++ examples/ansible.cfg | 3 + lib/ansible/constants.py | 1 + lib/ansible/executor/stats.py | 33 +++++++++ lib/ansible/modules/files/find.py | 1 + .../modules/utilities/logic/set_stats.py | 69 ++++++++++++++++++ lib/ansible/plugins/action/set_stats.py | 73 +++++++++++++++++++ lib/ansible/plugins/callback/default.py | 16 ++++ lib/ansible/plugins/strategy/__init__.py | 43 +++++++---- 9 files changed, 233 insertions(+), 16 deletions(-) create mode 100644 lib/ansible/modules/utilities/logic/set_stats.py create mode 100644 lib/ansible/plugins/action/set_stats.py diff --git a/docsite/rst/intro_configuration.rst b/docsite/rst/intro_configuration.rst index a87a7ed6153..d008cf4a96a 100644 --- a/docsite/rst/intro_configuration.rst +++ b/docsite/rst/intro_configuration.rst @@ -1165,6 +1165,16 @@ The default behavior is no:: libvirt_lxc_noseclabel = True +.. _show_custom_stats: + +show_custom_stats +================= + +.. versionadded:: 2.3 + +If enabled, this setting will display custom stats (set via set_stats plugin) when using the default callback. + + Galaxy Settings --------------- diff --git a/examples/ansible.cfg b/examples/ansible.cfg index 89da92a3f80..054b02dd150 100644 --- a/examples/ansible.cfg +++ b/examples/ansible.cfg @@ -276,6 +276,9 @@ # it is False, then the last specified argument is used and the others are ignored. #merge_multiple_cli_flags = False +# Controls showing custom stats at the end, off by default +#show_custom_stats = True + [privilege_escalation] #become=True #become_method=sudo diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index e486af6e417..13727e24dbc 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -232,6 +232,7 @@ DEFAULT_INVENTORY_IGNORE = get_config(p, DEFAULTS, 'inventory_ignore_extensions DEFAULT_VAR_COMPRESSION_LEVEL = get_config(p, DEFAULTS, 'var_compression_level', 'ANSIBLE_VAR_COMPRESSION_LEVEL', 0, value_type='integer') DEFAULT_INTERNAL_POLL_INTERVAL = get_config(p, DEFAULTS, 'internal_poll_interval', None, 0.001, value_type='float') ERROR_ON_MISSING_HANDLER = get_config(p, DEFAULTS, 'error_on_missing_handler', 'ANSIBLE_ERROR_ON_MISSING_HANDLER', True, value_type='boolean') +SHOW_CUSTOM_STATS = get_config(p, DEFAULTS, 'show_custom_stats', 'ANSIBLE_SHOW_CUSTOM_STATS', False, value_type='boolean') # static includes DEFAULT_TASK_INCLUDES_STATIC = get_config(p, DEFAULTS, 'task_includes_static', 'ANSIBLE_TASK_INCLUDES_STATIC', False, value_type='boolean') diff --git a/lib/ansible/executor/stats.py b/lib/ansible/executor/stats.py index 626b2959a47..989a380be7d 100644 --- a/lib/ansible/executor/stats.py +++ b/lib/ansible/executor/stats.py @@ -19,6 +19,8 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +from ansible.utils.vars import merge_hash + class AggregateStats: ''' holds stats about per-host activity during playbook runs ''' @@ -31,6 +33,9 @@ class AggregateStats: self.changed = {} self.skipped = {} + # user defined stats, which can be per host or global + self.custom = {} + def increment(self, what, host): ''' helper function to bump a statistic ''' @@ -49,3 +54,31 @@ class AggregateStats: skipped = self.skipped.get(host, 0) ) + def set_custom_stats(self, which, what, host=None): + ''' allow setting of a custom stat''' + + if host is None: + host = '_run' + if host not in self.custom: + self.custom[host] = {which: what} + else: + self.custom[host][which] = what + + def update_custom_stats(self, which, what, host=None): + ''' allow aggregation of a custom stat''' + + if host is None: + host = '_run' + if host not in self.custom or which not in self.custom[host]: + return self.set_custom_stats(which, what, host) + + # mismatching types + if type(what) != type(self.custom[host][which]): + return None + + if isinstance(what, dict): + self.custom[host][which] = merge_hash(self.custom[host][which], what) + else: + # let overloaded + take care of other types + self.custom[host][which] += what + diff --git a/lib/ansible/modules/files/find.py b/lib/ansible/modules/files/find.py index e2988aa36e9..eee57e44fcb 100644 --- a/lib/ansible/modules/files/find.py +++ b/lib/ansible/modules/files/find.py @@ -323,6 +323,7 @@ def main(): msg = '' looked = 0 for npath in params['paths']: + npath = os.path.expanduser(os.path.expandvars(npath)) if os.path.isdir(npath): ''' ignore followlinks for python version < 2.6 ''' diff --git a/lib/ansible/modules/utilities/logic/set_stats.py b/lib/ansible/modules/utilities/logic/set_stats.py new file mode 100644 index 00000000000..4c4e88b5134 --- /dev/null +++ b/lib/ansible/modules/utilities/logic/set_stats.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright 2016 Ansible RedHat, Inc +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +ANSIBLE_METADATA = {'status': ['preview'], + 'supported_by': 'community', + 'version': '1.0'} + +DOCUMENTATION = ''' +--- +author: "Brian Coca (@bcoca)" +module: set_stats +short_description: Set stats for the current ansible run +description: + - This module allows setting/accumulating stats on the current ansible run, either per host of for all hosts in the run. +options: + data: + description: + - A dictionary of which each key represents a stat (or variable) you want to keep track of + required: true + per_host: + description: + - boolean that indicates if the stats is per host or for all hosts in the run. + required: no + default: no + aggregate: + description: + - boolean that indicates if the provided value is aggregated to the existing stat C(yes) or will replace it C(no) + required: no + default: yes +version_added: "2.3" +''' + +EXAMPLES = ''' +# Aggregating packages_installed stat per host +- set_stats: + data: + packages_installed: 31 + +# Aggregating random stats for all hosts using complex arguments +- set_stats: + data: + one_stat: 11 + other_stat: "{{ local_var * 2 }}" + another_stat: "{{ some_registered_var.results | map(attribute='ansible_facts.some_fact') | list }}" + per_host: no + + +# setting stats (not aggregating) +- set_stats: + data: + the_answer: 42 + aggregate: no +''' diff --git a/lib/ansible/plugins/action/set_stats.py b/lib/ansible/plugins/action/set_stats.py new file mode 100644 index 00000000000..05366a8a164 --- /dev/null +++ b/lib/ansible/plugins/action/set_stats.py @@ -0,0 +1,73 @@ +# Copyright 2016 Ansible (RedHat, Inc) +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.compat.six import iteritems, string_types +from ansible.constants import mk_boolean as boolean +from ansible.plugins.action import ActionBase +from ansible.utils.vars import isidentifier + +class ActionModule(ActionBase): + + TRANSFERS_FILES = False + + #TODO: document this in non-empty set_stats.py module + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + + stats = {'data': {}, 'per_host': False, 'aggregate': True} + + if self._task.args: + data = self._task.args.get('data', {}) + + if not isinstance(data, dict): + data = self._templar.template(data, convert_bare=False, fail_on_undefined=True) + + if not isinstance(data, dict): + result['failed'] = True + result['msg'] = "The 'data' option needs to be a dictionary/hash" + return result + + # set boolean options, defaults are set above in stats init + for opt in ['per_host', 'aggregate']: + val = self._task.args.get(opt, None) + if val is not None: + if not isinstance(val, bool): + stats[opt] = boolean(self._templar.template(val)) + else: + stats[opt] = val + + for (k, v) in iteritems(data): + + k = self._templar.template(k) + + if not isidentifier(k): + result['failed'] = True + result['msg'] = "The variable name '%s' is not valid. Variables must start with a letter or underscore character, and contain only letters, numbers and underscores." % k + return result + + stats['data'][k] = self._templar.template(v) + + result['changed'] = False + result['ansible_stats'] = stats + + return result diff --git a/lib/ansible/plugins/callback/default.py b/lib/ansible/plugins/callback/default.py index 19e19286cf8..4b763735ef5 100644 --- a/lib/ansible/plugins/callback/default.py +++ b/lib/ansible/plugins/callback/default.py @@ -268,6 +268,22 @@ class CallbackModule(CallbackBase): self._display.display("", screen_only=True) + # print custom stats + if C.SHOW_CUSTOM_STATS and stats.custom: + self._display.banner("CUSTOM STATS: ") + # per host + #TODO: come up with 'pretty format' + for k in sorted(stats.custom.keys()): + if k == '_run': + continue + self._display.display('\t%s: %s' % (k, self._dump_results(stats.custom[k], indent=1).replace('\n',''))) + + # print per run custom stats + if '_run' in stats.custom: + self._display.display("", screen_only=True) + self._display.display('\tRUN: %s' % self._dump_results(stats.custom['_run'], indent=1).replace('\n','')) + self._display.display("", screen_only=True) + def v2_playbook_on_start(self, playbook): if self._display.verbosity > 1: from os.path import basename diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index f75ffdac97f..9c0d2a30e1f 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -229,6 +229,13 @@ class StrategyBase: return display.debug("exiting _queue_task() for %s/%s" % (host.name, task.action)) + def get_task_hosts(self, iterator, task_host, task): + if task.run_once: + host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts] + else: + host_list = [task_host] + return host_list + def _process_pending_results(self, iterator, one_pass=False, max_passes=None): ''' Reads results off the final queue and takes appropriate action @@ -348,10 +355,7 @@ class StrategyBase: run_once = templar.template(original_task.run_once) if original_task.register: - if run_once: - host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts] - else: - host_list = [original_host] + host_list = self.get_task_hosts(iterator, original_host, original_task) clean_copy = strip_internal_keys(task_result._result) if 'invocation' in clean_copy: @@ -477,7 +481,7 @@ class StrategyBase: # this task added a new group (group_by module) self._add_group(original_host, result_item) - elif 'ansible_facts' in result_item: + if 'ansible_facts' in result_item: # if delegated fact and we are delegating facts, we need to change target host for them if original_task.delegate_to is not None and original_task.delegate_facts: @@ -491,30 +495,37 @@ class StrategyBase: else: actual_host = original_host + host_list = self.get_task_hosts(iterator, actual_host, original_task) if original_task.action == 'include_vars': + for (var_name, var_value) in iteritems(result_item['ansible_facts']): # find the host we're actually referring too here, which may # be a host that is not really in inventory at all - - if run_once: - host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts] - else: - host_list = [actual_host] - for target_host in host_list: self._variable_manager.set_host_variable(target_host, var_name, var_value) else: - if run_once: - host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts] - else: - host_list = [actual_host] - for target_host in host_list: if original_task.action == 'set_fact': self._variable_manager.set_nonpersistent_facts(target_host, result_item['ansible_facts'].copy()) else: self._variable_manager.set_host_facts(target_host, result_item['ansible_facts'].copy()) + if 'ansible_stats' in result_item and 'data' in result_item['ansible_stats'] and result_item['ansible_stats']['data']: + + if 'per_host' not in result_item['ansible_stats'] or result_item['ansible_stats']['per_host']: + host_list = self.get_task_hosts(iterator, original_host, original_task) + else: + host_list = [None] + + data = result_item['ansible_stats']['data'] + aggregate = 'aggregate' in result_item['ansible_stats'] and result_item['ansible_stats']['aggregate'] + for myhost in host_list: + for k in data.keys(): + if aggregate: + self._tqm._stats.update_custom_stats(k, data[k], myhost) + else: + self._tqm._stats.set_custom_stats(k, data[k], myhost) + if 'diff' in task_result._result: if self._diff: self._tqm.send_callback('v2_on_file_diff', task_result)