allow modules to set custom stats (#18946)

can be per run or per host, also aggregate or not
set_stats action plugin as reference implementation
added doc stub
display stats in calblack
made custom stats showing configurable

docsite/rst/intro_configuration.rst

@@ -1165,6 +1165,16 @@ The default behavior is no::
 
     libvirt_lxc_noseclabel = True
 
+.. _show_custom_stats:
+
+show_custom_stats
+=================
+
+.. versionadded:: 2.3
+
+If enabled, this setting will display custom stats (set via set_stats plugin) when using the default callback.
+
+
 Galaxy Settings
 ---------------
 

examples/ansible.cfg

@@ -276,6 +276,9 @@
 # it is False, then the last specified argument is used and the others are ignored.
 #merge_multiple_cli_flags = False
 
+# Controls showing custom stats at the end, off by default
+#show_custom_stats = True
+
 [privilege_escalation]
 #become=True
 #become_method=sudo

lib/ansible/constants.py

@@ -232,6 +232,7 @@ DEFAULT_INVENTORY_IGNORE  = get_config(p, DEFAULTS, 'inventory_ignore_extensions
 DEFAULT_VAR_COMPRESSION_LEVEL = get_config(p, DEFAULTS, 'var_compression_level', 'ANSIBLE_VAR_COMPRESSION_LEVEL', 0, value_type='integer')
 DEFAULT_INTERNAL_POLL_INTERVAL = get_config(p, DEFAULTS, 'internal_poll_interval', None, 0.001, value_type='float')
 ERROR_ON_MISSING_HANDLER  = get_config(p, DEFAULTS, 'error_on_missing_handler', 'ANSIBLE_ERROR_ON_MISSING_HANDLER', True, value_type='boolean')
+SHOW_CUSTOM_STATS = get_config(p, DEFAULTS, 'show_custom_stats', 'ANSIBLE_SHOW_CUSTOM_STATS', False, value_type='boolean')
 
 # static includes
 DEFAULT_TASK_INCLUDES_STATIC    = get_config(p, DEFAULTS, 'task_includes_static', 'ANSIBLE_TASK_INCLUDES_STATIC', False, value_type='boolean')

lib/ansible/executor/stats.py

@@ -19,6 +19,8 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+from ansible.utils.vars import merge_hash
+
 class AggregateStats:
     ''' holds stats about per-host activity during playbook runs '''
 
@@ -31,6 +33,9 @@ class AggregateStats:
         self.changed   = {}
         self.skipped   = {}
 
+        # user defined stats, which can be per host or global
+        self.custom = {}
+
     def increment(self, what, host):
         ''' helper function to bump a statistic '''
 
@@ -49,3 +54,31 @@ class AggregateStats:
             skipped     = self.skipped.get(host, 0)
         )
 
+    def set_custom_stats(self, which, what, host=None):
+        ''' allow setting of a custom stat'''
+
+        if host is None:
+            host = '_run'
+        if host not in  self.custom:
+            self.custom[host] = {which: what}
+        else:
+            self.custom[host][which] = what
+
+    def update_custom_stats(self, which, what, host=None):
+        ''' allow aggregation of a custom stat'''
+
+        if host is None:
+            host = '_run'
+        if host not in self.custom or which not in self.custom[host]:
+            return self.set_custom_stats(which, what, host)
+
+        # mismatching types
+        if type(what) != type(self.custom[host][which]):
+            return None
+
+        if isinstance(what, dict):
+            self.custom[host][which] =  merge_hash(self.custom[host][which], what)
+        else:
+            # let overloaded + take care of other types
+            self.custom[host][which] += what
+

lib/ansible/modules/files/find.py

@@ -323,6 +323,7 @@ def main():
     msg = ''
     looked = 0
     for npath in params['paths']:
+        npath = os.path.expanduser(os.path.expandvars(npath))
         if os.path.isdir(npath):
 
             ''' ignore followlinks for python version < 2.6 '''

lib/ansible/modules/utilities/logic/set_stats.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2016 Ansible RedHat, Inc
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+ANSIBLE_METADATA = {'status': ['preview'],
+                    'supported_by': 'community',
+                    'version': '1.0'}
+
+DOCUMENTATION = '''
+---
+author: "Brian Coca (@bcoca)"
+module: set_stats
+short_description: Set stats for the current ansible run
+description:
+     - This module allows setting/accumulating stats on the current ansible run, either per host of for all hosts in the run.
+options:
+  data:
+    description:
+      - A dictionary of which each key represents a stat (or variable) you want to keep track of
+    required: true
+  per_host:
+    description:
+        - boolean that indicates if the stats is per host or for all hosts in the run.
+    required: no
+    default: no
+  aggregate:
+    description:
+        - boolean that indicates if the provided value is aggregated to the existing stat C(yes) or will replace it C(no)
+    required: no
+    default: yes
+version_added: "2.3"
+'''
+
+EXAMPLES = '''
+# Aggregating packages_installed stat per host
+- set_stats:
+    data:
+      packages_installed: 31
+
+# Aggregating random stats for all hosts using complex arguments
+- set_stats:
+    data:
+      one_stat: 11
+      other_stat: "{{ local_var * 2 }}"
+      another_stat: "{{ some_registered_var.results | map(attribute='ansible_facts.some_fact') | list }}"
+    per_host: no
+
+
+# setting stats (not aggregating)
+- set_stats:
+    data:
+      the_answer: 42
+    aggregate: no
+'''

lib/ansible/plugins/action/set_stats.py

@@ -0,0 +1,73 @@
+# Copyright 2016 Ansible (RedHat, Inc)
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.compat.six import iteritems, string_types
+from ansible.constants import mk_boolean as boolean
+from ansible.plugins.action import ActionBase
+from ansible.utils.vars import isidentifier
+
+class ActionModule(ActionBase):
+
+    TRANSFERS_FILES = False
+
+    #TODO: document this in non-empty set_stats.py module
+    def run(self, tmp=None, task_vars=None):
+        if task_vars is None:
+            task_vars = dict()
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+
+        stats = {'data': {}, 'per_host': False, 'aggregate': True}
+
+        if self._task.args:
+            data = self._task.args.get('data', {})
+
+            if not isinstance(data, dict):
+                data = self._templar.template(data, convert_bare=False, fail_on_undefined=True)
+
+            if not isinstance(data, dict):
+                result['failed'] = True
+                result['msg'] = "The 'data' option needs to be a dictionary/hash"
+                return result
+
+            # set boolean options, defaults are set above in stats init
+            for opt in ['per_host', 'aggregate']:
+                val = self._task.args.get(opt, None)
+                if val is not None:
+                    if not isinstance(val, bool):
+                        stats[opt] = boolean(self._templar.template(val))
+                    else:
+                        stats[opt] = val
+
+            for (k, v) in iteritems(data):
+
+                k = self._templar.template(k)
+
+                if not isidentifier(k):
+                    result['failed'] = True
+                    result['msg'] = "The variable name '%s' is not valid. Variables must start with a letter or underscore character, and contain only letters, numbers and underscores." % k
+                    return result
+
+                stats['data'][k] = self._templar.template(v)
+
+        result['changed'] = False
+        result['ansible_stats'] = stats
+
+        return result

lib/ansible/plugins/callback/default.py

@@ -268,6 +268,22 @@ class CallbackModule(CallbackBase):
 
         self._display.display("", screen_only=True)
 
+        # print custom stats
+        if C.SHOW_CUSTOM_STATS and stats.custom:
+            self._display.banner("CUSTOM STATS: ")
+            # per host
+            #TODO: come up with 'pretty format'
+            for k in sorted(stats.custom.keys()):
+                if k == '_run':
+                    continue
+                self._display.display('\t%s: %s' % (k, self._dump_results(stats.custom[k], indent=1).replace('\n','')))
+
+            # print per run custom stats
+            if '_run' in stats.custom:
+                self._display.display("", screen_only=True)
+                self._display.display('\tRUN: %s' % self._dump_results(stats.custom['_run'], indent=1).replace('\n',''))
+            self._display.display("", screen_only=True)
+
     def v2_playbook_on_start(self, playbook):
         if self._display.verbosity > 1:
             from os.path import basename

lib/ansible/plugins/strategy/__init__.py

@@ -229,6 +229,13 @@ class StrategyBase:
             return
         display.debug("exiting _queue_task() for %s/%s" % (host.name, task.action))
 
+    def get_task_hosts(self, iterator, task_host, task):
+        if task.run_once:
+            host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
+        else:
+            host_list = [task_host]
+        return host_list
+
     def _process_pending_results(self, iterator, one_pass=False, max_passes=None):
         '''
         Reads results off the final queue and takes appropriate action
@@ -348,10 +355,7 @@ class StrategyBase:
 
             run_once = templar.template(original_task.run_once)
             if original_task.register:
-                if run_once:
+                host_list = self.get_task_hosts(iterator, original_host, original_task)
-                    host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
-                else:
-                    host_list = [original_host]
 
                 clean_copy = strip_internal_keys(task_result._result)
                 if 'invocation' in clean_copy:
@@ -477,7 +481,7 @@ class StrategyBase:
                         # this task added a new group (group_by module)
                         self._add_group(original_host, result_item)
 
-                    elif 'ansible_facts' in result_item:
+                    if 'ansible_facts' in result_item:
 
                         # if delegated fact and we are delegating facts, we need to change target host for them
                         if original_task.delegate_to is not None and original_task.delegate_facts:
@@ -491,30 +495,37 @@ class StrategyBase:
                         else:
                             actual_host = original_host
 
+                        host_list = self.get_task_hosts(iterator, actual_host, original_task)
                         if original_task.action == 'include_vars':
+
                             for (var_name, var_value) in iteritems(result_item['ansible_facts']):
                                 # find the host we're actually referring too here, which may
                                 # be a host that is not really in inventory at all
-
-                                if run_once:
-                                    host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
-                                else:
-                                    host_list = [actual_host]
-
                                 for target_host in host_list:
                                     self._variable_manager.set_host_variable(target_host, var_name, var_value)
                         else:
-                            if run_once:
-                                host_list = [host for host in self._inventory.get_hosts(iterator._play.hosts) if host.name not in self._tqm._unreachable_hosts]
-                            else:
-                                host_list = [actual_host]
-
                             for target_host in host_list:
                                 if original_task.action == 'set_fact':
                                     self._variable_manager.set_nonpersistent_facts(target_host, result_item['ansible_facts'].copy())
                                 else:
                                     self._variable_manager.set_host_facts(target_host, result_item['ansible_facts'].copy())
 
+                    if 'ansible_stats' in result_item and 'data' in result_item['ansible_stats'] and result_item['ansible_stats']['data']:
+
+                        if 'per_host' not in result_item['ansible_stats'] or result_item['ansible_stats']['per_host']:
+                            host_list = self.get_task_hosts(iterator, original_host, original_task)
+                        else:
+                            host_list = [None]
+
+                        data = result_item['ansible_stats']['data']
+                        aggregate = 'aggregate' in result_item['ansible_stats'] and result_item['ansible_stats']['aggregate']
+                        for myhost in host_list:
+                            for k in data.keys():
+                                if aggregate:
+                                    self._tqm._stats.update_custom_stats(k, data[k], myhost)
+                                else:
+                                    self._tqm._stats.set_custom_stats(k, data[k], myhost)
+
                 if 'diff' in task_result._result:
                     if self._diff:
                         self._tqm.send_callback('v2_on_file_diff', task_result)