Make it so callback plugins can act on implicit/explicit meta tasks (#71009)

Change:
- Now sends meta tasks to the task start callback
- Lets callback plugins opt-in to receiving implicit tasks

Test Plan:
- New integration tests

Tickets:
- Indirectly fixes #71007 by allowing custom callbacks with this data

Signed-off-by: Rick Elrod <rick@elrod.me>

changelogs/fragments/71007_callback_on_meta.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - callback plugins - ``meta`` tasks now get sent to ``v2_playbook_on_task_start``. Explicit tasks are always sent. Plugins can opt in to receiving implicit ones.

docs/docsite/rst/dev_guide/developing_plugins.rst

@@ -278,6 +278,10 @@ Note that the ``CALLBACK_VERSION`` and ``CALLBACK_NAME`` definitions are require
 
 For example callback plugins, see the source code for the `callback plugins included with Ansible Core <https://github.com/ansible/ansible/tree/devel/lib/ansible/plugins/callback>`_
 
+New in ansible-base 2.11, callback plugins are notified (via ``v2_playbook_on_task_start``) of :ref:`meta<meta_module>` tasks. By default, only explicit ``meta`` tasks that users list in their plays are sent to callbacks.
+
+There are also some tasks which are generated internally and implicitly at various points in execution. Callback plugins can opt-in to receiving these implicit tasks as well, by setting ``self.wants_implicit_tasks = True``. Any ``Task`` object received by a callback hook will have an ``.implicit`` attribute, which can be consulted to determine whether the ``Task`` originated from within Ansible, or explicitly by the user.
+
 .. _developing_connection_plugins:
 
 Connection plugins

docs/docsite/rst/porting_guides/porting_guide_base_2.11.rst

@@ -66,6 +66,7 @@ Plugins
 =======
 
 * inventory plugins - ``CachePluginAdjudicator.flush()`` now calls the underlying cache plugin's ``flush()`` instead of only deleting keys that it knows about. Inventory plugins should use ``delete()`` to remove any specific keys. As a user, this means that when an inventory plugin calls its ``clear_cache()`` method, facts could also be flushed from the cache. To work around this, users can configure inventory plugins to use a cache backend that is independent of the facts cache.
+* callback plugins - ``meta`` task execution is now sent to ``v2_playbook_on_task_start`` like any other task. By default, only explicit meta tasks are sent there. Callback plugins can opt-in to receiving internal, implicitly created tasks to act on those as well, as noted in the plugin development documentation.
 
 Porting custom scripts
 ======================

lib/ansible/executor/task_queue_manager.py

@@ -34,6 +34,7 @@ from ansible.executor.task_result import TaskResult
 from ansible.module_utils.six import PY3, string_types
 from ansible.module_utils._text import to_text, to_native
 from ansible.playbook.play_context import PlayContext
+from ansible.playbook.task import Task
 from ansible.plugins.loader import callback_loader, strategy_loader, module_loader
 from ansible.plugins.callback import CallbackBase
 from ansible.template import Templar
@@ -359,6 +360,13 @@ class TaskQueueManager:
             if getattr(callback_plugin, 'disabled', False):
                 continue
 
+            # a plugin can opt in to implicit tasks (such as meta). It does this
+            # by declaring self.wants_implicit_tasks = True.
+            wants_implicit_tasks = getattr(
+                callback_plugin,
+                'wants_implicit_tasks',
+                False)
+
             # try to find v2 method, fallback to v1 method, ignore callback if no method found
             methods = []
             for possible in [method_name, 'v2_on_any']:
@@ -370,6 +378,12 @@ class TaskQueueManager:
 
             # send clean copies
             new_args = []
+
+            # If we end up being given an implicit task, we'll set this flag in
+            # the loop below. If the plugin doesn't care about those, then we
+            # check and continue to the next iteration of the outer loop.
+            is_implicit_task = False
+
             for arg in args:
                 # FIXME: add play/task cleaners
                 if isinstance(arg, TaskResult):
@@ -379,6 +393,12 @@ class TaskQueueManager:
                 else:
                     new_args.append(arg)
 
+                if isinstance(arg, Task) and arg.implicit:
+                    is_implicit_task = True
+
+            if is_implicit_task and not wants_implicit_tasks:
+                continue
+
             for method in methods:
                 try:
                     method(*new_args, **kwargs)

lib/ansible/playbook/play.py

@@ -272,6 +272,9 @@ class Play(Base, Taggable, CollectionSearch):
             loader=self._loader
         )
 
+        for task in flush_block.block:
+            task.implicit = True
+
         block_list = []
 
         block_list.extend(self.pre_tasks)

lib/ansible/playbook/task.py

@@ -97,6 +97,7 @@ class Task(Base, Conditional, Taggable, CollectionSearch):
 
         self._role = role
         self._parent = None
+        self.implicit = False
 
         if task_include:
             self._parent = task_include
@@ -411,6 +412,8 @@ class Task(Base, Conditional, Taggable, CollectionSearch):
         if self._role:
             new_me._role = self._role
 
+        new_me.implicit = self.implicit
+
         return new_me
 
     def serialize(self):
@@ -427,6 +430,8 @@ class Task(Base, Conditional, Taggable, CollectionSearch):
             if self._ansible_internal_redirect_list:
                 data['_ansible_internal_redirect_list'] = self._ansible_internal_redirect_list[:]
 
+            data['implicit'] = self.implicit
+
         return data
 
     def deserialize(self, data):
@@ -457,6 +462,8 @@ class Task(Base, Conditional, Taggable, CollectionSearch):
 
         self._ansible_internal_redirect_list = data.get('_ansible_internal_redirect_list', [])
 
+        self.implicit = data.get('implicit', False)
+
         super(Task, self).deserialize(data)
 
     def set_loader(self, loader):

lib/ansible/plugins/callback/__init__.py

@@ -74,6 +74,7 @@ class CallbackBase(AnsiblePlugin):
             self._display.vvvv('Loading callback plugin %s of type %s, v%s from %s' % (name, ctype, version, sys.modules[self.__module__].__file__))
 
         self.disabled = False
+        self.wants_implicit_tasks = False
 
         self._plugin_options = {}
         if options is not None:

lib/ansible/plugins/strategy/__init__.py

@@ -1138,22 +1138,20 @@ class StrategyBase:
 
         skipped = False
         msg = ''
-        # The top-level conditions should only compare meta_action
+        self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False)
+
+        # These don't support "when" conditionals
+        if meta_action in ('noop', 'flush_handlers', 'refresh_inventory', 'reset_connection') and task.when:
+            self._cond_not_supported_warn(meta_action)
+
         if meta_action == 'noop':
-            # FIXME: issue a callback for the noop here?
-            if task.when:
-                self._cond_not_supported_warn(meta_action)
             msg = "noop"
         elif meta_action == 'flush_handlers':
-            if task.when:
-                self._cond_not_supported_warn(meta_action)
             self._flushed_hosts[target_host] = True
             self.run_handlers(iterator, play_context)
             self._flushed_hosts[target_host] = False
             msg = "ran handlers"
         elif meta_action == 'refresh_inventory':
-            if task.when:
-                self._cond_not_supported_warn(meta_action)
             self._inventory.refresh_inventory()
             self._set_hosts_cache(iterator._play)
             msg = "inventory successfully refreshed"
@@ -1180,6 +1178,8 @@ class StrategyBase:
                     if host.name not in self._tqm._unreachable_hosts:
                         iterator._host_states[host.name].run_state = iterator.ITERATING_COMPLETE
                 msg = "ending play"
+            else:
+                skipped = True
         elif meta_action == 'end_host':
             if _evaluate_conditional(target_host):
                 iterator._host_states[target_host.name].run_state = iterator.ITERATING_COMPLETE
@@ -1211,9 +1211,6 @@ class StrategyBase:
             # a certain subset of variables exist.
             play_context.update_vars(all_vars)
 
-            if task.when:
-                self._cond_not_supported_warn(meta_action)
-
             if target_host in self._active_connections:
                 connection = Connection(self._active_connections[target_host])
                 del self._active_connections[target_host]

test/integration/targets/ansible/callback_plugins/callback_meta.py

@@ -0,0 +1,23 @@
+# (c) 2020 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.plugins.callback import CallbackBase
+import os
+
+
+class CallbackModule(CallbackBase):
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'stdout'
+    CALLBACK_NAME = 'callback_meta'
+
+    def __init__(self, *args, **kwargs):
+        super(CallbackModule, self).__init__(*args, **kwargs)
+        self.wants_implicit_tasks = os.environ.get('CB_WANTS_IMPLICIT', False)
+
+    def v2_playbook_on_task_start(self, task, is_conditional):
+        if task.implicit:
+            self._display.display('saw implicit task')
+        self._display.display(task.get_name())

test/integration/targets/ansible/playbook.yml

@@ -3,3 +3,6 @@
   tasks:
     - debug:
         msg: "{{ username }}"
+
+    - name: explicitly refresh inventory
+      meta: refresh_inventory

test/integration/targets/ansible/runme.sh

@@ -36,6 +36,18 @@ env -u ANSIBLE_PLAYBOOK_DIR ANSIBLE_CONFIG=./playbookdir_cfg.ini ansible localho
 # test adhoc callback triggers
 ANSIBLE_STDOUT_CALLBACK=callback_debug ANSIBLE_LOAD_CALLBACK_PLUGINS=1 ansible --playbook-dir . testhost -i ../../inventory -m ping | grep -E '^v2_' | diff -u adhoc-callback.stdout -
 
+# CB_WANTS_IMPLICIT isn't anything in Ansible itself.
+# Our test cb plugin just accepts it. It lets us avoid copypasting the whole
+# plugin just for two tests.
+CB_WANTS_IMPLICIT=1 ANSIBLE_STDOUT_CALLBACK=callback_meta ANSIBLE_LOAD_CALLBACK_PLUGINS=1 ansible-playbook -i ../../inventory --extra-vars @./vars.yml playbook.yml | grep 'saw implicit task'
+
+set +e
+if ANSIBLE_STDOUT_CALLBACK=callback_meta ANSIBLE_LOAD_CALLBACK_PLUGINS=1 ansible-playbook -i ../../inventory --extra-vars @./vars.yml playbook.yml | grep 'saw implicit task'; then
+  echo "Callback got implicit task and should not have"
+  exit 1
+fi
+set -e
+
 # Test that no tmp dirs are left behind when running ansible-config
 TMP_DIR=~/.ansible/tmptest
 if [[ -d "$TMP_DIR" ]]; then