Fix meta tasks breaking host/fork affinity with host_pinned (#83438)

Fixes #83294

changelogs/fragments/83294-meta-host_pinned-affinity.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - Fix for ``meta`` tasks breaking host/fork affinity with ``host_pinned`` strategy (https://github.com/ansible/ansible/issues/83294)

lib/ansible/plugins/strategy/free.py

@@ -95,6 +95,7 @@ class StrategyModule(StrategyBase):
 
             # try and find an unblocked host with a task to run
             host_results = []
+            meta_task_dummy_results_count = 0
             while True:
                 host = hosts_left[last_host]
                 display.debug("next free host: %s" % host)
@@ -181,6 +182,9 @@ class StrategyModule(StrategyBase):
                                 continue
 
                         if task.action in C._ACTION_META:
+                            if self._host_pinned:
+                                meta_task_dummy_results_count += 1
+                                workers_free -= 1
                             self._execute_meta(task, play_context, iterator, target_host=host)
                             self._blocked_hosts[host_name] = False
                         else:
@@ -220,7 +224,7 @@ class StrategyModule(StrategyBase):
             host_results.extend(results)
 
             # each result is counted as a worker being free again
-            workers_free += len(results)
+            workers_free += len(results) + meta_task_dummy_results_count
 
             self.update_active_connections(results)
 

test/integration/targets/strategy_host_pinned/aliases

@@ -0,0 +1 @@
+shippable/posix/group5

test/integration/targets/strategy_host_pinned/callback_plugins/callback_host_count.py

@@ -0,0 +1,43 @@
+# (c) 2024 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import annotations
+
+from ansible.plugins.callback import CallbackBase
+
+
+class CallbackModule(CallbackBase):
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'stdout'
+    CALLBACK_NAME = 'callback_host_count'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._executing_hosts_counter = 0
+
+    def v2_playbook_on_task_start(self, task, is_conditional):
+        self._display.display(task.name or task.action)
+
+        if task.name == "start":
+            self._executing_hosts_counter += 1
+
+        # NOTE assumes 2 forks
+        num_forks = 2
+        if self._executing_hosts_counter > num_forks:
+            # Exception is caught and turned into just a warning in TQM,
+            # so raise BaseException to fail the test
+            # To prevent seeing false positives in case the exception handling
+            # in TQM is changed and BaseException is swallowed, print something
+            # and ensure the test fails in runme.sh in such a case.
+            self._display.display("host_pinned_test_failed")
+            raise BaseException(
+                "host_pinned test failed, number of hosts executing: "
+                f"{self._executing_hosts_counter}, expected: {num_forks}"
+            )
+
+    def v2_playbook_on_handler_task_start(self, task):
+        self._display.display(task.name or task.action)
+
+    def v2_runner_on_ok(self, result):
+        if result._task.name == "end":
+            self._executing_hosts_counter -= 1

test/integration/targets/strategy_host_pinned/hosts

@@ -0,0 +1,14 @@
+localhost0
+localhost1
+localhost2
+localhost3
+localhost4
+localhost5
+localhost6
+localhost7
+localhost8
+localhost9
+
+[all:vars]
+ansible_connection=local
+ansible_python_interpreter={{ansible_playbook_python}}

test/integration/targets/strategy_host_pinned/playbook.yml

@@ -0,0 +1,46 @@
+# README - even the name of the tasks matter in this test, see callback_plugins/callback_host_count.py
+- hosts: all
+  gather_facts: false
+  strategy: host_pinned
+  pre_tasks:
+    # by executing in pre_tasks we ensure that "start" is the first task in the play,
+    # not an implicit "meta: flush_handlers" after pre_tasks
+    - name: start
+      debug:
+        msg: start
+
+    - ping:
+
+    - meta: noop
+  post_tasks:
+    # notifying a handler in post_tasks ensures the handler is the last task in the play,
+    # not an implicit "meta: flush_handlers" after post_tasks
+    - debug:
+      changed_when: true
+      notify: end
+  handlers:
+    - name: end
+      debug:
+        msg: end
+
+- hosts: localhost0,localhost1,localhost2
+  gather_facts: false
+  strategy: host_pinned
+  pre_tasks:
+    - name: start
+      debug:
+        msg: start
+
+    - command: sleep 3
+      when: inventory_hostname == "localhost0"
+
+    - meta: noop
+    - meta: noop
+  post_tasks:
+    - debug:
+      changed_when: true
+      notify: end
+  handlers:
+    - name: end
+      debug:
+        msg: end

test/integration/targets/strategy_host_pinned/runme.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+
+export ANSIBLE_STDOUT_CALLBACK=callback_host_count
+
+# the number of forks matter, see callback_plugins/callback_host_count.py
+ansible-playbook --inventory hosts --forks 2 playbook.yml | tee "${OUTPUT_DIR}/out.txt"
+
+[ "$(grep -c 'host_pinned_test_failed' "${OUTPUT_DIR}/out.txt")" -eq 0 ]