From 9cc56981b52f6e69a1c3af6784b6f53b1e6259fe Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 23 Aug 2018 20:16:32 -0400 Subject: [PATCH] Add new host_pinned strategy (#44586) The 'free' strategy still attempts to do all hosts per task before going to the next, it just doesn't wait for slow hosts, This strategy processes each host as fast as possible to the end of the play before trying to process another host in the pool. --- lib/ansible/plugins/strategy/free.py | 18 +++++++ lib/ansible/plugins/strategy/host_pinned.py | 49 +++++++++++++++++++ test/integration/targets/blocks/runme.sh | 8 +++ .../test_includes_host_pinned/tasks/inner.yml | 2 + .../test_includes_host_pinned/tasks/main.yml | 6 +++ .../includes/test_include_host_pinned.yml | 9 ++++ .../targets/includes/test_includes.yml | 2 + 7 files changed, 94 insertions(+) create mode 100644 lib/ansible/plugins/strategy/host_pinned.py create mode 100644 test/integration/targets/includes/roles/test_includes_host_pinned/tasks/inner.yml create mode 100644 test/integration/targets/includes/roles/test_includes_host_pinned/tasks/main.yml create mode 100644 test/integration/targets/includes/test_include_host_pinned.yml diff --git a/lib/ansible/plugins/strategy/free.py b/lib/ansible/plugins/strategy/free.py index d4b61376dab..207f63207be 100644 --- a/lib/ansible/plugins/strategy/free.py +++ b/lib/ansible/plugins/strategy/free.py @@ -58,6 +58,10 @@ class StrategyModule(StrategyBase): return [host for host in notified_hosts if host in self._flushed_hosts and self._flushed_hosts[host]] + def __init__(self, tqm): + super(StrategyModule, self).__init__(tqm) + self._host_pinned = False + def run(self, iterator, play_context): ''' The "free" strategy is a bit more complex, in that it allows tasks to @@ -77,6 +81,9 @@ class StrategyModule(StrategyBase): result = self._tqm.RUN_OK + # start with all workers being counted as being free + workers_free = len(self._workers) + work_to_do = True while work_to_do and not self._tqm._terminated: @@ -167,10 +174,18 @@ class StrategyModule(StrategyBase): "as tasks are executed independently on each host") self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) self._queue_task(host, task, task_vars, play_context) + # each task is counted as a worker being busy + workers_free -= 1 del task_vars else: display.debug("%s is blocked, skipping for now" % host_name) + # all workers have tasks to do (and the current host isn't done with the play). + # loop back to starting host and break out + if self._host_pinned and workers_free == 0 and work_to_do: + last_host = starting_host + break + # move on to the next host and make sure we # haven't gone past the end of our hosts list last_host += 1 @@ -184,6 +199,9 @@ class StrategyModule(StrategyBase): results = self._process_pending_results(iterator) host_results.extend(results) + # each result is counted as a worker being free again + workers_free += len(results) + self.update_active_connections(results) try: diff --git a/lib/ansible/plugins/strategy/host_pinned.py b/lib/ansible/plugins/strategy/host_pinned.py new file mode 100644 index 00000000000..e0c200c6158 --- /dev/null +++ b/lib/ansible/plugins/strategy/host_pinned.py @@ -0,0 +1,49 @@ +# (c) 2012-2014, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + strategy: host_pinned + short_description: Executes tasks on each host without interruption + description: + - Task execution is as fast as possible per host in batch as defined by C(serial) (default all). + Ansible will not start a play for a host unless the play can be finished without interruption by tasks for another host, + i.e. the number of hosts with an active play does not exceed the number of forks. + Ansible will not wait for other hosts to finish the current task before queuing the next task for a host that has finished. + Once a host is done with the play, it opens it's slot to a new host that was waiting to start. + Other than that, it behaves just like the "free" strategy. + version_added: "2.0" + author: Ansible Core Team +''' + +from ansible.plugins.strategy.free import StrategyModule as FreeStrategyModule + + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + +class StrategyModule(FreeStrategyModule): + + def __init__(self, tqm): + super(StrategyModule, self).__init__(tqm) + self._host_pinned = True diff --git a/test/integration/targets/blocks/runme.sh b/test/integration/targets/blocks/runme.sh index dd15a100834..20c89394240 100755 --- a/test/integration/targets/blocks/runme.sh +++ b/test/integration/targets/blocks/runme.sh @@ -18,3 +18,11 @@ env python -c \ 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' \ block_test_wo_colors.out [ "$(grep -c 'TEST COMPLETE' block_test.out)" = "$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ] +# cleanup the output log again, to make sure the test is clean +rm -f block_test.out block_test_wo_colors.out +# run test with host_pinned strategy and again count the completions +ansible-playbook -vv main.yml -i ../../inventory -e test_strategy=host_pinned "$@" | tee block_test.out +env python -c \ + 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' \ + block_test_wo_colors.out +[ "$(grep -c 'TEST COMPLETE' block_test.out)" = "$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ] diff --git a/test/integration/targets/includes/roles/test_includes_host_pinned/tasks/inner.yml b/test/integration/targets/includes/roles/test_includes_host_pinned/tasks/inner.yml new file mode 100644 index 00000000000..fa4ec93e4e5 --- /dev/null +++ b/test/integration/targets/includes/roles/test_includes_host_pinned/tasks/inner.yml @@ -0,0 +1,2 @@ +- set_fact: + inner_host_pinned: "reached" diff --git a/test/integration/targets/includes/roles/test_includes_host_pinned/tasks/main.yml b/test/integration/targets/includes/roles/test_includes_host_pinned/tasks/main.yml new file mode 100644 index 00000000000..7bc19faae1f --- /dev/null +++ b/test/integration/targets/includes/roles/test_includes_host_pinned/tasks/main.yml @@ -0,0 +1,6 @@ +- name: this needs to be here + debug: + msg: "hello" +- include: inner.yml + with_items: + - '1' diff --git a/test/integration/targets/includes/test_include_host_pinned.yml b/test/integration/targets/includes/test_include_host_pinned.yml new file mode 100644 index 00000000000..6ff92c66e13 --- /dev/null +++ b/test/integration/targets/includes/test_include_host_pinned.yml @@ -0,0 +1,9 @@ +- hosts: testhost + gather_facts: no + strategy: host_pinned + roles: + - test_includes_host_pinned + tasks: + - assert: + that: + - "inner_host_pinned == 'reached'" diff --git a/test/integration/targets/includes/test_includes.yml b/test/integration/targets/includes/test_includes.yml index ebc4c1c2a68..0bcebd4f9d3 100644 --- a/test/integration/targets/includes/test_includes.yml +++ b/test/integration/targets/includes/test_includes.yml @@ -3,3 +3,5 @@ - include: test_includes3.yml - include: test_include_free.yml + +- include: test_include_host_pinned.yml