mirror of https://github.com/ansible/ansible.git
T woerner max concurrent (#60702)
* play, block, task: New attribute forks With this it is possible to limit the number of concurrent task runs. forks can now be used in play, block and task. If forks is set in different levels in the chain, then the smallest value will be used for the task. The attribute has been added to the Base class as a list to easily provide all the values that have been set in the different levels of the chain. A warning has been added because of the conflict with run_once. forks will be ignored in this case. The forks limitation in StrategyBase._queue_task is not used for the free strategy. Signed-off-by: Thomas Woerner <twoerner@redhat.com> * Handle forks in free strategy The forks attribute for the free strategy is handled in run in the free StrategyModule. This is dony by counting the amount of tasks where the uuid is the same as the current task, that should be queued next. If this amount is bigger or equal to the forks attribute from the chain (task, block, play), then it will be skipped to the next host. Like it is also done with blocked_hosts. Signed-off-by: Thomas Woerner <twoerner@redhat.com> * Test cases for forks with linear and free strategy With ansible_python_interpreter defined in inventory file using ansible_playbook_python. Signed-off-by: Thomas Woerner <twoerner@redhat.com> * Changing forks keyword to throttle and adding some more docspull/61501/head
James Cammarata
5 years ago
committed by
GitHub
parent
7d1a981b61
commit
51b33b79c0
@ -0,0 +1,2 @@
|
||||
minor_changes:
|
||||
- Added new `throttle` keyword, which can be used at the task, block, or play level to limit the number of workers (up to the specified forks or serial setting) allowed.
|
||||
@ -0,0 +1 @@
|
||||
shippable/posix/group2
|
||||
@ -0,0 +1,6 @@
|
||||
[localhosts]
|
||||
testhost[00:11]
|
||||
|
||||
[localhosts:vars]
|
||||
ansible_connection=local
|
||||
ansible_python_interpreter="{{ ansible_playbook_python }}"
|
||||
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eux
|
||||
|
||||
# https://github.com/ansible/ansible/pull/42528
|
||||
ANSIBLE_STRATEGY='linear' ansible-playbook test_throttle.yml -vv -i inventory --forks 12 "$@"
|
||||
ANSIBLE_STRATEGY='free' ansible-playbook test_throttle.yml -vv -i inventory --forks 12 "$@"
|
||||
@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# read the args from sys.argv
|
||||
throttledir, inventory_hostname, max_throttle = sys.argv[1:]
|
||||
# format/create additional vars
|
||||
max_throttle = int(max_throttle)
|
||||
throttledir = os.path.expanduser(throttledir)
|
||||
throttlefile = os.path.join(throttledir, inventory_hostname)
|
||||
try:
|
||||
# create the file
|
||||
with(open(throttlefile, 'a')):
|
||||
os.utime(throttlefile, None)
|
||||
# count the number of files in the dir
|
||||
throttlelist = os.listdir(throttledir)
|
||||
print("tasks: %d/%d" % (len(throttlelist), max_throttle))
|
||||
# if we have too many files, fail
|
||||
if len(throttlelist) > max_throttle:
|
||||
print(throttlelist)
|
||||
raise ValueError("Too many concurrent tasks: %d/%d" % (len(throttlelist), max_throttle))
|
||||
finally:
|
||||
# remove the file, then wait to make sure it's gone
|
||||
os.unlink(throttlefile)
|
||||
while True:
|
||||
if not os.path.exists(throttlefile):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
@ -0,0 +1,59 @@
|
||||
---
|
||||
- hosts: localhosts
|
||||
gather_facts: false
|
||||
vars:
|
||||
throttledir: ~/ansible_testing/throttle.dir/
|
||||
tasks:
|
||||
- name: Clean throttledir '{{ throttledir }}'
|
||||
file:
|
||||
state: absent
|
||||
path: '{{ throttledir }}'
|
||||
ignore_errors: yes
|
||||
run_once: yes
|
||||
- name: Create throttledir '{{ throttledir }}'
|
||||
file:
|
||||
state: directory
|
||||
path: '{{ throttledir }}'
|
||||
run_once: yes
|
||||
- block:
|
||||
- name: "Test 1 (max throttle: 3)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 3"
|
||||
throttle: 3
|
||||
- block:
|
||||
- name: "Test 2 (max throttle: 5)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 5"
|
||||
throttle: 5
|
||||
- block:
|
||||
- name: "Test 3 (max throttle: 8)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 8"
|
||||
throttle: 8
|
||||
throttle: 6
|
||||
- block:
|
||||
- block:
|
||||
- name: "Test 4 (max throttle: 8)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 8"
|
||||
throttle: 8
|
||||
throttle: 6
|
||||
throttle: 12
|
||||
throttle: 15
|
||||
- block:
|
||||
- name: "Test 1 (max throttle: 3)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 3"
|
||||
throttle: 3
|
||||
- block:
|
||||
- name: "Test 2 (max throttle: 5)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 5"
|
||||
throttle: 5
|
||||
- block:
|
||||
- name: "Test 3 (max throttle: 6)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 6"
|
||||
throttle: 6
|
||||
throttle: 3
|
||||
- block:
|
||||
- block:
|
||||
- name: "Test 4 (max throttle: 8)"
|
||||
script: "test_throttle.py {{throttledir}} {{inventory_hostname}} 8"
|
||||
throttle: 8
|
||||
throttle: 6
|
||||
throttle: 4
|
||||
throttle: 2
|
||||
Loading…
Reference in New Issue