mirror of https://github.com/ansible/ansible.git
Performance improvements
James Cammarata
8 years ago
parent
80385a47bd
commit
d2b3b2c03e
@ -1,196 +0,0 @@
|
|||||||
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
|
|
||||||
#
|
|
||||||
# This file is part of Ansible
|
|
||||||
#
|
|
||||||
# Ansible is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# Ansible is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
# Make coding more python3-ish
|
|
||||||
from __future__ import (absolute_import, division, print_function)
|
|
||||||
__metaclass__ = type
|
|
||||||
|
|
||||||
from ansible.compat.six.moves import queue
|
|
||||||
from ansible.compat.six import iteritems, text_type
|
|
||||||
from ansible.vars import strip_internal_keys
|
|
||||||
|
|
||||||
import multiprocessing
|
|
||||||
import time
|
|
||||||
import traceback
|
|
||||||
|
|
||||||
# TODO: not needed if we use the cryptography library with its default RNG
|
|
||||||
# engine
|
|
||||||
HAS_ATFORK=True
|
|
||||||
try:
|
|
||||||
from Crypto.Random import atfork
|
|
||||||
except ImportError:
|
|
||||||
HAS_ATFORK=False
|
|
||||||
|
|
||||||
try:
|
|
||||||
from __main__ import display
|
|
||||||
except ImportError:
|
|
||||||
from ansible.utils.display import Display
|
|
||||||
display = Display()
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['ResultProcess']
|
|
||||||
|
|
||||||
|
|
||||||
class ResultProcess(multiprocessing.Process):
|
|
||||||
'''
|
|
||||||
The result worker thread, which reads results from the results
|
|
||||||
queue and fires off callbacks/etc. as necessary.
|
|
||||||
'''
|
|
||||||
|
|
||||||
def __init__(self, final_q, workers):
|
|
||||||
|
|
||||||
# takes a task queue manager as the sole param:
|
|
||||||
self._final_q = final_q
|
|
||||||
self._workers = workers
|
|
||||||
self._cur_worker = 0
|
|
||||||
self._terminated = False
|
|
||||||
|
|
||||||
super(ResultProcess, self).__init__()
|
|
||||||
|
|
||||||
def _send_result(self, result):
|
|
||||||
display.debug(u"sending result: %s" % ([text_type(x) for x in result],))
|
|
||||||
self._final_q.put(result)
|
|
||||||
display.debug("done sending result")
|
|
||||||
|
|
||||||
def _read_worker_result(self):
|
|
||||||
result = None
|
|
||||||
starting_point = self._cur_worker
|
|
||||||
while True:
|
|
||||||
(worker_prc, rslt_q) = self._workers[self._cur_worker]
|
|
||||||
self._cur_worker += 1
|
|
||||||
if self._cur_worker >= len(self._workers):
|
|
||||||
self._cur_worker = 0
|
|
||||||
|
|
||||||
try:
|
|
||||||
if not rslt_q.empty():
|
|
||||||
display.debug("worker %d has data to read" % self._cur_worker)
|
|
||||||
result = rslt_q.get()
|
|
||||||
display.debug("got a result from worker %d: %s" % (self._cur_worker, result))
|
|
||||||
break
|
|
||||||
except queue.Empty:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if self._cur_worker == starting_point:
|
|
||||||
break
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
def terminate(self):
|
|
||||||
self._terminated = True
|
|
||||||
super(ResultProcess, self).terminate()
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
'''
|
|
||||||
The main thread execution, which reads from the results queue
|
|
||||||
indefinitely and sends callbacks/etc. when results are received.
|
|
||||||
'''
|
|
||||||
|
|
||||||
if HAS_ATFORK:
|
|
||||||
atfork()
|
|
||||||
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
result = self._read_worker_result()
|
|
||||||
if result is None:
|
|
||||||
time.sleep(0.005)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# send callbacks for 'non final' results
|
|
||||||
if '_ansible_retry' in result._result:
|
|
||||||
self._send_result(('v2_runner_retry', result))
|
|
||||||
continue
|
|
||||||
elif '_ansible_item_result' in result._result:
|
|
||||||
if result.is_failed() or result.is_unreachable():
|
|
||||||
self._send_result(('v2_runner_item_on_failed', result))
|
|
||||||
elif result.is_skipped():
|
|
||||||
self._send_result(('v2_runner_item_on_skipped', result))
|
|
||||||
else:
|
|
||||||
self._send_result(('v2_runner_item_on_ok', result))
|
|
||||||
if 'diff' in result._result:
|
|
||||||
self._send_result(('v2_on_file_diff', result))
|
|
||||||
continue
|
|
||||||
|
|
||||||
clean_copy = strip_internal_keys(result._result)
|
|
||||||
if 'invocation' in clean_copy:
|
|
||||||
del clean_copy['invocation']
|
|
||||||
|
|
||||||
# if this task is registering a result, do it now
|
|
||||||
if result._task.register:
|
|
||||||
self._send_result(('register_host_var', result._host, result._task, clean_copy))
|
|
||||||
|
|
||||||
# send callbacks, execute other options based on the result status
|
|
||||||
# TODO: this should all be cleaned up and probably moved to a sub-function.
|
|
||||||
# the fact that this sometimes sends a TaskResult and other times
|
|
||||||
# sends a raw dictionary back may be confusing, but the result vs.
|
|
||||||
# results implementation for tasks with loops should be cleaned up
|
|
||||||
# better than this
|
|
||||||
if result.is_unreachable():
|
|
||||||
self._send_result(('host_unreachable', result))
|
|
||||||
elif result.is_failed():
|
|
||||||
self._send_result(('host_task_failed', result))
|
|
||||||
elif result.is_skipped():
|
|
||||||
self._send_result(('host_task_skipped', result))
|
|
||||||
else:
|
|
||||||
if result._task.loop:
|
|
||||||
# this task had a loop, and has more than one result, so
|
|
||||||
# loop over all of them instead of a single result
|
|
||||||
result_items = result._result.get('results', [])
|
|
||||||
else:
|
|
||||||
result_items = [ result._result ]
|
|
||||||
|
|
||||||
for result_item in result_items:
|
|
||||||
# if this task is notifying a handler, do it now
|
|
||||||
if '_ansible_notify' in result_item:
|
|
||||||
if result.is_changed():
|
|
||||||
# The shared dictionary for notified handlers is a proxy, which
|
|
||||||
# does not detect when sub-objects within the proxy are modified.
|
|
||||||
# So, per the docs, we reassign the list so the proxy picks up and
|
|
||||||
# notifies all other threads
|
|
||||||
for notify in result_item['_ansible_notify']:
|
|
||||||
self._send_result(('notify_handler', result, notify))
|
|
||||||
|
|
||||||
if 'add_host' in result_item:
|
|
||||||
# this task added a new host (add_host module)
|
|
||||||
self._send_result(('add_host', result_item))
|
|
||||||
elif 'add_group' in result_item:
|
|
||||||
# this task added a new group (group_by module)
|
|
||||||
self._send_result(('add_group', result._host, result_item))
|
|
||||||
elif 'ansible_facts' in result_item:
|
|
||||||
# if this task is registering facts, do that now
|
|
||||||
loop_var = 'item'
|
|
||||||
if result._task.loop_control:
|
|
||||||
loop_var = result._task.loop_control.loop_var or 'item'
|
|
||||||
item = result_item.get(loop_var, None)
|
|
||||||
if result._task.action == 'include_vars':
|
|
||||||
for (key, value) in iteritems(result_item['ansible_facts']):
|
|
||||||
self._send_result(('set_host_var', result._host, result._task, item, key, value))
|
|
||||||
else:
|
|
||||||
self._send_result(('set_host_facts', result._host, result._task, item, result_item['ansible_facts']))
|
|
||||||
|
|
||||||
# finally, send the ok for this task
|
|
||||||
self._send_result(('host_task_ok', result))
|
|
||||||
|
|
||||||
except queue.Empty:
|
|
||||||
pass
|
|
||||||
except (KeyboardInterrupt, SystemExit, IOError, EOFError):
|
|
||||||
break
|
|
||||||
except:
|
|
||||||
# TODO: we should probably send a proper callback here instead of
|
|
||||||
# simply dumping a stack trace on the screen
|
|
||||||
traceback.print_exc()
|
|
||||||
break
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue