From 7b12f843666d50430a96a8848e7a540ed28dffef Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 11:53:02 +0545 Subject: [PATCH 001/140] core: support CallError(str) for service.py. --- mitogen/core.py | 19 +++++++----- test.sh | 1 + tests/call_error_test.py | 66 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/call_error_test.py diff --git a/mitogen/core.py b/mitogen/core.py index e9026f03..4cac056c 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -95,13 +95,18 @@ class LatchError(Error): class CallError(Error): - def __init__(self, e): - s = '%s.%s: %s' % (type(e).__module__, type(e).__name__, e) - tb = sys.exc_info()[2] - if tb: - s += '\n' - s += ''.join(traceback.format_tb(tb)) - Error.__init__(self, s) + def __init__(self, fmt=None, *args): + if not isinstance(fmt, Exception): + Error.__init__(self, fmt, *args) + else: + e = fmt + fmt = '%s.%s: %s' % (type(e).__module__, type(e).__name__, e) + args = () + tb = sys.exc_info()[2] + if tb: + fmt += '\n' + fmt += ''.join(traceback.format_tb(tb)) + Error.__init__(self, fmt) def __reduce__(self): return (_unpickle_call_error, (self[0],)) diff --git a/test.sh b/test.sh index 6e678674..f254fcf9 100755 --- a/test.sh +++ b/test.sh @@ -36,6 +36,7 @@ run_test() } run_test tests/ansible_helpers_test.py +run_test tests/call_error_test.py run_test tests/call_function_test.py run_test tests/channel_test.py run_test tests/fakessh_test.py diff --git a/tests/call_error_test.py b/tests/call_error_test.py new file mode 100644 index 00000000..f96aae27 --- /dev/null +++ b/tests/call_error_test.py @@ -0,0 +1,66 @@ +import os +import pickle + +import unittest2 + +import mitogen.core + + +class ConstructorTest(unittest2.TestCase): + klass = mitogen.core.CallError + + def test_string_noargs(self): + e = self.klass('%s%s') + self.assertEquals(e[0], '%s%s') + + def test_string_args(self): + e = self.klass('%s%s', 1, 1) + self.assertEquals(e[0], '11') + + def test_from_exc(self): + ve = ValueError('eek') + e = self.klass(ve) + self.assertEquals(e[0], 'exceptions.ValueError: eek') + + def test_from_exc_tb(self): + try: + raise ValueError('eek') + except ValueError, ve: + e = self.klass(ve) + + self.assertTrue(e[0].startswith('exceptions.ValueError: eek')) + self.assertTrue('test_from_exc_tb' in e[0]) + + +class PickleTest(unittest2.TestCase): + klass = mitogen.core.CallError + + def test_string_noargs(self): + e = self.klass('%s%s') + e2 = pickle.loads(pickle.dumps(e)) + self.assertEquals(e2[0], '%s%s') + + def test_string_args(self): + e = self.klass('%s%s', 1, 1) + e2 = pickle.loads(pickle.dumps(e)) + self.assertEquals(e2[0], '11') + + def test_from_exc(self): + ve = ValueError('eek') + e = self.klass(ve) + e2 = pickle.loads(pickle.dumps(e)) + self.assertEquals(e2[0], 'exceptions.ValueError: eek') + + def test_from_exc_tb(self): + try: + raise ValueError('eek') + except ValueError, ve: + e = self.klass(ve) + + e2 = pickle.loads(pickle.dumps(e)) + self.assertTrue(e2[0].startswith('exceptions.ValueError: eek')) + self.assertTrue('test_from_exc_tb' in e2[0]) + + +if __name__ == '__main__': + unittest2.main() From 761cd9eaf8e7287afe4dc9d015ef60d6304ceb0f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 13:03:08 +0545 Subject: [PATCH 002/140] tests: import tty_create_child_test.py. --- tests/tty_create_child_test.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/tty_create_child_test.py diff --git a/tests/tty_create_child_test.py b/tests/tty_create_child_test.py new file mode 100644 index 00000000..dca1d375 --- /dev/null +++ b/tests/tty_create_child_test.py @@ -0,0 +1,34 @@ + +import os +import tempfile +import time +import unittest2 + +import mitogen.parent + + +class TtyCreateChildTest(unittest2.TestCase): + func = staticmethod(mitogen.parent.tty_create_child) + + def test_dev_tty_open_succeeds(self): + import logging + logging.basicConfig(level=logging.DEBUG) + tf = tempfile.NamedTemporaryFile() + try: + pid, fd = self.func( + 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,) + ) + # TODO: this waitpid hangs on OS X. Installing a SIGCHLD handler + # reveals the parent /is/ notified of the child's death, but + # calling waitpid() from within SIGCHLD yields "No such processes". + # Meanwhile, even inserting a sleep, the following call will hang. + waited_pid, status = os.waitpid(pid, 0) + self.assertEquals(pid, waited_pid) + self.assertEquals(0, status) + self.assertEquals('', tf.read()) + finally: + tf.close() + + +if __name__ == '__main__': + unittest2.main() From 03fcf057dd61e8a9330d18c3f2f73f01760487ac Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 07:25:11 +0000 Subject: [PATCH 003/140] tests: just call log_to_file() from testlib Now we can run test.sh with MITOGEN_LOG_LEVEL=debug and things just work. --- tests/importer_test.py | 7 ------- tests/testlib.py | 7 +++---- tests/tty_create_child_test.py | 3 +-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/importer_test.py b/tests/importer_test.py index 882db78f..1a654a2d 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -15,13 +15,6 @@ import mitogen.utils import testlib -import logging -logging.basicConfig(level=logging.DEBUG) -mitogen.core._v = True -mitogen.core._vv = True -mitogen.core.IOLOG.setLevel(logging.DEBUG) - - class ImporterMixin(testlib.RouterMixin): modname = None diff --git a/tests/testlib.py b/tests/testlib.py index a43c8b1d..fd41298b 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -10,6 +10,8 @@ import urlparse import unittest2 import mitogen.master +import mitogen.utils + if mitogen.is_master: # TODO: shouldn't be necessary. import docker @@ -17,10 +19,7 @@ if mitogen.is_master: # TODO: shouldn't be necessary. DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') sys.path.append(DATA_DIR) - -def set_debug(): - import logging - logging.getLogger('mitogen').setLevel(logging.DEBUG) +mitogen.utils.log_to_file() def data_path(suffix): diff --git a/tests/tty_create_child_test.py b/tests/tty_create_child_test.py index dca1d375..879373ca 100644 --- a/tests/tty_create_child_test.py +++ b/tests/tty_create_child_test.py @@ -4,6 +4,7 @@ import tempfile import time import unittest2 +import testlib import mitogen.parent @@ -11,8 +12,6 @@ class TtyCreateChildTest(unittest2.TestCase): func = staticmethod(mitogen.parent.tty_create_child) def test_dev_tty_open_succeeds(self): - import logging - logging.basicConfig(level=logging.DEBUG) tf = tempfile.NamedTemporaryFile() try: pid, fd = self.func( From 6118d4e6df90d6d8743cc41ad11b4c53cdf75f1f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 07:26:12 +0000 Subject: [PATCH 004/140] tests: set MITOGEN_LOG_LEVEL=debug in .travis.yml too. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3d8c7aee..a7f2f637 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - pip install -r dev_requirements.txt script: -- PYTHONPATH=. ${TRAVIS_BUILD_DIR}/test.sh +- MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/test.sh services: - docker From d6f49a003b10b27fae2493c87a13349b9e81afe4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 13:52:19 +0545 Subject: [PATCH 005/140] issue #106: ansible: beginnings of FileService. --- ansible_mitogen/services.py | 70 ++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 0a576351..98a231d4 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -27,9 +27,16 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +import logging +import zlib + +import mitogen import mitogen.service +LOG = logging.getLogger(__name__) + + class ContextService(mitogen.service.DeduplicatingService): """ Used by worker processes connecting back into the top-level process to @@ -40,7 +47,7 @@ class ContextService(mitogen.service.DeduplicatingService): https://mitogen.readthedocs.io/en/latest/api.html#context-factories This concentrates all SSH connections in the top-level process, which may - become a bottleneck. There are multiple ways to fix that: + become a bottleneck. There are multiple ways to fix that: * creating one .local() child context per CPU and sharding connections between them, using the master process to route messages, or * as above, but having each child create a unique UNIX listener and @@ -68,3 +75,64 @@ class ContextService(mitogen.service.DeduplicatingService): args.pop('discriminator', None) method = getattr(self.router, args.pop('method')) return method(**args) + + +class FileService(mitogen.service.Service): + """ + Primitive latency-inducing file server for old-style incantations of the + module runner. This is to be replaced later with a scheme that forwards + files known to be missing without the target having to ask for them, + avoiding a corresponding roundtrip per file. + + Paths must be explicitly added to the service by a trusted context before + they will be served to an untrusted context. + + :param tuple args: + Tuple of `(cmd, path)`, where: + - cmd: one of "register", "fetch", where: + - register: register a file that may be fetched + - fetch: fetch a file that was previously registered + - path: key of the file to fetch or register + + :returns: + Returns ``None` for "register", or the file data for "fetch". + + :raises mitogen.core.CallError: + Security violation occurred, either path not registered, or attempt to + register path from unprivileged context. + """ + handle = 501 + max_message_size = 1000 + + unprivileged_msg = 'Cannot register from unprivileged context.' + unregistered_msg = 'Path is not registered with FileService.' + + def __init__(self, router): + super(FileService, self).__init__(router) + self._paths = {} + + def validate_args(self, args): + return ( + isinstance(args, tuple) and + len(args) == 2 and + args[0] in ('register', 'fetch') and + isinstance(args[1], str) + ) + + def dispatch(self, args, msg): + cmd, path = msg + return getattr(self, cmd)(path, msg) + + def register(self, path, msg): + if msg.auth_id not in mitogen.parent_ids: + raise mitogen.core.CallError(self.unprivileged_msg) + + with open(path, 'rb') as fp: + self._paths[path] = zlib.compress(fp.read()) + + def fetch(self, path, msg): + if path not in self._paths: + raise mitogen.core.CallError(self.unregistered_msg) + + LOG.debug('Serving %r to context %r', path, msg.src_id) + return self._paths[path] From 15194abb8c34bda104852a83693571fef226645b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 14:15:09 +0545 Subject: [PATCH 006/140] Remove testlib.py from test.sh. --- test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test.sh b/test.sh index f254fcf9..f2ce7cae 100755 --- a/test.sh +++ b/test.sh @@ -54,7 +54,6 @@ run_test tests/responder_test.py run_test tests/router_test.py run_test tests/select_test.py run_test tests/ssh_test.py -run_test tests/testlib.py run_test tests/utils_test.py if [ "$fail" ]; then From a8a31728a032fd65ad7aa82637065198e0af6b11 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 14:31:13 +0545 Subject: [PATCH 007/140] tests: Import Latch soak test --- tests/soak/latch.py | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/soak/latch.py diff --git a/tests/soak/latch.py b/tests/soak/latch.py new file mode 100644 index 00000000..ce00cee0 --- /dev/null +++ b/tests/soak/latch.py @@ -0,0 +1,51 @@ +""" +Used for stressing Latch.get/put. Swap the number of producer/consumer threads +below to try both -- there are many conditions in the Latch code that require +testing of both. +""" + +import logging +import random +import threading +import time +import mitogen.core +import mitogen.utils + +mitogen.utils.log_to_file() +mitogen.core.IOLOG.setLevel(logging.DEBUG) +mitogen.core._v = True +mitogen.core._vv = True + +l = mitogen.core.Latch() +consumed = 0 +produced = 0 +crash = 0 + +def cons(): + global consumed, crash + try: + while 1: + g = l.get() + print 'got=%s consumed=%s produced=%s crash=%s' % (g, consumed, produced, crash) + consumed += 1 + time.sleep(g) + for x in xrange(int(g * 1000)): + pass + except: + crash += 1 + +def prod(): + global produced + while 1: + l.put(random.random()/10) + produced += 1 + time.sleep(random.random()/10) + +allc = [threading.Thread(target=cons) for x in range(64)] +allp = [threading.Thread(target=prod) for x in range(8)] +for th in allc+allp: + th.setDaemon(True) + th.start() + +raw_input() +exit() From 17aef51e6e4612b06ffc15a1efdc0c0aad2d8164 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 14:36:43 +0545 Subject: [PATCH 008/140] tests: Import .local() latency test --- tests/bench/local.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/bench/local.py diff --git a/tests/bench/local.py b/tests/bench/local.py new file mode 100644 index 00000000..25a67b00 --- /dev/null +++ b/tests/bench/local.py @@ -0,0 +1,22 @@ +""" +Measure latency of .local() setup. +""" + +import os +import socket +import mitogen +import time + + +@mitogen.main() #(log_level='DEBUG') +def main(router): + for x in xrange(1000): + t = time.time() + f = router.local()# debug=True) + tt = time.time() + print x, 1000 * (tt - t) + + print f + print 'EEK', f.call(socket.gethostname) + print 'MY PID', os.getpid() + print 'EEKERY', f.call(os.getpid) From ee0f21d57f5e4efc51e8f96910b6ee915444c678 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 14:15:43 +0545 Subject: [PATCH 009/140] core: remove Queue locking from broker loop. Move defer handling out of Broker and into Waker (where it belongs?). Now the lock must only be taken if Waker was actually woken. Knocks 400-item run_hostname_100_times from 10.62s to 10.05s (-5.3%). --- mitogen/core.py | 111 +++++++++++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 43 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 4cac056c..9bc0a2c8 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -import Queue import cPickle import cStringIO import collections @@ -41,6 +40,7 @@ import signal import socket import struct import sys +import thread import threading import time import traceback @@ -1039,14 +1039,19 @@ class Latch(object): class Waker(BasicStream): """ - :py:class:`BasicStream` subclass implementing the - `UNIX self-pipe trick`_. Used internally to wake the IO multiplexer when - some of its state has been changed by another thread. + :py:class:`BasicStream` subclass implementing the `UNIX self-pipe trick`_. + Used to wake the multiplexer when another thread needs to modify its state + (via a cross-thread function call). .. _UNIX self-pipe trick: https://cr.yp.to/docs/selfpipe.html """ + broker_ident = None + def __init__(self, broker): self._broker = broker + self._lock = threading.Lock() + self._deferred = [] + rfd, wfd = os.pipe() self.receive_side = Side(self, rfd) self.transmit_side = Side(self, wfd) @@ -1058,24 +1063,63 @@ class Waker(BasicStream): self.transmit_side.fd, ) - def on_receive(self, broker): + @property + def keep_alive(self): """ - Read a byte from the self-pipe. + Prevent immediate Broker shutdown while deferred functions remain. """ - self.receive_side.read(256) + self._lock.acquire() + try: + return len(self._deferred) + finally: + self._lock.release() - def wake(self): + def on_receive(self, broker): """ - Write a byte to the self-pipe, causing the IO multiplexer to wake up. - Nothing is written if the current thread is the IO multiplexer thread. + Drain the pipe and fire callbacks. Reading multiple bytes is safe since + new bytes corresponding to future .defer() calls are written only after + .defer() takes _lock: either a byte we read corresponds to something + already on the queue by the time we take _lock, or a byte remains + buffered, causing another wake up, because it was written after we + released _lock. """ - _vv and IOLOG.debug('%r.wake() [fd=%r]', self, self.transmit_side.fd) - if threading.currentThread() != self._broker._thread: + _vv and IOLOG.debug('%r.on_receive()', self) + self.receive_side.read(128) + self._lock.acquire() + try: + deferred = self._deferred + self._deferred = [] + finally: + self._lock.release() + + for func, args, kwargs in deferred: try: - self.transmit_side.write(' ') - except OSError, e: - if e[0] != errno.EBADF: - raise + func(*args, **kwargs) + except Exception: + LOG.exception('defer() crashed: %r(*%r, **%r)', + func, args, kwargs) + self._broker.shutdown() + + def defer(self, func, *args, **kwargs): + if thread.get_ident() == self.broker_ident: + _vv and IOLOG.debug('%r.defer() [immediate]', self) + return func(*args, **kwargs) + + _vv and IOLOG.debug('%r.defer() [fd=%r]', self, self.transmit_side.fd) + self._lock.acquire() + try: + self._deferred.append((func, args, kwargs)) + finally: + self._lock.release() + + # Wake the multiplexer by writing a byte. If the broker is in the midst + # of tearing itself down, the waker fd may already have been closed, so + # ignore EBADF here. + try: + self.transmit_side.write(' ') + except OSError, e: + if e[0] != errno.EBADF: + raise class IoLogger(BasicStream): @@ -1242,24 +1286,17 @@ class Broker(object): def __init__(self): self._alive = True - self._queue = Queue.Queue() - self._readers = [] - self._writers = [] self._waker = Waker(self) - self.start_receive(self._waker) + self.defer = self._waker.defer + self._readers = [self._waker.receive_side] + self._writers = [] self._thread = threading.Thread( target=_profile_hook, args=('broker', self._broker_main), name='mitogen-broker' ) self._thread.start() - - def defer(self, func, *args, **kwargs): - if threading.currentThread() == self._thread: - func(*args, **kwargs) - else: - self._queue.put((func, args, kwargs)) - self._waker.wake() + self._waker.broker_ident = self._thread.ident def _list_discard(self, lst, value): try: @@ -1296,19 +1333,8 @@ class Broker(object): LOG.exception('%r crashed', stream) stream.on_disconnect(self) - def _run_defer(self): - while not self._queue.empty(): - func, args, kwargs = self._queue.get() - try: - func(*args, **kwargs) - except Exception: - LOG.exception('defer() crashed: %r(*%r, **%r)', - func, args, kwargs) - self.shutdown() - def _loop_once(self, timeout=None): _vv and IOLOG.debug('%r._loop_once(%r)', self, timeout) - self._run_defer() #IOLOG.debug('readers = %r', self._readers) #IOLOG.debug('writers = %r', self._writers) @@ -1327,15 +1353,13 @@ class Broker(object): self._call(side.stream, side.stream.on_transmit) def keep_alive(self): - return (sum((side.keep_alive for side in self._readers), 0) + - (not self._queue.empty())) + return sum((side.keep_alive for side in self._readers), 0) def _broker_main(self): try: while self._alive: self._loop_once() - self._run_defer() fire(self, 'shutdown') for side in set(self._readers).union(self._writers): @@ -1361,8 +1385,9 @@ class Broker(object): def shutdown(self): _v and LOG.debug('%r.shutdown()', self) - self._alive = False - self._waker.wake() + def _shutdown(): + self._alive = False + self.defer(_shutdown) def join(self): self._thread.join() From 8676c40674fbbb4372f89b0b9804af0a4140a2a0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 15:08:17 +0545 Subject: [PATCH 010/140] core: make _start_transmit / _stop_transmit async-only For now at least, these APIs are always used in an asynchronous context, so stop using the defer mechanism. --- docs/api.rst | 6 +++--- docs/images/layout.graphml | 4 ++-- docs/internals.rst | 4 ++-- mitogen/core.py | 18 +++++++++--------- mitogen/fakessh.py | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 1364440d..e7846f9b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1062,11 +1062,11 @@ Broker Class Mark the :py:attr:`receive_side ` on `stream` as not ready for reading. Safe to call from any thread. - .. method:: start_transmit (stream) + .. method:: _start_transmit (stream) Mark the :py:attr:`transmit_side ` on `stream` as - ready for writing. Safe to call from any thread. When the associated - file descriptor becomes ready for writing, + ready for writing. Must only be called from the Broker thread. When the + associated file descriptor becomes ready for writing, :py:meth:`BasicStream.on_transmit` will be called. .. method:: stop_receive (stream) diff --git a/docs/images/layout.graphml b/docs/images/layout.graphml index 4aa1f95e..f21842bb 100644 --- a/docs/images/layout.graphml +++ b/docs/images/layout.graphml @@ -122,8 +122,8 @@ send(msg) - start_transmit(strm) -stop_transmit(strm) + _start_transmit(strm) +_stop_transmit(strm) diff --git a/docs/internals.rst b/docs/internals.rst index 7799011c..625f14ce 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -188,12 +188,12 @@ Stream Classes .. method:: on_transmit (broker) Called by :py:class:`Broker` when the stream's :py:attr:`transmit_side` - has been marked writeable using :py:meth:`Broker.start_transmit` and + has been marked writeable using :py:meth:`Broker._start_transmit` and the broker has detected the associated file descriptor is ready for writing. Subclasses must implement this method if - :py:meth:`Broker.start_transmit` is ever called on them. + :py:meth:`Broker._start_transmit` is ever called on them. .. method:: on_shutdown (broker) diff --git a/mitogen/core.py b/mitogen/core.py index 9bc0a2c8..a37d5f19 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -723,7 +723,7 @@ class BasicStream(object): def on_disconnect(self, broker): LOG.debug('%r.on_disconnect()', self) broker.stop_receive(self) - broker.stop_transmit(self) + broker._stop_transmit(self) if self.receive_side: self.receive_side.close() if self.transmit_side: @@ -834,7 +834,7 @@ class Stream(BasicStream): _vv and IOLOG.debug('%r.on_transmit() -> len %d', self, written) if not self._output_buf: - broker.stop_transmit(self) + broker._stop_transmit(self) def _send(self, msg): _vv and IOLOG.debug('%r._send(%r)', self, msg) @@ -842,7 +842,7 @@ class Stream(BasicStream): msg.auth_id, msg.handle, msg.reply_to or 0, len(msg.data)) + msg.data self._output_buf.append(pkt) - self._router.broker.start_transmit(self) + self._router.broker._start_transmit(self) def send(self, msg): """Send `data` to `handle`, and tell the broker we have output. May @@ -1317,14 +1317,14 @@ class Broker(object): IOLOG.debug('%r.stop_receive(%r)', self, stream) self.defer(self._list_discard, self._readers, stream.receive_side) - def start_transmit(self, stream): - IOLOG.debug('%r.start_transmit(%r)', self, stream) + def _start_transmit(self, stream): + IOLOG.debug('%r._start_transmit(%r)', self, stream) assert stream.transmit_side and stream.transmit_side.fd is not None - self.defer(self._list_add, self._writers, stream.transmit_side) + self._list_add(self._writers, stream.transmit_side) - def stop_transmit(self, stream): - IOLOG.debug('%r.stop_transmit(%r)', self, stream) - self.defer(self._list_discard, self._writers, stream.transmit_side) + def _stop_transmit(self, stream): + IOLOG.debug('%r._stop_transmit(%r)', self, stream) + self._list_discard(self._writers, stream.transmit_side) def _call(self, stream, func): try: diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index 13abfcfe..f5dcbe1c 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -62,14 +62,14 @@ class IoPump(mitogen.core.BasicStream): def write(self, s): self._output_buf += s - self._broker.start_transmit(self) + self._broker._start_transmit(self) def close(self): self._closed = True # If local process hasn't exitted yet, ensure its write buffer is # drained before lazily triggering disconnect in on_transmit. if self.transmit_side.fd is not None: - self._broker.start_transmit(self) + self._broker._start_transmit(self) def on_shutdown(self, broker): self.close() @@ -83,7 +83,7 @@ class IoPump(mitogen.core.BasicStream): self._output_buf = self._output_buf[written:] if not self._output_buf: - broker.stop_transmit(self) + broker._stop_transmit(self) if self._closed: self.on_disconnect(broker) From 692af860bab7041e1575ce6e912ff33820dff10e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 15:14:01 +0545 Subject: [PATCH 011/140] core: remove use of defer() from _async_route(). --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index a37d5f19..f6ddf23d 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1273,7 +1273,7 @@ class Router(object): self, msg, mitogen.context_id) return - stream.send(msg) + stream._send(msg) def route(self, msg): self.broker.defer(self._async_route, msg) From 0f29baa0777671b5d1bff28a0688dbb882416629 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 15:32:01 +0545 Subject: [PATCH 012/140] core: support pickling senders, Receiver.to_sender() CC @moreati, in case this impacts you --- mitogen/core.py | 22 +++++++++++++++++++++- tests/call_function_test.py | 19 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index f6ddf23d..5739e2b4 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -299,6 +299,9 @@ class Message(object): def _unpickle_context(self, context_id, name): return _unpickle_context(self.router, context_id, name) + def _unpickle_sender(self, context_id, dst_handle): + return _unpickle_sender(self.router, context_id, dst_handle) + def _find_global(self, module, func): """Return the class implementing `module_name.class_name` or raise `StreamError` if the module is not whitelisted.""" @@ -307,6 +310,8 @@ class Message(object): return _unpickle_call_error elif func == '_unpickle_dead': return _unpickle_dead + elif func == '_unpickle_sender': + return self._unpickle_sender elif func == '_unpickle_context': return self._unpickle_context @@ -366,6 +371,9 @@ class Sender(object): def __repr__(self): return 'Sender(%r, %r)' % (self.context, self.dst_handle) + def __reduce__(self): + return _unpickle_sender, (self.context.context_id, self.dst_handle) + def close(self): """Indicate this channel is closed to the remote side.""" _vv and IOLOG.debug('%r.close()', self) @@ -387,6 +395,14 @@ class Sender(object): ) +def _unpickle_sender(router, context_id, dst_handle): + if not (isinstance(router, Router) and + isinstance(context_id, (int, long)) and context_id >= 0 and + isinstance(dst_handle, (int, long)) and dst_handle > 0): + raise TypeError('cannot unpickle Sender: bad input') + return Sender(Context(router, context_id), dst_handle) + + class Receiver(object): notify = None raise_channelerror = True @@ -401,6 +417,10 @@ class Receiver(object): def __repr__(self): return 'Receiver(%r, %r)' % (self.router, self.handle) + def to_sender(self): + context = Context(self.router, mitogen.context_id) + return Sender(context, self.handle) + def _on_receive(self, msg): """Callback from the Stream; appends data to the internal queue.""" _vv and IOLOG.debug('%r._on_receive(%r)', self, msg) @@ -912,7 +932,7 @@ class Context(object): def _unpickle_context(router, context_id, name): if not (isinstance(router, Router) and - isinstance(context_id, (int, long)) and context_id > 0 and + isinstance(context_id, (int, long)) and context_id >= 0 and isinstance(name, basestring) and len(name) < 100): raise TypeError('cannot unpickle Context: bad input') return router.context_class(router, context_id, name) diff --git a/tests/call_function_test.py b/tests/call_function_test.py index cc6c32b6..301b7798 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -33,10 +33,16 @@ def func_accepts_returns_context(context): return context +def func_accepts_returns_sender(sender): + sender.put(123) + sender.close() + return sender + + class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): def setUp(self): super(CallFunctionTest, self).setUp() - self.local = self.router.local() + self.local = self.router.fork() def test_succeeds(self): self.assertEqual(3, self.local.call(function_that_adds_numbers, 1, 2)) @@ -87,6 +93,17 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(context.context_id, self.local.context_id) self.assertEqual(context.name, self.local.name) + def test_accepts_returns_sender(self): + recv = mitogen.core.Receiver(self.router) + sender = recv.to_sender() + sender2 = self.local.call(func_accepts_returns_sender, sender) + self.assertEquals(sender.context.context_id, + sender2.context.context_id) + self.assertEquals(sender.dst_handle, sender2.dst_handle) + self.assertEquals(123, recv.get().unpickle()) + self.assertRaises(mitogen.core.ChannelError, + lambda: recv.get().unpickle()) + if __name__ == '__main__': unittest2.main() From 085b3d21bd1ada99ab2b8d6fd02ff328764e4579 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 15:32:55 +0545 Subject: [PATCH 013/140] core: fix call_function_test regression Second time in 3 weeks. So stupid. This time write tests. --- mitogen/core.py | 5 +++-- test.sh | 1 + tests/receiver_test.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 tests/receiver_test.py diff --git a/mitogen/core.py b/mitogen/core.py index 5739e2b4..8218280d 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -441,13 +441,14 @@ class Receiver(object): if msg == _DEAD: raise ChannelError(ChannelError.local_msg) - msg.unpickle() # Cause .remote_msg to be thrown. return msg def __iter__(self): while True: try: - yield self.get() + msg = self.get() + msg.unpickle() # Cause .remote_msg to be thrown. + yield msg except ChannelError: return diff --git a/test.sh b/test.sh index f2ce7cae..ab545104 100755 --- a/test.sh +++ b/test.sh @@ -50,6 +50,7 @@ run_test tests/master_test.py run_test tests/module_finder_test.py run_test tests/nested_test.py run_test tests/parent_test.py +run_test tests/receiver_test.py run_test tests/responder_test.py run_test tests/router_test.py run_test tests/select_test.py diff --git a/tests/receiver_test.py b/tests/receiver_test.py new file mode 100644 index 00000000..1d8c0c7b --- /dev/null +++ b/tests/receiver_test.py @@ -0,0 +1,40 @@ + +import unittest2 + +import mitogen.core +import testlib + + +def yield_stuff_then_die(sender): + for x in xrange(5): + sender.put(x) + sender.close() + return 10 + + +class ConstructorTest(testlib.RouterMixin, testlib.TestCase): + def test_handle(self): + recv = mitogen.core.Receiver(self.router) + self.assertTrue(isinstance(recv.handle, int)) + self.assertTrue(recv.handle > 100) + self.router.route( + mitogen.core.Message.pickled( + 'hi', + dst_id=0, + handle=recv.handle, + ) + ) + self.assertEquals('hi', recv.get().unpickle()) + + +class IterationTest(testlib.RouterMixin, testlib.TestCase): + def test_dead_stops_iteration(self): + recv = mitogen.core.Receiver(self.router) + fork = self.router.fork() + ret = fork.call_async(yield_stuff_then_die, recv.to_sender()) + self.assertEquals(list(range(5)), list(m.unpickle() for m in recv)) + self.assertEquals(10, ret.get().unpickle()) + + +if __name__ == '__main__': + unittest2.main() From b48d63f33b075561330ecdacab85c2e4e63fdccf Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 15:38:44 +0545 Subject: [PATCH 014/140] docs: add to_sender() and update serialization notes --- docs/api.rst | 20 ++++++++++++++++++++ docs/getting_started.rst | 1 + 2 files changed, 21 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index e7846f9b..53ff60ac 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -932,6 +932,23 @@ Receiver Class Used by :py:class:`mitogen.master.Select` to implement waiting on multiple receivers. + .. py:method:: to_sender () + + Return a :py:class:`mitogen.core.Sender` configured to deliver messages + to this receiver. Since a Sender can be serialized, this makes it + convenient to pass `(context_id, handle)` pairs around:: + + def deliver_monthly_report(sender): + for line in open('monthly_report.txt'): + sender.send(line) + sender.close() + + remote = router.ssh(hostname='mainframe') + recv = mitogen.core.Receiver(router) + remote.call(deliver_monthly_report, recv.to_sender()) + for msg in recv: + print(msg) + .. py:method:: empty () Return ``True`` if calling :py:meth:`get` would block. @@ -997,6 +1014,9 @@ Sender Class Senders are used to send pickled messages to a handle in another context, it is the inverse of :py:class:`mitogen.core.Sender`. + Senders may be serialized, making them convenient to wire up data flows. + See :py:meth:`mitogen.core.Receiver.to_sender` for more information. + :param mitogen.core.Context context: Context to send messages to. :param int dst_handle: diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 846fa88c..a3bc7652 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -292,6 +292,7 @@ User-defined types may not be used, except for: * :py:class:`mitogen.core.CallError` * :py:class:`mitogen.core.Context` +* :py:class:`mitogen.core.Sender` * :py:class:`mitogen.core._DEAD` Subclasses of built-in types must be undecorated using From 80a97fbc9bc9aa35d5ade7d668ec035f1e435f9d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 15:43:48 +0545 Subject: [PATCH 015/140] core: Rename Sender.put() to Sender.send(). Been annoying me for months. --- docs/api.rst | 2 +- mitogen/core.py | 4 ++-- tests/call_function_test.py | 2 +- tests/receiver_test.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 53ff60ac..50c5bdab 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1027,7 +1027,7 @@ Sender Class Send :py:data:`_DEAD` to the remote end, causing :py:meth:`ChannelError` to be raised in any waiting thread. - .. py:method:: put (data) + .. py:method:: send (data) Send `data` to the remote end. diff --git a/mitogen/core.py b/mitogen/core.py index 8218280d..1e3b891a 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -384,9 +384,9 @@ class Sender(object): ) ) - def put(self, data): + def send(self, data): """Send `data` to the remote.""" - _vv and IOLOG.debug('%r.put(%r..)', self, data[:100]) + _vv and IOLOG.debug('%r.send(%r..)', self, data[:100]) self.context.send( Message.pickled( data, diff --git a/tests/call_function_test.py b/tests/call_function_test.py index 301b7798..d66864b4 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -34,7 +34,7 @@ def func_accepts_returns_context(context): def func_accepts_returns_sender(sender): - sender.put(123) + sender.send(123) sender.close() return sender diff --git a/tests/receiver_test.py b/tests/receiver_test.py index 1d8c0c7b..3d67cf80 100644 --- a/tests/receiver_test.py +++ b/tests/receiver_test.py @@ -7,7 +7,7 @@ import testlib def yield_stuff_then_die(sender): for x in xrange(5): - sender.put(x) + sender.send(x) sender.close() return 10 From 6db3588c93b6d18c50ca17f0c909623187058a79 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 16:06:18 +0545 Subject: [PATCH 016/140] Only call _start_transmit when required; closes #165. --- mitogen/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 1e3b891a..01d343d6 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -862,8 +862,10 @@ class Stream(BasicStream): pkt = struct.pack(self.HEADER_FMT, msg.dst_id, msg.src_id, msg.auth_id, msg.handle, msg.reply_to or 0, len(msg.data)) + msg.data + was_transmitting = len(self._output_buf) self._output_buf.append(pkt) - self._router.broker._start_transmit(self) + if not was_transmitting: + self._router.broker._start_transmit(self) def send(self, msg): """Send `data` to `handle`, and tell the broker we have output. May From e1af2db4ae91ecb647c25a43ceed2ab9e2c6592d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 17:59:23 +0545 Subject: [PATCH 017/140] issue #155: handle crash in the forking child better. This code path is probably only necessary during development, but it prevents tracebacks (etc.) getting written over the Stream socket, which naturally causes corruption. Instead keep whatever the parent has for stderr, manually write a traceback there and hard exit. --- mitogen/fork.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/mitogen/fork.py b/mitogen/fork.py index 7a6c87c1..36e4f0ca 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -31,6 +31,7 @@ import os import random import sys import threading +import traceback import mitogen.core import mitogen.parent @@ -65,6 +66,18 @@ def break_logging_locks(): handler.createLock() +def handle_child_crash(): + """ + Respond to _child_main() crashing by ensuring the relevant exception is + logged to /dev/tty. + """ + sys.stderr.write('\n\nFORKED CHILD PID %d CRASHED\n%s\n\n' % ( + os.getpid(), + traceback.format_exc(), + )) + os._exit(1) + + class Stream(mitogen.parent.Stream): #: Reference to the importer, if any, recovered from the parent. importer = None @@ -94,7 +107,13 @@ class Stream(mitogen.parent.Stream): return self.pid, fd else: parentfp.close() + self._wrap_child_main(childfp) + + def _wrap_child_main(self, childfp): + try: self._child_main(childfp) + except BaseException, e: + handle_child_crash() def _child_main(self, childfp): mitogen.core.Latch._on_fork() @@ -113,17 +132,18 @@ class Stream(mitogen.parent.Stream): # avoid ExternalContext.main() accidentally allocating new files over # the standard handles. os.dup2(childfp.fileno(), 0) - os.dup2(childfp.fileno(), 2) + os.dup2(sys.stderr.fileno(), 2) childfp.close() kwargs = self.get_main_kwargs() kwargs['core_src_fd'] = None kwargs['importer'] = self.importer kwargs['setup_package'] = False - mitogen.core.ExternalContext().main(**kwargs) - - # Don't trigger atexit handlers, they were copied from the parent. - os._exit(0) + try: + mitogen.core.ExternalContext().main(**kwargs) + finally: + # Don't trigger atexit handlers, they were copied from the parent. + os._exit(0) def _connect_bootstrap(self): # None required. From 1ff27ada4973f50682386a7ce1eb4d7b6c77003a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 18:54:55 +0545 Subject: [PATCH 018/140] Add maximum message size checks. Closes #151. --- ansible_mitogen/process.py | 2 +- mitogen/core.py | 24 ++++++++++++++--- mitogen/fakessh.py | 9 ++++--- mitogen/fork.py | 6 +++-- mitogen/master.py | 4 ++- mitogen/parent.py | 16 +++++++++--- tests/router_test.py | 53 ++++++++++++++++++++++++++++++++++++-- tests/testlib.py | 36 ++++++++++++++++++++++---- 8 files changed, 128 insertions(+), 22 deletions(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 1880f769..eb9bd2ff 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -134,7 +134,7 @@ class MuxProcess(object): """ Construct a Router, Broker, and mitogen.unix listener """ - self.router = mitogen.master.Router() + self.router = mitogen.master.Router(max_message_size=4096*1048576) self.router.responder.whitelist_prefix('ansible') self.router.responder.whitelist_prefix('ansible_mitogen') mitogen.core.listen(self.router.broker, 'shutdown', self.on_broker_shutdown) diff --git a/mitogen/core.py b/mitogen/core.py index 01d343d6..29e04a2a 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -812,6 +812,12 @@ class Stream(BasicStream): self._input_buf[0][:self.HEADER_LEN], ) + if msg_len > self._router.max_message_size: + LOG.error('Maximum message size exceeded (got %d, max %d)', + msg_len, self._router.max_message_size) + self.on_disconnect(broker) + return False + total_len = msg_len + self.HEADER_LEN if self._input_buf_len < total_len: _vv and IOLOG.debug( @@ -1191,6 +1197,7 @@ class IoLogger(BasicStream): class Router(object): context_class = Context + max_message_size = 128 * 1048576 def __init__(self, broker): self.broker = broker @@ -1274,6 +1281,11 @@ class Router(object): def _async_route(self, msg, stream=None): _vv and IOLOG.debug('%r._async_route(%r, %r)', self, msg, stream) + if len(msg.data) > self.max_message_size: + LOG.error('message too large (max %d bytes): %r', + self.max_message_size, msg) + return + # Perform source verification. if stream is not None: expected_stream = self._stream_by_id.get(msg.auth_id, @@ -1438,7 +1450,9 @@ class ExternalContext(object): _v and LOG.debug('%r: parent stream is gone, dying.', self) self.broker.shutdown() - def _setup_master(self, profiling, parent_id, context_id, in_fd, out_fd): + def _setup_master(self, max_message_size, profiling, parent_id, + context_id, in_fd, out_fd): + Router.max_message_size = max_message_size self.profiling = profiling if profiling: enable_profiling() @@ -1571,9 +1585,11 @@ class ExternalContext(object): self.dispatch_stopped = True def main(self, parent_ids, context_id, debug, profiling, log_level, - in_fd=100, out_fd=1, core_src_fd=101, setup_stdio=True, - setup_package=True, importer=None, whitelist=(), blacklist=()): - self._setup_master(profiling, parent_ids[0], context_id, in_fd, out_fd) + max_message_size, in_fd=100, out_fd=1, core_src_fd=101, + setup_stdio=True, setup_package=True, importer=None, + whitelist=(), blacklist=()): + self._setup_master(max_message_size, profiling, parent_ids[0], + context_id, in_fd, out_fd) try: try: self._setup_logging(debug, log_level) diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index f5dcbe1c..e07916ad 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -343,14 +343,15 @@ def run(dest, router, args, deadline=None, econtext=None): fp.write(inspect.getsource(mitogen.core)) fp.write('\n') fp.write('ExternalContext().main(**%r)\n' % ({ - 'parent_ids': parent_ids, 'context_id': context_id, + 'core_src_fd': None, 'debug': getattr(router, 'debug', False), - 'profiling': getattr(router, 'profiling', False), - 'log_level': mitogen.parent.get_log_level(), 'in_fd': sock2.fileno(), + 'log_level': mitogen.parent.get_log_level(), + 'max_message_size': router.max_message_size, 'out_fd': sock2.fileno(), - 'core_src_fd': None, + 'parent_ids': parent_ids, + 'profiling': getattr(router, 'profiling', False), 'setup_stdio': False, },)) finally: diff --git a/mitogen/fork.py b/mitogen/fork.py index 36e4f0ca..e4a8625a 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -85,9 +85,11 @@ class Stream(mitogen.parent.Stream): #: User-supplied function for cleaning up child process state. on_fork = None - def construct(self, old_router, on_fork=None, debug=False, profiling=False): + def construct(self, old_router, max_message_size, on_fork=None, + debug=False, profiling=False): # fork method only supports a tiny subset of options. - super(Stream, self).construct(debug=debug, profiling=profiling) + super(Stream, self).construct(max_message_size=max_message_size, + debug=debug, profiling=profiling) self.on_fork = on_fork responder = getattr(old_router, 'responder', None) diff --git a/mitogen/master.py b/mitogen/master.py index 4359a732..0cf5d451 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -646,9 +646,11 @@ class Router(mitogen.parent.Router): debug = False profiling = False - def __init__(self, broker=None): + def __init__(self, broker=None, max_message_size=None): if broker is None: broker = self.broker_class() + if max_message_size: + self.max_message_size = max_message_size super(Router, self).__init__(broker) self.upgrade() diff --git a/mitogen/parent.py b/mitogen/parent.py index 8a9a186a..599a4603 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -337,6 +337,10 @@ class Stream(mitogen.core.Stream): #: Set to the child's PID by connect(). pid = None + #: Passed via Router wrapper methods, must eventually be passed to + #: ExternalContext.main(). + max_message_size = None + def __init__(self, *args, **kwargs): super(Stream, self).__init__(*args, **kwargs) self.sent_modules = set(['mitogen', 'mitogen.core']) @@ -344,12 +348,13 @@ class Stream(mitogen.core.Stream): #: during disconnection. self.routes = set([self.remote_id]) - def construct(self, remote_name=None, python_path=None, debug=False, - connect_timeout=None, profiling=False, + def construct(self, max_message_size, remote_name=None, python_path=None, + debug=False, connect_timeout=None, profiling=False, old_router=None, **kwargs): """Get the named context running on the local machine, creating it if it does not exist.""" super(Stream, self).construct(**kwargs) + self.max_message_size = max_message_size if python_path: self.python_path = python_path if sys.platform == 'darwin' and self.python_path == '/usr/bin/python': @@ -367,6 +372,7 @@ class Stream(mitogen.core.Stream): self.remote_name = remote_name self.debug = debug self.profiling = profiling + self.max_message_size = max_message_size self.connect_deadline = time.time() + self.connect_timeout def on_shutdown(self, broker): @@ -441,6 +447,7 @@ class Stream(mitogen.core.Stream): ] def get_main_kwargs(self): + assert self.max_message_size is not None parent_ids = mitogen.parent_ids[:] parent_ids.insert(0, mitogen.context_id) return { @@ -451,6 +458,7 @@ class Stream(mitogen.core.Stream): 'log_level': get_log_level(), 'whitelist': self._router.get_module_whitelist(), 'blacklist': self._router.get_module_blacklist(), + 'max_message_size': self.max_message_size, } def get_preamble(self): @@ -703,7 +711,9 @@ class Router(mitogen.core.Router): def _connect(self, klass, name=None, **kwargs): context_id = self.allocate_id() context = self.context_class(self, context_id) - stream = klass(self, context_id, old_router=self, **kwargs) + kwargs['old_router'] = self + kwargs['max_message_size'] = self.max_message_size + stream = klass(self, context_id, **kwargs) if name is not None: stream.name = name stream.connect() diff --git a/tests/router_test.py b/tests/router_test.py index 3f460c3f..c6b4e2df 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -1,4 +1,6 @@ import Queue +import StringIO +import logging import subprocess import time @@ -8,7 +10,16 @@ import testlib import mitogen.master import mitogen.utils -mitogen.utils.log_to_file() + +@mitogen.core.takes_router +def return_router_max_message_size(router): + return router.max_message_size + + +def send_n_sized_reply(sender, n): + sender.send(' ' * n) + return 123 + class AddHandlerTest(unittest2.TestCase): klass = mitogen.master.Router @@ -21,6 +32,44 @@ class AddHandlerTest(unittest2.TestCase): self.assertEquals(queue.get(timeout=5), mitogen.core._DEAD) +class MessageSizeTest(testlib.BrokerMixin, unittest2.TestCase): + klass = mitogen.master.Router + + def test_local_exceeded(self): + router = self.klass(broker=self.broker, max_message_size=4096) + recv = mitogen.core.Receiver(router) + + logs = testlib.LogCapturer() + logs.start() + + sem = mitogen.core.Latch() + router.route(mitogen.core.Message.pickled(' '*8192)) + router.broker.defer(sem.put, ' ') # wlil always run after _async_route + sem.get() + + expect = 'message too large (max 4096 bytes)' + self.assertTrue(expect in logs.stop()) + + def test_remote_configured(self): + router = self.klass(broker=self.broker, max_message_size=4096) + remote = router.fork() + size = remote.call(return_router_max_message_size) + self.assertEquals(size, 4096) + + def test_remote_exceeded(self): + # Ensure new contexts receive a router with the same value. + router = self.klass(broker=self.broker, max_message_size=4096) + recv = mitogen.core.Receiver(router) + + logs = testlib.LogCapturer() + logs.start() + + remote = router.fork() + remote.call(send_n_sized_reply, recv.to_sender(), 8192) + + expect = 'message too large (max 4096 bytes)' + self.assertTrue(expect in logs.stop()) + + if __name__ == '__main__': unittest2.main() - diff --git a/tests/testlib.py b/tests/testlib.py index fd41298b..5d959b8d 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,4 +1,6 @@ +import StringIO +import logging import os import random import re @@ -113,6 +115,24 @@ def wait_for_port( % (host, port)) +class LogCapturer(object): + def __init__(self, name=None): + self.sio = StringIO.StringIO() + self.logger = logging.getLogger(name) + self.handler = logging.StreamHandler(self.sio) + self.old_propagate = self.logger.propagate + self.old_handlers = self.logger.handlers + + def start(self): + self.logger.handlers = [self.handler] + self.logger.propagate = False + + def stop(self): + self.logger.handlers = self.old_handlers + self.logger.propagate = self.old_propagate + return self.sio.getvalue() + + class TestCase(unittest2.TestCase): def assertRaises(self, exc, func, *args, **kwargs): """Like regular assertRaises, except return the exception that was @@ -156,19 +176,25 @@ class DockerizedSshDaemon(object): self.container.remove() -class RouterMixin(object): +class BrokerMixin(object): broker_class = mitogen.master.Broker - router_class = mitogen.master.Router def setUp(self): - super(RouterMixin, self).setUp() + super(BrokerMixin, self).setUp() self.broker = self.broker_class() - self.router = self.router_class(self.broker) def tearDown(self): self.broker.shutdown() self.broker.join() - super(RouterMixin, self).tearDown() + super(BrokerMixin, self).tearDown() + + +class RouterMixin(BrokerMixin): + router_class = mitogen.master.Router + + def setUp(self): + super(RouterMixin, self).setUp() + self.router = self.router_class(self.broker) class DockerMixin(RouterMixin): From 29e508fde9bb56dd3b53e47a1c99237cb5e4396d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 19:11:03 +0545 Subject: [PATCH 019/140] ssh: enable app-level keepalive by default; closes #77 --- mitogen/ssh.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 43f8d411..1893fdc9 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -61,7 +61,8 @@ class Stream(mitogen.parent.Stream): def construct(self, hostname, username=None, ssh_path=None, port=None, check_host_keys=True, password=None, identity_file=None, - compression=True, ssh_args=None, **kwargs): + compression=True, ssh_args=None, keepalive_enabled=True, + keepalive_count=3, keepalive_interval=15, **kwargs): super(Stream, self).construct(**kwargs) self.hostname = hostname self.username = username @@ -70,6 +71,9 @@ class Stream(mitogen.parent.Stream): self.password = password self.identity_file = identity_file self.compression = compression + self.keepalive_enabled = keepalive_enabled + self.keepalive_count = keepalive_count + self.keepalive_interval = keepalive_interval if ssh_path: self.ssh_path = ssh_path if ssh_args: @@ -89,6 +93,11 @@ class Stream(mitogen.parent.Stream): bits += ['-i', self.identity_file] if self.compression: bits += ['-o', 'Compression yes'] + if self.keepalive_enabled: + bits += [ + '-o', 'ServerAliveInterval %s' % (self.keepalive_interval,), + '-o', 'ServerAliveCountMax %s' % (self.keepalive_count,), + ] if not self.check_host_keys: bits += [ '-o', 'StrictHostKeyChecking no', From fe614aa9661650a8cd8985d3677ba4412b666eef Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 19:32:31 +0545 Subject: [PATCH 020/140] core: cleanup handlers on broker crash; closes #112. --- mitogen/core.py | 8 +++++++- tests/router_test.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 29e04a2a..b0fb8603 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1201,6 +1201,7 @@ class Router(object): def __init__(self, broker): self.broker = broker + listen(broker, 'crash', self._cleanup_handlers) listen(broker, 'shutdown', self.on_broker_shutdown) # Here seems as good a place as any. @@ -1230,7 +1231,11 @@ class Router(object): for context in self._context_by_id.itervalues(): context.on_shutdown(self.broker) - for _, func in self._handle_map.itervalues(): + self._cleanup_handlers() + + def _cleanup_handlers(self): + while self._handle_map: + _, (_, func) = self._handle_map.popitem() func(_DEAD) def register(self, context, stream): @@ -1415,6 +1420,7 @@ class Broker(object): side.stream.on_disconnect(self) except Exception: LOG.exception('_broker_main() crashed') + fire(self, 'crash') fire(self, 'exit') diff --git a/tests/router_test.py b/tests/router_test.py index c6b4e2df..053037fb 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -21,6 +21,36 @@ def send_n_sized_reply(sender, n): return 123 +class CrashTest(testlib.BrokerMixin, unittest2.TestCase): + # This is testing both Broker's ability to crash nicely, and Router's + # ability to respond to the crash event. + klass = mitogen.master.Router + + def _naughty(self): + raise ValueError('eek') + + def test_shutdown(self): + router = self.klass(self.broker) + + sem = mitogen.core.Latch() + router.add_handler(sem.put) + + log = testlib.LogCapturer('mitogen') + log.start() + + # Force a crash and ensure it wakes up. + self.broker._loop_once = self._naughty + self.broker.defer(lambda: None) + + # sem should have received _DEAD. + self.assertEquals(mitogen.core._DEAD, sem.get()) + + # Ensure it was logged. + expect = '_broker_main() crashed' + self.assertTrue(expect in log.stop()) + + + class AddHandlerTest(unittest2.TestCase): klass = mitogen.master.Router From 40b978c9b77dea8b9486071bf6a65383b79e965d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 20:24:54 +0545 Subject: [PATCH 021/140] core: Fix source verification. Previously: * src_id could be spoofed * auth_id was checked but the message was still delivered! --- mitogen/core.py | 21 ++++++++++---- tests/router_test.py | 68 ++++++++++++++++++++++++++++++++++++++++++++ tests/testlib.py | 18 ++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index b0fb8603..4c1e049b 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1292,12 +1292,21 @@ class Router(object): return # Perform source verification. - if stream is not None: - expected_stream = self._stream_by_id.get(msg.auth_id, - self._stream_by_id.get(mitogen.parent_id)) - if stream != expected_stream: - LOG.error('%r: bad source: got auth ID %r from %r, should be from %r', - self, msg, stream, expected_stream) + if stream: + parent = self._stream_by_id.get(mitogen.parent_id) + expect = self._stream_by_id.get(msg.auth_id, parent) + if stream != expect: + LOG.error('%r: bad auth_id: got %r via %r, not %r: %r', + self, msg.auth_id, stream, expect, msg) + return + + if msg.src_id != msg.auth_id: + expect = self._stream_by_id.get(msg.src_id, parent) + if stream != expect: + LOG.error('%r: bad src_id: got %r via %r, not %r: %r', + self, msg.src_id, stream, expect, msg) + return + if stream.auth_id is not None: msg.auth_id = stream.auth_id diff --git a/tests/router_test.py b/tests/router_test.py index 053037fb..c3d17b8b 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -11,6 +11,10 @@ import mitogen.master import mitogen.utils +def ping(): + return True + + @mitogen.core.takes_router def return_router_max_message_size(router): return router.max_message_size @@ -21,6 +25,70 @@ def send_n_sized_reply(sender, n): return 123 +class SourceVerifyTest(testlib.RouterMixin, unittest2.TestCase): + def setUp(self): + super(SourceVerifyTest, self).setUp() + # Create some children, ping them, and store what their messages look + # like so we can mess with them later. + self.child1 = self.router.fork() + self.child1_msg = self.child1.call_async(ping).get() + self.child1_stream = self.router._stream_by_id[self.child1.context_id] + + self.child2 = self.router.fork() + self.child2_msg = self.child2.call_async(ping).get() + self.child2_stream = self.router._stream_by_id[self.child2.context_id] + + def test_bad_auth_id(self): + # Deliver a message locally from child2, but using child1's stream. + log = testlib.LogCapturer() + log.start() + + # Used to ensure the message was dropped rather than routed after the + # error is logged. + recv = mitogen.core.Receiver(self.router) + self.child2_msg.handle = recv.handle + + self.broker.defer(self.router._async_route, + self.child2_msg, + stream=self.child1_stream) + + # Wait for IO loop to finish everything above. + self.sync_with_broker() + + # Ensure message wasn't forwarded. + self.assertTrue(recv.empty()) + + # Ensure error was logged. + expect = 'bad auth_id: got %d via' % (self.child2_msg.auth_id,) + self.assertTrue(expect in log.stop()) + + def test_bad_src_id(self): + # Deliver a message locally from child2 with the correct auth_id, but + # the wrong src_id. + log = testlib.LogCapturer() + log.start() + + # Used to ensure the message was dropped rather than routed after the + # error is logged. + recv = mitogen.core.Receiver(self.router) + self.child2_msg.handle = recv.handle + self.child2_msg.src_id = self.child1.context_id + + self.broker.defer(self.router._async_route, + self.child2_msg, + self.child2_stream) + + # Wait for IO loop to finish everything above. + self.sync_with_broker() + + # Ensure message wasn't forwarded. + self.assertTrue(recv.empty()) + + # Ensure error was lgoged. + expect = 'bad src_id: got %d via' % (self.child1_msg.src_id,) + self.assertTrue(expect in log.stop()) + + class CrashTest(testlib.BrokerMixin, unittest2.TestCase): # This is testing both Broker's ability to crash nicely, and Router's # ability to respond to the crash event. diff --git a/tests/testlib.py b/tests/testlib.py index 5d959b8d..cac5b1e9 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -11,6 +11,7 @@ import urlparse import unittest2 +import mitogen.core import mitogen.master import mitogen.utils @@ -115,6 +116,20 @@ def wait_for_port( % (host, port)) +def sync_with_broker(broker, timeout=10.0): + """ + Insert a synchronization barrier between the calling thread and the Broker + thread, ensuring it has completed at least one full IO loop before + returning. + + Used to block while asynchronous stuff (like defer()) happens on the + broker. + """ + sem = mitogen.core.Latch() + broker.defer(sem.put, None) + sem.get(timeout=10.0) + + class LogCapturer(object): def __init__(self, name=None): self.sio = StringIO.StringIO() @@ -188,6 +203,9 @@ class BrokerMixin(object): self.broker.join() super(BrokerMixin, self).tearDown() + def sync_with_broker(self): + sync_with_broker(self.broker) + class RouterMixin(BrokerMixin): router_class = mitogen.master.Router From f726ef86de333d7bd2b724d0cf255f238137799b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 21:33:18 +0545 Subject: [PATCH 022/140] tests: first_stage_test regression due to 1ff27ada4973f50682386a7ce1eb4d7b6c77003a --- tests/first_stage_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index ba12137d..d868f8b5 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -17,7 +17,7 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # * 3.x starting 2.7 def test_valid_syntax(self): - stream = mitogen.parent.Stream(self.router, 0) + stream = mitogen.parent.Stream(self.router, 0, max_message_size=123) args = stream.get_boot_command() # Executing the boot command will print "EC0" and expect to read from From 46a14d4ae28d3b5d86bccc9db3a77cc489ba5500 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 21:33:59 +0545 Subject: [PATCH 023/140] core: Fix logging crash if data is non-string. --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 4c1e049b..4f2bbd75 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -386,7 +386,7 @@ class Sender(object): def send(self, data): """Send `data` to the remote.""" - _vv and IOLOG.debug('%r.send(%r..)', self, data[:100]) + _vv and IOLOG.debug('%r.send(%r..)', self, repr(data)[:100]) self.context.send( Message.pickled( data, From 6670cba41cbd4b947e3d9441127994d49f7e0e35 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 21:34:21 +0545 Subject: [PATCH 024/140] Introduce handler policy functions; closes #138. Now you can specify a function to add_handler() that authenticates the message header, with has_parent_authority() and is_immediate_child() built in. --- docs/api.rst | 24 +++++++++++++++- mitogen/core.py | 68 ++++++++++++++++++++++++++++++++------------ mitogen/master.py | 15 ++++++++-- mitogen/parent.py | 17 ++++++++++- tests/router_test.py | 49 +++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 23 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 50c5bdab..2c4dc42f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -412,7 +412,7 @@ Router Class receive side to the I/O multiplexer. This This method remains public for now while hte design has not yet settled. - .. method:: add_handler (fn, handle=None, persist=True, respondent=None) + .. method:: add_handler (fn, handle=None, persist=True, respondent=None, policy=None) Invoke `fn(msg)` for each Message sent to `handle` from this context. Unregister after one invocation if `persist` is ``False``. If `handle` @@ -435,6 +435,28 @@ Router Class In future `respondent` will likely also be used to prevent other contexts from sending messages to the handle. + :param function policy: + Function invoked as `policy(msg, stream)` where `msg` is a + :py:class:`mitogen.core.Message` about to be delivered, and + `stream` is the :py:class:`mitogen.core.Stream` on which it was + received. The function must return :py:data:`True`, otherwise an + error is logged and delivery is refused. + + Two built-in policy functions exist: + + * :py:func:`mitogen.core.has_parent_authority`: requires the + message arrived from a parent context, or a context acting with a + parent context's authority (``auth_id``). + + * :py:func:`mitogen.parent.is_immediate_child`: requires the + message arrived from an immediately connected child, for use in + messaging patterns where either something becomes buggy or + insecure by permitting indirect upstream communication. + + In case of refusal, and the message's ``reply_to`` field is + nonzero, a :py:class:`mitogen.core.CallError` is delivered to the + sender indicating refusal occurred. + :return: `handle`, or if `handle` was ``None``, the newly allocated handle. diff --git a/mitogen/core.py b/mitogen/core.py index 4f2bbd75..2085a850 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -157,6 +157,10 @@ def _unpickle_dead(): _DEAD = Dead() +def has_parent_authority(msg, _stream): + return msg.auth_id in mitogen.parent_ids + + def listen(obj, name, func): signals = vars(obj).setdefault('_signals', {}) signals.setdefault(name, []).append(func) @@ -407,11 +411,17 @@ class Receiver(object): notify = None raise_channelerror = True - def __init__(self, router, handle=None, persist=True, respondent=None): + def __init__(self, router, handle=None, persist=True, + respondent=None, policy=None): self.router = router self.handle = handle # Avoid __repr__ crash in add_handler() - self.handle = router.add_handler(self._on_receive, handle, - persist, respondent) + self.handle = router.add_handler( + fn=self._on_receive, + handle=handle, + policy=policy, + persist=persist, + respondent=respondent, + ) self._latch = Latch() def __repr__(self): @@ -497,7 +507,11 @@ class Importer(object): # Presence of an entry in this map indicates in-flight GET_MODULE. self._callbacks = {} - router.add_handler(self._on_load_module, LOAD_MODULE) + router.add_handler( + fn=self._on_load_module, + handle=LOAD_MODULE, + policy=has_parent_authority, + ) self._cache = {} if core_src: self._cache['mitogen.core'] = ( @@ -1235,7 +1249,7 @@ class Router(object): def _cleanup_handlers(self): while self._handle_map: - _, (_, func) = self._handle_map.popitem() + _, (_, func, _) = self._handle_map.popitem() func(_DEAD) def register(self, context, stream): @@ -1245,18 +1259,22 @@ class Router(object): self.broker.start_receive(stream) listen(stream, 'disconnect', lambda: self.on_stream_disconnect(stream)) - def add_handler(self, fn, handle=None, persist=True, respondent=None): + def add_handler(self, fn, handle=None, persist=True, + policy=None, respondent=None): handle = handle or self._last_handle.next() _vv and IOLOG.debug('%r.add_handler(%r, %r, %r)', self, fn, handle, persist) - self._handle_map[handle] = persist, fn if respondent: + assert policy is None + def policy(msg, _stream): + return msg.src_id == respondent.context_id def on_disconnect(): if handle in self._handle_map: fn(_DEAD) del self._handle_map[handle] listen(respondent, 'disconnect', on_disconnect) + self._handle_map[handle] = persist, fn, policy return handle def on_shutdown(self, broker): @@ -1268,14 +1286,26 @@ class Router(object): _v and LOG.debug('%r.on_shutdown(): killing %r: %r', self, handle, fn) fn(_DEAD) - def _invoke(self, msg): + refused_msg = 'Refused by policy.' + + def _invoke(self, msg, stream): #IOLOG.debug('%r._invoke(%r)', self, msg) try: - persist, fn = self._handle_map[msg.handle] + persist, fn, policy = self._handle_map[msg.handle] except KeyError: LOG.error('%r: invalid handle: %r', self, msg) return + if policy and not policy(msg, stream): + LOG.error('%r: policy refused message: %r', self, msg) + if msg.reply_to: + self.route(Message.pickled( + CallError(self.refused_msg), + dst_id=msg.src_id, + handle=msg.reply_to + )) + return + if not persist: del self._handle_map[msg.handle] @@ -1311,7 +1341,7 @@ class Router(object): msg.auth_id = stream.auth_id if msg.dst_id == mitogen.context_id: - return self._invoke(msg) + return self._invoke(msg, stream) stream = self._stream_by_id.get(msg.dst_id) if stream is None: @@ -1456,10 +1486,8 @@ class ExternalContext(object): def _on_shutdown_msg(self, msg): _v and LOG.debug('_on_shutdown_msg(%r)', msg) - if msg != _DEAD and msg.auth_id not in mitogen.parent_ids: - LOG.warning('Ignoring SHUTDOWN from non-parent: %r', msg) - return - self.broker.shutdown() + if msg != _DEAD: + self.broker.shutdown() def _on_parent_disconnect(self): _v and LOG.debug('%r: parent stream is gone, dying.', self) @@ -1473,14 +1501,20 @@ class ExternalContext(object): enable_profiling() self.broker = Broker() self.router = Router(self.broker) - self.router.add_handler(self._on_shutdown_msg, SHUTDOWN) + self.router.add_handler( + fn=self._on_shutdown_msg, + handle=SHUTDOWN, + policy=has_parent_authority, + ) self.master = Context(self.router, 0, 'master') if parent_id == 0: self.parent = self.master else: self.parent = Context(self.router, parent_id, 'parent') - self.channel = Receiver(self.router, CALL_FUNCTION) + self.channel = Receiver(router=self.router, + handle=CALL_FUNCTION, + policy=has_parent_authority) self.stream = Stream(self.router, parent_id) self.stream.name = 'parent' self.stream.accept(in_fd, out_fd) @@ -1576,8 +1610,6 @@ class ExternalContext(object): def _dispatch_one(self, msg): data = msg.unpickle(throw=False) _v and LOG.debug('_dispatch_calls(%r)', data) - if msg.auth_id not in mitogen.parent_ids: - LOG.warning('CALL_FUNCTION from non-parent %r', msg.auth_id) modname, klass, func, args, kwargs = data obj = __import__(modname, {}, {}, ['']) diff --git a/mitogen/master.py b/mitogen/master.py index 0cf5d451..dca4eb46 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -288,7 +288,10 @@ class LogForwarder(object): def __init__(self, router): self._router = router self._cache = {} - router.add_handler(self._on_forward_log, mitogen.core.FORWARD_LOG) + router.add_handler( + fn=self._on_forward_log, + handle=mitogen.core.FORWARD_LOG, + ) def _on_forward_log(self, msg): if msg == mitogen.core._DEAD: @@ -524,7 +527,10 @@ class ModuleResponder(object): self._cache = {} # fullname -> pickled self.blacklist = [] self.whitelist = [''] - router.add_handler(self._on_get_module, mitogen.core.GET_MODULE) + router.add_handler( + fn=self._on_get_module, + handle=mitogen.core.GET_MODULE, + ) def __repr__(self): return 'ModuleResponder(%r)' % (self._router,) @@ -684,7 +690,10 @@ class IdAllocator(object): self.router = router self.next_id = 1 self.lock = threading.Lock() - router.add_handler(self.on_allocate_id, mitogen.core.ALLOCATE_ID) + router.add_handler( + fn=self.on_allocate_id, + handle=mitogen.core.ALLOCATE_ID, + ) def __repr__(self): return 'IdAllocator(%r)' % (self.router,) diff --git a/mitogen/parent.py b/mitogen/parent.py index 599a4603..41fe3676 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -78,6 +78,14 @@ def get_log_level(): return (LOG.level or logging.getLogger().level or logging.INFO) +def is_immediate_child(msg, stream): + """ + Handler policy that requires messages to arrive only from immediately + connected children. + """ + return msg.src_id == stream.remote_id + + def minimize_source(source): subber = lambda match: '""' + ('\n' * match.group(0).count('\n')) source = DOCSTRING_RE.sub(subber, source) @@ -554,11 +562,13 @@ class RouteMonitor(object): fn=self._on_add_route, handle=mitogen.core.ADD_ROUTE, persist=True, + policy=is_immediate_child, ) self.router.add_handler( fn=self._on_del_route, handle=mitogen.core.DEL_ROUTE, persist=True, + policy=is_immediate_child, ) def propagate(self, handle, target_id, name=None): @@ -795,7 +805,12 @@ class ModuleForwarder(object): self.router = router self.parent_context = parent_context self.importer = importer - router.add_handler(self._on_get_module, mitogen.core.GET_MODULE) + router.add_handler( + fn=self._on_get_module, + handle=mitogen.core.GET_MODULE, + persist=True, + policy=is_immediate_child, + ) def __repr__(self): return 'ModuleForwarder(%r)' % (self.router,) diff --git a/tests/router_test.py b/tests/router_test.py index c3d17b8b..2c0b7e60 100644 --- a/tests/router_test.py +++ b/tests/router_test.py @@ -89,6 +89,55 @@ class SourceVerifyTest(testlib.RouterMixin, unittest2.TestCase): self.assertTrue(expect in log.stop()) +class PolicyTest(testlib.RouterMixin, testlib.TestCase): + def test_allow_any(self): + # This guy gets everything. + recv = mitogen.core.Receiver(self.router) + recv.to_sender().send(123) + self.sync_with_broker() + self.assertFalse(recv.empty()) + self.assertEquals(123, recv.get().unpickle()) + + def test_refuse_all(self): + # Deliver a message locally from child2 with the correct auth_id, but + # the wrong src_id. + log = testlib.LogCapturer() + log.start() + + # This guy never gets anything. + recv = mitogen.core.Receiver( + router=self.router, + policy=(lambda msg, stream: False), + ) + + # This guy becomes the reply_to of our refused message. + reply_target = mitogen.core.Receiver(self.router) + + # Send the message. + self.router.route( + mitogen.core.Message( + dst_id=mitogen.context_id, + handle=recv.handle, + reply_to=reply_target.handle, + ) + ) + + # Wait for IO loop. + self.sync_with_broker() + + # Verify log. + expect = '%r: policy refused message: ' % (self.router,) + self.assertTrue(expect in log.stop()) + + # Verify message was not delivered. + self.assertTrue(recv.empty()) + + # Verify CallError received by reply_to target. + e = self.assertRaises(mitogen.core.CallError, + lambda: reply_target.get().unpickle()) + self.assertEquals(e[0], self.router.refused_msg) + + class CrashTest(testlib.BrokerMixin, unittest2.TestCase): # This is testing both Broker's ability to crash nicely, and Router's # ability to respond to the crash event. From bde177837379d0828eb4c6f69b727cdc69a53692 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 23:36:50 +0545 Subject: [PATCH 025/140] tests: merge tty_create_child() test into parent_test and fix hang --- tests/parent_test.py | 24 +++++++++++++++++++++++- tests/tty_create_child_test.py | 33 --------------------------------- 2 files changed, 23 insertions(+), 34 deletions(-) delete mode 100644 tests/tty_create_child_test.py diff --git a/tests/parent_test.py b/tests/parent_test.py index 169d237b..5473b015 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -1,11 +1,12 @@ import os import subprocess +import tempfile import time import unittest2 +import testlib import mitogen.parent -import testlib class ContextTest(testlib.RouterMixin, unittest2.TestCase): @@ -16,6 +17,27 @@ class ContextTest(testlib.RouterMixin, unittest2.TestCase): self.assertRaises(OSError, lambda: os.kill(pid, 0)) +class TtyCreateChildTest(unittest2.TestCase): + func = staticmethod(mitogen.parent.tty_create_child) + + def test_dev_tty_open_succeeds(self): + tf = tempfile.NamedTemporaryFile() + try: + pid, fd = self.func( + 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,) + ) + deadline = time.time() + 5.0 + for line in mitogen.parent.iter_read(fd, deadline): + self.assertEquals('hi\n', line) + break + waited_pid, status = os.waitpid(pid, 0) + self.assertEquals(pid, waited_pid) + self.assertEquals(0, status) + self.assertEquals('', tf.read()) + finally: + tf.close() + + class IterReadTest(unittest2.TestCase): func = staticmethod(mitogen.parent.iter_read) diff --git a/tests/tty_create_child_test.py b/tests/tty_create_child_test.py deleted file mode 100644 index 879373ca..00000000 --- a/tests/tty_create_child_test.py +++ /dev/null @@ -1,33 +0,0 @@ - -import os -import tempfile -import time -import unittest2 - -import testlib -import mitogen.parent - - -class TtyCreateChildTest(unittest2.TestCase): - func = staticmethod(mitogen.parent.tty_create_child) - - def test_dev_tty_open_succeeds(self): - tf = tempfile.NamedTemporaryFile() - try: - pid, fd = self.func( - 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,) - ) - # TODO: this waitpid hangs on OS X. Installing a SIGCHLD handler - # reveals the parent /is/ notified of the child's death, but - # calling waitpid() from within SIGCHLD yields "No such processes". - # Meanwhile, even inserting a sleep, the following call will hang. - waited_pid, status = os.waitpid(pid, 0) - self.assertEquals(pid, waited_pid) - self.assertEquals(0, status) - self.assertEquals('', tf.read()) - finally: - tf.close() - - -if __name__ == '__main__': - unittest2.main() From 4c433dbed15fb14fa781ddc113c76c88a1e0775c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 23:48:09 +0545 Subject: [PATCH 026/140] parent_test: Add explanation. --- tests/parent_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/parent_test.py b/tests/parent_test.py index 5473b015..da9f5e15 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -21,6 +21,15 @@ class TtyCreateChildTest(unittest2.TestCase): func = staticmethod(mitogen.parent.tty_create_child) def test_dev_tty_open_succeeds(self): + # In the early days of UNIX, a process that lacked a controlling TTY + # would acquire one simply by opening an existing TTY. Linux and OS X + # continue to follow this behaviour, however at least FreeBSD moved to + # requiring an explicit ioctl(). Linux supports it, but we don't yet + # use it there and anyway the behaviour will never change, so no point + # in fixing things that aren't broken. Below we test that + # getpass-loving apps like sudo and ssh get our slave PTY when they + # attempt to open /dev/tty, which is what they both do on attempting to + # read a password. tf = tempfile.NamedTemporaryFile() try: pid, fd = self.func( From bbb0f1bbd8449c5696eb4da1ba97049226cbdb05 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 29 Mar 2018 23:54:51 +0545 Subject: [PATCH 027/140] issue #155: fix double-fork behaviour and test it this time. --- docs/getting_started.rst | 1 - mitogen/fork.py | 12 ++++++++++-- tests/fork_test.py | 21 +++++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index a3bc7652..12cd9898 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -262,7 +262,6 @@ Recovering Mitogen Object References In Children ... - Recursion --------- diff --git a/mitogen/fork.py b/mitogen/fork.py index e4a8625a..5fd2281c 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -71,10 +71,12 @@ def handle_child_crash(): Respond to _child_main() crashing by ensuring the relevant exception is logged to /dev/tty. """ - sys.stderr.write('\n\nFORKED CHILD PID %d CRASHED\n%s\n\n' % ( + tty = open('/dev/tty', 'wb') + tty.write('\n\nFORKED CHILD PID %d CRASHED\n%s\n\n' % ( os.getpid(), traceback.format_exc(), )) + tty.close() os._exit(1) @@ -134,7 +136,13 @@ class Stream(mitogen.parent.Stream): # avoid ExternalContext.main() accidentally allocating new files over # the standard handles. os.dup2(childfp.fileno(), 0) - os.dup2(sys.stderr.fileno(), 2) + + # Avoid corrupting the stream on fork crash by dupping /dev/null over + # stderr. Instead, handle_child_crash() uses /dev/tty to log errors. + devnull = os.open('/dev/null', os.O_WRONLY) + if devnull != 2: + os.dup2(devnull, 2) + os.close(devnull) childfp.close() kwargs = self.get_main_kwargs() diff --git a/tests/fork_test.py b/tests/fork_test.py index 006f0b11..08c099ba 100644 --- a/tests/fork_test.py +++ b/tests/fork_test.py @@ -26,6 +26,10 @@ c_ssl.RAND_pseudo_bytes.argtypes = [ctypes.c_char_p, ctypes.c_int] c_ssl.RAND_pseudo_bytes.restype = ctypes.c_int +def ping(): + return 123 + + def random_random(): return random.random() @@ -54,5 +58,22 @@ class ForkTest(testlib.RouterMixin, unittest2.TestCase): RAND_pseudo_bytes()) +class DoubleChildTest(testlib.RouterMixin, unittest2.TestCase): + def test_okay(self): + # When forking from the master process, Mitogen had nothing to do with + # setting up stdio -- that was inherited wherever the Master is running + # (supervisor, iTerm, etc). When forking from a Mitogen child context + # however, Mitogen owns all of fd 0, 1, and 2, and during the fork + # procedure, it deletes all of these descriptors. That leaves the + # process in a weird state that must be handled by some combination of + # fork.py and ExternalContext.main(). + + # Below we simply test whether ExternalContext.main() managed to boot + # successfully. In future, we need lots more tests. + c1 = self.router.fork() + c2 = self.router.fork(via=c1) + self.assertEquals(123, c2.call(ping)) + + if __name__ == '__main__': unittest2.main() From e0c4d6b34838e1440df2b166d8303bf69f279ec1 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 00:13:22 +0545 Subject: [PATCH 028/140] ansible: Quick fix for #172. --- ansible_mitogen/logging.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 672f2b99..dc12e452 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -69,11 +69,12 @@ def setup(): """ display = find_display() - logging.getLogger('ansible_mitogen').handlers = [Handler(display.v)] - logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) + if display.verbosity > 2: + logging.getLogger('ansible_mitogen').handlers = [Handler(display.v)] + logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) - mitogen.core.LOG.handlers = [Handler(display.v)] - mitogen.core.LOG.setLevel(logging.DEBUG) + mitogen.core.LOG.handlers = [Handler(display.vv)] + mitogen.core.LOG.setLevel(logging.DEBUG) mitogen.core.IOLOG.handlers = [Handler(display.vvvv)] if display.verbosity > 3: From 36e1ae15fd881214c0a3230ff6ed736f7d12836c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 01:10:30 +0545 Subject: [PATCH 029/140] issue #172: prevent 'No handlers..' error being printed. --- ansible_mitogen/logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index dc12e452..3360051b 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -69,11 +69,11 @@ def setup(): """ display = find_display() + logging.getLogger('ansible_mitogen').handlers = [Handler(display.vvv)] + mitogen.core.LOG.handlers = [Handler(display.vvv)] + if display.verbosity > 2: - logging.getLogger('ansible_mitogen').handlers = [Handler(display.v)] logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) - - mitogen.core.LOG.handlers = [Handler(display.vv)] mitogen.core.LOG.setLevel(logging.DEBUG) mitogen.core.IOLOG.handlers = [Handler(display.vvvv)] From 8674ec42ddbf43090d801366a0794a7921ac122e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 01:37:17 +0545 Subject: [PATCH 030/140] docs: add new risk --- docs/ansible.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index bf4a05f8..d193ca89 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -131,9 +131,13 @@ High Risk `_ has received minimal testing. +* No mechanism exists yet to bound the number of interpreters created during a + run. For some playbooks that parameterize ``become_user`` over a large number + of user accounts, resource exhaustion may be triggered on the target machine. + * Only Ansible 2.4 is being used for development, with occasional tests under - 2.3 and 2.2. It should be more than possible to fully support at least 2.3, - if not also 2.2. + 2.5, 2.3 and 2.2. It should be more than possible to fully support at least + 2.3, if not also 2.2. Low Risk From ffdd1923976bd2f45192b59e5d39f18b41d94a27 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 02:22:07 +0545 Subject: [PATCH 031/140] issue #155: must catch select.error too. Regression caused by merging exception handlers in 9079176. --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 2085a850..e00f4cd0 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -214,7 +214,7 @@ def io_op(func, *args): while True: try: return func(*args), False - except OSError, e: + except (select.error, OSError), e: _vv and IOLOG.debug('io_op(%r) -> OSError: %s', func, e) if e.errno == errno.EINTR: continue From fa271fcc8e4757c1b606bd448320f98577cec852 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 02:24:30 +0545 Subject: [PATCH 032/140] issue #174: select.error interface differs to OSError Only OSError got the magical attribute treatment, select.error still behaves like a tuple. --- mitogen/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index e00f4cd0..beeedd63 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -216,9 +216,9 @@ def io_op(func, *args): return func(*args), False except (select.error, OSError), e: _vv and IOLOG.debug('io_op(%r) -> OSError: %s', func, e) - if e.errno == errno.EINTR: + if e[0] == errno.EINTR: continue - if e.errno in (errno.EIO, errno.ECONNRESET, errno.EPIPE): + if e[0] in (errno.EIO, errno.ECONNRESET, errno.EPIPE): return None, True raise From 682a4ca8d4cd8c38943e446d521e53754cbe4949 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 02:31:14 +0545 Subject: [PATCH 033/140] issue #174: reproduction. --- examples/playbook/issue_174.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 examples/playbook/issue_174.yml diff --git a/examples/playbook/issue_174.yml b/examples/playbook/issue_174.yml new file mode 100644 index 00000000..b68fdb24 --- /dev/null +++ b/examples/playbook/issue_174.yml @@ -0,0 +1,9 @@ +--- + +- hosts: all + gather_facts: false + tasks: + - name: add nginx ppa + become: yes + apt_repository: repo='ppa:nginx/stable' update_cache=yes + From 3d5bbb9a636ba36e6f3c6f598c9b8074bf25e274 Mon Sep 17 00:00:00 2001 From: Wesley Moore Date: Fri, 30 Mar 2018 13:00:43 +1100 Subject: [PATCH 034/140] Use become_pass for sudo password --- ansible_mitogen/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 2bb458d1..1c404410 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -198,7 +198,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): cast({ 'method': 'sudo', 'username': self._play_context.become_user, - 'password': self._play_context.password, + 'password': self._play_context.become_pass, 'python_path': python_path or self.python_path, 'sudo_path': self.sudo_path, 'connect_timeout': self._play_context.timeout, From a68e83346333b8ff7f484c1cb500b669edbbdaac Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 12:35:41 +0545 Subject: [PATCH 035/140] examples: add a ton of comments to mitop.py. --- examples/mitop.py | 87 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 17 deletions(-) diff --git a/examples/mitop.py b/examples/mitop.py index 84b52168..7b6d4e42 100644 --- a/examples/mitop.py +++ b/examples/mitop.py @@ -10,15 +10,28 @@ import mitogen.utils class Host(object): + """ + A target host from the perspective of the master process. + """ + #: String hostname. name = None + + #: mitogen.parent.Context used to call functions on the host. context = None + + #: mitogen.core.Receiver the target delivers state updates to. recv = None def __init__(self): - self.procs = {} #: pid -> Process() + #: Mapping of pid -> Process() for each process described + #: in the host's previous status update. + self.procs = {} class Process(object): + """ + A single process running on a target host. + """ host = None user = None pid = None @@ -30,14 +43,20 @@ class Process(object): rss = None -@mitogen.core.takes_router -def remote_main(context_id, handle, delay, router): - context = mitogen.core.Context(router, context_id) - sender = mitogen.core.Sender(context, handle) - +def child_main(sender, delay): + """ + Executed on the main thread of the Python interpreter running on the target + machine, using context.call() by the master. It simply sends the output of + the UNIX 'ps' command at regular intervals toward a Receiver on master. + + :param mitogen.core.Sender sender: + The Sender to use for delivering our result. This could target + anywhere, but the sender supplied by the master simply causes results + to be delivered to the master's associated per-host Receiver. + """ args = ['ps', '-axwwo', 'user,pid,ppid,pgid,%cpu,rss,command'] while True: - sender.put(subprocess.check_output(args)) + sender.send(subprocess.check_output(args)) time.sleep(delay) @@ -124,7 +143,12 @@ class Painter(object): self.stdscr.refresh() -def local_main(painter, router, select, delay): +def master_main(painter, router, select, delay): + """ + Loop until CTRL+C is pressed, waiting for the next result delivered by the + Select. Use parse_output() to turn that result ('ps' command output) into + rich data, and finally repaint the screen if the repaint delay has passed. + """ next_paint = 0 while True: msg = select.get() @@ -134,8 +158,13 @@ def local_main(painter, router, select, delay): painter.paint() -def main(router, argv): - mitogen.utils.log_to_file() +@mitogen.main() +def main(router): + """ + Main program entry point. @mitogen.main() is just a helper to handle + reliable setup/destruction of Broker, Router and the logging package. + """ + argv = sys.argv[1:] if not len(argv): print 'mitop: Need a list of SSH hosts to connect to.' sys.exit(1) @@ -144,36 +173,60 @@ def main(router, argv): select = mitogen.master.Select(oneshot=False) hosts = [] + # For each hostname on the command line, create a Host instance, a Mitogen + # connection, a Receiver to accept messages from the host, and finally + # start child_main() on the host to pump messages into the receiver. for hostname in argv: print 'Starting on', hostname host = Host() host.name = hostname + if host.name == 'localhost': host.context = router.local() else: host.context = router.ssh(hostname=host.name) + # A receiver wires up a handle (via Router.add_handler()) to an + # internal thread-safe queue object, which can be drained through calls + # to recv.get(). host.recv = mitogen.core.Receiver(router) host.recv.host = host + + # But we don't want to receive data from just one receiver, we want to + # receive data from many. In this case we can use a Select(). It knows + # how to efficiently sleep while waiting for the first message sent to + # many receivers. select.add(host.recv) - call_recv = host.context.call_async(remote_main, - mitogen.context_id, host.recv.handle, delay) + # The inverse of a Receiver is a Sender. Unlike receivers, senders are + # serializable, so we can call the .to_sender() helper method to create + # one equivalent to our host's receiver, and pass it directly to the + # host as a function parameter. + sender = host.recv.to_sender() + + # Finally invoke the function in the remote target. Since child_main() + # is an infinite loop, using .call() would block the parent, since + # child_main() never returns. Instead use .call_async(), which returns + # another Receiver. We also want to wait for results from receiver -- + # even child_main() never returns, if there is an exception, it will be + # delivered instead. + call_recv = host.context.call_async(child_main, sender, delay) + call_recv.host = host # Adding call_recv to the select will cause CallError to be thrown by - # .get() if startup in the context fails, halt local_main() and cause + # .get() if startup in the context fails, halt master_main() and cause # the exception to be printed. select.add(call_recv) hosts.append(host) + # Painter just wraps up all the prehistory ncurses code and keeps it out of + # master_main(). painter = Painter(hosts) try: try: - local_main(painter, router, select, delay) + master_main(painter, router, select, delay) except KeyboardInterrupt: + # Shut down gracefully when the user presses CTRL+C. pass finally: painter.close() - -if __name__ == '__main__': - mitogen.utils.run_with_router(main, sys.argv[1:]) From 9eccfb4972a7d94ff8fb0689440f8be5622908f6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 12:41:53 +0545 Subject: [PATCH 036/140] examples: add top-level doc --- examples/mitop.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/mitop.py b/examples/mitop.py index 7b6d4e42..cf37703a 100644 --- a/examples/mitop.py +++ b/examples/mitop.py @@ -1,3 +1,15 @@ +""" +mitop.py is a version of the UNIX top command that knows how to display process +lists from multiple machines in a single listing. + +This is a basic, initial version showing overall program layout. A future +version will extend it to: + + * Only notify the master of changed processes, rather than all processes. + * Runtime-reconfigurable filters and aggregations handled on the remote + machines rather than forcing a bottleneck in the master. + +""" import curses import subprocess From 76ac49dbdc3bf0190067c8ccc7add9461f9a953e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 12:48:26 +0545 Subject: [PATCH 037/140] examples: more comments. --- examples/mitop.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/examples/mitop.py b/examples/mitop.py index cf37703a..89de9aad 100644 --- a/examples/mitop.py +++ b/examples/mitop.py @@ -57,9 +57,9 @@ class Process(object): def child_main(sender, delay): """ - Executed on the main thread of the Python interpreter running on the target - machine, using context.call() by the master. It simply sends the output of - the UNIX 'ps' command at regular intervals toward a Receiver on master. + Executed on the main thread of the Python interpreter running on each + target machine, Context.call() from the master. It simply sends the output + of the UNIX 'ps' command at regular intervals toward a Receiver on master. :param mitogen.core.Sender sender: The Sender to use for delivering our result. This could target @@ -102,6 +102,9 @@ def parse_output(host, s): class Painter(object): + """ + This is ncurses (screen drawing) magic, you can ignore it. :) + """ def __init__(self, hosts): self.stdscr = curses.initscr() curses.start_color() @@ -219,15 +222,15 @@ def main(router): # Finally invoke the function in the remote target. Since child_main() # is an infinite loop, using .call() would block the parent, since # child_main() never returns. Instead use .call_async(), which returns - # another Receiver. We also want to wait for results from receiver -- - # even child_main() never returns, if there is an exception, it will be - # delivered instead. + # another Receiver. We also want to wait for results from it -- + # although child_main() never returns, if it crashes the exception will + # be delivered instead. call_recv = host.context.call_async(child_main, sender, delay) call_recv.host = host - # Adding call_recv to the select will cause CallError to be thrown by - # .get() if startup in the context fails, halt master_main() and cause - # the exception to be printed. + # Adding call_recv to the select will cause mitogen.core.CallError to + # be thrown by .get() if startup of any context fails, causing halt of + # master_main(), and the exception to be printed. select.add(call_recv) hosts.append(host) From 6958b8ff09eac0732e459fd5576a354d9daaf2bb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 13:05:47 +0545 Subject: [PATCH 038/140] docs: More getting started. --- docs/getting_started.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 12cd9898..af529b00 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -247,6 +247,24 @@ without the need for writing asynchronous code:: print 'Reply from %s: %s' % (recv.context, data) +Running Code That May Hang +-------------------------- + +When executing code that may hang due to, for example, talking to network peers +that may become unavailable, it is desirable to be able to recover control in +the case a remote call has hung. + +By specifying the `timeout` parameter to :meth:`Receiver.get` on the receiver +returned by `Context.call_async`, it becomes possible to wait for a function to +complete, but time out if its result does not become available. + +When a context has become hung like this, it is still possible to gracefully +terminate it using the :meth:`Context.shutdown` method. This method sends a +shutdown message to the target process, where its IO multiplexer thread can +still process it independently of the hung function running on on the target's +main thread. + + Recovering Mitogen Object References In Children ------------------------------------------------ From 9067a7b173b02e5482795138b60e158ec93524fa Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 14:42:54 +0545 Subject: [PATCH 039/140] ansible: Move setLevel() bits together. --- ansible_mitogen/logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 3360051b..ce4ea127 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -71,12 +71,12 @@ def setup(): logging.getLogger('ansible_mitogen').handlers = [Handler(display.vvv)] mitogen.core.LOG.handlers = [Handler(display.vvv)] + mitogen.core.IOLOG.handlers = [Handler(display.vvvv)] + mitogen.core.IOLOG.propagate = False if display.verbosity > 2: logging.getLogger('ansible_mitogen').setLevel(logging.DEBUG) mitogen.core.LOG.setLevel(logging.DEBUG) - mitogen.core.IOLOG.handlers = [Handler(display.vvvv)] if display.verbosity > 3: mitogen.core.IOLOG.setLevel(logging.DEBUG) - mitogen.core.IOLOG.propagate = False From 28cd17cf56ec573ad92a79b3566730c5230cdd82 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 15:55:38 +0545 Subject: [PATCH 040/140] issue #106: import skeletal new executor. --- ansible_mitogen/executor.py | 287 ++++++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 ansible_mitogen/executor.py diff --git a/ansible_mitogen/executor.py b/ansible_mitogen/executor.py new file mode 100644 index 00000000..00e91d05 --- /dev/null +++ b/ansible_mitogen/executor.py @@ -0,0 +1,287 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import json +import os +import tempfile + +import ansible_mitogen.helpers + +try: + from shlex import quote as shlex_quote +except ImportError: + from pipes import quote as shlex_quote + +# Prevent accidental import of an Ansible module from hanging on stdin read. +import ansible.module_utils.basic +ansible.module_utils.basic._ANSIBLE_ARGS = '{}' + + +class Exit(Exception): + """ + Raised when a module exits with success. + """ + def __init__(self, dct): + self.dct = dct + + +class ModuleError(Exception): + """ + Raised when a module voluntarily indicates failure via .fail_json(). + """ + def __init__(self, msg, dct): + Exception.__init__(self, msg) + self.dct = dct + + +class TemporaryEnvironment(object): + def __init__(self, env=None): + self.original = os.environ.copy() + self.env = env or {} + os.environ.update((k, str(v)) for k, v in self.env.iteritems()) + + def revert(self): + os.environ.clear() + os.environ.update(self.original) + + +class MethodOverrides(object): + @staticmethod + def exit_json(self, **kwargs): + """ + Replace AnsibleModule.exit_json() with something that doesn't try to + kill the process or JSON-encode the result dictionary. Instead, cause + Exit to be raised, with a `dct` attribute containing the successful + result dictionary. + """ + self.add_path_info(kwargs) + kwargs.setdefault('changed', False) + kwargs.setdefault('invocation', { + 'module_args': self.params + }) + kwargs = ansible.module_utils.basic.remove_values( + kwargs, + self.no_log_values, + ) + self.do_cleanup_files() + raise Exit(kwargs) + + @staticmethod + def fail_json(self, **kwargs): + """ + Replace AnsibleModule.fail_json() with something that raises + ModuleError, which includes a `dct` attribute. + """ + self.add_path_info(kwargs) + kwargs.setdefault('failed', True) + kwargs.setdefault('invocation', { + 'module_args': self.params + }) + kwargs = ansible.module_utils.basic.remove_values( + kwargs, + self.no_log_values, + ) + self.do_cleanup_files() + raise ModuleError(kwargs.get('msg'), kwargs) + + klass = ansible.module_utils.basic.AnsibleModule + + def __init__(self): + self._original_exit_json = self.klass.exit_json + self._original_fail_json = self.klass.fail_json + self.klass.exit_json = self.exit_json + self.klass.fail_json = self.fail_json + + def revert(self): + self.klass.exit_json = self._original_exit_json + self.klass.fail_json = self._original_fail_json + + +class ModuleArguments(object): + """ + Patch the ansible.module_utils.basic global arguments variable on + construction, and revert the changes on call to :meth:`revert`. + """ + def __init__(self, args): + self.original = ansible.module_utils.basic._ANSIBLE_ARGS + ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({ + 'ANSIBLE_MODULE_ARGS': args + }) + + def revert(self): + ansible.module_utils.basic._ANSIBLE_ARGS = self._original_args + + +class Runner(object): + def __init__(self, module, raw_params=None, args=None, env=None, + runner_params=None): + if args is None: + args = {} + if raw_params is not None: + args['_raw_params'] = raw_params + if runner_params is None: + runner_params = {} + + self.module = module + self.raw_params = raw_params + self.args = args + self.env = env + self.runner_params = runner_params + + def setup(self): + self._env = TemporaryEnvironment(self.env) + + def revert(self): + self._env.revert() + + def _run(self): + raise NotImplementedError() + + def run(self): + """ + Set up the process environment in preparation for running an Ansible + module. This monkey-patches the Ansible libraries in various places to + prevent it from trying to kill the process on completion, and to + prevent it from reading sys.stdin. + """ + self.setup() + try: + return self._run() + finally: + self.revert() + + +class PythonRunner(object): + """ + Execute a new-style Ansible module, where Module Replacer-related tricks + aren't required. + """ + def setup(self): + super(PythonRunner, self).setup() + self._overrides = MethodOverrides() + self._args = ModuleArguments(self.args) + + def revert(self): + super(PythonRunner, self).revert() + self._args.revert() + self._overrides.revert() + + def _run(self): + try: + mod = __import__(self.module, {}, {}, ['']) + # Ansible modules begin execution on import. Thus the above + # __import__ will cause either Exit or ModuleError to be raised. If + # we reach the line below, the module did not execute and must + # already have been imported for a previous invocation, so we need + # to invoke main explicitly. + mod.main() + except (Exit, ModuleError), e: + return json.dumps(e.dct) + + assert False, "Module returned no result." + + +class BinaryRunner(object): + def setup(self): + super(BinaryRunner, self).setup() + self._setup_binary() + self._setup_args() + + def _get_binary(self): + """ + Fetch the module binary from the master if necessary. + """ + return ansible_mitogen.helpers.get_file( + path=self.runner_params['path'], + ) + + def _get_args(self): + """ + Return the module arguments formatted as JSON. + """ + return json.dumps(self.args) + + def _setup_program(self): + """ + Create a temporary file containing the program code. The code is + fetched via :meth:`_get_binary`. + """ + self.bin_fp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen', + suffix='-binary', + ) + self.bin_fp.write(self._get_binary()) + self.bin_fp.flush() + os.chmod(self.fp.name, int('0700', 8)) + + def _setup_args(self): + """ + Create a temporary file containing the module's arguments. The + arguments are formatted via :meth:`_get_args`. + """ + self.args_fp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen', + suffix='-args', + ) + self.args_fp.write(self._get_args()) + self.args_fp.flush() + + def revert(self): + """ + Delete the temporary binary and argument files. + """ + self.args_fp.close() + self.bin_fp.close() + super(BinaryRunner, self).revert() + + def _run(self): + rc, stdout, stderr = ansible_mitogen.helpers.exec_args( + args=[self.bin_fp.name, self.args_fp.name], + ) + # ... + assert 0 + + +class WantJsonRunner(BinaryRunner): + def _get_binary(self): + s = super(WantJsonRunner, self)._get_binary() + # fix up shebang. + return s + + +class OldStyleRunner(BinaryRunner): + def _get_args(self): + """ + Mimic the argument formatting behaviour of + ActionBase._execute_module(). + """ + return ' '.join( + '%s=%s' % (key, shlex_quote(str(self.args[key]))) + for key in self.args + ) From 3dc90b76184137f278cdf5ba3322624be7dc47e2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 16:06:23 +0545 Subject: [PATCH 041/140] issue #106: import skeletal planner module. --- ansible_mitogen/planner.py | 186 +++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 ansible_mitogen/planner.py diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py new file mode 100644 index 00000000..6c9f8441 --- /dev/null +++ b/ansible_mitogen/planner.py @@ -0,0 +1,186 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" +This exists to detect every case defined in [0] and prepare arguments necessary +for the executor implementation running within the target, including preloading +any requisite files/Python modules known to be missing. + +[0] "Ansible Module Architecture", developing_program_flow_modules.html +""" + +from __future__ import absolute_import +from ansible.executor import module_common + +import mitogen +import mitogen.service +import ansible_mitogen.helpers + + +class Planner(object): + """ + A Planner receives a module name and the contents of its implementation + file, indicates whether or not it understands how to run the module, and + exports a method to run the module. + """ + def detect(self, name, source): + assert 0 + + def run(self, connection, name, source, args, env): + assert 0 + + +class JsonArgsPlanner(Planner): + """ + Script that has its interpreter directive and the task arguments + substituted into its source as a JSON string. + """ + def detect(self, name, source): + return module_common.REPLACER_JSONARGS in source + + def run(self, name, source, args, env): + path = None # TODO + mitogen.service.call(501, ('register', path)) + return { + 'func': 'run_json_args_module', + 'binary': source, + 'args': args, + 'env': env, + } + + +class WantJsonPlanner(Planner): + """ + If a module has the string WANT_JSON in it anywhere, Ansible treats it as a + non-native module that accepts a filename as its only command line + parameter. The filename is for a temporary file containing a JSON string + containing the module’s parameters. The module needs to open the file, read + and parse the parameters, operate on the data, and print its return data as + a JSON encoded dictionary to stdout before exiting. + + These types of modules are self-contained entities. As of Ansible 2.1, + Ansible only modifies them to change a shebang line if present. + """ + def detect(self, name, source): + return 'WANT_JSON' in source + + def run(self, name, source, args, env): + return { + 'func': 'run_want_json_module', + 'binary': source, + 'args': args, + 'env': env, + } + + +class ReplacerPlanner(Planner): + """ + The Module Replacer framework is the original framework implementing + new-style modules. It is essentially a preprocessor (like the C + Preprocessor for those familiar with that programming language). It does + straight substitutions of specific substring patterns in the module file. + There are two types of substitutions. + + * Replacements that only happen in the module file. These are public + replacement strings that modules can utilize to get helpful boilerplate + or access to arguments. + + "from ansible.module_utils.MOD_LIB_NAME import *" is replaced with the + contents of the ansible/module_utils/MOD_LIB_NAME.py. These should only + be used with new-style Python modules. + + "#<>" is equivalent to + "from ansible.module_utils.basic import *" and should also only apply to + new-style Python modules. + + "# POWERSHELL_COMMON" substitutes the contents of + "ansible/module_utils/powershell.ps1". It should only be used with + new-style Powershell modules. + """ + def detect(self, name, source): + return module_common.REPLACER in source + + def run(self, name, source, args, env): + return { + 'func': 'run_replacer_module', + 'binary': source, + 'args': args, + 'env': env, + } + + +class BinaryPlanner(Planner): + """ + Binary modules take their arguments and will return data to Ansible in the + same way as want JSON modules. + """ + helper = staticmethod(ansible_mitogen.helpers.run_binary) + + def detect(self, name, source): + return module_common._is_binary(source) + + def run(self, name, source, args, env): + return { + 'func': 'run_binary_module', + 'binary': source, + 'args': args, + 'env': env, + } + + +class PythonPlanner(Planner): + """ + The Ansiballz framework differs from module replacer in that it uses real + Python imports of things in ansible/module_utils instead of merely + preprocessing the module. + """ + helper = staticmethod(ansible_mitogen.helpers.run_module) + + def detect(self, name, source): + return True + + def run(self, name, source, args, env): + return { + 'func': 'run_python_module', + 'module': name, + 'args': args, + 'env': env + } + + +_planners = [ + # JsonArgsPlanner, + # WantJsonPlanner, + # ReplacerPlanner, + BinaryPlanner, + PythonPlanner, +] + + +def plan(): + pass From 841c2b13a14662e6b5c05983ef385c82d6addb24 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 01:17:29 +0100 Subject: [PATCH 042/140] fakessh_test: Apply timeout decorators to rsync tests timeoutcontext.timeout uses SIGALRM, hence it will only work on Unix like operating systems. --- dev_requirements.txt | 1 + tests/fakessh_test.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/dev_requirements.txt b/dev_requirements.txt index 7761ff4a..3d1b0624 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -9,6 +9,7 @@ pytest-catchlog==1.2.2 pytest==3.1.2 PyYAML==3.11; python_version < '2.7' PyYAML==3.12; python_version >= '2.7' +timeoutcontext==1.2.0 unittest2==1.1.0 # Fix InsecurePlatformWarning while creating py26 tox environment # https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings diff --git a/tests/fakessh_test.py b/tests/fakessh_test.py index 6e28be1d..8711b46b 100644 --- a/tests/fakessh_test.py +++ b/tests/fakessh_test.py @@ -2,6 +2,7 @@ import os import shutil +import timeoutcontext import unittest2 import mitogen.fakessh @@ -10,6 +11,7 @@ import testlib class RsyncTest(testlib.DockerMixin, unittest2.TestCase): + @timeoutcontext.timeout(5) def test_rsync_from_master(self): context = self.docker_ssh_any() @@ -25,6 +27,7 @@ class RsyncTest(testlib.DockerMixin, unittest2.TestCase): self.assertTrue(context.call(os.path.exists, '/tmp/data')) self.assertTrue(context.call(os.path.exists, '/tmp/data/simple_pkg/a.py')) + @timeoutcontext.timeout(5) def test_rsync_between_direct_children(self): # master -> SSH -> has-sudo-pubkey -> rsync(.ssh) -> master -> # has-sudo -> rsync From 0dcaeb21a2185ac23303d1902d0877d94f19c091 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 01:21:42 +0100 Subject: [PATCH 043/140] master_test: Don't assume __file__ points to source code When run under a test runner the unit tests are imported as modules. This triggers .pyc generation, after which __file__ resolves to the .pyc file. --- tests/master_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/master_test.py b/tests/master_test.py index 796c7084..cf16d6c5 100644 --- a/tests/master_test.py +++ b/tests/master_test.py @@ -1,3 +1,4 @@ +import inspect import unittest2 @@ -9,8 +10,10 @@ class ScanCodeImportsTest(unittest2.TestCase): func = staticmethod(mitogen.master.scan_code_imports) def test_simple(self): - co = compile(open(__file__).read(), __file__, 'exec') + source_path = inspect.getsourcefile(ScanCodeImportsTest) + co = compile(open(source_path).read(), source_path, 'exec') self.assertEquals(list(self.func(co)), [ + (-1, 'inspect', ()), (-1, 'unittest2', ()), (-1, 'testlib', ()), (-1, 'mitogen.master', ()), From a22294dda9e6241ea9005395845628b8bdb12b48 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 01:23:13 +0100 Subject: [PATCH 044/140] call_function_test: Fix assumption that we run as a script --- tests/call_function_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/call_function_test.py b/tests/call_function_test.py index d66864b4..de3f1f46 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -64,7 +64,10 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): def test_bad_return_value(self): exc = self.assertRaises(mitogen.core.StreamError, lambda: self.local.call(func_with_bad_return_value)) - self.assertEquals(exc[0], "cannot unpickle '__main__'/'CrazyType'") + self.assertEquals( + exc[0], + "cannot unpickle '%s'/'CrazyType'" % (__name__,), + ) def test_returns_dead(self): self.assertEqual(mitogen.core._DEAD, self.local.call(func_returns_dead)) From 7b8fef5284c4d7c44c295cf18f83e9d7372a9d66 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 01:32:13 +0100 Subject: [PATCH 045/140] tests: Make the tests directory an importable package Required for test discovery by e.g. unit2, pytest --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b From dc60f05a4057c870ab88ceb362e8e1c0d73048e2 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 01:42:31 +0100 Subject: [PATCH 046/140] tests: Switch to unit2 test runner, with coverage This means test files are imported as modules, not run as scripts. THey can still be run individually if so desired. Test coverage is measured, and an html report generated in htmlcov/. Test cases are automativally discovered, so they need not be listed twice. An overall passed/failed/skipped summary is printed, rather than for each file. Arguments passed to ./test are passed on to unit2. For instance ./test -v will print each test name as it is run. --- .gitignore | 2 ++ .travis.yml | 2 +- setup.cfg | 7 ++++++ test | 11 +++++++++ test.sh | 63 ------------------------------------------------- tests/README.md | 2 +- tox.ini | 2 +- 7 files changed, 23 insertions(+), 66 deletions(-) create mode 100755 test delete mode 100755 test.sh diff --git a/.gitignore b/.gitignore index 9a539564..9ec8ce9b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ +.coverage .venv **/.DS_Store MANIFEST build/ dist/ docs/_build +htmlcov/ *.egg-info diff --git a/.travis.yml b/.travis.yml index a7f2f637..8deda0c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - pip install -r dev_requirements.txt script: -- MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/test.sh +- MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/test services: - docker diff --git a/setup.cfg b/setup.cfg index 44668df3..92051682 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,10 @@ +[coverage:run] +branch = true +source = + mitogen +omit = + mitogen/compat/* + [flake8] ignore = E402,E128,W503 exclude = mitogen/compat diff --git a/test b/test new file mode 100755 index 00000000..aeb9c51c --- /dev/null +++ b/test @@ -0,0 +1,11 @@ +#/bin/sh + +UNIT2="$(which unit2)" + +coverage erase +coverage run "${UNIT2}" discover \ + --start-directory "tests" \ + --pattern '*_test.py' \ + "$@" +coverage html +echo coverage report is at "file://$(pwd)/htmlcov/index.html" diff --git a/test.sh b/test.sh deleted file mode 100755 index ab545104..00000000 --- a/test.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -timeout() -{ - python -c ' -import subprocess -import sys -import time - -deadline = time.time() + float(sys.argv[1]) -proc = subprocess.Popen(sys.argv[2:]) -while time.time() < deadline and proc.poll() is None: - time.sleep(1.0) - -if proc.poll() is not None: - sys.exit(proc.returncode) -proc.terminate() -print -print >> sys.stderr, "Timeout! Command was:", sys.argv[2:] -print -sys.exit(1) - ' "$@" -} - -trap 'sigint' INT -sigint() -{ - echo "SIGINT received, stopping.." - exit 1 -} - -run_test() -{ - echo "Running $1.." - timeout 10 python $1 || fail=$? -} - -run_test tests/ansible_helpers_test.py -run_test tests/call_error_test.py -run_test tests/call_function_test.py -run_test tests/channel_test.py -run_test tests/fakessh_test.py -run_test tests/first_stage_test.py -run_test tests/fork_test.py -run_test tests/id_allocation_test.py -run_test tests/importer_test.py -run_test tests/latch_test.py -run_test tests/local_test.py -run_test tests/master_test.py -run_test tests/module_finder_test.py -run_test tests/nested_test.py -run_test tests/parent_test.py -run_test tests/receiver_test.py -run_test tests/responder_test.py -run_test tests/router_test.py -run_test tests/select_test.py -run_test tests/ssh_test.py -run_test tests/utils_test.py - -if [ "$fail" ]; then - echo "AT LEAST ONE TEST FAILED" >&2 - exit 1 -fi diff --git a/tests/README.md b/tests/README.md index 0ac4bcb1..41c024b5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,4 +27,4 @@ and run the tests there. 1. Build the virtual environment ``virtualenv ../venv`` 1. Enable the virtual environment we just built ``source ../venv/bin/activate`` 1. Install Mitogen in pip editable mode ``pip install -e .`` -1. Run ``test.sh`` +1. Run ``test`` diff --git a/tox.ini b/tox.ini index f9eabeed..70de05df 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ deps = -r{toxinidir}/dev_requirements.txt commands = - {posargs:./test.sh} + {posargs:./test} [testenv:docs] basepython = python From 5e66f6c4fa52fe3bda0fceeb5d6ad6d7b86710c5 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 01:44:29 +0100 Subject: [PATCH 047/140] Ignore hidden directory containing tox environments --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9ec8ce9b..0a843e9b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .coverage +.tox .venv **/.DS_Store MANIFEST From b5848e71162581f85016382318671ad2dffff2e3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 01:45:22 +0100 Subject: [PATCH 048/140] Ignore compiled Python files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 0a843e9b..09a0cb67 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,13 @@ .tox .venv **/.DS_Store +*.pyc +*.pyd +*.pyo MANIFEST build/ dist/ docs/_build htmlcov/ *.egg-info +__pycache__/ From 94a082177d3edad7d27be61db635d5b2e9690144 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 02:04:41 +0100 Subject: [PATCH 049/140] tests: Add coverage as a dev requirement --- dev_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev_requirements.txt b/dev_requirements.txt index 3d1b0624..59bf43de 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,6 @@ -r docs/docs-requirements.txt ansible==2.3.1.0 +coverage==4.5.1 Django==1.6.11; python_version < '2.7' Django==1.11.5; python_version >= '2.7' # for module_finder_test https://github.com/docker/docker-py/archive/1.10.6.tar.gz; python_version < '2.7' From f92d88c24182142396646c464ab820d8fae14418 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 02:05:36 +0100 Subject: [PATCH 050/140] travis: Cache wheels and other pip artifacts Should speedup builds --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8deda0c0..815192a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ notifications: email: false language: python +cache: pip python: - "2.7" From 0896e95e2c6e065b36a42c5b05fae5686e1b4b70 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 1 Apr 2018 02:11:37 +0100 Subject: [PATCH 051/140] Set strict mode in test script Exits with an error if a command is not found, any undefined variable is used, or a command in a pipeline returns an error. Should make Travis detect failed tests. --- test | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test b/test index aeb9c51c..71ce18aa 100755 --- a/test +++ b/test @@ -1,4 +1,7 @@ #/bin/sh +set -o errexit +set -o nounset +set -o pipefail UNIT2="$(which unit2)" From 6eed3aa1fa20b072cf148a168fbe6a825b57d6eb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 15:54:17 +0100 Subject: [PATCH 052/140] issue #177: fetch and cache HOME value during connection setup. This ensures only 1 roundtrip is required for every invocation of _remote_expand_user(). --- ansible_mitogen/connection.py | 20 +++++++++++++++----- ansible_mitogen/mixins.py | 15 ++++++++++++--- ansible_mitogen/services.py | 13 +++++++++++-- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 1c404410..a81a4bd6 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -82,6 +82,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: Set to 'mitogen_ssh_discriminator' by on_action_run() mitogen_ssh_discriminator = None + #: Set after connection to the target context's home directory. + _homedir = None + def __init__(self, play_context, new_stdin, original_transport, **kwargs): assert ansible_mitogen.process.MuxProcess.unix_listener_path, ( 'The "mitogen" connection plug-in may only be instantiated ' @@ -125,6 +128,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): 'sudo' ) + @property + def homedir(self): + self._connect() + return self._homedir + @property def connected(self): return self.broker is not None @@ -232,18 +240,20 @@ class Connection(ansible.plugins.connection.ConnectionBase): if self.original_transport == 'local': if self._play_context.become: - self.context = self._connect_sudo(python_path=sys.executable) + self.context, self._homedir = self._connect_sudo( + python_path=sys.executable + ) else: - self.context = self._connect_local() + self.context, self._homedir = self._connect_local() return if self.original_transport == 'docker': - self.host = self._connect_docker() + self.host, self._homedir = self._connect_docker() elif self.original_transport == 'ssh': - self.host = self._connect_ssh() + self.host, self._homedir = self._connect_ssh() if self._play_context.become: - self.context = self._connect_sudo(via=self.host) + self.context, self._homedir = self._connect_sudo(via=self.host) else: self.context = self.host diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index ad020f67..70f63ec6 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -272,10 +272,19 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): Replace the base implementation's attempt to emulate os.path.expanduser() with an actual call to os.path.expanduser(). """ - LOG.debug('_remove_expand_user(%r, sudoable=%r)', path, sudoable) + LOG.debug('_remote_expand_user(%r, sudoable=%r)', path, sudoable) + if not path.startswith('~'): + # /home/foo -> /home/foo + return path + if path == '~': + # ~ -> /home/dmw + return self._connection.homedir + if path.startswith('~/'): + # ~/.ansible -> /home/dmw/.ansible + return os.path.join(self._connection.homedir, path[2:]) if path.startswith('~'): - path = self.call(os.path.expanduser, path) - return path + # ~root/.ansible -> /root/.ansible + return self.call(os.path.expanduser, path) def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 98a231d4..0266d533 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -28,6 +28,7 @@ from __future__ import absolute_import import logging +import os.path import zlib import mitogen @@ -62,7 +63,13 @@ class ContextService(mitogen.service.DeduplicatingService): existing connection, but popped from the list of arguments passed to the connection method. - :returns mitogen.master.Context: + :returns tuple: + Tuple of `(context, home_dir)`, where: + * `context` is the mitogen.master.Context referring to the target + context. + * `home_dir` is a cached copy of the remote directory. + + mitogen.master.Context: Corresponding Context instance. """ handle = 500 @@ -74,7 +81,9 @@ class ContextService(mitogen.service.DeduplicatingService): def get_response(self, args): args.pop('discriminator', None) method = getattr(self.router, args.pop('method')) - return method(**args) + context = method(**args) + home_dir = context.call(os.path.expanduser, '~') + return context, home_dir class FileService(mitogen.service.Service): From cf25437019abb688861ebb89d9b7354011bb1790 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 15:58:44 +0100 Subject: [PATCH 053/140] issue #177: populate Shell.tempdir global on creating a tempdir. It looks a lot like multiple calls to _make_tmp_path() will result in multiple temporary directories on the remote machine, only the last of which will be cleaned up. We must be bug-for-bug compatible for now, so ignore the problem in the meantime. --- ansible_mitogen/helpers.py | 16 ++++++++++++++++ ansible_mitogen/mixins.py | 20 +++++++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 1b686d7b..50dd3cec 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -34,6 +34,7 @@ import random import re import stat import subprocess +import tempfile import threading import mitogen.core @@ -173,6 +174,21 @@ def _async_main(job_id, module, raw_params, args, env): _result_by_job_id[job_id] = rc +def make_temp_directory:(base_dir): + """ + Handle creation of `base_dir` if it is absent, in addition to a unique + temporary directory within `base_dir`. + + :returns: + Newly created temporary directory. + """ + if not os.path.exists(base_dir): + os.makedirs(base_dir, mode=int('0700', 8)) + return tempfile.mkdtemp( + dir=base_dir, + prefix='ansible-mitogen-tmp-', + ) + def run_module_async(module, raw_params=None, args=None): """ Arrange for an Ansible module to be executed in a thread of the current diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 70f63ec6..151358f9 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -198,9 +198,20 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): with an actual call to mkdtemp(). """ LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) - path = self.call(tempfile.mkdtemp, prefix='ansible-mitogen-tmp-') + + # _make_tmp_path() is basically a global stashed away as Shell.tmpdir. + # The copy action plugin violates layering and grabs this attribute + # directly. + self._connection._shell.tmpdir = self.call( + ansible_mitogen.helpers.make_temp_directory:, + base_dir=self._remote_expand_user( + # ~/.ansible + self._connection._shell.get_option('remote_tmp') + ) + ) + LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) self._cleanup_remote_tmp = True - return path + return self._connection._shell.tmpdir def _remove_tmp_path(self, tmp_path): """ @@ -208,8 +219,11 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): shutil.rmtree(). """ LOG.debug('_remove_tmp_path(%r)', tmp_path) + if tmp_path is None: + tmp_path = self._connection._shell.tmpdir if self._should_remove_tmp_path(tmp_path): - return self.call(shutil.rmtree, tmp_path) + self.call(shutil.rmtree, tmp_path) + self._connection._shell.tmpdir = None def _transfer_data(self, remote_path, data): """ From 17b94c56f40cdad69c0c498467613cd069392e5d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 15:59:58 +0100 Subject: [PATCH 054/140] issue #177: import reproduction. --- examples/playbook/issue_177.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 examples/playbook/issue_177.yml diff --git a/examples/playbook/issue_177.yml b/examples/playbook/issue_177.yml new file mode 100644 index 00000000..ec7e3338 --- /dev/null +++ b/examples/playbook/issue_177.yml @@ -0,0 +1,11 @@ + + +- hosts: all + gather_facts: false + tasks: + - name: copy repo configs + copy: src=/etc/{{ item }} dest=/tmp/{{item}} mode=0644 + with_items: + - passwd + - hosts + From 98c15942f7b1602ef2f38a57b2d5ef5aef0eefe2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 16:38:22 +0100 Subject: [PATCH 055/140] issue #177: fix bizarre syntax error in last commit. --- ansible_mitogen/helpers.py | 2 +- ansible_mitogen/mixins.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 50dd3cec..9e8693d9 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -174,7 +174,7 @@ def _async_main(job_id, module, raw_params, args, env): _result_by_job_id[job_id] = rc -def make_temp_directory:(base_dir): +def make_temp_directory(base_dir): """ Handle creation of `base_dir` if it is absent, in addition to a unique temporary directory within `base_dir`. diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 151358f9..9cef0c3e 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -203,7 +203,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): # The copy action plugin violates layering and grabs this attribute # directly. self._connection._shell.tmpdir = self.call( - ansible_mitogen.helpers.make_temp_directory:, + ansible_mitogen.helpers.make_temp_directory, base_dir=self._remote_expand_user( # ~/.ansible self._connection._shell.get_option('remote_tmp') From 34a37a0ba599083963f521ddff4696273e387cac Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 30 Mar 2018 23:05:17 +0545 Subject: [PATCH 056/140] issue #106: ansible: rename and significant pad out runners.py Aiming to have working NativeRunner and BinaryRunner to begin with. --- ansible_mitogen/{executor.py => runner.py} | 147 ++++++++++++--------- 1 file changed, 83 insertions(+), 64 deletions(-) rename ansible_mitogen/{executor.py => runner.py} (71%) diff --git a/ansible_mitogen/executor.py b/ansible_mitogen/runner.py similarity index 71% rename from ansible_mitogen/executor.py rename to ansible_mitogen/runner.py index 00e91d05..559001f0 100644 --- a/ansible_mitogen/executor.py +++ b/ansible_mitogen/runner.py @@ -31,7 +31,7 @@ import json import os import tempfile -import ansible_mitogen.helpers +import ansible_mitogen.helpers # TODO: circular import try: from shlex import quote as shlex_quote @@ -43,23 +43,6 @@ import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' -class Exit(Exception): - """ - Raised when a module exits with success. - """ - def __init__(self, dct): - self.dct = dct - - -class ModuleError(Exception): - """ - Raised when a module voluntarily indicates failure via .fail_json(). - """ - def __init__(self, msg, dct): - Exception.__init__(self, msg) - self.dct = dct - - class TemporaryEnvironment(object): def __init__(self, env=None): self.original = os.environ.copy() @@ -71,44 +54,41 @@ class TemporaryEnvironment(object): os.environ.update(self.original) -class MethodOverrides(object): - @staticmethod - def exit_json(self, **kwargs): - """ - Replace AnsibleModule.exit_json() with something that doesn't try to - kill the process or JSON-encode the result dictionary. Instead, cause - Exit to be raised, with a `dct` attribute containing the successful - result dictionary. - """ - self.add_path_info(kwargs) - kwargs.setdefault('changed', False) +class NativeModuleExit(Exception): + """ + Capture the result of a call to `.exit_json()` or `.fail_json()` by a + native Ansible module. + """ + def __init__(self, ansible_module, **kwargs): + ansible_module.add_path_info(kwargs) kwargs.setdefault('invocation', { - 'module_args': self.params + 'module_args': ansible_module.params }) - kwargs = ansible.module_utils.basic.remove_values( + ansible_module.do_cleanup_files() + self.dct = ansible.module_utils.basic.remove_values( kwargs, self.no_log_values, ) - self.do_cleanup_files() - raise Exit(kwargs) + + +class NativeMethodOverrides(object): + @staticmethod + def exit_json(self, **kwargs): + """ + Raise exit_json() output as the `.dct` attribute of a + :class:`NativeModuleExit` exception`. + """ + kwargs.setdefault('changed', False) + return NativeModuleExit(self, **kwargs) @staticmethod def fail_json(self, **kwargs): """ - Replace AnsibleModule.fail_json() with something that raises - ModuleError, which includes a `dct` attribute. + Raise fail_json() output as the `.dct` attribute of a + :class:`NativeModuleExit` exception`. """ - self.add_path_info(kwargs) kwargs.setdefault('failed', True) - kwargs.setdefault('invocation', { - 'module_args': self.params - }) - kwargs = ansible.module_utils.basic.remove_values( - kwargs, - self.no_log_values, - ) - self.do_cleanup_files() - raise ModuleError(kwargs.get('msg'), kwargs) + return NativeModuleExit(self, **kwargs) klass = ansible.module_utils.basic.AnsibleModule @@ -119,14 +99,16 @@ class MethodOverrides(object): self.klass.fail_json = self.fail_json def revert(self): + """ + Restore prior state. + """ self.klass.exit_json = self._original_exit_json self.klass.fail_json = self._original_fail_json -class ModuleArguments(object): +class NativeModuleArguments(object): """ - Patch the ansible.module_utils.basic global arguments variable on - construction, and revert the changes on call to :meth:`revert`. + Patch ansible.module_utils.basic argument globals. """ def __init__(self, args): self.original = ansible.module_utils.basic._ANSIBLE_ARGS @@ -135,29 +117,44 @@ class ModuleArguments(object): }) def revert(self): + """ + Restore prior state. + """ ansible.module_utils.basic._ANSIBLE_ARGS = self._original_args class Runner(object): - def __init__(self, module, raw_params=None, args=None, env=None, - runner_params=None): + """ + Ansible module runner. After instantiation (with kwargs supplied by the + corresponding Planner), `.run()` is invoked, upon which `setup()`, + `_run()`, and `revert()` are invoked, with the return value of `_run()` + returned by `run()`. + + Subclasses may override `_run`()` and extend `setup()` and `revert()`. + """ + def __init__(self, module, raw_params=None, args=None, env=None): if args is None: args = {} if raw_params is not None: args['_raw_params'] = raw_params - if runner_params is None: - runner_params = {} self.module = module self.raw_params = raw_params self.args = args self.env = env - self.runner_params = runner_params def setup(self): + """ + Prepare the current process for running a module. The base + implementation simply prepares the environment. + """ self._env = TemporaryEnvironment(self.env) def revert(self): + """ + Revert any changes made to the process after running a module. The base + implementation simply restores the original environment. + """ self._env.revert() def _run(self): @@ -169,6 +166,9 @@ class Runner(object): module. This monkey-patches the Ansible libraries in various places to prevent it from trying to kill the process on completion, and to prevent it from reading sys.stdin. + + :returns: + Module result dictionary. """ self.setup() try: @@ -177,37 +177,53 @@ class Runner(object): self.revert() -class PythonRunner(object): +class NativeRunner(object): """ Execute a new-style Ansible module, where Module Replacer-related tricks aren't required. """ def setup(self): - super(PythonRunner, self).setup() - self._overrides = MethodOverrides() - self._args = ModuleArguments(self.args) + super(NativeRunner, self).setup() + self._overrides = NativeMethodOverrides() + self._args = NativeModuleArguments(self.args) def revert(self): - super(PythonRunner, self).revert() + super(NativeRunner, self).revert() self._args.revert() self._overrides.revert() + def module_fixups(mod): + """ + Apply fixups for known problems with mainline Ansible modules. + """ + if mod.__name__ == 'ansible.modules.packaging.os.yum_repository': + # https://github.com/dw/mitogen/issues/154 + mod.YumRepo.repofile = mod.configparser.RawConfigParser() + def _run(self): try: mod = __import__(self.module, {}, {}, ['']) + self.module_fixups(mod) # Ansible modules begin execution on import. Thus the above # __import__ will cause either Exit or ModuleError to be raised. If # we reach the line below, the module did not execute and must # already have been imported for a previous invocation, so we need # to invoke main explicitly. mod.main() - except (Exit, ModuleError), e: - return json.dumps(e.dct) + except NativeModuleExit, e: + return e.dct - assert False, "Module returned no result." + return { + 'failed': True, + 'msg': 'ansible_mitogen: module did not exit normally.' + } class BinaryRunner(object): + def __init__(self, path, **kwargs): + super(BinaryRunner, self).__init__(**kwargs) + self.path = path + def setup(self): super(BinaryRunner, self).setup() self._setup_binary() @@ -261,9 +277,12 @@ class BinaryRunner(object): super(BinaryRunner, self).revert() def _run(self): - rc, stdout, stderr = ansible_mitogen.helpers.exec_args( - args=[self.bin_fp.name, self.args_fp.name], - ) + try: + rc, stdout, stderr = ansible_mitogen.helpers.exec_args( + args=[self.bin_fp.name, self.args_fp.name], + ) + except Exception, e: + return # ... assert 0 From c891ab078abeab844011066d49fa1ceb535ce2b9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 31 Mar 2018 10:22:11 +0545 Subject: [PATCH 057/140] issue #106: working old-style native module execution Still abusing Python import mechanism, but one big step closer to eliminating that. --- ansible_mitogen/helpers.py | 152 +++++++++--------------------------- ansible_mitogen/mixins.py | 52 ++++++------- ansible_mitogen/planner.py | 156 ++++++++++++++++++++++++++++--------- ansible_mitogen/runner.py | 155 +++++++++++++++++++----------------- 4 files changed, 262 insertions(+), 253 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 9e8693d9..9f2e3c2b 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import json import operator import os @@ -38,10 +39,7 @@ import tempfile import threading import mitogen.core - -# Prevent accidental import of an Ansible module from hanging on stdin read. -import ansible.module_utils.basic -ansible.module_utils.basic._ANSIBLE_ARGS = '{}' +import ansible_mitogen.runner #: Mapping of job_id<->result dict _result_by_job_id = {} @@ -50,124 +48,26 @@ _result_by_job_id = {} _thread_by_job_id = {} -class Exit(Exception): - """ - Raised when a module exits with success. - """ - def __init__(self, dct): - self.dct = dct - - -class ModuleError(Exception): - """ - Raised when a module voluntarily indicates failure via .fail_json(). - """ - def __init__(self, msg, dct): - Exception.__init__(self, msg) - self.dct = dct - - -def monkey_exit_json(self, **kwargs): - """ - Replace AnsibleModule.exit_json() with something that doesn't try to kill - the process or JSON-encode the result dictionary. Instead, cause Exit to be - raised, with a `dct` attribute containing the successful result dictionary. - """ - self.add_path_info(kwargs) - kwargs.setdefault('changed', False) - kwargs.setdefault('invocation', { - 'module_args': self.params - }) - kwargs = ansible.module_utils.basic.remove_values( - kwargs, - self.no_log_values - ) - self.do_cleanup_files() - raise Exit(kwargs) - - -def monkey_fail_json(self, **kwargs): - """ - Replace AnsibleModule.fail_json() with something that raises ModuleError, - which includes a `dct` attribute. - """ - self.add_path_info(kwargs) - kwargs.setdefault('failed', True) - kwargs.setdefault('invocation', { - 'module_args': self.params - }) - kwargs = ansible.module_utils.basic.remove_values( - kwargs, - self.no_log_values - ) - self.do_cleanup_files() - raise ModuleError(kwargs.get('msg'), kwargs) - - -def module_fixups(mod): - """ - Apply fixups for known problems with mainline Ansible modules. - """ - if mod.__name__ == 'ansible.modules.packaging.os.yum_repository': - # https://github.com/dw/mitogen/issues/154 - mod.YumRepo.repofile = mod.configparser.RawConfigParser() - - -class TemporaryEnvironment(object): - def __init__(self, env=None): - self.original = os.environ.copy() - self.env = env or {} - os.environ.update((k, str(v)) for k, v in self.env.iteritems()) - - def revert(self): - os.environ.clear() - os.environ.update(self.original) - - -def run_module(module, raw_params=None, args=None, env=None): +def run_module(kwargs): """ Set up the process environment in preparation for running an Ansible module. This monkey-patches the Ansible libraries in various places to prevent it from trying to kill the process on completion, and to prevent it from reading sys.stdin. """ - if args is None: - args = {} - if raw_params is not None: - args['_raw_params'] = raw_params - - ansible.module_utils.basic.AnsibleModule.exit_json = monkey_exit_json - ansible.module_utils.basic.AnsibleModule.fail_json = monkey_fail_json - ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({ - 'ANSIBLE_MODULE_ARGS': args - }) + runner_name = kwargs.pop('runner_name') + klass = getattr(ansible_mitogen.runner, runner_name) + impl = klass(**kwargs) + return json.dumps(impl.run()) - temp_env = TemporaryEnvironment(env) - try: - try: - mod = __import__(module, {}, {}, ['']) - module_fixups(mod) - # Ansible modules begin execution on import. Thus the above __import__ - # will cause either Exit or ModuleError to be raised. If we reach the - # line below, the module did not execute and must already have been - # imported for a previous invocation, so we need to invoke main - # explicitly. - mod.main() - except (Exit, ModuleError), e: - result = json.dumps(e.dct) - finally: - temp_env.revert() - - return result - - -def _async_main(job_id, module, raw_params, args, env): + +def _async_main(job_id, runner_name, kwargs): """ Implementation for the thread that implements asynchronous module execution. """ try: - rc = run_module(module, raw_params, args, env) + rc = run_module(runner_name, kwargs) except Exception, e: rc = mitogen.core.CallError(e) @@ -189,7 +89,7 @@ def make_temp_directory(base_dir): prefix='ansible-mitogen-tmp-', ) -def run_module_async(module, raw_params=None, args=None): +def run_module_async(runner_name, kwargs): """ Arrange for an Ansible module to be executed in a thread of the current process, with results available via :py:func:`get_async_result`. @@ -200,9 +100,8 @@ def run_module_async(module, raw_params=None, args=None): target=_async_main, kwargs={ 'job_id': job_id, - 'module': module, - 'raw_params': raw_params, - 'args': args, + 'runner_name': runner_name, + 'kwargs': kwargs, } ) _thread_by_job_id[job_id].start() @@ -241,7 +140,7 @@ def get_user_shell(): return pw_shell or '/bin/sh' -def exec_command(cmd, in_data='', chdir=None, shell=None): +def exec_args(args, in_data='', chdir=None, shell=None): """ Run a command in a subprocess, emulating the argument handling behaviour of SSH. @@ -256,7 +155,7 @@ def exec_command(cmd, in_data='', chdir=None, shell=None): assert isinstance(cmd, basestring) proc = subprocess.Popen( - args=[get_user_shell(), '-c', cmd], + args=args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, @@ -266,6 +165,27 @@ def exec_command(cmd, in_data='', chdir=None, shell=None): return proc.returncode, stdout, stderr +def exec_command(cmd, in_data='', chdir=None, shell=None): + """ + Run a command in a subprocess, emulating the argument handling behaviour of + SSH. + + :param bytes cmd: + String command line, passed to user's shell. + :param bytes in_data: + Optional standard input for the command. + :return: + (return code, stdout bytes, stderr bytes) + """ + assert isinstance(cmd, basestring) + return _exec_command( + args=[get_user_shell(), '-c', cmd], + in_data=in_Data, + chdir=chdir, + shell=shell, + ) + + def read_path(path): """ Fetch the contents of a filesystem `path` as bytes. diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 9cef0c3e..d003ce2e 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -52,6 +52,7 @@ import mitogen.master from mitogen.utils import cast import ansible_mitogen.connection +import ansible_mitogen.planner import ansible_mitogen.helpers from ansible.module_utils._text import to_text @@ -59,22 +60,6 @@ from ansible.module_utils._text import to_text LOG = logging.getLogger(__name__) -def get_command_module_name(module_name): - """ - Given the name of an Ansible command module, return its canonical module - path within the ansible. - - :param module_name: - "shell" - :return: - "ansible.modules.commands.shell" - """ - path = module_loader.find_plugin(module_name, '') - relpath = os.path.relpath(path, os.path.dirname(ansible.__file__)) - root, _ = os.path.splitext(relpath) - return 'ansible.' + root.replace('/', '.') - - class ActionModuleMixin(ansible.plugins.action.ActionBase): """ The Mitogen-patched PluginLoader dynamically mixes this into every action @@ -308,29 +293,36 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): helpers.run_module() or helpers.run_module_async() in the target context. """ - if task_vars is None: - task_vars = {} if module_name is None: module_name = self._task.action if module_args is None: module_args = self._task.args + if task_vars is None: + task_vars = {} self._update_module_args(module_name, module_args, task_vars) - if wrap_async: - helper = ansible_mitogen.helpers.run_module_async - else: - helper = ansible_mitogen.helpers.run_module - env = {} self._compute_environment_string(env) - js = self.call( - helper, - get_command_module_name(module_name), - args=cast(module_args), - env=cast(env), + return ansible_mitogen.planner.invoke( + ansible_mitogen.planner.Invocation( + action=self, + connection=self._connection, + module_name=mitogen.utils.cast(module_name), + module_args=mitogen.utils.cast(module_args), + task_vars=task_vars, + tmp=tmp, + env=mitogen.utils.cast(env), + wrap_async=wrap_async, + ) ) + def _postprocess_response(self, js): + """ + Apply fixups mimicking ActionBase._execute_module(); this is copied + verbatim from action/__init__.py, the guts of _parse_returned_data are + garbage and should be removed or reimplemented once tests exist. + """ data = self._parse_returned_data({ 'rc': 0, 'stdout': js, @@ -351,8 +343,8 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): encoding_errors='surrogate_then_replace', chdir=None): """ - Replace the mad rat's nest of logic in the base implementation by - simply calling helpers.exec_command() in the target context. + Override the base implementation by simply calling + helpers.exec_command() in the target context. """ LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)', cmd, type(in_data), executable, chdir) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 6c9f8441..0fb99569 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -27,32 +27,69 @@ # POSSIBILITY OF SUCH DAMAGE. """ -This exists to detect every case defined in [0] and prepare arguments necessary -for the executor implementation running within the target, including preloading -any requisite files/Python modules known to be missing. +Classes to detect each case from [0] and prepare arguments necessary for the +corresponding Runner class within the target, including preloading requisite +files/modules known missing. [0] "Ansible Module Architecture", developing_program_flow_modules.html """ from __future__ import absolute_import +import logging +import os + from ansible.executor import module_common +import ansible.errors + +try: + from ansible.plugins.loader import module_loader +except ImportError: # Ansible <2.4 + from ansible.plugins import module_loader import mitogen import mitogen.service import ansible_mitogen.helpers +LOG = logging.getLogger(__name__) + + +class Invocation(object): + """ + Collect up a module's execution environment then use it to invoke + helpers.run_module() or helpers.run_module_async() in the target context. + """ + def __init__(self, action, connection, module_name, module_args, + task_vars, tmp, env, wrap_async): + #: Instance of the ActionBase subclass invoking the module. Required to + #: access some output postprocessing methods that don't belong in + #: ActionBase at all. + self.action = action + self.connection = connection + self.module_name = module_name + self.module_args = module_args + self.module_path = None + self.module_source = None + self.task_vars = task_vars + self.tmp = tmp + self.env = env + self.wrap_async = wrap_async + + def __repr__(self): + return 'Invocation(module_name=%s)' % (self.module_name,) + + class Planner(object): """ A Planner receives a module name and the contents of its implementation file, indicates whether or not it understands how to run the module, and exports a method to run the module. """ - def detect(self, name, source): - assert 0 + def detect(self, invocation): + raise NotImplementedError() - def run(self, connection, name, source, args, env): - assert 0 + def plan(self, invocation): + raise NotImplementedError() class JsonArgsPlanner(Planner): @@ -60,10 +97,10 @@ class JsonArgsPlanner(Planner): Script that has its interpreter directive and the task arguments substituted into its source as a JSON string. """ - def detect(self, name, source): - return module_common.REPLACER_JSONARGS in source + def detect(self, invocation): + return module_common.REPLACER_JSONARGS in invocation.module_source - def run(self, name, source, args, env): + def plan(self, invocation): path = None # TODO mitogen.service.call(501, ('register', path)) return { @@ -79,17 +116,17 @@ class WantJsonPlanner(Planner): If a module has the string WANT_JSON in it anywhere, Ansible treats it as a non-native module that accepts a filename as its only command line parameter. The filename is for a temporary file containing a JSON string - containing the module’s parameters. The module needs to open the file, read + containing the module's parameters. The module needs to open the file, read and parse the parameters, operate on the data, and print its return data as a JSON encoded dictionary to stdout before exiting. These types of modules are self-contained entities. As of Ansible 2.1, Ansible only modifies them to change a shebang line if present. """ - def detect(self, name, source): - return 'WANT_JSON' in source + def detect(self, invocation): + return 'WANT_JSON' in invocation.module_source - def run(self, name, source, args, env): + def plan(self, name, source, args, env): return { 'func': 'run_want_json_module', 'binary': source, @@ -122,10 +159,10 @@ class ReplacerPlanner(Planner): "ansible/module_utils/powershell.ps1". It should only be used with new-style Powershell modules. """ - def detect(self, name, source): - return module_common.REPLACER in source + def detect(self, invocation): + return module_common.REPLACER in invocation.module_source - def run(self, name, source, args, env): + def plan(self, name, source, args, env): return { 'func': 'run_replacer_module', 'binary': source, @@ -139,37 +176,49 @@ class BinaryPlanner(Planner): Binary modules take their arguments and will return data to Ansible in the same way as want JSON modules. """ - helper = staticmethod(ansible_mitogen.helpers.run_binary) - - def detect(self, name, source): - return module_common._is_binary(source) + def detect(self, invocation): + return module_common._is_binary(invocation.module_source) - def run(self, name, source, args, env): + def plan(self, name, source, args, env): return { - 'func': 'run_binary_module', + 'runner_name': 'BinaryRunner', 'binary': source, 'args': args, 'env': env, } -class PythonPlanner(Planner): +class NativePlanner(Planner): """ The Ansiballz framework differs from module replacer in that it uses real Python imports of things in ansible/module_utils instead of merely preprocessing the module. """ - helper = staticmethod(ansible_mitogen.helpers.run_module) - - def detect(self, name, source): + def detect(self, invocation): return True - def run(self, name, source, args, env): + def get_command_module_name(self, module_name): + """ + Given the name of an Ansible command module, return its canonical + module path within the ansible. + + :param module_name: + "shell" + :return: + "ansible.modules.commands.shell" + """ + path = module_loader.find_plugin(module_name, '') + relpath = os.path.relpath(path, os.path.dirname(ansible.__file__)) + root, _ = os.path.splitext(relpath) + return 'ansible.' + root.replace('/', '.') + + def plan(self, invocation): return { - 'func': 'run_python_module', - 'module': name, - 'args': args, - 'env': env + 'runner_name': 'NativeRunner', + 'module': invocation.module_name, + 'mod_name': self.get_command_module_name(invocation.module_name), + 'args': invocation.module_args, + 'env': invocation.env, } @@ -178,9 +227,46 @@ _planners = [ # WantJsonPlanner, # ReplacerPlanner, BinaryPlanner, - PythonPlanner, + NativePlanner, ] -def plan(): - pass +NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' +CRASHED_MSG = 'Mitogen: internal error: ' + + +def get_module_data(name): + path = module_loader.find_plugin(name, '') + with open(path, 'rb') as fp: + source = fp.read() + return path, source + + +def invoke(invocation): + """ + Find a suitable Planner that knows how to run `invocation`. + """ + (invocation.module_path, + invocation.module_source) = get_module_data(invocation.module_name) + + for klass in _planners: + planner = klass() + if planner.detect(invocation): + break + else: + raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation)) + + kwargs = planner.plan(invocation) + if invocation.wrap_async: + helper = ansible_mitogen.helpers.run_module_async + else: + helper = ansible_mitogen.helpers.run_module + + try: + js = invocation.connection.call(helper, kwargs) + except mitogen.core.CallError as e: + LOG.exception('invocation crashed: %r', invocation) + summary = str(e).splitlines()[0] + raise ansible.errors.AnsibleInternalError(CRASHED_MSG + summary) + + return invocation.action._postprocess_response(js) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 559001f0..2d92626e 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -43,6 +43,60 @@ import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' +class Runner(object): + """ + Ansible module runner. After instantiation (with kwargs supplied by the + corresponding Planner), `.run()` is invoked, upon which `setup()`, + `_run()`, and `revert()` are invoked, with the return value of `_run()` + returned by `run()`. + + Subclasses may override `_run`()` and extend `setup()` and `revert()`. + """ + def __init__(self, module, raw_params=None, args=None, env=None): + if args is None: + args = {} + if raw_params is not None: + args['_raw_params'] = raw_params + + self.module = module + self.raw_params = raw_params + self.args = args + self.env = env + + def setup(self): + """ + Prepare the current process for running a module. The base + implementation simply prepares the environment. + """ + self._env = TemporaryEnvironment(self.env) + + def revert(self): + """ + Revert any changes made to the process after running a module. The base + implementation simply restores the original environment. + """ + self._env.revert() + + def _run(self): + raise NotImplementedError() + + def run(self): + """ + Set up the process environment in preparation for running an Ansible + module. This monkey-patches the Ansible libraries in various places to + prevent it from trying to kill the process on completion, and to + prevent it from reading sys.stdin. + + :returns: + Module result dictionary. + """ + self.setup() + try: + return self._run() + finally: + self.revert() + + class TemporaryEnvironment(object): def __init__(self, env=None): self.original = os.environ.copy() @@ -67,7 +121,7 @@ class NativeModuleExit(Exception): ansible_module.do_cleanup_files() self.dct = ansible.module_utils.basic.remove_values( kwargs, - self.no_log_values, + ansible_module.no_log_values, ) @@ -79,7 +133,7 @@ class NativeMethodOverrides(object): :class:`NativeModuleExit` exception`. """ kwargs.setdefault('changed', False) - return NativeModuleExit(self, **kwargs) + raise NativeModuleExit(self, **kwargs) @staticmethod def fail_json(self, **kwargs): @@ -88,7 +142,7 @@ class NativeMethodOverrides(object): :class:`NativeModuleExit` exception`. """ kwargs.setdefault('failed', True) - return NativeModuleExit(self, **kwargs) + raise NativeModuleExit(self, **kwargs) klass = ansible.module_utils.basic.AnsibleModule @@ -120,68 +174,18 @@ class NativeModuleArguments(object): """ Restore prior state. """ - ansible.module_utils.basic._ANSIBLE_ARGS = self._original_args + ansible.module_utils.basic._ANSIBLE_ARGS = self.original -class Runner(object): - """ - Ansible module runner. After instantiation (with kwargs supplied by the - corresponding Planner), `.run()` is invoked, upon which `setup()`, - `_run()`, and `revert()` are invoked, with the return value of `_run()` - returned by `run()`. - - Subclasses may override `_run`()` and extend `setup()` and `revert()`. - """ - def __init__(self, module, raw_params=None, args=None, env=None): - if args is None: - args = {} - if raw_params is not None: - args['_raw_params'] = raw_params - - self.module = module - self.raw_params = raw_params - self.args = args - self.env = env - - def setup(self): - """ - Prepare the current process for running a module. The base - implementation simply prepares the environment. - """ - self._env = TemporaryEnvironment(self.env) - - def revert(self): - """ - Revert any changes made to the process after running a module. The base - implementation simply restores the original environment. - """ - self._env.revert() - - def _run(self): - raise NotImplementedError() - - def run(self): - """ - Set up the process environment in preparation for running an Ansible - module. This monkey-patches the Ansible libraries in various places to - prevent it from trying to kill the process on completion, and to - prevent it from reading sys.stdin. - - :returns: - Module result dictionary. - """ - self.setup() - try: - return self._run() - finally: - self.revert() - - -class NativeRunner(object): +class NativeRunner(Runner): """ Execute a new-style Ansible module, where Module Replacer-related tricks aren't required. """ + def __init__(self, mod_name, **kwargs): + super(NativeRunner, self).__init__(**kwargs) + self.mod_name = mod_name + def setup(self): super(NativeRunner, self).setup() self._overrides = NativeMethodOverrides() @@ -192,18 +196,18 @@ class NativeRunner(object): self._args.revert() self._overrides.revert() - def module_fixups(mod): - """ - Apply fixups for known problems with mainline Ansible modules. - """ - if mod.__name__ == 'ansible.modules.packaging.os.yum_repository': - # https://github.com/dw/mitogen/issues/154 - mod.YumRepo.repofile = mod.configparser.RawConfigParser() + def _fixup__default(self, mod): + pass + + def _fixup__yum_repository(self, mod): + # https://github.com/dw/mitogen/issues/154 + mod.YumRepo.repofile = mod.configparser.RawConfigParser() def _run(self): + fixup = getattr(self, '_fixup__' + self.module, self._fixup__default) try: - mod = __import__(self.module, {}, {}, ['']) - self.module_fixups(mod) + mod = __import__(self.mod_name, {}, {}, ['']) + fixup(mod) # Ansible modules begin execution on import. Thus the above # __import__ will cause either Exit or ModuleError to be raised. If # we reach the line below, the module did not execute and must @@ -219,7 +223,7 @@ class NativeRunner(object): } -class BinaryRunner(object): +class BinaryRunner(Runner): def __init__(self, path, **kwargs): super(BinaryRunner, self).__init__(**kwargs) self.path = path @@ -282,9 +286,16 @@ class BinaryRunner(object): args=[self.bin_fp.name, self.args_fp.name], ) except Exception, e: - return - # ... - assert 0 + return { + 'failed': True, + 'msg': '%s: %s' % (type(e), e), + } + + return { + 'rc': rc, + 'stdout': stdout, + 'stderr': stderr + } class WantJsonRunner(BinaryRunner): From de27fb3a28ff52ece2f574ac5ae4743c9dd88cd3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 31 Mar 2018 17:28:51 +0100 Subject: [PATCH 058/140] issue #174: test all io_op() logic. --- tests/io_op_test.py | 119 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/io_op_test.py diff --git a/tests/io_op_test.py b/tests/io_op_test.py new file mode 100644 index 00000000..8ec204b6 --- /dev/null +++ b/tests/io_op_test.py @@ -0,0 +1,119 @@ + +import errno +import select + +import mock +import unittest2 + +import testlib +import mitogen.core + + +class RestartTest(object): + func = staticmethod(mitogen.core.io_op) + exception_class = None + + def test_eintr_restarts(self): + m = mock.Mock() + m.side_effect = [ + self.exception_class(errno.EINTR), + self.exception_class(errno.EINTR), + self.exception_class(errno.EINTR), + 'yay', + ] + rc, disconnected = self.func(m, 'input') + self.assertEquals(rc, 'yay') + self.assertFalse(disconnected) + self.assertEquals(4, m.call_count) + self.assertEquals(m.mock_calls, [ + mock.call('input'), + mock.call('input'), + mock.call('input'), + mock.call('input'), + ]) + + +class SelectRestartTest(RestartTest, testlib.TestCase): + exception_class = select.error + + +class OsErrorRestartTest(RestartTest, testlib.TestCase): + exception_class = OSError + + +class DisconnectTest(object): + func = staticmethod(mitogen.core.io_op) + errno = None + exception_class = None + + def test_disconnection(self): + m = mock.Mock() + m.side_effect = self.exception_class(self.errno) + rc, disconnected = self.func(m, 'input') + self.assertEquals(rc, None) + self.assertTrue(disconnected) + self.assertEquals(1, m.call_count) + self.assertEquals(m.mock_calls, [ + mock.call('input'), + ]) + + +class SelectDisconnectEioTest(DisconnectTest, testlib.TestCase): + errno = errno.EIO + exception_class = select.error + + +class SelectDisconnectEconnresetTest(DisconnectTest, testlib.TestCase): + errno = errno.ECONNRESET + exception_class = select.error + + +class SelectDisconnectEpipeTest(DisconnectTest, testlib.TestCase): + errno = errno.EPIPE + exception_class = select.error + + +class OsErrorDisconnectEioTest(DisconnectTest, testlib.TestCase): + errno = errno.EIO + exception_class = OSError + + +class OsErrorDisconnectEconnresetTest(DisconnectTest, testlib.TestCase): + errno = errno.ECONNRESET + exception_class = OSError + + +class OsErrorDisconnectEpipeTest(DisconnectTest, testlib.TestCase): + errno = errno.EPIPE + exception_class = OSError + + +class ExceptionTest(object): + func = staticmethod(mitogen.core.io_op) + errno = None + exception_class = None + + def test_exception(self): + m = mock.Mock() + m.side_effect = self.exception_class(self.errno) + e = self.assertRaises(self.exception_class, + lambda: self.func(m, 'input')) + self.assertEquals(e, m.side_effect) + self.assertEquals(1, m.call_count) + self.assertEquals(m.mock_calls, [ + mock.call('input'), + ]) + + +class SelectExceptionTest(ExceptionTest, testlib.TestCase): + errno = errno.EBADF + exception_class = select.error + + +class OsErrorExceptionTest(ExceptionTest, testlib.TestCase): + errno = errno.EBADF + exception_class = OSError + + +if __name__ == '__main__': + unittest2.main() From 2470f486e1e121159dc9d660b1966b4766506ce2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 10:48:20 +0100 Subject: [PATCH 059/140] issue 106: ansible: make the context name available For use later to track/deduplicate streaming uploads to targets. --- ansible_mitogen/connection.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a81a4bd6..bd0d79ff 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -257,6 +257,14 @@ class Connection(ansible.plugins.connection.ConnectionBase): else: self.context = self.host + def get_context_name(self): + """ + Return the name of the target context we issue commands against, i.e. a + unique string useful as a key for related data, such as a list of + modules uploaded to the target. + """ + return self.context.name + def close(self): """ Arrange for the mitogen.master.Router running in the worker to From 43e4f5009af58fd187267fb23a62f2bedc232af6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:24:57 +0100 Subject: [PATCH 060/140] issue #106: remove 2 needless Invocation attributes. --- ansible_mitogen/mixins.py | 2 -- ansible_mitogen/planner.py | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index d003ce2e..2ed7139a 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -310,8 +310,6 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): connection=self._connection, module_name=mitogen.utils.cast(module_name), module_args=mitogen.utils.cast(module_args), - task_vars=task_vars, - tmp=tmp, env=mitogen.utils.cast(env), wrap_async=wrap_async, ) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 0fb99569..c0bb1d23 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -60,18 +60,16 @@ class Invocation(object): helpers.run_module() or helpers.run_module_async() in the target context. """ def __init__(self, action, connection, module_name, module_args, - task_vars, tmp, env, wrap_async): - #: Instance of the ActionBase subclass invoking the module. Required to - #: access some output postprocessing methods that don't belong in - #: ActionBase at all. + env, wrap_async): + #: ActionBase instance invoking the module. Required to access some + #: output postprocessing methods that don't belong in ActionBase at + #: all. self.action = action self.connection = connection self.module_name = module_name self.module_args = module_args self.module_path = None self.module_source = None - self.task_vars = task_vars - self.tmp = tmp self.env = env self.wrap_async = wrap_async From 504032e6e87ad9aaa2ee002fbee7aa52a86bcded Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:26:38 +0100 Subject: [PATCH 061/140] issue #106: has_parent_authority should accept own context ID. When a stream (such as unix.connect()) has its auth_id set to the current context's, we should allow those requests too, since the request is working with the privilege of the current context. --- mitogen/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index beeedd63..7fcad0ac 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -157,8 +157,9 @@ def _unpickle_dead(): _DEAD = Dead() -def has_parent_authority(msg, _stream): - return msg.auth_id in mitogen.parent_ids +def has_parent_authority(msg, _stream=None): + return (msg.auth_id == mitogen.context_id or + msg.auth_id in mitogen.parent_ids) def listen(obj, name, func): From 17bfb596d01f15d6620259bfcc16d8676692fdf9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:27:37 +0100 Subject: [PATCH 062/140] issue #106: mitogen.service missing from modules list. --- mitogen/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitogen/core.py b/mitogen/core.py index 7fcad0ac..60971c61 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -492,6 +492,7 @@ class Importer(object): 'fork', 'master', 'parent', + 'service', 'ssh', 'sudo', 'utils', From 8f175bf7a8b078fb55163d5f35773ae3f94931bc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:27:51 +0100 Subject: [PATCH 063/140] issue #106: _unpickle_context() did not allow nameless contexts. These are generated by any child calling .context_by_id() without previously knowing the context's name, such as Contexts set in ExternalContext.master and ExternalContext.parent. --- mitogen/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 60971c61..d60fbafc 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -957,8 +957,10 @@ class Context(object): def _unpickle_context(router, context_id, name): if not (isinstance(router, Router) and - isinstance(context_id, (int, long)) and context_id >= 0 and - isinstance(name, basestring) and len(name) < 100): + isinstance(context_id, (int, long)) and context_id >= 0 and ( + (name is None) or + (isinstance(name, basestring) and len(name) < 100)) + ): raise TypeError('cannot unpickle Context: bad input') return router.context_class(router, context_id, name) From f6d436783c401fb9a54b56a8471c12f02283ba0c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:28:42 +0100 Subject: [PATCH 064/140] issue #106: add Service.__repr__, reply to bad calls * Don't hang callers that fail validate_args(), instead tell them their message was rejected. * Add Service.repr for nicer logging. --- mitogen/service.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mitogen/service.py b/mitogen/service.py index b9fad1fe..018fa17d 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -82,6 +82,12 @@ class Service(object): self.handle = self.recv.handle self.running = True + def __repr__(self): + return '%s.%s()' % ( + self.__class__.__module__, + self.__class__.__name__, + ) + def validate_args(self, args): return ( isinstance(args, dict) and @@ -108,6 +114,7 @@ class Service(object): isinstance(args, mitogen.core.CallError) or not self.validate_args(args)): LOG.warning('Received junk message: %r', args) + msg.reply(mitogen.core.CallError('Received junk message')) return try: From dbaca05ac81333c77c94c2ea1ae6b8a7fdd2b5fb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:29:52 +0100 Subject: [PATCH 065/140] issue #106: Runner module docstring --- ansible_mitogen/runner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 2d92626e..3d55d806 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -26,6 +26,15 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. + +""" +These classes implement execution for each style of Ansible module. They are +instantiated in the target context by way of helpers.py::run_module(). + +Each class in here has a corresponding Planner class in planners.py that knows +how to build arguments for it, preseed related data, etc. +""" + from __future__ import absolute_import import json import os From 8fffb34752c72dca951714a34603a81b02220a6e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:30:35 +0100 Subject: [PATCH 066/140] issue #106: helpers.get_file(), command logging. * Add helpers.get_file() that calls back up into FileService as necessary. This is a stopgap measure. * Add logging to exec_args() to simplify debugging of binary runners. --- ansible_mitogen/helpers.py | 43 +++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 9f2e3c2b..f561c877 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -28,6 +28,7 @@ from __future__ import absolute_import import json +import logging import operator import os import pwd @@ -37,9 +38,18 @@ import stat import subprocess import tempfile import threading +import zlib import mitogen.core +import mitogen.service import ansible_mitogen.runner +import ansible_mitogen.services + + +LOG = logging.getLogger(__name__) + +#: Caching of fetched file data. +_file_cache = {} #: Mapping of job_id<->result dict _result_by_job_id = {} @@ -48,6 +58,32 @@ _result_by_job_id = {} _thread_by_job_id = {} +def get_file(context, path): + """ + Basic in-memory caching module fetcher. This generates an one roundtrip for + every previously unseen module, so it is only temporary. + + :param context: + Context we should direct FileService requests to. For now (and probably + forever) this is just the top-level Mitogen connection manager process. + :param path: + Path to fetch from FileService, must previously have been registered by + a privileged context using the `register` command. + :returns: + Bytestring file data. + """ + if path not in _file_cache: + _file_cache[path] = zlib.decompress( + mitogen.service.call( + context, + ansible_mitogen.services.FileService.handle, + ('fetch', path) + ) + ) + + return _file_cache[path] + + def run_module(kwargs): """ Set up the process environment in preparation for running an Ansible @@ -145,14 +181,15 @@ def exec_args(args, in_data='', chdir=None, shell=None): Run a command in a subprocess, emulating the argument handling behaviour of SSH. - :param bytes cmd: - String command line, passed to user's shell. + :param list[str]: + Argument vector. :param bytes in_data: Optional standard input for the command. :return: (return code, stdout bytes, stderr bytes) """ - assert isinstance(cmd, basestring) + LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir) + assert isinstance(args, list) proc = subprocess.Popen( args=args, From 6aac37e157d93acca36c7da16baf46e7b8478cd4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:31:33 +0100 Subject: [PATCH 067/140] issue #106: allow any context to contact FileService. Also fix privilege check for register command. --- ansible_mitogen/services.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 0266d533..058dac1c 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -112,6 +112,9 @@ class FileService(mitogen.service.Service): """ handle = 501 max_message_size = 1000 + policies = ( + mitogen.service.AllowAny(), + ) unprivileged_msg = 'Cannot register from unprivileged context.' unregistered_msg = 'Path is not registered with FileService.' @@ -133,7 +136,7 @@ class FileService(mitogen.service.Service): return getattr(self, cmd)(path, msg) def register(self, path, msg): - if msg.auth_id not in mitogen.parent_ids: + if not mitogen.core.has_parent_authority(msg): raise mitogen.core.CallError(self.unprivileged_msg) with open(path, 'rb') as fp: From 1a040cf5c0d95f45b7088564c52eabc54f7cf1b7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:32:29 +0100 Subject: [PATCH 068/140] issue #106: get FileService working. --- ansible_mitogen/services.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 058dac1c..7bc59135 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -128,17 +128,21 @@ class FileService(mitogen.service.Service): isinstance(args, tuple) and len(args) == 2 and args[0] in ('register', 'fetch') and - isinstance(args[1], str) + isinstance(args[1], basestring) ) def dispatch(self, args, msg): - cmd, path = msg + cmd, path = args return getattr(self, cmd)(path, msg) def register(self, path, msg): if not mitogen.core.has_parent_authority(msg): raise mitogen.core.CallError(self.unprivileged_msg) + if path in self._paths: + return + + LOG.info('%r: registering %r', self, path) with open(path, 'rb') as fp: self._paths[path] = zlib.compress(fp.read()) From 0dd5e04eae4b99fba3ab4b4220665e0b7b7d5724 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:32:45 +0100 Subject: [PATCH 069/140] issue #106: partially working BinaryRunner/Planner. Refactor planner.py to look a lot more like runner.py. This 'structural cutpaste' looks messy -- probably we can simplify this code, even though it's pretty simple already. --- ansible_mitogen/planner.py | 114 +++++++++++++++++++------------------ ansible_mitogen/process.py | 3 +- ansible_mitogen/runner.py | 21 ++++--- 3 files changed, 72 insertions(+), 66 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index c0bb1d23..11b9d7a6 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -49,6 +49,7 @@ except ImportError: # Ansible <2.4 import mitogen import mitogen.service import ansible_mitogen.helpers +import ansible_mitogen.services LOG = logging.getLogger(__name__) @@ -65,14 +66,25 @@ class Invocation(object): #: output postprocessing methods that don't belong in ActionBase at #: all. self.action = action + #: Ansible connection to use to contact the target. Must be an + #: ansible_mitogen connection. self.connection = connection + #: Name of the module ('command', 'shell', etc.) to execute. self.module_name = module_name + #: Final module arguments. self.module_args = module_args - self.module_path = None - self.module_source = None + #: Final module environment. self.env = env + #: Boolean, if :py:data:`True`, launch the module asynchronously. self.wrap_async = wrap_async + #: Initially ``None``, but set by :func:`invoke`. The path on the + #: master to the module's implementation file. + self.module_path = None + #: Initially ``None``, but set by :func:`invoke`. The raw source or + #: binary contents of the module. + self.module_source = None + def __repr__(self): return 'Invocation(module_name=%s)' % (self.module_name,) @@ -90,50 +102,34 @@ class Planner(object): raise NotImplementedError() -class JsonArgsPlanner(Planner): - """ - Script that has its interpreter directive and the task arguments - substituted into its source as a JSON string. +class BinaryPlanner(Planner): """ - def detect(self, invocation): - return module_common.REPLACER_JSONARGS in invocation.module_source - - def plan(self, invocation): - path = None # TODO - mitogen.service.call(501, ('register', path)) - return { - 'func': 'run_json_args_module', - 'binary': source, - 'args': args, - 'env': env, - } - - -class WantJsonPlanner(Planner): + Binary modules take their arguments and will return data to Ansible in the + same way as want JSON modules. """ - If a module has the string WANT_JSON in it anywhere, Ansible treats it as a - non-native module that accepts a filename as its only command line - parameter. The filename is for a temporary file containing a JSON string - containing the module's parameters. The module needs to open the file, read - and parse the parameters, operate on the data, and print its return data as - a JSON encoded dictionary to stdout before exiting. + runner_name = 'BinaryRunner' - These types of modules are self-contained entities. As of Ansible 2.1, - Ansible only modifies them to change a shebang line if present. - """ def detect(self, invocation): - return 'WANT_JSON' in invocation.module_source + return module_common._is_binary(invocation.module_source) - def plan(self, name, source, args, env): + def plan(self, invocation): + invocation.connection._connect() + mitogen.service.call( + invocation.connection.parent, + ansible_mitogen.services.FileService.handle, + ('register', invocation.module_path) + ) return { - 'func': 'run_want_json_module', - 'binary': source, - 'args': args, - 'env': env, + 'runner_name': self.runner_name, + 'module': invocation.module_name, + 'service_context': invocation.connection.parent, + 'path': invocation.module_path, + 'args': invocation.module_args, + 'env': invocation.env, } -class ReplacerPlanner(Planner): +class ReplacerPlanner(BinaryPlanner): """ The Module Replacer framework is the original framework implementing new-style modules. It is essentially a preprocessor (like the C @@ -157,33 +153,39 @@ class ReplacerPlanner(Planner): "ansible/module_utils/powershell.ps1". It should only be used with new-style Powershell modules. """ + runner_name = 'ReplacerRunner' + def detect(self, invocation): return module_common.REPLACER in invocation.module_source - def plan(self, name, source, args, env): - return { - 'func': 'run_replacer_module', - 'binary': source, - 'args': args, - 'env': env, - } - -class BinaryPlanner(Planner): +class JsonArgsPlanner(BinaryPlanner): """ - Binary modules take their arguments and will return data to Ansible in the - same way as want JSON modules. + Script that has its interpreter directive and the task arguments + substituted into its source as a JSON string. """ + runner_name = 'JsonArgsRunner' + def detect(self, invocation): - return module_common._is_binary(invocation.module_source) + return module_common.REPLACER_JSONARGS in invocation.module_source - def plan(self, name, source, args, env): - return { - 'runner_name': 'BinaryRunner', - 'binary': source, - 'args': args, - 'env': env, - } + +class WantJsonPlanner(BinaryPlanner): + """ + If a module has the string WANT_JSON in it anywhere, Ansible treats it as a + non-native module that accepts a filename as its only command line + parameter. The filename is for a temporary file containing a JSON string + containing the module's parameters. The module needs to open the file, read + and parse the parameters, operate on the data, and print its return data as + a JSON encoded dictionary to stdout before exiting. + + These types of modules are self-contained entities. As of Ansible 2.1, + Ansible only modifies them to change a shebang line if present. + """ + runner_name = 'WantJsonRunner' + + def detect(self, invocation): + return 'WANT_JSON' in invocation.module_source class NativePlanner(Planner): diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index eb9bd2ff..8febea90 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -153,7 +153,8 @@ class MuxProcess(object): self.pool = mitogen.service.Pool( router=self.router, services=[ - ansible_mitogen.services.ContextService(self.router) + ansible_mitogen.services.ContextService(self.router), + ansible_mitogen.services.FileService(self.router), ], size=int(os.environ.get('MITOGEN_POOL_SIZE', '16')), ) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 3d55d806..84d06297 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -233,21 +233,24 @@ class NativeRunner(Runner): class BinaryRunner(Runner): - def __init__(self, path, **kwargs): + def __init__(self, path, service_context, **kwargs): + print 'derp', kwargs super(BinaryRunner, self).__init__(**kwargs) self.path = path + self.service_context = service_context def setup(self): super(BinaryRunner, self).setup() - self._setup_binary() + self._setup_program() self._setup_args() - def _get_binary(self): + def _get_program(self): """ Fetch the module binary from the master if necessary. """ return ansible_mitogen.helpers.get_file( - path=self.runner_params['path'], + context=self.service_context, + path=self.path, ) def _get_args(self): @@ -259,15 +262,15 @@ class BinaryRunner(Runner): def _setup_program(self): """ Create a temporary file containing the program code. The code is - fetched via :meth:`_get_binary`. + fetched via :meth:`_get_program`. """ self.bin_fp = tempfile.NamedTemporaryFile( prefix='ansible_mitogen', suffix='-binary', ) - self.bin_fp.write(self._get_binary()) + self.bin_fp.write(self._get_program()) self.bin_fp.flush() - os.chmod(self.fp.name, int('0700', 8)) + os.chmod(self.bin_fp.name, int('0700', 8)) def _setup_args(self): """ @@ -308,8 +311,8 @@ class BinaryRunner(Runner): class WantJsonRunner(BinaryRunner): - def _get_binary(self): - s = super(WantJsonRunner, self)._get_binary() + def _get_program(self): + s = super(WantJsonRunner, self)._get_program() # fix up shebang. return s From 23366b4580558837c422ba45672cf537f8e92665 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 11:58:06 +0100 Subject: [PATCH 070/140] issue #106: import binary modules. --- examples/playbook/Makefile | 4 ++++ examples/playbook/modules/binary_producing_json.c | 13 +++++++++++++ examples/playbook/modules/binary_producing_junk.c | 9 +++++++++ examples/playbook/modules/single_null_binary | Bin 0 -> 2 bytes 4 files changed, 26 insertions(+) create mode 100644 examples/playbook/Makefile create mode 100644 examples/playbook/modules/binary_producing_json.c create mode 100644 examples/playbook/modules/binary_producing_junk.c create mode 100755 examples/playbook/modules/single_null_binary diff --git a/examples/playbook/Makefile b/examples/playbook/Makefile new file mode 100644 index 00000000..8c9e4480 --- /dev/null +++ b/examples/playbook/Makefile @@ -0,0 +1,4 @@ + +all: \ + modules/binary_producing_junk \ + modules/binary_producing_json diff --git a/examples/playbook/modules/binary_producing_json.c b/examples/playbook/modules/binary_producing_json.c new file mode 100644 index 00000000..989e5e3e --- /dev/null +++ b/examples/playbook/modules/binary_producing_json.c @@ -0,0 +1,13 @@ +#include + + +int main(void) +{ + fprintf(stderr, "binary_producing_json: oh noes\n"); + printf("{" + "\"changed\": true, " + "\"failed\": false, " + "\"msg\": \"Hello, world.\"" + "}\n"); + return 0; +} diff --git a/examples/playbook/modules/binary_producing_junk.c b/examples/playbook/modules/binary_producing_junk.c new file mode 100644 index 00000000..f6b68462 --- /dev/null +++ b/examples/playbook/modules/binary_producing_junk.c @@ -0,0 +1,9 @@ +#include + + +int main(void) +{ + fprintf(stderr, "binary_producing_junk: oh noes\n"); + printf("Hello, world.\n"); + return 0; +} diff --git a/examples/playbook/modules/single_null_binary b/examples/playbook/modules/single_null_binary new file mode 100755 index 0000000000000000000000000000000000000000..1f2a4f5ef3df7f7456d91c961da36fc58904f2f1 GIT binary patch literal 2 JcmZSJ0ssIE01E&B literal 0 HcmV?d00001 From df6daaf3c4d0be274eaf2899b591315daa89546e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 12:56:53 +0100 Subject: [PATCH 071/140] issue #106: working/semantically compatible binary support. --- ansible_mitogen/helpers.py | 2 +- ansible_mitogen/mixins.py | 18 +++++++++++------- ansible_mitogen/runner.py | 26 +++++++++++++++++++++----- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index f561c877..270f3244 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -94,7 +94,7 @@ def run_module(kwargs): runner_name = kwargs.pop('runner_name') klass = getattr(ansible_mitogen.runner, runner_name) impl = klass(**kwargs) - return json.dumps(impl.run()) + return impl.run() def _async_main(job_id, runner_name, kwargs): diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2ed7139a..831d77eb 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -315,18 +315,22 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): ) ) - def _postprocess_response(self, js): + def _postprocess_response(self, result): """ Apply fixups mimicking ActionBase._execute_module(); this is copied verbatim from action/__init__.py, the guts of _parse_returned_data are garbage and should be removed or reimplemented once tests exist. + + :param dict result: + Dictionary with format:: + + { + "rc": int, + "stdout": "stdout data", + "stderr": "stderr data" + } """ - data = self._parse_returned_data({ - 'rc': 0, - 'stdout': js, - 'stdout_lines': [js], - 'stderr': '' - }) + data = self._parse_returned_data(result) # Cutpasted from the base implementation. if 'stdout' in data and 'stdout_lines' not in data: diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 84d06297..ad94ef7c 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -87,6 +87,16 @@ class Runner(object): self._env.revert() def _run(self): + """ + The _run() method is expected to return a dictionary in the form of + ActionBase._low_level_execute_command() output, i.e. having:: + + { + "rc": int, + "stdout": "stdout data", + "stderr": "stderr data" + } + """ raise NotImplementedError() def run(self): @@ -224,11 +234,16 @@ class NativeRunner(Runner): # to invoke main explicitly. mod.main() except NativeModuleExit, e: - return e.dct + return { + 'rc': 0, + 'stdout': json.dumps(e.dct), + 'stderr': '', + } return { - 'failed': True, - 'msg': 'ansible_mitogen: module did not exit normally.' + 'rc': 1, + 'stdout': '', + 'stderr': 'ansible_mitogen: module did not exit normally.', } @@ -299,8 +314,9 @@ class BinaryRunner(Runner): ) except Exception, e: return { - 'failed': True, - 'msg': '%s: %s' % (type(e), e), + 'rc': 1, + 'stdout': '', + 'stderr': '%s: %s' % (type(e), e), } return { From 16b64392e2092cad084c38356aed5a6c574baeb2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 18:19:34 +0100 Subject: [PATCH 072/140] issue #106: support WANT_JSON modules. --- ansible_mitogen/mixins.py | 2 + ansible_mitogen/planner.py | 61 +++++++++++++- ansible_mitogen/runner.py | 157 +++++++++++++++++++++++++++---------- 3 files changed, 173 insertions(+), 47 deletions(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 831d77eb..013e6b81 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -310,6 +310,8 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): connection=self._connection, module_name=mitogen.utils.cast(module_name), module_args=mitogen.utils.cast(module_args), + task_vars=task_vars, + templar=self._templar, env=mitogen.utils.cast(env), wrap_async=wrap_async, ) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 11b9d7a6..ec14fdcc 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -55,13 +55,41 @@ import ansible_mitogen.services LOG = logging.getLogger(__name__) +def parse_script_interpreter(source): + """ + Extract the script interpreter and its sole argument from the module + source code. + + :returns: + Tuple of `(interpreter, arg)`, where `intepreter` is the script + interpreter and `arg` is its solve argument if present, otherwise + :py:data:`None`. + """ + # Linux requires first 2 bytes with no whitespace, pretty sure it's the + # same everywhere. See binfmt_script.c. + if not source.startswith('#!'): + return None, None + + # Find terminating newline. Assume last byte of binprm_buf if absent. + nl = source.find('\n', 0, 128) + if nl == -1: + nl = min(128, len(source)) + + # Split once on the first run of whitespace. If no whitespace exists, + # bits just contains the interpreter filename. + bits = source[2:nl].strip().split(None, 1) + if len(bits) == 1: + return bits[0], None + return bits[0], bits[1] + + class Invocation(object): """ Collect up a module's execution environment then use it to invoke helpers.run_module() or helpers.run_module_async() in the target context. """ def __init__(self, action, connection, module_name, module_args, - env, wrap_async): + task_vars, templar, env, wrap_async): #: ActionBase instance invoking the module. Required to access some #: output postprocessing methods that don't belong in ActionBase at #: all. @@ -73,6 +101,10 @@ class Invocation(object): self.module_name = module_name #: Final module arguments. self.module_args = module_args + #: Task variables, needed to extract ansible_*_interpreter. + self.task_vars = task_vars + #: Templar, needed to extract ansible_*_interpreter. + self.templar = templar #: Final module environment. self.env = env #: Boolean, if :py:data:`True`, launch the module asynchronously. @@ -129,6 +161,27 @@ class BinaryPlanner(Planner): } +class ScriptPlanner(BinaryPlanner): + """ + Common functionality for script module planners -- handle interpreter + detection and rewrite. + """ + def plan(self, invocation): + kwargs = super(ScriptPlanner, self).plan(invocation) + interpreter, arg = parse_script_interpreter(invocation.module_source) + shebang, _ = module_common._get_shebang( + interpreter=interpreter, + task_vars=invocation.task_vars, + templar=invocation.templar, + ) + if shebang: + interpreter = shebang[2:] + + kwargs['interpreter'] = interpreter + kwargs['interpreter_arg'] = arg + return kwargs + + class ReplacerPlanner(BinaryPlanner): """ The Module Replacer framework is the original framework implementing @@ -159,7 +212,7 @@ class ReplacerPlanner(BinaryPlanner): return module_common.REPLACER in invocation.module_source -class JsonArgsPlanner(BinaryPlanner): +class JsonArgsPlanner(ScriptPlanner): """ Script that has its interpreter directive and the task arguments substituted into its source as a JSON string. @@ -170,7 +223,7 @@ class JsonArgsPlanner(BinaryPlanner): return module_common.REPLACER_JSONARGS in invocation.module_source -class WantJsonPlanner(BinaryPlanner): +class WantJsonPlanner(ScriptPlanner): """ If a module has the string WANT_JSON in it anywhere, Ansible treats it as a non-native module that accepts a filename as its only command line @@ -224,7 +277,7 @@ class NativePlanner(Planner): _planners = [ # JsonArgsPlanner, - # WantJsonPlanner, + WantJsonPlanner, # ReplacerPlanner, BinaryPlanner, NativePlanner, diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index ad94ef7c..cf970a6b 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -37,6 +37,7 @@ how to build arguments for it, preseed related data, etc. from __future__ import absolute_import import json +import logging import os import tempfile @@ -52,6 +53,9 @@ import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' +LOG = logging.getLogger(__name__) + + class Runner(object): """ Ansible module runner. After instantiation (with kwargs supplied by the @@ -247,70 +251,52 @@ class NativeRunner(Runner): } -class BinaryRunner(Runner): +class ProgramRunner(Runner): def __init__(self, path, service_context, **kwargs): - print 'derp', kwargs - super(BinaryRunner, self).__init__(**kwargs) + super(ProgramRunner, self).__init__(**kwargs) self.path = path self.service_context = service_context def setup(self): - super(BinaryRunner, self).setup() + super(ProgramRunner, self).setup() self._setup_program() - self._setup_args() - - def _get_program(self): - """ - Fetch the module binary from the master if necessary. - """ - return ansible_mitogen.helpers.get_file( - context=self.service_context, - path=self.path, - ) - - def _get_args(self): - """ - Return the module arguments formatted as JSON. - """ - return json.dumps(self.args) def _setup_program(self): """ Create a temporary file containing the program code. The code is fetched via :meth:`_get_program`. """ - self.bin_fp = tempfile.NamedTemporaryFile( + self.program_fp = tempfile.NamedTemporaryFile( prefix='ansible_mitogen', suffix='-binary', ) - self.bin_fp.write(self._get_program()) - self.bin_fp.flush() - os.chmod(self.bin_fp.name, int('0700', 8)) + self.program_fp.write(self._get_program()) + self.program_fp.flush() + os.chmod(self.program_fp.name, int('0700', 8)) - def _setup_args(self): + def _get_program(self): """ - Create a temporary file containing the module's arguments. The - arguments are formatted via :meth:`_get_args`. + Fetch the module binary from the master if necessary. """ - self.args_fp = tempfile.NamedTemporaryFile( - prefix='ansible_mitogen', - suffix='-args', + return ansible_mitogen.helpers.get_file( + context=self.service_context, + path=self.path, ) - self.args_fp.write(self._get_args()) - self.args_fp.flush() + + def _get_program_args(self): + return [self.program_fp.name] def revert(self): """ - Delete the temporary binary and argument files. + Delete the temporary program file. """ - self.args_fp.close() - self.bin_fp.close() - super(BinaryRunner, self).revert() + super(ProgramRunner, self).revert() + self.program_fp.close() def _run(self): try: rc, stdout, stderr = ansible_mitogen.helpers.exec_args( - args=[self.bin_fp.name, self.args_fp.name], + args=self._get_program_args(), ) except Exception, e: return { @@ -326,15 +312,100 @@ class BinaryRunner(Runner): } -class WantJsonRunner(BinaryRunner): +class ArgsFileRunner(Runner): + def setup(self): + super(ArgsFileRunner, self).setup() + self._setup_args() + + def _setup_args(self): + """ + Create a temporary file containing the module's arguments. The + arguments are formatted via :meth:`_get_args`. + """ + self.args_fp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen', + suffix='-args', + ) + self.args_fp.write(self._get_args_contents()) + self.args_fp.flush() + + def _get_args_contents(self): + """ + Return the module arguments formatted as JSON. + """ + return json.dumps(self.args) + + def _get_program_args(self): + return [self.program_fp.name, self.args_fp.name] + + def revert(self): + """ + Delete the temporary argument file. + """ + super(ArgsFileRunner, self).revert() + self.args_fp.close() + + +class BinaryRunner(ArgsFileRunner, ProgramRunner): + pass + + +class ScriptRunner(ProgramRunner): + def __init__(self, interpreter, interpreter_arg, **kwargs): + super(ScriptRunner, self).__init__(**kwargs) + self.interpreter = interpreter + self.interpreter_arg = interpreter_arg + + b_ENCODING_STRING = b'# -*- coding: utf-8 -*-' + def _get_program(self): - s = super(WantJsonRunner, self)._get_program() - # fix up shebang. - return s + return self._rewrite_source( + super(ScriptRunner, self)._get_program() + ) + + def _rewrite_source(self, s): + """ + Mutate the source according to the per-task parameters. + """ + # Couldn't find shebang, so let shell run it, because shell assumes + # executables like this are just shell scripts. + LOG.debug('++++++++++++++ %s', self.interpreter) + if not self.interpreter: + return s + + shebang = '#!' + self.interpreter + if self.interpreter_arg: + shebang += ' ' + self.interpreter_arg + + new = [shebang] + if os.path.basename(self.interpreter).startswith('python'): + new.append(self.b_ENCODING_STRING) + + _, _, rest = s.partition('\n') + new.append(rest) + return '\n'.join(new) + + +class JsonArgsFileRunner(ScriptRunner): + JSON_ARGS = '<>' + + def _get_args_contents(self): + return json.dump(self.args) + + def _rewrite_source(self, s): + return ( + super(JsonArgsFileRunner, self)._rewrite_source(s) + .replace(self.JSON_ARGS, self._get_args_contents()) + ) + + +class WantJsonRunner(ArgsFileRunner, ScriptRunner): + pass + -class OldStyleRunner(BinaryRunner): - def _get_args(self): +class OldStyleRunner(ScriptRunner): + def _get_args_contents(self): """ Mimic the argument formatting behaviour of ActionBase._execute_module(). From a954d54644099cced940dc1a13fa2efa74914606 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 19:01:18 +0100 Subject: [PATCH 073/140] issue #106: support old-style too. --- ansible_mitogen/planner.py | 30 ++++++++++++++++++++++++------ ansible_mitogen/runner.py | 34 ++++++++++++++++------------------ 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index ec14fdcc..b5f1bca0 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -241,14 +241,16 @@ class WantJsonPlanner(ScriptPlanner): return 'WANT_JSON' in invocation.module_source -class NativePlanner(Planner): +class NewStylePlanner(Planner): """ The Ansiballz framework differs from module replacer in that it uses real Python imports of things in ansible/module_utils instead of merely preprocessing the module. """ + runner_name = 'NewStyleRunner' + def detect(self, invocation): - return True + return 'from ansible.module_utils.' in invocation.module_source def get_command_module_name(self, module_name): """ @@ -267,7 +269,7 @@ class NativePlanner(Planner): def plan(self, invocation): return { - 'runner_name': 'NativeRunner', + 'runner_name': self.runner_name, 'module': invocation.module_name, 'mod_name': self.get_command_module_name(invocation.module_name), 'args': invocation.module_args, @@ -275,12 +277,28 @@ class NativePlanner(Planner): } +class ReplacerPlanner(NewStylePlanner): + runner_name = 'ReplacerRunner' + + def detect(self, invocation): + return module_common.REPLACER in invocation.module_source + + +class OldStylePlanner(ScriptPlanner): + runner_name = 'OldStyleRunner' + + def detect(self, invocation): + # Everything else. + return True + + _planners = [ + BinaryPlanner, + # ReplacerPlanner, + NewStylePlanner, # JsonArgsPlanner, WantJsonPlanner, - # ReplacerPlanner, - BinaryPlanner, - NativePlanner, + OldStylePlanner, ] diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index cf970a6b..ed421cca 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -131,7 +131,7 @@ class TemporaryEnvironment(object): os.environ.update(self.original) -class NativeModuleExit(Exception): +class NewStyleModuleExit(Exception): """ Capture the result of a call to `.exit_json()` or `.fail_json()` by a native Ansible module. @@ -148,24 +148,24 @@ class NativeModuleExit(Exception): ) -class NativeMethodOverrides(object): +class NewStyleMethodOverrides(object): @staticmethod def exit_json(self, **kwargs): """ Raise exit_json() output as the `.dct` attribute of a - :class:`NativeModuleExit` exception`. + :class:`NewStyleModuleExit` exception`. """ kwargs.setdefault('changed', False) - raise NativeModuleExit(self, **kwargs) + raise NewStyleModuleExit(self, **kwargs) @staticmethod def fail_json(self, **kwargs): """ Raise fail_json() output as the `.dct` attribute of a - :class:`NativeModuleExit` exception`. + :class:`NewStyleModuleExit` exception`. """ kwargs.setdefault('failed', True) - raise NativeModuleExit(self, **kwargs) + raise NewStyleModuleExit(self, **kwargs) klass = ansible.module_utils.basic.AnsibleModule @@ -183,7 +183,7 @@ class NativeMethodOverrides(object): self.klass.fail_json = self._original_fail_json -class NativeModuleArguments(object): +class NewStyleModuleArguments(object): """ Patch ansible.module_utils.basic argument globals. """ @@ -200,22 +200,22 @@ class NativeModuleArguments(object): ansible.module_utils.basic._ANSIBLE_ARGS = self.original -class NativeRunner(Runner): +class NewStyleRunner(Runner): """ Execute a new-style Ansible module, where Module Replacer-related tricks aren't required. """ def __init__(self, mod_name, **kwargs): - super(NativeRunner, self).__init__(**kwargs) + super(NewStyleRunner, self).__init__(**kwargs) self.mod_name = mod_name def setup(self): - super(NativeRunner, self).setup() - self._overrides = NativeMethodOverrides() - self._args = NativeModuleArguments(self.args) + super(NewStyleRunner, self).setup() + self._overrides = NewStyleMethodOverrides() + self._args = NewStyleModuleArguments(self.args) def revert(self): - super(NativeRunner, self).revert() + super(NewStyleRunner, self).revert() self._args.revert() self._overrides.revert() @@ -237,7 +237,7 @@ class NativeRunner(Runner): # already have been imported for a previous invocation, so we need # to invoke main explicitly. mod.main() - except NativeModuleExit, e: + except NewStyleModuleExit, e: return { 'rc': 0, 'stdout': json.dumps(e.dct), @@ -369,7 +369,6 @@ class ScriptRunner(ProgramRunner): """ # Couldn't find shebang, so let shell run it, because shell assumes # executables like this are just shell scripts. - LOG.debug('++++++++++++++ %s', self.interpreter) if not self.interpreter: return s @@ -386,7 +385,7 @@ class ScriptRunner(ProgramRunner): return '\n'.join(new) -class JsonArgsFileRunner(ScriptRunner): +class JsonArgsFileRunner(ArgsFileRunner, ScriptRunner): JSON_ARGS = '<>' def _get_args_contents(self): @@ -403,8 +402,7 @@ class WantJsonRunner(ArgsFileRunner, ScriptRunner): pass - -class OldStyleRunner(ScriptRunner): +class OldStyleRunner(ArgsFileRunner, ScriptRunner): def _get_args_contents(self): """ Mimic the argument formatting behaviour of From 8cc20856a82f0eb63b1e1e1fefb479ae8d451ae8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 21:10:04 +0100 Subject: [PATCH 074/140] issue #106: support custom new-style modules + 'reexec by default' Rather than assume any structure about the Python code: * Delete the exit_json/fail_json monkeypatches. * Patch SystemExit rather than a magic monkeypatch-thrown exception * Setup fake cStringIO stdin, stdout, stderr and return those along with SystemExit exit status * Setup _ANSIBLE_ARGS as we used to, since we still want to override that with '{}' to prevent accidental import hangs, but also provide the same string via sys.stdin. * Compile the module bytecode once and re-execute it for every invocation. May change this back again later, once some benchmarks are done. * Remove the fixups stuff for now, it's handled by ^ above. Should support any "somewhat new style" Python module, including those that just give up and dump stuff to stdout directly. --- ansible_mitogen/helpers.py | 10 +++ ansible_mitogen/planner.py | 26 +----- ansible_mitogen/runner.py | 169 ++++++++++++++----------------------- 3 files changed, 74 insertions(+), 131 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 270f3244..b107d000 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -51,6 +51,9 @@ LOG = logging.getLogger(__name__) #: Caching of fetched file data. _file_cache = {} +#: Caching of compiled new-style module bytecode. +_bytecode_cache = {} + #: Mapping of job_id<->result dict _result_by_job_id = {} @@ -84,6 +87,13 @@ def get_file(context, path): return _file_cache[path] +def get_bytecode(context, path): + if path not in _bytecode_cache: + source = get_file(context, path) + _bytecode_cache[path] = compile(source, path, 'exec') + return _bytecode_cache[path] + + def run_module(kwargs): """ Set up the process environment in preparation for running an Ansible diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index b5f1bca0..ae632ab9 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -241,7 +241,7 @@ class WantJsonPlanner(ScriptPlanner): return 'WANT_JSON' in invocation.module_source -class NewStylePlanner(Planner): +class NewStylePlanner(ScriptPlanner): """ The Ansiballz framework differs from module replacer in that it uses real Python imports of things in ansible/module_utils instead of merely @@ -252,30 +252,6 @@ class NewStylePlanner(Planner): def detect(self, invocation): return 'from ansible.module_utils.' in invocation.module_source - def get_command_module_name(self, module_name): - """ - Given the name of an Ansible command module, return its canonical - module path within the ansible. - - :param module_name: - "shell" - :return: - "ansible.modules.commands.shell" - """ - path = module_loader.find_plugin(module_name, '') - relpath = os.path.relpath(path, os.path.dirname(ansible.__file__)) - root, _ = os.path.splitext(relpath) - return 'ansible.' + root.replace('/', '.') - - def plan(self, invocation): - return { - 'runner_name': self.runner_name, - 'module': invocation.module_name, - 'mod_name': self.get_command_module_name(invocation.module_name), - 'args': invocation.module_args, - 'env': invocation.env, - } - class ReplacerPlanner(NewStylePlanner): runner_name = 'ReplacerRunner' diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index ed421cca..d6b3d48d 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -36,10 +36,13 @@ how to build arguments for it, preseed related data, etc. """ from __future__ import absolute_import +import cStringIO import json import logging import os +import sys import tempfile +import types import ansible_mitogen.helpers # TODO: circular import @@ -131,124 +134,37 @@ class TemporaryEnvironment(object): os.environ.update(self.original) -class NewStyleModuleExit(Exception): - """ - Capture the result of a call to `.exit_json()` or `.fail_json()` by a - native Ansible module. - """ - def __init__(self, ansible_module, **kwargs): - ansible_module.add_path_info(kwargs) - kwargs.setdefault('invocation', { - 'module_args': ansible_module.params - }) - ansible_module.do_cleanup_files() - self.dct = ansible.module_utils.basic.remove_values( - kwargs, - ansible_module.no_log_values, - ) - - -class NewStyleMethodOverrides(object): - @staticmethod - def exit_json(self, **kwargs): - """ - Raise exit_json() output as the `.dct` attribute of a - :class:`NewStyleModuleExit` exception`. - """ - kwargs.setdefault('changed', False) - raise NewStyleModuleExit(self, **kwargs) - - @staticmethod - def fail_json(self, **kwargs): - """ - Raise fail_json() output as the `.dct` attribute of a - :class:`NewStyleModuleExit` exception`. - """ - kwargs.setdefault('failed', True) - raise NewStyleModuleExit(self, **kwargs) - - klass = ansible.module_utils.basic.AnsibleModule - - def __init__(self): - self._original_exit_json = self.klass.exit_json - self._original_fail_json = self.klass.fail_json - self.klass.exit_json = self.exit_json - self.klass.fail_json = self.fail_json +class TemporaryArgv(object): + def __init__(self, argv): + self.original = sys.argv[:] + sys.argv[:] = argv def revert(self): - """ - Restore prior state. - """ - self.klass.exit_json = self._original_exit_json - self.klass.fail_json = self._original_fail_json + sys.argv[:] = self.original -class NewStyleModuleArguments(object): +class NewStyleStdio(object): """ Patch ansible.module_utils.basic argument globals. """ def __init__(self, args): - self.original = ansible.module_utils.basic._ANSIBLE_ARGS + self.original_stdout = sys.stdout + self.original_stderr = sys.stderr + self.original_stdin = sys.stdin + sys.stdout = cStringIO.StringIO() + sys.stderr = cStringIO.StringIO() ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({ 'ANSIBLE_MODULE_ARGS': args }) + sys.stdin = cStringIO.StringIO( + ansible.module_utils.basic._ANSIBLE_ARGS + ) def revert(self): - """ - Restore prior state. - """ - ansible.module_utils.basic._ANSIBLE_ARGS = self.original - - -class NewStyleRunner(Runner): - """ - Execute a new-style Ansible module, where Module Replacer-related tricks - aren't required. - """ - def __init__(self, mod_name, **kwargs): - super(NewStyleRunner, self).__init__(**kwargs) - self.mod_name = mod_name - - def setup(self): - super(NewStyleRunner, self).setup() - self._overrides = NewStyleMethodOverrides() - self._args = NewStyleModuleArguments(self.args) - - def revert(self): - super(NewStyleRunner, self).revert() - self._args.revert() - self._overrides.revert() - - def _fixup__default(self, mod): - pass - - def _fixup__yum_repository(self, mod): - # https://github.com/dw/mitogen/issues/154 - mod.YumRepo.repofile = mod.configparser.RawConfigParser() - - def _run(self): - fixup = getattr(self, '_fixup__' + self.module, self._fixup__default) - try: - mod = __import__(self.mod_name, {}, {}, ['']) - fixup(mod) - # Ansible modules begin execution on import. Thus the above - # __import__ will cause either Exit or ModuleError to be raised. If - # we reach the line below, the module did not execute and must - # already have been imported for a previous invocation, so we need - # to invoke main explicitly. - mod.main() - except NewStyleModuleExit, e: - return { - 'rc': 0, - 'stdout': json.dumps(e.dct), - 'stderr': '', - } - - return { - 'rc': 1, - 'stdout': '', - 'stderr': 'ansible_mitogen: module did not exit normally.', - } + sys.stdout = self.original_stdout + sys.stderr = self.original_stderr + sys.stdin = self.original_stdin + ansible.module_utils.basic._ANSIBLE_ARGS = '{}' class ProgramRunner(Runner): @@ -385,6 +301,47 @@ class ScriptRunner(ProgramRunner): return '\n'.join(new) +class NewStyleRunner(ScriptRunner): + """ + Execute a new-style Ansible module, where Module Replacer-related tricks + aren't required. + """ + def setup(self): + super(NewStyleRunner, self).setup() + self._stdio = NewStyleStdio(self.args) + self._argv = TemporaryArgv([self.path]) + + def revert(self): + super(NewStyleRunner, self).revert() + self._stdio.revert() + + def _get_bytecode(self): + """ + Fetch the module binary from the master if necessary. + """ + return ansible_mitogen.helpers.get_bytecode( + context=self.service_context, + path=self.path, + ) + + def _run(self): + bytecode = self._get_bytecode() + mod = types.ModuleType('__main__') + d = vars(mod) + e = None + + try: + exec bytecode in d, d + except SystemExit, e: + pass + + return { + 'rc': e[0] if e else 2, + 'stdout': sys.stdout.getvalue(), + 'stderr': sys.stderr.getvalue(), + } + + class JsonArgsFileRunner(ArgsFileRunner, ScriptRunner): JSON_ARGS = '<>' @@ -411,4 +368,4 @@ class OldStyleRunner(ArgsFileRunner, ScriptRunner): return ' '.join( '%s=%s' % (key, shlex_quote(str(self.args[key]))) for key in self.args - ) + ) + ' ' # Bug-for-bug :( From 971b366162c2b271ec469be2266dc78eb2597550 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 21:20:23 +0100 Subject: [PATCH 075/140] issue #106: import many more test cases --- examples/playbook/modules/bin_bash_module.sh | 5 +++ examples/playbook/modules/json_args_python.py | 13 ++++++++ examples/playbook/modules/old_style_module.sh | 16 +++++++++ .../modules/python_new_style_module.py | 28 ++++++++++++++++ .../modules/python_want_json_module.py | 33 +++++++++++++++++++ examples/playbook/modules/want_json_module.sh | 16 +++++++++ 6 files changed, 111 insertions(+) create mode 100755 examples/playbook/modules/bin_bash_module.sh create mode 100644 examples/playbook/modules/json_args_python.py create mode 100755 examples/playbook/modules/old_style_module.sh create mode 100755 examples/playbook/modules/python_new_style_module.py create mode 100755 examples/playbook/modules/python_want_json_module.py create mode 100755 examples/playbook/modules/want_json_module.sh diff --git a/examples/playbook/modules/bin_bash_module.sh b/examples/playbook/modules/bin_bash_module.sh new file mode 100755 index 00000000..01abff0c --- /dev/null +++ b/examples/playbook/modules/bin_bash_module.sh @@ -0,0 +1,5 @@ +#!/bin/bash +exec >/tmp/derp +echo "$1" +cat "$1" + diff --git a/examples/playbook/modules/json_args_python.py b/examples/playbook/modules/json_args_python.py new file mode 100644 index 00000000..45689584 --- /dev/null +++ b/examples/playbook/modules/json_args_python.py @@ -0,0 +1,13 @@ +#!/usr/bin/python +# I am an Ansible Python JSONARGS module. I should receive an encoding string. + +import json +import sys + +json_arguments = """<>""" + +print "{" +print " \"changed\": false," +print " \"msg\": \"Here is my input\"," +print " \"input\": [%s]" % (json_arguments,) +print "}" diff --git a/examples/playbook/modules/old_style_module.sh b/examples/playbook/modules/old_style_module.sh new file mode 100755 index 00000000..47e6afbd --- /dev/null +++ b/examples/playbook/modules/old_style_module.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# I am an Ansible old-style module. + +INPUT=$1 + +[ ! -r "$INPUT" ] && { + echo "Usage: $0 " >&2 + exit 1 +} + +echo "{" +echo " \"changed\": false," +echo " \"msg\": \"Here is my input\"," +echo " \"filname\": \"$INPUT\"," +echo " \"input\": [\"$(cat $INPUT | tr \" \' )\"]" +echo "}" diff --git a/examples/playbook/modules/python_new_style_module.py b/examples/playbook/modules/python_new_style_module.py new file mode 100755 index 00000000..06d031b1 --- /dev/null +++ b/examples/playbook/modules/python_new_style_module.py @@ -0,0 +1,28 @@ +#!/usr/bin/python +# I am an Ansible new-style Python module. I should receive an encoding string. + +import json +import sys + +# This is the magic marker Ansible looks for: +# from ansible.module_utils. + + +def usage(): + sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) + sys.exit(1) + +# Also must slurp in our own source code, to verify the encoding string was +# added. +print 'WTFFFFFFFFFFFFFFFF %r' % (sys.argv,) +with open(sys.argv[0]) as fp: + me = fp.read() + +input_json = sys.stdin.read() + +print "{" +print " \"changed\": false," +print " \"msg\": \"Here is my input\"," +print " \"source\": [%s]," % (json.dumps(me),) +print " \"input\": [%s]" % (input_json,) +print "}" diff --git a/examples/playbook/modules/python_want_json_module.py b/examples/playbook/modules/python_want_json_module.py new file mode 100755 index 00000000..bd12704e --- /dev/null +++ b/examples/playbook/modules/python_want_json_module.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +# I am an Ansible Python WANT_JSON module. I should receive an encoding string. + +import json +import sys + +WANT_JSON = 1 + + +def usage(): + sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) + sys.exit(1) + +if len(sys.argv) < 2: + usage() + +# Also must slurp in our own source code, to verify the encoding string was +# added. +with open(sys.argv[0]) as fp: + me = fp.read() + +try: + with open(sys.argv[1]) as fp: + input_json = fp.read() +except IOError: + usage() + +print "{" +print " \"changed\": false," +print " \"msg\": \"Here is my input\"," +print " \"source\": [%s]," % (json.dumps(me),) +print " \"input\": [%s]" % (input_json,) +print "}" diff --git a/examples/playbook/modules/want_json_module.sh b/examples/playbook/modules/want_json_module.sh new file mode 100755 index 00000000..6053eacd --- /dev/null +++ b/examples/playbook/modules/want_json_module.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# I am an Ansible WANT_JSON module. + +WANT_JSON=1 +INPUT=$1 + +[ ! -r "$INPUT" ] && { + echo "Usage: $0 " >&2 + exit 1 +} + +echo "{" +echo " \"changed\": false," +echo " \"msg\": \"Here is my input\"," +echo " \"input\": [$(< $INPUT)]" +echo "}" From a98a51a32885ffe6496c13099d132845514d2f00 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 21:20:58 +0100 Subject: [PATCH 076/140] issue #106: handle JSONARGS modules too. --- ansible_mitogen/planner.py | 2 +- ansible_mitogen/runner.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index ae632ab9..e7b553c6 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -272,7 +272,7 @@ _planners = [ BinaryPlanner, # ReplacerPlanner, NewStylePlanner, - # JsonArgsPlanner, + JsonArgsPlanner, WantJsonPlanner, OldStylePlanner, ] diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index d6b3d48d..a6b77977 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -342,15 +342,15 @@ class NewStyleRunner(ScriptRunner): } -class JsonArgsFileRunner(ArgsFileRunner, ScriptRunner): +class JsonArgsRunner(ScriptRunner): JSON_ARGS = '<>' def _get_args_contents(self): - return json.dump(self.args) + return json.dumps(self.args) def _rewrite_source(self, s): return ( - super(JsonArgsFileRunner, self)._rewrite_source(s) + super(JsonArgsRunner, self)._rewrite_source(s) .replace(self.JSON_ARGS, self._get_args_contents()) ) From 2c17d60ffd2afa9ff2541c224a2495e05cc55ee0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 21:24:31 +0100 Subject: [PATCH 077/140] issue #106: import basic regtest.py --- examples/playbook/regtest.py | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 examples/playbook/regtest.py diff --git a/examples/playbook/regtest.py b/examples/playbook/regtest.py new file mode 100644 index 00000000..15fc9e3c --- /dev/null +++ b/examples/playbook/regtest.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import difflib +import logging +import re +import subprocess +import tempfile + + +LOG = logging.getLogger(__name__) + +suffixes = [ + '-m bin_bash_module', + '-m binary_producing_json', + '-m binary_producing_junk', + '-m old_style_module', + '-m python_new_style_module', + '-m python_want_json_module', + '-m single_null_binary', + '-m want_json_module', + '-m json_args_python', + '-m setup', +] + +fixups = [ + ('Shared connection to localhost closed\\.(\r\n)?', ''), # TODO +] + + +def fixup(s): + for regex, to in fixups: + s = re.sub(regex, to, s, re.DOTALL|re.M) + return s + + +def run(s): + LOG.debug('running: %r', s) + with tempfile.NamedTemporaryFile() as fp: + # https://www.systutorials.com/docs/linux/man/1-ansible-playbook/#lbAG + returncode = subprocess.call(s, stdout=fp, stderr=fp, shell=True) + fp.write('\nReturn code: %s\n' % (returncode,)) + fp.seek(0) + return fp.read() + + +logging.basicConfig(level=logging.DEBUG) + +for suffix in suffixes: + ansible = run('ansible localhost %s' % (suffix,)) + mitogen = run('ANSIBLE_STRATEGY=mitogen ansible localhost %s' % (suffix,)) + + diff = list(difflib.unified_diff( + a=fixup(ansible).splitlines(), + b=fixup(mitogen).splitlines(), + fromfile='ansible-output.txt', + tofile='mitogen-output.txt', + )) + if diff: + print '++ differ! suffix: %r' % (suffix,) + for line in diff: + print line + print + print From 41ca3ad94be573f287bd63b2d7fa3591a45a5d74 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 21:25:44 +0100 Subject: [PATCH 078/140] issue #106: delete junk from example module. --- examples/playbook/modules/python_new_style_module.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/playbook/modules/python_new_style_module.py b/examples/playbook/modules/python_new_style_module.py index 06d031b1..1ae50d50 100755 --- a/examples/playbook/modules/python_new_style_module.py +++ b/examples/playbook/modules/python_new_style_module.py @@ -14,7 +14,6 @@ def usage(): # Also must slurp in our own source code, to verify the encoding string was # added. -print 'WTFFFFFFFFFFFFFFFF %r' % (sys.argv,) with open(sys.argv[0]) as fp: me = fp.read() From a5e4a6f346213fc80b480a6768ce60a5b127ecb9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 21:40:06 +0100 Subject: [PATCH 079/140] issue #106: move helpers.get_bytecode() into NewStyleRunner --- ansible_mitogen/helpers.py | 10 ---------- ansible_mitogen/runner.py | 31 ++++++++++++++++++++----------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index b107d000..270f3244 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -51,9 +51,6 @@ LOG = logging.getLogger(__name__) #: Caching of fetched file data. _file_cache = {} -#: Caching of compiled new-style module bytecode. -_bytecode_cache = {} - #: Mapping of job_id<->result dict _result_by_job_id = {} @@ -87,13 +84,6 @@ def get_file(context, path): return _file_cache[path] -def get_bytecode(context, path): - if path not in _bytecode_cache: - source = get_file(context, path) - _bytecode_cache[path] = compile(source, path, 'exec') - return _bytecode_cache[path] - - def run_module(kwargs): """ Set up the process environment in preparation for running an Ansible diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index a6b77977..f5cb1283 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -306,32 +306,41 @@ class NewStyleRunner(ScriptRunner): Execute a new-style Ansible module, where Module Replacer-related tricks aren't required. """ + #: path => new-style module bytecode. + _code_by_path = {} + def setup(self): super(NewStyleRunner, self).setup() self._stdio = NewStyleStdio(self.args) self._argv = TemporaryArgv([self.path]) def revert(self): - super(NewStyleRunner, self).revert() + self._argv.revert() self._stdio.revert() + super(NewStyleRunner, self).revert() - def _get_bytecode(self): - """ - Fetch the module binary from the master if necessary. - """ - return ansible_mitogen.helpers.get_bytecode( - context=self.service_context, - path=self.path, - ) + def _get_code(self): + try: + return self._code_by_path[self.path] + except KeyError: + return self._code_by_path.setdefault(self.path, compile( + source=ansible_mitogen.helpers.get_file( + context=self.service_context, + path=self.path, + ), + filename=self.path, + mode='exec', + dont_inherit=True, + )) def _run(self): - bytecode = self._get_bytecode() + code = self._get_code() mod = types.ModuleType('__main__') d = vars(mod) e = None try: - exec bytecode in d, d + exec code in d, d except SystemExit, e: pass From e5723e4f5f1bd69d8e222a46ecf7664a224e4aab Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 21:59:59 +0100 Subject: [PATCH 080/140] ansible: fix _make_tmp_path() regression on 2.3.x. Due to issue #177. --- ansible_mitogen/mixins.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 013e6b81..56eaa281 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -184,15 +184,18 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) + try: + remote_tmp = self._connection._shell.get_option('remote_tmp') + except AttributeError: + # Required for <2.4.x. + remote_tmp = '~/.ansible' + # _make_tmp_path() is basically a global stashed away as Shell.tmpdir. # The copy action plugin violates layering and grabs this attribute # directly. self._connection._shell.tmpdir = self.call( ansible_mitogen.helpers.make_temp_directory, - base_dir=self._remote_expand_user( - # ~/.ansible - self._connection._shell.get_option('remote_tmp') - ) + base_dir=self._remote_expand_user(remote_tmp), ) LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) self._cleanup_remote_tmp = True From 822502125d3653d8d63c60f1a24defa26d2fb1cd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 22:04:05 +0100 Subject: [PATCH 081/140] issue #106: 2.3.x compatible get_shebang-alike. --- ansible_mitogen/planner.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index e7b553c6..49c0be06 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -166,20 +166,24 @@ class ScriptPlanner(BinaryPlanner): Common functionality for script module planners -- handle interpreter detection and rewrite. """ + def _rewrite_interpreter(self, interpreter, task_vars, templar): + key = u'ansible_%s_interpreter' % os.path.basename(interpreter).strip() + try: + return templar.template(task_vars[key].strip()) + except KeyError: + return interpreter + def plan(self, invocation): kwargs = super(ScriptPlanner, self).plan(invocation) interpreter, arg = parse_script_interpreter(invocation.module_source) - shebang, _ = module_common._get_shebang( - interpreter=interpreter, - task_vars=invocation.task_vars, - templar=invocation.templar, + return dict(kwargs, + interpreter_arg=arg, + interpreter=self._rewrite_interpreter( + interpreter=interpreter, + task_vars=invocation.task_vars, + templar=invocation.templar, + ) ) - if shebang: - interpreter = shebang[2:] - - kwargs['interpreter'] = interpreter - kwargs['interpreter_arg'] = arg - return kwargs class ReplacerPlanner(BinaryPlanner): From 6dcefd631a4feab1a1b39ad3951031db20241e86 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 22:08:46 +0100 Subject: [PATCH 082/140] issue #106: docs: remove built-in only limitation :> --- docs/ansible.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index d193ca89..ee0f8595 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -113,9 +113,6 @@ This is a proof of concept: issues below are exclusively due to code immaturity. High Risk ~~~~~~~~~ -* For now only **built-in Python command modules work**, however almost all - modules shipped with Ansible are Python-based. - * Transfer of large (i.e. GB-sized) files using certain Ansible-internal APIs, such as triggered via the ``copy`` module, will cause corresponding temporary memory and CPU spikes on both host and target machine, due to delivering the @@ -172,6 +169,10 @@ Low Risk what that behaviour is supposed to be. See `Ansible#14377`_ for related discussion. +* "Module Replacer" style modules are not yet supported. These rarely appear in + practice, and light Github code searches failed to reveal many examples of + them. + .. _Ansible#14377: https://github.com/ansible/ansible/issues/14377 From 470d8399a346b9ca1b5e083c0f99b8bbd361f59b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 22:31:45 +0100 Subject: [PATCH 083/140] ansible: document planner.Planner. --- ansible_mitogen/planner.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 49c0be06..8fcd639c 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -128,9 +128,27 @@ class Planner(object): exports a method to run the module. """ def detect(self, invocation): + """ + Return true if the supplied `invocation` matches the module type + implemented by this planner. + """ raise NotImplementedError() def plan(self, invocation): + """ + If :meth:`detect` returned :data:`True`, plan for the module's + execution, including granting access to or delivering any files to it + that are known to be absent, and finally return a dict:: + + { + # Name of the class from runners.py that implements the + # target-side execution of this module type. + "runner_name": "...", + + # Remaining keys are passed to the constructor of the class + # named by `runner_name`. + } + """ raise NotImplementedError() From 8425b196e7463baa10ff6854261ca230f450ac0f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 22:34:53 +0100 Subject: [PATCH 084/140] docs: merge duplicate risks --- docs/ansible.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index ee0f8595..8d4dacc6 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -126,7 +126,8 @@ High Risk * `Asynchronous Actions And Polling `_ has received - minimal testing. + minimal testing. Jobs execute in a thread of the target Python interpreter. + This will fixed shortly. * No mechanism exists yet to bound the number of interpreters created during a run. For some playbooks that parameterize ``become_user`` over a large number @@ -204,9 +205,6 @@ Behavioural Differences connection to host closed`` to appear in ``stderr`` output of every executed command. This never manifests with the Mitogen extension. -* Asynchronous support is very primitive, and jobs execute in a thread of the - target Python interpreter. This will fixed shortly. - * Local commands are executed in a reuseable Python interpreter created identically to interpreters used on remote hosts. At present only one such interpreter per ``become_user`` exists, and so only one action may be From 586318c475670d4c1794e39f1d4b44b5e889b0ed Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 1 Apr 2018 23:49:43 +0100 Subject: [PATCH 085/140] Fix preamble_size.py. --- preamble_size.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preamble_size.py b/preamble_size.py index 9af20795..83e29bc8 100644 --- a/preamble_size.py +++ b/preamble_size.py @@ -14,7 +14,7 @@ import mitogen.sudo router = mitogen.master.Router() context = mitogen.parent.Context(router, 0) -stream = mitogen.ssh.Stream(router, 0, hostname='foo') +stream = mitogen.ssh.Stream(router, 0, max_message_size=0, hostname='foo') print 'SSH command size: %s' % (len(' '.join(stream.get_boot_command())),) print 'Preamble size: %s (%.2fKiB)' % ( From 380ef7376db93bb6974fb70d30a0a7ba290b6cdf Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 00:01:28 +0100 Subject: [PATCH 086/140] ansible: Add support for free strategy. --- ansible_mitogen/plugins/strategy/mitogen.py | 10 +++- .../plugins/strategy/mitogen_free.py | 60 +++++++++++++++++++ .../plugins/strategy/mitogen_linear.py | 60 +++++++++++++++++++ ansible_mitogen/strategy.py | 13 ++-- docs/ansible.rst | 8 +-- examples/playbook/ansible.cfg | 2 +- 6 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 ansible_mitogen/plugins/strategy/mitogen_free.py create mode 100644 ansible_mitogen/plugins/strategy/mitogen_linear.py diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index 77d1e06e..3ef522b4 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -51,6 +51,10 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir -from ansible_mitogen.strategy import StrategyModule -del os -del sys +import ansible_mitogen.strategy +import ansible.plugins.strategy.linear + + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, + ansible.plugins.strategy.linear.StrategyModule): + pass diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py new file mode 100644 index 00000000..b7ee03bc --- /dev/null +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -0,0 +1,60 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import os.path +import sys + +# +# This is not the real Strategy implementation module, it simply exists as a +# proxy to the real module, which is loaded using Python's regular import +# mechanism, to prevent Ansible's PluginLoader from making up a fake name that +# results in ansible_mitogen plugin modules being loaded twice: once by +# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is +# stuffed into sys.modules even though attempting to import it will trigger an +# ImportError, and once under its canonical name, "ansible_mitogen.strategy". +# +# Therefore we have a proxy module that imports it under the real name, and +# sets up the duff PluginLoader-imported module to just contain objects from +# the real module, so duplicate types don't exist in memory, and things like +# debuggers and isinstance() work predictably. +# + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.strategy +import ansible.plugins.strategy.free + + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, + ansible.plugins.strategy.free.StrategyModule): + pass diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py new file mode 100644 index 00000000..3ef522b4 --- /dev/null +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -0,0 +1,60 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import os.path +import sys + +# +# This is not the real Strategy implementation module, it simply exists as a +# proxy to the real module, which is loaded using Python's regular import +# mechanism, to prevent Ansible's PluginLoader from making up a fake name that +# results in ansible_mitogen plugin modules being loaded twice: once by +# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is +# stuffed into sys.modules even though attempting to import it will trigger an +# ImportError, and once under its canonical name, "ansible_mitogen.strategy". +# +# Therefore we have a proxy module that imports it under the real name, and +# sets up the duff PluginLoader-imported module to just contain objects from +# the real module, so duplicate types don't exist in memory, and things like +# debuggers and isinstance() work predictably. +# + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.strategy +import ansible.plugins.strategy.linear + + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, + ansible.plugins.strategy.linear.StrategyModule): + pass diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 078ba921..1ad7099a 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -30,7 +30,6 @@ from __future__ import absolute_import import os import ansible.errors -import ansible.plugins.strategy.linear import ansible_mitogen.mixins import ansible_mitogen.process @@ -83,12 +82,12 @@ def wrap_connection_loader__get(name, play_context, new_stdin, **kwargs): return connection_loader__get(name, play_context, new_stdin, **kwargs) -class StrategyModule(ansible.plugins.strategy.linear.StrategyModule): +class StrategyMixin(object): """ - This strategy enhances the default "linear" strategy by arranging for - various Mitogen services to be initialized in the Ansible top-level - process, and for worker processes to grow support for using those top-level - services to communicate with and execute modules on remote hosts. + This mix-in enhances any built-in strategy by arranging for various Mitogen + services to be initialized in the Ansible top-level process, and for worker + processes to grow support for using those top-level services to communicate + with and execute modules on remote hosts. Mitogen: @@ -182,6 +181,6 @@ class StrategyModule(ansible.plugins.strategy.linear.StrategyModule): self._add_connection_plugin_path() self._install_wrappers() try: - return super(StrategyModule, self).run(iterator, play_context) + return super(StrategyMixin, self).run(iterator, play_context) finally: self._remove_wrappers() diff --git a/docs/ansible.rst b/docs/ansible.rst index 8d4dacc6..d164d85f 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -97,10 +97,12 @@ Installation [defaults] strategy_plugins = /path/to/mitogen-master/ansible_mitogen/plugins/strategy - strategy = mitogen + strategy = mitogen_linear The ``strategy`` key is optional. If omitted, you can set the - ``ANSIBLE_STRATEGY=mitogen`` environment variable on a per-run basis. + ``ANSIBLE_STRATEGY=mitogen_linear`` environment variable on a per-run basis. + Like ``mitogen_linear``, the ``mitogen_free`` strategy also exists to mimic + the built-in ``free`` strategy. 4. Cross your fingers and try it. @@ -146,8 +148,6 @@ Low Risk * Only the ``sudo`` become method is available, however adding new methods is straightforward, and eventually at least ``su`` will be included. -* The only supported strategy is ``linear``, which is Ansible's default. - * In some cases ``remote_tmp`` may not be respected. * The extension's performance benefits do not scale perfectly linearly with the diff --git a/examples/playbook/ansible.cfg b/examples/playbook/ansible.cfg index 81be60b1..479e42ab 100644 --- a/examples/playbook/ansible.cfg +++ b/examples/playbook/ansible.cfg @@ -1,7 +1,7 @@ [defaults] inventory = hosts strategy_plugins = ../../ansible_mitogen/plugins/strategy -strategy = mitogen +strategy = mitogen_linear library = modules retry_files_enabled = False forks = 50 From f6b82bb8db4c5baeb0cfd2ca346869f43878a528 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 07:22:24 +0100 Subject: [PATCH 087/140] examples: add make output to gitignore. --- examples/playbook/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 examples/playbook/.gitignore diff --git a/examples/playbook/.gitignore b/examples/playbook/.gitignore new file mode 100644 index 00000000..5d172bda --- /dev/null +++ b/examples/playbook/.gitignore @@ -0,0 +1,2 @@ +modules/binary_producing_junk +modules/binary_producing_json From 95ca29262a0d588f52068e6d7b37e16c7c5cde4c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 07:23:54 +0100 Subject: [PATCH 088/140] examples: add README.md to playbook/. --- examples/playbook/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 examples/playbook/README.md diff --git a/examples/playbook/README.md b/examples/playbook/README.md new file mode 100644 index 00000000..46fb3777 --- /dev/null +++ b/examples/playbook/README.md @@ -0,0 +1,9 @@ + +# playbooks Directory + +Although this lives under `examples/`, it is more or less an organically +growing collection of regression tests used for development relating to user +bug reports. + +This will be tidied up over time, meanwhile, the playbooks here are a useful +demonstrator for what does and doesn't work. From 047458a8b301c23a4b8197ce68b0a3b07b7f3f09 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 08:05:37 +0100 Subject: [PATCH 089/140] "examples": start adding structure to regression tests. --- ansible_mitogen/planner.py | 3 ++ examples/playbook/.gitignore | 4 +-- examples/playbook/Makefile | 4 +-- ... => action__low_level_execute_command.yml} | 4 --- examples/playbook/ansible.cfg | 3 +- examples/playbook/async_polling.yml | 4 --- examples/playbook/issue_109.yml | 3 -- examples/playbook/issue_113.yml | 3 -- examples/playbook/issue_118.yml | 2 -- examples/playbook/issue_122.yml | 1 - examples/playbook/issue_131.yml | 5 ---- examples/playbook/issue_140.yml | 5 ---- examples/playbook/issue_152.yml | 3 -- examples/playbook/issue_152b.yml | 3 -- examples/playbook/issue_154.yml | 3 -- examples/playbook/issue_174.yml | 4 --- examples/playbook/issue_177.yml | 4 --- examples/playbook/modules/bin_bash_module.sh | 5 ---- ...ule.sh => custom_bash_old_style_module.sh} | 4 +-- ...ule.sh => custom_bash_want_json_module.sh} | 0 ..._json.c => custom_binary_producing_json.c} | 0 ..._junk.c => custom_binary_producing_junk.c} | 0 .../modules/custom_binary_single_null | Bin 0 -> 1 bytes ...n.py => custom_python_json_args_module.py} | 0 ...e.py => custom_python_new_style_module.py} | 0 ...e.py => custom_python_want_json_module.py} | 0 examples/playbook/modules/single_null_binary | Bin 2 -> 0 bytes examples/playbook/non_python_modules.yml | 6 ---- examples/playbook/playbook__become_flags.yml | 27 ++++++++++++++++++ ...egate_to.yml => playbook__delegate_to.yml} | 4 --- ...ironment.yml => playbook__environment.yml} | 4 --- examples/playbook/runner.yml | 9 ++++++ ...yml => runner__builtin_command_module.yml} | 5 +--- .../runner__custom_bash_old_style_module.yml | 5 ++++ .../runner__custom_bash_want_json_module.yml | 5 ++++ .../runner__custom_binary_producing_json.yml | 5 ++++ .../runner__custom_binary_producing_junk.yml | 6 ++++ .../runner__custom_binary_single_null.yml | 6 ++++ ...runner__custom_python_json_args_module.yml | 5 ++++ ...runner__custom_python_new_style_module.yml | 5 ++++ ...runner__custom_python_want_json_module.yml | 5 ++++ 41 files changed, 90 insertions(+), 74 deletions(-) rename examples/playbook/{low_level_execute_command.yml => action__low_level_execute_command.yml} (96%) delete mode 100755 examples/playbook/modules/bin_bash_module.sh rename examples/playbook/modules/{old_style_module.sh => custom_bash_old_style_module.sh} (76%) rename examples/playbook/modules/{want_json_module.sh => custom_bash_want_json_module.sh} (100%) rename examples/playbook/modules/{binary_producing_json.c => custom_binary_producing_json.c} (100%) rename examples/playbook/modules/{binary_producing_junk.c => custom_binary_producing_junk.c} (100%) create mode 100644 examples/playbook/modules/custom_binary_single_null rename examples/playbook/modules/{json_args_python.py => custom_python_json_args_module.py} (100%) mode change 100644 => 100755 rename examples/playbook/modules/{python_new_style_module.py => custom_python_new_style_module.py} (100%) rename examples/playbook/modules/{python_want_json_module.py => custom_python_want_json_module.py} (100%) delete mode 100755 examples/playbook/modules/single_null_binary delete mode 100644 examples/playbook/non_python_modules.yml create mode 100644 examples/playbook/playbook__become_flags.yml rename examples/playbook/{delegate_to.yml => playbook__delegate_to.yml} (98%) rename examples/playbook/{environment.yml => playbook__environment.yml} (89%) create mode 100644 examples/playbook/runner.yml rename examples/playbook/{run_hostname_100_times.yml => runner__builtin_command_module.yml} (53%) create mode 100644 examples/playbook/runner__custom_bash_old_style_module.yml create mode 100644 examples/playbook/runner__custom_bash_want_json_module.yml create mode 100644 examples/playbook/runner__custom_binary_producing_json.yml create mode 100644 examples/playbook/runner__custom_binary_producing_junk.yml create mode 100644 examples/playbook/runner__custom_binary_single_null.yml create mode 100644 examples/playbook/runner__custom_python_json_args_module.yml create mode 100644 examples/playbook/runner__custom_python_new_style_module.yml create mode 100644 examples/playbook/runner__custom_python_want_json_module.yml diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 8fcd639c..b940fbbb 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -185,6 +185,9 @@ class ScriptPlanner(BinaryPlanner): detection and rewrite. """ def _rewrite_interpreter(self, interpreter, task_vars, templar): + if interpreter is None: + return None + key = u'ansible_%s_interpreter' % os.path.basename(interpreter).strip() try: return templar.template(task_vars[key].strip()) diff --git a/examples/playbook/.gitignore b/examples/playbook/.gitignore index 5d172bda..14f1a005 100644 --- a/examples/playbook/.gitignore +++ b/examples/playbook/.gitignore @@ -1,2 +1,2 @@ -modules/binary_producing_junk -modules/binary_producing_json +modules/custom_binary_producing_junk +modules/custom_binary_producing_json diff --git a/examples/playbook/Makefile b/examples/playbook/Makefile index 8c9e4480..f37789ee 100644 --- a/examples/playbook/Makefile +++ b/examples/playbook/Makefile @@ -1,4 +1,4 @@ all: \ - modules/binary_producing_junk \ - modules/binary_producing_json + modules/custom_binary_producing_junk \ + modules/custom_binary_producing_json diff --git a/examples/playbook/low_level_execute_command.yml b/examples/playbook/action__low_level_execute_command.yml similarity index 96% rename from examples/playbook/low_level_execute_command.yml rename to examples/playbook/action__low_level_execute_command.yml index 10fb8d79..419af489 100644 --- a/examples/playbook/low_level_execute_command.yml +++ b/examples/playbook/action__low_level_execute_command.yml @@ -1,11 +1,7 @@ ---- - # Verify the behaviour of _low_level_execute_command(). - hosts: all - gather_facts: false tasks: - # "echo -en" to test we actually hit bash shell too. - name: Run raw module without sudo raw: 'echo -en $((1 + 1))' diff --git a/examples/playbook/ansible.cfg b/examples/playbook/ansible.cfg index 479e42ab..2b3c6193 100644 --- a/examples/playbook/ansible.cfg +++ b/examples/playbook/ansible.cfg @@ -1,7 +1,8 @@ [defaults] inventory = hosts +gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy -strategy = mitogen_linear +#strategy = mitogen_linear library = modules retry_files_enabled = False forks = 50 diff --git a/examples/playbook/async_polling.yml b/examples/playbook/async_polling.yml index f4dcd96d..20aa211c 100644 --- a/examples/playbook/async_polling.yml +++ b/examples/playbook/async_polling.yml @@ -1,9 +1,5 @@ ---- - - hosts: all - gather_facts: false tasks: - - name: simulate long running op (3 sec), wait for up to 5 sec, poll every 1 sec command: /bin/sleep 2 async: 4 diff --git a/examples/playbook/issue_109.yml b/examples/playbook/issue_109.yml index 15e66c21..b47a225f 100644 --- a/examples/playbook/issue_109.yml +++ b/examples/playbook/issue_109.yml @@ -1,8 +1,5 @@ ---- - # Reproduction for issue #109. - hosts: all roles: - issue_109 - gather_facts: no diff --git a/examples/playbook/issue_113.yml b/examples/playbook/issue_113.yml index 870cd44a..d213ba02 100644 --- a/examples/playbook/issue_113.yml +++ b/examples/playbook/issue_113.yml @@ -1,7 +1,4 @@ ---- - - hosts: all - gather_facts: false tasks: - name: Get auth token diff --git a/examples/playbook/issue_118.yml b/examples/playbook/issue_118.yml index 0e2b6751..5b920db0 100644 --- a/examples/playbook/issue_118.yml +++ b/examples/playbook/issue_118.yml @@ -1,5 +1,3 @@ ---- - # issue #118 repro: chmod +x not happening during script upload # - name: saytrue diff --git a/examples/playbook/issue_122.yml b/examples/playbook/issue_122.yml index 4bba6ad6..d72ecf96 100644 --- a/examples/playbook/issue_122.yml +++ b/examples/playbook/issue_122.yml @@ -1,4 +1,3 @@ - - hosts: all tasks: - script: scripts/print_env.sh diff --git a/examples/playbook/issue_131.yml b/examples/playbook/issue_131.yml index 9c3aa0f4..a271c46b 100644 --- a/examples/playbook/issue_131.yml +++ b/examples/playbook/issue_131.yml @@ -1,13 +1,9 @@ ---- - # Hopeful reproduction for issue #131. # Run lots of steps (rather than just one) so WorkerProcess and suchlike # machinery is constantly recreated. - hosts: all - gather_facts: no tasks: - - shell: "true" - shell: "true" - shell: "true" @@ -58,4 +54,3 @@ - shell: "true" - shell: "true" - shell: "true" - diff --git a/examples/playbook/issue_140.yml b/examples/playbook/issue_140.yml index e540bcea..f9dc2d2b 100644 --- a/examples/playbook/issue_140.yml +++ b/examples/playbook/issue_140.yml @@ -1,11 +1,7 @@ ---- - # Reproduction for issue #140. - hosts: all - gather_facts: no tasks: - - name: Create file tree connection: local shell: > @@ -26,4 +22,3 @@ with_filetree: - filetree when: item.state == 'file' - diff --git a/examples/playbook/issue_152.yml b/examples/playbook/issue_152.yml index af7d257a..dff424b1 100644 --- a/examples/playbook/issue_152.yml +++ b/examples/playbook/issue_152.yml @@ -1,8 +1,5 @@ - - - hosts: all tasks: - - name: Make virtualenv pip: virtualenv: /tmp/issue_151_virtualenv diff --git a/examples/playbook/issue_152b.yml b/examples/playbook/issue_152b.yml index 617de96a..962f1d6f 100644 --- a/examples/playbook/issue_152b.yml +++ b/examples/playbook/issue_152b.yml @@ -1,4 +1,3 @@ - # issue #152 (b): local connections were not receiving # ansible_python_interpreter treatment, breaking virtualenvs. @@ -9,7 +8,5 @@ # - Run ansible-playbook ... with the virtualenv activated. Observe success. - hosts: all - gather_facts: false tasks: - - local_action: cloudformation_facts diff --git a/examples/playbook/issue_154.yml b/examples/playbook/issue_154.yml index d262abb3..28c476e7 100644 --- a/examples/playbook/issue_154.yml +++ b/examples/playbook/issue_154.yml @@ -1,6 +1,4 @@ - - hosts: all - gather_facts: no become: true vars: repo_baseurl: "http://myurl.com" @@ -11,7 +9,6 @@ - repo: demo-repo2 description: Misc packages url: "{{repo_baseurl}}/repo2" - tasks: - name: Create multiple yum repos yum_repository: diff --git a/examples/playbook/issue_174.yml b/examples/playbook/issue_174.yml index b68fdb24..c64cc70f 100644 --- a/examples/playbook/issue_174.yml +++ b/examples/playbook/issue_174.yml @@ -1,9 +1,5 @@ ---- - - hosts: all - gather_facts: false tasks: - name: add nginx ppa become: yes apt_repository: repo='ppa:nginx/stable' update_cache=yes - diff --git a/examples/playbook/issue_177.yml b/examples/playbook/issue_177.yml index ec7e3338..5137b73d 100644 --- a/examples/playbook/issue_177.yml +++ b/examples/playbook/issue_177.yml @@ -1,11 +1,7 @@ - - - hosts: all - gather_facts: false tasks: - name: copy repo configs copy: src=/etc/{{ item }} dest=/tmp/{{item}} mode=0644 with_items: - passwd - hosts - diff --git a/examples/playbook/modules/bin_bash_module.sh b/examples/playbook/modules/bin_bash_module.sh deleted file mode 100755 index 01abff0c..00000000 --- a/examples/playbook/modules/bin_bash_module.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -exec >/tmp/derp -echo "$1" -cat "$1" - diff --git a/examples/playbook/modules/old_style_module.sh b/examples/playbook/modules/custom_bash_old_style_module.sh similarity index 76% rename from examples/playbook/modules/old_style_module.sh rename to examples/playbook/modules/custom_bash_old_style_module.sh index 47e6afbd..144789cb 100755 --- a/examples/playbook/modules/old_style_module.sh +++ b/examples/playbook/modules/custom_bash_old_style_module.sh @@ -4,13 +4,13 @@ INPUT=$1 [ ! -r "$INPUT" ] && { - echo "Usage: $0 " >&2 + echo "Usage: $0 " >&2 exit 1 } echo "{" echo " \"changed\": false," echo " \"msg\": \"Here is my input\"," -echo " \"filname\": \"$INPUT\"," +echo " \"filename\": \"$INPUT\"," echo " \"input\": [\"$(cat $INPUT | tr \" \' )\"]" echo "}" diff --git a/examples/playbook/modules/want_json_module.sh b/examples/playbook/modules/custom_bash_want_json_module.sh similarity index 100% rename from examples/playbook/modules/want_json_module.sh rename to examples/playbook/modules/custom_bash_want_json_module.sh diff --git a/examples/playbook/modules/binary_producing_json.c b/examples/playbook/modules/custom_binary_producing_json.c similarity index 100% rename from examples/playbook/modules/binary_producing_json.c rename to examples/playbook/modules/custom_binary_producing_json.c diff --git a/examples/playbook/modules/binary_producing_junk.c b/examples/playbook/modules/custom_binary_producing_junk.c similarity index 100% rename from examples/playbook/modules/binary_producing_junk.c rename to examples/playbook/modules/custom_binary_producing_junk.c diff --git a/examples/playbook/modules/custom_binary_single_null b/examples/playbook/modules/custom_binary_single_null new file mode 100644 index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d GIT binary patch literal 1 IcmZPo000310RR91 literal 0 HcmV?d00001 diff --git a/examples/playbook/modules/json_args_python.py b/examples/playbook/modules/custom_python_json_args_module.py old mode 100644 new mode 100755 similarity index 100% rename from examples/playbook/modules/json_args_python.py rename to examples/playbook/modules/custom_python_json_args_module.py diff --git a/examples/playbook/modules/python_new_style_module.py b/examples/playbook/modules/custom_python_new_style_module.py similarity index 100% rename from examples/playbook/modules/python_new_style_module.py rename to examples/playbook/modules/custom_python_new_style_module.py diff --git a/examples/playbook/modules/python_want_json_module.py b/examples/playbook/modules/custom_python_want_json_module.py similarity index 100% rename from examples/playbook/modules/python_want_json_module.py rename to examples/playbook/modules/custom_python_want_json_module.py diff --git a/examples/playbook/modules/single_null_binary b/examples/playbook/modules/single_null_binary deleted file mode 100755 index 1f2a4f5ef3df7f7456d91c961da36fc58904f2f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2 JcmZSJ0ssIE01E&B diff --git a/examples/playbook/non_python_modules.yml b/examples/playbook/non_python_modules.yml deleted file mode 100644 index 5fb6cc75..00000000 --- a/examples/playbook/non_python_modules.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -- hosts: all - gather_facts: false - tasks: - - bin_bash_module: diff --git a/examples/playbook/playbook__become_flags.yml b/examples/playbook/playbook__become_flags.yml new file mode 100644 index 00000000..74ebf059 --- /dev/null +++ b/examples/playbook/playbook__become_flags.yml @@ -0,0 +1,27 @@ +# This must be run with FOO=2 set in the environment. + +# +# Test sudo_flags respects -E. +# + +- hosts: all + tasks: + - name: "without -E" + become: true + shell: "echo $FOO" + register: out + + - assert: + that: "out.stdout == ''" + +- hosts: all + become_flags: -E + tasks: + - name: "with -E" + become: true + shell: "set" + register: out2 + + - debug: msg={{out2}} + - assert: + that: "out2.stdout == '2'" diff --git a/examples/playbook/delegate_to.yml b/examples/playbook/playbook__delegate_to.yml similarity index 98% rename from examples/playbook/delegate_to.yml rename to examples/playbook/playbook__delegate_to.yml index b4f85112..dae2afd8 100644 --- a/examples/playbook/delegate_to.yml +++ b/examples/playbook/playbook__delegate_to.yml @@ -1,9 +1,5 @@ ---- - - hosts: all - gather_facts: false tasks: - # # delegate_to, no sudo # diff --git a/examples/playbook/environment.yml b/examples/playbook/playbook__environment.yml similarity index 89% rename from examples/playbook/environment.yml rename to examples/playbook/playbook__environment.yml index c24bf083..fd70174f 100644 --- a/examples/playbook/environment.yml +++ b/examples/playbook/playbook__environment.yml @@ -1,10 +1,7 @@ ---- # Ensure environment: is preserved during call. - hosts: all - gather_facts: false tasks: - - shell: echo $SOME_ENV environment: SOME_ENV: 123 @@ -14,4 +11,3 @@ - assert: that: "result.stdout == '123'" - diff --git a/examples/playbook/runner.yml b/examples/playbook/runner.yml new file mode 100644 index 00000000..fb8baa64 --- /dev/null +++ b/examples/playbook/runner.yml @@ -0,0 +1,9 @@ +- import_playbook: runner__builtin_command_module.yml +- import_playbook: runner__custom_bash_old_style_module.yml +- import_playbook: runner__custom_bash_want_json_module.yml +- import_playbook: runner__custom_binary_producing_json.yml +- import_playbook: runner__custom_binary_producing_junk.yml +- import_playbook: runner__custom_binary_single_null.yml +- import_playbook: runner__custom_python_json_args_module.yml +- import_playbook: runner__custom_python_new_style_module.yml +- import_playbook: runner__custom_python_want_json_module.yml diff --git a/examples/playbook/run_hostname_100_times.yml b/examples/playbook/runner__builtin_command_module.yml similarity index 53% rename from examples/playbook/run_hostname_100_times.yml rename to examples/playbook/runner__builtin_command_module.yml index 22c74138..c0b0c6e2 100644 --- a/examples/playbook/run_hostname_100_times.yml +++ b/examples/playbook/runner__builtin_command_module.yml @@ -1,8 +1,5 @@ ---- - - hosts: all - gather_facts: false tasks: - name: "Run hostname" command: hostname - with_sequence: start=1 end=100 + with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_bash_old_style_module.yml b/examples/playbook/runner__custom_bash_old_style_module.yml new file mode 100644 index 00000000..f7ebefa8 --- /dev/null +++ b/examples/playbook/runner__custom_bash_old_style_module.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_bash_old_style_module: + foo: true + with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_bash_want_json_module.yml b/examples/playbook/runner__custom_bash_want_json_module.yml new file mode 100644 index 00000000..bcf49b50 --- /dev/null +++ b/examples/playbook/runner__custom_bash_want_json_module.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_bash_want_json_module: + foo: true + with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_binary_producing_json.yml b/examples/playbook/runner__custom_binary_producing_json.yml new file mode 100644 index 00000000..bc0ced40 --- /dev/null +++ b/examples/playbook/runner__custom_binary_producing_json.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_binary_producing_json: + foo: true + with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_binary_producing_junk.yml b/examples/playbook/runner__custom_binary_producing_junk.yml new file mode 100644 index 00000000..b806752d --- /dev/null +++ b/examples/playbook/runner__custom_binary_producing_junk.yml @@ -0,0 +1,6 @@ +- hosts: all + tasks: + - custom_binary_producing_junk: + foo: true + with_sequence: start=1 end={{end|default(100)}} + ignore_errors: true diff --git a/examples/playbook/runner__custom_binary_single_null.yml b/examples/playbook/runner__custom_binary_single_null.yml new file mode 100644 index 00000000..a830e5fe --- /dev/null +++ b/examples/playbook/runner__custom_binary_single_null.yml @@ -0,0 +1,6 @@ +- hosts: all + tasks: + - custom_binary_single_null: + foo: true + with_sequence: start=1 end={{end|default(100)}} + ignore_errors: true diff --git a/examples/playbook/runner__custom_python_json_args_module.yml b/examples/playbook/runner__custom_python_json_args_module.yml new file mode 100644 index 00000000..f281184a --- /dev/null +++ b/examples/playbook/runner__custom_python_json_args_module.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_python_json_args_module: + foo: true + with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_python_new_style_module.yml b/examples/playbook/runner__custom_python_new_style_module.yml new file mode 100644 index 00000000..1a2421d3 --- /dev/null +++ b/examples/playbook/runner__custom_python_new_style_module.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_python_new_style_module: + foo: true + with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_python_want_json_module.yml b/examples/playbook/runner__custom_python_want_json_module.yml new file mode 100644 index 00000000..b149808f --- /dev/null +++ b/examples/playbook/runner__custom_python_want_json_module.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_python_want_json_module: + foo: true + with_sequence: start=1 end={{end|default(100)}} From eb402c8bd4b7722ce4d81796f4b1ac692192c4c9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 07:26:52 +0000 Subject: [PATCH 090/140] tests/bench: import wrapper script used for blog charts --- tests/bench/linux_record_cpu_net.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 tests/bench/linux_record_cpu_net.sh diff --git a/tests/bench/linux_record_cpu_net.sh b/tests/bench/linux_record_cpu_net.sh new file mode 100755 index 00000000..bc5c44ee --- /dev/null +++ b/tests/bench/linux_record_cpu_net.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# +# Wrap a run of Ansible playbook so that CPU usage counters and network +# activity are logged to files. +# + +[ ! "$1" ] && exit 1 +sudo tcpdump -w $1-out.cap -s 0 host k1.botanicus.net & +date +%s.%N > $1-task-clock.csv +perf stat -x, -I 25 -e task-clock --append -o $1-task-clock.csv ansible-playbook run_hostname_100_times.yml +sudo pkill -f tcpdump From 8ca8b43df1c0ba94739b1c03a65e215dd1d80944 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 07:30:06 +0000 Subject: [PATCH 091/140] tests/bench: import "slightly more reliable time" script --- tests/bench/megatime.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tests/bench/megatime.py diff --git a/tests/bench/megatime.py b/tests/bench/megatime.py new file mode 100755 index 00000000..0964424d --- /dev/null +++ b/tests/bench/megatime.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import sys +import os +import time + + +times = [] +for x in xrange(5): + t0 = time.time() + os.spawnvp(os.P_WAIT, sys.argv[1], sys.argv[1:]) + t = time.time() - t0 + times.append(t) + print '+++', t + +print 'all:', times +print 'min %s max %s diff %s' % (min(times), max(times), (max(times) - min(times))) From d2c009f70fee2a8daab0912c4da9f67d787dd213 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 08:33:52 +0100 Subject: [PATCH 092/140] "examples": rename regtest.py -> compare_output_test.py Incomplete, but getting better all the time. --- .../{regtest.py => compare_output_test.py} | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) rename examples/playbook/{regtest.py => compare_output_test.py} (79%) mode change 100644 => 100755 diff --git a/examples/playbook/regtest.py b/examples/playbook/compare_output_test.py old mode 100644 new mode 100755 similarity index 79% rename from examples/playbook/regtest.py rename to examples/playbook/compare_output_test.py index 15fc9e3c..1ad2f01f --- a/examples/playbook/regtest.py +++ b/examples/playbook/compare_output_test.py @@ -10,15 +10,14 @@ import tempfile LOG = logging.getLogger(__name__) suffixes = [ - '-m bin_bash_module', - '-m binary_producing_json', - '-m binary_producing_junk', - '-m old_style_module', - '-m python_new_style_module', - '-m python_want_json_module', - '-m single_null_binary', - '-m want_json_module', - '-m json_args_python', + '-m custom_bash_old_style_module', + '-m custom_bash_want_json_module', + '-m custom_binary_producing_json', + '-m custom_binary_producing_junk', + '-m custom_binary_single_null', + '-m custom_python_json_args_module', + '-m custom_python_new_style_module', + '-m custom_python_want_json_module', '-m setup', ] From 1fab6d9c25f0254f1bf4376fc76da05378703e46 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 08:59:36 +0100 Subject: [PATCH 093/140] ansible: permit execve() of temporary files on Linux. --- ansible_mitogen/runner.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index f5cb1283..c9ab8427 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -59,6 +59,19 @@ ansible.module_utils.basic._ANSIBLE_ARGS = '{}' LOG = logging.getLogger(__name__) +def reopen_readonly(fp): + """ + Replace the file descriptor belonging to the file object `fp` with one + open on the same file (`fp.name`), but opened with :py:data:`os.O_RDONLY`. + This enables temporary files to be executed on Linux, which usually theows + ``ETXTBUSY`` if any writeable handle exists pointing to a file passed to + `execve()`. + """ + fd = os.open(fp.name, os.O_RDONLY) + os.dup2(fd, fp.fileno()) + os.close(fd) + + class Runner(object): """ Ansible module runner. After instantiation (with kwargs supplied by the @@ -189,6 +202,7 @@ class ProgramRunner(Runner): self.program_fp.write(self._get_program()) self.program_fp.flush() os.chmod(self.program_fp.name, int('0700', 8)) + reopen_readonly(self.program_fp) def _get_program(self): """ @@ -244,6 +258,7 @@ class ArgsFileRunner(Runner): ) self.args_fp.write(self._get_args_contents()) self.args_fp.flush() + reopen_readonly(self.program_fp) def _get_args_contents(self): """ From fe9bf1d81ddfa2beeee5a2296ecefc67c1ac4506 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 08:08:14 +0000 Subject: [PATCH 094/140] ansible: match Ansible behaviour when script lacks interpreter line. --- ansible_mitogen/planner.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index b940fbbb..11531e4a 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -184,25 +184,27 @@ class ScriptPlanner(BinaryPlanner): Common functionality for script module planners -- handle interpreter detection and rewrite. """ - def _rewrite_interpreter(self, interpreter, task_vars, templar): - if interpreter is None: - return None - + def _rewrite_interpreter(self, invocation, interpreter): key = u'ansible_%s_interpreter' % os.path.basename(interpreter).strip() try: - return templar.template(task_vars[key].strip()) + template = invocation.task_vars[key].strip() + return invocation.templar.template(template) except KeyError: return interpreter def plan(self, invocation): kwargs = super(ScriptPlanner, self).plan(invocation) interpreter, arg = parse_script_interpreter(invocation.module_source) + if interpreter is None: + raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % ( + invocation.module_name, + )) + return dict(kwargs, interpreter_arg=arg, interpreter=self._rewrite_interpreter( interpreter=interpreter, - task_vars=invocation.task_vars, - templar=invocation.templar, + invocation=invocation ) ) @@ -305,6 +307,7 @@ _planners = [ NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' CRASHED_MSG = 'Mitogen: internal error: ' +NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line' def get_module_data(name): From 293567e9d58a2ff841d37ad18fb1ed90ccb71fb3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 08:24:17 +0000 Subject: [PATCH 095/140] ansible: fix low_level_execute_command regression. --- ansible_mitogen/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 270f3244..20f69c19 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -215,9 +215,9 @@ def exec_command(cmd, in_data='', chdir=None, shell=None): (return code, stdout bytes, stderr bytes) """ assert isinstance(cmd, basestring) - return _exec_command( + return exec_args( args=[get_user_shell(), '-c', cmd], - in_data=in_Data, + in_data=in_data, chdir=chdir, shell=shell, ) From 20044ba95640c247f6074ae1b146261a882a1c01 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 09:26:52 +0100 Subject: [PATCH 096/140] "examples": import all.yml --- examples/playbook/all.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 examples/playbook/all.yml diff --git a/examples/playbook/all.yml b/examples/playbook/all.yml new file mode 100644 index 00000000..c6dcb95b --- /dev/null +++ b/examples/playbook/all.yml @@ -0,0 +1,7 @@ + +# +# This playbook imports all tests that are known to work at present. +# + +- import_playbook: action__low_level_execute_command.yml +- import_playbook: runner.yml From e2542c1683e5b0917050b8d2af06c8eef06cc571 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 09:54:48 +0100 Subject: [PATCH 097/140] "examples": add perl script regression tests. --- .../modules/custom_perl_json_args_module.pl | 15 +++++++++++++ .../modules/custom_perl_want_json_module.pl | 21 +++++++++++++++++++ examples/playbook/runner.yml | 2 ++ .../runner__custom_perl_json_args_module.yml | 5 +++++ .../runner__custom_perl_want_json_module.yml | 5 +++++ 5 files changed, 48 insertions(+) create mode 100644 examples/playbook/modules/custom_perl_json_args_module.pl create mode 100644 examples/playbook/modules/custom_perl_want_json_module.pl create mode 100644 examples/playbook/runner__custom_perl_json_args_module.yml create mode 100644 examples/playbook/runner__custom_perl_want_json_module.yml diff --git a/examples/playbook/modules/custom_perl_json_args_module.pl b/examples/playbook/modules/custom_perl_json_args_module.pl new file mode 100644 index 00000000..c999ca6c --- /dev/null +++ b/examples/playbook/modules/custom_perl_json_args_module.pl @@ -0,0 +1,15 @@ +#!/usr/bin/perl + +binmode STDOUT, ":utf8"; +use utf8; + +use JSON; + +my $json_args = <<'END_MESSAGE'; +<> +END_MESSAGE + +print encode_json({ + message => "I am a perl script! Here is my input.", + input => [decode_json($json_args)] +}); diff --git a/examples/playbook/modules/custom_perl_want_json_module.pl b/examples/playbook/modules/custom_perl_want_json_module.pl new file mode 100644 index 00000000..8b45e5b4 --- /dev/null +++ b/examples/playbook/modules/custom_perl_want_json_module.pl @@ -0,0 +1,21 @@ +#!/usr/bin/perl + +binmode STDOUT, ":utf8"; +use utf8; + +my $WANT_JSON = 1; + +use JSON; + +my $json; +{ + local $/; #Enable 'slurp' mode + open my $fh, "<", $ARGV[0]; + $json_args = <$fh>; + close $fh; +} + +print encode_json({ + message => "I am a want JSON perl script! Here is my input.", + input => [decode_json($json_args)] +}); diff --git a/examples/playbook/runner.yml b/examples/playbook/runner.yml index fb8baa64..4a2fed81 100644 --- a/examples/playbook/runner.yml +++ b/examples/playbook/runner.yml @@ -4,6 +4,8 @@ - import_playbook: runner__custom_binary_producing_json.yml - import_playbook: runner__custom_binary_producing_junk.yml - import_playbook: runner__custom_binary_single_null.yml +- import_playbook: runner__custom_perl_json_args_module.yml +- import_playbook: runner__custom_perl_want_json_module.yml - import_playbook: runner__custom_python_json_args_module.yml - import_playbook: runner__custom_python_new_style_module.yml - import_playbook: runner__custom_python_want_json_module.yml diff --git a/examples/playbook/runner__custom_perl_json_args_module.yml b/examples/playbook/runner__custom_perl_json_args_module.yml new file mode 100644 index 00000000..1ee24ccb --- /dev/null +++ b/examples/playbook/runner__custom_perl_json_args_module.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_perl_json_args_module: + foo: true + with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_perl_want_json_module.yml b/examples/playbook/runner__custom_perl_want_json_module.yml new file mode 100644 index 00000000..4b821bef --- /dev/null +++ b/examples/playbook/runner__custom_perl_want_json_module.yml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + - custom_perl_want_json_module: + foo: true + with_sequence: start=1 end={{end|default(100)}} From a731be32a2ae92c590bf914d605c7f98c14ecbf3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 10:37:23 +0100 Subject: [PATCH 098/140] ansible: use _ansible_shell_executable in ScriptRunner. Now we match Ansible error output and exit status. Ansible: $ ansible localhost -e end=2 -m custom_binary_single_null localhost | FAILED! => { "changed": false, "module_stderr": "Shared connection to localhost closed.\r\n", "module_stdout": "/bin/sh: /Users/dmw/.ansible/tmp/ansible-tmp-1522661797.42-158833651208060/custom_binary_single_null: cannot execute binary file\r\n", "msg": "MODULE FAILURE", "rc": 126 } Mitogen now: localhost | FAILED! => { "changed": false, "module_stderr": "/bin/sh: /var/folders/gw/f6w3dgy16fsg5y4kdthbqycw0000gn/T/ansible_mitogenAYF8LM-binary: cannot execute binary file\n", "module_stdout": "", "msg": "MODULE FAILURE", "rc": 126 } Previously: localhost | FAILED! => { "changed": false, "module_stderr": ": [Errno 8] Exec format error", "module_stdout": "", "msg": "MODULE FAILURE", "rc": 1 } --- ansible_mitogen/runner.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index c9ab8427..6cfa4166 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -214,7 +214,11 @@ class ProgramRunner(Runner): ) def _get_program_args(self): - return [self.program_fp.name] + return [ + self.args['_ansible_shell_executable'], + '-c', + self.program_fp.name + ] def revert(self): """ @@ -229,6 +233,7 @@ class ProgramRunner(Runner): args=self._get_program_args(), ) except Exception, e: + LOG.exception('While running %s', self._get_program_args()) return { 'rc': 1, 'stdout': '', @@ -267,7 +272,11 @@ class ArgsFileRunner(Runner): return json.dumps(self.args) def _get_program_args(self): - return [self.program_fp.name, self.args_fp.name] + return [ + self.args['_ansible_shell_executable'], + '-c', + "%s %s" % (self.program_fp.name, self.args_fp.name), + ] def revert(self): """ From 0e648dbd53b1abbdc2b18c6d7e83834bba336910 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 2 Apr 2018 11:35:33 +0100 Subject: [PATCH 099/140] ansible: tidy up planner.py. --- ansible_mitogen/planner.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 11531e4a..8d216af1 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -53,6 +53,9 @@ import ansible_mitogen.services LOG = logging.getLogger(__name__) +NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' +CRASHED_MSG = 'Mitogen: internal error: ' +NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line' def parse_script_interpreter(source): @@ -62,7 +65,7 @@ def parse_script_interpreter(source): :returns: Tuple of `(interpreter, arg)`, where `intepreter` is the script - interpreter and `arg` is its solve argument if present, otherwise + interpreter and `arg` is its sole argument if present, otherwise :py:data:`None`. """ # Linux requires first 2 bytes with no whitespace, pretty sure it's the @@ -305,11 +308,6 @@ _planners = [ ] -NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' -CRASHED_MSG = 'Mitogen: internal error: ' -NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line' - - def get_module_data(name): path = module_loader.find_plugin(name, '') with open(path, 'rb') as fp: From 38311336e16aed4988f1cdcd40de5effc87ceeaa Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 3 Apr 2018 11:06:05 +0100 Subject: [PATCH 100/140] docs: link to Ansible video demo --- docs/ansible.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index d164d85f..a5a09eb6 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -57,6 +57,19 @@ quickly as possible. relating to those files in cross-account scenarios are entirely avoided. +Demo +---- + +This demonstrates Ansible running a subset of the Mitogen integration tests +concurrent to an equivalent run using the extension. + +.. raw:: html + + + + Testimonials ------------ @@ -214,8 +227,8 @@ Behavioural Differences release. -Demo ----- +Sample Profiles +--------------- Local VM connection ~~~~~~~~~~~~~~~~~~~ From de5028ac18ba1c2f6b6a992d41774fc34361c7df Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 11:00:24 +0100 Subject: [PATCH 101/140] issue #164: arrange for DebOps common.yml to run under Travis. --- .travis.yml | 6 +++++- .travis/debops_tests.sh | 34 ++++++++++++++++++++++++++++++++++ .travis/mitogen_tests.sh | 4 ++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100755 .travis/debops_tests.sh create mode 100644 .travis/mitogen_tests.sh diff --git a/.travis.yml b/.travis.yml index 815192a6..ca59ff82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,15 @@ cache: pip python: - "2.7" +env: +- MODE=mitogen +- MODE=debops + install: - pip install -r dev_requirements.txt script: -- MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/test +- ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh services: - docker diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh new file mode 100755 index 00000000..0e3e3607 --- /dev/null +++ b/.travis/debops_tests.sh @@ -0,0 +1,34 @@ +#!/bin/bash -ex +# Run some invocations of DebOps. + +TMPDIR="/tmp/debops-$$" +TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" + +function on_exit() +{ + [ "$KEEP" ] || { + rm -rvf "$TMPDIR" || true + docker kill target || true + } +} + +trap on_exit EXIT +mkdir "$TMPDIR" + +docker run --rm --detach --name=target d2mw/mitogen-test /bin/sleep 86400 + +pip install -U debops==0.7.2 ansible==2.4.3.0 +debops-init "$TMPDIR/project" +cd "$TMPDIR/project" + +cat > .debops.cfg <<-EOF +[ansible defaults] +strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy +strategy = mitogen_linear +EOF + +cat >> ansible/inventory/hosts <<-EOF +target ansible_python_interpreter=/usr/bin/python2.7 ansible_connection=docker +EOF + +debops common diff --git a/.travis/mitogen_tests.sh b/.travis/mitogen_tests.sh new file mode 100644 index 00000000..01d008b7 --- /dev/null +++ b/.travis/mitogen_tests.sh @@ -0,0 +1,4 @@ +#!/bin/bash -ex +# Run the Mitogen tests. + +MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/test From 5bf566466749eeeaa3ab695101e7f20ba0a38e0b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 12:30:48 +0100 Subject: [PATCH 102/140] issue #164: speed up DH generation step --- .travis/debops_tests.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh index 0e3e3607..4e0a3557 100755 --- a/.travis/debops_tests.sh +++ b/.travis/debops_tests.sh @@ -27,8 +27,13 @@ strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy strategy = mitogen_linear EOF -cat >> ansible/inventory/hosts <<-EOF -target ansible_python_interpreter=/usr/bin/python2.7 ansible_connection=docker +cat > ansible/inventory/host_vars/target.yml <<-EOF +ansible_connection: docker +ansible_python_interpreter: /usr/bin/python2.7 + +# Speed up slow DH generation. +dhparam__bits: [128, 64] EOF +echo target >> ansible/inventory/hosts debops common From 2128ffafce4178f8ea0af86c09a929dcff466ff3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 12:35:07 +0100 Subject: [PATCH 103/140] issue #164: cure type error. --- .travis/debops_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh index 4e0a3557..4698cd8f 100755 --- a/.travis/debops_tests.sh +++ b/.travis/debops_tests.sh @@ -32,7 +32,7 @@ ansible_connection: docker ansible_python_interpreter: /usr/bin/python2.7 # Speed up slow DH generation. -dhparam__bits: [128, 64] +dhparam__bits: ["128", "64"] EOF echo target >> ansible/inventory/hosts From 4c842751d00fa5ca5f7828e9b09715245c3c3f8b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 12:47:53 +0100 Subject: [PATCH 104/140] issue #164: run twice to make timing comparable to old reports --- .travis/debops_tests.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh index 4698cd8f..d5cac2cb 100755 --- a/.travis/debops_tests.sh +++ b/.travis/debops_tests.sh @@ -15,8 +15,13 @@ function on_exit() trap on_exit EXIT mkdir "$TMPDIR" + +echo travis_fold:start:docker_setup docker run --rm --detach --name=target d2mw/mitogen-test /bin/sleep 86400 +echo travis_fold:end:docker_setup + +echo travis_fold:start:job_setup pip install -U debops==0.7.2 ansible==2.4.3.0 debops-init "$TMPDIR/project" cd "$TMPDIR/project" @@ -36,4 +41,14 @@ dhparam__bits: ["128", "64"] EOF echo target >> ansible/inventory/hosts -debops common +echo travis_fold:end:job_setup + + +echo travis_fold:start:first_run +/usr/bin/time debops common +echo travis_fold:end:first_run + + +echo travis_fold:start:second_run +/usr/bin/time debops common +echo travis_fold:end:second_run From 058ddeee581abefe7c1187287fa77ac7ef874385 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 12:59:22 +0100 Subject: [PATCH 105/140] issue #164: run against 4 targets. --- .travis/debops_tests.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh index d5cac2cb..9a53c9e8 100755 --- a/.travis/debops_tests.sh +++ b/.travis/debops_tests.sh @@ -6,10 +6,12 @@ TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" function on_exit() { + echo travis_fold:start:cleanup [ "$KEEP" ] || { rm -rvf "$TMPDIR" || true docker kill target || true } + echo travis_fold:end:cleanup } trap on_exit EXIT @@ -17,7 +19,10 @@ mkdir "$TMPDIR" echo travis_fold:start:docker_setup -docker run --rm --detach --name=target d2mw/mitogen-test /bin/sleep 86400 +docker run --rm --detach --name=target1 d2mw/mitogen-test /bin/sleep 86400 +docker run --rm --detach --name=target2 d2mw/mitogen-test /bin/sleep 86400 +docker run --rm --detach --name=target3 d2mw/mitogen-test /bin/sleep 86400 +docker run --rm --detach --name=target4 d2mw/mitogen-test /bin/sleep 86400 echo travis_fold:end:docker_setup @@ -32,7 +37,7 @@ strategy_plugins = ${TRAVIS_BUILD_DIR}/ansible_mitogen/plugins/strategy strategy = mitogen_linear EOF -cat > ansible/inventory/host_vars/target.yml <<-EOF +cat > ansible/inventory/group_vars/debops_all_hosts.yml <<-EOF ansible_connection: docker ansible_python_interpreter: /usr/bin/python2.7 @@ -40,7 +45,12 @@ ansible_python_interpreter: /usr/bin/python2.7 dhparam__bits: ["128", "64"] EOF -echo target >> ansible/inventory/hosts +cat ansible/inventory/hosts <<-EOF +target1 +target2 +target3 +target4 +EOF echo travis_fold:end:job_setup From b9afde0e61b2fc422031b8ef6b12e04dec292300 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 13:11:35 +0100 Subject: [PATCH 106/140] issue #164: typo. --- .travis/debops_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh index 9a53c9e8..d099a443 100755 --- a/.travis/debops_tests.sh +++ b/.travis/debops_tests.sh @@ -45,7 +45,7 @@ ansible_python_interpreter: /usr/bin/python2.7 dhparam__bits: ["128", "64"] EOF -cat ansible/inventory/hosts <<-EOF +cat > ansible/inventory/hosts <<-EOF target1 target2 target3 From 8249fa2019a638b1915617c8d0f08ff9b616796b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 13:11:35 +0100 Subject: [PATCH 107/140] issue #164: typo x2. --- .travis/debops_tests.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh index d099a443..f77b9d25 100755 --- a/.travis/debops_tests.sh +++ b/.travis/debops_tests.sh @@ -9,7 +9,10 @@ function on_exit() echo travis_fold:start:cleanup [ "$KEEP" ] || { rm -rvf "$TMPDIR" || true - docker kill target || true + docker kill target1 || true + docker kill target2 || true + docker kill target3 || true + docker kill target4 || true } echo travis_fold:end:cleanup } @@ -45,7 +48,7 @@ ansible_python_interpreter: /usr/bin/python2.7 dhparam__bits: ["128", "64"] EOF -cat > ansible/inventory/hosts <<-EOF +cat >> ansible/inventory/hosts <<-EOF target1 target2 target3 From 4805f3cf3694fcb399aed964bd317ac80488a1b8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 14:04:22 +0100 Subject: [PATCH 108/140] issue #164: slightly flatten docker image layers --- tests/build_docker_image.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/build_docker_image.py b/tests/build_docker_image.py index 99a34ab0..68ba4100 100755 --- a/tests/build_docker_image.py +++ b/tests/build_docker_image.py @@ -11,7 +11,7 @@ DOCKERFILE = r""" FROM debian:stable RUN apt-get update RUN \ - apt-get install -y python2.7 openssh-server sudo rsync git && \ + apt-get install -y python2.7 openssh-server sudo rsync git strace && \ apt-get clean RUN \ mkdir /var/run/sshd && \ @@ -25,7 +25,8 @@ RUN \ ( echo 'root:x' | chpasswd; ) && \ ( echo 'has-sudo:y' | chpasswd; ) && \ ( echo 'has-sudo-nopw:y' | chpasswd; ) && \ - mkdir ~has-sudo-pubkey/.ssh + mkdir ~has-sudo-pubkey/.ssh && \ + { echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; } COPY data/docker/has-sudo-pubkey.key.pub /home/has-sudo-pubkey/.ssh/authorized_keys RUN \ @@ -41,7 +42,6 @@ RUN echo "export VISIBLE=now" >> /etc/profile EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] -RUN apt-get install -y strace && apt-get clean && { echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; } """ From e0381606af35367f57d1896cdf310904c40e70fd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 14:05:57 +0100 Subject: [PATCH 109/140] Ensure remote_tmp is respected everywhere. Logic is still somewhat different from Ansible: we don't have to care about sudo/non-sudo cases, etc. --- ansible_mitogen/mixins.py | 21 ++++++++++++++------- ansible_mitogen/planner.py | 6 +++++- ansible_mitogen/runner.py | 25 +++++++++++++++++++------ docs/ansible.rst | 2 -- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 56eaa281..094c4510 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -177,6 +177,18 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ assert False, "_is_pipelining_enabled() should never be called." + def _get_remote_tmp(self): + """ + Mitogen-only: return the 'remote_tmp' setting. + """ + try: + s = self._connection._shell.get_option('remote_tmp') + except AttributeError: + # Required for <2.4.x. + s = '~/.ansible' + + return self._remote_expand_user(s) + def _make_tmp_path(self, remote_user=None): """ Replace the base implementation's use of shell to implement mkdtemp() @@ -184,18 +196,12 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) - try: - remote_tmp = self._connection._shell.get_option('remote_tmp') - except AttributeError: - # Required for <2.4.x. - remote_tmp = '~/.ansible' - # _make_tmp_path() is basically a global stashed away as Shell.tmpdir. # The copy action plugin violates layering and grabs this attribute # directly. self._connection._shell.tmpdir = self.call( ansible_mitogen.helpers.make_temp_directory, - base_dir=self._remote_expand_user(remote_tmp), + base_dir=self._get_remote_tmp(), ) LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) self._cleanup_remote_tmp = True @@ -313,6 +319,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): connection=self._connection, module_name=mitogen.utils.cast(module_name), module_args=mitogen.utils.cast(module_args), + remote_tmp=mitogen.utils.cast(self._get_remote_tmp()), task_vars=task_vars, templar=self._templar, env=mitogen.utils.cast(env), diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 8d216af1..93461584 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -92,7 +92,7 @@ class Invocation(object): helpers.run_module() or helpers.run_module_async() in the target context. """ def __init__(self, action, connection, module_name, module_args, - task_vars, templar, env, wrap_async): + remote_tmp, task_vars, templar, env, wrap_async): #: ActionBase instance invoking the module. Required to access some #: output postprocessing methods that don't belong in ActionBase at #: all. @@ -104,6 +104,9 @@ class Invocation(object): self.module_name = module_name #: Final module arguments. self.module_args = module_args + #: Value of 'remote_tmp' parameter, to allow target to create temporary + #: files in correct location. + self.remote_tmp = remote_tmp #: Task variables, needed to extract ansible_*_interpreter. self.task_vars = task_vars #: Templar, needed to extract ansible_*_interpreter. @@ -179,6 +182,7 @@ class BinaryPlanner(Planner): 'path': invocation.module_path, 'args': invocation.module_args, 'env': invocation.env, + 'remote_tmp': invocation.remote_tmp, } diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 6cfa4166..475e1b6d 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -40,6 +40,7 @@ import cStringIO import json import logging import os +import shutil import sys import tempfile import types @@ -81,16 +82,25 @@ class Runner(object): Subclasses may override `_run`()` and extend `setup()` and `revert()`. """ - def __init__(self, module, raw_params=None, args=None, env=None): + def __init__(self, module, remote_tmp, raw_params=None, args=None, env=None): if args is None: args = {} if raw_params is not None: args['_raw_params'] = raw_params self.module = module + self.remote_tmp = os.path.expanduser(remote_tmp) self.raw_params = raw_params self.args = args self.env = env + self._temp_dir = None + + def get_temp_dir(self): + if not self._temp_dir: + self._temp_dir = ansible_mitogen.helpers.make_temp_directory( + self.remote_tmp, + ) + return self._temp_dir def setup(self): """ @@ -105,6 +115,8 @@ class Runner(object): implementation simply restores the original environment. """ self._env.revert() + if self._temp_dir: + shutil.rmtree(self._temp_dir) def _run(self): """ @@ -195,9 +207,9 @@ class ProgramRunner(Runner): Create a temporary file containing the program code. The code is fetched via :meth:`_get_program`. """ - self.program_fp = tempfile.NamedTemporaryFile( - prefix='ansible_mitogen', - suffix='-binary', + self.program_fp = open( + os.path.join(self.get_temp_dir(), self.module), + 'wb' ) self.program_fp.write(self._get_program()) self.program_fp.flush() @@ -224,8 +236,8 @@ class ProgramRunner(Runner): """ Delete the temporary program file. """ - super(ProgramRunner, self).revert() self.program_fp.close() + super(ProgramRunner, self).revert() def _run(self): try: @@ -260,6 +272,7 @@ class ArgsFileRunner(Runner): self.args_fp = tempfile.NamedTemporaryFile( prefix='ansible_mitogen', suffix='-args', + dir=self.get_temp_dir(), ) self.args_fp.write(self._get_args_contents()) self.args_fp.flush() @@ -282,8 +295,8 @@ class ArgsFileRunner(Runner): """ Delete the temporary argument file. """ - super(ArgsFileRunner, self).revert() self.args_fp.close() + super(ArgsFileRunner, self).revert() class BinaryRunner(ArgsFileRunner, ProgramRunner): diff --git a/docs/ansible.rst b/docs/ansible.rst index a5a09eb6..d0b309f3 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -161,8 +161,6 @@ Low Risk * Only the ``sudo`` become method is available, however adding new methods is straightforward, and eventually at least ``su`` will be included. -* In some cases ``remote_tmp`` may not be respected. - * The extension's performance benefits do not scale perfectly linearly with the number of targets. This is a subject of ongoing investigation and improvements will appear in time. From aa8d7a02501ebaa60a874c92452e8144e9b571db Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 15:38:39 +0100 Subject: [PATCH 110/140] issue #164: verify remote_tmp respected by code running remotely. --- examples/playbook/ansible.cfg | 3 +++ .../playbook/modules/bash_return_paths.sh | 19 +++++++++++++++++++ examples/playbook/runner.yml | 1 + examples/playbook/runner__remote_tmp.yml | 16 ++++++++++++++++ 4 files changed, 39 insertions(+) create mode 100755 examples/playbook/modules/bash_return_paths.sh create mode 100644 examples/playbook/runner__remote_tmp.yml diff --git a/examples/playbook/ansible.cfg b/examples/playbook/ansible.cfg index 2b3c6193..e9ecb706 100644 --- a/examples/playbook/ansible.cfg +++ b/examples/playbook/ansible.cfg @@ -7,6 +7,9 @@ library = modules retry_files_enabled = False forks = 50 +# Required by runner__remote_tmp.yml +remote_tmp = ~/.ansible/mitogen-tests/ + [ssh_connection] ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s pipelining = True diff --git a/examples/playbook/modules/bash_return_paths.sh b/examples/playbook/modules/bash_return_paths.sh new file mode 100755 index 00000000..d6282084 --- /dev/null +++ b/examples/playbook/modules/bash_return_paths.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# I am an Ansible WANT_JSON module that returns the paths to its argv[0] and +# args file. + +INPUT="$1" + +[ ! -r "$INPUT" ] && { + echo "Usage: $0 " >&2 + exit 1 +} + +echo "{" +echo " \"changed\": false," +echo " \"msg\": \"Here is my input\"," +echo " \"input\": [$(< $INPUT)]," +echo " \"argv0\": \"$0\"," +echo " \"argv1\": \"$1\"" +echo "}" diff --git a/examples/playbook/runner.yml b/examples/playbook/runner.yml index 4a2fed81..51583ba3 100644 --- a/examples/playbook/runner.yml +++ b/examples/playbook/runner.yml @@ -9,3 +9,4 @@ - import_playbook: runner__custom_python_json_args_module.yml - import_playbook: runner__custom_python_new_style_module.yml - import_playbook: runner__custom_python_want_json_module.yml +- import_playbook: runner__remote_tmp.yml diff --git a/examples/playbook/runner__remote_tmp.yml b/examples/playbook/runner__remote_tmp.yml new file mode 100644 index 00000000..74f18480 --- /dev/null +++ b/examples/playbook/runner__remote_tmp.yml @@ -0,0 +1,16 @@ +# +# The ansible.cfg remote_tmp setting should be copied to the target and used +# when generating temporary paths created by the runner.py code executing +# remotely. +# +- hosts: all + gather_facts: true + tasks: + - bash_return_paths: + register: output + + - assert: + that: output.argv0.startswith('%s/.ansible/mitogen-tests/' % ansible_user_dir) + + - assert: + that: output.argv1.startswith('%s/.ansible/mitogen-tests/' % ansible_user_dir) From 49aa8834b020e9b3c03bc6fd6c2517de4fc1817b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 15:54:31 +0100 Subject: [PATCH 111/140] issue #164: split "examples" out into regression/integration tests. --- examples/playbook/README.md | 9 --------- {examples/playbook => tests/ansible}/.gitignore | 0 {examples/playbook => tests/ansible}/Makefile | 0 tests/ansible/README.md | 8 ++++++++ {examples/playbook => tests/ansible}/all.yml | 0 {examples/playbook => tests/ansible}/ansible.cfg | 3 +-- .../ansible}/compare_output_test.py | 0 .../ansible}/gcloud-ansible-playbook.py | 0 {examples/playbook => tests/ansible}/hosts | 0 {examples/playbook => tests/ansible}/hosts.docker | 0 .../action__low_level_execute_command.yml | 0 .../ansible/integration}/async_polling.yml | 0 .../ansible/integration}/playbook__become_flags.yml | 0 .../ansible/integration}/playbook__delegate_to.yml | 0 .../ansible/integration}/playbook__environment.yml | 0 .../ansible/integration}/runner.yml | 0 .../integration}/runner__builtin_command_module.yml | 2 +- .../runner__custom_bash_old_style_module.yml | 0 .../runner__custom_bash_want_json_module.yml | 0 .../runner__custom_binary_producing_json.yml | 0 .../runner__custom_binary_producing_junk.yml | 0 .../runner__custom_binary_single_null.yml | 0 .../runner__custom_perl_json_args_module.yml | 0 .../runner__custom_perl_want_json_module.yml | 0 .../runner__custom_python_json_args_module.yml | 0 .../runner__custom_python_new_style_module.yml | 0 .../runner__custom_python_want_json_module.yml | 0 .../ansible/integration}/runner__remote_tmp.yml | 0 .../ansible}/modules/bash_return_paths.sh | 0 .../modules/custom_bash_old_style_module.sh | 0 .../modules/custom_bash_want_json_module.sh | 0 .../ansible}/modules/custom_binary_producing_json.c | 0 .../ansible}/modules/custom_binary_producing_junk.c | 0 .../ansible}/modules/custom_binary_single_null | Bin .../modules/custom_perl_json_args_module.pl | 0 .../modules/custom_perl_want_json_module.pl | 0 .../modules/custom_python_json_args_module.py | 0 .../modules/custom_python_new_style_module.py | 0 .../modules/custom_python_want_json_module.py | 0 .../ansible/regression}/issue_109.yml | 0 .../ansible/regression}/issue_113.yml | 0 .../ansible/regression}/issue_118.yml | 0 .../ansible/regression}/issue_122.yml | 0 .../ansible/regression}/issue_131.yml | 0 .../ansible/regression}/issue_140.yml | 0 .../ansible/regression}/issue_152.yml | 0 .../ansible/regression}/issue_152b.yml | 0 .../ansible/regression}/issue_154.yml | 0 .../ansible/regression}/issue_174.yml | 0 .../ansible/regression}/issue_177.yml | 0 .../regression}/roles/issue_109/tasks/main.yml | 0 .../roles/issue_109_add_ssh_key/tasks/main.yml | 0 .../roles/issue_109_gather_facts/tasks/main.yml | 0 .../ansible/regression}/scripts/issue_118_saytrue | 0 .../ansible/regression}/scripts/print_env.sh | 0 55 files changed, 10 insertions(+), 12 deletions(-) delete mode 100644 examples/playbook/README.md rename {examples/playbook => tests/ansible}/.gitignore (100%) rename {examples/playbook => tests/ansible}/Makefile (100%) create mode 100644 tests/ansible/README.md rename {examples/playbook => tests/ansible}/all.yml (100%) rename {examples/playbook => tests/ansible}/ansible.cfg (83%) rename {examples/playbook => tests/ansible}/compare_output_test.py (100%) rename {examples/playbook => tests/ansible}/gcloud-ansible-playbook.py (100%) rename {examples/playbook => tests/ansible}/hosts (100%) rename {examples/playbook => tests/ansible}/hosts.docker (100%) rename {examples/playbook => tests/ansible/integration}/action__low_level_execute_command.yml (100%) rename {examples/playbook => tests/ansible/integration}/async_polling.yml (100%) rename {examples/playbook => tests/ansible/integration}/playbook__become_flags.yml (100%) rename {examples/playbook => tests/ansible/integration}/playbook__delegate_to.yml (100%) rename {examples/playbook => tests/ansible/integration}/playbook__environment.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__builtin_command_module.yml (74%) rename {examples/playbook => tests/ansible/integration}/runner__custom_bash_old_style_module.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_bash_want_json_module.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_binary_producing_json.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_binary_producing_junk.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_binary_single_null.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_perl_json_args_module.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_perl_want_json_module.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_python_json_args_module.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_python_new_style_module.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__custom_python_want_json_module.yml (100%) rename {examples/playbook => tests/ansible/integration}/runner__remote_tmp.yml (100%) rename {examples/playbook => tests/ansible}/modules/bash_return_paths.sh (100%) rename {examples/playbook => tests/ansible}/modules/custom_bash_old_style_module.sh (100%) rename {examples/playbook => tests/ansible}/modules/custom_bash_want_json_module.sh (100%) rename {examples/playbook => tests/ansible}/modules/custom_binary_producing_json.c (100%) rename {examples/playbook => tests/ansible}/modules/custom_binary_producing_junk.c (100%) rename {examples/playbook => tests/ansible}/modules/custom_binary_single_null (100%) rename {examples/playbook => tests/ansible}/modules/custom_perl_json_args_module.pl (100%) rename {examples/playbook => tests/ansible}/modules/custom_perl_want_json_module.pl (100%) rename {examples/playbook => tests/ansible}/modules/custom_python_json_args_module.py (100%) rename {examples/playbook => tests/ansible}/modules/custom_python_new_style_module.py (100%) rename {examples/playbook => tests/ansible}/modules/custom_python_want_json_module.py (100%) rename {examples/playbook => tests/ansible/regression}/issue_109.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_113.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_118.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_122.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_131.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_140.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_152.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_152b.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_154.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_174.yml (100%) rename {examples/playbook => tests/ansible/regression}/issue_177.yml (100%) rename {examples/playbook => tests/ansible/regression}/roles/issue_109/tasks/main.yml (100%) rename {examples/playbook => tests/ansible/regression}/roles/issue_109_add_ssh_key/tasks/main.yml (100%) rename {examples/playbook => tests/ansible/regression}/roles/issue_109_gather_facts/tasks/main.yml (100%) rename {examples/playbook => tests/ansible/regression}/scripts/issue_118_saytrue (100%) rename {examples/playbook => tests/ansible/regression}/scripts/print_env.sh (100%) diff --git a/examples/playbook/README.md b/examples/playbook/README.md deleted file mode 100644 index 46fb3777..00000000 --- a/examples/playbook/README.md +++ /dev/null @@ -1,9 +0,0 @@ - -# playbooks Directory - -Although this lives under `examples/`, it is more or less an organically -growing collection of regression tests used for development relating to user -bug reports. - -This will be tidied up over time, meanwhile, the playbooks here are a useful -demonstrator for what does and doesn't work. diff --git a/examples/playbook/.gitignore b/tests/ansible/.gitignore similarity index 100% rename from examples/playbook/.gitignore rename to tests/ansible/.gitignore diff --git a/examples/playbook/Makefile b/tests/ansible/Makefile similarity index 100% rename from examples/playbook/Makefile rename to tests/ansible/Makefile diff --git a/tests/ansible/README.md b/tests/ansible/README.md new file mode 100644 index 00000000..399cec3a --- /dev/null +++ b/tests/ansible/README.md @@ -0,0 +1,8 @@ + +# ``tests/ansible`` Directory + +This is an an organically growing collection of integration and regression +tests used for development and end-user bug reports. + +It will be tidied up over time, meanwhile, the playbooks here are a useful +demonstrator for what does and doesn't work. diff --git a/examples/playbook/all.yml b/tests/ansible/all.yml similarity index 100% rename from examples/playbook/all.yml rename to tests/ansible/all.yml diff --git a/examples/playbook/ansible.cfg b/tests/ansible/ansible.cfg similarity index 83% rename from examples/playbook/ansible.cfg rename to tests/ansible/ansible.cfg index e9ecb706..73da0159 100644 --- a/examples/playbook/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -2,12 +2,11 @@ inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy -#strategy = mitogen_linear library = modules retry_files_enabled = False forks = 50 -# Required by runner__remote_tmp.yml +# Required by integration/runner__remote_tmp.yml remote_tmp = ~/.ansible/mitogen-tests/ [ssh_connection] diff --git a/examples/playbook/compare_output_test.py b/tests/ansible/compare_output_test.py similarity index 100% rename from examples/playbook/compare_output_test.py rename to tests/ansible/compare_output_test.py diff --git a/examples/playbook/gcloud-ansible-playbook.py b/tests/ansible/gcloud-ansible-playbook.py similarity index 100% rename from examples/playbook/gcloud-ansible-playbook.py rename to tests/ansible/gcloud-ansible-playbook.py diff --git a/examples/playbook/hosts b/tests/ansible/hosts similarity index 100% rename from examples/playbook/hosts rename to tests/ansible/hosts diff --git a/examples/playbook/hosts.docker b/tests/ansible/hosts.docker similarity index 100% rename from examples/playbook/hosts.docker rename to tests/ansible/hosts.docker diff --git a/examples/playbook/action__low_level_execute_command.yml b/tests/ansible/integration/action__low_level_execute_command.yml similarity index 100% rename from examples/playbook/action__low_level_execute_command.yml rename to tests/ansible/integration/action__low_level_execute_command.yml diff --git a/examples/playbook/async_polling.yml b/tests/ansible/integration/async_polling.yml similarity index 100% rename from examples/playbook/async_polling.yml rename to tests/ansible/integration/async_polling.yml diff --git a/examples/playbook/playbook__become_flags.yml b/tests/ansible/integration/playbook__become_flags.yml similarity index 100% rename from examples/playbook/playbook__become_flags.yml rename to tests/ansible/integration/playbook__become_flags.yml diff --git a/examples/playbook/playbook__delegate_to.yml b/tests/ansible/integration/playbook__delegate_to.yml similarity index 100% rename from examples/playbook/playbook__delegate_to.yml rename to tests/ansible/integration/playbook__delegate_to.yml diff --git a/examples/playbook/playbook__environment.yml b/tests/ansible/integration/playbook__environment.yml similarity index 100% rename from examples/playbook/playbook__environment.yml rename to tests/ansible/integration/playbook__environment.yml diff --git a/examples/playbook/runner.yml b/tests/ansible/integration/runner.yml similarity index 100% rename from examples/playbook/runner.yml rename to tests/ansible/integration/runner.yml diff --git a/examples/playbook/runner__builtin_command_module.yml b/tests/ansible/integration/runner__builtin_command_module.yml similarity index 74% rename from examples/playbook/runner__builtin_command_module.yml rename to tests/ansible/integration/runner__builtin_command_module.yml index c0b0c6e2..389756d5 100644 --- a/examples/playbook/runner__builtin_command_module.yml +++ b/tests/ansible/integration/runner__builtin_command_module.yml @@ -1,5 +1,5 @@ - hosts: all tasks: - - name: "Run hostname" + - name: builtin_command_module command: hostname with_sequence: start=1 end={{end|default(100)}} diff --git a/examples/playbook/runner__custom_bash_old_style_module.yml b/tests/ansible/integration/runner__custom_bash_old_style_module.yml similarity index 100% rename from examples/playbook/runner__custom_bash_old_style_module.yml rename to tests/ansible/integration/runner__custom_bash_old_style_module.yml diff --git a/examples/playbook/runner__custom_bash_want_json_module.yml b/tests/ansible/integration/runner__custom_bash_want_json_module.yml similarity index 100% rename from examples/playbook/runner__custom_bash_want_json_module.yml rename to tests/ansible/integration/runner__custom_bash_want_json_module.yml diff --git a/examples/playbook/runner__custom_binary_producing_json.yml b/tests/ansible/integration/runner__custom_binary_producing_json.yml similarity index 100% rename from examples/playbook/runner__custom_binary_producing_json.yml rename to tests/ansible/integration/runner__custom_binary_producing_json.yml diff --git a/examples/playbook/runner__custom_binary_producing_junk.yml b/tests/ansible/integration/runner__custom_binary_producing_junk.yml similarity index 100% rename from examples/playbook/runner__custom_binary_producing_junk.yml rename to tests/ansible/integration/runner__custom_binary_producing_junk.yml diff --git a/examples/playbook/runner__custom_binary_single_null.yml b/tests/ansible/integration/runner__custom_binary_single_null.yml similarity index 100% rename from examples/playbook/runner__custom_binary_single_null.yml rename to tests/ansible/integration/runner__custom_binary_single_null.yml diff --git a/examples/playbook/runner__custom_perl_json_args_module.yml b/tests/ansible/integration/runner__custom_perl_json_args_module.yml similarity index 100% rename from examples/playbook/runner__custom_perl_json_args_module.yml rename to tests/ansible/integration/runner__custom_perl_json_args_module.yml diff --git a/examples/playbook/runner__custom_perl_want_json_module.yml b/tests/ansible/integration/runner__custom_perl_want_json_module.yml similarity index 100% rename from examples/playbook/runner__custom_perl_want_json_module.yml rename to tests/ansible/integration/runner__custom_perl_want_json_module.yml diff --git a/examples/playbook/runner__custom_python_json_args_module.yml b/tests/ansible/integration/runner__custom_python_json_args_module.yml similarity index 100% rename from examples/playbook/runner__custom_python_json_args_module.yml rename to tests/ansible/integration/runner__custom_python_json_args_module.yml diff --git a/examples/playbook/runner__custom_python_new_style_module.yml b/tests/ansible/integration/runner__custom_python_new_style_module.yml similarity index 100% rename from examples/playbook/runner__custom_python_new_style_module.yml rename to tests/ansible/integration/runner__custom_python_new_style_module.yml diff --git a/examples/playbook/runner__custom_python_want_json_module.yml b/tests/ansible/integration/runner__custom_python_want_json_module.yml similarity index 100% rename from examples/playbook/runner__custom_python_want_json_module.yml rename to tests/ansible/integration/runner__custom_python_want_json_module.yml diff --git a/examples/playbook/runner__remote_tmp.yml b/tests/ansible/integration/runner__remote_tmp.yml similarity index 100% rename from examples/playbook/runner__remote_tmp.yml rename to tests/ansible/integration/runner__remote_tmp.yml diff --git a/examples/playbook/modules/bash_return_paths.sh b/tests/ansible/modules/bash_return_paths.sh similarity index 100% rename from examples/playbook/modules/bash_return_paths.sh rename to tests/ansible/modules/bash_return_paths.sh diff --git a/examples/playbook/modules/custom_bash_old_style_module.sh b/tests/ansible/modules/custom_bash_old_style_module.sh similarity index 100% rename from examples/playbook/modules/custom_bash_old_style_module.sh rename to tests/ansible/modules/custom_bash_old_style_module.sh diff --git a/examples/playbook/modules/custom_bash_want_json_module.sh b/tests/ansible/modules/custom_bash_want_json_module.sh similarity index 100% rename from examples/playbook/modules/custom_bash_want_json_module.sh rename to tests/ansible/modules/custom_bash_want_json_module.sh diff --git a/examples/playbook/modules/custom_binary_producing_json.c b/tests/ansible/modules/custom_binary_producing_json.c similarity index 100% rename from examples/playbook/modules/custom_binary_producing_json.c rename to tests/ansible/modules/custom_binary_producing_json.c diff --git a/examples/playbook/modules/custom_binary_producing_junk.c b/tests/ansible/modules/custom_binary_producing_junk.c similarity index 100% rename from examples/playbook/modules/custom_binary_producing_junk.c rename to tests/ansible/modules/custom_binary_producing_junk.c diff --git a/examples/playbook/modules/custom_binary_single_null b/tests/ansible/modules/custom_binary_single_null similarity index 100% rename from examples/playbook/modules/custom_binary_single_null rename to tests/ansible/modules/custom_binary_single_null diff --git a/examples/playbook/modules/custom_perl_json_args_module.pl b/tests/ansible/modules/custom_perl_json_args_module.pl similarity index 100% rename from examples/playbook/modules/custom_perl_json_args_module.pl rename to tests/ansible/modules/custom_perl_json_args_module.pl diff --git a/examples/playbook/modules/custom_perl_want_json_module.pl b/tests/ansible/modules/custom_perl_want_json_module.pl similarity index 100% rename from examples/playbook/modules/custom_perl_want_json_module.pl rename to tests/ansible/modules/custom_perl_want_json_module.pl diff --git a/examples/playbook/modules/custom_python_json_args_module.py b/tests/ansible/modules/custom_python_json_args_module.py similarity index 100% rename from examples/playbook/modules/custom_python_json_args_module.py rename to tests/ansible/modules/custom_python_json_args_module.py diff --git a/examples/playbook/modules/custom_python_new_style_module.py b/tests/ansible/modules/custom_python_new_style_module.py similarity index 100% rename from examples/playbook/modules/custom_python_new_style_module.py rename to tests/ansible/modules/custom_python_new_style_module.py diff --git a/examples/playbook/modules/custom_python_want_json_module.py b/tests/ansible/modules/custom_python_want_json_module.py similarity index 100% rename from examples/playbook/modules/custom_python_want_json_module.py rename to tests/ansible/modules/custom_python_want_json_module.py diff --git a/examples/playbook/issue_109.yml b/tests/ansible/regression/issue_109.yml similarity index 100% rename from examples/playbook/issue_109.yml rename to tests/ansible/regression/issue_109.yml diff --git a/examples/playbook/issue_113.yml b/tests/ansible/regression/issue_113.yml similarity index 100% rename from examples/playbook/issue_113.yml rename to tests/ansible/regression/issue_113.yml diff --git a/examples/playbook/issue_118.yml b/tests/ansible/regression/issue_118.yml similarity index 100% rename from examples/playbook/issue_118.yml rename to tests/ansible/regression/issue_118.yml diff --git a/examples/playbook/issue_122.yml b/tests/ansible/regression/issue_122.yml similarity index 100% rename from examples/playbook/issue_122.yml rename to tests/ansible/regression/issue_122.yml diff --git a/examples/playbook/issue_131.yml b/tests/ansible/regression/issue_131.yml similarity index 100% rename from examples/playbook/issue_131.yml rename to tests/ansible/regression/issue_131.yml diff --git a/examples/playbook/issue_140.yml b/tests/ansible/regression/issue_140.yml similarity index 100% rename from examples/playbook/issue_140.yml rename to tests/ansible/regression/issue_140.yml diff --git a/examples/playbook/issue_152.yml b/tests/ansible/regression/issue_152.yml similarity index 100% rename from examples/playbook/issue_152.yml rename to tests/ansible/regression/issue_152.yml diff --git a/examples/playbook/issue_152b.yml b/tests/ansible/regression/issue_152b.yml similarity index 100% rename from examples/playbook/issue_152b.yml rename to tests/ansible/regression/issue_152b.yml diff --git a/examples/playbook/issue_154.yml b/tests/ansible/regression/issue_154.yml similarity index 100% rename from examples/playbook/issue_154.yml rename to tests/ansible/regression/issue_154.yml diff --git a/examples/playbook/issue_174.yml b/tests/ansible/regression/issue_174.yml similarity index 100% rename from examples/playbook/issue_174.yml rename to tests/ansible/regression/issue_174.yml diff --git a/examples/playbook/issue_177.yml b/tests/ansible/regression/issue_177.yml similarity index 100% rename from examples/playbook/issue_177.yml rename to tests/ansible/regression/issue_177.yml diff --git a/examples/playbook/roles/issue_109/tasks/main.yml b/tests/ansible/regression/roles/issue_109/tasks/main.yml similarity index 100% rename from examples/playbook/roles/issue_109/tasks/main.yml rename to tests/ansible/regression/roles/issue_109/tasks/main.yml diff --git a/examples/playbook/roles/issue_109_add_ssh_key/tasks/main.yml b/tests/ansible/regression/roles/issue_109_add_ssh_key/tasks/main.yml similarity index 100% rename from examples/playbook/roles/issue_109_add_ssh_key/tasks/main.yml rename to tests/ansible/regression/roles/issue_109_add_ssh_key/tasks/main.yml diff --git a/examples/playbook/roles/issue_109_gather_facts/tasks/main.yml b/tests/ansible/regression/roles/issue_109_gather_facts/tasks/main.yml similarity index 100% rename from examples/playbook/roles/issue_109_gather_facts/tasks/main.yml rename to tests/ansible/regression/roles/issue_109_gather_facts/tasks/main.yml diff --git a/examples/playbook/scripts/issue_118_saytrue b/tests/ansible/regression/scripts/issue_118_saytrue similarity index 100% rename from examples/playbook/scripts/issue_118_saytrue rename to tests/ansible/regression/scripts/issue_118_saytrue diff --git a/examples/playbook/scripts/print_env.sh b/tests/ansible/regression/scripts/print_env.sh similarity index 100% rename from examples/playbook/scripts/print_env.sh rename to tests/ansible/regression/scripts/print_env.sh From 475d459185f0d187bc390d0f221e38a56e455d6b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 16:00:09 +0100 Subject: [PATCH 112/140] issue #164: rename 'test' to 'run_tests' to avoid tab complete conflict --- .travis/mitogen_tests.sh | 2 +- test => run_tests | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename test => run_tests (100%) diff --git a/.travis/mitogen_tests.sh b/.travis/mitogen_tests.sh index 01d008b7..8b317251 100644 --- a/.travis/mitogen_tests.sh +++ b/.travis/mitogen_tests.sh @@ -1,4 +1,4 @@ #!/bin/bash -ex # Run the Mitogen tests. -MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/test +MITOGEN_LOG_LEVEL=debug PYTHONPATH=. ${TRAVIS_BUILD_DIR}/run_tests diff --git a/test b/run_tests similarity index 100% rename from test rename to run_tests From 563639961d55665b26e394c5d7570bad3caa4acd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 16:08:38 +0100 Subject: [PATCH 113/140] issue #164: dir structure is gross, but at least tab completion works :> --- tests/{ansible_helpers_test.py => ansible/tests/helpers_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{ansible_helpers_test.py => ansible/tests/helpers_test.py} (100%) diff --git a/tests/ansible_helpers_test.py b/tests/ansible/tests/helpers_test.py similarity index 100% rename from tests/ansible_helpers_test.py rename to tests/ansible/tests/helpers_test.py From ae75a0ca8cc99cd932fc10f7ac59dfc69f3156d5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 16:11:19 +0100 Subject: [PATCH 114/140] issue #164: rearrange playbooks a little more --- tests/ansible/README.md | 8 ++++++++ tests/ansible/all.yml | 8 ++------ tests/ansible/integration/all.yml | 7 +++++++ tests/ansible/regression/all.yml | 11 +++++++++++ 4 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 tests/ansible/integration/all.yml create mode 100644 tests/ansible/regression/all.yml diff --git a/tests/ansible/README.md b/tests/ansible/README.md index 399cec3a..7eb04769 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -6,3 +6,11 @@ tests used for development and end-user bug reports. It will be tidied up over time, meanwhile, the playbooks here are a useful demonstrator for what does and doesn't work. + + + +## Running Everything + +``` +ANSIBLE_STRATEGY=mitogen_linear ansible-playbook all.yml +``` diff --git a/tests/ansible/all.yml b/tests/ansible/all.yml index c6dcb95b..a68831f7 100644 --- a/tests/ansible/all.yml +++ b/tests/ansible/all.yml @@ -1,7 +1,3 @@ +- import_playbook: regression/all.yml +- import_playbook: integration/all.yml -# -# This playbook imports all tests that are known to work at present. -# - -- import_playbook: action__low_level_execute_command.yml -- import_playbook: runner.yml diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml new file mode 100644 index 00000000..c6dcb95b --- /dev/null +++ b/tests/ansible/integration/all.yml @@ -0,0 +1,7 @@ + +# +# This playbook imports all tests that are known to work at present. +# + +- import_playbook: action__low_level_execute_command.yml +- import_playbook: runner.yml diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml new file mode 100644 index 00000000..c601f919 --- /dev/null +++ b/tests/ansible/regression/all.yml @@ -0,0 +1,11 @@ +- import_playbook: issue_109.yml +- import_playbook: issue_113.yml +- import_playbook: issue_118.yml +- import_playbook: issue_122.yml +- import_playbook: issue_131.yml +- import_playbook: issue_140.yml +- import_playbook: issue_152.yml +- import_playbook: issue_152b.yml +- import_playbook: issue_154.yml +- import_playbook: issue_174.yml +- import_playbook: issue_177.yml From 3ebe600389f33274c41aa922e254165902b418e9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 20:01:23 +0100 Subject: [PATCH 115/140] issue #164: convert "examples" into actual tests - Add new Travis mode, "ansible_tests.sh" that runs integrations/all.yml. Slowly build this up over time to cover more of the existing junk. - Add basic assertions on the output of the existing runner__* files. - Wire up 2.4.3/2.5.0 jobs in Travis. --- .travis.yml | 2 + .travis/ansible_tests.sh | 45 +++++++++++++++++++ .../action__low_level_execute_command.yml | 4 ++ .../runner__builtin_command_module.yml | 15 ++++++- .../runner__custom_bash_old_style_module.yml | 12 ++++- .../runner__custom_bash_want_json_module.yml | 12 ++++- .../runner__custom_binary_producing_json.yml | 12 ++++- .../runner__custom_binary_producing_junk.yml | 13 +++++- .../runner__custom_binary_single_null.yml | 13 +++++- .../runner__custom_perl_json_args_module.yml | 13 +++++- .../runner__custom_perl_want_json_module.yml | 13 +++++- ...runner__custom_python_json_args_module.yml | 13 +++++- ...runner__custom_python_new_style_module.yml | 13 +++++- ...runner__custom_python_want_json_module.yml | 13 +++++- 14 files changed, 171 insertions(+), 22 deletions(-) create mode 100755 .travis/ansible_tests.sh diff --git a/.travis.yml b/.travis.yml index ca59ff82..deb3c0fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ python: env: - MODE=mitogen - MODE=debops +- MODE=ansible ANSIBLE_VERSION=2.4.3.0 +- MODE=ansible ANSIBLE_VERSION=2.5.0 install: - pip install -r dev_requirements.txt diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh new file mode 100755 index 00000000..89f99cf7 --- /dev/null +++ b/.travis/ansible_tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash -ex +# Run tests/ansible/integration/all.yml under Ansible and Ansible-Mitogen + +TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" +TMPDIR="/tmp/ansible-tests-$$" +ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.4.3.0}" + +function on_exit() +{ + rm -rf "$TMPDIR" + docker kill target || true +} + +trap on_exit EXIT +mkdir "$TMPDIR" + + +echo travis_fold:start:docker_setup +docker run --rm --detach --name=target d2mw/mitogen-test /bin/sleep 86400 +echo travis_fold:end:docker_setup + + +echo travis_fold:start:job_setup +pip install -U ansible==${ANSIBLE_VERSION}" +cd ${TRAVIS_BUILD_DIR}/tests/ansible + +cat >> ${TMPDIR}/hosts <<-EOF +localhost +target ansible_connection=docker ansible_python_interpreter=/usr/bin/python2.7 +EOF +echo travis_fold:end:job_setup + + +echo travis_fold:start:mitogen_linear +ANSIBLE_STRATEGY=mitogen_linear /usr/bin/time ansible-playbook \ + integration/all.yml \ + -i "${TMPDIR}/hosts" +echo travis_fold:end:mitogen_linear + + +echo travis_fold:start:vanilla_ansible +/usr/bin/time ansible-playbook \ + integration/all.yml \ + -i "${TMPDIR}/hosts" +echo travis_fold:end:vanilla_ansible diff --git a/tests/ansible/integration/action__low_level_execute_command.yml b/tests/ansible/integration/action__low_level_execute_command.yml index 419af489..c938537e 100644 --- a/tests/ansible/integration/action__low_level_execute_command.yml +++ b/tests/ansible/integration/action__low_level_execute_command.yml @@ -2,6 +2,10 @@ - hosts: all tasks: + - name: integration/action__low_level_execute_command.yml + assert: + that: true + # "echo -en" to test we actually hit bash shell too. - name: Run raw module without sudo raw: 'echo -en $((1 + 1))' diff --git a/tests/ansible/integration/runner__builtin_command_module.yml b/tests/ansible/integration/runner__builtin_command_module.yml index 389756d5..171207d0 100644 --- a/tests/ansible/integration/runner__builtin_command_module.yml +++ b/tests/ansible/integration/runner__builtin_command_module.yml @@ -1,5 +1,16 @@ - hosts: all + gather_facts: true tasks: - - name: builtin_command_module + - name: integration/runner__builtin_command_module.yml command: hostname - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + out.changed and + out.results[0].changed and + out.results[0].cmd == ['hostname'] and + out.results[0].item == '1' and + out.results[0].rc == 0 and + (out.results[0].stdout == ansible_nodename) diff --git a/tests/ansible/integration/runner__custom_bash_old_style_module.yml b/tests/ansible/integration/runner__custom_bash_old_style_module.yml index f7ebefa8..34b7a5b8 100644 --- a/tests/ansible/integration/runner__custom_bash_old_style_module.yml +++ b/tests/ansible/integration/runner__custom_bash_old_style_module.yml @@ -1,5 +1,13 @@ - hosts: all tasks: - - custom_bash_old_style_module: + - name: integration/runner__custom_bash_old_style_module.yml + custom_bash_old_style_module: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + (not out.changed) and + (not out.results[0].changed) and + out.results[0].msg == 'Here is my input' diff --git a/tests/ansible/integration/runner__custom_bash_want_json_module.yml b/tests/ansible/integration/runner__custom_bash_want_json_module.yml index bcf49b50..7933ba65 100644 --- a/tests/ansible/integration/runner__custom_bash_want_json_module.yml +++ b/tests/ansible/integration/runner__custom_bash_want_json_module.yml @@ -1,5 +1,13 @@ - hosts: all tasks: - - custom_bash_want_json_module: + - name: integration/runner__custom_bash_want_json_module.yml + custom_bash_want_json_module: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + (not out.changed) and + (not out.results[0].changed) and + out.results[0].msg == 'Here is my input' diff --git a/tests/ansible/integration/runner__custom_binary_producing_json.yml b/tests/ansible/integration/runner__custom_binary_producing_json.yml index bc0ced40..c04a7552 100644 --- a/tests/ansible/integration/runner__custom_binary_producing_json.yml +++ b/tests/ansible/integration/runner__custom_binary_producing_json.yml @@ -1,5 +1,13 @@ - hosts: all tasks: - - custom_binary_producing_json: + - name: integration/runner__custom_binary_producing_json.yml + custom_binary_producing_json: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + out.changed and + out.results[0].changed and + out.results[0].msg == 'Hello, world.' diff --git a/tests/ansible/integration/runner__custom_binary_producing_junk.yml b/tests/ansible/integration/runner__custom_binary_producing_junk.yml index b806752d..dbf734ee 100644 --- a/tests/ansible/integration/runner__custom_binary_producing_junk.yml +++ b/tests/ansible/integration/runner__custom_binary_producing_junk.yml @@ -1,6 +1,15 @@ - hosts: all tasks: - - custom_binary_producing_junk: + - name: integration/runner__custom_binary_producing_junk.yml + custom_binary_producing_junk: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} ignore_errors: true + register: out + + - assert: + that: | + out.failed and + out.results[0].failed and + out.results[0].msg == 'MODULE FAILURE' and + out.results[0].rc == 0 diff --git a/tests/ansible/integration/runner__custom_binary_single_null.yml b/tests/ansible/integration/runner__custom_binary_single_null.yml index a830e5fe..9c468f27 100644 --- a/tests/ansible/integration/runner__custom_binary_single_null.yml +++ b/tests/ansible/integration/runner__custom_binary_single_null.yml @@ -1,6 +1,15 @@ - hosts: all tasks: - - custom_binary_single_null: + - name: integration/runner__custom_binary_single_null.yml + custom_binary_single_null: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} ignore_errors: true + register: out + + - assert: + that: | + out.failed and + out.results[0].failed and + out.results[0].msg == 'MODULE FAILURE' and + out.results[0].rc == 126 diff --git a/tests/ansible/integration/runner__custom_perl_json_args_module.yml b/tests/ansible/integration/runner__custom_perl_json_args_module.yml index 1ee24ccb..d0ac467a 100644 --- a/tests/ansible/integration/runner__custom_perl_json_args_module.yml +++ b/tests/ansible/integration/runner__custom_perl_json_args_module.yml @@ -1,5 +1,14 @@ - hosts: all tasks: - - custom_perl_json_args_module: + - name: integration/runner__custom_perl_json_args_module.yml + custom_perl_json_args_module: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + (not out.changed) and + (not out.results[0].changed) and + out.results[0].input[0].foo and + out.results[0].message == 'I am a perl script! Here is my input.' diff --git a/tests/ansible/integration/runner__custom_perl_want_json_module.yml b/tests/ansible/integration/runner__custom_perl_want_json_module.yml index 4b821bef..0eaf7768 100644 --- a/tests/ansible/integration/runner__custom_perl_want_json_module.yml +++ b/tests/ansible/integration/runner__custom_perl_want_json_module.yml @@ -1,5 +1,14 @@ - hosts: all tasks: - - custom_perl_want_json_module: + - name: integration/runner__custom_perl_want_json_module.yml + custom_perl_want_json_module: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + (not out.changed) and + (not out.results[0].changed) and + out.results[0].input[0].foo and + out.results[0].message == 'I am a want JSON perl script! Here is my input.' diff --git a/tests/ansible/integration/runner__custom_python_json_args_module.yml b/tests/ansible/integration/runner__custom_python_json_args_module.yml index f281184a..377043b6 100644 --- a/tests/ansible/integration/runner__custom_python_json_args_module.yml +++ b/tests/ansible/integration/runner__custom_python_json_args_module.yml @@ -1,5 +1,14 @@ - hosts: all tasks: - - custom_python_json_args_module: + - name: integration/runner__custom_python_json_args_module.yml + custom_python_json_args_module: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + (not out.changed) and + (not out.results[0].changed) and + out.results[0].input[0].foo and + out.results[0].msg == 'Here is my input' diff --git a/tests/ansible/integration/runner__custom_python_new_style_module.yml b/tests/ansible/integration/runner__custom_python_new_style_module.yml index 1a2421d3..b01733e1 100644 --- a/tests/ansible/integration/runner__custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner__custom_python_new_style_module.yml @@ -1,5 +1,14 @@ - hosts: all tasks: - - custom_python_new_style_module: + - name: integration/runner__custom_python_new_style_module.yml + custom_python_new_style_module: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + (not out.changed) and + (not out.results[0].changed) and + out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and + out.results[0].msg == 'Here is my input' diff --git a/tests/ansible/integration/runner__custom_python_want_json_module.yml b/tests/ansible/integration/runner__custom_python_want_json_module.yml index b149808f..642ac81a 100644 --- a/tests/ansible/integration/runner__custom_python_want_json_module.yml +++ b/tests/ansible/integration/runner__custom_python_want_json_module.yml @@ -1,5 +1,14 @@ - hosts: all tasks: - - custom_python_want_json_module: + - name: integration/runner__custom_python_want_json_module.yml + custom_python_want_json_module: foo: true - with_sequence: start=1 end={{end|default(100)}} + with_sequence: start=1 end={{end|default(1)}} + register: out + + - assert: + that: | + (not out.changed) and + (not out.results[0].changed) and + out.results[0].input[0].foo and + out.results[0].msg == 'Here is my input' From 26cc0f27245642d55948e4985f86cb821dffe04a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 4 Apr 2018 20:19:46 +0100 Subject: [PATCH 116/140] issue #164: fix remote_tmp handling on <2.5 --- .travis/ansible_tests.sh | 2 +- ansible_mitogen/mixins.py | 4 ++-- tests/ansible/integration/runner__remote_tmp.yml | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index 89f99cf7..4fc69891 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -21,7 +21,7 @@ echo travis_fold:end:docker_setup echo travis_fold:start:job_setup -pip install -U ansible==${ANSIBLE_VERSION}" +pip install -U ansible=="${ANSIBLE_VERSION}" cd ${TRAVIS_BUILD_DIR}/tests/ansible cat >> ${TMPDIR}/hosts <<-EOF diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 094c4510..f2b8aedc 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -39,6 +39,7 @@ from ansible.module_utils._text import to_bytes from ansible.parsing.utils.jsonify import jsonify import ansible +import ansible.constants import ansible.plugins import ansible.plugins.action @@ -184,8 +185,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): try: s = self._connection._shell.get_option('remote_tmp') except AttributeError: - # Required for <2.4.x. - s = '~/.ansible' + s = ansible.constants.DEFAULT_REMOTE_TMP # <=2.4.x return self._remote_expand_user(s) diff --git a/tests/ansible/integration/runner__remote_tmp.yml b/tests/ansible/integration/runner__remote_tmp.yml index 74f18480..47c55b99 100644 --- a/tests/ansible/integration/runner__remote_tmp.yml +++ b/tests/ansible/integration/runner__remote_tmp.yml @@ -6,7 +6,8 @@ - hosts: all gather_facts: true tasks: - - bash_return_paths: + - name: integration/runner__remote_tmp.yml + bash_return_paths: register: output - assert: From c5ca2e87ea34f24ced709d17bcdc6428afc206a3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 00:11:29 +0100 Subject: [PATCH 117/140] issue #164: stop tests on first failure --- .../ansible/integration/action__low_level_execute_command.yml | 1 + tests/ansible/integration/async_polling.yml | 1 + tests/ansible/integration/playbook__become_flags.yml | 2 ++ tests/ansible/integration/playbook__delegate_to.yml | 1 + tests/ansible/integration/playbook__environment.yml | 1 + tests/ansible/integration/runner__builtin_command_module.yml | 1 + .../integration/runner__custom_bash_old_style_module.yml | 1 + .../integration/runner__custom_bash_want_json_module.yml | 1 + .../integration/runner__custom_binary_producing_json.yml | 1 + .../integration/runner__custom_binary_producing_junk.yml | 4 ++++ .../ansible/integration/runner__custom_binary_single_null.yml | 3 +++ .../integration/runner__custom_perl_json_args_module.yml | 1 + .../integration/runner__custom_perl_want_json_module.yml | 1 + .../integration/runner__custom_python_json_args_module.yml | 1 + .../integration/runner__custom_python_new_style_module.yml | 1 + .../integration/runner__custom_python_want_json_module.yml | 1 + tests/ansible/integration/runner__remote_tmp.yml | 1 + 17 files changed, 23 insertions(+) diff --git a/tests/ansible/integration/action__low_level_execute_command.yml b/tests/ansible/integration/action__low_level_execute_command.yml index c938537e..fd0217bc 100644 --- a/tests/ansible/integration/action__low_level_execute_command.yml +++ b/tests/ansible/integration/action__low_level_execute_command.yml @@ -1,6 +1,7 @@ # Verify the behaviour of _low_level_execute_command(). - hosts: all + any_errors_fatal: true tasks: - name: integration/action__low_level_execute_command.yml assert: diff --git a/tests/ansible/integration/async_polling.yml b/tests/ansible/integration/async_polling.yml index 20aa211c..b08394a7 100644 --- a/tests/ansible/integration/async_polling.yml +++ b/tests/ansible/integration/async_polling.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: simulate long running op (3 sec), wait for up to 5 sec, poll every 1 sec command: /bin/sleep 2 diff --git a/tests/ansible/integration/playbook__become_flags.yml b/tests/ansible/integration/playbook__become_flags.yml index 74ebf059..61148169 100644 --- a/tests/ansible/integration/playbook__become_flags.yml +++ b/tests/ansible/integration/playbook__become_flags.yml @@ -5,6 +5,7 @@ # - hosts: all + any_errors_fatal: true tasks: - name: "without -E" become: true @@ -15,6 +16,7 @@ that: "out.stdout == ''" - hosts: all + any_errors_fatal: true become_flags: -E tasks: - name: "with -E" diff --git a/tests/ansible/integration/playbook__delegate_to.yml b/tests/ansible/integration/playbook__delegate_to.yml index dae2afd8..beb8bdc3 100644 --- a/tests/ansible/integration/playbook__delegate_to.yml +++ b/tests/ansible/integration/playbook__delegate_to.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: # # delegate_to, no sudo diff --git a/tests/ansible/integration/playbook__environment.yml b/tests/ansible/integration/playbook__environment.yml index fd70174f..8d956d49 100644 --- a/tests/ansible/integration/playbook__environment.yml +++ b/tests/ansible/integration/playbook__environment.yml @@ -1,6 +1,7 @@ # Ensure environment: is preserved during call. - hosts: all + any_errors_fatal: true tasks: - shell: echo $SOME_ENV environment: diff --git a/tests/ansible/integration/runner__builtin_command_module.yml b/tests/ansible/integration/runner__builtin_command_module.yml index 171207d0..ca94a604 100644 --- a/tests/ansible/integration/runner__builtin_command_module.yml +++ b/tests/ansible/integration/runner__builtin_command_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true gather_facts: true tasks: - name: integration/runner__builtin_command_module.yml diff --git a/tests/ansible/integration/runner__custom_bash_old_style_module.yml b/tests/ansible/integration/runner__custom_bash_old_style_module.yml index 34b7a5b8..3aa9fe52 100644 --- a/tests/ansible/integration/runner__custom_bash_old_style_module.yml +++ b/tests/ansible/integration/runner__custom_bash_old_style_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_bash_old_style_module.yml custom_bash_old_style_module: diff --git a/tests/ansible/integration/runner__custom_bash_want_json_module.yml b/tests/ansible/integration/runner__custom_bash_want_json_module.yml index 7933ba65..85e83e3e 100644 --- a/tests/ansible/integration/runner__custom_bash_want_json_module.yml +++ b/tests/ansible/integration/runner__custom_bash_want_json_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_bash_want_json_module.yml custom_bash_want_json_module: diff --git a/tests/ansible/integration/runner__custom_binary_producing_json.yml b/tests/ansible/integration/runner__custom_binary_producing_json.yml index c04a7552..559d89b1 100644 --- a/tests/ansible/integration/runner__custom_binary_producing_json.yml +++ b/tests/ansible/integration/runner__custom_binary_producing_json.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_binary_producing_json.yml custom_binary_producing_json: diff --git a/tests/ansible/integration/runner__custom_binary_producing_junk.yml b/tests/ansible/integration/runner__custom_binary_producing_junk.yml index dbf734ee..d9614e7d 100644 --- a/tests/ansible/integration/runner__custom_binary_producing_junk.yml +++ b/tests/ansible/integration/runner__custom_binary_producing_junk.yml @@ -7,6 +7,10 @@ ignore_errors: true register: out + +- hosts: all + any_errors_fatal: true + tasks: - assert: that: | out.failed and diff --git a/tests/ansible/integration/runner__custom_binary_single_null.yml b/tests/ansible/integration/runner__custom_binary_single_null.yml index 9c468f27..c521e3b2 100644 --- a/tests/ansible/integration/runner__custom_binary_single_null.yml +++ b/tests/ansible/integration/runner__custom_binary_single_null.yml @@ -7,6 +7,9 @@ ignore_errors: true register: out +- hosts: all + any_errors_fatal: true + tasks: - assert: that: | out.failed and diff --git a/tests/ansible/integration/runner__custom_perl_json_args_module.yml b/tests/ansible/integration/runner__custom_perl_json_args_module.yml index d0ac467a..1777798a 100644 --- a/tests/ansible/integration/runner__custom_perl_json_args_module.yml +++ b/tests/ansible/integration/runner__custom_perl_json_args_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_perl_json_args_module.yml custom_perl_json_args_module: diff --git a/tests/ansible/integration/runner__custom_perl_want_json_module.yml b/tests/ansible/integration/runner__custom_perl_want_json_module.yml index 0eaf7768..dfe0894c 100644 --- a/tests/ansible/integration/runner__custom_perl_want_json_module.yml +++ b/tests/ansible/integration/runner__custom_perl_want_json_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_perl_want_json_module.yml custom_perl_want_json_module: diff --git a/tests/ansible/integration/runner__custom_python_json_args_module.yml b/tests/ansible/integration/runner__custom_python_json_args_module.yml index 377043b6..027280df 100644 --- a/tests/ansible/integration/runner__custom_python_json_args_module.yml +++ b/tests/ansible/integration/runner__custom_python_json_args_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_python_json_args_module.yml custom_python_json_args_module: diff --git a/tests/ansible/integration/runner__custom_python_new_style_module.yml b/tests/ansible/integration/runner__custom_python_new_style_module.yml index b01733e1..fce315ea 100644 --- a/tests/ansible/integration/runner__custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner__custom_python_new_style_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_python_new_style_module.yml custom_python_new_style_module: diff --git a/tests/ansible/integration/runner__custom_python_want_json_module.yml b/tests/ansible/integration/runner__custom_python_want_json_module.yml index 642ac81a..82dc2efd 100644 --- a/tests/ansible/integration/runner__custom_python_want_json_module.yml +++ b/tests/ansible/integration/runner__custom_python_want_json_module.yml @@ -1,4 +1,5 @@ - hosts: all + any_errors_fatal: true tasks: - name: integration/runner__custom_python_want_json_module.yml custom_python_want_json_module: diff --git a/tests/ansible/integration/runner__remote_tmp.yml b/tests/ansible/integration/runner__remote_tmp.yml index 47c55b99..dfa85ba4 100644 --- a/tests/ansible/integration/runner__remote_tmp.yml +++ b/tests/ansible/integration/runner__remote_tmp.yml @@ -4,6 +4,7 @@ # remotely. # - hosts: all + any_errors_fatal: true gather_facts: true tasks: - name: integration/runner__remote_tmp.yml From 6aeb4e9f05ee257d191c5f23861fc1711105788f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 00:41:14 +0100 Subject: [PATCH 118/140] issue #164: precisely emulate Ansible's stdio behaviour. * Use identical logic to select when stdout/stderr are merged, so 'stdout', 'stdout_lines', 'stderr', 'stderr_lines' contain the same output before/after the extension. * When stdout/stderr are merged, synthesize carriage returns just like the TTY layer. * Mimic the SSH connection multiplexing message on stderr. Not really for user code, but so compare_output_test.sh needs fewer fixups. --- ansible_mitogen/connection.py | 18 ++++++-- ansible_mitogen/helpers.py | 21 +++++++-- ansible_mitogen/mixins.py | 10 ++-- ansible_mitogen/runner.py | 1 + docs/ansible.rst | 46 ++++++++++--------- .../action__low_level_execute_command.yml | 7 +-- .../runner__custom_binary_single_null.yml | 8 +++- 7 files changed, 73 insertions(+), 38 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index bd0d79ff..a538f743 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -303,7 +303,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): LOG.debug('Call %s%r took %d ms', func.func_name, args, 1000 * (time.time() - t0)) - def exec_command(self, cmd, in_data='', sudoable=True): + def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None): """ Implement exec_command() by calling the corresponding ansible_mitogen.helpers function in the target. @@ -315,8 +315,20 @@ class Connection(ansible.plugins.connection.ConnectionBase): :returns: (return code, stdout bytes, stderr bytes) """ - return self.call(ansible_mitogen.helpers.exec_command, - cast(cmd), cast(in_data)) + emulate_tty = (not in_data and sudoable) + rc, stdout, stderr = self.call( + ansible_mitogen.helpers.exec_command, + cmd=cast(cmd), + in_data=cast(in_data), + chdir=mitogen_chdir, + emulate_tty=emulate_tty, + ) + + stderr += 'Shared connection to %s closed.%s' % ( + self._play_context.remote_addr, + ('\r\n' if emulate_tty else '\n'), + ) + return rc, stdout, stderr def fetch_file(self, in_path, out_path): """ diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 20f69c19..167afbb5 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -176,7 +176,7 @@ def get_user_shell(): return pw_shell or '/bin/sh' -def exec_args(args, in_data='', chdir=None, shell=None): +def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False): """ Run a command in a subprocess, emulating the argument handling behaviour of SSH. @@ -185,24 +185,36 @@ def exec_args(args, in_data='', chdir=None, shell=None): Argument vector. :param bytes in_data: Optional standard input for the command. + :param bool emulate_tty: + If :data:`True`, arrange for stdout and stderr to be merged into the + stdout pipe and for LF to be translated into CRLF, emulating the + behaviour of a TTY. :return: (return code, stdout bytes, stderr bytes) """ LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir) assert isinstance(args, list) + if emulate_tty: + stderr = subprocess.STDOUT + else: + stderr = subprocess.PIPE + proc = subprocess.Popen( args=args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=stderr, stdin=subprocess.PIPE, cwd=chdir, ) stdout, stderr = proc.communicate(in_data) - return proc.returncode, stdout, stderr + + if emulate_tty: + stdout = stdout.replace('\n', '\r\n') + return proc.returncode, stdout, stderr or '' -def exec_command(cmd, in_data='', chdir=None, shell=None): +def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False): """ Run a command in a subprocess, emulating the argument handling behaviour of SSH. @@ -220,6 +232,7 @@ def exec_command(cmd, in_data='', chdir=None, shell=None): in_data=in_data, chdir=chdir, shell=shell, + emulate_tty=emulate_tty, ) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index f2b8aedc..2151e6b3 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -367,11 +367,11 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): if executable: cmd = executable + ' -c ' + commands.mkarg(cmd) - rc, stdout, stderr = self.call( - ansible_mitogen.helpers.exec_command, - cast(cmd), - cast(in_data), - chdir=cast(chdir), + rc, stdout, stderr = self._connection.exec_command( + cmd=cast(cmd), + in_data=cast(in_data), + sudoable=sudoable, + mitogen_chdir=cast(chdir), ) stdout_text = to_text(stdout, errors=encoding_errors) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 475e1b6d..ae6bd270 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -243,6 +243,7 @@ class ProgramRunner(Runner): try: rc, stdout, stderr = ansible_mitogen.helpers.exec_args( args=self._get_program_args(), + emulate_tty=True, ) except Exception, e: LOG.exception('While running %s', self._get_program_args()) diff --git a/docs/ansible.rst b/docs/ansible.rst index d0b309f3..fdccdbf1 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -165,28 +165,10 @@ Low Risk number of targets. This is a subject of ongoing investigation and improvements will appear in time. -* Ansible defaults to requiring pseudo TTYs for most SSH invocations, in order - to allow it to handle ``sudo`` with ``requiretty`` enabled, however it - disables pseudo TTYs for certain commands where standard input is required or - ``sudo`` is not in use. Mitogen does not require this, as it can simply call - :py:func:`pty.openpty` from the SSH user account during ``sudo`` setup. - - A major downside to Ansible's default is that stdout and stderr of any - resulting executed command are merged, with additional carriage return - characters synthesized in the output by the TTY layer. Neither of these - problems are apparent using the Mitogen extension, which may break some - playbooks. - - A future version will emulate Ansible's behaviour, once it is clear precisely - what that behaviour is supposed to be. See `Ansible#14377`_ for related - discussion. - * "Module Replacer" style modules are not yet supported. These rarely appear in practice, and light Github code searches failed to reveal many examples of them. -.. _Ansible#14377: https://github.com/ansible/ansible/issues/14377 - Behavioural Differences ----------------------- @@ -212,10 +194,6 @@ Behavioural Differences captured and returned to the host machine, where it can be viewed as desired with ``-vvv``. -* Ansible with SSH multiplexing enabled causes a string like ``Shared - connection to host closed`` to appear in ``stderr`` output of every executed - command. This never manifests with the Mitogen extension. - * Local commands are executed in a reuseable Python interpreter created identically to interpreters used on remote hosts. At present only one such interpreter per ``become_user`` exists, and so only one action may be @@ -357,6 +335,30 @@ plug-ins are unlikely to attempt similar patches, so the risk to an established configuration should be minimal. +Standard IO +~~~~~~~~~~~ + +Ansible uses pseudo TTYs for most invocations, to allow it to handle typing +passwords interactively, however it disables pseudo TTYs for certain commands +where standard input is required or ``sudo`` is not in use. Additionally when +SSH multiplexing is enabled, a string like ``Shared connection to localhost +closed\r\n`` appears in ``stderr`` of every invocation. + +Mitogen does not naturally require either of these, as command output is +embedded within the SSH stream, and it can simply call :py:func:`pty.openpty` +in every location an interactive password must be typed. + +A major downside to Ansible's behaviour is that ``stdout`` and ``stderr`` are +merged together into a single ``stdout`` variable, with carriage returns +inserted in the output by the TTY layer. However ugly, the extension emulates +all of this behaviour precisely, to avoid breaking playbooks that expect +certain text to appear in certain variables with certain linefeed characters. + +See `Ansible#14377`_ for related discussion. + +.. _Ansible#14377: https://github.com/ansible/ansible/issues/14377 + + Flag Emulation ~~~~~~~~~~~~~~ diff --git a/tests/ansible/integration/action__low_level_execute_command.yml b/tests/ansible/integration/action__low_level_execute_command.yml index fd0217bc..7cb6c410 100644 --- a/tests/ansible/integration/action__low_level_execute_command.yml +++ b/tests/ansible/integration/action__low_level_execute_command.yml @@ -28,6 +28,7 @@ - debug: msg={{raw}} - name: Verify raw module output. assert: - that: - - 'raw.rc == 0' - - 'raw.stdout_lines == ["root"]' + that: | + raw.rc == 0 and + raw.stdout == "root\r\n" and + raw.stdout_lines == ["root"] diff --git a/tests/ansible/integration/runner__custom_binary_single_null.yml b/tests/ansible/integration/runner__custom_binary_single_null.yml index c521e3b2..4933a8ec 100644 --- a/tests/ansible/integration/runner__custom_binary_single_null.yml +++ b/tests/ansible/integration/runner__custom_binary_single_null.yml @@ -15,4 +15,10 @@ out.failed and out.results[0].failed and out.results[0].msg == 'MODULE FAILURE' and - out.results[0].rc == 126 + out.results[0].module_stdout.startswith('/bin/sh: ') and + out.results[0].module_stdout.endswith('/custom_binary_single_null: cannot execute binary file\r\n') + + +# Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the +# return value and always returned 0. +# out.results[0].rc == 126 From d503956493ec6429ce6e0331632cb70a405527bc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 02:55:17 +0100 Subject: [PATCH 119/140] ansible: Remove duplicate casts already done in Connection --- ansible_mitogen/mixins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2151e6b3..2cd9d30a 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -368,10 +368,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): cmd = executable + ' -c ' + commands.mkarg(cmd) rc, stdout, stderr = self._connection.exec_command( - cmd=cast(cmd), - in_data=cast(in_data), + cmd=cmd, + in_data=in_data, sudoable=sudoable, - mitogen_chdir=cast(chdir), + mitogen_chdir=chdir, ) stdout_text = to_text(stdout, errors=encoding_errors) From 0247561fc71df8d5493634743aadb5f242935c69 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 17:26:43 +0100 Subject: [PATCH 120/140] issue #164: rename lib/modules --- tests/ansible/ansible.cfg | 2 +- .../{ => lib}/modules/bash_return_paths.sh | 0 .../modules/custom_bash_old_style_module.sh | 0 .../modules/custom_bash_want_json_module.sh | 0 .../lib/modules/custom_binary_producing_json | Bin 0 -> 8532 bytes .../modules/custom_binary_producing_json.c | 0 .../lib/modules/custom_binary_producing_junk | Bin 0 -> 8532 bytes .../modules/custom_binary_producing_junk.c | 0 .../modules/custom_binary_single_null | Bin .../modules/custom_perl_json_args_module.pl | 0 .../modules/custom_perl_want_json_module.pl | 0 .../custom_python_detect_environment.py | 23 ++++++++++++++++++ .../modules/custom_python_json_args_module.py | 0 .../modules/custom_python_new_style_module.py | 0 .../modules/custom_python_want_json_module.py | 0 15 files changed, 24 insertions(+), 1 deletion(-) rename tests/ansible/{ => lib}/modules/bash_return_paths.sh (100%) rename tests/ansible/{ => lib}/modules/custom_bash_old_style_module.sh (100%) rename tests/ansible/{ => lib}/modules/custom_bash_want_json_module.sh (100%) create mode 100755 tests/ansible/lib/modules/custom_binary_producing_json rename tests/ansible/{ => lib}/modules/custom_binary_producing_json.c (100%) create mode 100755 tests/ansible/lib/modules/custom_binary_producing_junk rename tests/ansible/{ => lib}/modules/custom_binary_producing_junk.c (100%) rename tests/ansible/{ => lib}/modules/custom_binary_single_null (100%) rename tests/ansible/{ => lib}/modules/custom_perl_json_args_module.pl (100%) rename tests/ansible/{ => lib}/modules/custom_perl_want_json_module.pl (100%) create mode 100644 tests/ansible/lib/modules/custom_python_detect_environment.py rename tests/ansible/{ => lib}/modules/custom_python_json_args_module.py (100%) rename tests/ansible/{ => lib}/modules/custom_python_new_style_module.py (100%) rename tests/ansible/{ => lib}/modules/custom_python_want_json_module.py (100%) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 73da0159..7f6524e8 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -2,7 +2,7 @@ inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy -library = modules +library = lib/modules retry_files_enabled = False forks = 50 diff --git a/tests/ansible/modules/bash_return_paths.sh b/tests/ansible/lib/modules/bash_return_paths.sh similarity index 100% rename from tests/ansible/modules/bash_return_paths.sh rename to tests/ansible/lib/modules/bash_return_paths.sh diff --git a/tests/ansible/modules/custom_bash_old_style_module.sh b/tests/ansible/lib/modules/custom_bash_old_style_module.sh similarity index 100% rename from tests/ansible/modules/custom_bash_old_style_module.sh rename to tests/ansible/lib/modules/custom_bash_old_style_module.sh diff --git a/tests/ansible/modules/custom_bash_want_json_module.sh b/tests/ansible/lib/modules/custom_bash_want_json_module.sh similarity index 100% rename from tests/ansible/modules/custom_bash_want_json_module.sh rename to tests/ansible/lib/modules/custom_bash_want_json_module.sh diff --git a/tests/ansible/lib/modules/custom_binary_producing_json b/tests/ansible/lib/modules/custom_binary_producing_json new file mode 100755 index 0000000000000000000000000000000000000000..867e5a53f647373af4b1544a4a21dde104a7d7de GIT binary patch literal 8532 zcmeHN&ubGw6n@dFB^He$64BxsZ2duPt0ndMtk-fA%!I?UHJTe?`diiUWn1$WKnRhsg1n zb{{FswV@o@5Ftch&IGVRN#=XAh98Ez@~R)WL)lVw&|Dk};RlV9-*P4M^)eyI*QFvN z91ZAH`Bo}Nm`_wqyKZ{yOXl0J@@-e~z~H?lOX{2Q1J81%RX{S|q{??u^#jMHEjM=v z(QvL14f=btHp^?4ohj zhyzDiZnqeNvyN><*FMp#qAO}y+-l3X_f4c{BEtfEc|S97>9Ko7p%`AZs?PBA9~4U zr>J#rzW=#dKQL?kNBX%As}zpzWrh3A86khW6b3(9Gy|Fe&46Y=GoTsJ3}^=aKL*Yh z=HC@=e99CS4p>}$;r5Djis*6vCmyA6d*~j0*a~am@MTu>`S0x5z}DWU^Z9S{PYO34 zW-dKgZ1#L&U$f^6Ux`jGbiZIVdal{?i1lI>&X4b#&1Mx}UmstyRkEC-H!JF%TW(D8 z=bO0fyUwuTRt?8B{WM+8O;wA|v{}v#8-dp_`;1(rXxS21inb3&&7bDpT*0($x6ioZ zdUm-#cP-7oYKgYso-A>V-@W)Gy|Fe&46Y=GoTsJ4E#q7tjlH3qTF>O0!P7BQW_ayoll3YDLST9&A!9 zr#Ib*Wr+;9*QVsjpWK+(C~>%-5$$)dhACwSU*mW%fpHcX@5lZJ&M|Qx{GWhjB=Q-2 WI=r-AA?uB?(t7OZftN58=B3}^!1&Gp literal 0 HcmV?d00001 diff --git a/tests/ansible/modules/custom_binary_producing_json.c b/tests/ansible/lib/modules/custom_binary_producing_json.c similarity index 100% rename from tests/ansible/modules/custom_binary_producing_json.c rename to tests/ansible/lib/modules/custom_binary_producing_json.c diff --git a/tests/ansible/lib/modules/custom_binary_producing_junk b/tests/ansible/lib/modules/custom_binary_producing_junk new file mode 100755 index 0000000000000000000000000000000000000000..d87110c9ad3e7e14ecaad938cc67003def434e9f GIT binary patch literal 8532 zcmeHN&1(}u6o0W*ix!QbQt@kBTECE{fdp$ILd z5UBnI;@yJ>52B!gdJ-=p;?*xuPaZ_X_0~3`m3i>?eaxFTZ+^1_o0+`$@#WW2 zqF5WzO%`CnlH{1K)#e}huyApDcWgLH-#wsm2A1;isn0`@^z|+&<;Cv zsC-M5BgiN6wo|e__C@pMRK8vn4;#2QsHMKKvhNk#iz*OC^?$cT6q1M)-_yTZC&}DBQQvai$!f& z+5U6Bjv$=yeuRDD0{(iPLb#47S0}PXN0fZ8C0~cCFDwgoG@ot@;bi`nuPc%dw%$K{ zkHhj8_SMnhV@FblhQmE!MQ~#qG%CR)LLQ|`e-H250caKxu<2mAjb%s!mTaOCmWTsJ zUCh&L1GDySq7jzMnIUyTEVya^=mD&Mw1S!6v%w zsn7BB+@1o@Ha)xe4U4C#pO|`jcJK~;+zf4Q|0NbPsqgGq&DyT#GpTPg&(c#*;^!aN z>V5awSMPhkPokr98*j5no~rlVWLe9@`RRSVUeDtz_S0L|vIRHeRYl1w<|`@#k}Pf?ea28+m2J*XKgCAB7_8Uzdi1_6VBLBJqj5HJWB1PlTO0fT@+z##CCA+V~a>m*8E zH6(C#owGf!L?=9|4dJp*UXBYsn_xfH5a)GqtG30(pG-z_CUnO oU7+5Mvk%;3VjTRRfN3c7S$sR(TCb4xYN#}(9Zt9fp&&2)28|-=-v9sr literal 0 HcmV?d00001 diff --git a/tests/ansible/modules/custom_binary_producing_junk.c b/tests/ansible/lib/modules/custom_binary_producing_junk.c similarity index 100% rename from tests/ansible/modules/custom_binary_producing_junk.c rename to tests/ansible/lib/modules/custom_binary_producing_junk.c diff --git a/tests/ansible/modules/custom_binary_single_null b/tests/ansible/lib/modules/custom_binary_single_null similarity index 100% rename from tests/ansible/modules/custom_binary_single_null rename to tests/ansible/lib/modules/custom_binary_single_null diff --git a/tests/ansible/modules/custom_perl_json_args_module.pl b/tests/ansible/lib/modules/custom_perl_json_args_module.pl similarity index 100% rename from tests/ansible/modules/custom_perl_json_args_module.pl rename to tests/ansible/lib/modules/custom_perl_json_args_module.pl diff --git a/tests/ansible/modules/custom_perl_want_json_module.pl b/tests/ansible/lib/modules/custom_perl_want_json_module.pl similarity index 100% rename from tests/ansible/modules/custom_perl_want_json_module.pl rename to tests/ansible/lib/modules/custom_perl_want_json_module.pl diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py new file mode 100644 index 00000000..8e29249a --- /dev/null +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -0,0 +1,23 @@ +#!/usr/bin/python +# I am an Ansible new-style Python module. I return details about the Python +# interpreter I run within. + +from ansible.module_utils.basic import AnsibleModule + +import os +import pwd +import socket +import sys + + +def main(): + module = AnsibleModule(argument_spec={}) + module.exit_json( + sys_executable=sys.executable, + mitogen_loaded='mitogen.core' in sys.modules, + hostname=socket.gethostname(), + username=pwd.getpwuid(os.getuid()).pw_name, + ) + +if __name__ == '__main__': + main() diff --git a/tests/ansible/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py similarity index 100% rename from tests/ansible/modules/custom_python_json_args_module.py rename to tests/ansible/lib/modules/custom_python_json_args_module.py diff --git a/tests/ansible/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py similarity index 100% rename from tests/ansible/modules/custom_python_new_style_module.py rename to tests/ansible/lib/modules/custom_python_new_style_module.py diff --git a/tests/ansible/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py similarity index 100% rename from tests/ansible/modules/custom_python_want_json_module.py rename to tests/ansible/lib/modules/custom_python_want_json_module.py From 48a0938d04bf30e40df56f3914876c740369c406 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 17:27:02 +0100 Subject: [PATCH 121/140] issue #164: add action module to return active strategy. --- tests/ansible/ansible.cfg | 1 + .../ansible/lib/action/determine_strategy.py | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tests/ansible/lib/action/determine_strategy.py diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 7f6524e8..e41b2ba0 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -2,6 +2,7 @@ inventory = hosts gathering = explicit strategy_plugins = ../../ansible_mitogen/plugins/strategy +action_plugins = lib/action library = lib/modules retry_files_enabled = False forks = 50 diff --git a/tests/ansible/lib/action/determine_strategy.py b/tests/ansible/lib/action/determine_strategy.py new file mode 100644 index 00000000..b4b067c1 --- /dev/null +++ b/tests/ansible/lib/action/determine_strategy.py @@ -0,0 +1,25 @@ + +import sys + +from ansible.plugins.strategy import StrategyBase +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def _get_strategy_name(self): + frame = sys._getframe() + while frame: + st = frame.f_locals.get('self') + if isinstance(st, StrategyBase): + return '%s.%s' % (type(st).__module__, type(st).__name__) + frame = frame.f_back + return '' + + def run(self, tmp=None, task_vars=None): + return { + 'changed': False, + 'ansible_facts': { + 'strategy': self._get_strategy_name(), + 'is_mitogen': 'ansible_mitogen' in self._get_strategy_name(), + } + } From 20ecd0af02cf2c3af2eb99d7d9fa60f98abcefe7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 17:27:16 +0100 Subject: [PATCH 122/140] issue #164: fix makefile --- tests/ansible/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ansible/Makefile b/tests/ansible/Makefile index f37789ee..26428081 100644 --- a/tests/ansible/Makefile +++ b/tests/ansible/Makefile @@ -1,4 +1,4 @@ all: \ - modules/custom_binary_producing_junk \ - modules/custom_binary_producing_json + lib/modules/custom_binary_producing_junk \ + lib/modules/custom_binary_producing_json From 680dc1bf68f3a4bdc414a9886c8b54a31b1c7828 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 17:27:42 +0100 Subject: [PATCH 123/140] issue #164: basic connection loader tests. --- tests/ansible/integration/all.yml | 1 + .../ansible/integration/connection_loader/all.yml | 3 +++ .../connection_loader/local_blemished.yml | 14 ++++++++++++++ .../connection_loader/paramiko_unblemished.yml | 12 ++++++++++++ .../connection_loader/ssh_blemished.yml | 14 ++++++++++++++ 5 files changed, 44 insertions(+) create mode 100644 tests/ansible/integration/connection_loader/all.yml create mode 100644 tests/ansible/integration/connection_loader/local_blemished.yml create mode 100644 tests/ansible/integration/connection_loader/paramiko_unblemished.yml create mode 100644 tests/ansible/integration/connection_loader/ssh_blemished.yml diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index c6dcb95b..87459301 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -3,5 +3,6 @@ # This playbook imports all tests that are known to work at present. # +- import_playbook: connection_loader/all.yml - import_playbook: action__low_level_execute_command.yml - import_playbook: runner.yml diff --git a/tests/ansible/integration/connection_loader/all.yml b/tests/ansible/integration/connection_loader/all.yml new file mode 100644 index 00000000..7a44bb2f --- /dev/null +++ b/tests/ansible/integration/connection_loader/all.yml @@ -0,0 +1,3 @@ +- import_playbook: local_blemished.yml +- import_playbook: paramiko_unblemished.yml +- import_playbook: ssh_blemished.yml diff --git a/tests/ansible/integration/connection_loader/local_blemished.yml b/tests/ansible/integration/connection_loader/local_blemished.yml new file mode 100644 index 00000000..be9873b2 --- /dev/null +++ b/tests/ansible/integration/connection_loader/local_blemished.yml @@ -0,0 +1,14 @@ +# Ensure 'local' connections are grabbed. + +- hosts: all + any_errors_fatal: true + tasks: + - name: integration/connection_loader__local_blemished.yml + determine_strategy: + + - custom_python_detect_environment: + connection: local + register: out + + - assert: + that: out.mitogen_loaded or not is_mitogen diff --git a/tests/ansible/integration/connection_loader/paramiko_unblemished.yml b/tests/ansible/integration/connection_loader/paramiko_unblemished.yml new file mode 100644 index 00000000..4959b672 --- /dev/null +++ b/tests/ansible/integration/connection_loader/paramiko_unblemished.yml @@ -0,0 +1,12 @@ +# Ensure paramiko connections aren't grabbed. + +- hosts: all + any_errors_fatal: true + tasks: + - name: integration/connection_loader__paramiko_unblemished.yml + custom_python_detect_environment: + connection: paramiko + register: out + + - assert: + that: not out.mitogen_loaded diff --git a/tests/ansible/integration/connection_loader/ssh_blemished.yml b/tests/ansible/integration/connection_loader/ssh_blemished.yml new file mode 100644 index 00000000..6b295c7e --- /dev/null +++ b/tests/ansible/integration/connection_loader/ssh_blemished.yml @@ -0,0 +1,14 @@ +# Ensure 'ssh' connections are grabbed. + +- hosts: all + any_errors_fatal: true + tasks: + - name: integration/connection_loader__ssh_blemished.yml + determine_strategy: + + - custom_python_detect_environment: + connection: ssh + register: out + + - assert: + that: out.mitogen_loaded or not is_mitogen From d068a36c1eeb5410b4a106357dbe565d0cd009b3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 17:32:56 +0100 Subject: [PATCH 124/140] issue #164: more dir layout contortions. all.yml slurps in tests from each file/subdir in the CWD. --- tests/ansible/integration/action/all.yml | 2 ++ .../low_level_execute_command.yml} | 0 tests/ansible/integration/all.yml | 5 +++-- tests/ansible/integration/playbook_semantics/all.yml | 3 +++ .../become_flags.yml} | 0 .../delegate_to.yml} | 0 .../environment.yml} | 0 tests/ansible/integration/runner.yml | 12 ------------ tests/ansible/integration/runner/all.yml | 12 ++++++++++++ .../builtin_command_module.yml} | 0 .../custom_bash_old_style_module.yml} | 0 .../custom_bash_want_json_module.yml} | 0 .../custom_binary_producing_json.yml} | 0 .../custom_binary_producing_junk.yml} | 0 .../custom_binary_single_null.yml} | 0 .../custom_perl_json_args_module.yml} | 0 .../custom_perl_want_json_module.yml} | 0 .../custom_python_json_args_module.yml} | 0 .../custom_python_new_style_module.yml} | 0 .../custom_python_want_json_module.yml} | 0 .../remote_tmp.yml} | 0 21 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 tests/ansible/integration/action/all.yml rename tests/ansible/integration/{action__low_level_execute_command.yml => action/low_level_execute_command.yml} (100%) create mode 100644 tests/ansible/integration/playbook_semantics/all.yml rename tests/ansible/integration/{playbook__become_flags.yml => playbook_semantics/become_flags.yml} (100%) rename tests/ansible/integration/{playbook__delegate_to.yml => playbook_semantics/delegate_to.yml} (100%) rename tests/ansible/integration/{playbook__environment.yml => playbook_semantics/environment.yml} (100%) delete mode 100644 tests/ansible/integration/runner.yml create mode 100644 tests/ansible/integration/runner/all.yml rename tests/ansible/integration/{runner__builtin_command_module.yml => runner/builtin_command_module.yml} (100%) rename tests/ansible/integration/{runner__custom_bash_old_style_module.yml => runner/custom_bash_old_style_module.yml} (100%) rename tests/ansible/integration/{runner__custom_bash_want_json_module.yml => runner/custom_bash_want_json_module.yml} (100%) rename tests/ansible/integration/{runner__custom_binary_producing_json.yml => runner/custom_binary_producing_json.yml} (100%) rename tests/ansible/integration/{runner__custom_binary_producing_junk.yml => runner/custom_binary_producing_junk.yml} (100%) rename tests/ansible/integration/{runner__custom_binary_single_null.yml => runner/custom_binary_single_null.yml} (100%) rename tests/ansible/integration/{runner__custom_perl_json_args_module.yml => runner/custom_perl_json_args_module.yml} (100%) rename tests/ansible/integration/{runner__custom_perl_want_json_module.yml => runner/custom_perl_want_json_module.yml} (100%) rename tests/ansible/integration/{runner__custom_python_json_args_module.yml => runner/custom_python_json_args_module.yml} (100%) rename tests/ansible/integration/{runner__custom_python_new_style_module.yml => runner/custom_python_new_style_module.yml} (100%) rename tests/ansible/integration/{runner__custom_python_want_json_module.yml => runner/custom_python_want_json_module.yml} (100%) rename tests/ansible/integration/{runner__remote_tmp.yml => runner/remote_tmp.yml} (100%) diff --git a/tests/ansible/integration/action/all.yml b/tests/ansible/integration/action/all.yml new file mode 100644 index 00000000..0f61d043 --- /dev/null +++ b/tests/ansible/integration/action/all.yml @@ -0,0 +1,2 @@ +- import_playbook: level_execute_command.yml + diff --git a/tests/ansible/integration/action__low_level_execute_command.yml b/tests/ansible/integration/action/low_level_execute_command.yml similarity index 100% rename from tests/ansible/integration/action__low_level_execute_command.yml rename to tests/ansible/integration/action/low_level_execute_command.yml diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index 87459301..c9bb1908 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -3,6 +3,7 @@ # This playbook imports all tests that are known to work at present. # +- import_playbook: action/all.yml - import_playbook: connection_loader/all.yml -- import_playbook: action__low_level_execute_command.yml -- import_playbook: runner.yml +- import_playbook: runner/all.yml +- import_playbook: playbook_semantics/all.yml diff --git a/tests/ansible/integration/playbook_semantics/all.yml b/tests/ansible/integration/playbook_semantics/all.yml new file mode 100644 index 00000000..40fa70b7 --- /dev/null +++ b/tests/ansible/integration/playbook_semantics/all.yml @@ -0,0 +1,3 @@ +- import_playbook: become_flags.yml +- import_playbook: delegate_to.yml +- import_playbook: environment.yml diff --git a/tests/ansible/integration/playbook__become_flags.yml b/tests/ansible/integration/playbook_semantics/become_flags.yml similarity index 100% rename from tests/ansible/integration/playbook__become_flags.yml rename to tests/ansible/integration/playbook_semantics/become_flags.yml diff --git a/tests/ansible/integration/playbook__delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml similarity index 100% rename from tests/ansible/integration/playbook__delegate_to.yml rename to tests/ansible/integration/playbook_semantics/delegate_to.yml diff --git a/tests/ansible/integration/playbook__environment.yml b/tests/ansible/integration/playbook_semantics/environment.yml similarity index 100% rename from tests/ansible/integration/playbook__environment.yml rename to tests/ansible/integration/playbook_semantics/environment.yml diff --git a/tests/ansible/integration/runner.yml b/tests/ansible/integration/runner.yml deleted file mode 100644 index 51583ba3..00000000 --- a/tests/ansible/integration/runner.yml +++ /dev/null @@ -1,12 +0,0 @@ -- import_playbook: runner__builtin_command_module.yml -- import_playbook: runner__custom_bash_old_style_module.yml -- import_playbook: runner__custom_bash_want_json_module.yml -- import_playbook: runner__custom_binary_producing_json.yml -- import_playbook: runner__custom_binary_producing_junk.yml -- import_playbook: runner__custom_binary_single_null.yml -- import_playbook: runner__custom_perl_json_args_module.yml -- import_playbook: runner__custom_perl_want_json_module.yml -- import_playbook: runner__custom_python_json_args_module.yml -- import_playbook: runner__custom_python_new_style_module.yml -- import_playbook: runner__custom_python_want_json_module.yml -- import_playbook: runner__remote_tmp.yml diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml new file mode 100644 index 00000000..b2424b6b --- /dev/null +++ b/tests/ansible/integration/runner/all.yml @@ -0,0 +1,12 @@ +- import_playbook: builtin_command_module.yml +- import_playbook: custom_bash_old_style_module.yml +- import_playbook: custom_bash_want_json_module.yml +- import_playbook: custom_binary_producing_json.yml +- import_playbook: custom_binary_producing_junk.yml +- import_playbook: custom_binary_single_null.yml +- import_playbook: custom_perl_json_args_module.yml +- import_playbook: custom_perl_want_json_module.yml +- import_playbook: custom_python_json_args_module.yml +- import_playbook: custom_python_new_style_module.yml +- import_playbook: custom_python_want_json_module.yml +- import_playbook: remote_tmp.yml diff --git a/tests/ansible/integration/runner__builtin_command_module.yml b/tests/ansible/integration/runner/builtin_command_module.yml similarity index 100% rename from tests/ansible/integration/runner__builtin_command_module.yml rename to tests/ansible/integration/runner/builtin_command_module.yml diff --git a/tests/ansible/integration/runner__custom_bash_old_style_module.yml b/tests/ansible/integration/runner/custom_bash_old_style_module.yml similarity index 100% rename from tests/ansible/integration/runner__custom_bash_old_style_module.yml rename to tests/ansible/integration/runner/custom_bash_old_style_module.yml diff --git a/tests/ansible/integration/runner__custom_bash_want_json_module.yml b/tests/ansible/integration/runner/custom_bash_want_json_module.yml similarity index 100% rename from tests/ansible/integration/runner__custom_bash_want_json_module.yml rename to tests/ansible/integration/runner/custom_bash_want_json_module.yml diff --git a/tests/ansible/integration/runner__custom_binary_producing_json.yml b/tests/ansible/integration/runner/custom_binary_producing_json.yml similarity index 100% rename from tests/ansible/integration/runner__custom_binary_producing_json.yml rename to tests/ansible/integration/runner/custom_binary_producing_json.yml diff --git a/tests/ansible/integration/runner__custom_binary_producing_junk.yml b/tests/ansible/integration/runner/custom_binary_producing_junk.yml similarity index 100% rename from tests/ansible/integration/runner__custom_binary_producing_junk.yml rename to tests/ansible/integration/runner/custom_binary_producing_junk.yml diff --git a/tests/ansible/integration/runner__custom_binary_single_null.yml b/tests/ansible/integration/runner/custom_binary_single_null.yml similarity index 100% rename from tests/ansible/integration/runner__custom_binary_single_null.yml rename to tests/ansible/integration/runner/custom_binary_single_null.yml diff --git a/tests/ansible/integration/runner__custom_perl_json_args_module.yml b/tests/ansible/integration/runner/custom_perl_json_args_module.yml similarity index 100% rename from tests/ansible/integration/runner__custom_perl_json_args_module.yml rename to tests/ansible/integration/runner/custom_perl_json_args_module.yml diff --git a/tests/ansible/integration/runner__custom_perl_want_json_module.yml b/tests/ansible/integration/runner/custom_perl_want_json_module.yml similarity index 100% rename from tests/ansible/integration/runner__custom_perl_want_json_module.yml rename to tests/ansible/integration/runner/custom_perl_want_json_module.yml diff --git a/tests/ansible/integration/runner__custom_python_json_args_module.yml b/tests/ansible/integration/runner/custom_python_json_args_module.yml similarity index 100% rename from tests/ansible/integration/runner__custom_python_json_args_module.yml rename to tests/ansible/integration/runner/custom_python_json_args_module.yml diff --git a/tests/ansible/integration/runner__custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml similarity index 100% rename from tests/ansible/integration/runner__custom_python_new_style_module.yml rename to tests/ansible/integration/runner/custom_python_new_style_module.yml diff --git a/tests/ansible/integration/runner__custom_python_want_json_module.yml b/tests/ansible/integration/runner/custom_python_want_json_module.yml similarity index 100% rename from tests/ansible/integration/runner__custom_python_want_json_module.yml rename to tests/ansible/integration/runner/custom_python_want_json_module.yml diff --git a/tests/ansible/integration/runner__remote_tmp.yml b/tests/ansible/integration/runner/remote_tmp.yml similarity index 100% rename from tests/ansible/integration/runner__remote_tmp.yml rename to tests/ansible/integration/runner/remote_tmp.yml From e4b49997d67c1de12155a81911f3c359af497306 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 16:34:35 +0000 Subject: [PATCH 125/140] issue #164: whups, delete checked in binaries. --- tests/ansible/.gitignore | 4 ++-- .../lib/modules/custom_binary_producing_json | Bin 8532 -> 0 bytes .../lib/modules/custom_binary_producing_junk | Bin 8532 -> 0 bytes 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100755 tests/ansible/lib/modules/custom_binary_producing_json delete mode 100755 tests/ansible/lib/modules/custom_binary_producing_junk diff --git a/tests/ansible/.gitignore b/tests/ansible/.gitignore index 14f1a005..1ea0ada7 100644 --- a/tests/ansible/.gitignore +++ b/tests/ansible/.gitignore @@ -1,2 +1,2 @@ -modules/custom_binary_producing_junk -modules/custom_binary_producing_json +lib/modules/custom_binary_producing_junk +lib/modules/custom_binary_producing_json diff --git a/tests/ansible/lib/modules/custom_binary_producing_json b/tests/ansible/lib/modules/custom_binary_producing_json deleted file mode 100755 index 867e5a53f647373af4b1544a4a21dde104a7d7de..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8532 zcmeHN&ubGw6n@dFB^He$64BxsZ2duPt0ndMtk-fA%!I?UHJTe?`diiUWn1$WKnRhsg1n zb{{FswV@o@5Ftch&IGVRN#=XAh98Ez@~R)WL)lVw&|Dk};RlV9-*P4M^)eyI*QFvN z91ZAH`Bo}Nm`_wqyKZ{yOXl0J@@-e~z~H?lOX{2Q1J81%RX{S|q{??u^#jMHEjM=v z(QvL14f=btHp^?4ohj zhyzDiZnqeNvyN><*FMp#qAO}y+-l3X_f4c{BEtfEc|S97>9Ko7p%`AZs?PBA9~4U zr>J#rzW=#dKQL?kNBX%As}zpzWrh3A86khW6b3(9Gy|Fe&46Y=GoTsJ3}^=aKL*Yh z=HC@=e99CS4p>}$;r5Djis*6vCmyA6d*~j0*a~am@MTu>`S0x5z}DWU^Z9S{PYO34 zW-dKgZ1#L&U$f^6Ux`jGbiZIVdal{?i1lI>&X4b#&1Mx}UmstyRkEC-H!JF%TW(D8 z=bO0fyUwuTRt?8B{WM+8O;wA|v{}v#8-dp_`;1(rXxS21inb3&&7bDpT*0($x6ioZ zdUm-#cP-7oYKgYso-A>V-@W)Gy|Fe&46Y=GoTsJ4E#q7tjlH3qTF>O0!P7BQW_ayoll3YDLST9&A!9 zr#Ib*Wr+;9*QVsjpWK+(C~>%-5$$)dhACwSU*mW%fpHcX@5lZJ&M|Qx{GWhjB=Q-2 WI=r-AA?uB?(t7OZftN58=B3}^!1&Gp diff --git a/tests/ansible/lib/modules/custom_binary_producing_junk b/tests/ansible/lib/modules/custom_binary_producing_junk deleted file mode 100755 index d87110c9ad3e7e14ecaad938cc67003def434e9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8532 zcmeHN&1(}u6o0W*ix!QbQt@kBTECE{fdp$ILd z5UBnI;@yJ>52B!gdJ-=p;?*xuPaZ_X_0~3`m3i>?eaxFTZ+^1_o0+`$@#WW2 zqF5WzO%`CnlH{1K)#e}huyApDcWgLH-#wsm2A1;isn0`@^z|+&<;Cv zsC-M5BgiN6wo|e__C@pMRK8vn4;#2QsHMKKvhNk#iz*OC^?$cT6q1M)-_yTZC&}DBQQvai$!f& z+5U6Bjv$=yeuRDD0{(iPLb#47S0}PXN0fZ8C0~cCFDwgoG@ot@;bi`nuPc%dw%$K{ zkHhj8_SMnhV@FblhQmE!MQ~#qG%CR)LLQ|`e-H250caKxu<2mAjb%s!mTaOCmWTsJ zUCh&L1GDySq7jzMnIUyTEVya^=mD&Mw1S!6v%w zsn7BB+@1o@Ha)xe4U4C#pO|`jcJK~;+zf4Q|0NbPsqgGq&DyT#GpTPg&(c#*;^!aN z>V5awSMPhkPokr98*j5no~rlVWLe9@`RRSVUeDtz_S0L|vIRHeRYl1w<|`@#k}Pf?ea28+m2J*XKgCAB7_8Uzdi1_6VBLBJqj5HJWB1PlTO0fT@+z##CCA+V~a>m*8E zH6(C#owGf!L?=9|4dJp*UXBYsn_xfH5a)GqtG30(pG-z_CUnO oU7+5Mvk%;3VjTRRfN3c7S$sR(TCb4xYN#}(9Zt9fp&&2)28|-=-v9sr From db894478f81221a73f1dfd4a8db01c00fc8991ca Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 17:42:43 +0100 Subject: [PATCH 126/140] issue #164: make become_flags work without FOO=2 env var. --- .../integration/playbook_semantics/become_flags.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/ansible/integration/playbook_semantics/become_flags.yml b/tests/ansible/integration/playbook_semantics/become_flags.yml index 61148169..df59bfec 100644 --- a/tests/ansible/integration/playbook_semantics/become_flags.yml +++ b/tests/ansible/integration/playbook_semantics/become_flags.yml @@ -1,5 +1,3 @@ -# This must be run with FOO=2 set in the environment. - # # Test sudo_flags respects -E. # @@ -7,6 +5,10 @@ - hosts: all any_errors_fatal: true tasks: + - name: integration/playbook_semantics/become_flags.yml + assert: + that: true + - name: "without -E" become: true shell: "echo $FOO" @@ -21,9 +23,10 @@ tasks: - name: "with -E" become: true - shell: "set" + shell: "echo $FOO" register: out2 + environment: + FOO: 2 - - debug: msg={{out2}} - assert: that: "out2.stdout == '2'" From 29288b236b6a1db6abc6244df287823a2b30b3bc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 19:34:44 +0100 Subject: [PATCH 127/140] issue #164: import run_ansible_playbook.sh. --- tests/ansible/README.md | 8 +++++++- tests/ansible/run_ansible_playbook.sh | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100755 tests/ansible/run_ansible_playbook.sh diff --git a/tests/ansible/README.md b/tests/ansible/README.md index 7eb04769..fe343125 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -8,9 +8,15 @@ It will be tidied up over time, meanwhile, the playbooks here are a useful demonstrator for what does and doesn't work. +## ``run_ansible_playbook.sh`` + +This is necessary to set some environment variables used by future tests, as +there appears to be no better way to inject them into the top-level process +environment before the Mitogen connection process forks. + ## Running Everything ``` -ANSIBLE_STRATEGY=mitogen_linear ansible-playbook all.yml +ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml ``` diff --git a/tests/ansible/run_ansible_playbook.sh b/tests/ansible/run_ansible_playbook.sh new file mode 100755 index 00000000..1df82047 --- /dev/null +++ b/tests/ansible/run_ansible_playbook.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Wrap ansible-playbook, setting up some test of the test environment. + +# Used by delegate_to.yml to ensure "sudo -E" preserves environment. +export I_WAS_PRESERVED=1 + +exec ansible-playbook "$@" From 4a823c7a27c932ca5872892775992ee396a300bd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 19:35:03 +0100 Subject: [PATCH 128/140] issue #164: missing cast() for _remote_file_exists(). --- ansible_mitogen/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2cd9d30a..0efa95b4 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -162,7 +162,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): target user account. """ LOG.debug('_remote_file_exists(%r)', path) - return self.call(os.path.exists, path) + return self.call(os.path.exists, mitogen.utils.cast(path)) def _configure_module(self, module_name, module_args, task_vars=None): """ From b9d4ec57b390f52778d1585b1b9aed42669d16d8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 19:35:21 +0100 Subject: [PATCH 129/140] issue #164: some more ActionMixin tests. --- tests/ansible/integration/action/all.yml | 6 ++- .../integration/action/make_tmp_path.yml | 25 ++++++++++ .../integration/action/remote_file_exists.yml | 38 +++++++++++++++ .../integration/action/transfer_data.yml | 47 +++++++++++++++++++ .../playbook_semantics/become_flags.yml | 6 +-- .../ansible/lib/action/action_passthrough.py | 28 +++++++++++ 6 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 tests/ansible/integration/action/make_tmp_path.yml create mode 100644 tests/ansible/integration/action/remote_file_exists.yml create mode 100644 tests/ansible/integration/action/transfer_data.yml create mode 100644 tests/ansible/lib/action/action_passthrough.py diff --git a/tests/ansible/integration/action/all.yml b/tests/ansible/integration/action/all.yml index 0f61d043..d7f2b583 100644 --- a/tests/ansible/integration/action/all.yml +++ b/tests/ansible/integration/action/all.yml @@ -1,2 +1,4 @@ -- import_playbook: level_execute_command.yml - +- import_playbook: remote_file_exists.yml +- import_playbook: low_level_execute_command.yml +- import_playbook: make_tmp_path.yml +- import_playbook: transfer_data.yml diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml new file mode 100644 index 00000000..d8fdbb43 --- /dev/null +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -0,0 +1,25 @@ + +- hosts: all + any_errors_fatal: true + tasks: + - name: integration/action/make_tmp_path.yml + assert: + that: true + + - action_passthrough: + method: _make_tmp_path + register: out + + - assert: + that: out.result.startswith(ansible_remote_tmp|expanduser) + + - stat: + path: "{{out.result}}" + register: st + + - assert: + that: st.stat.exists and st.stat.isdir and st.stat.mode == "0700" + + - file: + path: "{{out.result}}" + state: absent diff --git a/tests/ansible/integration/action/remote_file_exists.yml b/tests/ansible/integration/action/remote_file_exists.yml new file mode 100644 index 00000000..a4d2459f --- /dev/null +++ b/tests/ansible/integration/action/remote_file_exists.yml @@ -0,0 +1,38 @@ + +- hosts: all + any_errors_fatal: true + tasks: + - name: integration/action/remote_file_exists.yml + assert: + that: true + + - file: + path: /tmp/does-not-exist + state: absent + + - action_passthrough: + method: _remote_file_exists + args: ['/tmp/does-not-exist'] + register: out + + - assert: + that: out.result == False + + # --- + + - copy: + dest: /tmp/does-exist + content: "I think, therefore I am" + + - action_passthrough: + method: _remote_file_exists + args: ['/tmp/does-exist'] + register: out + + - assert: + that: out.result == True + + - file: + path: /tmp/does-exist + state: absent + diff --git a/tests/ansible/integration/action/transfer_data.yml b/tests/ansible/integration/action/transfer_data.yml new file mode 100644 index 00000000..ad4ffa56 --- /dev/null +++ b/tests/ansible/integration/action/transfer_data.yml @@ -0,0 +1,47 @@ + +- hosts: all + any_errors_fatal: true + tasks: + - name: integration/action/transfer_data.yml + file: + path: /tmp/transfer-data + state: absent + + # Ensure it JSON-encodes dicts. + - action_passthrough: + method: _transfer_data + kwargs: + remote_path: /tmp/transfer-data + data: { + "I am JSON": true + } + + - slurp: + src: /tmp/transfer-data + register: out + + - assert: + that: | + out.content.decode('base64') == '{"I am JSON": true}' + + + # Ensure it handles strings. + - action_passthrough: + method: _transfer_data + kwargs: + remote_path: /tmp/transfer-data + data: "I am text." + + - slurp: + src: /tmp/transfer-data + register: out + + - debug: msg={{out}} + + - assert: + that: + out.content.decode('base64') == 'I am text.' + + - file: + path: /tmp/transfer-data + state: absent diff --git a/tests/ansible/integration/playbook_semantics/become_flags.yml b/tests/ansible/integration/playbook_semantics/become_flags.yml index df59bfec..5922b062 100644 --- a/tests/ansible/integration/playbook_semantics/become_flags.yml +++ b/tests/ansible/integration/playbook_semantics/become_flags.yml @@ -11,7 +11,7 @@ - name: "without -E" become: true - shell: "echo $FOO" + shell: "echo $I_WAS_PRESERVED" register: out - assert: @@ -23,10 +23,10 @@ tasks: - name: "with -E" become: true - shell: "echo $FOO" + shell: "echo $I_WAS_PRESERVED" register: out2 environment: - FOO: 2 + I_WAS_PRESERVED: 2 - assert: that: "out2.stdout == '2'" diff --git a/tests/ansible/lib/action/action_passthrough.py b/tests/ansible/lib/action/action_passthrough.py new file mode 100644 index 00000000..2748a932 --- /dev/null +++ b/tests/ansible/lib/action/action_passthrough.py @@ -0,0 +1,28 @@ + +import traceback +import sys + +from ansible.plugins.strategy import StrategyBase +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + try: + method = getattr(self, self._task.args['method']) + args = tuple(self._task.args.get('args', ())) + kwargs = self._task.args.get('kwargs', {}) + + return { + 'changed': False, + 'failed': False, + 'result': method(*args, **kwargs) + } + except Exception as e: + traceback.print_exc() + return { + 'changed': False, + 'failed': True, + 'msg': str(e), + 'result': e, + } From 7fd88868a6c8939f1b37336b3b20e2c124ddac3a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 20:14:14 +0100 Subject: [PATCH 130/140] ansible: raise AnsibleConnectionFailure on connection failure; closes #183 Before: $ ANSIBLE_STRATEGY=mitogen ansible -i derp, derp -m setup An exception occurred during task execution. To see the full traceback, use -vvv. The error was: (''.join(bits)[-300:],) derp | FAILED! => { "msg": "Unexpected failure during module execution.", "stdout": "" } After: $ ANSIBLE_STRATEGY=mitogen ansible -i derp, derp -m setup derp | UNREACHABLE! => { "changed": false, "msg": "EOF on stream; last 300 bytes received: 'ssh: Could not resolve hostname derp: nodename nor servname provided, or not known\\r\\n'", "unreachable": true } --- ansible_mitogen/connection.py | 121 +++++++++++++++++----------------- ansible_mitogen/services.py | 16 ++++- 2 files changed, 75 insertions(+), 62 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a538f743..db182930 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -37,7 +37,7 @@ import ansible.errors import ansible.plugins.connection import mitogen.unix -from mitogen.utils import cast +import mitogen.utils import ansible_mitogen.helpers import ansible_mitogen.process @@ -137,59 +137,63 @@ class Connection(ansible.plugins.connection.ConnectionBase): def connected(self): return self.broker is not None + def _wrap_connect(self, args): + dct = mitogen.service.call( + context=self.parent, + handle=ContextService.handle, + obj=mitogen.utils.cast(args), + ) + + if dct['msg']: + raise ansible.errors.AnsibleConnectionFailure(dct['msg']) + + return dct['context'], dct['home_dir'] + def _connect_local(self): """ Fetch a reference to the local() Context from ContextService in the master process. """ - return mitogen.service.call(self.parent, ContextService.handle, cast({ + return self._wrap_connect({ 'method': 'local', 'python_path': self.python_path, - })) + }) def _connect_ssh(self): """ Fetch a reference to an SSH Context matching the play context from ContextService in the master process. """ - return mitogen.service.call( - self.parent, - ContextService.handle, - cast({ - 'method': 'ssh', - 'check_host_keys': False, # TODO - 'hostname': self._play_context.remote_addr, - 'discriminator': self.mitogen_ssh_discriminator, - 'username': self._play_context.remote_user, - 'password': self._play_context.password, - 'port': self._play_context.port, - 'python_path': self.python_path, - 'identity_file': self._play_context.private_key_file, - 'ssh_path': self._play_context.ssh_executable, - 'connect_timeout': self.ansible_ssh_timeout, - 'ssh_args': [ - term - for s in ( - getattr(self._play_context, 'ssh_args', ''), - getattr(self._play_context, 'ssh_common_args', ''), - getattr(self._play_context, 'ssh_extra_args', '') - ) - for term in shlex.split(s or '') - ] - }) - ) + return self._wrap_connect({ + 'method': 'ssh', + 'check_host_keys': False, # TODO + 'hostname': self._play_context.remote_addr, + 'discriminator': self.mitogen_ssh_discriminator, + 'username': self._play_context.remote_user, + 'password': self._play_context.password, + 'port': self._play_context.port, + 'python_path': self.python_path, + 'identity_file': self._play_context.private_key_file, + 'ssh_path': self._play_context.ssh_executable, + 'connect_timeout': self.ansible_ssh_timeout, + 'ssh_args': [ + term + for s in ( + getattr(self._play_context, 'ssh_args', ''), + getattr(self._play_context, 'ssh_common_args', ''), + getattr(self._play_context, 'ssh_extra_args', '') + ) + for term in shlex.split(s or '') + ] + }) def _connect_docker(self): - return mitogen.service.call( - self.parent, - ContextService.handle, - cast({ - 'method': 'docker', - 'container': self._play_context.remote_addr, - 'python_path': self.python_path, - 'connect_timeout': self._play_context.timeout, - }) - ) + return self._wrap_connect({ + 'method': 'docker', + 'container': self._play_context.remote_addr, + 'python_path': self.python_path, + 'connect_timeout': self._play_context.timeout, + }) def _connect_sudo(self, via=None, python_path=None): """ @@ -200,23 +204,19 @@ class Connection(ansible.plugins.connection.ConnectionBase): Parent Context of the sudo Context. For Ansible, this should always be a Context returned by _connect_ssh(). """ - return mitogen.service.call( - self.parent, - ContextService.handle, - cast({ - 'method': 'sudo', - 'username': self._play_context.become_user, - 'password': self._play_context.become_pass, - 'python_path': python_path or self.python_path, - 'sudo_path': self.sudo_path, - 'connect_timeout': self._play_context.timeout, - 'via': via, - 'sudo_args': shlex.split( - self._play_context.sudo_flags or - self._play_context.become_flags or '' - ), - }) - ) + return self._wrap_connect({ + 'method': 'sudo', + 'username': self._play_context.become_user, + 'password': self._play_context.become_pass, + 'python_path': python_path or self.python_path, + 'sudo_path': self.sudo_path, + 'connect_timeout': self._play_context.timeout, + 'via': via, + 'sudo_args': shlex.split( + self._play_context.sudo_flags or + self._play_context.become_flags or '' + ), + }) def _connect(self): """ @@ -318,8 +318,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): emulate_tty = (not in_data and sudoable) rc, stdout, stderr = self.call( ansible_mitogen.helpers.exec_command, - cmd=cast(cmd), - in_data=cast(in_data), + cmd=mitogen.utils.cast(cmd), + in_data=mitogen.utils.cast(in_data), chdir=mitogen_chdir, emulate_tty=emulate_tty, ) @@ -341,7 +341,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): Local filesystem path to write. """ output = self.call(ansible_mitogen.helpers.read_path, - cast(in_path)) + mitogen.utils.cast(in_path)) ansible_mitogen.helpers.write_path(out_path, output) def put_data(self, out_path, data): @@ -355,7 +355,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): Remote filesystem path to write. """ self.call(ansible_mitogen.helpers.write_path, - cast(out_path), cast(data)) + mitogen.utils.cast(out_path), + mitogen.utils.cast(data)) def put_file(self, in_path, out_path): """ diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 7bc59135..18782f20 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -81,9 +81,21 @@ class ContextService(mitogen.service.DeduplicatingService): def get_response(self, args): args.pop('discriminator', None) method = getattr(self.router, args.pop('method')) - context = method(**args) + try: + context = method(**args) + except mitogen.core.StreamError as e: + return { + 'context': None, + 'home_dir': None, + 'msg': str(e), + } + home_dir = context.call(os.path.expanduser, '~') - return context, home_dir + return { + 'context': context, + 'home_dir': home_dir, + 'msg': None, + } class FileService(mitogen.service.Service): From 35fdd97f9aafa04cac1f8ef9b9f8637c944c3032 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 20:24:11 +0100 Subject: [PATCH 131/140] issue #164: utility to print Docker hostname for use from shell scripts. --- tests/show_docker_hostname.py | 12 ++++++++++++ tests/testlib.py | 14 +++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 tests/show_docker_hostname.py diff --git a/tests/show_docker_hostname.py b/tests/show_docker_hostname.py new file mode 100644 index 00000000..1dc1cb98 --- /dev/null +++ b/tests/show_docker_hostname.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +""" +For use by the Travis scripts, just print out the hostname of the Docker +daemon from the environment. +""" + +import docker +import testlib + +docker = docker.from_env(version='auto') +print testlib.get_docker_host(docker) diff --git a/tests/testlib.py b/tests/testlib.py index cac5b1e9..199e8a46 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -161,6 +161,14 @@ class TestCase(unittest2.TestCase): assert 0, '%r did not raise %r' % (func, exc) +def get_docker_host(docker): + if docker.api.base_url == 'http+docker://localunixsocket': + return 'localhost' + + parsed = urlparse.urlparse(docker.api.base_url) + return parsed.netloc.partition(':')[0] + + class DockerizedSshDaemon(object): def __init__(self): self.docker = docker.from_env(version='auto') @@ -177,11 +185,7 @@ class DockerizedSshDaemon(object): self.host = self.get_host() def get_host(self): - if self.docker.api.base_url == 'http+docker://localunixsocket': - return 'localhost' - - parsed = urlparse.urlparse(self.docker.api.base_url) - return parsed.netloc.partition(':')[0] + return get_docker_host(self.docker) def wait_for_sshd(self): wait_for_port(self.get_host(), int(self.port), pattern='OpenSSH') From 998a1209cc83c29596ab4e9dd4f5a77b8bb0c36c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 21:48:43 +0100 Subject: [PATCH 132/140] issue #183: make PasswordErrors subclass of StreamError. --- mitogen/ssh.py | 2 +- mitogen/sudo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 1893fdc9..d59bc159 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -43,7 +43,7 @@ PASSWORD_PROMPT = 'password' PERMDENIED_PROMPT = 'permission denied' -class PasswordError(mitogen.core.Error): +class PasswordError(mitogen.core.StreamError): pass diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 3295083e..96c37e90 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -98,7 +98,7 @@ def parse_sudo_flags(args): return opts -class PasswordError(mitogen.core.Error): +class PasswordError(mitogen.core.StreamError): pass From cd098ef158e9b9cf160fcd652a3b17b84ce4f7c3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 22:40:02 +0100 Subject: [PATCH 133/140] issue #183: re-raise StreamError in calling context. This allows catching just StreamError regardless of via=None or via=. Deserves a more general solution, but it's easy to fix up later. --- mitogen/parent.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 41fe3676..da82bd66 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -314,12 +314,24 @@ METHOD_NAMES = { @mitogen.core.takes_econtext def _proxy_connect(name, method_name, kwargs, econtext): mitogen.parent.upgrade_router(econtext) - context = econtext.router._connect( - klass=METHOD_NAMES[method_name](), - name=name, - **kwargs - ) - return context.context_id, context.name + try: + context = econtext.router._connect( + klass=METHOD_NAMES[method_name](), + name=name, + **kwargs + ) + except mitogen.core.StreamError, e: + return { + 'id': None, + 'name': None, + 'msg': str(e), + } + + return { + 'id': context.context_id, + 'name': context.name, + 'msg': None, + } class Stream(mitogen.core.Stream): @@ -743,14 +755,16 @@ class Router(mitogen.core.Router): return self._connect(klass, name=name, **kwargs) def proxy_connect(self, via_context, method_name, name=None, **kwargs): - context_id, name = via_context.call(_proxy_connect, + resp = via_context.call(_proxy_connect, name=name, method_name=method_name, kwargs=kwargs ) - name = '%s.%s' % (via_context.name, name) + if resp['msg'] is not None: + raise mitogen.core.StreamError(resp['msg']) - context = self.context_class(self, context_id, name=name) + name = '%s.%s' % (via_context.name, resp['name']) + context = self.context_class(self, resp['id'], name=name) context.via = via_context self._context_by_id[context.context_id] = context return context From bc4a6b39bf8c2882d094afd3d4464edc5560430a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 22:52:02 +0100 Subject: [PATCH 134/140] issue #164: teach debops_tests.sh to use SSH Login with a non-privileged account over SSH rather than just jumping straight in as root via Docker. --- .travis/debops_tests.sh | 52 ++++++++++++++++++++++--------------- tests/build_docker_image.py | 1 + 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/.travis/debops_tests.sh b/.travis/debops_tests.sh index f77b9d25..6e423aec 100755 --- a/.travis/debops_tests.sh +++ b/.travis/debops_tests.sh @@ -3,16 +3,18 @@ TMPDIR="/tmp/debops-$$" TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" +TARGET_COUNT="${TARGET_COUNT:-4}" + function on_exit() { echo travis_fold:start:cleanup [ "$KEEP" ] || { - rm -rvf "$TMPDIR" || true - docker kill target1 || true - docker kill target2 || true - docker kill target3 || true - docker kill target4 || true + rm -rf "$TMPDIR" || true + for i in $(seq $TARGET_COUNT) + do + docker kill target$i || true + done } echo travis_fold:end:cleanup } @@ -21,16 +23,8 @@ trap on_exit EXIT mkdir "$TMPDIR" -echo travis_fold:start:docker_setup -docker run --rm --detach --name=target1 d2mw/mitogen-test /bin/sleep 86400 -docker run --rm --detach --name=target2 d2mw/mitogen-test /bin/sleep 86400 -docker run --rm --detach --name=target3 d2mw/mitogen-test /bin/sleep 86400 -docker run --rm --detach --name=target4 d2mw/mitogen-test /bin/sleep 86400 -echo travis_fold:end:docker_setup - - echo travis_fold:start:job_setup -pip install -U debops==0.7.2 ansible==2.4.3.0 +pip install -qqqU debops==0.7.2 ansible==2.4.3.0 debops-init "$TMPDIR/project" cd "$TMPDIR/project" @@ -41,19 +35,35 @@ strategy = mitogen_linear EOF cat > ansible/inventory/group_vars/debops_all_hosts.yml <<-EOF -ansible_connection: docker ansible_python_interpreter: /usr/bin/python2.7 +ansible_user: has-sudo-pubkey +ansible_become_pass: y +ansible_ssh_private_key_file: ${TRAVIS_BUILD_DIR}/tests/data/docker/has-sudo-pubkey.key + # Speed up slow DH generation. dhparam__bits: ["128", "64"] EOF -cat >> ansible/inventory/hosts <<-EOF -target1 -target2 -target3 -target4 -EOF +DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)" + +for i in $(seq $TARGET_COUNT) +do + port=$((2200 + $i)) + docker run \ + --rm \ + --detach \ + --publish 0.0.0.0:$port:22/tcp \ + --name=target$i \ + d2mw/mitogen-test + + echo \ + target$i \ + ansible_host=$DOCKER_HOSTNAME \ + ansible_port=$port \ + >> ansible/inventory/hosts +done + echo travis_fold:end:job_setup diff --git a/tests/build_docker_image.py b/tests/build_docker_image.py index 68ba4100..02172e04 100755 --- a/tests/build_docker_image.py +++ b/tests/build_docker_image.py @@ -24,6 +24,7 @@ RUN \ useradd -m webapp && \ ( echo 'root:x' | chpasswd; ) && \ ( echo 'has-sudo:y' | chpasswd; ) && \ + ( echo 'has-sudo-pubkey:y' | chpasswd; ) && \ ( echo 'has-sudo-nopw:y' | chpasswd; ) && \ mkdir ~has-sudo-pubkey/.ssh && \ { echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; } From f655be1455eeba2a5450277acc9443f908203ab9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 5 Apr 2018 22:53:19 +0100 Subject: [PATCH 135/140] ssh: fix password prompt check when running with -vvv Can only happen by hacking -vvv into ssh.py at present, but that will probably be exposed via a constructor parameter in future. --- mitogen/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index d59bc159..15e9ac02 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -39,7 +39,7 @@ import mitogen.parent LOG = logging.getLogger('mitogen') -PASSWORD_PROMPT = 'password' +PASSWORD_PROMPT = 'password:' PERMDENIED_PROMPT = 'permission denied' From b247c320d2088d2a7914b292703a3b1e14834db7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 6 Apr 2018 15:10:02 +0100 Subject: [PATCH 136/140] issue #164: rename tests for clarity --- ...yml => issue_152__local_action_wrong_interpreter.yml} | 9 ++++++++- ...ue_152.yml => issue_152__virtualenv_python_fails.yml} | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) rename tests/ansible/regression/{issue_152b.yml => issue_152__local_action_wrong_interpreter.yml} (65%) rename tests/ansible/regression/{issue_152.yml => issue_152__virtualenv_python_fails.yml} (94%) diff --git a/tests/ansible/regression/issue_152b.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml similarity index 65% rename from tests/ansible/regression/issue_152b.yml rename to tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index 962f1d6f..e4b0adbc 100644 --- a/tests/ansible/regression/issue_152b.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -9,4 +9,11 @@ - hosts: all tasks: - - local_action: cloudformation_facts + - name: regression/issue_152__local_action_wrong_interpreter.yml + connection: local + become: true + shell: pip uninstall boto3 + ignore_errors: true + + - cloudformation_facts: + connection: local diff --git a/tests/ansible/regression/issue_152.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml similarity index 94% rename from tests/ansible/regression/issue_152.yml rename to tests/ansible/regression/issue_152__virtualenv_python_fails.yml index dff424b1..e6c60c99 100644 --- a/tests/ansible/regression/issue_152.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -8,8 +8,6 @@ - name: Use virtualenv for the Python interpeter set_fact: ansible_python_interpreter=/tmp/issue_151_virtualenv/bin/python - - command: sleep 123123 - - name: Ensure the app DB user exists postgresql_user: db: postgres From b5953146193db40d019975f113b19d5cebf94895 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 6 Apr 2018 16:49:54 +0100 Subject: [PATCH 137/140] docs: fix intensely annoying _prefix, 2 years later. --- .gitignore | 1 - docs/.gitignore | 1 + docs/Makefile | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 docs/.gitignore diff --git a/.gitignore b/.gitignore index 09a0cb67..458cf82c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ MANIFEST build/ dist/ -docs/_build htmlcov/ *.egg-info __pycache__/ diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +build diff --git a/docs/Makefile b/docs/Makefile index ce73f29a..bc394d34 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ default: SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = -BUILDDIR = _build +BUILDDIR = build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) From 432ebbca896909454a756278ddf81f7350632985 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 6 Apr 2018 17:09:14 +0100 Subject: [PATCH 138/140] issue #106: docs: initial docs for how modules execute. --- docs/ansible.rst | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docs/ansible.rst b/docs/ansible.rst index fdccdbf1..e0a5037f 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -203,6 +203,56 @@ Behavioural Differences release. +How Modules Execute +------------------- + +Ansible usually modifies and recompresses a module each time it runs on a +target, work that must be repeated for every playbook step. With the extension +much of this work is pushed to the target, allowing pristine copies of the +module to be cached by the target, reducing the necessity to re-transfer +modified modules for every invocation. + +**Binary** + * Native executables detected using a complex heuristic. + * Arguments are supplied as a JSON file whose path is the sole script + parameter. + * Downloaded from the master on first use, and cached in the target for the + remainder of the run. + +**Module Replacer** + * Python scripts detected by the presence of + ``#<>`` appearing in their source. + * This type is not yet supported. + +**New-Style** + * Python scripts detected by the presence of ``from ansible.module_utils.`` + appearing in their source. + * Arguments are supplied as JSON written to ``sys.stdin`` of the target + interpreter. + +**JSON_ARGS** + * Detected by the presence of ``INCLUDE_ANSIBLE_MODULE_JSON_ARGS`` + appearing in the script source. + * The interpreter directive (``#!interpreter``) is adjusted to match the + corresponding value of ``{{ansible_*_interpreter}}`` if one is set. + * Arguments are supplied as JSON mixed into the script as a replacement for + ``INCLUDE_ANSIBLE_MODULE_JSON_ARGS``. + +**WANT_JSON** + * Detected by the presence of ``WANT_JSON`` appearing in the script source. + * The interpreter directive is adjusted to match the corresponding value + of ``{{ansible_*_interpreter}}`` if one is set. + * Arguments are supplied as a JSON file whose path is the sole script + parameter. + +**Old Style** + * Files not matching any of the above tests. + * The interpreter directive is adjusted to match the corresponding value + of ``{{ansible_*_interpreter}}`` if one is set. + * Arguments are supplied as a file whose path is the sole script parameter. + The format of the file is "``key=repr(value)[ key2=repr(value2)[ ..]]``". + + Sample Profiles --------------- From a643f13ebe4b9ecff9d38c26c114f3346b2edd0b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 6 Apr 2018 17:18:37 +0100 Subject: [PATCH 139/140] issue #106: docs: tidyup. --- docs/ansible.rst | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index e0a5037f..15ccc07a 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -65,7 +65,7 @@ concurrent to an equivalent run using the extension. .. raw:: html -