From 26fdbf70e0cd3505b84fb9b630b29a8ba89a2e66 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 06:18:03 +0000 Subject: [PATCH 01/10] tests: good_load_module_size check can't be exact Since message contains pathnames that vary across machines --- tests/responder_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/responder_test.py b/tests/responder_test.py index dd790880..3b5f8e98 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -76,7 +76,7 @@ class GoodModulesTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(256, context.call(plain_old_module.pow, 2, 8)) self.assertEquals(1, self.router.responder.get_module_count) self.assertEquals(1, self.router.responder.good_load_module_count) - self.assertEquals(359, self.router.responder.good_load_module_size) + self.assertLess(300, self.router.responder.good_load_module_size) def test_simple_pkg(self): # Ensure success of a simple package containing two submodules, one of @@ -87,7 +87,7 @@ class GoodModulesTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(2, self.router.responder.get_module_count) self.assertEquals(3, self.router.responder.good_load_module_count) self.assertEquals(0, self.router.responder.bad_load_module_count) - self.assertEquals(537, self.router.responder.good_load_module_size) + self.assertLess(450, self.router.responder.good_load_module_size) def test_self_contained_program(self): # Ensure a program composed of a single script can be imported @@ -171,7 +171,7 @@ class ForwardTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(256, c2.call(plain_old_module.pow, 2, 8)) self.assertEquals(3, self.router.responder.get_module_count) self.assertEquals(5, self.router.responder.good_load_module_count) - self.assertEquals(28148, self.router.responder.good_load_module_size) + self.assertLess(27000, self.router.responder.good_load_module_size) class BlacklistTest(testlib.TestCase): From b6840aab753d826c698015d7a4ad8cef25da71c0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 06:24:23 +0000 Subject: [PATCH 02/10] issue #459: one line stats output during shutdown CI logs are too noisy. --- mitogen/master.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 25e36c8d..04399ab1 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -879,16 +879,13 @@ class Router(mitogen.parent.Router): ) LOG.debug( - '%(self)r: stats:\n' - ' GET_MODULE requests: %(get_module_count)d\n' - ' GET_MODULE runtime: %(get_module_ms)d ms\n' - ' LOAD_MODULE responses: %(good_load_module_count)d\n' - ' Negative LOAD_MODULE responses: ' - '%(bad_load_module_count)d\n' - ' LOAD_MODULE total bytes sent: ' - '%(good_load_module_size_kb).02f kb\n' - ' LOAD_MODULE avg bytes sent: ' - '%(good_load_module_size_avg).02f kb' + '%(self)r: stats: ' + '%(get_module_count)d module requests in ' + '%(get_module_ms)d ms, ' + '%(good_load_module_count)d sent, ' + '%(bad_load_module_count)d negative responses. ' + 'Sent %(good_load_module_size_kb).01f kb total, ' + '%(good_load_module_size_avg).01f kb avg.' % dct ) From b0e7c1a315008993f48756579bafc106fce2f3a7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 06:46:22 +0000 Subject: [PATCH 03/10] tests: fix /etc/environment test on vanilla Use "meta: reset_connection" to ensure /etc/environment is reloaded. Looks like this entire feature can be ripped out! --- .../ansible/integration/runner/_etc_environment_global.yml | 6 ++++++ tests/ansible/integration/runner/_reset_conn.yml | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 tests/ansible/integration/runner/_reset_conn.yml diff --git a/tests/ansible/integration/runner/_etc_environment_global.yml b/tests/ansible/integration/runner/_etc_environment_global.yml index e24a038a..db7961d1 100644 --- a/tests/ansible/integration/runner/_etc_environment_global.yml +++ b/tests/ansible/integration/runner/_etc_environment_global.yml @@ -17,6 +17,9 @@ MAGIC_ETC_ENV=555 become: true +- include_tasks: _reset_conn.yml + when: not is_mitogen + - shell: echo $MAGIC_ETC_ENV register: echo @@ -28,6 +31,9 @@ state: absent become: true +- include_tasks: _reset_conn.yml + when: not is_mitogen + - shell: echo $MAGIC_ETC_ENV register: echo diff --git a/tests/ansible/integration/runner/_reset_conn.yml b/tests/ansible/integration/runner/_reset_conn.yml new file mode 100644 index 00000000..30f1b0c0 --- /dev/null +++ b/tests/ansible/integration/runner/_reset_conn.yml @@ -0,0 +1,2 @@ + +- meta: reset_connection From cce1dbf3b141b7ede1b3a734ed4b2593005b7b1e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 07:08:11 +0000 Subject: [PATCH 04/10] tests: quieten a bunch of spam printed during run --- mitogen/master.py | 4 ++-- mitogen/service.py | 2 +- tests/data/six_brokenpkg/__init__.py | 4 ++-- tests/service_test.py | 18 ++++++++++++++---- tests/two_three_compat_test.py | 1 + tests/unix_test.py | 14 +++++++++++--- 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 04399ab1..7a9f1dd9 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -714,8 +714,8 @@ class ModuleResponder(object): if tup[2] and is_stdlib_path(tup[2]): # Prevent loading of 2.x<->3.x stdlib modules! This costs one # RTT per hit, so a client-side solution is also required. - LOG.warning('%r: refusing to serve stdlib module %r', - self, fullname) + LOG.debug('%r: refusing to serve stdlib module %r', + self, fullname) self._send_module_load_failed(stream, fullname) return diff --git a/mitogen/service.py b/mitogen/service.py index 5f6f4048..d1285386 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -542,7 +542,7 @@ class Pool(object): msg = self._select.get() except (mitogen.core.ChannelError, mitogen.core.LatchError): e = sys.exc_info()[1] - LOG.info('%r: channel or latch closed, exitting: %s', self, e) + LOG.debug('%r: channel or latch closed, exitting: %s', self, e) return func = self._func_by_recv[msg.receiver] diff --git a/tests/data/six_brokenpkg/__init__.py b/tests/data/six_brokenpkg/__init__.py index 42e633c7..e5944b83 100644 --- a/tests/data/six_brokenpkg/__init__.py +++ b/tests/data/six_brokenpkg/__init__.py @@ -29,9 +29,9 @@ import os.path try: import six as _system_six - print('six_brokenpkg: using system six:', _system_six) + #print('six_brokenpkg: using system six:', _system_six) except ImportError: - print('six_brokenpkg: no system six available') + #print('six_brokenpkg: no system six available') _system_six = None if _system_six: diff --git a/tests/service_test.py b/tests/service_test.py index 0ff16032..e57548da 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -54,8 +54,13 @@ class ActivationTest(testlib.RouterMixin, testlib.TestCase): l1 = self.router.fork() l2 = self.router.fork() l1.call_service(MyService, 'get_id') # force framework activation - exc = self.assertRaises(mitogen.core.CallError, - lambda: l2.call(call_service_in, l1, MyService2.name(), 'get_id')) + capture = testlib.LogCapturer() + capture.start() + try: + exc = self.assertRaises(mitogen.core.CallError, + lambda: l2.call(call_service_in, l1, MyService2.name(), 'get_id')) + finally: + capture.stop() msg = mitogen.service.Activator.not_active_msg % (MyService2.name(),) self.assertTrue(msg in exc.args[0]) @@ -80,8 +85,13 @@ class PermissionTest(testlib.RouterMixin, testlib.TestCase): l1 = self.router.fork() l1.call_service(MyService, 'get_id') l2 = self.router.fork() - exc = self.assertRaises(mitogen.core.CallError, lambda: - l2.call(call_service_in, l1, MyService.name(), 'privileged_op')) + capture = testlib.LogCapturer() + capture.start() + try: + exc = self.assertRaises(mitogen.core.CallError, lambda: + l2.call(call_service_in, l1, MyService.name(), 'privileged_op')) + finally: + capture.stop() msg = mitogen.service.Invoker.unauthorized_msg % ( u'privileged_op', MyService.name(), diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index babbbe39..f6f092d0 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -13,6 +13,7 @@ import testlib def roundtrip(*args): return args + class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: python_path = 'python2' diff --git a/tests/unix_test.py b/tests/unix_test.py index ee9499ba..300efb84 100644 --- a/tests/unix_test.py +++ b/tests/unix_test.py @@ -62,9 +62,17 @@ class ListenerTest(testlib.RouterMixin, testlib.TestCase): def test_constructor_basic(self): listener = self.klass(router=self.router) - self.assertFalse(mitogen.unix.is_path_dead(listener.path)) - os.unlink(listener.path) - + capture = testlib.LogCapturer() + capture.start() + try: + self.assertFalse(mitogen.unix.is_path_dead(listener.path)) + os.unlink(listener.path) + # ensure we catch 0 byte read error log message + self.broker.shutdown() + self.broker.join() + self.broker_shutdown = True + finally: + capture.stop() class ClientTest(testlib.TestCase): klass = mitogen.unix.Listener From 6592598999d993da7c8726c63be5053f87afdd26 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 07:15:53 +0000 Subject: [PATCH 05/10] tests: stray socket file left during unix_test. --- tests/unix_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unix_test.py b/tests/unix_test.py index 300efb84..0bc5fc81 100644 --- a/tests/unix_test.py +++ b/tests/unix_test.py @@ -74,6 +74,7 @@ class ListenerTest(testlib.RouterMixin, testlib.TestCase): finally: capture.stop() + class ClientTest(testlib.TestCase): klass = mitogen.unix.Listener @@ -97,6 +98,7 @@ class ClientTest(testlib.TestCase): self.assertEquals(0, resp['auth_id']) router.broker.shutdown() router.broker.join() + os.unlink(path) def _test_simple_server(self, path): router = mitogen.master.Router() From 5ef94eb3e28c09dfe5d128be34f40e42bc0dfb39 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 07:29:11 +0000 Subject: [PATCH 06/10] issue #456: loosen Waker.defer() shutdown test a little Allow messages to continue being queued during the shutdown period, right up until the final loop iteration, even though this is racy, as too many things depend on .defer() during exit right now. This doesn't hurt the spirit of the check: it still catches the worst situation where $user accidentally shut down Broker then tried to continue using it. --- mitogen/core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 8c03145f..97c12bcd 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2122,8 +2122,8 @@ class Waker(BasicStream): broker_shutdown_msg = ( "An attempt was made to enqueue a message with a Broker that has " - "already begun shutting down. It is likely your program called " - "Broker.shutdown() too early." + "already exitted. It is likely your program called Broker.shutdown() " + "too early." ) def defer(self, func, *args, **kwargs): @@ -2138,7 +2138,7 @@ class Waker(BasicStream): if threading.currentThread().ident == self.broker_ident: _vv and IOLOG.debug('%r.defer() [immediate]', self) return func(*args, **kwargs) - if not self._broker._alive: + if self._broker._exitted: raise Error(self.broker_shutdown_msg) _vv and IOLOG.debug('%r.defer() [fd=%r]', self, self.transmit_side.fd) @@ -2564,6 +2564,7 @@ class Broker(object): def __init__(self, poller_class=None): self._alive = True + self._exitted = False self._waker = Waker(self) #: Arrange for `func(\*args, \**kwargs)` to be executed on the broker #: thread, or immediately if the current thread is the broker thread. @@ -2724,6 +2725,7 @@ class Broker(object): except Exception: LOG.exception('_broker_main() crashed') + self._exitted = True self._broker_exit() fire(self, 'exit') From 57b652eddca3aeb93be38c6b9490da946c91836c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 07:36:31 +0000 Subject: [PATCH 07/10] parent: remove unused imports The stray functools import must have been there forever! Instant 4kb knocked off wire footprint. --- mitogen/parent.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index eb17a1b7..8d9fbd2a 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -53,16 +53,6 @@ import zlib # Absolute imports for <2.5. select = __import__('select') -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -try: - from functools import lru_cache -except ImportError: - from mitogen.compat.functools import lru_cache - import mitogen.core from mitogen.core import b from mitogen.core import LOG From 386d869d5f1ad54d37f53c12f6b3a56e0947b833 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 08:11:54 +0000 Subject: [PATCH 08/10] tests: fix responder_test after removing unused imports --- tests/responder_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/responder_test.py b/tests/responder_test.py index 3b5f8e98..60e4ae67 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -67,12 +67,12 @@ class NeutralizeMainTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(bits[-3:], ['def', 'main():', 'pass']) - class GoodModulesTest(testlib.RouterMixin, testlib.TestCase): def test_plain_old_module(self): # The simplest case: a top-level module with no interesting imports or # package machinery damage. context = self.router.local() + self.assertEquals(256, context.call(plain_old_module.pow, 2, 8)) self.assertEquals(1, self.router.responder.get_module_count) self.assertEquals(1, self.router.responder.good_load_module_count) @@ -170,8 +170,8 @@ class ForwardTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(256, c2.call(plain_old_module.pow, 2, 8)) self.assertEquals(3, self.router.responder.get_module_count) - self.assertEquals(5, self.router.responder.good_load_module_count) - self.assertLess(27000, self.router.responder.good_load_module_size) + self.assertEquals(3, self.router.responder.good_load_module_count) + self.assertLess(23000, self.router.responder.good_load_module_size) class BlacklistTest(testlib.TestCase): From 822978520f625958e828e1c6ade23cc2e582b570 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 08:13:04 +0000 Subject: [PATCH 09/10] issue #446: update Receiver.__iter__ to match iter() previously relied on the fake dead message being enqueued. --- mitogen/core.py | 5 +++-- tests/receiver_test.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 97c12bcd..d6a2fbb4 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -894,8 +894,9 @@ class Receiver(object): until :class:`ChannelError` is raised. """ while True: - msg = self.get(throw_dead=False) - if msg.is_dead: + try: + msg = self.get() + except ChannelError: return yield msg diff --git a/tests/receiver_test.py b/tests/receiver_test.py index 550b4525..8942ba29 100644 --- a/tests/receiver_test.py +++ b/tests/receiver_test.py @@ -37,6 +37,27 @@ class IterationTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(list(range(5)), list(m.unpickle() for m in recv)) self.assertEquals(10, ret.get().unpickle()) + def iter_and_put(self, recv, latch): + try: + for msg in recv: + latch.put(msg) + except Exception: + latch.put(sys.exc_info()[1]) + + def test_close_stops_iteration(self): + recv = mitogen.core.Receiver(self.router) + latch = mitogen.core.Latch() + t = threading.Thread( + target=self.iter_and_put, + args=(recv, latch), + ) + t.start() + t.join(0.1) + recv.close() + t.join() + self.assertTrue(latch.empty()) + + class CloseTest(testlib.RouterMixin, testlib.TestCase): def wait(self, latch, wait_recv): From 5761652e02e17bd41128880f84d1a6c999b6b4d8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 20 Jan 2019 08:30:55 +0000 Subject: [PATCH 10/10] core: allow Router.shutdown() to succeed after exit. For join_thread(): Exception in thread mitogen.master.join_thread_async: Traceback (most recent call last): File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/home/dmw/src/mitogen/mitogen/master.py", line 249, in _watch watcher.on_join() File "/home/dmw/src/mitogen/mitogen/master.py", line 816, in shutdown super(Broker, self).shutdown() File "/home/dmw/src/mitogen/mitogen/core.py", line 2741, in shutdown self.defer(_shutdown) File "/home/dmw/src/mitogen/mitogen/core.py", line 2142, in defer raise Error(self.broker_shutdown_msg) Error: An attempt was made to enqueue a message with a Broker that has already exitted. It is likely your program called Broker.shutdown() too early. --- mitogen/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index d6a2fbb4..10336771 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2738,7 +2738,8 @@ class Broker(object): _v and LOG.debug('%r.shutdown()', self) def _shutdown(): self._alive = False - self.defer(_shutdown) + if self._alive and not self._exitted: + self.defer(_shutdown) def join(self): """