From 8a4caea84f6768ed9c1e67c6f860b2d99e17f4b2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 07:29:21 +0000 Subject: [PATCH 1/7] ci: Allow DISTROS="debian*32" variable, and KEEP=1 --- .ci/ci_lib.py | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 3147877a..d06fedda 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -158,23 +158,37 @@ def make_containers(): firstbit = lambda s: (s+'-').split('-')[0] secondbit = lambda s: (s+'-').split('-')[1] - return [ - { - "distro": firstbit(distro), - "name": "target-%s-%s" % (distro, i), - "hostname": docker_hostname, - "port": BASE_PORT + i, - "python_path": ( - '/usr/bin/python3' - if secondbit(distro) == 'py3' - else '/usr/bin/python' - ) - } - for i, distro in enumerate(DISTROS, 1) - ] + i = 1 + lst = [] + + for distro in DISTROS: + distro, star, count = distro.rpartition('*') + if star: + count = int(count) + else: + count = 1 + + for x in range(count): + lst.append({ + "distro": firstbit(distro), + "name": "target-%s-%s" % (distro, i), + "hostname": docker_hostname, + "port": BASE_PORT + i, + "python_path": ( + '/usr/bin/python3' + if secondbit(distro) == 'py3' + else '/usr/bin/python' + ) + }) + i += 1 + + return lst def start_containers(containers): + if os.environ.get('KEEP'): + return + run_batches([ [ "docker rm -f %(name)s || true" % container, From dc4b27c6bff76cfe9ba1316ef2524c87bf5071d1 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 07:43:45 +0000 Subject: [PATCH 2/7] master: keep is_stdlib_path() result as negative cache entry On 32x Docker run of issue_140__thread_pileup.yml Before: ncalls tottime percall cumtime percall filename:lineno(function) 1218500 1.716 0.000 7.325 0.000 /home/dmw/src/mitogen/mitogen/master.py:118(is_stdlib_path) After: ncalls tottime percall cumtime percall filename:lineno(function) 166 0.000 0.000 0.001 0.000 /home/dmw/src/mitogen/mitogen/master.py:123(is_stdlib_path) --- mitogen/master.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 2930b42b..39342e5e 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -768,13 +768,22 @@ class ModuleResponder(object): return (fullname, None, None, None, ()) def _build_tuple(self, fullname): - if mitogen.core.is_blacklisted_import(self, fullname): - raise ImportError('blacklisted') - if fullname in self._cache: return self._cache[fullname] + if mitogen.core.is_blacklisted_import(self, fullname): + raise ImportError('blacklisted') + path, source, is_pkg = self._finder.get_module_source(fullname) + if path and is_stdlib_path(path): + # Prevent loading of 2.x<->3.x stdlib modules! This costs one + # RTT per hit, so a client-side solution is also required. + LOG.debug('%r: refusing to serve stdlib module %r', + self, fullname) + tup = self._make_negative_response(fullname) + self._cache[fullname] = tup + return tup + if source is None: # TODO: make this .warning() or similar again once importer has its # own logging category. @@ -839,14 +848,6 @@ class ModuleResponder(object): def _send_module_and_related(self, stream, fullname): try: tup = self._build_tuple(fullname) - if tup[2] and is_stdlib_path(tup[2]): - # Prevent loading of 2.x<->3.x stdlib modules! This costs one - # RTT per hit, so a client-side solution is also required. - LOG.debug('%r: refusing to serve stdlib module %r', - self, fullname) - self._send_module_load_failed(stream, fullname) - return - for name in tup[4]: # related parent, _, _ = str_partition(name, '.') if parent != fullname and parent not in stream.sent_modules: @@ -893,8 +894,8 @@ class ModuleResponder(object): path.append(fullname) fullname, _, _ = str_rpartition(fullname, u'.') + stream = self._router.stream_by_id(context.context_id) for fullname in reversed(path): - stream = self._router.stream_by_id(context.context_id) self._send_module_and_related(stream, fullname) self._send_forward_module(stream, context, fullname) From d4a0b70e15ec6413f5216dd14cb6b26d86ad2e44 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 07:59:02 +0000 Subject: [PATCH 3/7] core: ensure broker profiling output reaches disk Profiler hasn't been used much since the hard exit was added. --- mitogen/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 4bc754ba..aaaed1ba 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -540,7 +540,7 @@ def enable_profiling(): try: return func(*args) finally: - profiler.dump_stats('/tmp/mitogen.%d.%s.pstat' % (os.getpid(), name)) + profiler.dump_stats('/tmp/mitogen.stats.%d.%s.pstat' % (os.getpid(), name)) profiler.create_stats() fp = open('/tmp/mitogen.stats.%d.%s.log' % (os.getpid(), name), 'w') try: @@ -2798,8 +2798,7 @@ class Broker(object): (self._waker.receive_side, self._waker.on_receive) ) self._thread = threading.Thread( - target=_profile_hook, - args=('broker', self._broker_main), + target=self._broker_main, name='mitogen-broker' ) self._thread.start() @@ -2932,7 +2931,7 @@ class Broker(object): 'more child processes still connected to ' 'our stdout/stderr pipes.', self) - def _broker_main(self): + def _do_broker_main(self): """ Broker thread main function. Dispatches IO events until :meth:`shutdown` is called. @@ -2950,6 +2949,9 @@ class Broker(object): self._exitted = True self._broker_exit() + + def _broker_main(self): + _profile_hook('mitogen-broker', self._do_broker_main) fire(self, 'exit') def shutdown(self): From a18a083c944e3f7963ae81113161f0cd053d348d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 17:11:51 +0000 Subject: [PATCH 4/7] issue #260: avoid start_transmit()/on_transmit()/stop_transmit() Previous transmit sequence was: Router._async_route -> Stream._send -> Broker._start_transmit -> Broker.loop -> Stream.on_transmit -> socket.write -> Broker.stop_transmit New sequence, when socket buffer can hold message is: Router._async_route -> Stream._send -> socket.write bench/roundtrip.py Before: 240 usec after: 178 usec Stat before: 5088.276050 task-clock (msec) # 0.997 CPUs utilized 185,568 context-switches # 0.036 M/sec 0 cpu-migrations # 0.000 K/sec 18,923 page-faults # 0.004 M/sec 13,063,871,501 cycles # 2.567 GHz 12,834,579,684 instructions # 0.98 insn per cycle 2,669,820,684 branches # 524.700 M/sec 107,296,033 branch-misses # 4.02% of all branches 5.105018296 seconds time elapsed 2.350970000 seconds user 0.345497000 seconds sys Stat after: 4019.208047 task-clock (msec) # 0.998 CPUs utilized 249,471 context-switches # 0.062 M/sec 0 cpu-migrations # 0.000 K/sec 20,990 page-faults # 0.005 M/sec 10,312,535,979 cycles # 2.566 GHz 11,586,365,995 instructions # 1.12 insn per cycle 2,392,933,370 branches # 595.374 M/sec 75,432,205 branch-misses # 3.15% of all branches 4.028763347 seconds time elapsed 3.367051000 seconds user 0.652962000 seconds sys --- mitogen/core.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mitogen/core.py b/mitogen/core.py index aaaed1ba..7ceba02c 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1682,6 +1682,22 @@ class Stream(BasicStream): pkt = struct.pack(self.HEADER_FMT, self.HEADER_MAGIC, msg.dst_id, msg.src_id, msg.auth_id, msg.handle, msg.reply_to or 0, len(msg.data)) + msg.data + + if not self._output_buf_len: + # Modifying epoll/Kqueue state is expensive, as is needless broker + # loop iterations. Rather than wait for writeability, simply + # attempt to write immediately, and only fall back to + # start_transmit()/on_transmit() if an error occurred or the socket + # buffer was full. + try: + n = self.transmit_side.write(pkt) + if n: + if n == len(pkt): + return + pkt = pkt[n:] + except OSError: + pass + if not self._output_buf_len: self._router.broker._start_transmit(self) self._output_buf.append(pkt) From 26c6c6d0488aac2baa9593cae383eb6b81224898 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 17:17:57 +0000 Subject: [PATCH 5/7] issue #498: fix shutdown crash Traceback (most recent call last): File "", line 2707, in _invoke File "", line 2480, in _on_del_route NameError: global name 'target_id' is not defined --- mitogen/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 7ceba02c..c94126a6 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2473,7 +2473,8 @@ class Router(object): return target_id_s, _, name = bytes_partition(msg.data, b(':')) - context = self._context_by_id.get(int(target_id_s, 10)) + target_id = int(target_id_s, 10) + context = self._context_by_id.get(target_id) if context: fire(context, 'disconnect') else: From 696cee57dd71b8766ac5456e72fe82cc5b7dde39 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 17:11:51 +0000 Subject: [PATCH 6/7] issue #260: hide force-disconnect messages. Parent is always force-disconnected, and now it shows up quite visibly on every exit. --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index c94126a6..9a61d4cd 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2923,7 +2923,7 @@ class Broker(object): to shut down gracefully, then discard the :class:`Poller`. """ for _, (side, _) in self.poller.readers + self.poller.writers: - LOG.error('_broker_main() force disconnecting %r', side) + LOG.debug('_broker_main() force disconnecting %r', side) side.stream.on_disconnect(self) self.poller.close() From 49a8745a45f7d32a4f2360b6649e007cbc229162 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 17:39:46 +0000 Subject: [PATCH 7/7] ci: fix incorrect partition/rpartition from 8a4caea84f6768ed9c1e67c6f860b2d99e17f4b2 --- .ci/ci_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index d06fedda..10e9d11e 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -162,7 +162,7 @@ def make_containers(): lst = [] for distro in DISTROS: - distro, star, count = distro.rpartition('*') + distro, star, count = distro.partition('*') if star: count = int(count) else: