From b8a0e0f929d1432ea396c517525f123ce7d459db Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 29 Jan 2019 07:43:45 +0000 Subject: [PATCH] master: cache sent/forwarded module names On 32x Docker run of issue_140__thread_pileup.yml Before: ncalls tottime percall cumtime percall filename:lineno(function) 384807 3.595 0.000 12.207 0.000 /home/dmw/src/mitogen/mitogen/master.py:896(_forward_one_module) 1218352 4.867 0.000 7.302 0.000 /home/dmw/src/mitogen/mitogen/master.py:853(_send_module_and_related) After: ncalls tottime percall cumtime percall filename:lineno(function) 384807 3.839 0.000 6.543 0.000 /home/dmw/src/mitogen/mitogen/master.py:902(_forward_one_module) 1218352 0.723 0.000 0.898 0.000 /home/dmw/src/mitogen/mitogen/master.py:856(_send_module_and_related) --- mitogen/master.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mitogen/master.py b/mitogen/master.py index 39342e5e..be1e205e 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -707,6 +707,9 @@ class ModuleResponder(object): self.blacklist = [] self.whitelist = [''] + #: Context -> set([fullname, ..]) + self._forwarded_by_context = {} + #: Number of GET_MODULE messages received. self.get_module_count = 0 #: Total time spent in uncached GET_MODULE. @@ -846,6 +849,9 @@ class ModuleResponder(object): ) def _send_module_and_related(self, stream, fullname): + if fullname in stream.sent_modules: + return + try: tup = self._build_tuple(fullname) for name in tup[4]: # related @@ -889,6 +895,14 @@ class ModuleResponder(object): ) def _forward_one_module(self, context, fullname): + forwarded = self._forwarded_by_context.get(context) + if forwarded is None: + forwarded = set() + self._forwarded_by_context[context] = forwarded + + if fullname in forwarded: + return + path = [] while fullname: path.append(fullname)