master: keep is_stdlib_path() result as negative cache entry

On 32x Docker run of issue_140__thread_pileup.yml

Before:

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
  1218500    1.716    0.000    7.325    0.000 /home/dmw/src/mitogen/mitogen/master.py:118(is_stdlib_path)

After:

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
      166    0.000    0.000    0.001    0.000 /home/dmw/src/mitogen/mitogen/master.py:123(is_stdlib_path)
issue510
David Wilson 6 years ago
parent 8a4caea84f
commit dc4b27c6bf

@ -768,13 +768,22 @@ class ModuleResponder(object):
return (fullname, None, None, None, ()) return (fullname, None, None, None, ())
def _build_tuple(self, fullname): def _build_tuple(self, fullname):
if mitogen.core.is_blacklisted_import(self, fullname):
raise ImportError('blacklisted')
if fullname in self._cache: if fullname in self._cache:
return self._cache[fullname] return self._cache[fullname]
if mitogen.core.is_blacklisted_import(self, fullname):
raise ImportError('blacklisted')
path, source, is_pkg = self._finder.get_module_source(fullname) path, source, is_pkg = self._finder.get_module_source(fullname)
if path and is_stdlib_path(path):
# Prevent loading of 2.x<->3.x stdlib modules! This costs one
# RTT per hit, so a client-side solution is also required.
LOG.debug('%r: refusing to serve stdlib module %r',
self, fullname)
tup = self._make_negative_response(fullname)
self._cache[fullname] = tup
return tup
if source is None: if source is None:
# TODO: make this .warning() or similar again once importer has its # TODO: make this .warning() or similar again once importer has its
# own logging category. # own logging category.
@ -839,14 +848,6 @@ class ModuleResponder(object):
def _send_module_and_related(self, stream, fullname): def _send_module_and_related(self, stream, fullname):
try: try:
tup = self._build_tuple(fullname) tup = self._build_tuple(fullname)
if tup[2] and is_stdlib_path(tup[2]):
# Prevent loading of 2.x<->3.x stdlib modules! This costs one
# RTT per hit, so a client-side solution is also required.
LOG.debug('%r: refusing to serve stdlib module %r',
self, fullname)
self._send_module_load_failed(stream, fullname)
return
for name in tup[4]: # related for name in tup[4]: # related
parent, _, _ = str_partition(name, '.') parent, _, _ = str_partition(name, '.')
if parent != fullname and parent not in stream.sent_modules: if parent != fullname and parent not in stream.sent_modules:
@ -893,8 +894,8 @@ class ModuleResponder(object):
path.append(fullname) path.append(fullname)
fullname, _, _ = str_rpartition(fullname, u'.') fullname, _, _ = str_rpartition(fullname, u'.')
stream = self._router.stream_by_id(context.context_id)
for fullname in reversed(path): for fullname in reversed(path):
stream = self._router.stream_by_id(context.context_id)
self._send_module_and_related(stream, fullname) self._send_module_and_related(stream, fullname)
self._send_forward_module(stream, context, fullname) self._send_forward_module(stream, context, fullname)

Loading…
Cancel
Save