issue #508: master: minify all Mitogen/ansible_mitogen sources.
Minify-safe files are marked with a magical "# !mitogen: minify_safe"
comment anywhere in the file, which activates the minifier. The result
is naturally cached by ModuleResponder, therefore lru_cache is gone too.
Given:
import os, mitogen
@mitogen.main()
def main(router):
c = router.ssh(hostname='k3')
c.call(os.getpid)
router.sudo(via=c)
SSH footprint drops from 56.2 KiB to 42.75 KiB (-23.9%)
Ansible "shell: hostname" drops 149.26 KiB to 117.42 KiB (-21.3%)
pull/564/head
David Wilson
6 years ago
parent
cfb94e463f
commit
0e193c223c
@ -1,288 +0,0 @@
|
|||||||
# encoding: utf-8
|
|
||||||
"""Selected backports from Python stdlib functools module
|
|
||||||
"""
|
|
||||||
# Written by Nick Coghlan <ncoghlan at gmail.com>,
|
|
||||||
# Raymond Hettinger <python at rcn.com>,
|
|
||||||
# and Łukasz Langa <lukasz at langa.pl>.
|
|
||||||
# Copyright (C) 2006-2013 Python Software Foundation.
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
'update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
|
|
||||||
'lru_cache',
|
|
||||||
]
|
|
||||||
|
|
||||||
from threading import RLock
|
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
### update_wrapper() and wraps() decorator
|
|
||||||
################################################################################
|
|
||||||
|
|
||||||
# update_wrapper() and wraps() are tools to help write
|
|
||||||
# wrapper functions that can handle naive introspection
|
|
||||||
|
|
||||||
WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
|
|
||||||
'__annotations__')
|
|
||||||
WRAPPER_UPDATES = ('__dict__',)
|
|
||||||
def update_wrapper(wrapper,
|
|
||||||
wrapped,
|
|
||||||
assigned = WRAPPER_ASSIGNMENTS,
|
|
||||||
updated = WRAPPER_UPDATES):
|
|
||||||
"""Update a wrapper function to look like the wrapped function
|
|
||||||
wrapper is the function to be updated
|
|
||||||
wrapped is the original function
|
|
||||||
assigned is a tuple naming the attributes assigned directly
|
|
||||||
from the wrapped function to the wrapper function (defaults to
|
|
||||||
functools.WRAPPER_ASSIGNMENTS)
|
|
||||||
updated is a tuple naming the attributes of the wrapper that
|
|
||||||
are updated with the corresponding attribute from the wrapped
|
|
||||||
function (defaults to functools.WRAPPER_UPDATES)
|
|
||||||
"""
|
|
||||||
for attr in assigned:
|
|
||||||
try:
|
|
||||||
value = getattr(wrapped, attr)
|
|
||||||
except AttributeError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
setattr(wrapper, attr, value)
|
|
||||||
for attr in updated:
|
|
||||||
getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
|
|
||||||
# Issue #17482: set __wrapped__ last so we don't inadvertently copy it
|
|
||||||
# from the wrapped function when updating __dict__
|
|
||||||
wrapper.__wrapped__ = wrapped
|
|
||||||
# Return the wrapper so this can be used as a decorator via partial()
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
def wraps(wrapped,
|
|
||||||
assigned = WRAPPER_ASSIGNMENTS,
|
|
||||||
updated = WRAPPER_UPDATES):
|
|
||||||
"""Decorator factory to apply update_wrapper() to a wrapper function
|
|
||||||
Returns a decorator that invokes update_wrapper() with the decorated
|
|
||||||
function as the wrapper argument and the arguments to wraps() as the
|
|
||||||
remaining arguments. Default arguments are as for update_wrapper().
|
|
||||||
This is a convenience function to simplify applying partial() to
|
|
||||||
update_wrapper().
|
|
||||||
"""
|
|
||||||
return partial(update_wrapper, wrapped=wrapped,
|
|
||||||
assigned=assigned, updated=updated)
|
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
### partial() argument application
|
|
||||||
################################################################################
|
|
||||||
|
|
||||||
# Purely functional, no descriptor behaviour
|
|
||||||
def partial(func, *args, **keywords):
|
|
||||||
"""New function with partial application of the given arguments
|
|
||||||
and keywords.
|
|
||||||
"""
|
|
||||||
if hasattr(func, 'func'):
|
|
||||||
args = func.args + args
|
|
||||||
tmpkw = func.keywords.copy()
|
|
||||||
tmpkw.update(keywords)
|
|
||||||
keywords = tmpkw
|
|
||||||
del tmpkw
|
|
||||||
func = func.func
|
|
||||||
|
|
||||||
def newfunc(*fargs, **fkeywords):
|
|
||||||
newkeywords = keywords.copy()
|
|
||||||
newkeywords.update(fkeywords)
|
|
||||||
return func(*(args + fargs), **newkeywords)
|
|
||||||
newfunc.func = func
|
|
||||||
newfunc.args = args
|
|
||||||
newfunc.keywords = keywords
|
|
||||||
return newfunc
|
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
### LRU Cache function decorator
|
|
||||||
################################################################################
|
|
||||||
|
|
||||||
class _HashedSeq(list):
|
|
||||||
""" This class guarantees that hash() will be called no more than once
|
|
||||||
per element. This is important because the lru_cache() will hash
|
|
||||||
the key multiple times on a cache miss.
|
|
||||||
"""
|
|
||||||
|
|
||||||
__slots__ = 'hashvalue'
|
|
||||||
|
|
||||||
def __init__(self, tup, hash=hash):
|
|
||||||
self[:] = tup
|
|
||||||
self.hashvalue = hash(tup)
|
|
||||||
|
|
||||||
def __hash__(self):
|
|
||||||
return self.hashvalue
|
|
||||||
|
|
||||||
def _make_key(args, kwds, typed,
|
|
||||||
kwd_mark = (object(),),
|
|
||||||
fasttypes = set([int, str, frozenset, type(None)]),
|
|
||||||
sorted=sorted, tuple=tuple, type=type, len=len):
|
|
||||||
"""Make a cache key from optionally typed positional and keyword arguments
|
|
||||||
The key is constructed in a way that is flat as possible rather than
|
|
||||||
as a nested structure that would take more memory.
|
|
||||||
If there is only a single argument and its data type is known to cache
|
|
||||||
its hash value, then that argument is returned without a wrapper. This
|
|
||||||
saves space and improves lookup speed.
|
|
||||||
"""
|
|
||||||
key = args
|
|
||||||
if kwds:
|
|
||||||
sorted_items = sorted(kwds.items())
|
|
||||||
key += kwd_mark
|
|
||||||
for item in sorted_items:
|
|
||||||
key += item
|
|
||||||
if typed:
|
|
||||||
key += tuple(type(v) for v in args)
|
|
||||||
if kwds:
|
|
||||||
key += tuple(type(v) for k, v in sorted_items)
|
|
||||||
elif len(key) == 1 and type(key[0]) in fasttypes:
|
|
||||||
return key[0]
|
|
||||||
return _HashedSeq(key)
|
|
||||||
|
|
||||||
def lru_cache(maxsize=128, typed=False):
|
|
||||||
"""Least-recently-used cache decorator.
|
|
||||||
If *maxsize* is set to None, the LRU features are disabled and the cache
|
|
||||||
can grow without bound.
|
|
||||||
If *typed* is True, arguments of different types will be cached separately.
|
|
||||||
For example, f(3.0) and f(3) will be treated as distinct calls with
|
|
||||||
distinct results.
|
|
||||||
Arguments to the cached function must be hashable.
|
|
||||||
View the cache statistics named tuple (hits, misses, maxsize, currsize)
|
|
||||||
with f.cache_info(). Clear the cache and statistics with f.cache_clear().
|
|
||||||
Access the underlying function with f.__wrapped__.
|
|
||||||
See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Users should only access the lru_cache through its public API:
|
|
||||||
# cache_info, cache_clear, and f.__wrapped__
|
|
||||||
# The internals of the lru_cache are encapsulated for thread safety and
|
|
||||||
# to allow the implementation to change (including a possible C version).
|
|
||||||
|
|
||||||
# Early detection of an erroneous call to @lru_cache without any arguments
|
|
||||||
# resulting in the inner function being passed to maxsize instead of an
|
|
||||||
# integer or None.
|
|
||||||
if maxsize is not None and not isinstance(maxsize, int):
|
|
||||||
raise TypeError('Expected maxsize to be an integer or None')
|
|
||||||
|
|
||||||
def decorating_function(user_function):
|
|
||||||
wrapper = _lru_cache_wrapper(user_function, maxsize, typed)
|
|
||||||
return update_wrapper(wrapper, user_function)
|
|
||||||
|
|
||||||
return decorating_function
|
|
||||||
|
|
||||||
def _lru_cache_wrapper(user_function, maxsize, typed):
|
|
||||||
# Constants shared by all lru cache instances:
|
|
||||||
sentinel = object() # unique object used to signal cache misses
|
|
||||||
make_key = _make_key # build a key from the function arguments
|
|
||||||
PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
|
|
||||||
|
|
||||||
cache = {}
|
|
||||||
cache_get = cache.get # bound method to lookup a key or return None
|
|
||||||
lock = RLock() # because linkedlist updates aren't threadsafe
|
|
||||||
root = [] # root of the circular doubly linked list
|
|
||||||
root[:] = [root, root, None, None] # initialize by pointing to self
|
|
||||||
hits_misses_full_root = [0, 0, False, root]
|
|
||||||
HITS,MISSES,FULL,ROOT = 0, 1, 2, 3
|
|
||||||
|
|
||||||
if maxsize == 0:
|
|
||||||
|
|
||||||
def wrapper(*args, **kwds):
|
|
||||||
# No caching -- just a statistics update after a successful call
|
|
||||||
result = user_function(*args, **kwds)
|
|
||||||
hits_misses_full_root[MISSES] += 1
|
|
||||||
return result
|
|
||||||
|
|
||||||
elif maxsize is None:
|
|
||||||
|
|
||||||
def wrapper(*args, **kwds):
|
|
||||||
# Simple caching without ordering or size limit
|
|
||||||
key = make_key(args, kwds, typed)
|
|
||||||
result = cache_get(key, sentinel)
|
|
||||||
if result is not sentinel:
|
|
||||||
hits_misses_full_root[HITS] += 1
|
|
||||||
return result
|
|
||||||
result = user_function(*args, **kwds)
|
|
||||||
cache[key] = result
|
|
||||||
hits_misses_full_root[MISSES] += 1
|
|
||||||
return result
|
|
||||||
|
|
||||||
else:
|
|
||||||
|
|
||||||
def wrapper(*args, **kwds):
|
|
||||||
# Size limited caching that tracks accesses by recency
|
|
||||||
key = make_key(args, kwds, typed)
|
|
||||||
lock.acquire()
|
|
||||||
try:
|
|
||||||
link = cache_get(key)
|
|
||||||
if link is not None:
|
|
||||||
# Move the link to the front of the circular queue
|
|
||||||
root = hits_misses_full_root[ROOT]
|
|
||||||
link_prev, link_next, _key, result = link
|
|
||||||
link_prev[NEXT] = link_next
|
|
||||||
link_next[PREV] = link_prev
|
|
||||||
last = root[PREV]
|
|
||||||
last[NEXT] = root[PREV] = link
|
|
||||||
link[PREV] = last
|
|
||||||
link[NEXT] = root
|
|
||||||
hits_misses_full_root[HITS] += 1
|
|
||||||
return result
|
|
||||||
finally:
|
|
||||||
lock.release()
|
|
||||||
result = user_function(*args, **kwds)
|
|
||||||
lock.acquire()
|
|
||||||
try:
|
|
||||||
if key in cache:
|
|
||||||
# Getting here means that this same key was added to the
|
|
||||||
# cache while the lock was released. Since the link
|
|
||||||
# update is already done, we need only return the
|
|
||||||
# computed result and update the count of misses.
|
|
||||||
pass
|
|
||||||
elif hits_misses_full_root[FULL]:
|
|
||||||
# Use the old root to store the new key and result.
|
|
||||||
oldroot = root = hits_misses_full_root[ROOT]
|
|
||||||
oldroot[KEY] = key
|
|
||||||
oldroot[RESULT] = result
|
|
||||||
# Empty the oldest link and make it the new root.
|
|
||||||
# Keep a reference to the old key and old result to
|
|
||||||
# prevent their ref counts from going to zero during the
|
|
||||||
# update. That will prevent potentially arbitrary object
|
|
||||||
# clean-up code (i.e. __del__) from running while we're
|
|
||||||
# still adjusting the links.
|
|
||||||
root = hits_misses_full_root[ROOT] = oldroot[NEXT]
|
|
||||||
oldkey = root[KEY]
|
|
||||||
oldresult = root[RESULT]
|
|
||||||
root[KEY] = root[RESULT] = None
|
|
||||||
# Now update the cache dictionary.
|
|
||||||
del cache[oldkey]
|
|
||||||
# Save the potentially reentrant cache[key] assignment
|
|
||||||
# for last, after the root and links have been put in
|
|
||||||
# a consistent state.
|
|
||||||
cache[key] = oldroot
|
|
||||||
else:
|
|
||||||
# Put result in a new link at the front of the queue.
|
|
||||||
root = hits_misses_full_root[ROOT]
|
|
||||||
last = root[PREV]
|
|
||||||
link = [last, root, key, result]
|
|
||||||
last[NEXT] = root[PREV] = cache[key] = link
|
|
||||||
# Use the __len__() method instead of the len() function
|
|
||||||
# which could potentially be wrapped in an lru_cache itself.
|
|
||||||
hits_misses_full_root[FULL] = (cache.__len__() >= maxsize)
|
|
||||||
hits_misses_full_root[MISSES]
|
|
||||||
finally:
|
|
||||||
lock.release()
|
|
||||||
return result
|
|
||||||
|
|
||||||
def cache_clear():
|
|
||||||
"""Clear the cache and cache statistics"""
|
|
||||||
lock.acquire()
|
|
||||||
try:
|
|
||||||
cache.clear()
|
|
||||||
root = hits_misses_full_root[ROOT]
|
|
||||||
root[:] = [root, root, None, None]
|
|
||||||
hits_misses_full[HITS] = 0
|
|
||||||
hits_misses_full[MISSES] = 0
|
|
||||||
hits_misses_full[FULL] = False
|
|
||||||
finally:
|
|
||||||
lock.release()
|
|
||||||
|
|
||||||
wrapper.cache_clear = cache_clear
|
|
||||||
return wrapper
|
|
||||||
Loading…
Reference in New Issue