From 2c7f2b4a9994ee2c564e4bf227d08efe2f7bc94d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 17 Jul 2022 10:35:28 +0100 Subject: [PATCH] Experimental: mitogen: Remove minification of Python source code Done on the basis that sending a few more bytes of the network is quicker than tokenizing and untokeninzing the modules. --- ansible_mitogen/runner.py | 1 - ansible_mitogen/services.py | 1 - ansible_mitogen/target.py | 1 - docs/internals.rst | 4 - mitogen/__init__.py | 1 - mitogen/buildah.py | 1 - mitogen/compat/pkgutil.py | 1 - mitogen/compat/tokenize.py | 453 ------------------------------------ mitogen/core.py | 2 - mitogen/debug.py | 1 - mitogen/doas.py | 1 - mitogen/docker.py | 1 - mitogen/fakessh.py | 1 - mitogen/fork.py | 1 - mitogen/jail.py | 1 - mitogen/kubectl.py | 1 - mitogen/lxc.py | 1 - mitogen/lxd.py | 1 - mitogen/master.py | 19 +- mitogen/minify.py | 143 ------------ mitogen/os_fork.py | 1 - mitogen/parent.py | 3 +- mitogen/podman.py | 1 - mitogen/profiler.py | 1 - mitogen/select.py | 1 - mitogen/service.py | 3 +- mitogen/setns.py | 1 - mitogen/ssh.py | 1 - mitogen/su.py | 1 - mitogen/sudo.py | 1 - mitogen/unix.py | 1 - mitogen/utils.py | 1 - preamble_size.py | 5 +- tests/minify_test.py | 112 --------- tests/module_finder_test.py | 1 - tests/responder_test.py | 2 +- 36 files changed, 6 insertions(+), 766 deletions(-) delete mode 100644 mitogen/compat/tokenize.py delete mode 100644 mitogen/minify.py delete mode 100644 tests/minify_test.py diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index c4cb71ff..4562ca7e 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ These classes implement execution for each style of Ansible module. They are diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index b0f5c70e..a57f1c5b 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ Classes in this file define Mitogen 'services' that run (initially) within the diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 7d907d62..eb2e1abc 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ Helper functions intended to be executed on the target. These are entrypoints diff --git a/docs/internals.rst b/docs/internals.rst index 7f44d7b0..13704376 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -273,10 +273,6 @@ Helpers .. currentmodule:: mitogen.master .. autofunction:: get_child_modules -.. currentmodule:: mitogen.minify -.. autofunction:: minimize_source - - .. _signals: Signals diff --git a/mitogen/__init__.py b/mitogen/__init__.py index db4e8b3e..a1691702 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would diff --git a/mitogen/buildah.py b/mitogen/buildah.py index 7a1e3f80..f90fa3ff 100644 --- a/mitogen/buildah.py +++ b/mitogen/buildah.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import logging diff --git a/mitogen/compat/pkgutil.py b/mitogen/compat/pkgutil.py index 15eb2afa..2a213b57 100644 --- a/mitogen/compat/pkgutil.py +++ b/mitogen/compat/pkgutil.py @@ -1,6 +1,5 @@ """Utilities to support packages.""" -# !mitogen: minify_safe # NOTE: This module must remain compatible with Python 2.3, as it is shared # by setuptools for distribution with Python 2.3 and up. diff --git a/mitogen/compat/tokenize.py b/mitogen/compat/tokenize.py deleted file mode 100644 index 0473c6a5..00000000 --- a/mitogen/compat/tokenize.py +++ /dev/null @@ -1,453 +0,0 @@ -"""Tokenization help for Python programs. - -generate_tokens(readline) is a generator that breaks a stream of -text into Python tokens. It accepts a readline-like method which is called -repeatedly to get the next line of input (or "" for EOF). It generates -5-tuples with these members: - - the token type (see token.py) - the token (a string) - the starting (row, column) indices of the token (a 2-tuple of ints) - the ending (row, column) indices of the token (a 2-tuple of ints) - the original line (string) - -It is designed to match the working of the Python tokenizer exactly, except -that it produces COMMENT tokens for comments and gives type OP for all -operators - -Older entry points - tokenize_loop(readline, tokeneater) - tokenize(readline, tokeneater=printtoken) -are the same, except instead of generating tokens, tokeneater is a callback -function to which the 5 fields described above are passed as 5 arguments, -each time a new token is found.""" - -# !mitogen: minify_safe - -__author__ = 'Ka-Ping Yee ' -__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' - 'Skip Montanaro, Raymond Hettinger') - -from itertools import chain -import string, re -from token import * - -import token -__all__ = [x for x in dir(token) if not x.startswith("_")] -__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"] -del token - -COMMENT = N_TOKENS -tok_name[COMMENT] = 'COMMENT' -NL = N_TOKENS + 1 -tok_name[NL] = 'NL' -N_TOKENS += 2 - -def group(*choices): return '(' + '|'.join(choices) + ')' -def any(*choices): return group(*choices) + '*' -def maybe(*choices): return group(*choices) + '?' - -Whitespace = r'[ \f\t]*' -Comment = r'#[^\r\n]*' -Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'[a-zA-Z_]\w*' - -Hexnumber = r'0[xX][\da-fA-F]+[lL]?' -Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?' -Binnumber = r'0[bB][01]+[lL]?' -Decnumber = r'[1-9]\d*[lL]?' -Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent -Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') -Number = group(Imagnumber, Floatnumber, Intnumber) - -# Tail end of ' string. -Single = r"[^'\\]*(?:\\.[^'\\]*)*'" -# Tail end of " string. -Double = r'[^"\\]*(?:\\.[^"\\]*)*"' -# Tail end of ''' string. -Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" -# Tail end of """ string. -Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') -# Single-line ' or " string. -String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') - -# Because of leftmost-then-longest match semantics, be sure to put the -# longest operators first (e.g., if = came before ==, == would get -# recognized as two instances of =). -Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", - r"//=?", - r"[+\-*/%&|^=<>]=?", - r"~") - -Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,`@]') -Funny = group(Operator, Bracket, Special) - -PlainToken = group(Number, Funny, String, Name) -Token = Ignore + PlainToken - -# First (or only) line of ' or " string. -ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + - group("'", r'\\\r?\n'), - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + - group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) -PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) - -tokenprog, pseudoprog, single3prog, double3prog = map( - re.compile, (Token, PseudoToken, Single3, Double3)) -endprogs = {"'": re.compile(Single), '"': re.compile(Double), - "'''": single3prog, '"""': double3prog, - "r'''": single3prog, 'r"""': double3prog, - "u'''": single3prog, 'u"""': double3prog, - "ur'''": single3prog, 'ur"""': double3prog, - "R'''": single3prog, 'R"""': double3prog, - "U'''": single3prog, 'U"""': double3prog, - "uR'''": single3prog, 'uR"""': double3prog, - "Ur'''": single3prog, 'Ur"""': double3prog, - "UR'''": single3prog, 'UR"""': double3prog, - "b'''": single3prog, 'b"""': double3prog, - "br'''": single3prog, 'br"""': double3prog, - "B'''": single3prog, 'B"""': double3prog, - "bR'''": single3prog, 'bR"""': double3prog, - "Br'''": single3prog, 'Br"""': double3prog, - "BR'''": single3prog, 'BR"""': double3prog, - 'r': None, 'R': None, 'u': None, 'U': None, - 'b': None, 'B': None} - -triple_quoted = {} -for t in ("'''", '"""', - "r'''", 'r"""', "R'''", 'R"""', - "u'''", 'u"""', "U'''", 'U"""', - "ur'''", 'ur"""', "Ur'''", 'Ur"""', - "uR'''", 'uR"""', "UR'''", 'UR"""', - "b'''", 'b"""', "B'''", 'B"""', - "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""'): - triple_quoted[t] = t -single_quoted = {} -for t in ("'", '"', - "r'", 'r"', "R'", 'R"', - "u'", 'u"', "U'", 'U"', - "ur'", 'ur"', "Ur'", 'Ur"', - "uR'", 'uR"', "UR'", 'UR"', - "b'", 'b"', "B'", 'B"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"' ): - single_quoted[t] = t - -tabsize = 8 - -class TokenError(Exception): pass - -class StopTokenizing(Exception): pass - -def printtoken(type, token, srow_scol, erow_ecol, line): # for testing - srow, scol = srow_scol - erow, ecol = erow_ecol - print("%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token))) - -def tokenize(readline, tokeneater=printtoken): - """ - The tokenize() function accepts two parameters: one representing the - input stream, and one providing an output mechanism for tokenize(). - - The first parameter, readline, must be a callable object which provides - the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. - - The second parameter, tokeneater, must also be a callable object. It is - called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). - """ - try: - tokenize_loop(readline, tokeneater) - except StopTokenizing: - pass - -# backwards compatible interface -def tokenize_loop(readline, tokeneater): - for token_info in generate_tokens(readline): - tokeneater(*token_info) - -class Untokenizer: - - def __init__(self): - self.tokens = [] - self.prev_row = 1 - self.prev_col = 0 - - def add_whitespace(self, start): - row, col = start - if row < self.prev_row or row == self.prev_row and col < self.prev_col: - raise ValueError("start ({},{}) precedes previous end ({},{})" - .format(row, col, self.prev_row, self.prev_col)) - row_offset = row - self.prev_row - if row_offset: - self.tokens.append("\\\n" * row_offset) - self.prev_col = 0 - col_offset = col - self.prev_col - if col_offset: - self.tokens.append(" " * col_offset) - - def untokenize(self, iterable): - it = iter(iterable) - indents = [] - startline = False - for t in it: - if len(t) == 2: - self.compat(t, it) - break - tok_type, token, start, end, line = t - if tok_type == ENDMARKER: - break - if tok_type == INDENT: - indents.append(token) - continue - elif tok_type == DEDENT: - indents.pop() - self.prev_row, self.prev_col = end - continue - elif tok_type in (NEWLINE, NL): - startline = True - elif startline and indents: - indent = indents[-1] - if start[1] >= len(indent): - self.tokens.append(indent) - self.prev_col = len(indent) - startline = False - self.add_whitespace(start) - self.tokens.append(token) - self.prev_row, self.prev_col = end - if tok_type in (NEWLINE, NL): - self.prev_row += 1 - self.prev_col = 0 - return "".join(self.tokens) - - def compat(self, token, iterable): - indents = [] - toks_append = self.tokens.append - startline = token[0] in (NEWLINE, NL) - prevstring = False - - for tok in chain([token], iterable): - toknum, tokval = tok[:2] - - if toknum in (NAME, NUMBER): - tokval += ' ' - - # Insert a space between two consecutive strings - if toknum == STRING: - if prevstring: - tokval = ' ' + tokval - prevstring = True - else: - prevstring = False - - if toknum == INDENT: - indents.append(tokval) - continue - elif toknum == DEDENT: - indents.pop() - continue - elif toknum in (NEWLINE, NL): - startline = True - elif startline and indents: - toks_append(indents[-1]) - startline = False - toks_append(tokval) - -def untokenize(iterable): - """Transform tokens back into Python source code. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited intput: - # Output text will tokenize the back to the input - t1 = [tok[:2] for tok in generate_tokens(f.readline)] - newcode = untokenize(t1) - readline = iter(newcode.splitlines(1)).next - t2 = [tok[:2] for tok in generate_tokens(readline)] - assert t1 == t2 - """ - ut = Untokenizer() - return ut.untokenize(iterable) - -def generate_tokens(readline): - """ - The generate_tokens() generator requires one argument, readline, which - must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as a string. Alternately, readline - can be a callable function terminating with StopIteration: - readline = open(myfile).next # Example of alternate readline - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple (srow, scol) of ints specifying the row and - column where the token begins in the source; a 2-tuple (erow, ecol) of - ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the - logical line; continuation lines are included. - """ - lnum = parenlev = continued = 0 - namechars, numchars = string.ascii_letters + '_', '0123456789' - contstr, needcont = '', 0 - contline = None - indents = [0] - - while 1: # loop over lines in stream - try: - line = readline() - except StopIteration: - line = '' - lnum += 1 - pos, max = 0, len(line) - - if contstr: # continued string - if not line: - raise TokenError("EOF in multi-line string", strstart) - endmatch = endprog.match(line) - if endmatch: - pos = end = endmatch.end(0) - yield (STRING, contstr + line[:end], - strstart, (lnum, end), contline + line) - contstr, needcont = '', 0 - contline = None - elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield (ERRORTOKEN, contstr + line, - strstart, (lnum, len(line)), contline) - contstr = '' - contline = None - continue - else: - contstr = contstr + line - contline = contline + line - continue - - elif parenlev == 0 and not continued: # new statement - if not line: break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == ' ': - column += 1 - elif line[pos] == '\t': - column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': - column = 0 - else: - break - pos += 1 - if pos == max: - break - - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - nl_pos = pos + len(comment_token) - yield (COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - yield (NL, line[nl_pos:], - (lnum, nl_pos), (lnum, len(line)), line) - else: - yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], - (lnum, pos), (lnum, len(line)), line) - continue - - if column > indents[-1]: # count indents or dedents - indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - while column < indents[-1]: - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("", lnum, pos, line)) - indents = indents[:-1] - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) - - else: # continued statement - if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) - continued = 0 - - while pos < max: - pseudomatch = pseudoprog.match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - if start == end: - continue - token, initial = line[start:end], line[start] - - if initial in numchars or \ - (initial == '.' and token != '.'): # ordinary number - yield (NUMBER, token, spos, epos, line) - elif initial in '\r\n': - if parenlev > 0: - n = NL - else: - n = NEWLINE - yield (n, token, spos, epos, line) - elif initial == '#': - assert not token.endswith("\n") - yield (COMMENT, token, spos, epos, line) - elif token in triple_quoted: - endprog = endprogs[token] - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - yield (STRING, token, spos, (lnum, pos), line) - else: - strstart = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: - if token[-1] == '\n': # continued string - strstart = (lnum, start) - endprog = (endprogs[initial] or endprogs[token[1]] or - endprogs[token[2]]) - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - yield (STRING, token, spos, epos, line) - elif initial in namechars: # ordinary name - yield (NAME, token, spos, epos, line) - elif initial == '\\': # continued stmt - continued = 1 - else: - if initial in '([{': - parenlev += 1 - elif initial in ')]}': - parenlev -= 1 - yield (OP, token, spos, epos, line) - else: - yield (ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos+1), line) - pos += 1 - - for indent in indents[1:]: # pop remaining indent levels - yield (DEDENT, '', (lnum, 0), (lnum, 0), '') - yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') - -if __name__ == '__main__': # testing - import sys - if len(sys.argv) > 1: - tokenize(open(sys.argv[1]).readline) - else: - tokenize(sys.stdin.readline) diff --git a/mitogen/core.py b/mitogen/core.py index bee722e6..989baaec 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ This module implements most package functionality, but remains separate from @@ -1265,7 +1264,6 @@ class Importer(object): 'lxc', 'lxd', 'master', - 'minify', 'os_fork', 'parent', 'podman', diff --git a/mitogen/debug.py b/mitogen/debug.py index dbab550e..0c386d7c 100644 --- a/mitogen/debug.py +++ b/mitogen/debug.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ Basic signal handler for dumping thread stacks. diff --git a/mitogen/doas.py b/mitogen/doas.py index 5b212b9b..ac1ce971 100644 --- a/mitogen/doas.py +++ b/mitogen/doas.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import logging import re diff --git a/mitogen/docker.py b/mitogen/docker.py index 48848c89..db8f1179 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import logging diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index 2d660248..cc3a0e6d 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ :mod:`mitogen.fakessh` is a stream implementation that starts a subprocess with diff --git a/mitogen/fork.py b/mitogen/fork.py index f0c2d7e7..daadc246 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import errno import logging diff --git a/mitogen/jail.py b/mitogen/jail.py index 4da7eb0d..cbecf5fd 100644 --- a/mitogen/jail.py +++ b/mitogen/jail.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import mitogen.core import mitogen.parent diff --git a/mitogen/kubectl.py b/mitogen/kubectl.py index 5d3994ae..26a40d76 100644 --- a/mitogen/kubectl.py +++ b/mitogen/kubectl.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import mitogen.parent diff --git a/mitogen/lxc.py b/mitogen/lxc.py index 21dfef59..9b811e64 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import mitogen.parent diff --git a/mitogen/lxd.py b/mitogen/lxd.py index 09034abf..374794b4 100644 --- a/mitogen/lxd.py +++ b/mitogen/lxd.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import mitogen.parent diff --git a/mitogen/master.py b/mitogen/master.py index 4fb535f0..18b46a5d 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ This module implements functionality required by master processes, such as @@ -62,7 +61,6 @@ if not hasattr(pkgutil, 'find_loader'): import mitogen import mitogen.core -import mitogen.minify import mitogen.parent from mitogen.core import b @@ -231,7 +229,7 @@ def _get_core_source(): Master version of parent.get_core_source(). """ source = inspect.getsource(mitogen.core) - return mitogen.minify.minimize_source(source) + return source if mitogen.is_master: @@ -980,8 +978,6 @@ class ModuleResponder(object): self.get_module_count = 0 #: Total time spent in uncached GET_MODULE. self.get_module_secs = 0.0 - #: Total time spent minifying modules. - self.minify_secs = 0.0 #: Number of successful LOAD_MODULE messages sent. self.good_load_module_count = 0 #: Total bytes in successful LOAD_MODULE payloads. @@ -1040,8 +1036,6 @@ class ModuleResponder(object): def _make_negative_response(self, fullname): return (fullname, None, None, None, ()) - minify_safe_re = re.compile(b(r'\s+#\s*!mitogen:\s*minify_safe')) - def _build_tuple(self, fullname): if fullname in self._cache: return self._cache[fullname] @@ -1066,12 +1060,6 @@ class ModuleResponder(object): self._cache[fullname] = tup return tup - if self.minify_safe_re.search(source): - # If the module contains a magic marker, it's safe to minify. - t0 = mitogen.core.now() - source = mitogen.minify.minimize_source(source).encode('utf-8') - self.minify_secs += mitogen.core.now() - t0 - if is_pkg: pkg_present = get_child_modules(path, fullname) self._log.debug('%s is a package at %s with submodules %r', @@ -1310,7 +1298,6 @@ class Router(mitogen.parent.Router): super(Router, self)._on_broker_exit() dct = self.get_stats() dct['self'] = self - dct['minify_ms'] = 1000 * dct['minify_secs'] dct['get_module_ms'] = 1000 * dct['get_module_secs'] dct['good_load_module_size_kb'] = dct['good_load_module_size'] / 1024.0 dct['good_load_module_size_avg'] = ( @@ -1325,7 +1312,6 @@ class Router(mitogen.parent.Router): '%(get_module_count)d module requests in ' '%(get_module_ms)d ms, ' '%(good_load_module_count)d sent ' - '(%(minify_ms)d ms minify time), ' '%(bad_load_module_count)d negative responses. ' 'Sent %(good_load_module_size_kb).01f kb total, ' '%(good_load_module_size_avg).01f kb avg.' @@ -1350,8 +1336,6 @@ class Router(mitogen.parent.Router): :data:`mitogen.core.LOAD_MODULE` message payloads. * `bad_load_module_count`: Integer count of negative :data:`mitogen.core.LOAD_MODULE` messages sent. - * `minify_secs`: CPU seconds spent minifying modules marked - minify-safe. """ return { 'get_module_count': self.responder.get_module_count, @@ -1359,7 +1343,6 @@ class Router(mitogen.parent.Router): 'good_load_module_count': self.responder.good_load_module_count, 'good_load_module_size': self.responder.good_load_module_size, 'bad_load_module_count': self.responder.bad_load_module_count, - 'minify_secs': self.responder.minify_secs, } def enable_debug(self): diff --git a/mitogen/minify.py b/mitogen/minify.py deleted file mode 100644 index 09fdc4eb..00000000 --- a/mitogen/minify.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2017, Alex Willmer -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -# !mitogen: minify_safe - -import sys - -try: - from io import StringIO -except ImportError: - from StringIO import StringIO - -import mitogen.core - -if sys.version_info < (2, 7, 11): - from mitogen.compat import tokenize -else: - import tokenize - - -def minimize_source(source): - """ - Remove comments and docstrings from Python `source`, preserving line - numbers and syntax of empty blocks. - - :param str source: - The source to minimize. - - :returns str: - The minimized source. - """ - source = mitogen.core.to_text(source) - tokens = tokenize.generate_tokens(StringIO(source).readline) - tokens = strip_comments(tokens) - tokens = strip_docstrings(tokens) - tokens = reindent(tokens) - return tokenize.untokenize(tokens) - - -def strip_comments(tokens): - """ - Drop comment tokens from a `tokenize` stream. - - Comments on lines 1-2 are kept, to preserve hashbang and encoding. - Trailing whitespace is remove from all lines. - """ - prev_typ = None - prev_end_col = 0 - for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens: - if typ in (tokenize.NL, tokenize.NEWLINE): - if prev_typ in (tokenize.NL, tokenize.NEWLINE): - start_col = 0 - else: - start_col = prev_end_col - end_col = start_col + 1 - elif typ == tokenize.COMMENT and start_row > 2: - continue - prev_typ = typ - prev_end_col = end_col - yield typ, tok, (start_row, start_col), (end_row, end_col), line - - -def strip_docstrings(tokens): - """ - Replace docstring tokens with NL tokens in a `tokenize` stream. - - Any STRING token not part of an expression is deemed a docstring. - Indented docstrings are not yet recognised. - """ - stack = [] - state = 'wait_string' - for t in tokens: - typ = t[0] - if state == 'wait_string': - if typ in (tokenize.NL, tokenize.COMMENT): - yield t - elif typ in (tokenize.DEDENT, tokenize.INDENT, tokenize.STRING): - stack.append(t) - elif typ == tokenize.NEWLINE: - stack.append(t) - start_line, end_line = stack[0][2][0], stack[-1][3][0]+1 - for i in range(start_line, end_line): - yield tokenize.NL, '\n', (i, 0), (i,1), '\n' - for t in stack: - if t[0] in (tokenize.DEDENT, tokenize.INDENT): - yield t[0], t[1], (i+1, t[2][1]), (i+1, t[3][1]), t[4] - del stack[:] - else: - stack.append(t) - for t in stack: yield t - del stack[:] - state = 'wait_newline' - elif state == 'wait_newline': - if typ == tokenize.NEWLINE: - state = 'wait_string' - yield t - - -def reindent(tokens, indent=' '): - """ - Replace existing indentation in a token steam, with `indent`. - """ - old_levels = [] - old_level = 0 - new_level = 0 - for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens: - if typ == tokenize.INDENT: - old_levels.append(old_level) - old_level = len(tok) - new_level += 1 - tok = indent * new_level - elif typ == tokenize.DEDENT: - old_level = old_levels.pop() - new_level -= 1 - start_col = max(0, start_col - old_level + new_level) - if start_row == end_row: - end_col = start_col + len(tok) - yield typ, tok, (start_row, start_col), (end_row, end_col), line diff --git a/mitogen/os_fork.py b/mitogen/os_fork.py index 9c649d07..c11a398c 100644 --- a/mitogen/os_fork.py +++ b/mitogen/os_fork.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ Support for operating in a mixed threading/forking environment. diff --git a/mitogen/parent.py b/mitogen/parent.py index 32aa3cb6..005f0f19 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ This module defines functionality common to master and parent processes. It is @@ -189,7 +188,7 @@ def _get_core_source(): def get_core_source_partial(): """ - _get_core_source() is expensive, even with @lru_cache in minify.py, threads + _get_core_source() is expensive, even with caching, threads can enter it simultaneously causing severe slowdowns. """ global _core_source_partial diff --git a/mitogen/podman.py b/mitogen/podman.py index acc46a33..e83c20db 100644 --- a/mitogen/podman.py +++ b/mitogen/podman.py @@ -27,7 +27,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import logging diff --git a/mitogen/profiler.py b/mitogen/profiler.py index 512a593e..90a22a4a 100644 --- a/mitogen/profiler.py +++ b/mitogen/profiler.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ mitogen.profiler diff --git a/mitogen/select.py b/mitogen/select.py index 2d87574f..62ed98eb 100644 --- a/mitogen/select.py +++ b/mitogen/select.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import mitogen.core diff --git a/mitogen/service.py b/mitogen/service.py index 0e5f6419..4e897c06 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import grp import logging @@ -212,7 +211,7 @@ class Error(Exception): """ Raised when an error occurs configuring a service or pool. """ - pass # cope with minify_source() bug. + pass class Policy(object): diff --git a/mitogen/setns.py b/mitogen/setns.py index 46a50301..a3a3c7f3 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import ctypes import grp diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 656dc72c..459b1e73 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ Construct new children via the OpenSSH client. diff --git a/mitogen/su.py b/mitogen/su.py index 080c9782..0abf53a3 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import logging import re diff --git a/mitogen/sudo.py b/mitogen/sudo.py index a1a7b8af..d57b4bd6 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import base64 import logging diff --git a/mitogen/unix.py b/mitogen/unix.py index 1af1c0ec..8bf97d90 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe """ Permit connection of additional contexts that may act with the authority of diff --git a/mitogen/utils.py b/mitogen/utils.py index 71f7c35f..695c33b3 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -26,7 +26,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# !mitogen: minify_safe import datetime import functools diff --git a/preamble_size.py b/preamble_size.py index efb46eae..280d4f11 100755 --- a/preamble_size.py +++ b/preamble_size.py @@ -11,7 +11,6 @@ import zlib import mitogen.fakessh import mitogen.fork import mitogen.master -import mitogen.minify import mitogen.parent import mitogen.select import mitogen.service @@ -58,8 +57,8 @@ for mod in ( ): original = inspect.getsource(mod) original_size = len(original) - minimized = mitogen.minify.minimize_source(original) - minimized_size = len(minimized) + minimized = original + minimized_size = original_size compressed = zlib.compress(minimized.encode(), 9) compressed_size = len(compressed) print( diff --git a/tests/minify_test.py b/tests/minify_test.py deleted file mode 100644 index 97a51ed1..00000000 --- a/tests/minify_test.py +++ /dev/null @@ -1,112 +0,0 @@ -import codecs -import glob -import pprint -import sys - -import mitogen.minify -import testlib - - -def read_sample(fname): - sample_path = testlib.data_path('minimize_samples/' + fname) - sample_file = open(sample_path) - sample = sample_file.read() - sample_file.close() - return sample - - -class MinimizeSourceTest(testlib.TestCase): - func = staticmethod(mitogen.minify.minimize_source) - - def test_class(self): - original = read_sample('class.py') - expected = read_sample('class_min.py') - self.assertEqual(expected, self.func(original)) - - def test_comment(self): - original = read_sample('comment.py') - expected = read_sample('comment_min.py') - self.assertEqual(expected, self.func(original)) - - def test_def(self): - original = read_sample('def.py') - expected = read_sample('def_min.py') - self.assertEqual(expected, self.func(original)) - - def test_hashbang(self): - original = read_sample('hashbang.py') - expected = read_sample('hashbang_min.py') - self.assertEqual(expected, self.func(original)) - - def test_mod(self): - original = read_sample('mod.py') - expected = read_sample('mod_min.py') - self.assertEqual(expected, self.func(original)) - - def test_pass(self): - original = read_sample('pass.py') - expected = read_sample('pass_min.py') - self.assertEqual(expected, self.func(original)) - - def test_obstacle_course(self): - original = read_sample('obstacle_course.py') - expected = read_sample('obstacle_course_min.py') - self.assertEqual(expected, self.func(original)) - - -class MitogenCoreTest(testlib.TestCase): - # Verify minimize_source() succeeds for all built-in modules. - func = staticmethod(mitogen.minify.minimize_source) - - def read_source(self, name): - fp = codecs.open(name, encoding='utf-8') - try: - return fp.read() - finally: - fp.close() - - def _test_syntax_valid(self, minified, name): - compile(minified, name, 'exec') - - def _test_line_counts_match(self, original, minified): - self.assertEqual(original.count('\n'), - minified.count('\n')) - - def _test_non_blank_lines_match(self, name, original, minified): - # Verify first token matches. We just want to ensure line numbers make - # sense, this is good enough. - olines = original.splitlines() - mlines = minified.splitlines() - for i, (orig, mini) in enumerate(zip(olines, mlines)): - if i < 2: - assert orig == mini - continue - - owords = orig.split() - mwords = mini.split() - assert len(mwords) == 0 or (mwords[0] == owords[0]), pprint.pformat({ - 'line': i+1, - 'filename': name, - 'owords': owords, - 'mwords': mwords, - }) - - PY_24_25_SKIP = [ - # cProfile unsupported on 2.4, 2.6+ syntax is fine here. - 'mitogen/profiler.py', - ] - - def test_minify_all(self): - for name in glob.glob('mitogen/*.py'): - if name in self.PY_24_25_SKIP and sys.version_info < (2, 6): - continue - original = self.read_source(name) - try: - minified = self.func(original) - except Exception: - print('file was: ' + name) - raise - - self._test_syntax_valid(minified, name) - self._test_line_counts_match(original, minified) - self._test_non_blank_lines_match(name, original, minified) diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index ff18bbc5..4fc1afa7 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -285,7 +285,6 @@ class FindRelatedTest(testlib.TestCase): if sys.version_info < (2, 7): SIMPLE_EXPECT.add('mitogen.compat') - SIMPLE_EXPECT.add('mitogen.compat.tokenize') if sys.version_info < (2, 6): SIMPLE_EXPECT.add('mitogen.compat') SIMPLE_EXPECT.add('mitogen.compat.pkgutil') diff --git a/tests/responder_test.py b/tests/responder_test.py index 1131b421..8a0ec29b 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -193,7 +193,7 @@ class ForwardTest(testlib.RouterMixin, testlib.TestCase): self.assertEqual(2+os_fork, self.router.responder.get_module_count) self.assertEqual(2+os_fork, self.router.responder.good_load_module_count) self.assertLess(10000, self.router.responder.good_load_module_size) - self.assertGreater(40000, self.router.responder.good_load_module_size) + self.assertGreater(41000, self.router.responder.good_load_module_size) class BlacklistTest(testlib.TestCase):