minimize_source: Implement reindentation

mitogen/parent.py

@@ -82,6 +82,7 @@ def minimize_source(source):
     tokens = tokenize.generate_tokens(cStringIO.StringIO(source).readline)
     tokens = strip_comments(tokens)
     tokens = strip_docstrings(tokens)
+    tokens = reindent(tokens)
     return tokenize.untokenize(tokens)
 
 
@@ -142,6 +143,25 @@ def strip_docstrings(tokens):
             yield t
 
 
+def reindent(tokens, indent=' '):
+    old_levels = []
+    old_level = 0
+    new_level = 0
+    for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
+        if typ == tokenize.INDENT:
+            old_levels.append(old_level)
+            old_level = len(tok)
+            new_level += 1
+            tok = indent * new_level
+        elif typ == tokenize.DEDENT:
+            old_level = old_levels.pop()
+            new_level -= 1
+        start_col = max(0, start_col - old_level + new_level)
+        if start_row == end_row:
+            end_col = start_col + len(tok)
+        yield typ, tok, (start_row, start_col), (end_row, end_col), line
+
+
 def flags(names):
     """Return the result of ORing a set of (space separated) :py:mod:`termios`
     module constants together."""

tests/data/minimize_samples/class_min.py

@@ -2,7 +2,7 @@ class C:
 
 
 
-    def method(self):
+ def method(self):
 
 
-        pass
+  pass

tests/data/minimize_samples/def_min.py

@@ -1,4 +1,4 @@
 def f():
 
 
-    pass
+ pass

tests/data/minimize_samples/obstacle_course_min.py

@@ -29,21 +29,21 @@ b,
 def f2():
 
 
-    pass
+ pass
 class c:
 
 
 
 
-    f='justastring'
+ f='justastring'
 
 
 @f1
 class c2(object):
 
-    def __init__(self):
+ def __init__(self):
 
-        def inner(): pass
-        d = {'a':0}
-        for x in '':
+  def inner(): pass
+  d = {'a':0}
+  for x in '':