blib2to3: Never put prefixes on INDENT leaves either

diff --git a/blib2to3/README b/blib2to3/README
index 773aaf6d069a04cf32521f23cd30a0b8faa7643f..6f9da929cdb3794e5f08dea56ef39cd727726526 100644 (file)
--- a/blib2to3/README
+++ b/blib2to3/README
@@ -7,5 +7,6 @@ Reasons for forking:
   *args and **kwargs
 - backport of GH-6143 that restores the ability to reformat legacy usage of
   `async`
-- better ability to debug (better reprs for starters)
+- better ability to debug (better reprs)
+- INDENT and DEDENT don't hold whitespace and comment prefixes
 - ability to Cythonize

diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index eabc72b779571f2aa2b46b25ce81f2d495660819..cc7b415f47fcd30e711d23fbfc80fc89d2b2af5b 100644 (file)
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -69,7 +69,7 @@ class Driver(object):
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)
-            if type == token.DEDENT:
+            if type in {token.INDENT, token.DEDENT}:
                 _prefix = prefix
                 prefix = ""
             if p.addtoken(type, value, (prefix, start)):
@@ -77,7 +77,7 @@ class Driver(object):
                     self.logger.debug("Stop.")
                 break
             prefix = ""
-            if type == token.DEDENT:
+            if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end
             if value.endswith("\n"):

diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 6dada4745aaad230dab453b1bc4e2e923f63b80d..669c3f183bb8907a01e863167be8c1c2680b1e85 100644 (file)
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -430,6 +430,10 @@ def generate_tokens(readline):
                 yield stashed
                 stashed = None
 
+            if column > indents[-1]:           # count indents
+                indents.append(column)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+
             if line[pos] in '#\r\n':           # skip comments or blank lines
                 if line[pos] == '#':
                     comment_token = line[pos:].rstrip('\r\n')
@@ -443,10 +447,7 @@ def generate_tokens(readline):
                            (lnum, pos), (lnum, len(line)), line)
                 continue
 
-            if column > indents[-1]:           # count indents or dedents
-                indents.append(column)
-                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
-            while column < indents[-1]:
+            while column < indents[-1]:         # count dedents
                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",

diff --git a/tests/debug_visitor.out b/tests/debug_visitor.out
index 92b72806a02e2e7e6c01a8557b0256bf40246cc7..45027243bc19851ced34d417b63fe595df8e8e44 100644 (file)
--- a/tests/debug_visitor.out
+++ b/tests/debug_visitor.out
@@ -546,7 +546,7 @@ file_input
                     simple_stmt
                       power
                         NAME
- "                # We don't have to handle prefixes for `Node` objects since\n                # that delegates to the first child anyway.\n"
+ "# We don't have to handle prefixes for `Node` objects since\n                # that delegates to the first child anyway.\n                "
  'out'
                         trailer
                           LPAR

diff --git a/tests/empty_lines.py b/tests/empty_lines.py
index 6419f6132955d80ec4e1039aa6c933d50922006b..8f00ddced4bc4b15c64d3fee4994ae16c108e6dc 100644 (file)
--- a/tests/empty_lines.py
+++ b/tests/empty_lines.py
@@ -105,6 +105,7 @@ def f():
     if not prev:
         prevp = preceding_leaf(p)
         if not prevp or prevp.type in OPENING_BRACKETS:
+
             return NO
 
         if prevp.type == token.EQUAL: