From: Łukasz Langa <lukasz@langa.pl>
Date: Sat, 24 Mar 2018 00:12:20 +0000 (-0700)
Subject: blib2to3: Never put prefixes on INDENT leaves either
X-Git-Url: https://git.madduck.net/etc/vim.git/commitdiff_plain/8c565d86843649dc1467b5909b446021e4417bba?ds=sidebyside;hp=7e1c5b2ba6355f1a6916cf5545dc577cf2e6c727

blib2to3: Never put prefixes on INDENT leaves either
---

diff --git a/blib2to3/README b/blib2to3/README
index 773aaf6..6f9da92 100644
--- a/blib2to3/README
+++ b/blib2to3/README
@@ -7,5 +7,6 @@ Reasons for forking:
   *args and **kwargs
 - backport of GH-6143 that restores the ability to reformat legacy usage of
   `async`
-- better ability to debug (better reprs for starters)
+- better ability to debug (better reprs)
+- INDENT and DEDENT don't hold whitespace and comment prefixes
 - ability to Cythonize
diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index eabc72b..cc7b415 100644
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -69,7 +69,7 @@ class Driver(object):
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)
-            if type == token.DEDENT:
+            if type in {token.INDENT, token.DEDENT}:
                 _prefix = prefix
                 prefix = ""
             if p.addtoken(type, value, (prefix, start)):
@@ -77,7 +77,7 @@ class Driver(object):
                     self.logger.debug("Stop.")
                 break
             prefix = ""
-            if type == token.DEDENT:
+            if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end
             if value.endswith("\n"):
diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 6dada47..669c3f1 100644
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -430,6 +430,10 @@ def generate_tokens(readline):
                 yield stashed
                 stashed = None
 
+            if column > indents[-1]:           # count indents
+                indents.append(column)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+
             if line[pos] in '#\r\n':           # skip comments or blank lines
                 if line[pos] == '#':
                     comment_token = line[pos:].rstrip('\r\n')
@@ -443,10 +447,7 @@ def generate_tokens(readline):
                            (lnum, pos), (lnum, len(line)), line)
                 continue
 
-            if column > indents[-1]:           # count indents or dedents
-                indents.append(column)
-                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
-            while column < indents[-1]:
+            while column < indents[-1]:         # count dedents
                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",
diff --git a/tests/debug_visitor.out b/tests/debug_visitor.out
index 92b7280..4502724 100644
--- a/tests/debug_visitor.out
+++ b/tests/debug_visitor.out
@@ -546,7 +546,7 @@ file_input
                     simple_stmt
                       power
                         NAME
- "                # We don't have to handle prefixes for `Node` objects since\n                # that delegates to the first child anyway.\n"
+ "# We don't have to handle prefixes for `Node` objects since\n                # that delegates to the first child anyway.\n                "
  'out'
                         trailer
                           LPAR
diff --git a/tests/empty_lines.py b/tests/empty_lines.py
index 6419f61..8f00ddc 100644
--- a/tests/empty_lines.py
+++ b/tests/empty_lines.py
@@ -105,6 +105,7 @@ def f():
     if not prev:
         prevp = preceding_leaf(p)
         if not prevp or prevp.type in OPENING_BRACKETS:
+
             return NO
 
         if prevp.type == token.EQUAL: