From 9138a75b759ecb690d63924503f88bfbc82d4862 Mon Sep 17 00:00:00 2001
From: =?utf8?q?=C5=81ukasz=20Langa?= <lukasz@langa.pl>
Date: Wed, 11 Apr 2018 23:07:56 -0700
Subject: [PATCH] Fix parsing of unaligned standalone comments

Fixes #99
Fixes #112
---
 README.md                  |  4 ++++
 blib2to3/pgen2/driver.py   |  6 ++++++
 blib2to3/pgen2/tokenize.py |  8 ++++----
 tests/comments2.py         | 18 +++++++++++++++++-
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 06b7324..d41da0c 100644
--- a/README.md
+++ b/README.md
@@ -491,6 +491,10 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Change Log
 
+### 18.4a2 (unreleased)
+
+* fixed parsing of unaligned standalone comments (#99, #112)
+
 ### 18.4a1
 
 * added `--quiet` (#78)
diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index cc7b415..5cdd2e5 100644
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -77,6 +77,12 @@ class Driver(object):
                     self.logger.debug("Stop.")
                 break
             prefix = ""
+            if type == token.INDENT:
+                if _prefix.startswith(value):
+                    # Don't double-indent.  Since we're delaying the prefix that
+                    # would normally belong to INDENT, we need to put the value
+                    # at the end versus at the beginning.
+                    _prefix = _prefix[len(value):] + value
             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end
diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 4f03130..bd99454 100644
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -412,10 +412,6 @@ def generate_tokens(readline):
                 yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
                 continue
 
-            if column > indents[-1]:           # count indents
-                indents.append(column)
-                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
-
             if line[pos] == '#':               # skip comments
                 comment_token = line[pos:].rstrip('\r\n')
                 nl_pos = pos + len(comment_token)
@@ -425,6 +421,10 @@ def generate_tokens(readline):
                         (lnum, nl_pos), (lnum, len(line)), line)
                 continue
 
+            if column > indents[-1]:           # count indents
+                indents.append(column)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+
             while column < indents[-1]:        # count dedents
                 if column not in indents:
                     raise IndentationError(
diff --git a/tests/comments2.py b/tests/comments2.py
index 49ef2dc..848ddb1 100644
--- a/tests/comments2.py
+++ b/tests/comments2.py
@@ -23,6 +23,14 @@ __all__ = [
     'Generator',
 ]
 
+if 'PYTHON' in os.environ:
+    add_compiler(compiler_from_env())
+else:
+    # for compiler in compilers.values():
+         # add_compiler(compiler)
+    add_compiler(compilers[(7.0, 32)])
+    # add_compiler(compilers[(7.1, 64)])
+
 # Comment before function.
 def inline_comments_in_brackets_ruin_everything():
     if typedargslist:
@@ -97,7 +105,7 @@ short
             # and round and round we go
         # and round and round we go
 
-    # let's return
+   # let's return
     return Node(
         syms.simple_stmt,
         [
@@ -144,6 +152,14 @@ __all__ = [
     "Generator",
 ]
 
+if "PYTHON" in os.environ:
+    add_compiler(compiler_from_env())
+else:
+    # for compiler in compilers.values():
+    # add_compiler(compiler)
+    add_compiler(compilers[(7.0, 32)])
+# add_compiler(compilers[(7.1, 64)])
+
 # Comment before function.
 
 
-- 
2.39.2