Fix parsing of unaligned standalone comments

Fixes #99
Fixes #112

diff --git a/README.md b/README.md
index 06b732466eb3b305307992f35a23df99cc476956..d41da0cf49bcd8bed68b788bb8bf352fd7e75671 100644 (file)
--- a/README.md
+++ b/README.md
@@ -491,6 +491,10 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Change Log
 
+### 18.4a2 (unreleased)
+
+* fixed parsing of unaligned standalone comments (#99, #112)
+
 ### 18.4a1
 
 * added `--quiet` (#78)

diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index cc7b415f47fcd30e711d23fbfc80fc89d2b2af5b..5cdd2e59e6156cecd8ffa3cebc822b4be20ef7ae 100644 (file)
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -77,6 +77,12 @@ class Driver(object):
                     self.logger.debug("Stop.")
                 break
             prefix = ""
+            if type == token.INDENT:
+                if _prefix.startswith(value):
+                    # Don't double-indent.  Since we're delaying the prefix that
+                    # would normally belong to INDENT, we need to put the value
+                    # at the end versus at the beginning.
+                    _prefix = _prefix[len(value):] + value
             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end

diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 4f031306378ab20d13aec985e3c8715f59c8606a..bd9945498bd49adb40514134646de3d0a60c41b4 100644 (file)
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -412,10 +412,6 @@ def generate_tokens(readline):
                 yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
                 continue
 
-            if column > indents[-1]:           # count indents
-                indents.append(column)
-                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
-
             if line[pos] == '#':               # skip comments
                 comment_token = line[pos:].rstrip('\r\n')
                 nl_pos = pos + len(comment_token)
@@ -425,6 +421,10 @@ def generate_tokens(readline):
                         (lnum, nl_pos), (lnum, len(line)), line)
                 continue
 
+            if column > indents[-1]:           # count indents
+                indents.append(column)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+
             while column < indents[-1]:        # count dedents
                 if column not in indents:
                     raise IndentationError(

diff --git a/tests/comments2.py b/tests/comments2.py
index 49ef2dce09a696c8e83c72c2b7ab54e5f352af5c..848ddb1ae8541a9dcf25b429e6481fe5da204cf4 100644 (file)
--- a/tests/comments2.py
+++ b/tests/comments2.py
@@ -23,6 +23,14 @@ __all__ = [
     'Generator',
 ]
 
+if 'PYTHON' in os.environ:
+    add_compiler(compiler_from_env())
+else:
+    # for compiler in compilers.values():
+         # add_compiler(compiler)
+    add_compiler(compilers[(7.0, 32)])
+    # add_compiler(compilers[(7.1, 64)])
+
 # Comment before function.
 def inline_comments_in_brackets_ruin_everything():
     if typedargslist:
@@ -97,7 +105,7 @@ short
             # and round and round we go
         # and round and round we go
 
-    # let's return
+   # let's return
     return Node(
         syms.simple_stmt,
         [
@@ -144,6 +152,14 @@ __all__ = [
     "Generator",
 ]
 
+if "PYTHON" in os.environ:
+    add_compiler(compiler_from_env())
+else:
+    # for compiler in compilers.values():
+    # add_compiler(compiler)
+    add_compiler(compilers[(7.0, 32)])
+# add_compiler(compilers[(7.1, 64)])
+
 # Comment before function.