Don't crash and burn on empty lines with trailing whitespace

Fixes #80

diff --git a/README.md b/README.md
index 1cd1ef0ba4c0ce1a8f2eb11f35c7458727c3297b..1e28d5299aae4251a4048a0165f026db7e9cf1ca 100644 (file)
--- a/README.md
+++ b/README.md
@@ -333,6 +333,12 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Change Log
 
 
 ## Change Log
 

+### 18.3a5 (unreleased)
+
+* fixed 18.3a4 regression: don't crash and burn on empty lines with
+  trailing whitespace (#80)
+
+

 ### 18.3a4
 
 * `# fmt: off` and `# fmt: on` are implemented (#5)
 ### 18.3a4
 
 * `# fmt: off` and `# fmt: on` are implemented (#5)

diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 669c3f183bb8907a01e863167be8c1c2680b1e85..b6bbf4ec7dde2d912690be28a938bfb7f0742cc0 100644 (file)
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -430,24 +430,24 @@ def generate_tokens(readline):
                 yield stashed
                 stashed = None
 
                 yield stashed
                 stashed = None
 

+            if line[pos] in '\r\n':            # skip blank lines
+                yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
+                continue
+

             if column > indents[-1]:           # count indents
                 indents.append(column)
                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
 
             if column > indents[-1]:           # count indents
                 indents.append(column)
                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
 

-            if line[pos] in '#\r\n':           # skip comments or blank lines
-                if line[pos] == '#':
-                    comment_token = line[pos:].rstrip('\r\n')
-                    nl_pos = pos + len(comment_token)
-                    yield (COMMENT, comment_token,
-                           (lnum, pos), (lnum, pos + len(comment_token)), line)
-                    yield (NL, line[nl_pos:],
-                           (lnum, nl_pos), (lnum, len(line)), line)
-                else:
-                    yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
-                           (lnum, pos), (lnum, len(line)), line)
+            if line[pos] == '#':               # skip comments
+                comment_token = line[pos:].rstrip('\r\n')
+                nl_pos = pos + len(comment_token)
+                yield (COMMENT, comment_token,
+                        (lnum, pos), (lnum, pos + len(comment_token)), line)
+                yield (NL, line[nl_pos:],
+                        (lnum, nl_pos), (lnum, len(line)), line)

                 continue
 
                 continue
 

-            while column < indents[-1]:         # count dedents
+            while column < indents[-1]:        # count dedents

                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",
                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",

diff --git a/tests/function.py b/tests/function.py
index 08ab2b80a576f52a04fa6c9847bdfc90f6429e75..387e441b4d3dbd7a6813ce53e7668ca11e8c58d2 100644 (file)
--- a/tests/function.py
+++ b/tests/function.py
@@ -34,6 +34,7 @@ def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r''
 def spaces_types(a: int = 1, b: tuple = (), c: list = [], d: dict = {}, e: bool = True, f: int = -1, g: int = 1 if False else 2, h: str = "", i: str = r''): ...
 def spaces2(result= _core.Value(None)):
  ...
 def spaces_types(a: int = 1, b: tuple = (), c: list = [], d: dict = {}, e: bool = True, f: int = -1, g: int = 1 if False else 2, h: str = "", i: str = r''): ...
 def spaces2(result= _core.Value(None)):
  ...

+    # EMPTY LINE WITH WHITESPACE (this comment will be removed)

 def example(session):
     result = session.query(models.Customer.id).filter(
         models.Customer.account_id == account_id,
 def example(session):
     result = session.query(models.Customer.id).filter(
         models.Customer.account_id == account_id,

diff --git a/tests/test_black.py b/tests/test_black.py
index 62b4b1a7dad5cf30a00e45176a0d6565a4d736f3..759bda5af3dbea3c83036853b443cc4ada8de43d 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -17,6 +17,7 @@ ff = partial(black.format_file_in_place, line_length=ll, fast=True)
 fs = partial(black.format_str, line_length=ll)
 THIS_FILE = Path(__file__)
 THIS_DIR = THIS_FILE.parent
 fs = partial(black.format_str, line_length=ll)
 THIS_FILE = Path(__file__)
 THIS_DIR = THIS_FILE.parent

+EMPTY_LINE = '# EMPTY LINE WITH WHITESPACE' + ' (this comment will be removed)'

 
 
 def dump_to_stderr(*output: str) -> str:
 
 
 def dump_to_stderr(*output: str) -> str:


@@ -33,6 +34,7 @@ def read_data(name: str) -> Tuple[str, str]:
         lines = test.readlines()
     result = _input
     for line in lines:
         lines = test.readlines()
     result = _input
     for line in lines:

+        line = line.replace(EMPTY_LINE, '')

         if line.rstrip() == '# output':
             result = _output
             continue
         if line.rstrip() == '# output':
             result = _output
             continue