From: Ɓukasz Langa Date: Thu, 12 Apr 2018 06:07:56 +0000 (-0700) Subject: Fix parsing of unaligned standalone comments X-Git-Url: https://git.madduck.net/etc/vim.git/commitdiff_plain/9138a75b759ecb690d63924503f88bfbc82d4862?ds=inline Fix parsing of unaligned standalone comments Fixes #99 Fixes #112 --- diff --git a/README.md b/README.md index 06b7324..d41da0c 100644 --- a/README.md +++ b/README.md @@ -491,6 +491,10 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md). ## Change Log +### 18.4a2 (unreleased) + +* fixed parsing of unaligned standalone comments (#99, #112) + ### 18.4a1 * added `--quiet` (#78) diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py index cc7b415..5cdd2e5 100644 --- a/blib2to3/pgen2/driver.py +++ b/blib2to3/pgen2/driver.py @@ -77,6 +77,12 @@ class Driver(object): self.logger.debug("Stop.") break prefix = "" + if type == token.INDENT: + if _prefix.startswith(value): + # Don't double-indent. Since we're delaying the prefix that + # would normally belong to INDENT, we need to put the value + # at the end versus at the beginning. + _prefix = _prefix[len(value):] + value if type in {token.INDENT, token.DEDENT}: prefix = _prefix lineno, column = end diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py index 4f03130..bd99454 100644 --- a/blib2to3/pgen2/tokenize.py +++ b/blib2to3/pgen2/tokenize.py @@ -412,10 +412,6 @@ def generate_tokens(readline): yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line) continue - if column > indents[-1]: # count indents - indents.append(column) - yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - if line[pos] == '#': # skip comments comment_token = line[pos:].rstrip('\r\n') nl_pos = pos + len(comment_token) @@ -425,6 +421,10 @@ def generate_tokens(readline): (lnum, nl_pos), (lnum, len(line)), line) continue + if column > indents[-1]: # count indents + indents.append(column) + yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + while column < indents[-1]: # count dedents if column not in indents: raise IndentationError( diff --git a/tests/comments2.py b/tests/comments2.py index 49ef2dc..848ddb1 100644 --- a/tests/comments2.py +++ b/tests/comments2.py @@ -23,6 +23,14 @@ __all__ = [ 'Generator', ] +if 'PYTHON' in os.environ: + add_compiler(compiler_from_env()) +else: + # for compiler in compilers.values(): + # add_compiler(compiler) + add_compiler(compilers[(7.0, 32)]) + # add_compiler(compilers[(7.1, 64)]) + # Comment before function. def inline_comments_in_brackets_ruin_everything(): if typedargslist: @@ -97,7 +105,7 @@ short # and round and round we go # and round and round we go - # let's return + # let's return return Node( syms.simple_stmt, [ @@ -144,6 +152,14 @@ __all__ = [ "Generator", ] +if "PYTHON" in os.environ: + add_compiler(compiler_from_env()) +else: + # for compiler in compilers.values(): + # add_compiler(compiler) + add_compiler(compilers[(7.0, 32)]) +# add_compiler(compilers[(7.1, 64)]) + # Comment before function.