[blib2to3] Make the grammar pickles faster

[etc/vim.git] / blib2to3 / pgen2 / tokenize.py

diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 45afc5f4e53fcf7d0f1f602638804a93726d50c1..b6bbf4ec7dde2d912690be28a938bfb7f0742cc0 100644 (file)
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -31,7 +31,7 @@ __credits__ = \
 
 import string, re
 from codecs import BOM_UTF8, lookup
 
 import string, re
 from codecs import BOM_UTF8, lookup

-from lib2to3.pgen2.token import *
+from blib2to3.pgen2.token import *

 
 from . import token
 __all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize",
 
 from . import token
 __all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize",


@@ -430,23 +430,24 @@ def generate_tokens(readline):
                 yield stashed
                 stashed = None
 
                 yield stashed
                 stashed = None
 

-            if line[pos] in '#\r\n':           # skip comments or blank lines
-                if line[pos] == '#':
-                    comment_token = line[pos:].rstrip('\r\n')
-                    nl_pos = pos + len(comment_token)
-                    yield (COMMENT, comment_token,
-                           (lnum, pos), (lnum, pos + len(comment_token)), line)
-                    yield (NL, line[nl_pos:],
-                           (lnum, nl_pos), (lnum, len(line)), line)
-                else:
-                    yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
-                           (lnum, pos), (lnum, len(line)), line)
+            if line[pos] in '\r\n':            # skip blank lines
+                yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)

                 continue
 
                 continue
 

-            if column > indents[-1]:           # count indents or dedents
+            if column > indents[-1]:           # count indents

                 indents.append(column)
                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
                 indents.append(column)
                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)

-            while column < indents[-1]:
+
+            if line[pos] == '#':               # skip comments
+                comment_token = line[pos:].rstrip('\r\n')
+                nl_pos = pos + len(comment_token)
+                yield (COMMENT, comment_token,
+                        (lnum, pos), (lnum, pos + len(comment_token)), line)
+                yield (NL, line[nl_pos:],
+                        (lnum, nl_pos), (lnum, len(line)), line)
+                continue
+
+            while column < indents[-1]:        # count dedents

                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",
                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",