Add support for always tokenizing async/await as keywords

[etc/vim.git] / blib2to3 / pgen2 / driver.py

diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index af7ca1799ca8dc733871755115c5d4ad89c5692a..e681b526a2219ae46dff248bb991173622c7698e 100644 (file)
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -29,12 +29,19 @@ from . import grammar, parse, token, tokenize, pgen
 
 class Driver(object):
 
-    def __init__(self, grammar, convert=None, logger=None):
+    def __init__(
+        self,
+        grammar,
+        convert=None,
+        logger=None,
+        tokenizer_config=tokenize.TokenizerConfig(),
+    ):
         self.grammar = grammar
         if logger is None:
-            logger = logging.getLogger()
+            logger = logging.getLogger(__name__)
         self.logger = logger
         self.convert = convert
+        self.tokenizer_config = tokenizer_config
 
     def parse_tokens(self, tokens, debug=False):
         """Parse a series of tokens and return the syntax tree."""
@@ -70,24 +77,19 @@ class Driver(object):
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)
-            if type in {token.INDENT, token.DEDENT}:
-                _prefix = prefix
+            if type == token.INDENT:
+                indent_columns.append(len(value))
+                _prefix = prefix + value
                 prefix = ""
-            if type == token.DEDENT:
+                value = ""
+            elif type == token.DEDENT:
                 _indent_col = indent_columns.pop()
-                prefix, _prefix = self._partially_consume_prefix(_prefix, _indent_col)
+                prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col)
             if p.addtoken(type, value, (prefix, start)):
                 if debug:
                     self.logger.debug("Stop.")
                 break
             prefix = ""
-            if type == token.INDENT:
-                indent_columns.append(len(value))
-                if _prefix.startswith(value):
-                    # Don't double-indent.  Since we're delaying the prefix that
-                    # would normally belong to INDENT, we need to put the value
-                    # at the end versus at the beginning.
-                    _prefix = _prefix[len(value):] + value
             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end
@@ -102,7 +104,7 @@ class Driver(object):
 
     def parse_stream_raw(self, stream, debug=False):
         """Parse a stream and return the syntax tree."""
-        tokens = tokenize.generate_tokens(stream.readline)
+        tokens = tokenize.generate_tokens(stream.readline, config=self.tokenizer_config)
         return self.parse_tokens(tokens, debug)
 
     def parse_stream(self, stream, debug=False):
@@ -116,7 +118,10 @@ class Driver(object):
 
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
-        tokens = tokenize.generate_tokens(io.StringIO(text).readline)
+        tokens = tokenize.generate_tokens(
+            io.StringIO(text).readline,
+            config=self.tokenizer_config,
+        )
         return self.parse_tokens(tokens, debug)
 
     def _partially_consume_prefix(self, prefix, column):
@@ -136,12 +141,10 @@ class Driver(object):
                     current_line = ""
                     current_column = 0
                     wait_for_nl = False
-            elif char == ' ':
+            elif char in ' \t':
                 current_column += 1
-            elif char == '\t':
-                current_column += 4
             elif char == '\n':
-                # enexpected empty line
+                # unexpected empty line
                 current_column = 0
             else:
                 # indent is finished
@@ -164,7 +167,7 @@ def load_grammar(gt="Grammar.txt", gp=None,
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
     if logger is None:
-        logger = logging.getLogger()
+        logger = logging.getLogger(__name__)
     gp = _generate_pickle_name(gt) if gp is None else gp
     if force or not _newer(gp, gt):
         logger.info("Generating grammar tables from %s", gt)