Add support for always tokenizing async/await as keywords

[etc/vim.git] / blib2to3 / pgen2 / driver.py

diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index 63b60bbad56f533924752f2dd86aea148ffd4f5a..e681b526a2219ae46dff248bb991173622c7698e 100644 (file)
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -29,12 +29,19 @@ from . import grammar, parse, token, tokenize, pgen
 
 class Driver(object):
 
-    def __init__(self, grammar, convert=None, logger=None):
+    def __init__(
+        self,
+        grammar,
+        convert=None,
+        logger=None,
+        tokenizer_config=tokenize.TokenizerConfig(),
+    ):
         self.grammar = grammar
         if logger is None:
             logger = logging.getLogger(__name__)
         self.logger = logger
         self.convert = convert
+        self.tokenizer_config = tokenizer_config
 
     def parse_tokens(self, tokens, debug=False):
         """Parse a series of tokens and return the syntax tree."""
@@ -97,7 +104,7 @@ class Driver(object):
 
     def parse_stream_raw(self, stream, debug=False):
         """Parse a stream and return the syntax tree."""
-        tokens = tokenize.generate_tokens(stream.readline)
+        tokens = tokenize.generate_tokens(stream.readline, config=self.tokenizer_config)
         return self.parse_tokens(tokens, debug)
 
     def parse_stream(self, stream, debug=False):
@@ -111,7 +118,10 @@ class Driver(object):
 
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
-        tokens = tokenize.generate_tokens(io.StringIO(text).readline)
+        tokens = tokenize.generate_tokens(
+            io.StringIO(text).readline,
+            config=self.tokenizer_config,
+        )
         return self.parse_tokens(tokens, debug)
 
     def _partially_consume_prefix(self, prefix, column):