Used by: add pandas and Pillow (#1057)

[etc/vim.git] / blib2to3 / pgen2 / driver.py

diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index a51ffc30b9fe9195c1ff936d403387e7fe8b4110..6452c57a1fa4b4a9e4c35541205c4919d6e48ddc 100644 (file)
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -29,10 +29,15 @@ from . import grammar, parse, token, tokenize, pgen
 
 class Driver(object):
 
 
 class Driver(object):
 

-    def __init__(self, grammar, convert=None, logger=None):
+    def __init__(
+        self,
+        grammar,
+        convert=None,
+        logger=None,
+    ):

         self.grammar = grammar
         if logger is None:
         self.grammar = grammar
         if logger is None:

-            logger = logging.getLogger()
+            logger = logging.getLogger(__name__)

         self.logger = logger
         self.convert = convert
 
         self.logger = logger
         self.convert = convert
 


@@ -97,7 +102,7 @@ class Driver(object):
 
     def parse_stream_raw(self, stream, debug=False):
         """Parse a stream and return the syntax tree."""
 
     def parse_stream_raw(self, stream, debug=False):
         """Parse a stream and return the syntax tree."""

-        tokens = tokenize.generate_tokens(stream.readline)
+        tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar)

         return self.parse_tokens(tokens, debug)
 
     def parse_stream(self, stream, debug=False):
         return self.parse_tokens(tokens, debug)
 
     def parse_stream(self, stream, debug=False):


@@ -111,7 +116,10 @@ class Driver(object):
 
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
 
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""

-        tokens = tokenize.generate_tokens(io.StringIO(text).readline)
+        tokens = tokenize.generate_tokens(
+            io.StringIO(text).readline,
+            grammar=self.grammar
+        )

         return self.parse_tokens(tokens, debug)
 
     def _partially_consume_prefix(self, prefix, column):
         return self.parse_tokens(tokens, debug)
 
     def _partially_consume_prefix(self, prefix, column):


@@ -131,12 +139,10 @@ class Driver(object):
                     current_line = ""
                     current_column = 0
                     wait_for_nl = False
                     current_line = ""
                     current_column = 0
                     wait_for_nl = False

-            elif char == ' ':
+            elif char in ' \t':

                 current_column += 1
                 current_column += 1

-            elif char == '\t':
-                current_column += 4

             elif char == '\n':
             elif char == '\n':

-                # enexpected empty line
+                # unexpected empty line

                 current_column = 0
             else:
                 # indent is finished
                 current_column = 0
             else:
                 # indent is finished


@@ -159,7 +165,7 @@ def load_grammar(gt="Grammar.txt", gp=None,
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
     if logger is None:
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
     if logger is None:

-        logger = logging.getLogger()
+        logger = logging.getLogger(__name__)

     gp = _generate_pickle_name(gt) if gp is None else gp
     if force or not _newer(gp, gt):
         logger.info("Generating grammar tables from %s", gt)
     gp = _generate_pickle_name(gt) if gp is None else gp
     if force or not _newer(gp, gt):
         logger.info("Generating grammar tables from %s", gt)