Minor README updates (#842)

[etc/vim.git] / blib2to3 / pgen2 / driver.py

diff --git a/blib2to3/pgen2/driver.py b/blib2to3/pgen2/driver.py
index 2265c26443d2f45280112653f5f25eda59360b07..63b60bbad56f533924752f2dd86aea148ffd4f5a 100644 (file)
--- a/blib2to3/pgen2/driver.py
+++ b/blib2to3/pgen2/driver.py
@@ -32,7 +32,7 @@ class Driver(object):
     def __init__(self, grammar, convert=None, logger=None):
         self.grammar = grammar
         if logger is None:
     def __init__(self, grammar, convert=None, logger=None):
         self.grammar = grammar
         if logger is None:

-            logger = logging.getLogger()
+            logger = logging.getLogger(__name__)

         self.logger = logger
         self.convert = convert
 
         self.logger = logger
         self.convert = convert
 


@@ -70,24 +70,19 @@ class Driver(object):
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)
             if debug:
                 self.logger.debug("%s %r (prefix=%r)",
                                   token.tok_name[type], value, prefix)

-            if type in {token.INDENT, token.DEDENT}:
-                _prefix = prefix
+            if type == token.INDENT:
+                indent_columns.append(len(value))
+                _prefix = prefix + value

                 prefix = ""
                 prefix = ""

-            if type == token.DEDENT:
+                value = ""
+            elif type == token.DEDENT:

                 _indent_col = indent_columns.pop()
                 _indent_col = indent_columns.pop()

-                prefix, _prefix = self._partially_consume_prefix(_prefix, _indent_col)
+                prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col)

             if p.addtoken(type, value, (prefix, start)):
                 if debug:
                     self.logger.debug("Stop.")
                 break
             prefix = ""
             if p.addtoken(type, value, (prefix, start)):
                 if debug:
                     self.logger.debug("Stop.")
                 break
             prefix = ""

-            if type == token.INDENT:
-                indent_columns.append(len(value))
-                if _prefix.startswith(value):
-                    # Don't double-indent.  Since we're delaying the prefix that
-                    # would normally belong to INDENT, we need to put the value
-                    # at the end versus at the beginning.
-                    _prefix = _prefix[len(value):] + value

             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end
             if type in {token.INDENT, token.DEDENT}:
                 prefix = _prefix
             lineno, column = end


@@ -136,12 +131,10 @@ class Driver(object):
                     current_line = ""
                     current_column = 0
                     wait_for_nl = False
                     current_line = ""
                     current_column = 0
                     wait_for_nl = False

-            elif char == ' ':
+            elif char in ' \t':

                 current_column += 1
                 current_column += 1

-            elif char == '\t':
-                current_column += 4

             elif char == '\n':
             elif char == '\n':

-                # enexpected empty line
+                # unexpected empty line

                 current_column = 0
             else:
                 # indent is finished
                 current_column = 0
             else:
                 # indent is finished


@@ -149,18 +142,22 @@ class Driver(object):
         return ''.join(lines), current_line
 
 
         return ''.join(lines), current_line
 
 

-def _generate_pickle_name(gt):
+def _generate_pickle_name(gt, cache_dir=None):

     head, tail = os.path.splitext(gt)
     if tail == ".txt":
         tail = ""
     head, tail = os.path.splitext(gt)
     if tail == ".txt":
         tail = ""

-    return head + tail + ".".join(map(str, sys.version_info)) + ".pickle"
+    name = head + tail + ".".join(map(str, sys.version_info)) + ".pickle"
+    if cache_dir:
+        return os.path.join(cache_dir, os.path.basename(name))
+    else:
+        return name

 
 
 def load_grammar(gt="Grammar.txt", gp=None,
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
     if logger is None:
 
 
 def load_grammar(gt="Grammar.txt", gp=None,
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
     if logger is None:

-        logger = logging.getLogger()
+        logger = logging.getLogger(__name__)

     gp = _generate_pickle_name(gt) if gp is None else gp
     if force or not _newer(gp, gt):
         logger.info("Generating grammar tables from %s", gt)
     gp = _generate_pickle_name(gt) if gp is None else gp
     if force or not _newer(gp, gt):
         logger.info("Generating grammar tables from %s", gt)


@@ -186,7 +183,7 @@ def _newer(a, b):
     return os.path.getmtime(a) >= os.path.getmtime(b)
 
 
     return os.path.getmtime(a) >= os.path.getmtime(b)
 
 

-def load_packaged_grammar(package, grammar_source):
+def load_packaged_grammar(package, grammar_source, cache_dir=None):

     """Normally, loads a pickled grammar by doing
         pkgutil.get_data(package, pickled_grammar)
     where *pickled_grammar* is computed from *grammar_source* by adding the
     """Normally, loads a pickled grammar by doing
         pkgutil.get_data(package, pickled_grammar)
     where *pickled_grammar* is computed from *grammar_source* by adding the


@@ -198,8 +195,9 @@ def load_packaged_grammar(package, grammar_source):
 
     """
     if os.path.isfile(grammar_source):
 
     """
     if os.path.isfile(grammar_source):

-        return load_grammar(grammar_source)
-    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
+        gp = _generate_pickle_name(grammar_source, cache_dir) if cache_dir else None
+        return load_grammar(grammar_source, gp=gp)
+    pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir)

     data = pkgutil.get_data(package, pickled_name)
     g = grammar.Grammar()
     g.loads(data)
     data = pkgutil.get_data(package, pickled_name)
     g = grammar.Grammar()
     g.loads(data)