Add SECURITY.md (#3612)

[etc/vim.git] / src / blib2to3 / pgen2 / driver.py

diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py
index 5edd75b1333991d692a8f57ac84101b1a609e916..1741b33c5103d0719b499309ecd0e6049c11cfa4 100644 (file)
--- a/src/blib2to3/pgen2/driver.py
+++ b/src/blib2to3/pgen2/driver.py
@@ -23,6 +23,7 @@ import pkgutil
 import sys
 from typing import (
     Any,
 import sys
 from typing import (
     Any,

+    cast,

     IO,
     Iterable,
     List,
     IO,
     Iterable,
     List,


@@ -34,14 +35,15 @@ from typing import (
     Generic,
     Union,
 )
     Generic,
     Union,
 )

+from contextlib import contextmanager

 from dataclasses import dataclass, field
 
 # Pgen imports
 from . import grammar, parse, token, tokenize, pgen
 from logging import Logger
 from dataclasses import dataclass, field
 
 # Pgen imports
 from . import grammar, parse, token, tokenize, pgen
 from logging import Logger

-from blib2to3.pytree import _Convert, NL
+from blib2to3.pytree import NL

 from blib2to3.pgen2.grammar import Grammar
 from blib2to3.pgen2.grammar import Grammar

-from contextlib import contextmanager
+from blib2to3.pgen2.tokenize import GoodTokenInfo

 
 Path = Union[str, "os.PathLike[str]"]
 
 
 Path = Union[str, "os.PathLike[str]"]
 


@@ -105,7 +107,7 @@ class TokenProxy:
 
     def can_advance(self, to: int) -> bool:
         # Try to eat, fail if it can't. The eat operation is cached
 
     def can_advance(self, to: int) -> bool:
         # Try to eat, fail if it can't. The eat operation is cached

-        # so there wont be any additional cost of eating here
+        # so there won't be any additional cost of eating here

         try:
             self.eat(to)
         except StopIteration:
         try:
             self.eat(to)
         except StopIteration:


@@ -115,29 +117,23 @@ class TokenProxy:
 
 
 class Driver(object):
 
 
 class Driver(object):

-    def __init__(
-        self,
-        grammar: Grammar,
-        convert: Optional[_Convert] = None,
-        logger: Optional[Logger] = None,
-    ) -> None:
+    def __init__(self, grammar: Grammar, logger: Optional[Logger] = None) -> None:

         self.grammar = grammar
         if logger is None:
             logger = logging.getLogger(__name__)
         self.logger = logger
         self.grammar = grammar
         if logger is None:
             logger = logging.getLogger(__name__)
         self.logger = logger

-        self.convert = convert

 
 

-    def parse_tokens(self, tokens: Iterable[Any], debug: bool = False) -> NL:
+    def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> NL:

         """Parse a series of tokens and return the syntax tree."""
         # XXX Move the prefix computation into a wrapper around tokenize.
         proxy = TokenProxy(tokens)
 
         """Parse a series of tokens and return the syntax tree."""
         # XXX Move the prefix computation into a wrapper around tokenize.
         proxy = TokenProxy(tokens)
 

-        p = parse.Parser(self.grammar, self.convert)
+        p = parse.Parser(self.grammar)

         p.setup(proxy=proxy)
 
         lineno = 1
         column = 0
         p.setup(proxy=proxy)
 
         lineno = 1
         column = 0

-        indent_columns = []
+        indent_columns: List[int] = []

         type = value = start = end = line_text = None
         prefix = ""
 
         type = value = start = end = line_text = None
         prefix = ""
 


@@ -163,6 +159,7 @@ class Driver(object):
             if type == token.OP:
                 type = grammar.opmap[value]
             if debug:
             if type == token.OP:
                 type = grammar.opmap[value]
             if debug:

+                assert type is not None

                 self.logger.debug(
                     "%s %r (prefix=%r)", token.tok_name[type], value, prefix
                 )
                 self.logger.debug(
                     "%s %r (prefix=%r)", token.tok_name[type], value, prefix
                 )


@@ -174,7 +171,7 @@ class Driver(object):
             elif type == token.DEDENT:
                 _indent_col = indent_columns.pop()
                 prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col)
             elif type == token.DEDENT:
                 _indent_col = indent_columns.pop()
                 prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col)

-            if p.addtoken(type, value, (prefix, start)):
+            if p.addtoken(cast(int, type), value, (prefix, start)):

                 if debug:
                     self.logger.debug("Stop.")
                 break
                 if debug:
                     self.logger.debug("Stop.")
                 break


@@ -266,14 +263,13 @@ def load_grammar(
         logger = logging.getLogger(__name__)
     gp = _generate_pickle_name(gt) if gp is None else gp
     if force or not _newer(gp, gt):
         logger = logging.getLogger(__name__)
     gp = _generate_pickle_name(gt) if gp is None else gp
     if force or not _newer(gp, gt):

-        logger.info("Generating grammar tables from %s", gt)

         g: grammar.Grammar = pgen.generate_grammar(gt)
         if save:
         g: grammar.Grammar = pgen.generate_grammar(gt)
         if save:

-            logger.info("Writing grammar tables to %s", gp)

             try:
                 g.dump(gp)
             try:
                 g.dump(gp)

-            except OSError as e:
-                logger.info("Writing failed: %s", e)
+            except OSError:
+                # Ignore error, caching is not vital.
+                pass

     else:
         g = grammar.Grammar()
         g.load(gp)
     else:
         g = grammar.Grammar()
         g.load(gp)