]>
git.madduck.net Git - etc/vim.git/blobdiff - blib2to3/pgen2/driver.py
madduck's git repository
Every one of the projects in this repository is available at the canonical
URL git://git.madduck.net/madduck/pub/<projectpath> — see
each project's metadata for the exact URL.
All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@ git. madduck. net .
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
SSH access, as well as push access can be individually
arranged .
If you use my repositories frequently, consider adding the following
snippet to ~/.gitconfig and using the third clone URL listed for each
project:
[url "git://git.madduck.net/madduck/"]
insteadOf = madduck:
import logging
import pkgutil
import sys
import logging
import pkgutil
import sys
+from typing import (
+ Any,
+ Callable,
+ IO,
+ Iterable,
+ List,
+ Optional,
+ Text,
+ Tuple,
+ Union,
+ Sequence,
+)
# Pgen imports
from . import grammar, parse, token, tokenize, pgen
# Pgen imports
from . import grammar, parse, token, tokenize, pgen
+from logging import Logger
+from blib2to3.pytree import _Convert, NL
+from blib2to3.pgen2.grammar import Grammar
+
+Path = Union[str, "os.PathLike[str]"]
- def __init__(self, grammar, convert=None, logger=None):
+ def __init__(
+ self,
+ grammar: Grammar,
+ convert: Optional[_Convert] = None,
+ logger: Optional[Logger] = None,
+ ) -> None:
self.grammar = grammar
if logger is None:
logger = logging.getLogger(__name__)
self.logger = logger
self.convert = convert
self.grammar = grammar
if logger is None:
logger = logging.getLogger(__name__)
self.logger = logger
self.convert = convert
- def parse_tokens(self, tokens, debug=False) :
+ def parse_tokens(self, tokens: Iterable[Any], debug: bool = False) -> NL :
"""Parse a series of tokens and return the syntax tree."""
# XXX Move the prefix computation into a wrapper around tokenize.
p = parse.Parser(self.grammar, self.convert)
"""Parse a series of tokens and return the syntax tree."""
# XXX Move the prefix computation into a wrapper around tokenize.
p = parse.Parser(self.grammar, self.convert)
column = 0
else:
# We never broke out -- EOF is too soon (how can this happen???)
column = 0
else:
# We never broke out -- EOF is too soon (how can this happen???)
+ assert start is not None
raise parse.ParseError("incomplete input", type, value, (prefix, start))
raise parse.ParseError("incomplete input", type, value, (prefix, start))
+ assert p.rootnode is not None
- def parse_stream_raw(self, stream, debug=False) :
+ def parse_stream_raw(self, stream: IO[Text], debug: bool = False) -> NL :
"""Parse a stream and return the syntax tree."""
tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar)
return self.parse_tokens(tokens, debug)
"""Parse a stream and return the syntax tree."""
tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar)
return self.parse_tokens(tokens, debug)
- def parse_stream(self, stream, debug=False) :
+ def parse_stream(self, stream: IO[Text], debug: bool = False) -> NL :
"""Parse a stream and return the syntax tree."""
return self.parse_stream_raw(stream, debug)
"""Parse a stream and return the syntax tree."""
return self.parse_stream_raw(stream, debug)
- def parse_file(self, filename, encoding=None, debug=False):
+ def parse_file(
+ self, filename: Path, encoding: Optional[Text] = None, debug: bool = False,
+ ) -> NL:
"""Parse a file and return the syntax tree."""
with io.open(filename, "r", encoding=encoding) as stream:
return self.parse_stream(stream, debug)
"""Parse a file and return the syntax tree."""
with io.open(filename, "r", encoding=encoding) as stream:
return self.parse_stream(stream, debug)
- def parse_string(self, text, debug=False) :
+ def parse_string(self, text: Text, debug: bool = False) -> NL :
"""Parse a string and return the syntax tree."""
tokens = tokenize.generate_tokens(
io.StringIO(text).readline, grammar=self.grammar
)
return self.parse_tokens(tokens, debug)
"""Parse a string and return the syntax tree."""
tokens = tokenize.generate_tokens(
io.StringIO(text).readline, grammar=self.grammar
)
return self.parse_tokens(tokens, debug)
- def _partially_consume_prefix(self, prefix, column) :
- lines = []
+ def _partially_consume_prefix(self, prefix: Text, column: int) -> Tuple[Text, Text] :
+ lines: List[str] = []
current_line = ""
current_column = 0
wait_for_nl = False
current_line = ""
current_column = 0
wait_for_nl = False
return "".join(lines), current_line
return "".join(lines), current_line
-def _generate_pickle_name(gt, cache_dir=None) :
+def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> Text :
head, tail = os.path.splitext(gt)
if tail == ".txt":
tail = ""
head, tail = os.path.splitext(gt)
if tail == ".txt":
tail = ""
-def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None):
+def load_grammar(
+ gt: Text = "Grammar.txt",
+ gp: Optional[Text] = None,
+ save: bool = True,
+ force: bool = False,
+ logger: Optional[Logger] = None,
+) -> Grammar:
"""Load the grammar (maybe from a pickle)."""
if logger is None:
logger = logging.getLogger(__name__)
gp = _generate_pickle_name(gt) if gp is None else gp
if force or not _newer(gp, gt):
logger.info("Generating grammar tables from %s", gt)
"""Load the grammar (maybe from a pickle)."""
if logger is None:
logger = logging.getLogger(__name__)
gp = _generate_pickle_name(gt) if gp is None else gp
if force or not _newer(gp, gt):
logger.info("Generating grammar tables from %s", gt)
- g = pgen.generate_grammar(gt)
+ g: grammar.Grammar = pgen.generate_grammar(gt)
if save:
logger.info("Writing grammar tables to %s", gp)
try:
if save:
logger.info("Writing grammar tables to %s", gp)
try:
+def _newer(a: Text, b: Text) -> bool :
"""Inquire whether file a was written since file b."""
if not os.path.exists(a):
return False
"""Inquire whether file a was written since file b."""
if not os.path.exists(a):
return False
return os.path.getmtime(a) >= os.path.getmtime(b)
return os.path.getmtime(a) >= os.path.getmtime(b)
-def load_packaged_grammar(package, grammar_source, cache_dir=None):
+def load_packaged_grammar(
+ package: str, grammar_source: Text, cache_dir: Optional[Path] = None
+) -> grammar.Grammar:
"""Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
"""Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
return load_grammar(grammar_source, gp=gp)
pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir)
data = pkgutil.get_data(package, pickled_name)
return load_grammar(grammar_source, gp=gp)
pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir)
data = pkgutil.get_data(package, pickled_name)
+ assert data is not None
g = grammar.Grammar()
g.loads(data)
return g
g = grammar.Grammar()
g.loads(data)
return g
+def main(*args: Text) -> bool :
"""Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
"""
if not args:
"""Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
"""
if not args:
+ args = tuple(sys.argv[1:])
logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s")
for gt in args:
load_grammar(gt, save=True, force=True)
logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s")
for gt in args:
load_grammar(gt, save=True, force=True)
projects / etc / vim.git / blobdiff