X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/47480ca33143b250cb33ba501f201838269173ce..43b53a034c9e3ada9295bf7f93be473b66b74f75:/.vim/bundle/black/src/blib2to3/pgen2/driver.py diff --git a/.vim/bundle/black/src/blib2to3/pgen2/driver.py b/.vim/bundle/black/src/blib2to3/pgen2/driver.py deleted file mode 100644 index e629843f..00000000 --- a/.vim/bundle/black/src/blib2to3/pgen2/driver.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -# Modifications: -# Copyright 2006 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Parser driver. - -This provides a high-level interface to parse a file into a syntax tree. - -""" - -__author__ = "Guido van Rossum " - -__all__ = ["Driver", "load_grammar"] - -# Python imports -import io -import logging -import os -import pkgutil -import sys -from contextlib import contextmanager -from dataclasses import dataclass, field -from logging import Logger -from typing import IO, Any, Iterable, Iterator, List, Optional, Tuple, Union, cast - -from blib2to3.pgen2.grammar import Grammar -from blib2to3.pgen2.tokenize import GoodTokenInfo -from blib2to3.pytree import NL - -# Pgen imports -from . import grammar, parse, pgen, token, tokenize - -Path = Union[str, "os.PathLike[str]"] - - -@dataclass -class ReleaseRange: - start: int - end: Optional[int] = None - tokens: List[Any] = field(default_factory=list) - - def lock(self) -> None: - total_eaten = len(self.tokens) - self.end = self.start + total_eaten - - -class TokenProxy: - def __init__(self, generator: Any) -> None: - self._tokens = generator - self._counter = 0 - self._release_ranges: List[ReleaseRange] = [] - - @contextmanager - def release(self) -> Iterator["TokenProxy"]: - release_range = ReleaseRange(self._counter) - self._release_ranges.append(release_range) - try: - yield self - finally: - # Lock the last release range to the final position that - # has been eaten. - release_range.lock() - - def eat(self, point: int) -> Any: - eaten_tokens = self._release_ranges[-1].tokens - if point < len(eaten_tokens): - return eaten_tokens[point] - else: - while point >= len(eaten_tokens): - token = next(self._tokens) - eaten_tokens.append(token) - return token - - def __iter__(self) -> "TokenProxy": - return self - - def __next__(self) -> Any: - # If the current position is already compromised (looked up) - # return the eaten token, if not just go further on the given - # token producer. - for release_range in self._release_ranges: - assert release_range.end is not None - - start, end = release_range.start, release_range.end - if start <= self._counter < end: - token = release_range.tokens[self._counter - start] - break - else: - token = next(self._tokens) - self._counter += 1 - return token - - def can_advance(self, to: int) -> bool: - # Try to eat, fail if it can't. The eat operation is cached - # so there won't be any additional cost of eating here - try: - self.eat(to) - except StopIteration: - return False - else: - return True - - -class Driver: - def __init__(self, grammar: Grammar, logger: Optional[Logger] = None) -> None: - self.grammar = grammar - if logger is None: - logger = logging.getLogger(__name__) - self.logger = logger - - def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> NL: - """Parse a series of tokens and return the syntax tree.""" - # XXX Move the prefix computation into a wrapper around tokenize. - proxy = TokenProxy(tokens) - - p = parse.Parser(self.grammar) - p.setup(proxy=proxy) - - lineno = 1 - column = 0 - indent_columns: List[int] = [] - type = value = start = end = line_text = None - prefix = "" - - for quintuple in proxy: - type, value, start, end, line_text = quintuple - if start != (lineno, column): - assert (lineno, column) <= start, ((lineno, column), start) - s_lineno, s_column = start - if lineno < s_lineno: - prefix += "\n" * (s_lineno - lineno) - lineno = s_lineno - column = 0 - if column < s_column: - prefix += line_text[column:s_column] - column = s_column - if type in (tokenize.COMMENT, tokenize.NL): - prefix += value - lineno, column = end - if value.endswith("\n"): - lineno += 1 - column = 0 - continue - if type == token.OP: - type = grammar.opmap[value] - if debug: - assert type is not None - self.logger.debug( - "%s %r (prefix=%r)", token.tok_name[type], value, prefix - ) - if type == token.INDENT: - indent_columns.append(len(value)) - _prefix = prefix + value - prefix = "" - value = "" - elif type == token.DEDENT: - _indent_col = indent_columns.pop() - prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col) - if p.addtoken(cast(int, type), value, (prefix, start)): - if debug: - self.logger.debug("Stop.") - break - prefix = "" - if type in {token.INDENT, token.DEDENT}: - prefix = _prefix - lineno, column = end - if value.endswith("\n"): - lineno += 1 - column = 0 - else: - # We never broke out -- EOF is too soon (how can this happen???) - assert start is not None - raise parse.ParseError("incomplete input", type, value, (prefix, start)) - assert p.rootnode is not None - return p.rootnode - - def parse_stream_raw(self, stream: IO[str], debug: bool = False) -> NL: - """Parse a stream and return the syntax tree.""" - tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar) - return self.parse_tokens(tokens, debug) - - def parse_stream(self, stream: IO[str], debug: bool = False) -> NL: - """Parse a stream and return the syntax tree.""" - return self.parse_stream_raw(stream, debug) - - def parse_file( - self, filename: Path, encoding: Optional[str] = None, debug: bool = False - ) -> NL: - """Parse a file and return the syntax tree.""" - with open(filename, encoding=encoding) as stream: - return self.parse_stream(stream, debug) - - def parse_string(self, text: str, debug: bool = False) -> NL: - """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens( - io.StringIO(text).readline, grammar=self.grammar - ) - return self.parse_tokens(tokens, debug) - - def _partially_consume_prefix(self, prefix: str, column: int) -> Tuple[str, str]: - lines: List[str] = [] - current_line = "" - current_column = 0 - wait_for_nl = False - for char in prefix: - current_line += char - if wait_for_nl: - if char == "\n": - if current_line.strip() and current_column < column: - res = "".join(lines) - return res, prefix[len(res) :] - - lines.append(current_line) - current_line = "" - current_column = 0 - wait_for_nl = False - elif char in " \t": - current_column += 1 - elif char == "\n": - # unexpected empty line - current_column = 0 - else: - # indent is finished - wait_for_nl = True - return "".join(lines), current_line - - -def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> str: - head, tail = os.path.splitext(gt) - if tail == ".txt": - tail = "" - name = head + tail + ".".join(map(str, sys.version_info)) + ".pickle" - if cache_dir: - return os.path.join(cache_dir, os.path.basename(name)) - else: - return name - - -def load_grammar( - gt: str = "Grammar.txt", - gp: Optional[str] = None, - save: bool = True, - force: bool = False, - logger: Optional[Logger] = None, -) -> Grammar: - """Load the grammar (maybe from a pickle).""" - if logger is None: - logger = logging.getLogger(__name__) - gp = _generate_pickle_name(gt) if gp is None else gp - if force or not _newer(gp, gt): - g: grammar.Grammar = pgen.generate_grammar(gt) - if save: - try: - g.dump(gp) - except OSError: - # Ignore error, caching is not vital. - pass - else: - g = grammar.Grammar() - g.load(gp) - return g - - -def _newer(a: str, b: str) -> bool: - """Inquire whether file a was written since file b.""" - if not os.path.exists(a): - return False - if not os.path.exists(b): - return True - return os.path.getmtime(a) >= os.path.getmtime(b) - - -def load_packaged_grammar( - package: str, grammar_source: str, cache_dir: Optional[Path] = None -) -> grammar.Grammar: - """Normally, loads a pickled grammar by doing - pkgutil.get_data(package, pickled_grammar) - where *pickled_grammar* is computed from *grammar_source* by adding the - Python version and using a ``.pickle`` extension. - - However, if *grammar_source* is an extant file, load_grammar(grammar_source) - is called instead. This facilitates using a packaged grammar file when needed - but preserves load_grammar's automatic regeneration behavior when possible. - - """ - if os.path.isfile(grammar_source): - gp = _generate_pickle_name(grammar_source, cache_dir) if cache_dir else None - return load_grammar(grammar_source, gp=gp) - pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir) - data = pkgutil.get_data(package, pickled_name) - assert data is not None - g = grammar.Grammar() - g.loads(data) - return g - - -def main(*args: str) -> bool: - """Main program, when run as a script: produce grammar pickle files. - - Calls load_grammar for each argument, a path to a grammar text file. - """ - if not args: - args = tuple(sys.argv[1:]) - logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s") - for gt in args: - load_grammar(gt, save=True, force=True) - return True - - -if __name__ == "__main__": - sys.exit(int(not main()))