+++ /dev/null
-# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
-# Licensed to PSF under a Contributor Agreement.
-
-"""This module defines the data structures used to represent a grammar.
-
-These are a bit arcane because they are derived from the data
-structures used by Python's 'pgen' parser generator.
-
-There's also a table here mapping operators to their names in the
-token module; the Python tokenize module reports all operators as the
-fallback token code OP, but the parser needs the actual token code.
-
-"""
-
-# Python imports
-import os
-import pickle
-import tempfile
-from typing import Any, Dict, List, Optional, Text, Tuple, TypeVar, Union
-
-# Local imports
-from . import token
-
-_P = TypeVar("_P", bound="Grammar")
-Label = Tuple[int, Optional[Text]]
-DFA = List[List[Tuple[int, int]]]
-DFAS = Tuple[DFA, Dict[int, int]]
-Path = Union[str, "os.PathLike[str]"]
-
-
-class Grammar(object):
- """Pgen parsing tables conversion class.
-
- Once initialized, this class supplies the grammar tables for the
- parsing engine implemented by parse.py. The parsing engine
- accesses the instance variables directly. The class here does not
- provide initialization of the tables; several subclasses exist to
- do this (see the conv and pgen modules).
-
- The load() method reads the tables from a pickle file, which is
- much faster than the other ways offered by subclasses. The pickle
- file is written by calling dump() (after loading the grammar
- tables using a subclass). The report() method prints a readable
- representation of the tables to stdout, for debugging.
-
- The instance variables are as follows:
-
- symbol2number -- a dict mapping symbol names to numbers. Symbol
- numbers are always 256 or higher, to distinguish
- them from token numbers, which are between 0 and
- 255 (inclusive).
-
- number2symbol -- a dict mapping numbers to symbol names;
- these two are each other's inverse.
-
- states -- a list of DFAs, where each DFA is a list of
- states, each state is a list of arcs, and each
- arc is a (i, j) pair where i is a label and j is
- a state number. The DFA number is the index into
- this list. (This name is slightly confusing.)
- Final states are represented by a special arc of
- the form (0, j) where j is its own state number.
-
- dfas -- a dict mapping symbol numbers to (DFA, first)
- pairs, where DFA is an item from the states list
- above, and first is a set of tokens that can
- begin this grammar rule (represented by a dict
- whose values are always 1).
-
- labels -- a list of (x, y) pairs where x is either a token
- number or a symbol number, and y is either None
- or a string; the strings are keywords. The label
- number is the index in this list; label numbers
- are used to mark state transitions (arcs) in the
- DFAs.
-
- start -- the number of the grammar's start symbol.
-
- keywords -- a dict mapping keyword strings to arc labels.
-
- tokens -- a dict mapping token numbers to arc labels.
-
- """
-
- def __init__(self) -> None:
- self.symbol2number: Dict[str, int] = {}
- self.number2symbol: Dict[int, str] = {}
- self.states: List[DFA] = []
- self.dfas: Dict[int, DFAS] = {}
- self.labels: List[Label] = [(0, "EMPTY")]
- self.keywords: Dict[str, int] = {}
- self.tokens: Dict[int, int] = {}
- self.symbol2label: Dict[str, int] = {}
- self.start = 256
- # Python 3.7+ parses async as a keyword, not an identifier
- self.async_keywords = False
-
- def dump(self, filename: Path) -> None:
- """Dump the grammar tables to a pickle file."""
-
- # mypyc generates objects that don't have a __dict__, but they
- # do have __getstate__ methods that will return an equivalent
- # dictionary
- if hasattr(self, "__dict__"):
- d = self.__dict__
- else:
- d = self.__getstate__() # type: ignore
-
- with tempfile.NamedTemporaryFile(
- dir=os.path.dirname(filename), delete=False
- ) as f:
- pickle.dump(d, f, pickle.HIGHEST_PROTOCOL)
- os.replace(f.name, filename)
-
- def _update(self, attrs: Dict[str, Any]) -> None:
- for k, v in attrs.items():
- setattr(self, k, v)
-
- def load(self, filename: Path) -> None:
- """Load the grammar tables from a pickle file."""
- with open(filename, "rb") as f:
- d = pickle.load(f)
- self._update(d)
-
- def loads(self, pkl: bytes) -> None:
- """Load the grammar tables from a pickle bytes object."""
- self._update(pickle.loads(pkl))
-
- def copy(self: _P) -> _P:
- """
- Copy the grammar.
- """
- new = self.__class__()
- for dict_attr in (
- "symbol2number",
- "number2symbol",
- "dfas",
- "keywords",
- "tokens",
- "symbol2label",
- ):
- setattr(new, dict_attr, getattr(self, dict_attr).copy())
- new.labels = self.labels[:]
- new.states = self.states[:]
- new.start = self.start
- new.async_keywords = self.async_keywords
- return new
-
- def report(self) -> None:
- """Dump the grammar tables to standard output, for debugging."""
- from pprint import pprint
-
- print("s2n")
- pprint(self.symbol2number)
- print("n2s")
- pprint(self.number2symbol)
- print("states")
- pprint(self.states)
- print("dfas")
- pprint(self.dfas)
- print("labels")
- pprint(self.labels)
- print("start", self.start)
-
-
-# Map from operator to number (since tokenize doesn't do this)
-
-opmap_raw = """
-( LPAR
-) RPAR
-[ LSQB
-] RSQB
-: COLON
-, COMMA
-; SEMI
-+ PLUS
-- MINUS
-* STAR
-/ SLASH
-| VBAR
-& AMPER
-< LESS
-> GREATER
-= EQUAL
-. DOT
-% PERCENT
-` BACKQUOTE
-{ LBRACE
-} RBRACE
-@ AT
-@= ATEQUAL
-== EQEQUAL
-!= NOTEQUAL
-<> NOTEQUAL
-<= LESSEQUAL
->= GREATEREQUAL
-~ TILDE
-^ CIRCUMFLEX
-<< LEFTSHIFT
->> RIGHTSHIFT
-** DOUBLESTAR
-+= PLUSEQUAL
--= MINEQUAL
-*= STAREQUAL
-/= SLASHEQUAL
-%= PERCENTEQUAL
-&= AMPEREQUAL
-|= VBAREQUAL
-^= CIRCUMFLEXEQUAL
-<<= LEFTSHIFTEQUAL
->>= RIGHTSHIFTEQUAL
-**= DOUBLESTAREQUAL
-// DOUBLESLASH
-//= DOUBLESLASHEQUAL
--> RARROW
-:= COLONEQUAL
-"""
-
-opmap = {}
-for line in opmap_raw.splitlines():
- if line:
- op, name = line.split()
- opmap[op] = getattr(token, name)
projects / etc / vim.git / blobdiff
