]>
git.madduck.net Git - etc/vim.git/blobdiff - src/blib2to3/pgen2/pgen.py
madduck's git repository
Every one of the projects in this repository is available at the canonical
URL git://git.madduck.net/madduck/pub/<projectpath> — see
each project's metadata for the exact URL.
All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@ git. madduck. net .
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
SSH access, as well as push access can be individually
arranged .
If you use my repositories frequently, consider adding the following
snippet to ~/.gitconfig and using the third clone URL listed for each
project:
[url "git://git.madduck.net/madduck/"]
insteadOf = madduck:
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
-# Pgen imports
-from . import grammar, token, tokenize
-
-from blib2to3.pgen2 import grammar
-from blib2to3.pgen2.tokenize import GoodTokenInfo
-import os
+from blib2to3.pgen2 import grammar, token, tokenize
+from blib2to3.pgen2.tokenize import GoodTokenInfo
Path = Union[str, "os.PathLike[str]"]
Path = Union[str, "os.PathLike[str]"]
-class ParserGenerator(object):
-
generator: Iterator[GoodTokenInfo]
generator: Iterator[GoodTokenInfo]
- first: Dict[Text, Optional[Dict[Text , int]]]
+ first: Dict[str, Optional[Dict[str , int]]]
- def __init__(self, filename: Path, stream: Optional[IO[Text ]] = None) -> None:
+ def __init__(self, filename: Path, stream: Optional[IO[str ]] = None) -> None:
close_stream = None
if stream is None:
close_stream = None
if stream is None:
- stream = open(filename)
+ stream = open(filename, encoding="utf-8" )
close_stream = stream.close
self.filename = filename
self.stream = stream
close_stream = stream.close
self.filename = filename
self.stream = stream
c.start = c.symbol2number[self.startsymbol]
return c
c.start = c.symbol2number[self.startsymbol]
return c
- def make_first(self, c: PgenGrammar, name: Text ) -> Dict[int, int]:
+ def make_first(self, c: PgenGrammar, name: str ) -> Dict[int, int]:
rawfirst = self.first[name]
assert rawfirst is not None
first = {}
rawfirst = self.first[name]
assert rawfirst is not None
first = {}
first[ilabel] = 1
return first
first[ilabel] = 1
return first
- def make_label(self, c: PgenGrammar, label: Text ) -> int:
+ def make_label(self, c: PgenGrammar, label: str ) -> int:
# XXX Maybe this should be a method on a subclass of converter?
ilabel = len(c.labels)
if label[0].isalpha():
# XXX Maybe this should be a method on a subclass of converter?
ilabel = len(c.labels)
if label[0].isalpha():
assert label[0] in ('"', "'"), label
value = eval(label)
if value[0].isalpha():
assert label[0] in ('"', "'"), label
value = eval(label)
if value[0].isalpha():
+ if label[0] == '"':
+ keywords = c.soft_keywords
+ else:
+ keywords = c.keywords
+
- if value in c. keywords:
- return c. keywords[value]
+ if value in keywords:
+ return keywords[value]
else:
c.labels.append((token.NAME, value))
else:
c.labels.append((token.NAME, value))
- c. keywords[value] = ilabel
+ keywords[value] = ilabel
return ilabel
else:
# An operator (any non-numeric token)
return ilabel
else:
# An operator (any non-numeric token)
self.calcfirst(name)
# print name, self.first[name].keys()
self.calcfirst(name)
# print name, self.first[name].keys()
- def calcfirst(self, name: Text ) -> None:
+ def calcfirst(self, name: str ) -> None:
dfa = self.dfas[name]
self.first[name] = None # dummy to detect left recursion
state = dfa[0]
totalset: Dict[str, int] = {}
overlapcheck = {}
dfa = self.dfas[name]
self.first[name] = None # dummy to detect left recursion
state = dfa[0]
totalset: Dict[str, int] = {}
overlapcheck = {}
- for label, next in state.arcs.items() :
+ for label in state.arcs :
if label in self.dfas:
if label in self.first:
fset = self.first[label]
if label in self.dfas:
if label in self.first:
fset = self.first[label]
inverse[symbol] = label
self.first[name] = totalset
inverse[symbol] = label
self.first[name] = totalset
- def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text ]:
+ def parse(self) -> Tuple[Dict[str, List["DFAState"]], str ]:
dfas = {}
startsymbol: Optional[str] = None
# MSTART: (NEWLINE | RULE)* ENDMARKER
dfas = {}
startsymbol: Optional[str] = None
# MSTART: (NEWLINE | RULE)* ENDMARKER
# self.dump_nfa(name, a, z)
dfa = self.make_dfa(a, z)
# self.dump_dfa(name, dfa)
# self.dump_nfa(name, a, z)
dfa = self.make_dfa(a, z)
# self.dump_dfa(name, dfa)
dfas[name] = dfa
# print name, oldlen, newlen
if startsymbol is None:
dfas[name] = dfa
# print name, oldlen, newlen
if startsymbol is None:
state.addarc(st, label)
return states # List of DFAState instances; first one is start
state.addarc(st, label)
return states # List of DFAState instances; first one is start
- def dump_nfa(self, name: Text , start: "NFAState", finish: "NFAState") -> None:
+ def dump_nfa(self, name: str , start: "NFAState", finish: "NFAState") -> None:
print("Dump of NFA for", name)
todo = [start]
for i, state in enumerate(todo):
print("Dump of NFA for", name)
todo = [start]
for i, state in enumerate(todo):
else:
print(" %s -> %d" % (label, j))
else:
print(" %s -> %d" % (label, j))
- def dump_dfa(self, name: Text , dfa: Sequence["DFAState"]) -> None:
+ def dump_dfa(self, name: str , dfa: Sequence["DFAState"]) -> None:
print("Dump of DFA for", name)
for i, state in enumerate(dfa):
print(" State", i, state.isfinal and "(final)" or "")
print("Dump of DFA for", name)
for i, state in enumerate(dfa):
print(" State", i, state.isfinal and "(final)" or "")
self.raise_error(
"expected (...) or NAME or STRING, got %s/%s", self.type, self.value
)
self.raise_error(
"expected (...) or NAME or STRING, got %s/%s", self.type, self.value
)
- def expect(self, type: int, value: Optional[Any] = None) -> Text :
+ def expect(self, type: int, value: Optional[Any] = None) -> str :
if self.type != type or (value is not None and self.value != value):
self.raise_error(
"expected %s/%s, got %s/%s", type, value, self.type, self.value
if self.type != type or (value is not None and self.value != value):
self.raise_error(
"expected %s/%s, got %s/%s", type, value, self.type, self.value
if args:
try:
msg = msg % args
if args:
try:
msg = msg % args
msg = " ".join([msg] + list(map(str, args)))
raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line))
msg = " ".join([msg] + list(map(str, args)))
raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line))
-class NFAState(object) :
- arcs: List[Tuple[Optional[Text ], "NFAState"]]
+class NFAState:
+ arcs: List[Tuple[Optional[str ], "NFAState"]]
def __init__(self) -> None:
self.arcs = [] # list of (label, NFAState) pairs
def __init__(self) -> None:
self.arcs = [] # list of (label, NFAState) pairs
- def addarc(self, next: "NFAState", label: Optional[Text ] = None) -> None:
+ def addarc(self, next: "NFAState", label: Optional[str ] = None) -> None:
assert label is None or isinstance(label, str)
assert isinstance(next, NFAState)
self.arcs.append((label, next))
assert label is None or isinstance(label, str)
assert isinstance(next, NFAState)
self.arcs.append((label, next))
nfaset: Dict[NFAState, Any]
isfinal: bool
nfaset: Dict[NFAState, Any]
isfinal: bool
- arcs: Dict[Text , "DFAState"]
+ arcs: Dict[str , "DFAState"]
def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None:
assert isinstance(nfaset, dict)
def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None:
assert isinstance(nfaset, dict)
self.isfinal = final in nfaset
self.arcs = {} # map from label to DFAState
self.isfinal = final in nfaset
self.arcs = {} # map from label to DFAState
- def addarc(self, next: "DFAState", label: Text ) -> None:
+ def addarc(self, next: "DFAState", label: str ) -> None:
assert isinstance(label, str)
assert label not in self.arcs
assert isinstance(next, DFAState)
assert isinstance(label, str)
assert label not in self.arcs
assert isinstance(next, DFAState)
projects / etc / vim.git / blobdiff