madduck's git repository
Every one of the projects in this repository is available at the canonical
URL git://git.madduck.net/madduck/pub/<projectpath> — see
each project's metadata for the exact URL.
All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
SSH access, as well as push access can be individually
arranged.
If you use my repositories frequently, consider adding the following
snippet to ~/.gitconfig and using the third clone URL listed for each
project:
[url "git://git.madduck.net/madduck/"]
insteadOf = madduck:
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
-# Pgen imports
-from . import grammar, token, tokenize
-
-from blib2to3.pgen2 import grammar
-from blib2to3.pgen2.tokenize import GoodTokenInfo
-import os
+from blib2to3.pgen2 import grammar, token, tokenize
+from blib2to3.pgen2.tokenize import GoodTokenInfo
Path = Union[str, "os.PathLike[str]"]
Path = Union[str, "os.PathLike[str]"]
-class ParserGenerator(object):
-
generator: Iterator[GoodTokenInfo]
generator: Iterator[GoodTokenInfo]
- first: Dict[Text, Optional[Dict[Text, int]]]
+ first: Dict[str, Optional[Dict[str, int]]]
- def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None:
+ def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None:
close_stream = None
if stream is None:
close_stream = None
if stream is None:
- stream = open(filename)
+ stream = open(filename, encoding="utf-8")
close_stream = stream.close
self.filename = filename
self.stream = stream
close_stream = stream.close
self.filename = filename
self.stream = stream
c.start = c.symbol2number[self.startsymbol]
return c
c.start = c.symbol2number[self.startsymbol]
return c
- def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]:
+ def make_first(self, c: PgenGrammar, name: str) -> Dict[int, int]:
rawfirst = self.first[name]
assert rawfirst is not None
first = {}
rawfirst = self.first[name]
assert rawfirst is not None
first = {}
first[ilabel] = 1
return first
first[ilabel] = 1
return first
- def make_label(self, c: PgenGrammar, label: Text) -> int:
+ def make_label(self, c: PgenGrammar, label: str) -> int:
# XXX Maybe this should be a method on a subclass of converter?
ilabel = len(c.labels)
if label[0].isalpha():
# XXX Maybe this should be a method on a subclass of converter?
ilabel = len(c.labels)
if label[0].isalpha():
assert label[0] in ('"', "'"), label
value = eval(label)
if value[0].isalpha():
assert label[0] in ('"', "'"), label
value = eval(label)
if value[0].isalpha():
+ if label[0] == '"':
+ keywords = c.soft_keywords
+ else:
+ keywords = c.keywords
+
- if value in c.keywords:
- return c.keywords[value]
+ if value in keywords:
+ return keywords[value]
else:
c.labels.append((token.NAME, value))
else:
c.labels.append((token.NAME, value))
- c.keywords[value] = ilabel
+ keywords[value] = ilabel
return ilabel
else:
# An operator (any non-numeric token)
return ilabel
else:
# An operator (any non-numeric token)
self.calcfirst(name)
# print name, self.first[name].keys()
self.calcfirst(name)
# print name, self.first[name].keys()
- def calcfirst(self, name: Text) -> None:
+ def calcfirst(self, name: str) -> None:
dfa = self.dfas[name]
self.first[name] = None # dummy to detect left recursion
state = dfa[0]
totalset: Dict[str, int] = {}
overlapcheck = {}
dfa = self.dfas[name]
self.first[name] = None # dummy to detect left recursion
state = dfa[0]
totalset: Dict[str, int] = {}
overlapcheck = {}
- for label, next in state.arcs.items():
+ for label in state.arcs:
if label in self.dfas:
if label in self.first:
fset = self.first[label]
if label in self.dfas:
if label in self.first:
fset = self.first[label]
if symbol in inverse:
raise ValueError(
"rule %s is ambiguous; %s is in the first sets of %s as well"
if symbol in inverse:
raise ValueError(
"rule %s is ambiguous; %s is in the first sets of %s as well"
- " as %s"
- % (name, symbol, label, inverse[symbol])
+ " as %s" % (name, symbol, label, inverse[symbol])
)
inverse[symbol] = label
self.first[name] = totalset
)
inverse[symbol] = label
self.first[name] = totalset
- def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text]:
+ def parse(self) -> Tuple[Dict[str, List["DFAState"]], str]:
dfas = {}
startsymbol: Optional[str] = None
# MSTART: (NEWLINE | RULE)* ENDMARKER
dfas = {}
startsymbol: Optional[str] = None
# MSTART: (NEWLINE | RULE)* ENDMARKER
# self.dump_nfa(name, a, z)
dfa = self.make_dfa(a, z)
# self.dump_dfa(name, dfa)
# self.dump_nfa(name, a, z)
dfa = self.make_dfa(a, z)
# self.dump_dfa(name, dfa)
dfas[name] = dfa
# print name, oldlen, newlen
if startsymbol is None:
dfas[name] = dfa
# print name, oldlen, newlen
if startsymbol is None:
state.addarc(st, label)
return states # List of DFAState instances; first one is start
state.addarc(st, label)
return states # List of DFAState instances; first one is start
- def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None:
+ def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None:
print("Dump of NFA for", name)
todo = [start]
for i, state in enumerate(todo):
print("Dump of NFA for", name)
todo = [start]
for i, state in enumerate(todo):
else:
print(" %s -> %d" % (label, j))
else:
print(" %s -> %d" % (label, j))
- def dump_dfa(self, name: Text, dfa: Sequence["DFAState"]) -> None:
+ def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None:
print("Dump of DFA for", name)
for i, state in enumerate(dfa):
print(" State", i, state.isfinal and "(final)" or "")
print("Dump of DFA for", name)
for i, state in enumerate(dfa):
print(" State", i, state.isfinal and "(final)" or "")
self.raise_error(
"expected (...) or NAME or STRING, got %s/%s", self.type, self.value
)
self.raise_error(
"expected (...) or NAME or STRING, got %s/%s", self.type, self.value
)
- def expect(self, type: int, value: Optional[Any] = None) -> Text:
+ def expect(self, type: int, value: Optional[Any] = None) -> str:
if self.type != type or (value is not None and self.value != value):
self.raise_error(
"expected %s/%s, got %s/%s", type, value, self.type, self.value
if self.type != type or (value is not None and self.value != value):
self.raise_error(
"expected %s/%s, got %s/%s", type, value, self.type, self.value
if args:
try:
msg = msg % args
if args:
try:
msg = msg % args
msg = " ".join([msg] + list(map(str, args)))
raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line))
msg = " ".join([msg] + list(map(str, args)))
raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line))
-class NFAState(object):
- arcs: List[Tuple[Optional[Text], "NFAState"]]
+class NFAState:
+ arcs: List[Tuple[Optional[str], "NFAState"]]
def __init__(self) -> None:
self.arcs = [] # list of (label, NFAState) pairs
def __init__(self) -> None:
self.arcs = [] # list of (label, NFAState) pairs
- def addarc(self, next: "NFAState", label: Optional[Text] = None) -> None:
+ def addarc(self, next: "NFAState", label: Optional[str] = None) -> None:
assert label is None or isinstance(label, str)
assert isinstance(next, NFAState)
self.arcs.append((label, next))
assert label is None or isinstance(label, str)
assert isinstance(next, NFAState)
self.arcs.append((label, next))
nfaset: Dict[NFAState, Any]
isfinal: bool
nfaset: Dict[NFAState, Any]
isfinal: bool
- arcs: Dict[Text, "DFAState"]
+ arcs: Dict[str, "DFAState"]
def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None:
assert isinstance(nfaset, dict)
def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None:
assert isinstance(nfaset, dict)
self.isfinal = final in nfaset
self.arcs = {} # map from label to DFAState
self.isfinal = final in nfaset
self.arcs = {} # map from label to DFAState
- def addarc(self, next: "DFAState", label: Text) -> None:
+ def addarc(self, next: "DFAState", label: str) -> None:
assert isinstance(label, str)
assert label not in self.arcs
assert isinstance(next, DFAState)
assert isinstance(label, str)
assert label not in self.arcs
assert isinstance(next, DFAState)
projects / etc / vim.git / blobdiff