]>
git.madduck.net Git - etc/vim.git/blobdiff - src/blib2to3/pgen2/tokenize.py
madduck's git repository
Every one of the projects in this repository is available at the canonical
URL git://git.madduck.net/madduck/pub/<projectpath> — see
each project's metadata for the exact URL.
All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@ git. madduck. net .
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
SSH access, as well as push access can be individually
arranged .
If you use my repositories frequently, consider adding the following
snippet to ~/.gitconfig and using the third clone URL listed for each
project:
[url "git://git.madduck.net/madduck/"]
insteadOf = madduck:
Iterator,
List,
Optional,
Iterator,
List,
Optional,
__author__ = "Ka-Ping Yee <ping@lfw.org>"
__credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro"
__author__ = "Ka-Ping Yee <ping@lfw.org>"
__credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro"
from codecs import BOM_UTF8, lookup
from blib2to3.pgen2.token import *
from codecs import BOM_UTF8, lookup
from blib2to3.pgen2.token import *
+def group(*choices: str) -> str :
return "(" + "|".join(choices) + ")"
return "(" + "|".join(choices) + ")"
+def any(*choices: str) -> str :
return group(*choices) + "*"
return group(*choices) + "*"
+def maybe(*choices: str) -> str :
return group(*choices) + "?"
return group(*choices) + "?"
+def _combinations(*l: str) -> Set[str] :
return set(x + y for x in l for y in l + ("",) if x.casefold() != y.casefold())
return set(x + y for x in l for y in l + ("",) if x.casefold() != y.casefold())
Comment = r"#[^\r\n]*"
Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment)
Name = ( # this is invalid but it's fine because Name comes after Number in all groups
Comment = r"#[^\r\n]*"
Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment)
Name = ( # this is invalid but it's fine because Name comes after Number in all groups
+ r"[^\s#\(\)\[\]\{\}+\-*/!@$%^&=|;:'\",\.<>/?`~\\] +"
)
Binnumber = r"0[bB]_?[01]+(?:_[01]+)*"
)
Binnumber = r"0[bB]_?[01]+(?:_[01]+)*"
'"""': double3prog,
**{f"{prefix}'''": single3prog for prefix in _strprefixes},
**{f'{prefix}"""': double3prog for prefix in _strprefixes},
'"""': double3prog,
**{f"{prefix}'''": single3prog for prefix in _strprefixes},
**{f'{prefix}"""': double3prog for prefix in _strprefixes},
- **{prefix: None for prefix in _strprefixes},
}
triple_quoted: Final = (
}
triple_quoted: Final = (
-def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing
- (srow, scol) = xxx_todo_changeme
- (erow, ecol) = xxx_todo_changeme1
+Coord = Tuple[int, int]
+
+
+def printtoken(
+ type: int, token: Text, srow_col: Coord, erow_col: Coord, line: Text
+) -> None: # for testing
+ (srow, scol) = srow_col
+ (erow, ecol) = erow_col
print(
"%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token))
)
print(
"%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token))
)
TokenEater = Callable[[int, Text, Coord, Coord, Text], None]
TokenEater = Callable[[int, Text, Coord, Coord, Text], None]
# backwards compatible interface
# backwards compatible interface
-def tokenize_loop(readline, tokeneater) :
+def tokenize_loop(readline: Callable[[], Text], tokeneater: TokenEater) -> None :
for token_info in generate_tokens(readline):
tokeneater(*token_info)
for token_info in generate_tokens(readline):
tokeneater(*token_info)
tokens: List[Text]
prev_row: int
prev_col: int
tokens: List[Text]
prev_row: int
prev_col: int
cookie_re = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.ASCII)
cookie_re = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.ASCII)
-blank_re = re.compile(br "^[ \t\f]*(?:[#\r\n]|$)", re.ASCII)
+blank_re = re.compile(rb "^[ \t\f]*(?:[#\r\n]|$)", re.ASCII)
def _get_normal_name(orig_enc: str) -> str:
def _get_normal_name(orig_enc: str) -> str:
logical line; continuation lines are included.
"""
lnum = parenlev = continued = 0
logical line; continuation lines are included.
"""
lnum = parenlev = continued = 0
- numchars: Final = "0123456789"
+ numchars: Final[str] = "0123456789"
contstr, needcont = "", 0
contline: Optional[str] = None
indents = [0]
contstr, needcont = "", 0
contline: Optional[str] = None
indents = [0]
):
if token[-1] == "\n": # continued string
strstart = (lnum, start)
):
if token[-1] == "\n": # continued string
strstart = (lnum, start)
- endprog = (
- endprogs[initial]
- or endprogs[token[1]]
- or endprogs[token[2]]
+ maybe_ endprog = (
+ endprogs.get(initial)
+ or endprogs.get(token[1])
+ or endprogs.get(token[2])
+ assert (
+ maybe_endprog is not None
+ ), f"endprog not found for {token}"
+ endprog = maybe_endprog
contstr, needcont = line[start:], 1
contline = line
break
contstr, needcont = line[start:], 1
contline = line
break
if token in ("def", "for"):
if stashed and stashed[0] == NAME and stashed[1] == "async":
if token in ("def", "for"):
if stashed and stashed[0] == NAME and stashed[1] == "async":
if token == "def":
async_def = True
async_def_indent = indents[-1]
if token == "def":
async_def = True
async_def_indent = indents[-1]
projects / etc / vim.git / blobdiff