]>
git.madduck.net Git - etc/vim.git/blobdiff - src/blib2to3/pgen2/tokenize.py
madduck's git repository
Every one of the projects in this repository is available at the canonical
URL git://git.madduck.net/madduck/pub/<projectpath> — see
each project's metadata for the exact URL.
All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@ git. madduck. net .
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
SSH access, as well as push access can be individually
arranged .
If you use my repositories frequently, consider adding the following
snippet to ~/.gitconfig and using the third clone URL listed for each
project:
[url "git://git.madduck.net/madduck/"]
insteadOf = madduck:
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found."""
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found."""
from typing import (
Callable,
Iterable,
from typing import (
Callable,
Iterable,
+
+if sys.version_info >= (3, 8):
+ from typing import Final
+else:
+ from typing_extensions import Final
+
from blib2to3.pgen2.token import *
from blib2to3.pgen2.grammar import Grammar
__author__ = "Ka-Ping Yee <ping@lfw.org>"
__credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro"
from blib2to3.pgen2.token import *
from blib2to3.pgen2.grammar import Grammar
__author__ = "Ka-Ping Yee <ping@lfw.org>"
__credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro"
from codecs import BOM_UTF8, lookup
from blib2to3.pgen2.token import *
from codecs import BOM_UTF8, lookup
from blib2to3.pgen2.token import *
Comment = r"#[^\r\n]*"
Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment)
Name = ( # this is invalid but it's fine because Name comes after Number in all groups
Comment = r"#[^\r\n]*"
Ignore = Whitespace + any(r"\\\r?\n" + Whitespace) + maybe(Comment)
Name = ( # this is invalid but it's fine because Name comes after Number in all groups
+ r"[^\s#\(\)\[\]\{\}+\-*/!@$%^&=|;:'\",\.<>/?`~\\] +"
)
Binnumber = r"0[bB]_?[01]+(?:_[01]+)*"
)
Binnumber = r"0[bB]_?[01]+(?:_[01]+)*"
PseudoExtras = group(r"\\\r?\n", Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
PseudoExtras = group(r"\\\r?\n", Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
-pseudoprog = re.compile(PseudoToken, re.UNICODE)
+pseudoprog: Final = re.compile(PseudoToken, re.UNICODE)
single3prog = re.compile(Single3)
double3prog = re.compile(Double3)
single3prog = re.compile(Single3)
double3prog = re.compile(Double3)
| {"u", "U", "ur", "uR", "Ur", "UR"}
)
| {"u", "U", "ur", "uR", "Ur", "UR"}
)
"'": re.compile(Single),
'"': re.compile(Double),
"'''": single3prog,
"'": re.compile(Single),
'"': re.compile(Double),
"'''": single3prog,
**{prefix: None for prefix in _strprefixes},
}
**{prefix: None for prefix in _strprefixes},
}
+triple_quoted: Final = (
{"'''", '"""'}
| {f"{prefix}'''" for prefix in _strprefixes}
| {f'{prefix}"""' for prefix in _strprefixes}
)
{"'''", '"""'}
| {f"{prefix}'''" for prefix in _strprefixes}
| {f'{prefix}"""' for prefix in _strprefixes}
)
+single_quoted: Final = (
{"'", '"'}
| {f"{prefix}'" for prefix in _strprefixes}
| {f'{prefix}"' for prefix in _strprefixes}
{"'", '"'}
| {f"{prefix}'" for prefix in _strprefixes}
| {f'{prefix}"' for prefix in _strprefixes}
cookie_re = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.ASCII)
cookie_re = re.compile(r"^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)", re.ASCII)
-blank_re = re.compile(br "^[ \t\f]*(?:[#\r\n]|$)", re.ASCII)
+blank_re = re.compile(rb "^[ \t\f]*(?:[#\r\n]|$)", re.ASCII)
def _get_normal_name(orig_enc: str) -> str:
def _get_normal_name(orig_enc: str) -> str:
logical line; continuation lines are included.
"""
lnum = parenlev = continued = 0
logical line; continuation lines are included.
"""
lnum = parenlev = continued = 0
- numchars = "0123456789"
+ numchars: Final = "0123456789"
contstr, needcont = "", 0
contline: Optional[str] = None
indents = [0]
contstr, needcont = "", 0
contline: Optional[str] = None
indents = [0]
# `await` as keywords.
async_keywords = False if grammar is None else grammar.async_keywords
# 'stashed' and 'async_*' are used for async/await parsing
# `await` as keywords.
async_keywords = False if grammar is None else grammar.async_keywords
# 'stashed' and 'async_*' are used for async/await parsing
+ stashed: Optional[GoodTokenInfo] = None
async_def = False
async_def_indent = 0
async_def_nl = False
async_def = False
async_def_indent = 0
async_def_nl = False
line = readline()
except StopIteration:
line = ""
line = readline()
except StopIteration:
line = ""
pos, max = 0, len(line)
if contstr: # continued string
pos, max = 0, len(line)
if contstr: # continued string
column = 0
while pos < max: # measure leading whitespace
if line[pos] == " ":
column = 0
while pos < max: # measure leading whitespace
if line[pos] == " ":
elif line[pos] == "\t":
column = (column // tabsize + 1) * tabsize
elif line[pos] == "\f":
column = 0
else:
break
elif line[pos] == "\t":
column = (column // tabsize + 1) * tabsize
elif line[pos] == "\f":
column = 0
else:
break
COMMENT,
comment_token,
(lnum, pos),
COMMENT,
comment_token,
(lnum, pos),
- (lnum, pos + len(comment_token) ),
line,
)
yield (NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line)
line,
)
yield (NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line)
continued = 1
else:
if initial in "([{":
continued = 1
else:
if initial in "([{":
- parenlev = parenlev + 1
- parenlev = parenlev - 1
if stashed:
yield stashed
stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line)
if stashed:
yield stashed
stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line)
if stashed:
yield stashed
if stashed:
yield stashed
projects / etc / vim.git / blobdiff