X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/e401b6bb1e1c0ed534bba59d9dc908caf7ba898c..a07871b9cd1b5a1469271be6aaac5766f5a0e0fc:/src/black/parsing.py
diff --git a/src/black/parsing.py b/src/black/parsing.py
index 13fa67e..70ed99c 100644
--- a/src/black/parsing.py
+++ b/src/black/parsing.py
@@ -11,16 +11,14 @@ if sys.version_info < (3, 8):
else:
from typing import Final
-# lib2to3 fork
-from blib2to3.pytree import Node, Leaf
+from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature
+from black.nodes import syms
from blib2to3 import pygram
from blib2to3.pgen2 import driver
from blib2to3.pgen2.grammar import Grammar
from blib2to3.pgen2.parse import ParseError
from blib2to3.pgen2.tokenize import TokenError
-
-from black.mode import TargetVersion, Feature, supports_feature
-from black.nodes import syms
+from blib2to3.pytree import Leaf, Node
ast3: Any
@@ -29,8 +27,7 @@ _IS_PYPY = platform.python_implementation() == "PyPy"
try:
from typed_ast import ast3
except ImportError:
- # Either our python version is too low, or we're on pypy
- if sys.version_info < (3, 7) or (sys.version_info < (3, 8) and not _IS_PYPY):
+ if sys.version_info < (3, 8) and not _IS_PYPY:
print(
"The typed_ast package is required but not installed.\n"
"You can upgrade to Python 3.8+ or install typed_ast with\n"
@@ -42,7 +39,6 @@ except ImportError:
ast3 = ast
-PY310_HINT: Final = "Consider using --target-version py310 to parse Python 3.10 code."
PY2_HINT: Final = "Python 2 support was removed in version 22.0."
@@ -54,16 +50,15 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
if not target_versions:
# No target_version specified, so try all grammars.
return [
- # Python 3.7+
+ # Python 3.7-3.9
pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
# Python 3.0-3.6
pygram.python_grammar_no_print_statement_no_exec_statement,
+ # Python 3.10+
+ pygram.python_grammar_soft_keywords,
]
grammars = []
- if supports_feature(target_versions, Feature.PATTERN_MATCHING):
- # Python 3.10+
- grammars.append(pygram.python_grammar_soft_keywords)
# If we have to parse both, try to parse async as a keyword first
if not supports_feature(
target_versions, Feature.ASYNC_IDENTIFIERS
@@ -75,6 +70,10 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
# Python 3.0-3.6
grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
+ if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions):
+ # Python 3.10+
+ grammars.append(pygram.python_grammar_soft_keywords)
+
# At least one of the above branches must have been taken, because every Python
# version has exactly one of the two 'ASYNC_*' flags
return grammars
@@ -86,6 +85,7 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -
src_txt += "\n"
grammars = get_grammars(set(target_versions))
+ errors = {}
for grammar in grammars:
drv = driver.Driver(grammar)
try:
@@ -99,20 +99,21 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -
faulty_line = lines[lineno - 1]
except IndexError:
faulty_line = "<line number missing in source>"
- exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}")
+ errors[grammar.version] = InvalidInput(
+ f"Cannot parse: {lineno}:{column}: {faulty_line}"
+ )
except TokenError as te:
# In edge cases these are raised; and typically don't have a "faulty_line".
lineno, column = te.args[1]
- exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {te.args[0]}")
+ errors[grammar.version] = InvalidInput(
+ f"Cannot parse: {lineno}:{column}: {te.args[0]}"
+ )
else:
- if pygram.python_grammar_soft_keywords not in grammars and matches_grammar(
- src_txt, pygram.python_grammar_soft_keywords
- ):
- original_msg = exc.args[0]
- msg = f"{original_msg}\n{PY310_HINT}"
- raise InvalidInput(msg) from None
+ # Choose the latest version when raising the actual parsing error.
+ assert len(errors) >= 1
+ exc = errors[max(errors)]
if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar(
src_txt, pygram.python_grammar_no_print_statement
@@ -145,18 +146,29 @@ def lib2to3_unparse(node: Node) -> str:
def parse_single_version(
- src: str, version: Tuple[int, int]
+ src: str, version: Tuple[int, int], *, type_comments: bool
) -> Union[ast.AST, ast3.AST]:
filename = "<unknown>"
- # typed_ast is needed because of feature version limitations in the builtin ast
+ # typed-ast is needed because of feature version limitations in the builtin ast 3.8>
if sys.version_info >= (3, 8) and version >= (3,):
- return ast.parse(src, filename, feature_version=version)
- elif version >= (3,):
- if _IS_PYPY:
- return ast3.parse(src, filename)
+ return ast.parse(
+ src, filename, feature_version=version, type_comments=type_comments
+ )
+
+ if _IS_PYPY:
+ # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
+ # not much we can do as typed-ast won't work either.
+ if sys.version_info >= (3, 8):
+ return ast3.parse(src, filename, type_comments=type_comments)
else:
+ return ast3.parse(src, filename)
+ else:
+ if type_comments:
+ # Typed-ast is guaranteed to be used here and automatically tracks type
+ # comments separately.
return ast3.parse(src, filename, feature_version=version[1])
- raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!")
+ else:
+ return ast.parse(src, filename)
def parse_ast(src: str) -> Union[ast.AST, ast3.AST]:
@@ -166,11 +178,18 @@ def parse_ast(src: str) -> Union[ast.AST, ast3.AST]:
first_error = ""
for version in sorted(versions, reverse=True):
try:
- return parse_single_version(src, version)
+ return parse_single_version(src, version, type_comments=True)
except SyntaxError as e:
if not first_error:
first_error = str(e)
+ # Try to parse without type comments
+ for version in sorted(versions, reverse=True):
+ try:
+ return parse_single_version(src, version, type_comments=False)
+ except SyntaxError:
+ pass
+
raise SyntaxError(first_error)
@@ -206,7 +225,7 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st
break
try:
- value = getattr(node, field)
+ value: object = getattr(node, field)
except AttributeError:
continue
@@ -221,8 +240,8 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st
and isinstance(node, (ast.Delete, ast3.Delete))
and isinstance(item, (ast.Tuple, ast3.Tuple))
):
- for item in item.elts:
- yield from stringify_ast(item, depth + 2)
+ for elt in item.elts:
+ yield from stringify_ast(elt, depth + 2)
elif isinstance(item, (ast.AST, ast3.AST)):
yield from stringify_ast(item, depth + 2)
@@ -237,6 +256,7 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st
yield from stringify_ast(value, depth + 2)
else:
+ normalized: object
# Constant strings may be indented across newlines, if they are
# docstrings; fold spaces after newlines when comparing. Similarly,
# trailing and leading space may be removed.