Parse Python code and perform AST validation.
"""
import ast
+import platform
import sys
-from typing import Iterable, Iterator, List, Set, Union
+from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union
+
+if sys.version_info < (3, 8):
+ from typing_extensions import Final
+else:
+ from typing import Final
# lib2to3 fork
from blib2to3.pytree import Node, Leaf
-from blib2to3 import pygram, pytree
+from blib2to3 import pygram
from blib2to3.pgen2 import driver
from blib2to3.pgen2.grammar import Grammar
from blib2to3.pgen2.parse import ParseError
from black.mode import TargetVersion, Feature, supports_feature
from black.nodes import syms
+ast3: Any
+ast27: Any
+
+_IS_PYPY = platform.python_implementation() == "PyPy"
+
try:
from typed_ast import ast3, ast27
except ImportError:
- if sys.version_info < (3, 8):
+ # Either our python version is too low, or we're on pypy
+ if sys.version_info < (3, 7) or (sys.version_info < (3, 8) and not _IS_PYPY):
print(
"The typed_ast package is required but not installed.\n"
"You can upgrade to Python 3.8+ or install typed_ast with\n"
# Python 3-compatible code, so only try Python 3 grammar.
grammars = []
+ if supports_feature(target_versions, Feature.PATTERN_MATCHING):
+ # Python 3.10+
+ grammars.append(pygram.python_grammar_soft_keywords)
# If we have to parse both, try to parse async as a keyword first
- if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS):
- # Python 3.7+
+ if not supports_feature(
+ target_versions, Feature.ASYNC_IDENTIFIERS
+ ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
+ # Python 3.7-3.9
grammars.append(
pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
)
src_txt += "\n"
for grammar in get_grammars(set(target_versions)):
- drv = driver.Driver(grammar, pytree.convert)
+ drv = driver.Driver(grammar)
try:
result = drv.parse_string(src_txt, True)
break
return code
-def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]:
+def parse_single_version(
+ src: str, version: Tuple[int, int]
+) -> Union[ast.AST, ast3.AST, ast27.AST]:
filename = "<unknown>"
- if sys.version_info >= (3, 8):
- # TODO: support Python 4+ ;)
- for minor_version in range(sys.version_info[1], 4, -1):
- try:
- return ast.parse(src, filename, feature_version=(3, minor_version))
- except SyntaxError:
- continue
- else:
- for feature_version in (7, 6):
- try:
- return ast3.parse(src, filename, feature_version=feature_version)
- except SyntaxError:
- continue
- if ast27.__name__ == "ast":
- raise SyntaxError(
- "The requested source code has invalid Python 3 syntax.\n"
- "If you are trying to format Python 2 files please reinstall Black"
- " with the 'python2' extra: `python3 -m pip install black[python2]`."
- )
- return ast27.parse(src)
+ # typed_ast is needed because of feature version limitations in the builtin ast
+ if sys.version_info >= (3, 8) and version >= (3,):
+ return ast.parse(src, filename, feature_version=version)
+ elif version >= (3,):
+ if _IS_PYPY:
+ return ast3.parse(src, filename)
+ else:
+ return ast3.parse(src, filename, feature_version=version[1])
+ elif version == (2, 7):
+ return ast27.parse(src)
+ raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!")
+
+
+def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]:
+ # TODO: support Python 4+ ;)
+ versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]
+
+ if ast27.__name__ != "ast":
+ versions.append((2, 7))
+
+ first_error = ""
+ for version in sorted(versions, reverse=True):
+ try:
+ return parse_single_version(src, version)
+ except SyntaxError as e:
+ if not first_error:
+ first_error = str(e)
+
+ raise SyntaxError(first_error)
+
+
+ast3_AST: Final[Type[ast3.AST]] = ast3.AST
+ast27_AST: Final[Type[ast27.AST]] = ast27.AST
def stringify_ast(
yield f"{' ' * depth}{node.__class__.__name__}("
+ type_ignore_classes: Tuple[Type[Any], ...]
for field in sorted(node._fields): # noqa: F402
- # TypeIgnore has only one field 'lineno' which breaks this comparison
- type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore)
- if sys.version_info >= (3, 8):
- type_ignore_classes += (ast.TypeIgnore,)
- if isinstance(node, type_ignore_classes):
- break
+ # TypeIgnore will not be present using pypy < 3.8, so need for this
+ if not (_IS_PYPY and sys.version_info < (3, 8)):
+ # TypeIgnore has only one field 'lineno' which breaks this comparison
+ type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore)
+ if sys.version_info >= (3, 8):
+ type_ignore_classes += (ast.TypeIgnore,)
+ if isinstance(node, type_ignore_classes):
+ break
try:
value = getattr(node, field)
elif isinstance(item, (ast.AST, ast3.AST, ast27.AST)):
yield from stringify_ast(item, depth + 2)
- elif isinstance(value, (ast.AST, ast3.AST, ast27.AST)):
+ # Note that we are referencing the typed-ast ASTs via global variables and not
+ # direct module attribute accesses because that breaks mypyc. It's probably
+ # something to do with the ast3 / ast27 variables being marked as Any leading
+ # mypy to think this branch is always taken, leaving the rest of the code
+ # unanalyzed. Tighting up the types for the typed-ast AST types avoids the
+ # mypyc crash.
+ elif isinstance(value, (ast.AST, ast3_AST, ast27_AST)):
yield from stringify_ast(value, depth + 2)
else:
projects / etc / vim.git / blobdiff