X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/e401b6bb1e1c0ed534bba59d9dc908caf7ba898c..a07871b9cd1b5a1469271be6aaac5766f5a0e0fc:/src/black/parsing.py diff --git a/src/black/parsing.py b/src/black/parsing.py index 13fa67e..70ed99c 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -11,16 +11,14 @@ if sys.version_info < (3, 8): else: from typing import Final -# lib2to3 fork -from blib2to3.pytree import Node, Leaf +from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature +from black.nodes import syms from blib2to3 import pygram from blib2to3.pgen2 import driver from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.parse import ParseError from blib2to3.pgen2.tokenize import TokenError - -from black.mode import TargetVersion, Feature, supports_feature -from black.nodes import syms +from blib2to3.pytree import Leaf, Node ast3: Any @@ -29,8 +27,7 @@ _IS_PYPY = platform.python_implementation() == "PyPy" try: from typed_ast import ast3 except ImportError: - # Either our python version is too low, or we're on pypy - if sys.version_info < (3, 7) or (sys.version_info < (3, 8) and not _IS_PYPY): + if sys.version_info < (3, 8) and not _IS_PYPY: print( "The typed_ast package is required but not installed.\n" "You can upgrade to Python 3.8+ or install typed_ast with\n" @@ -42,7 +39,6 @@ except ImportError: ast3 = ast -PY310_HINT: Final = "Consider using --target-version py310 to parse Python 3.10 code." PY2_HINT: Final = "Python 2 support was removed in version 22.0." @@ -54,16 +50,15 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: if not target_versions: # No target_version specified, so try all grammars. return [ - # Python 3.7+ + # Python 3.7-3.9 pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords, # Python 3.0-3.6 pygram.python_grammar_no_print_statement_no_exec_statement, + # Python 3.10+ + pygram.python_grammar_soft_keywords, ] grammars = [] - if supports_feature(target_versions, Feature.PATTERN_MATCHING): - # Python 3.10+ - grammars.append(pygram.python_grammar_soft_keywords) # If we have to parse both, try to parse async as a keyword first if not supports_feature( target_versions, Feature.ASYNC_IDENTIFIERS @@ -75,6 +70,10 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): # Python 3.0-3.6 grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement) + if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions): + # Python 3.10+ + grammars.append(pygram.python_grammar_soft_keywords) + # At least one of the above branches must have been taken, because every Python # version has exactly one of the two 'ASYNC_*' flags return grammars @@ -86,6 +85,7 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) - src_txt += "\n" grammars = get_grammars(set(target_versions)) + errors = {} for grammar in grammars: drv = driver.Driver(grammar) try: @@ -99,20 +99,21 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) - faulty_line = lines[lineno - 1] except IndexError: faulty_line = "" - exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}") + errors[grammar.version] = InvalidInput( + f"Cannot parse: {lineno}:{column}: {faulty_line}" + ) except TokenError as te: # In edge cases these are raised; and typically don't have a "faulty_line". lineno, column = te.args[1] - exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {te.args[0]}") + errors[grammar.version] = InvalidInput( + f"Cannot parse: {lineno}:{column}: {te.args[0]}" + ) else: - if pygram.python_grammar_soft_keywords not in grammars and matches_grammar( - src_txt, pygram.python_grammar_soft_keywords - ): - original_msg = exc.args[0] - msg = f"{original_msg}\n{PY310_HINT}" - raise InvalidInput(msg) from None + # Choose the latest version when raising the actual parsing error. + assert len(errors) >= 1 + exc = errors[max(errors)] if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar( src_txt, pygram.python_grammar_no_print_statement @@ -145,18 +146,29 @@ def lib2to3_unparse(node: Node) -> str: def parse_single_version( - src: str, version: Tuple[int, int] + src: str, version: Tuple[int, int], *, type_comments: bool ) -> Union[ast.AST, ast3.AST]: filename = "" - # typed_ast is needed because of feature version limitations in the builtin ast + # typed-ast is needed because of feature version limitations in the builtin ast 3.8> if sys.version_info >= (3, 8) and version >= (3,): - return ast.parse(src, filename, feature_version=version) - elif version >= (3,): - if _IS_PYPY: - return ast3.parse(src, filename) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) + + if _IS_PYPY: + # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's + # not much we can do as typed-ast won't work either. + if sys.version_info >= (3, 8): + return ast3.parse(src, filename, type_comments=type_comments) else: + return ast3.parse(src, filename) + else: + if type_comments: + # Typed-ast is guaranteed to be used here and automatically tracks type + # comments separately. return ast3.parse(src, filename, feature_version=version[1]) - raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!") + else: + return ast.parse(src, filename) def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: @@ -166,11 +178,18 @@ def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: first_error = "" for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version) + return parse_single_version(src, version, type_comments=True) except SyntaxError as e: if not first_error: first_error = str(e) + # Try to parse without type comments + for version in sorted(versions, reverse=True): + try: + return parse_single_version(src, version, type_comments=False) + except SyntaxError: + pass + raise SyntaxError(first_error) @@ -206,7 +225,7 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st break try: - value = getattr(node, field) + value: object = getattr(node, field) except AttributeError: continue @@ -221,8 +240,8 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st and isinstance(node, (ast.Delete, ast3.Delete)) and isinstance(item, (ast.Tuple, ast3.Tuple)) ): - for item in item.elts: - yield from stringify_ast(item, depth + 2) + for elt in item.elts: + yield from stringify_ast(elt, depth + 2) elif isinstance(item, (ast.AST, ast3.AST)): yield from stringify_ast(item, depth + 2) @@ -237,6 +256,7 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st yield from stringify_ast(value, depth + 2) else: + normalized: object # Constant strings may be indented across newlines, if they are # docstrings; fold spaces after newlines when comparing. Similarly, # trailing and leading space may be removed.