X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/1e0ec543ff3a7de715c8ee3359c8defb2c2c0e0d..bb588073ab286a9f1f8d839ab2cebe13011dd22c:/src/black/parsing.py diff --git a/src/black/parsing.py b/src/black/parsing.py index fc540ad..ea282d1 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -1,33 +1,19 @@ """ Parse Python code and perform AST validation. """ + import ast import sys -from typing import Iterable, Iterator, List, Set, Union, Tuple +from typing import Iterable, Iterator, List, Set, Tuple -# lib2to3 fork -from blib2to3.pytree import Node, Leaf -from blib2to3 import pygram, pytree +from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature +from black.nodes import syms +from blib2to3 import pygram from blib2to3.pgen2 import driver from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.parse import ParseError - -from black.mode import TargetVersion, Feature, supports_feature -from black.nodes import syms - -try: - from typed_ast import ast3, ast27 -except ImportError: - if sys.version_info < (3, 8): - print( - "The typed_ast package is required but not installed.\n" - "You can upgrade to Python 3.8+ or install typed_ast with\n" - "`python3 -m pip install typed-ast`.", - file=sys.stderr, - ) - sys.exit(1) - else: - ast3 = ast27 = ast +from blib2to3.pgen2.tokenize import TokenError +from blib2to3.pytree import Leaf, Node class InvalidInput(ValueError): @@ -38,39 +24,28 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: if not target_versions: # No target_version specified, so try all grammars. return [ - # Python 3.7+ - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords, + # Python 3.7-3.9 + pygram.python_grammar_async_keywords, # Python 3.0-3.6 - pygram.python_grammar_no_print_statement_no_exec_statement, - # Python 2.7 with future print_function import - pygram.python_grammar_no_print_statement, - # Python 2.7 - pygram.python_grammar, - ] - - if all(version.is_python2() for version in target_versions): - # Python 2-only code, so try Python 2 grammars. - return [ - # Python 2.7 with future print_function import - pygram.python_grammar_no_print_statement, - # Python 2.7 pygram.python_grammar, + # Python 3.10+ + pygram.python_grammar_soft_keywords, ] - # Python 3-compatible code, so only try Python 3 grammar. grammars = [] - if supports_feature(target_versions, Feature.PATTERN_MATCHING): - # Python 3.10+ - grammars.append(pygram.python_grammar_soft_keywords) # If we have to parse both, try to parse async as a keyword first - if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS): - # Python 3.7+ - grammars.append( - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords - ) + if not supports_feature( + target_versions, Feature.ASYNC_IDENTIFIERS + ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING): + # Python 3.7-3.9 + grammars.append(pygram.python_grammar_async_keywords) if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): # Python 3.0-3.6 - grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement) + grammars.append(pygram.python_grammar) + if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions): + # Python 3.10+ + grammars.append(pygram.python_grammar_soft_keywords) + # At least one of the above branches must have been taken, because every Python # version has exactly one of the two 'ASYNC_*' flags return grammars @@ -81,8 +56,10 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) - if not src_txt.endswith("\n"): src_txt += "\n" - for grammar in get_grammars(set(target_versions)): - drv = driver.Driver(grammar, pytree.convert) + grammars = get_grammars(set(target_versions)) + errors = {} + for grammar in grammars: + drv = driver.Driver(grammar) try: result = drv.parse_string(src_txt, True) break @@ -94,8 +71,21 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) - faulty_line = lines[lineno - 1] except IndexError: faulty_line = "" - exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}") + errors[grammar.version] = InvalidInput( + f"Cannot parse: {lineno}:{column}: {faulty_line}" + ) + + except TokenError as te: + # In edge cases these are raised; and typically don't have a "faulty_line". + lineno, column = te.args[1] + errors[grammar.version] = InvalidInput( + f"Cannot parse: {lineno}:{column}: {te.args[0]}" + ) + else: + # Choose the latest version when raising the actual parsing error. + assert len(errors) >= 1 + exc = errors[max(errors)] raise exc from None if isinstance(result, Leaf): @@ -103,6 +93,16 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) - return result +def matches_grammar(src_txt: str, grammar: Grammar) -> bool: + drv = driver.Driver(grammar) + try: + drv.parse_string(src_txt, True) + except (ParseError, TokenError, IndentationError): + return False + else: + return True + + def lib2to3_unparse(node: Node) -> str: """Given a lib2to3 node, return its string representation.""" code = str(node) @@ -110,56 +110,68 @@ def lib2to3_unparse(node: Node) -> str: def parse_single_version( - src: str, version: Tuple[int, int] -) -> Union[ast.AST, ast3.AST, ast27.AST]: + src: str, version: Tuple[int, int], *, type_comments: bool +) -> ast.AST: filename = "" - # typed_ast is needed because of feature version limitations in the builtin ast - if sys.version_info >= (3, 8) and version >= (3,): - return ast.parse(src, filename, feature_version=version) - elif version >= (3,): - return ast3.parse(src, filename, feature_version=version[1]) - elif version == (2, 7): - return ast27.parse(src) - raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!") + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) -def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]: +def parse_ast(src: str) -> ast.AST: # TODO: support Python 4+ ;) versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)] - if ast27.__name__ != "ast": - versions.append((2, 7)) - first_error = "" for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version) + return parse_single_version(src, version, type_comments=True) except SyntaxError as e: if not first_error: first_error = str(e) + # Try to parse without type comments + for version in sorted(versions, reverse=True): + try: + return parse_single_version(src, version, type_comments=False) + except SyntaxError: + pass + raise SyntaxError(first_error) -def stringify_ast( - node: Union[ast.AST, ast3.AST, ast27.AST], depth: int = 0 -) -> Iterator[str]: +def _normalize(lineend: str, value: str) -> str: + # To normalize, we strip any leading and trailing space from + # each line... + stripped: List[str] = [i.strip() for i in value.splitlines()] + normalized = lineend.join(stripped) + # ...and remove any blank lines at the beginning and end of + # the whole string + return normalized.strip() + + +def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: """Simple visitor generating strings to compare ASTs by content.""" - node = fixup_ast_constants(node) + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.kind == "u" + ): + # It's a quirk of history that we strip the u prefix over here. We used to + # rewrite the AST nodes for Python version compatibility and we never copied + # over the kind + node.kind = None yield f"{' ' * depth}{node.__class__.__name__}(" for field in sorted(node._fields): # noqa: F402 # TypeIgnore has only one field 'lineno' which breaks this comparison - type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore) - if sys.version_info >= (3, 8): - type_ignore_classes += (ast.TypeIgnore,) - if isinstance(node, type_ignore_classes): + if isinstance(node, ast.TypeIgnore): break try: - value = getattr(node, field) + value: object = getattr(node, field) except AttributeError: continue @@ -171,56 +183,34 @@ def stringify_ast( # parentheses and they change the AST. if ( field == "targets" - and isinstance(node, (ast.Delete, ast3.Delete, ast27.Delete)) - and isinstance(item, (ast.Tuple, ast3.Tuple, ast27.Tuple)) + and isinstance(node, ast.Delete) + and isinstance(item, ast.Tuple) ): - for item in item.elts: - yield from stringify_ast(item, depth + 2) + for elt in item.elts: + yield from stringify_ast(elt, depth + 2) - elif isinstance(item, (ast.AST, ast3.AST, ast27.AST)): + elif isinstance(item, ast.AST): yield from stringify_ast(item, depth + 2) - elif isinstance(value, (ast.AST, ast3.AST, ast27.AST)): + elif isinstance(value, ast.AST): yield from stringify_ast(value, depth + 2) else: - # Constant strings may be indented across newlines, if they are - # docstrings; fold spaces after newlines when comparing. Similarly, - # trailing and leading space may be removed. - # Note that when formatting Python 2 code, at least with Windows - # line-endings, docstrings can end up here as bytes instead of - # str so make sure that we handle both cases. + normalized: object if ( isinstance(node, ast.Constant) and field == "value" - and isinstance(value, (str, bytes)) + and isinstance(value, str) ): - lineend = "\n" if isinstance(value, str) else b"\n" - # To normalize, we strip any leading and trailing space from - # each line... - stripped = [line.strip() for line in value.splitlines()] - normalized = lineend.join(stripped) # type: ignore[attr-defined] - # ...and remove any blank lines at the beginning and end of - # the whole string - normalized = normalized.strip() + # Constant strings may be indented across newlines, if they are + # docstrings; fold spaces after newlines when comparing. Similarly, + # trailing and leading space may be removed. + normalized = _normalize("\n", value) + elif field == "type_comment" and isinstance(value, str): + # Trailing whitespace in type comments is removed. + normalized = value.rstrip() else: normalized = value yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" yield f"{' ' * depth}) # /{node.__class__.__name__}" - - -def fixup_ast_constants( - node: Union[ast.AST, ast3.AST, ast27.AST] -) -> Union[ast.AST, ast3.AST, ast27.AST]: - """Map ast nodes deprecated in 3.8 to Constant.""" - if isinstance(node, (ast.Str, ast3.Str, ast27.Str, ast.Bytes, ast3.Bytes)): - return ast.Constant(value=node.s) - - if isinstance(node, (ast.Num, ast3.Num, ast27.Num)): - return ast.Constant(value=node.n) - - if isinstance(node, (ast.NameConstant, ast3.NameConstant)): - return ast.Constant(value=node.value) - - return node