Fix parser bug where "type" was misinterpreted as a keyword inside a match (#3950)

[etc/vim.git] / src / black / parsing.py

diff --git a/src/black/parsing.py b/src/black/parsing.py
index f7d9fdc2f134f8689f25e4f1d67ab4bb415546bb..ea282d1805cfea68ccf406e7e92e676de16f5fca 100644 (file)
--- a/src/black/parsing.py
+++ b/src/black/parsing.py
@@ -1,17 +1,12 @@
 """
 Parse Python code and perform AST validation.
 """
 """
 Parse Python code and perform AST validation.
 """

+

 import ast
 import ast

-import platform

 import sys
 import sys

-from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union
-
-if sys.version_info < (3, 8):
-    from typing_extensions import Final
-else:
-    from typing import Final
+from typing import Iterable, Iterator, List, Set, Tuple

 
 

-from black.mode import Feature, TargetVersion, supports_feature
+from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature

 from black.nodes import syms
 from blib2to3 import pygram
 from blib2to3.pgen2 import driver
 from black.nodes import syms
 from blib2to3 import pygram
 from blib2to3.pgen2 import driver


@@ -20,27 +15,6 @@ from blib2to3.pgen2.parse import ParseError
 from blib2to3.pgen2.tokenize import TokenError
 from blib2to3.pytree import Leaf, Node
 
 from blib2to3.pgen2.tokenize import TokenError
 from blib2to3.pytree import Leaf, Node
 

-ast3: Any
-
-_IS_PYPY = platform.python_implementation() == "PyPy"
-
-try:
-    from typed_ast import ast3
-except ImportError:
-    if sys.version_info < (3, 8) and not _IS_PYPY:
-        print(
-            "The typed_ast package is required but not installed.\n"
-            "You can upgrade to Python 3.8+ or install typed_ast with\n"
-            "`python3 -m pip install typed-ast`.",
-            file=sys.stderr,
-        )
-        sys.exit(1)
-    else:
-        ast3 = ast
-
-
-PY2_HINT: Final = "Python 2 support was removed in version 22.0."
-

 
 class InvalidInput(ValueError):
     """Raised when input source code fails all parse attempts."""
 
 class InvalidInput(ValueError):
     """Raised when input source code fails all parse attempts."""


@@ -50,10 +24,10 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
     if not target_versions:
         # No target_version specified, so try all grammars.
         return [
     if not target_versions:
         # No target_version specified, so try all grammars.
         return [

-            # Python 3.7+
-            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
+            # Python 3.7-3.9
+            pygram.python_grammar_async_keywords,

             # Python 3.0-3.6
             # Python 3.0-3.6

-            pygram.python_grammar_no_print_statement_no_exec_statement,
+            pygram.python_grammar,

             # Python 3.10+
             pygram.python_grammar_soft_keywords,
         ]
             # Python 3.10+
             pygram.python_grammar_soft_keywords,
         ]


@@ -64,13 +38,11 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
         target_versions, Feature.ASYNC_IDENTIFIERS
     ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
         # Python 3.7-3.9
         target_versions, Feature.ASYNC_IDENTIFIERS
     ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
         # Python 3.7-3.9

-        grammars.append(
-            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
-        )
+        grammars.append(pygram.python_grammar_async_keywords)

     if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
         # Python 3.0-3.6
     if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
         # Python 3.0-3.6

-        grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
-    if supports_feature(target_versions, Feature.PATTERN_MATCHING):
+        grammars.append(pygram.python_grammar)
+    if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions):

         # Python 3.10+
         grammars.append(pygram.python_grammar_soft_keywords)
 
         # Python 3.10+
         grammars.append(pygram.python_grammar_soft_keywords)
 


@@ -114,14 +86,6 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -
         # Choose the latest version when raising the actual parsing error.
         assert len(errors) >= 1
         exc = errors[max(errors)]
         # Choose the latest version when raising the actual parsing error.
         assert len(errors) >= 1
         exc = errors[max(errors)]

-
-        if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar(
-            src_txt, pygram.python_grammar_no_print_statement
-        ):
-            original_msg = exc.args[0]
-            msg = f"{original_msg}\n{PY2_HINT}"
-            raise InvalidInput(msg) from None
-

         raise exc from None
 
     if isinstance(result, Leaf):
         raise exc from None
 
     if isinstance(result, Leaf):


@@ -146,44 +110,34 @@ def lib2to3_unparse(node: Node) -> str:
 
 
 def parse_single_version(
 
 
 def parse_single_version(

-    src: str, version: Tuple[int, int]
-) -> Union[ast.AST, ast3.AST]:
+    src: str, version: Tuple[int, int], *, type_comments: bool
+) -> ast.AST:

     filename = "<unknown>"
     filename = "<unknown>"

-    # typed-ast is needed because of feature version limitations in the builtin ast 3.8>
-    if sys.version_info >= (3, 8) and version >= (3,):
-        return ast.parse(src, filename, feature_version=version, type_comments=True)
-
-    if _IS_PYPY:
-        # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
-        # not much we can do as typed-ast won't work either.
-        if sys.version_info >= (3, 8):
-            return ast3.parse(src, filename, type_comments=True)
-        else:
-            return ast3.parse(src, filename)
-    else:
-        # Typed-ast is guaranteed to be used here and automatically tracks type
-        # comments separately.
-        return ast3.parse(src, filename, feature_version=version[1])
-
-    raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!")
+    return ast.parse(
+        src, filename, feature_version=version, type_comments=type_comments
+    )

 
 
 
 

-def parse_ast(src: str) -> Union[ast.AST, ast3.AST]:
+def parse_ast(src: str) -> ast.AST:

     # TODO: support Python 4+ ;)
     versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]
 
     first_error = ""
     for version in sorted(versions, reverse=True):
         try:
     # TODO: support Python 4+ ;)
     versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]
 
     first_error = ""
     for version in sorted(versions, reverse=True):
         try:

-            return parse_single_version(src, version)
+            return parse_single_version(src, version, type_comments=True)

         except SyntaxError as e:
             if not first_error:
                 first_error = str(e)
 
         except SyntaxError as e:
             if not first_error:
                 first_error = str(e)
 

-    raise SyntaxError(first_error)
-
+    # Try to parse without type comments
+    for version in sorted(versions, reverse=True):
+        try:
+            return parse_single_version(src, version, type_comments=False)
+        except SyntaxError:
+            pass

 
 

-ast3_AST: Final[Type[ast3.AST]] = ast3.AST
+    raise SyntaxError(first_error)

 
 
 def _normalize(lineend: str, value: str) -> str:
 
 
 def _normalize(lineend: str, value: str) -> str:


@@ -196,23 +150,25 @@ def _normalize(lineend: str, value: str) -> str:
     return normalized.strip()
 
 
     return normalized.strip()
 
 

-def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]:
+def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]:

     """Simple visitor generating strings to compare ASTs by content."""
 
     """Simple visitor generating strings to compare ASTs by content."""
 

-    node = fixup_ast_constants(node)
+    if (
+        isinstance(node, ast.Constant)
+        and isinstance(node.value, str)
+        and node.kind == "u"
+    ):
+        # It's a quirk of history that we strip the u prefix over here. We used to
+        # rewrite the AST nodes for Python version compatibility and we never copied
+        # over the kind
+        node.kind = None

 
     yield f"{'  ' * depth}{node.__class__.__name__}("
 
 
     yield f"{'  ' * depth}{node.__class__.__name__}("
 

-    type_ignore_classes: Tuple[Type[Any], ...]

     for field in sorted(node._fields):  # noqa: F402
     for field in sorted(node._fields):  # noqa: F402

-        # TypeIgnore will not be present using pypy < 3.8, so need for this
-        if not (_IS_PYPY and sys.version_info < (3, 8)):
-            # TypeIgnore has only one field 'lineno' which breaks this comparison
-            type_ignore_classes = (ast3.TypeIgnore,)
-            if sys.version_info >= (3, 8):
-                type_ignore_classes += (ast.TypeIgnore,)
-            if isinstance(node, type_ignore_classes):
-                break
+        # TypeIgnore has only one field 'lineno' which breaks this comparison
+        if isinstance(node, ast.TypeIgnore):
+            break

 
         try:
             value: object = getattr(node, field)
 
         try:
             value: object = getattr(node, field)


@@ -227,51 +183,34 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st
                 # parentheses and they change the AST.
                 if (
                     field == "targets"
                 # parentheses and they change the AST.
                 if (
                     field == "targets"

-                    and isinstance(node, (ast.Delete, ast3.Delete))
-                    and isinstance(item, (ast.Tuple, ast3.Tuple))
+                    and isinstance(node, ast.Delete)
+                    and isinstance(item, ast.Tuple)

                 ):
                     for elt in item.elts:
                         yield from stringify_ast(elt, depth + 2)
 
                 ):
                     for elt in item.elts:
                         yield from stringify_ast(elt, depth + 2)
 

-                elif isinstance(item, (ast.AST, ast3.AST)):
+                elif isinstance(item, ast.AST):

                     yield from stringify_ast(item, depth + 2)
 
                     yield from stringify_ast(item, depth + 2)
 

-        # Note that we are referencing the typed-ast ASTs via global variables and not
-        # direct module attribute accesses because that breaks mypyc. It's probably
-        # something to do with the ast3 variables being marked as Any leading
-        # mypy to think this branch is always taken, leaving the rest of the code
-        # unanalyzed. Tighting up the types for the typed-ast AST types avoids the
-        # mypyc crash.
-        elif isinstance(value, (ast.AST, ast3_AST)):
+        elif isinstance(value, ast.AST):

             yield from stringify_ast(value, depth + 2)
 
         else:
             normalized: object
             yield from stringify_ast(value, depth + 2)
 
         else:
             normalized: object

-            # Constant strings may be indented across newlines, if they are
-            # docstrings; fold spaces after newlines when comparing. Similarly,
-            # trailing and leading space may be removed.

             if (
                 isinstance(node, ast.Constant)
                 and field == "value"
                 and isinstance(value, str)
             ):
             if (
                 isinstance(node, ast.Constant)
                 and field == "value"
                 and isinstance(value, str)
             ):

+                # Constant strings may be indented across newlines, if they are
+                # docstrings; fold spaces after newlines when comparing. Similarly,
+                # trailing and leading space may be removed.

                 normalized = _normalize("\n", value)
                 normalized = _normalize("\n", value)

+            elif field == "type_comment" and isinstance(value, str):
+                # Trailing whitespace in type comments is removed.
+                normalized = value.rstrip()

             else:
                 normalized = value
             yield f"{'  ' * (depth+2)}{normalized!r},  # {value.__class__.__name__}"
 
     yield f"{'  ' * depth})  # /{node.__class__.__name__}"
             else:
                 normalized = value
             yield f"{'  ' * (depth+2)}{normalized!r},  # {value.__class__.__name__}"
 
     yield f"{'  ' * depth})  # /{node.__class__.__name__}"

-
-
-def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]:
-    """Map ast nodes deprecated in 3.8 to Constant."""
-    if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)):
-        return ast.Constant(value=node.s)
-
-    if isinstance(node, (ast.Num, ast3.Num)):
-        return ast.Constant(value=node.n)
-
-    if isinstance(node, (ast.NameConstant, ast3.NameConstant)):
-        return ast.Constant(value=node.value)
-
-    return node