Don't let TokenError bubble up from lib2to3_parse (GH-2343)

[etc/vim.git] / src / black / parsing.py

diff --git a/src/black/parsing.py b/src/black/parsing.py
index 32cfa5239f117e00c2ce969baab6ee3118a5b519..b673027022f44b4a2ebd5a9b4b3bd44d0b163b3b 100644 (file)
--- a/src/black/parsing.py
+++ b/src/black/parsing.py
@@ -17,6 +17,7 @@ from blib2to3 import pygram
 from blib2to3.pgen2 import driver
 from blib2to3.pgen2.grammar import Grammar
 from blib2to3.pgen2.parse import ParseError
+from blib2to3.pgen2.tokenize import TokenError
 
 from black.mode import TargetVersion, Feature, supports_feature
 from black.nodes import syms
@@ -75,8 +76,10 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
         # Python 3.10+
         grammars.append(pygram.python_grammar_soft_keywords)
     # If we have to parse both, try to parse async as a keyword first
-    if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS):
-        # Python 3.7+
+    if not supports_feature(
+        target_versions, Feature.ASYNC_IDENTIFIERS
+    ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
+        # Python 3.7-3.9
         grammars.append(
             pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
         )
@@ -107,6 +110,12 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -
             except IndexError:
                 faulty_line = "<line number missing in source>"
             exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}")
+
+        except TokenError as te:
+            # In edge cases these are raised; and typically don't have a "faulty_line".
+            lineno, column = te.args[1]
+            exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {te.args[0]}")
+
     else:
         raise exc from None