Don't let TokenError bubble up from lib2to3_parse (GH-2343)

[etc/vim.git] / src / black / comments.py

diff --git a/src/black/comments.py b/src/black/comments.py
index 415e391b2a7270d5047bd948fa15d823f29a0b2e..28b9117101d7aa0eb19ce64190b6c2a83b83a4fd 100644 (file)
--- a/src/black/comments.py
+++ b/src/black/comments.py
@@ -1,8 +1,14 @@
+import sys

 from dataclasses import dataclass
 from functools import lru_cache
 from dataclasses import dataclass
 from functools import lru_cache

-import regex as re
+import re

 from typing import Iterator, List, Optional, Union
 
 from typing import Iterator, List, Optional, Union
 

+if sys.version_info >= (3, 8):
+    from typing import Final
+else:
+    from typing_extensions import Final
+

 from blib2to3.pytree import Node, Leaf
 from blib2to3.pgen2 import token
 
 from blib2to3.pytree import Node, Leaf
 from blib2to3.pgen2 import token
 


@@ -12,11 +18,10 @@ from black.nodes import STANDALONE_COMMENT, WHITESPACE
 # types
 LN = Union[Leaf, Node]
 
 # types
 LN = Union[Leaf, Node]
 

-
-FMT_OFF = {"# fmt: off", "# fmt:off", "# yapf: disable"}
-FMT_SKIP = {"# fmt: skip", "# fmt:skip"}
-FMT_PASS = {*FMT_OFF, *FMT_SKIP}
-FMT_ON = {"# fmt: on", "# fmt:on", "# yapf: enable"}
+FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"}
+FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"}
+FMT_PASS: Final = {*FMT_OFF, *FMT_SKIP}
+FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"}

 
 
 @dataclass
 
 
 @dataclass


@@ -159,7 +164,10 @@ def convert_one_fmt_off_pair(node: Node) -> bool:
             first = ignored_nodes[0]  # Can be a container node with the `leaf`.
             parent = first.parent
             prefix = first.prefix
             first = ignored_nodes[0]  # Can be a container node with the `leaf`.
             parent = first.parent
             prefix = first.prefix

-            first.prefix = prefix[comment.consumed :]
+            if comment.value in FMT_OFF:
+                first.prefix = prefix[comment.consumed :]
+            if comment.value in FMT_SKIP:
+                first.prefix = ""

             hidden_value = "".join(str(n) for n in ignored_nodes)
             if comment.value in FMT_OFF:
                 hidden_value = comment.value + "\n" + hidden_value
             hidden_value = "".join(str(n) for n in ignored_nodes)
             if comment.value in FMT_OFF:
                 hidden_value = comment.value + "\n" + hidden_value