Copy over comments when hugging power ops (#2874)

Otherwise they'd be deleted which was a regression in 22.1.0 (oops! my
bad!). Also type comments are now tracked in the AST safety check on all
compatible platforms to error out if this happens again.

Overall the line rewriting code has been rewritten to do "the right
thing (tm)", I hope this fixes other potential bugs in the code (fwiw I
got to drop the bugfix in blib2to3.pytree.Leaf.clone since now bracket
metadata is properly copied over).

Fixes #2873

diff --git a/CHANGES.md b/CHANGES.md
index a0607edefa5eb4dd3a828409a978f17437d33b8f..249f7752beacbe53630f12351fc2489675e5a1cd 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -10,6 +10,9 @@
 
 <!-- Changes that affect Black's stable style -->
 
+- Comments are no longer deleted when a line had spaces removed around power operators
+  (#2874)
+
 ### Preview style
 
 <!-- Changes that affect Black's preview style -->
@@ -47,6 +50,10 @@
 
 <!-- Changes to the parser or to version autodetection -->
 
+- Type comments are now included in the AST equivalence check consistently so accidental
+  deletion raises an error. Though type comments can't be tracked when running on PyPy
+  3.7 due to standard library limitations. (#2874)
+
 ### Performance
 
 <!-- Changes that improve Black's performance. -->

diff --git a/src/black/parsing.py b/src/black/parsing.py
index 1272656794873ec67696bd1fbc40d4364ea76bdf..d1ad7d2c671a605d5a8745f39f47de355ae22cae 100644 (file)
--- a/src/black/parsing.py
+++ b/src/black/parsing.py
@@ -152,14 +152,22 @@ def parse_single_version(
     src: str, version: Tuple[int, int]
 ) -> Union[ast.AST, ast3.AST]:
     filename = "<unknown>"
-    # typed_ast is needed because of feature version limitations in the builtin ast
+    # typed-ast is needed because of feature version limitations in the builtin ast 3.8>
     if sys.version_info >= (3, 8) and version >= (3,):
-        return ast.parse(src, filename, feature_version=version)
-    elif version >= (3,):
-        if _IS_PYPY:
-            return ast3.parse(src, filename)
+        return ast.parse(src, filename, feature_version=version, type_comments=True)
+
+    if _IS_PYPY:
+        # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
+        # not much we can do as typed-ast won't work either.
+        if sys.version_info >= (3, 8):
+            return ast3.parse(src, filename, type_comments=True)
         else:
-            return ast3.parse(src, filename, feature_version=version[1])
+            return ast3.parse(src, filename)
+    else:
+        # Typed-ast is guaranteed to be used here and automatically tracks type
+        # comments separately.
+        return ast3.parse(src, filename, feature_version=version[1])
+
     raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!")
 
 

diff --git a/src/black/trans.py b/src/black/trans.py
index 01aa80eaaf84760f765ae34eef651d7336e28c09..28d9250adc1fe78d7e788b2d6fd382d00d42073b 100644 (file)
--- a/src/black/trans.py
+++ b/src/black/trans.py
@@ -121,7 +121,7 @@ def hug_power_op(line: Line, features: Collection[Feature]) -> Iterator[Line]:
 
         return False
 
-    leaves: List[Leaf] = []
+    new_line = line.clone()
     should_hug = False
     for idx, leaf in enumerate(line.leaves):
         new_leaf = leaf.clone()
@@ -139,18 +139,14 @@ def hug_power_op(line: Line, features: Collection[Feature]) -> Iterator[Line]:
         if should_hug:
             new_leaf.prefix = ""
 
-        leaves.append(new_leaf)
-
-    yield Line(
-        mode=line.mode,
-        depth=line.depth,
-        leaves=leaves,
-        comments=line.comments,
-        bracket_tracker=line.bracket_tracker,
-        inside_brackets=line.inside_brackets,
-        should_split_rhs=line.should_split_rhs,
-        magic_trailing_comma=line.magic_trailing_comma,
-    )
+        # We have to be careful to make a new line properly:
+        # - bracket related metadata must be maintained (handled by Line.append)
+        # - comments need to copied over, updating the leaf IDs they're attached to
+        new_line.append(new_leaf, preformatted=True)
+        for comment_leaf in line.comments_after(leaf):
+            new_line.append(comment_leaf, preformatted=True)
+
+    yield new_line
 
 
 class StringTransformer(ABC):

diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py
index b203ce5b2ac897b895aa28f31d81c4c38809c4e6..10b4690218e50209b5b00cce54195cd76112bb44 100644 (file)
--- a/src/blib2to3/pytree.py
+++ b/src/blib2to3/pytree.py
@@ -451,7 +451,6 @@ class Leaf(Base):
             self.value,
             (self.prefix, (self.lineno, self.column)),
             fixers_applied=self.fixers_applied,
-            opening_bracket=self.opening_bracket,
         )
 
     def leaves(self) -> Iterator["Leaf"]:

diff --git a/tests/data/simple_cases/power_op_spacing.py b/tests/data/simple_cases/power_op_spacing.py
index 87dde7f39dcd34d9a67eba52f18a4eda0a352dde..c95fa788fc3eba93de835ab39c37138c9e7dbbe6 100644 (file)
--- a/tests/data/simple_cases/power_op_spacing.py
+++ b/tests/data/simple_cases/power_op_spacing.py
@@ -49,6 +49,20 @@ p = {(k, k**2): v**2.0 for k, v in pairs}
 q = [10.5**i for i in range(6)]
 
 
+# WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873)
+if hasattr(view, "sum_of_weights"):
+    return np.divide(  # type: ignore[no-any-return]
+        view.variance,  # type: ignore[union-attr]
+        view.sum_of_weights,  # type: ignore[union-attr]
+        out=np.full(view.sum_of_weights.shape, np.nan),  # type: ignore[union-attr]
+        where=view.sum_of_weights**2 > view.sum_of_weights_squared,  # type: ignore[union-attr]
+    )
+
+return np.divide(
+    where=view.sum_of_weights_of_weight_long**2 > view.sum_of_weights_squared,  # type: ignore
+)
+
+
 # output
 
 
@@ -101,3 +115,17 @@ n = count <= 10**5.0
 o = settings(max_examples=10**6.0)
 p = {(k, k**2): v**2.0 for k, v in pairs}
 q = [10.5**i for i in range(6)]
+
+
+# WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873)
+if hasattr(view, "sum_of_weights"):
+    return np.divide(  # type: ignore[no-any-return]
+        view.variance,  # type: ignore[union-attr]
+        view.sum_of_weights,  # type: ignore[union-attr]
+        out=np.full(view.sum_of_weights.shape, np.nan),  # type: ignore[union-attr]
+        where=view.sum_of_weights**2 > view.sum_of_weights_squared,  # type: ignore[union-attr]
+    )
+
+return np.divide(
+    where=view.sum_of_weights_of_weight_long**2 > view.sum_of_weights_squared,  # type: ignore
+)