Handle Docstrings as bytes + strip all whitespace (#2037)

(fixes #1844, fixes #1923, fixes #1851, fixes #2002, fixes #2103)

diff --git a/CHANGES.md b/CHANGES.md
index f911fdf77ae949beeaef44c43d34b6902d3eb161..2999abfff4ebf08417cb67abde609910640bb366 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -52,6 +52,9 @@
 
 - Exclude `venv` directory by default (#1683)
 
+- Fixed "Black produced code that is not equivalent to the source" when formatting
+  Python 2 docstrings (#2037)
+
 #### _Packaging_
 
 - Self-contained native _Black_ binaries are now provided for releases via GitHub

diff --git a/src/black/__init__.py b/src/black/__init__.py
index efa82f41160ac12f1665052fc7ccfc459d71cdc0..5c9ab7516508b78ee23eebb49fbe934912548554 100644 (file)
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -6466,12 +6466,22 @@ def _stringify_ast(
             # Constant strings may be indented across newlines, if they are
             # docstrings; fold spaces after newlines when comparing. Similarly,
             # trailing and leading space may be removed.
+            # Note that when formatting Python 2 code, at least with Windows
+            # line-endings, docstrings can end up here as bytes instead of
+            # str so make sure that we handle both cases.
             if (
                 isinstance(node, ast.Constant)
                 and field == "value"
-                and isinstance(value, str)
+                and isinstance(value, (str, bytes))
             ):
-                normalized = re.sub(r" *\n[ \t]*", "\n", value).strip()
+                lineend = "\n" if isinstance(value, str) else b"\n"
+                # To normalize, we strip any leading and trailing space from
+                # each line...
+                stripped = [line.strip() for line in value.splitlines()]
+                normalized = lineend.join(stripped)  # type: ignore[attr-defined]
+                # ...and remove any blank lines at the beginning and end of
+                # the whole string
+                normalized = normalized.strip()
             else:
                 normalized = value
             yield f"{'  ' * (depth+2)}{normalized!r},  # {value.__class__.__name__}"

diff --git a/tests/test_black.py b/tests/test_black.py
index c603233efc40b58e874e5c76461ee74f6f491ded..201edfa2de522b17b84a153333a969100c933e87 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -1918,6 +1918,26 @@ class BlackTestCase(BlackBaseTestCase):
         actual = diff_header.sub(DETERMINISTIC_HEADER, actual)
         self.assertEqual(actual, expected)
 
+    def test_docstring_reformat_for_py27(self) -> None:
+        """
+        Check that stripping trailing whitespace from Python 2 docstrings
+        doesn't trigger a "not equivalent to source" error
+        """
+        source = (
+            b'def foo():\r\n    """Testing\r\n    Testing """\r\n    print "Foo"\r\n'
+        )
+        expected = 'def foo():\n    """Testing\n    Testing"""\n    print "Foo"\n'
+
+        result = CliRunner().invoke(
+            black.main,
+            ["-", "-q", "--target-version=py27"],
+            input=BytesIO(source),
+        )
+
+        self.assertEqual(result.exit_code, 0)
+        actual = result.output
+        self.assertFormatEqual(actual, expected)
+
 
 with open(black.__file__, "r", encoding="utf-8") as _bf:
     black_source_lines = _bf.readlines()