Correctly handle inline tabs in docstrings (#1810)

The `fix_docstring` function expanded all tabs, which caused a
difference in the AST representation when those tabs were inline and not
leading. This changes the function to only expand leading tabs so inline
tabs are preserved.

Fixes #1601.

diff --git a/src/black/__init__.py b/src/black/__init__.py
index 7a7456ad78b7217c93a5f85993b998bdb281c6d7..7e13a5d33f5169e35d195dd4f8bbbb05ef9f524a 100644 (file)
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -6744,13 +6744,33 @@ def is_docstring(leaf: Leaf) -> bool:
     return False
 
 
     return False
 
 

+def lines_with_leading_tabs_expanded(s: str) -> List[str]:
+    """
+    Splits string into lines and expands only leading tabs (following the normal
+    Python rules)
+    """
+    lines = []
+    for line in s.splitlines():
+        # Find the index of the first non-whitespace character after a string of
+        # whitespace that includes at least one tab
+        match = re.match(r"\s*\t+\s*(\S)", line)
+        if match:
+            first_non_whitespace_idx = match.start(1)
+
+            lines.append(
+                line[:first_non_whitespace_idx].expandtabs()
+                + line[first_non_whitespace_idx:]
+            )
+        else:
+            lines.append(line)
+    return lines
+
+

 def fix_docstring(docstring: str, prefix: str) -> str:
     # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation
     if not docstring:
         return ""
 def fix_docstring(docstring: str, prefix: str) -> str:
     # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation
     if not docstring:
         return ""

-    # Convert tabs to spaces (following the normal Python rules)
-    # and split into a list of lines:
-    lines = docstring.expandtabs().splitlines()
+    lines = lines_with_leading_tabs_expanded(docstring)

     # Determine minimum indentation (first line doesn't count):
     indent = sys.maxsize
     for line in lines[1:]:
     # Determine minimum indentation (first line doesn't count):
     indent = sys.maxsize
     for line in lines[1:]:

diff --git a/tests/data/docstring.py b/tests/data/docstring.py
index 2d3d73a101cff55a75031e21899aad6af6233d49..5c6985d0e0851cd71018a1ae7eb8b34d14891348 100644 (file)
--- a/tests/data/docstring.py
+++ b/tests/data/docstring.py
@@ -110,6 +110,32 @@ def ignored_docstring():
     """a => \
 b"""  
 
     """a => \
 b"""  
 

+
+def docstring_with_inline_tabs_and_space_indentation():
+    """hey
+
+    tab        separated       value
+       tab at start of line and then a tab     separated       value
+                               multiple tabs at the beginning  and     inline
+                       mixed tabs and spaces at beginning. next line has mixed tabs and spaces only.
+                                               
+    line ends with some tabs           
+    """
+
+
+def docstring_with_inline_tabs_and_tab_indentation():
+       """hey
+
+       tab     separated       value
+               tab at start of line and then a tab     separated       value
+                                       multiple tabs at the beginning  and     inline
+                               mixed tabs and spaces at beginning. next line has mixed tabs and spaces only.
+                                                       
+       line ends with some tabs                
+       """
+       pass
+        
+

 # output
 
 class MyClass:
 # output
 
 class MyClass:


@@ -222,4 +248,29 @@ def believe_it_or_not_this_is_in_the_py_stdlib():
 
 def ignored_docstring():
     """a => \
 
 def ignored_docstring():
     """a => \

-b"""
\ No newline at end of file
+b"""
+
+
+def docstring_with_inline_tabs_and_space_indentation():
+    """hey
+
+    tab        separated       value
+        tab at start of line and then a tab    separated       value
+                                multiple tabs at the beginning and     inline
+                        mixed tabs and spaces at beginning. next line has mixed tabs and spaces only.
+
+    line ends with some tabs
+    """
+
+
+def docstring_with_inline_tabs_and_tab_indentation():
+    """hey
+
+    tab        separated       value
+            tab at start of line and then a tab        separated       value
+                                    multiple tabs at the beginning     and     inline
+                            mixed tabs and spaces at beginning. next line has mixed tabs and spaces only.
+
+    line ends with some tabs
+    """
+    pass