Return `NothingChanged` if non-Python cell magic is detected, to avoid tokenize error...

Fixes https://github.com/psf/black/issues/2627 , a non-Python cell magic such as `%%writeline` can legitimately contain "incorrect" indentation, however this causes `tokenize-rt` to return an error. To avoid this, `validate_cell` should early detect cell magics (just like it detects `TransformerManager` transformations).

Test added too, in the shape of a "badly indented" `%%writefile` within `test_non_python_magics`.

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Co-authored-by: Marco Edward Gorelli <marcogorelli@protonmail.com>

diff --git a/CHANGES.md b/CHANGES.md
index 235919ec7acbbbe9be293dbb5262e8ae82ef2001..57af2c5deae0d6d8ec633969a074c0823b4b11cc 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -4,6 +4,9 @@
 
 ### _Black_
 
 
 ### _Black_
 

+- Cell magics are now only processed if they are known Python cell magics. Earlier, all
+  cell magics were tokenized, leading to possible indentation errors e.g. with
+  `%%writefile`. (#2630)

 - Fixed Python 3.10 support on platforms without ProcessPoolExecutor (#2631)
 - Fixed `match` statements with open sequence subjects, like `match a, b:` (#2639)
 - Fixed assignment to environment variables in Jupyter Notebooks (#2642)
 - Fixed Python 3.10 support on platforms without ProcessPoolExecutor (#2631)
 - Fixed `match` statements with open sequence subjects, like `match a, b:` (#2639)
 - Fixed assignment to environment variables in Jupyter Notebooks (#2642)

diff --git a/docs/faq.md b/docs/faq.md
index 72bae6b389df003c70647599d14d83616fb74bde..88bf35b1e6a40d744c60e152ca2456a90054dc85 100644 (file)
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -47,6 +47,7 @@ _Black_ is timid about formatting Jupyter Notebooks. Cells containing any of the
 following will not be formatted:
 
 - automagics (e.g. `pip install black`)
 following will not be formatted:
 
 - automagics (e.g. `pip install black`)

+- non-Python cell magics (e.g. `%%writeline`)

 - multiline magics, e.g.:
 
   ```python
 - multiline magics, e.g.:
 
   ```python

diff --git a/src/black/__init__.py b/src/black/__init__.py
index d99c48a1b04f7686f50678db7421b4ea2210eb0f..c2b52e6eadb07936e41d53c59f61abf37bbb1e9d 100644 (file)
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -57,6 +57,7 @@ from black.handle_ipynb_magics import (
     remove_trailing_semicolon,
     put_trailing_semicolon_back,
     TRANSFORMED_MAGICS,
     remove_trailing_semicolon,
     put_trailing_semicolon_back,
     TRANSFORMED_MAGICS,

+    PYTHON_CELL_MAGICS,

     jupyter_dependencies_are_installed,
 )
 
     jupyter_dependencies_are_installed,
 )
 


@@ -943,7 +944,9 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo
 
 
 def validate_cell(src: str) -> None:
 
 
 def validate_cell(src: str) -> None:

-    """Check that cell does not already contain TransformerManager transformations.
+    """Check that cell does not already contain TransformerManager transformations,
+    or non-Python cell magics, which might cause tokenizer_rt to break because of
+    indentations.

 
     If a cell contains ``!ls``, then it'll be transformed to
     ``get_ipython().system('ls')``. However, if the cell originally contained
 
     If a cell contains ``!ls``, then it'll be transformed to
     ``get_ipython().system('ls')``. However, if the cell originally contained


@@ -959,6 +962,8 @@ def validate_cell(src: str) -> None:
     """
     if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS):
         raise NothingChanged
     """
     if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS):
         raise NothingChanged

+    if src[:2] == "%%" and src.split()[0][2:] not in PYTHON_CELL_MAGICS:
+        raise NothingChanged

 
 
 def format_cell(src: str, *, fast: bool, mode: Mode) -> str:
 
 
 def format_cell(src: str, *, fast: bool, mode: Mode) -> str:

diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py
index 8ae9d2e2e6ca8d9095338660fc914e4f54292709..a0ed56baafcf86781b8c06da06922a52d2b5308a 100644 (file)
--- a/src/black/handle_ipynb_magics.py
+++ b/src/black/handle_ipynb_magics.py
@@ -37,20 +37,15 @@ TOKENS_TO_IGNORE = frozenset(
         "ESCAPED_NL",
     )
 )
         "ESCAPED_NL",
     )
 )

-NON_PYTHON_CELL_MAGICS = frozenset(
+PYTHON_CELL_MAGICS = frozenset(

     (
     (

-        "bash",
-        "html",
-        "javascript",
-        "js",
-        "latex",
-        "markdown",
-        "perl",
-        "ruby",
-        "script",
-        "sh",
-        "svg",
-        "writefile",
+        "capture",
+        "prun",
+        "pypy",
+        "python",
+        "python3",
+        "time",
+        "timeit",

     )
 )
 TOKEN_HEX = secrets.token_hex
     )
 )
 TOKEN_HEX = secrets.token_hex


@@ -230,8 +225,6 @@ def replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]:
     cell_magic_finder.visit(tree)
     if cell_magic_finder.cell_magic is None:
         return src, replacements
     cell_magic_finder.visit(tree)
     if cell_magic_finder.cell_magic is None:
         return src, replacements

-    if cell_magic_finder.cell_magic.name in NON_PYTHON_CELL_MAGICS:
-        raise NothingChanged

     header = cell_magic_finder.cell_magic.header
     mask = get_token(src, header)
     replacements.append(Replacement(mask=mask, src=header))
     header = cell_magic_finder.cell_magic.header
     mask = get_token(src, header)
     replacements.append(Replacement(mask=mask, src=header))

diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py
index 5ecdf00b7bce8d50637fff3364ff9637ca84d206..141e865815a5c9f84aed85cf46e411cce6070879 100644 (file)
--- a/tests/test_ipynb.py
+++ b/tests/test_ipynb.py
@@ -106,6 +106,7 @@ def test_magic(src: str, expected: str) -> None:
     (
         "%%bash\n2+2",
         "%%html --isolated\n2+2",
     (
         "%%bash\n2+2",
         "%%html --isolated\n2+2",

+        "%%writefile e.txt\n  meh\n meh",

     ),
 )
 def test_non_python_magics(src: str) -> None:
     ),
 )
 def test_non_python_magics(src: str) -> None:


@@ -132,9 +133,9 @@ def test_magic_noop() -> None:
 
 
 def test_cell_magic_with_magic() -> None:
 
 
 def test_cell_magic_with_magic() -> None:

-    src = "%%t -n1\nls =!ls"
+    src = "%%timeit -n1\nls =!ls"

     result = format_cell(src, fast=True, mode=JUPYTER_MODE)
     result = format_cell(src, fast=True, mode=JUPYTER_MODE)

-    expected = "%%t -n1\nls = !ls"
+    expected = "%%timeit -n1\nls = !ls"

     assert result == expected
 
 
     assert result == expected