From 4da0851809e024760d3861ff43309125de34157a Mon Sep 17 00:00:00 2001 From: Antonio Ossa-Guerra Date: Thu, 6 Oct 2022 19:17:32 -0300 Subject: [PATCH] Add option to skip the first line of source code (#3299) * Add option to skip the first line in source file This commit adds a CLi option to skip the first line in the source files, just like the Cpython command line allows [1]. By enabling the flag, using `-x` or `--skip-source-first-line`, the first line is removed temporarilly while the remaining contents are formatted. The first line is added back before returning the formatted output. [1]: https://docs.python.org/dev/using/cmdline.html#cmdoption-x Signed-off-by: Antonio Ossa Guerra * Add tests for `--skip-source-first-line` option When the flag is disabled (default), black formats the entire source file, as in every line. In the other hand, if the flag is enabled, by using `-x` or `--skip-source-first-line`, the first line is retained while the rest of the source is formatted and then is added back. These tests use an empty Python file that contains invalid syntax in its first line (`invalid_header.py`, at `miscellaneous/`). First, Black is invoked without enabling the flag which should result in an exit code different than 0. When the flag is enabled, Black is expected to return a successful exit code and the header is expected to be retained (even if its not valid Python syntax). Signed-off-by: Antonio Ossa Guerra * Support skip source first line option for blackd The recently added option can be added as an acceptable header for blackd. The arguments are passed in such a way that using the new header will activate the skip source first line behaviour as expected Signed-off-by: Antonio Ossa Guerra * Add skip source first line option to blackd docs The new option can be passed to blackd as a header. This commit updates the blackd docs to include the new header. Signed-off-by: Antonio Ossa Guerra * Update CHANGES.md Include the new Black option to skip the first line of source code in the configuration section Signed-off-by: Antonio Ossa Guerra * Update skip first line test including valid syntax Including valid Python syntax help us make sure that the file is still actually valid after skipping the first line of the source file (which contains invalid Python syntax) Signed-off-by: Antonio Ossa Guerra * Skip first source line at `format_file_in_place` Instead of skipping the first source line at `format_file_contents`, do it before. This allow us to find the correct newline and encoding on the actual source code (everything that's after the header). This change is also applied at Blackd: take the header before passing the source to `format_file_contents` and put the header back once we get the formatted result. Signed-off-by: Antonio Ossa Guerra * Test output newlines when skipping first line When skipping the first line of source code, the reference newline must be taken from the second line of the file instead of the first one, in case that the file mixes more than one kind of newline character Signed-off-by: Antonio Ossa Guerra * Test that Blackd also skips first line correctly Simliarly to the Black tests, we first compare that Blackd fails when the first line is invalid Python syntax and then check that the result is the expected when tha flag is activated Signed-off-by: Antonio Ossa Guerra * Use the content encoding to decode the header When decoding the header to put it back at the top of the contents of the file, use the same encoding used in the content. This should be a better "guess" that using the default value Signed-off-by: Antonio Ossa Guerra --- CHANGES.md | 2 ++ .../black_as_a_server.md | 3 +++ src/black/__init__.py | 13 ++++++++++ src/black/mode.py | 2 ++ src/blackd/__init__.py | 16 +++++++++++++ tests/data/miscellaneous/invalid_header.py | 2 ++ tests/test_black.py | 24 +++++++++++++++++++ tests/test_blackd.py | 14 +++++++++++ 8 files changed, 76 insertions(+) create mode 100644 tests/data/miscellaneous/invalid_header.py diff --git a/CHANGES.md b/CHANGES.md index 4ff1816..ffdbd9c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,6 +28,8 @@ - `.ipynb_checkpoints` directories are now excluded by default (#3293) +- Add `--skip-source-first-line` / `-x` option to ignore the first line of source code + while formatting (#3299) ### Packaging diff --git a/docs/usage_and_configuration/black_as_a_server.md b/docs/usage_and_configuration/black_as_a_server.md index a2d4252..f24fb34 100644 --- a/docs/usage_and_configuration/black_as_a_server.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -50,6 +50,9 @@ is rejected with `HTTP 501` (Not Implemented). The headers controlling how source code is formatted are: - `X-Line-Length`: corresponds to the `--line-length` command line flag. +- `X-Skip-Source-First-Line`: corresponds to the `--skip-source-first-line` command line + flag. If present and its value is not an empty string, the first line of the source + code will be ignored. - `X-Skip-String-Normalization`: corresponds to the `--skip-string-normalization` command line flag. If present and its value is not the empty string, no string normalization will be performed. diff --git a/src/black/__init__.py b/src/black/__init__.py index 5b8c974..afd71e5 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -248,6 +248,12 @@ def validate_regex( ), default=[], ) +@click.option( + "-x", + "--skip-source-first-line", + is_flag=True, + help="Skip the first line of the source code.", +) @click.option( "-S", "--skip-string-normalization", @@ -428,6 +434,7 @@ def main( # noqa: C901 pyi: bool, ipynb: bool, python_cell_magics: Sequence[str], + skip_source_first_line: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, experimental_string_processing: bool, @@ -528,6 +535,7 @@ def main( # noqa: C901 line_length=line_length, is_pyi=pyi, is_ipynb=ipynb, + skip_source_first_line=skip_source_first_line, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, experimental_string_processing=experimental_string_processing, @@ -790,7 +798,10 @@ def format_file_in_place( mode = replace(mode, is_ipynb=True) then = datetime.utcfromtimestamp(src.stat().st_mtime) + header = b"" with open(src, "rb") as buf: + if mode.skip_source_first_line: + header = buf.readline() src_contents, encoding, newline = decode_bytes(buf.read()) try: dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) @@ -800,6 +811,8 @@ def format_file_in_place( raise ValueError( f"File '{src}' cannot be parsed as valid Jupyter notebook." ) from None + src_contents = header.decode(encoding) + src_contents + dst_contents = header.decode(encoding) + dst_contents if write_back == WriteBack.YES: with open(src, "w", encoding=encoding, newline=newline) as f: diff --git a/src/black/mode.py b/src/black/mode.py index 6c0847e..e3c3645 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -170,6 +170,7 @@ class Mode: string_normalization: bool = True is_pyi: bool = False is_ipynb: bool = False + skip_source_first_line: bool = False magic_trailing_comma: bool = True experimental_string_processing: bool = False python_cell_magics: Set[str] = field(default_factory=set) @@ -208,6 +209,7 @@ class Mode: str(int(self.string_normalization)), str(int(self.is_pyi)), str(int(self.is_ipynb)), + str(int(self.skip_source_first_line)), str(int(self.magic_trailing_comma)), str(int(self.experimental_string_processing)), str(int(self.preview)), diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 6bbc7c5..ba4750b 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -30,6 +30,7 @@ _stop_signal = asyncio.Event() PROTOCOL_VERSION_HEADER = "X-Protocol-Version" LINE_LENGTH_HEADER = "X-Line-Length" PYTHON_VARIANT_HEADER = "X-Python-Variant" +SKIP_SOURCE_FIRST_LINE = "X-Skip-Source-First-Line" SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization" SKIP_MAGIC_TRAILING_COMMA = "X-Skip-Magic-Trailing-Comma" PREVIEW = "X-Preview" @@ -40,6 +41,7 @@ BLACK_HEADERS = [ PROTOCOL_VERSION_HEADER, LINE_LENGTH_HEADER, PYTHON_VARIANT_HEADER, + SKIP_SOURCE_FIRST_LINE, SKIP_STRING_NORMALIZATION_HEADER, SKIP_MAGIC_TRAILING_COMMA, PREVIEW, @@ -111,6 +113,9 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: skip_magic_trailing_comma = bool( request.headers.get(SKIP_MAGIC_TRAILING_COMMA, False) ) + skip_source_first_line = bool( + request.headers.get(SKIP_SOURCE_FIRST_LINE, False) + ) preview = bool(request.headers.get(PREVIEW, False)) fast = False if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast": @@ -119,6 +124,7 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: target_versions=versions, is_pyi=pyi, line_length=line_length, + skip_source_first_line=skip_source_first_line, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, preview=preview, @@ -128,6 +134,12 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: req_str = req_bytes.decode(charset) then = datetime.utcnow() + header = "" + if skip_source_first_line: + first_newline_position: int = req_str.find("\n") + 1 + header = req_str[:first_newline_position] + req_str = req_str[first_newline_position:] + loop = asyncio.get_event_loop() formatted_str = await loop.run_in_executor( executor, partial(black.format_file_contents, req_str, fast=fast, mode=mode) @@ -140,6 +152,10 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: if formatted_str == req_str: raise black.NothingChanged + # Put the source first line back + req_str = header + req_str + formatted_str = header + formatted_str + # Only output the diff in the HTTP response only_diff = bool(request.headers.get(DIFF_HEADER, False)) if only_diff: diff --git a/tests/data/miscellaneous/invalid_header.py b/tests/data/miscellaneous/invalid_header.py new file mode 100644 index 0000000..fb49e2f --- /dev/null +++ b/tests/data/miscellaneous/invalid_header.py @@ -0,0 +1,2 @@ +This is not valid Python syntax +y = "This is valid syntax" diff --git a/tests/test_black.py b/tests/test_black.py index 96e6f1e..5d0175d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -341,6 +341,30 @@ class BlackTestCase(BlackBaseTestCase): black.assert_equivalent(source, not_normalized) black.assert_stable(source, not_normalized, mode=mode) + def test_skip_source_first_line(self) -> None: + source, _ = read_data("miscellaneous", "invalid_header") + tmp_file = Path(black.dump_to_file(source)) + # Full source should fail (invalid syntax at header) + self.invokeBlack([str(tmp_file), "--diff", "--check"], exit_code=123) + # So, skipping the first line should work + result = BlackRunner().invoke( + black.main, [str(tmp_file), "-x", f"--config={EMPTY_CONFIG}"] + ) + self.assertEqual(result.exit_code, 0) + with open(tmp_file, encoding="utf8") as f: + actual = f.read() + self.assertFormatEqual(source, actual) + + def test_skip_source_first_line_when_mixing_newlines(self) -> None: + code_mixing_newlines = b"Header will be skipped\r\ni = [1,2,3]\nj = [1,2,3]\n" + expected = b"Header will be skipped\r\ni = [1, 2, 3]\nj = [1, 2, 3]\n" + with TemporaryDirectory() as workspace: + test_file = Path(workspace) / "skip_header.py" + test_file.write_bytes(code_mixing_newlines) + mode = replace(DEFAULT_MODE, skip_source_first_line=True) + ff(test_file, mode=mode, write_back=black.WriteBack.YES) + self.assertEqual(test_file.read_bytes(), expected) + def test_skip_magic_trailing_comma(self) -> None: source, _ = read_data("simple_cases", "expression") expected, _ = read_data( diff --git a/tests/test_blackd.py b/tests/test_blackd.py index db9a165..5b6461f 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -177,6 +177,20 @@ class BlackDTestCase(AioHTTPTestCase): # type: ignore[misc] ) self.assertEqual(response.status, 400) + @unittest_run_loop + async def test_blackd_skip_first_source_line(self) -> None: + invalid_first_line = b"Header will be skipped\r\ni = [1,2,3]\nj = [1,2,3]\n" + expected_result = b"Header will be skipped\r\ni = [1, 2, 3]\nj = [1, 2, 3]\n" + response = await self.client.post("/", data=invalid_first_line) + self.assertEqual(response.status, 400) + response = await self.client.post( + "/", + data=invalid_first_line, + headers={blackd.SKIP_SOURCE_FIRST_LINE: "true"}, + ) + self.assertEqual(response.status, 200) + self.assertEqual(await response.read(), expected_result) + @unittest_run_loop async def test_blackd_preview(self) -> None: response = await self.client.post( -- 2.39.5