From a23f521fee1b283b10000d94267ef93069b7f7ec Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Feb 2021 17:37:25 +0000 Subject: [PATCH 01/16] Brevity: only use the variables required to convey the intended expressions --- src/black/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 44017b59..67eef953 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5971,7 +5971,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf elif leaf.type in CLOSING_BRACKETS: prev = line.leaves[index - 1] if index > 0 else None if ( - (line.should_explode or line.magic_trailing_comma) + line.magic_trailing_comma and prev and prev.type == token.COMMA and not is_one_tuple_between( @@ -5999,7 +5999,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf yield omit if ( - (line.should_explode or line.magic_trailing_comma) + line.magic_trailing_comma and prev and prev.type == token.COMMA and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves) @@ -6659,9 +6659,7 @@ def can_omit_invisible_parens( # unnecessary. return True - if ( - line.should_explode or line.magic_trailing_comma - ) and penultimate.type == token.COMMA: + if line.magic_trailing_comma and penultimate.type == token.COMMA: # The rightmost non-omitted bracket pair is the one we want to explode on. return True -- 2.39.5 From 51141f1af4aab0e0c7f71932ffe06482a884f1d5 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Feb 2021 17:38:55 +0000 Subject: [PATCH 02/16] Consistency: use variable names that correspond to the methods they invoke --- src/black/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 67eef953..c836b2b4 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1480,7 +1480,7 @@ class Line: comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict) bracket_tracker: BracketTracker = field(default_factory=BracketTracker) inside_brackets: bool = False - should_explode: bool = False + should_split: bool = False magic_trailing_comma: Optional[Leaf] = None def append(self, leaf: Leaf, preformatted: bool = False) -> None: @@ -1792,7 +1792,7 @@ class Line: mode=self.mode, depth=self.depth, inside_brackets=self.inside_brackets, - should_explode=self.should_explode, + should_split=self.should_split, magic_trailing_comma=self.magic_trailing_comma, ) @@ -2712,7 +2712,7 @@ def transform_line( transformers: List[Transformer] if ( not line.contains_uncollapsable_type_comments() - and not (line.should_explode or line.magic_trailing_comma) + and not (line.should_split or line.magic_trailing_comma) and ( is_line_short_enough(line, line_length=mode.line_length, line_str=line_str) or line.contains_unsplittable_type_ignore() @@ -4386,7 +4386,7 @@ class StringParenWrapper(CustomSplitMapMixin, BaseStringSplitter): mode=line.mode, depth=line.depth + 1, inside_brackets=True, - should_explode=line.should_explode, + should_split=line.should_split, magic_trailing_comma=line.magic_trailing_comma, ) string_leaf = Leaf(token.STRING, string_value) @@ -5009,7 +5009,7 @@ def bracket_split_build_line( for comment_after in original.comments_after(leaf): result.append(comment_after, preformatted=True) if is_body and should_split(result, opening_bracket): - result.should_explode = True + result.should_split = True return result @@ -5949,7 +5949,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf """ omit: Set[LeafID] = set() - if not line.should_explode and not line.magic_trailing_comma: + if not line.should_split and not line.magic_trailing_comma: yield omit length = 4 * line.depth @@ -6632,7 +6632,7 @@ def can_omit_invisible_parens( penultimate = line.leaves[-2] last = line.leaves[-1] - if line.should_explode or line.magic_trailing_comma: + if line.should_split or line.magic_trailing_comma: try: penultimate, last = last_two_except(line.leaves, omit=omit_on_explode) except LookupError: -- 2.39.5 From 89c42a0011721e164738c17a192452ee77d9cc99 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Feb 2021 17:40:42 +0000 Subject: [PATCH 03/16] Clarity: special case: avoid using variables that have the same names as methods --- src/black/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index c836b2b4..7f727b18 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5008,7 +5008,7 @@ def bracket_split_build_line( result.append(leaf, preformatted=True) for comment_after in original.comments_after(leaf): result.append(comment_after, preformatted=True) - if is_body and should_split(result, opening_bracket): + if is_body and should_split_line(result, opening_bracket): result.should_split = True return result @@ -5813,7 +5813,7 @@ def ensure_visible(leaf: Leaf) -> None: leaf.value = ")" -def should_split(line: Line, opening_bracket: Leaf) -> bool: +def should_split_line(line: Line, opening_bracket: Leaf) -> bool: """Should `line` be immediately split with `delimiter_split()` after RHS?""" if not (opening_bracket.parent and opening_bracket.value in "[{("): -- 2.39.5 From 829331a8777aa758c6fc2a5032d38d8fbb5b5ac6 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Feb 2021 17:45:45 +0000 Subject: [PATCH 04/16] Simplification: only use special-case token retrieval logic when magic trailing comma is present --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 7f727b18..56f180ac 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -6632,7 +6632,7 @@ def can_omit_invisible_parens( penultimate = line.leaves[-2] last = line.leaves[-1] - if line.should_split or line.magic_trailing_comma: + if line.magic_trailing_comma: try: penultimate, last = last_two_except(line.leaves, omit=omit_on_explode) except LookupError: -- 2.39.5 From e0d766727dc87f5c5f39ef751d0bf23fc5ff31a0 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Feb 2021 17:48:38 +0000 Subject: [PATCH 05/16] Simplification: only yield empty omit list when magic trailing comma is present --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 56f180ac..3eea33ce 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5949,7 +5949,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf """ omit: Set[LeafID] = set() - if not line.should_split and not line.magic_trailing_comma: + if not line.magic_trailing_comma: yield omit length = 4 * line.depth -- 2.39.5 From 24700806f681f2809a2b85999871e291c36dd948 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Feb 2021 18:07:43 +0000 Subject: [PATCH 06/16] Cleanup: remove unused / redundant variables from conditionals --- src/black/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 3eea33ce..c2a4bda6 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5971,8 +5971,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf elif leaf.type in CLOSING_BRACKETS: prev = line.leaves[index - 1] if index > 0 else None if ( - line.magic_trailing_comma - and prev + prev and prev.type == token.COMMA and not is_one_tuple_between( leaf.opening_bracket, leaf, line.leaves @@ -5999,8 +5998,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf yield omit if ( - line.magic_trailing_comma - and prev + prev and prev.type == token.COMMA and not is_one_tuple_between(leaf.opening_bracket, leaf, line.leaves) ): @@ -6659,7 +6657,7 @@ def can_omit_invisible_parens( # unnecessary. return True - if line.magic_trailing_comma and penultimate.type == token.COMMA: + if penultimate.type == token.COMMA: # The rightmost non-omitted bracket pair is the one we want to explode on. return True -- 2.39.5 From 22127c633eba10d41519fb562c1252f859e2d7fa Mon Sep 17 00:00:00 2001 From: James Addison Date: Tue, 9 Feb 2021 21:13:57 +0000 Subject: [PATCH 07/16] Readability: reduce boolean nesting --- src/black/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index c2a4bda6..c2b0ad43 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -2712,7 +2712,8 @@ def transform_line( transformers: List[Transformer] if ( not line.contains_uncollapsable_type_comments() - and not (line.should_split or line.magic_trailing_comma) + and not line.should_split + and not line.magic_trailing_comma and ( is_line_short_enough(line, line_length=mode.line_length, line_str=line_str) or line.contains_unsplittable_type_ignore() -- 2.39.5 From 0cbe19c813559d6642e71832242264ab8d5a5d59 Mon Sep 17 00:00:00 2001 From: James Addison Date: Wed, 10 Feb 2021 12:33:50 +0000 Subject: [PATCH 08/16] Minimize changes: more closely resemble original conditional logic --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index c2b0ad43..0b734adc 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -6658,7 +6658,7 @@ def can_omit_invisible_parens( # unnecessary. return True - if penultimate.type == token.COMMA: + if line.magic_trailing_comma and penultimate.type == token.COMMA: # The rightmost non-omitted bracket pair is the one we want to explode on. return True -- 2.39.5 From cd4295dd9888f491cfd9aae060d7832b7f831b24 Mon Sep 17 00:00:00 2001 From: "Paul \"TBBle\" Hampson" Date: Mon, 22 Feb 2021 17:43:23 +1100 Subject: [PATCH 09/16] Indicate that a final newline was added in --diff (#1897) (#1897) Fixes: #1662 Work-around for https://bugs.python.org/issue2142 The test has to slightly mess with its input data, because the utility functions default to ensuring the test data has a final newline, which defeats the point of the test. Signed-off-by: Paul "TBBle" Hampson --- CHANGES.md | 2 ++ docs/authors.md | 1 + src/black/__init__.py | 23 ++++++++++++++++------- tests/data/missing_final_newline.diff | 8 ++++++++ tests/data/missing_final_newline.py | 3 +++ tests/test_black.py | 23 ++++++++++++++++++++++- 6 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 tests/data/missing_final_newline.diff create mode 100644 tests/data/missing_final_newline.py diff --git a/CHANGES.md b/CHANGES.md index 5f7ca4f4..3fd7ad40 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,6 +28,8 @@ - speed up caching by avoiding pathlib (#1950) +- `--diff` correctly indicates when a file doesn't end in a newline (#1662) + #### _Packaging_ - Self-contained native _Black_ binaries are now provided for releases via GitHub diff --git a/docs/authors.md b/docs/authors.md index cdf5046c..9f2ea057 100644 --- a/docs/authors.md +++ b/docs/authors.md @@ -132,6 +132,7 @@ Multiple contributions by: - [Pablo Galindo](mailto:Pablogsal@gmail.com) - [Paul Ganssle](mailto:p.ganssle@gmail.com) - [Paul Meinhardt](mailto:mnhrdt@gmail.com) +- [Paul "TBBle" Hampson](mailto:Paul.Hampson@Pobox.com) - [Peter Bengtsson](mailto:mail@peterbe.com) - [Peter Grayson](mailto:pete@jpgrayson.net) - [Peter Stensmyr](mailto:peter.stensmyr@gmail.com) diff --git a/src/black/__init__.py b/src/black/__init__.py index 0b734adc..c1907d9b 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -6425,14 +6425,14 @@ def assert_stable(src: str, dst: str, mode: Mode) -> None: @mypyc_attr(patchable=True) -def dump_to_file(*output: str) -> str: +def dump_to_file(*output: str, ensure_final_newline: bool = True) -> str: """Dump `output` to a temporary file. Return path to the file.""" with tempfile.NamedTemporaryFile( mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8" ) as f: for lines in output: f.write(lines) - if lines and lines[-1] != "\n": + if ensure_final_newline and lines and lines[-1] != "\n": f.write("\n") return f.name @@ -6450,11 +6450,20 @@ def diff(a: str, b: str, a_name: str, b_name: str) -> str: """Return a unified diff string between strings `a` and `b`.""" import difflib - a_lines = [line + "\n" for line in a.splitlines()] - b_lines = [line + "\n" for line in b.splitlines()] - return "".join( - difflib.unified_diff(a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5) - ) + a_lines = [line for line in a.splitlines(keepends=True)] + b_lines = [line for line in b.splitlines(keepends=True)] + diff_lines = [] + for line in difflib.unified_diff( + a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5 + ): + # Work around https://bugs.python.org/issue2142 + # See https://www.gnu.org/software/diffutils/manual/html_node/Incomplete-Lines.html + if line[-1] == "\n": + diff_lines.append(line) + else: + diff_lines.append(line + "\n") + diff_lines.append("\\ No newline at end of file\n") + return "".join(diff_lines) def cancel(tasks: Iterable["asyncio.Task[Any]"]) -> None: diff --git a/tests/data/missing_final_newline.diff b/tests/data/missing_final_newline.diff new file mode 100644 index 00000000..6d991c74 --- /dev/null +++ b/tests/data/missing_final_newline.diff @@ -0,0 +1,8 @@ +--- [Deterministic header] ++++ [Deterministic header] +@@ -1,3 +1,3 @@ + # A comment-only file, with no final EOL character + # This triggers https://bugs.python.org/issue2142 +-# This is the line without the EOL character +\ No newline at end of file ++# This is the line without the EOL character diff --git a/tests/data/missing_final_newline.py b/tests/data/missing_final_newline.py new file mode 100644 index 00000000..687e1367 --- /dev/null +++ b/tests/data/missing_final_newline.py @@ -0,0 +1,3 @@ +# A comment-only file, with no final EOL character +# This triggers https://bugs.python.org/issue2142 +# This is the line without the EOL character \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index a2efef82..5d14ceda 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -300,7 +300,6 @@ class BlackTestCase(BlackBaseTestCase): os.unlink(tmp_file) actual = result.output actual = diff_header.sub(DETERMINISTIC_HEADER, actual) - actual = actual.rstrip() + "\n" # the diff output has a trailing space if expected != actual: dump = black.dump_to_file(actual) msg = ( @@ -1798,6 +1797,28 @@ class BlackTestCase(BlackBaseTestCase): output = black.format_str(source, mode=DEFAULT_MODE) black.assert_stable(source, output, mode=DEFAULT_MODE) + def test_bpo_2142_workaround(self) -> None: + + # https://bugs.python.org/issue2142 + + source, _ = read_data("missing_final_newline.py") + # read_data adds a trailing newline + source = source.rstrip() + expected, _ = read_data("missing_final_newline.diff") + tmp_file = Path(black.dump_to_file(source, ensure_final_newline=False)) + diff_header = re.compile( + rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + ) + try: + result = BlackRunner().invoke(black.main, ["--diff", str(tmp_file)]) + self.assertEqual(result.exit_code, 0) + finally: + os.unlink(tmp_file) + actual = result.output + actual = diff_header.sub(DETERMINISTIC_HEADER, actual) + self.assertEqual(actual, expected) + with open(black.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() -- 2.39.5 From 24e8dad575f2ae373e64b583f6ad103cf9193781 Mon Sep 17 00:00:00 2001 From: James <50501825+Gobot1234@users.noreply.github.com> Date: Mon, 22 Feb 2021 15:42:05 +0000 Subject: [PATCH 10/16] Fix for enum changes in 3.10 (#1999) --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index c1907d9b..e09f08df 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -883,7 +883,7 @@ def format_file_in_place( dst_name = f"{src}\t{now} +0000" diff_contents = diff(src_contents, dst_contents, src_name, dst_name) - if write_back == write_back.COLOR_DIFF: + if write_back == WriteBack.COLOR_DIFF: diff_contents = color_diff(diff_contents) with lock or nullcontext(): -- 2.39.5 From fe4a9d6bee21c471139e07fa27b464187477556c Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 22 Feb 2021 15:46:38 +0000 Subject: [PATCH 11/16] Fixup: update function name in docs to match source (#1997) --- docs/reference/reference_functions.rst | 2 +- src/black/__init__.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/reference_functions.rst b/docs/reference/reference_functions.rst index a7184115..bffce7e9 100644 --- a/docs/reference/reference_functions.rst +++ b/docs/reference/reference_functions.rst @@ -171,7 +171,7 @@ Utilities .. autofunction:: black.re_compile_maybe_verbose -.. autofunction:: black.should_split_body_explode +.. autofunction:: black.should_split_line .. autofunction:: black.shutdown diff --git a/src/black/__init__.py b/src/black/__init__.py index e09f08df..07251d55 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1480,7 +1480,7 @@ class Line: comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict) bracket_tracker: BracketTracker = field(default_factory=BracketTracker) inside_brackets: bool = False - should_split: bool = False + should_split_rhs: bool = False magic_trailing_comma: Optional[Leaf] = None def append(self, leaf: Leaf, preformatted: bool = False) -> None: @@ -1792,7 +1792,7 @@ class Line: mode=self.mode, depth=self.depth, inside_brackets=self.inside_brackets, - should_split=self.should_split, + should_split_rhs=self.should_split_rhs, magic_trailing_comma=self.magic_trailing_comma, ) @@ -2712,7 +2712,7 @@ def transform_line( transformers: List[Transformer] if ( not line.contains_uncollapsable_type_comments() - and not line.should_split + and not line.should_split_rhs and not line.magic_trailing_comma and ( is_line_short_enough(line, line_length=mode.line_length, line_str=line_str) @@ -4387,7 +4387,7 @@ class StringParenWrapper(CustomSplitMapMixin, BaseStringSplitter): mode=line.mode, depth=line.depth + 1, inside_brackets=True, - should_split=line.should_split, + should_split_rhs=line.should_split_rhs, magic_trailing_comma=line.magic_trailing_comma, ) string_leaf = Leaf(token.STRING, string_value) @@ -5010,7 +5010,7 @@ def bracket_split_build_line( for comment_after in original.comments_after(leaf): result.append(comment_after, preformatted=True) if is_body and should_split_line(result, opening_bracket): - result.should_split = True + result.should_split_rhs = True return result -- 2.39.5 From e1c86f987eca7e532f7d69f7ff4b9c70432fabbf Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 22 Feb 2021 15:49:38 +0000 Subject: [PATCH 12/16] Fuzzer testing: less strict special-case regex match passthrough for multi-line EOF exceptions (#1998) --- fuzz.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzz.py b/fuzz.py index 6e206869..a9ca8eff 100644 --- a/fuzz.py +++ b/fuzz.py @@ -50,9 +50,9 @@ def test_idempotent_any_syntatically_valid_python( # TODO: remove this try-except block when issues are resolved. return except TokenError as e: - if ( + if ( # Special-case logic for backslashes followed by newlines or end-of-input e.args[0] == "EOF in multi-line statement" - and re.search(r"\r?\n\\\r?\n", src_contents) is not None + and re.search(r"\\($|\r?\n)", src_contents) is not None ): # This is a bug - if it's valid Python code, as above, Black should be # able to cope with it. See issue #1012. -- 2.39.5 From b06cd15666e5d766347cda0434dc6c828a96c74a Mon Sep 17 00:00:00 2001 From: tpilewicz <31728184+tpilewicz@users.noreply.github.com> Date: Wed, 24 Feb 2021 12:56:56 +0100 Subject: [PATCH 13/16] Wrap arithmetic and binary arithmetic expressions in invisible parentheses (#2001) --- src/black/__init__.py | 13 ++++++++ tests/data/expression.diff | 30 +++++++++++++++++-- tests/data/expression.py | 24 +++++++++++++++ .../expression_skip_magic_trailing_comma.diff | 30 +++++++++++++++++-- 4 files changed, 91 insertions(+), 6 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 07251d55..69194686 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -2049,6 +2049,8 @@ class LineGenerator(Visitor[Line]): def visit_simple_stmt(self, node: Node) -> Iterator[Line]: """Visit a statement without nested statements.""" + if first_child_is_arith(node): + wrap_in_parentheses(node, node.children[0], visible=False) is_suite_like = node.parent and node.parent.type in STATEMENT if is_suite_like: if self.mode.is_pyi and is_stub_body(node): @@ -5613,6 +5615,17 @@ def unwrap_singleton_parenthesis(node: LN) -> Optional[LN]: return wrapped +def first_child_is_arith(node: Node) -> bool: + """Whether first child is an arithmetic or a binary arithmetic expression""" + expr_types = { + syms.arith_expr, + syms.shift_expr, + syms.xor_expr, + syms.and_expr, + } + return bool(node.children and node.children[0].type in expr_types) + + def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: """Wrap `child` in parentheses. diff --git a/tests/data/expression.diff b/tests/data/expression.diff index 684f92cd..721a07d2 100644 --- a/tests/data/expression.diff +++ b/tests/data/expression.diff @@ -166,7 +166,7 @@ slice[0:1:2] slice[:] slice[:-1] -@@ -137,113 +173,180 @@ +@@ -137,118 +173,199 @@ numpy[-(c + 1) :, d] numpy[:, l[-2]] numpy[:, ::-1] @@ -346,6 +346,9 @@ - return True -if ( - ~ aaaaaaaaaaaaaaaa.a + aaaaaaaaaaaaaaaa.b - aaaaaaaaaaaaaaaa.c * aaaaaaaaaaaaaaaa.d @ aaaaaaaaaaaaaaaa.e | aaaaaaaaaaaaaaaa.f & aaaaaaaaaaaaaaaa.g % aaaaaaaaaaaaaaaa.h ^ aaaaaaaaaaaaaaaa.i << aaaaaaaaaaaaaaaa.k >> aaaaaaaaaaaaaaaa.l ** aaaaaaaaaaaaaaaa.m // aaaaaaaaaaaaaaaa.n +-): +- return True +-aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa - aaaaaaaaaaaaaaaa * (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) / (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) +a = ( + aaaa.bbbb.cccc.dddd.eeee.ffff.gggg.hhhh.iiii.jjjj.kkkk.llll.mmmm.nnnn.oooo.pppp + in qqqq.rrrr.ssss.tttt.uuuu.vvvv.xxxx.yyyy.zzzz @@ -417,7 +420,28 @@ + ^ aaaaaaaaaaaaaaaa.i + << aaaaaaaaaaaaaaaa.k + >> aaaaaaaaaaaaaaaa.l ** aaaaaaaaaaaaaaaa.m // aaaaaaaaaaaaaaaa.n - ): - return True ++): ++ return True ++( ++ aaaaaaaaaaaaaaaa ++ + aaaaaaaaaaaaaaaa ++ - aaaaaaaaaaaaaaaa ++ * (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) ++ / (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) ++) + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa +-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa << aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++( ++ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ >> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ << aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++) + bbbb >> bbbb * bbbb +-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ^bbbb.a & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa^aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++( ++ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ ^ bbbb.a & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ ^ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++) last_call() # standalone comment at ENDMARKER diff --git a/tests/data/expression.py b/tests/data/expression.py index 8e63bdcd..d13450cd 100644 --- a/tests/data/expression.py +++ b/tests/data/expression.py @@ -245,6 +245,11 @@ if ( ~ aaaaaaaaaaaaaaaa.a + aaaaaaaaaaaaaaaa.b - aaaaaaaaaaaaaaaa.c * aaaaaaaaaaaaaaaa.d @ aaaaaaaaaaaaaaaa.e | aaaaaaaaaaaaaaaa.f & aaaaaaaaaaaaaaaa.g % aaaaaaaaaaaaaaaa.h ^ aaaaaaaaaaaaaaaa.i << aaaaaaaaaaaaaaaa.k >> aaaaaaaaaaaaaaaa.l ** aaaaaaaaaaaaaaaa.m // aaaaaaaaaaaaaaaa.n ): return True +aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa - aaaaaaaaaaaaaaaa * (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) / (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) +aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa << aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +bbbb >> bbbb * bbbb +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ^bbbb.a & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa^aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa last_call() # standalone comment at ENDMARKER @@ -602,5 +607,24 @@ if ( >> aaaaaaaaaaaaaaaa.l ** aaaaaaaaaaaaaaaa.m // aaaaaaaaaaaaaaaa.n ): return True +( + aaaaaaaaaaaaaaaa + + aaaaaaaaaaaaaaaa + - aaaaaaaaaaaaaaaa + * (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) + / (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) +) +aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa +( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + >> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + << aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +) +bbbb >> bbbb * bbbb +( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ^ bbbb.a & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ^ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +) last_call() # standalone comment at ENDMARKER diff --git a/tests/data/expression_skip_magic_trailing_comma.diff b/tests/data/expression_skip_magic_trailing_comma.diff index 8a0225bc..4a8a95c7 100644 --- a/tests/data/expression_skip_magic_trailing_comma.diff +++ b/tests/data/expression_skip_magic_trailing_comma.diff @@ -149,7 +149,7 @@ slice[0:1:2] slice[:] slice[:-1] -@@ -137,113 +156,178 @@ +@@ -137,118 +156,197 @@ numpy[-(c + 1) :, d] numpy[:, l[-2]] numpy[:, ::-1] @@ -327,6 +327,9 @@ - return True -if ( - ~ aaaaaaaaaaaaaaaa.a + aaaaaaaaaaaaaaaa.b - aaaaaaaaaaaaaaaa.c * aaaaaaaaaaaaaaaa.d @ aaaaaaaaaaaaaaaa.e | aaaaaaaaaaaaaaaa.f & aaaaaaaaaaaaaaaa.g % aaaaaaaaaaaaaaaa.h ^ aaaaaaaaaaaaaaaa.i << aaaaaaaaaaaaaaaa.k >> aaaaaaaaaaaaaaaa.l ** aaaaaaaaaaaaaaaa.m // aaaaaaaaaaaaaaaa.n +-): +- return True +-aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa - aaaaaaaaaaaaaaaa * (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) / (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) +a = ( + aaaa.bbbb.cccc.dddd.eeee.ffff.gggg.hhhh.iiii.jjjj.kkkk.llll.mmmm.nnnn.oooo.pppp + in qqqq.rrrr.ssss.tttt.uuuu.vvvv.xxxx.yyyy.zzzz @@ -398,7 +401,28 @@ + ^ aaaaaaaaaaaaaaaa.i + << aaaaaaaaaaaaaaaa.k + >> aaaaaaaaaaaaaaaa.l ** aaaaaaaaaaaaaaaa.m // aaaaaaaaaaaaaaaa.n - ): - return True ++): ++ return True ++( ++ aaaaaaaaaaaaaaaa ++ + aaaaaaaaaaaaaaaa ++ - aaaaaaaaaaaaaaaa ++ * (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) ++ / (aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa) ++) + aaaaaaaaaaaaaaaa + aaaaaaaaaaaaaaaa +-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa << aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++( ++ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ >> aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ << aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++) + bbbb >> bbbb * bbbb +-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ^bbbb.a & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa^aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++( ++ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ ^ bbbb.a & aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++ ^ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ++) last_call() # standalone comment at ENDMARKER -- 2.39.5 From 858225d34dc49ca353f9e573dd82dd5845766115 Mon Sep 17 00:00:00 2001 From: Rishav Kundu Date: Sun, 28 Feb 2021 06:50:23 +0530 Subject: [PATCH 14/16] Strip redundant parentheses from assignment exprs (#1906) Fixes #1656 --- src/black/__init__.py | 10 +++++--- tests/data/pep_572.py | 4 +-- tests/data/pep_572_remove_parens.py | 38 +++++++++++++++++++++++++++++ tests/data/remove_parens.py | 2 -- tests/test_black.py | 9 +++++++ 5 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 tests/data/pep_572_remove_parens.py diff --git a/src/black/__init__.py b/src/black/__init__.py index 69194686..a1d16d9b 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5370,10 +5370,7 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None: check_lpar = True if check_lpar: - if is_walrus_assignment(child): - pass - - elif child.type == syms.atom: + if child.type == syms.atom: if maybe_make_parens_invisible_in_atom(child, parent=node): wrap_in_parentheses(node, child, visible=False) elif is_one_tuple(child): @@ -5545,6 +5542,7 @@ def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool: Returns whether the node should itself be wrapped in invisible parentheses. """ + if ( node.type != syms.atom or is_empty_tuple(node) @@ -5554,6 +5552,10 @@ def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool: ): return False + if is_walrus_assignment(node): + if parent.type in [syms.annassign, syms.expr_stmt]: + return False + first = node.children[0] last = node.children[-1] if first.type == token.LPAR and last.type == token.RPAR: diff --git a/tests/data/pep_572.py b/tests/data/pep_572.py index 637b3bb3..c6867f26 100644 --- a/tests/data/pep_572.py +++ b/tests/data/pep_572.py @@ -2,7 +2,7 @@ (a := a) if (match := pattern.search(data)) is None: pass -if (match := pattern.search(data)): +if match := pattern.search(data): pass [y := f(x), y ** 2, y ** 3] filtered_data = [y for x in data if (y := f(x)) is None] @@ -43,5 +43,5 @@ foo(c=(b := 2), a=1) while x := f(x): pass -while (x := f(x)): +while x := f(x): pass diff --git a/tests/data/pep_572_remove_parens.py b/tests/data/pep_572_remove_parens.py new file mode 100644 index 00000000..04cc75bc --- /dev/null +++ b/tests/data/pep_572_remove_parens.py @@ -0,0 +1,38 @@ +if (foo := 0): + pass + +if (foo := 1): + pass + +if (y := 5 + 5): + pass + +y = (x := 0) + +y += (x := 0) + +(y := 5 + 5) + +test: int = (test2 := 2) + +a, b = (test := (1, 2)) + +# output +if foo := 0: + pass + +if foo := 1: + pass + +if y := 5 + 5: + pass + +y = (x := 0) + +y += (x := 0) + +(y := 5 + 5) + +test: int = (test2 := 2) + +a, b = (test := (1, 2)) diff --git a/tests/data/remove_parens.py b/tests/data/remove_parens.py index afc34010..abd5f71f 100644 --- a/tests/data/remove_parens.py +++ b/tests/data/remove_parens.py @@ -54,7 +54,6 @@ def example7(): def example8(): return (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((None))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) - # output x = 1 x = 1.2 @@ -141,4 +140,3 @@ def example7(): def example8(): return None - diff --git a/tests/test_black.py b/tests/test_black.py index 5d14ceda..9c3cc643 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -263,6 +263,15 @@ class BlackTestCase(BlackBaseTestCase): if sys.version_info >= (3, 8): black.assert_equivalent(source, actual) + @patch("black.dump_to_file", dump_to_stderr) + def test_pep_572_remove_parens(self) -> None: + source, expected = read_data("pep_572_remove_parens") + actual = fs(source) + self.assertFormatEqual(expected, actual) + black.assert_stable(source, actual, DEFAULT_MODE) + if sys.version_info >= (3, 8): + black.assert_equivalent(source, actual) + def test_pep_572_version_detection(self) -> None: source, _ = read_data("pep_572") root = black.lib2to3_parse(source) -- 2.39.5 From beecd6fd0a9103aa91b1019dcf8fc774b667ea6c Mon Sep 17 00:00:00 2001 From: Joshua Cannon Date: Mon, 1 Mar 2021 16:07:36 -0600 Subject: [PATCH 15/16] Add --extend-exclude parameter (#2005) Look ma! I contribute to open source! Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- README.md | 30 +++++------- docs/change_log.md | 2 + docs/installation_and_usage.md | 7 ++- docs/pyproject_toml.md | 22 ++------- pyproject.toml | 13 +---- src/black/__init__.py | 78 ++++++++++++++++++++---------- tests/test_black.py | 86 ++++++++++++++++++++++++++++------ 7 files changed, 148 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 178f763c..5f8b52cb 100644 --- a/README.md +++ b/README.md @@ -135,11 +135,17 @@ Options: hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_bu ild|buck-out|build|dist)/] + --extend-exclude TEXT Like --exclude, but adds additional files + and directories on top of the excluded + ones (useful if you simply want to add to + the default). + --force-exclude TEXT Like --exclude, but files and directories matching this regex will be excluded even when they are passed explicitly as arguments. + --stdin-filename TEXT The name of the file when passing it through stdin. Useful to make sure Black will respect --force-exclude option on some @@ -151,7 +157,7 @@ Options: -v, --verbose Also emit messages to stderr about files that were not changed or were ignored due to - --exclude=. + exclusion patterns. --version Show the version and exit. --config FILE Read configuration from FILE path. @@ -263,7 +269,7 @@ above. What seems like a bug might be intended behaviour. _Black_ is able to read project-specific default values for its command line options from a `pyproject.toml` file. This is especially useful for specifying custom -`--include` and `--exclude` patterns for your project. +`--include` and `--exclude`/`--extend-exclude` patterns for your project. **Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is "No". _Black_ is all about sensible defaults. @@ -313,25 +319,10 @@ expressions by Black. Use `[ ]` to denote a significant space character. line-length = 88 target-version = ['py37'] include = '\.pyi?$' -exclude = ''' +extend-exclude = ''' # A regex preceded with ^/ will apply only to files and directories # in the root of the project. -^/( - ( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - )/ - | foo.py # also separately exclude a file named foo.py in - # the root of the project -) +^/foo.py # exclude a file named foo.py in the root of the project (in addition to the defaults) ''' ``` @@ -616,6 +607,7 @@ Multiple contributions by: - [Joseph Larson](mailto:larson.joseph@gmail.com) - [Josh Bode](mailto:joshbode@fastmail.com) - [Josh Holland](mailto:anowlcalledjosh@gmail.com) +- [Joshua Cannon](mailto:joshdcannon@gmail.com) - [José Padilla](mailto:jpadilla@webapplicate.com) - [Juan Luis Cano Rodríguez](mailto:hello@juanlu.space) - [kaiix](mailto:kvn.hou@gmail.com) diff --git a/docs/change_log.md b/docs/change_log.md index 066be76c..01c27553 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -28,6 +28,8 @@ - use lowercase hex strings (#1692) +- added `--extend-exclude` argument (#1571) + #### _Packaging_ - Self-contained native _Black_ binaries are now provided for releases via GitHub diff --git a/docs/installation_and_usage.md b/docs/installation_and_usage.md index ee45c934..bb554eb6 100644 --- a/docs/installation_and_usage.md +++ b/docs/installation_and_usage.md @@ -95,6 +95,11 @@ Options: when they are passed explicitly as arguments. + --extend-exclude TEXT Like --exclude, but adds additional files + and directories on top of the excluded + ones. (useful if you simply want to add to + the default) + --stdin-filename TEXT The name of the file when passing it through stdin. Useful to make sure Black will respect --force-exclude option on some @@ -106,7 +111,7 @@ Options: -v, --verbose Also emit messages to stderr about files that were not changed or were ignored due to - --exclude=. + exclusion patterns. --version Show the version and exit. --config FILE Read configuration from FILE path. diff --git a/docs/pyproject_toml.md b/docs/pyproject_toml.md index 453f533b..9acc4c03 100644 --- a/docs/pyproject_toml.md +++ b/docs/pyproject_toml.md @@ -4,7 +4,8 @@ _Black_ is able to read project-specific default values for its command line options from a `pyproject.toml` file. This is especially useful for specifying custom -`--include` and `--exclude` patterns for your project. +`--include` and `--exclude`/`--force-exclude`/`--extend-exclude` patterns for your +project. **Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is "No". _Black_ is all about sensible defaults. @@ -54,25 +55,10 @@ expressions by Black. Use `[ ]` to denote a significant space character. line-length = 88 target-version = ['py37'] include = '\.pyi?$' -exclude = ''' +extend-exclude = ''' # A regex preceded with ^/ will apply only to files and directories # in the root of the project. -^/( - ( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - )/ - | foo.py # also separately exclude a file named foo.py in - # the root of the project -) +^/foo.py # exclude a file named foo.py in the root of the project (in addition to the defaults) ''' ``` diff --git a/pyproject.toml b/pyproject.toml index 9d4da0bf..7f632f28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,19 +9,8 @@ line-length = 88 target-version = ['py36', 'py37', 'py38'] include = '\.pyi?$' -exclude = ''' +extend-exclude = ''' /( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - # The following are specific to Black, you probably don't want those. | blib2to3 | tests/data diff --git a/src/black/__init__.py b/src/black/__init__.py index a1d16d9b..e21e2af5 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -461,6 +461,14 @@ def target_version_option_callback( ), show_default=True, ) +@click.option( + "--extend-exclude", + type=str, + help=( + "Like --exclude, but adds additional files and directories on top of the" + " excluded ones. (Useful if you simply want to add to the default)" + ), +) @click.option( "--force-exclude", type=str, @@ -493,7 +501,7 @@ def target_version_option_callback( is_flag=True, help=( "Also emit messages to stderr about files that were not changed or were ignored" - " due to --exclude=." + " due to exclusion patterns." ), ) @click.version_option(version=__version__) @@ -537,6 +545,7 @@ def main( verbose: bool, include: str, exclude: str, + extend_exclude: Optional[str], force_exclude: Optional[str], stdin_filename: Optional[str], src: Tuple[str, ...], @@ -570,6 +579,7 @@ def main( verbose=verbose, include=include, exclude=exclude, + extend_exclude=extend_exclude, force_exclude=force_exclude, report=report, stdin_filename=stdin_filename, @@ -602,6 +612,18 @@ def main( ctx.exit(report.return_code) +def test_regex( + ctx: click.Context, + regex_name: str, + regex: Optional[str], +) -> Optional[Pattern]: + try: + return re_compile_maybe_verbose(regex) if regex is not None else None + except re.error: + err(f"Invalid regular expression for {regex_name} given: {regex!r}") + ctx.exit(2) + + def get_sources( *, ctx: click.Context, @@ -610,28 +632,18 @@ def get_sources( verbose: bool, include: str, exclude: str, + extend_exclude: Optional[str], force_exclude: Optional[str], report: "Report", stdin_filename: Optional[str], ) -> Set[Path]: """Compute the set of files to be formatted.""" - try: - include_regex = re_compile_maybe_verbose(include) - except re.error: - err(f"Invalid regular expression for include given: {include!r}") - ctx.exit(2) - try: - exclude_regex = re_compile_maybe_verbose(exclude) - except re.error: - err(f"Invalid regular expression for exclude given: {exclude!r}") - ctx.exit(2) - try: - force_exclude_regex = ( - re_compile_maybe_verbose(force_exclude) if force_exclude else None - ) - except re.error: - err(f"Invalid regular expression for force_exclude given: {force_exclude!r}") - ctx.exit(2) + + include_regex = test_regex(ctx, "include", include) + exclude_regex = test_regex(ctx, "exclude", exclude) + assert exclude_regex is not None + extend_exclude_regex = test_regex(ctx, "extend_exclude", extend_exclude) + force_exclude_regex = test_regex(ctx, "force_exclude", force_exclude) root = find_project_root(src) sources: Set[Path] = set() @@ -672,6 +684,7 @@ def get_sources( root, include_regex, exclude_regex, + extend_exclude_regex, force_exclude_regex, report, gitignore, @@ -6112,17 +6125,27 @@ def normalize_path_maybe_ignore( return normalized_path +def path_is_excluded( + normalized_path: str, + pattern: Optional[Pattern[str]], +) -> bool: + match = pattern.search(normalized_path) if pattern else None + return bool(match and match.group(0)) + + def gen_python_files( paths: Iterable[Path], root: Path, include: Optional[Pattern[str]], exclude: Pattern[str], + extend_exclude: Optional[Pattern[str]], force_exclude: Optional[Pattern[str]], report: "Report", gitignore: PathSpec, ) -> Iterator[Path]: """Generate all files under `path` whose paths are not excluded by the - `exclude_regex` or `force_exclude` regexes, but are included by the `include` regex. + `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, + but are included by the `include` regex. Symbolic links pointing outside of the `root` directory are ignored. @@ -6139,20 +6162,22 @@ def gen_python_files( report.path_ignored(child, "matches the .gitignore file content") continue - # Then ignore with `--exclude` and `--force-exclude` options. + # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. normalized_path = "/" + normalized_path if child.is_dir(): normalized_path += "/" - exclude_match = exclude.search(normalized_path) if exclude else None - if exclude_match and exclude_match.group(0): + if path_is_excluded(normalized_path, exclude): report.path_ignored(child, "matches the --exclude regular expression") continue - force_exclude_match = ( - force_exclude.search(normalized_path) if force_exclude else None - ) - if force_exclude_match and force_exclude_match.group(0): + if path_is_excluded(normalized_path, extend_exclude): + report.path_ignored( + child, "matches the --extend-exclude regular expression" + ) + continue + + if path_is_excluded(normalized_path, force_exclude): report.path_ignored(child, "matches the --force-exclude regular expression") continue @@ -6162,6 +6187,7 @@ def gen_python_files( root, include, exclude, + extend_exclude, force_exclude, report, gitignore, diff --git a/tests/test_black.py b/tests/test_black.py index 9c3cc643..ba1869aa 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1346,7 +1346,14 @@ class BlackTestCase(BlackBaseTestCase): this_abs = THIS_DIR.resolve() sources.extend( black.gen_python_files( - path.iterdir(), this_abs, include, exclude, None, report, gitignore + path.iterdir(), + this_abs, + include, + exclude, + None, + None, + report, + gitignore, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1370,6 +1377,7 @@ class BlackTestCase(BlackBaseTestCase): verbose=False, include=include, exclude=exclude, + extend_exclude=None, force_exclude=None, report=report, stdin_filename=None, @@ -1392,6 +1400,7 @@ class BlackTestCase(BlackBaseTestCase): verbose=False, include=include, exclude=exclude, + extend_exclude=None, force_exclude=None, report=report, stdin_filename=None, @@ -1415,6 +1424,7 @@ class BlackTestCase(BlackBaseTestCase): verbose=False, include=include, exclude=exclude, + extend_exclude=None, force_exclude=None, report=report, stdin_filename=stdin_filename, @@ -1442,6 +1452,35 @@ class BlackTestCase(BlackBaseTestCase): verbose=False, include=include, exclude=exclude, + extend_exclude=None, + force_exclude=None, + report=report, + stdin_filename=stdin_filename, + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + + @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) + def test_get_sources_with_stdin_filename_and_extend_exclude(self) -> None: + # Extend exclude shouldn't exclude stdin_filename since it is mimicking the + # file being passed directly. This is the same as + # test_exclude_for_issue_1572 + path = THIS_DIR / "data" / "include_exclude_tests" + include = "" + extend_exclude = r"/exclude/|a\.py" + src = "-" + report = black.Report() + stdin_filename = str(path / "b/exclude/a.py") + expected = [Path(f"__BLACK_STDIN_FILENAME__{stdin_filename}")] + sources = list( + black.get_sources( + ctx=FakeContext(), + src=(src,), + quiet=True, + verbose=False, + include=include, + exclude="", + extend_exclude=extend_exclude, force_exclude=None, report=report, stdin_filename=stdin_filename, @@ -1467,6 +1506,7 @@ class BlackTestCase(BlackBaseTestCase): verbose=False, include=include, exclude="", + extend_exclude=None, force_exclude=force_exclude, report=report, stdin_filename=stdin_filename, @@ -1551,7 +1591,14 @@ class BlackTestCase(BlackBaseTestCase): this_abs = THIS_DIR.resolve() sources.extend( black.gen_python_files( - path.iterdir(), this_abs, include, exclude, None, report, gitignore + path.iterdir(), + this_abs, + include, + exclude, + None, + None, + report, + gitignore, ) ) self.assertEqual(sorted(expected), sorted(sources)) @@ -1581,25 +1628,21 @@ class BlackTestCase(BlackBaseTestCase): empty, re.compile(black.DEFAULT_EXCLUDES), None, + None, report, gitignore, ) ) self.assertEqual(sorted(expected), sorted(sources)) - def test_empty_exclude(self) -> None: + def test_extend_exclude(self) -> None: path = THIS_DIR / "data" / "include_exclude_tests" report = black.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) - empty = re.compile(r"") sources: List[Path] = [] expected = [ - Path(path / "b/dont_exclude/a.py"), - Path(path / "b/dont_exclude/a.pyi"), Path(path / "b/exclude/a.py"), - Path(path / "b/exclude/a.pyi"), - Path(path / "b/.definitely_exclude/a.py"), - Path(path / "b/.definitely_exclude/a.pyi"), + Path(path / "b/dont_exclude/a.py"), ] this_abs = THIS_DIR.resolve() sources.extend( @@ -1607,7 +1650,8 @@ class BlackTestCase(BlackBaseTestCase): path.iterdir(), this_abs, re.compile(black.DEFAULT_INCLUDES), - empty, + re.compile(r"\.pyi$"), + re.compile(r"\.definitely_exclude"), None, report, gitignore, @@ -1615,8 +1659,8 @@ class BlackTestCase(BlackBaseTestCase): ) self.assertEqual(sorted(expected), sorted(sources)) - def test_invalid_include_exclude(self) -> None: - for option in ["--include", "--exclude"]: + def test_invalid_cli_regex(self) -> None: + for option in ["--include", "--exclude", "--extend-exclude", "--force-exclude"]: self.invokeBlack(["-", option, "**()(!!*)"], exit_code=2) def test_preserves_line_endings(self) -> None: @@ -1665,7 +1709,14 @@ class BlackTestCase(BlackBaseTestCase): try: list( black.gen_python_files( - path.iterdir(), root, include, exclude, None, report, gitignore + path.iterdir(), + root, + include, + exclude, + None, + None, + report, + gitignore, ) ) except ValueError as ve: @@ -1679,7 +1730,14 @@ class BlackTestCase(BlackBaseTestCase): with self.assertRaises(ValueError): list( black.gen_python_files( - path.iterdir(), root, include, exclude, None, report, gitignore + path.iterdir(), + root, + include, + exclude, + None, + None, + report, + gitignore, ) ) path.iterdir.assert_called() -- 2.39.5 From cac18293d5a6bd6b34a953f9cb5413f9826e505f Mon Sep 17 00:00:00 2001 From: Austin Pray <71290498+austinpray-mixpanel@users.noreply.github.com> Date: Mon, 1 Mar 2021 18:35:57 -0600 Subject: [PATCH 16/16] Adds --stdin-filename back to changelog (#2017) * Adds --stdin-filename back to changelog Looks like this went missing https://github.com/psf/black/commit/dea81b7ad5cfa04c3572771c34af823449d0a8f3#diff-729efdd61772b108539268bdbfd7472521bdc05a7cae6113f62ed2649a3ad9c7 * Update CHANGES.md Co-authored-by: Jelle Zijlstra * Update CHANGES.md Co-authored-by: Jelle Zijlstra Co-authored-by: Jelle Zijlstra --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3fd7ad40..90e5143f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,6 +30,9 @@ - `--diff` correctly indicates when a file doesn't end in a newline (#1662) +- Added `--stdin-filename` argument to allow stdin to respect `--force-exclude` rules + (#1780) + #### _Packaging_ - Self-contained native _Black_ binaries are now provided for releases via GitHub -- 2.39.5