All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
2 String transformers that can split and merge strings.
6 from abc import ABC, abstractmethod
7 from collections import defaultdict
8 from dataclasses import dataclass
28 from mypy_extensions import trait
30 from black.comments import contains_pragma_comment
31 from black.lines import Line, append_leaves
32 from black.mode import Feature, Mode
33 from black.nodes import (
40 is_part_of_annotation,
45 from black.rusty import Err, Ok, Result
46 from black.strings import (
47 assert_is_leaf_string,
51 normalize_string_quotes,
54 from blib2to3.pgen2 import token
55 from blib2to3.pytree import Leaf, Node
58 class CannotTransform(Exception):
59 """Base class for errors raised by Transformers."""
64 LN = Union[Leaf, Node]
65 Transformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]]
70 TResult = Result[T, CannotTransform] # (T)ransform Result
71 TMatchResult = TResult[List[Index]]
73 SPLIT_SAFE_CHARS = frozenset(["\u3001", "\u3002", "\uff0c"]) # East Asian stops
76 def TErr(err_msg: str) -> Err[CannotTransform]:
79 Convenience function used when working with the TResult type.
81 cant_transform = CannotTransform(err_msg)
82 return Err(cant_transform)
86 line: Line, features: Collection[Feature], mode: Mode
88 """A transformer which normalizes spacing around power operators."""
90 # Performance optimization to avoid unnecessary Leaf clones and other ops.
91 for leaf in line.leaves:
92 if leaf.type == token.DOUBLESTAR:
95 raise CannotTransform("No doublestar token was found in the line.")
97 def is_simple_lookup(index: int, step: Literal[1, -1]) -> bool:
98 # Brackets and parentheses indicate calls, subscripts, etc. ...
99 # basically stuff that doesn't count as "simple". Only a NAME lookup
100 # or dotted lookup (eg. NAME.NAME) is OK.
102 disallowed = {token.RPAR, token.RSQB}
104 disallowed = {token.LPAR, token.LSQB}
106 while 0 <= index < len(line.leaves):
107 current = line.leaves[index]
108 if current.type in disallowed:
110 if current.type not in {token.NAME, token.DOT} or current.value == "for":
111 # If the current token isn't disallowed, we'll assume this is simple as
112 # only the disallowed tokens are semantically attached to this lookup
113 # expression we're checking. Also, stop early if we hit the 'for' bit
114 # of a comprehension.
121 def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool:
122 # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple
123 # lookup (see above), with or without a preceding unary operator.
124 start = line.leaves[index]
125 if start.type in {token.NAME, token.NUMBER}:
126 return is_simple_lookup(index, step=(1 if kind == "exponent" else -1))
128 if start.type in {token.PLUS, token.MINUS, token.TILDE}:
129 if line.leaves[index + 1].type in {token.NAME, token.NUMBER}:
130 # step is always one as bases with a preceding unary op will be checked
131 # for simplicity starting from the next token (so it'll hit the check
133 return is_simple_lookup(index + 1, step=1)
137 new_line = line.clone()
139 for idx, leaf in enumerate(line.leaves):
140 new_leaf = leaf.clone()
146 (0 < idx < len(line.leaves) - 1)
147 and leaf.type == token.DOUBLESTAR
148 and is_simple_operand(idx - 1, kind="base")
149 and line.leaves[idx - 1].value != "lambda"
150 and is_simple_operand(idx + 1, kind="exponent")
155 # We have to be careful to make a new line properly:
156 # - bracket related metadata must be maintained (handled by Line.append)
157 # - comments need to copied over, updating the leaf IDs they're attached to
158 new_line.append(new_leaf, preformatted=True)
159 for comment_leaf in line.comments_after(leaf):
160 new_line.append(comment_leaf, preformatted=True)
165 class StringTransformer(ABC):
167 An implementation of the Transformer protocol that relies on its
168 subclasses overriding the template methods `do_match(...)` and
171 This Transformer works exclusively on strings (for example, by merging
174 The following sections can be found among the docstrings of each concrete
175 StringTransformer subclass.
178 Which requirements must be met of the given Line for this
179 StringTransformer to be applied?
182 If the given Line meets all of the above requirements, which string
183 transformations can you expect to be applied to it by this
187 What contractual agreements does this StringTransformer have with other
188 StringTransfomers? Such collaborations should be eliminated/minimized
192 __name__: Final = "StringTransformer"
194 # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with
196 def __init__(self, line_length: int, normalize_strings: bool) -> None:
197 self.line_length = line_length
198 self.normalize_strings = normalize_strings
201 def do_match(self, line: Line) -> TMatchResult:
204 * Ok(string_indices) such that for each index, `line.leaves[index]`
205 is our target string if a match was able to be made. For
206 transformers that don't result in more lines (e.g. StringMerger,
207 StringParenStripper), multiple matches and transforms are done at
208 once to reduce the complexity.
210 * Err(CannotTransform), if no match could be made.
215 self, line: Line, string_indices: List[int]
216 ) -> Iterator[TResult[Line]]:
219 * Ok(new_line) where new_line is the new transformed line.
221 * Err(CannotTransform) if the transformation failed for some reason. The
222 `do_match(...)` template method should usually be used to reject
223 the form of the given Line, but in some cases it is difficult to
224 know whether or not a Line meets the StringTransformer's
225 requirements until the transformation is already midway.
228 This method should NOT mutate @line directly, but it MAY mutate the
229 Line's underlying Node structure. (WARNING: If the underlying Node
230 structure IS altered, then this method should NOT be allowed to
231 yield an CannotTransform after that point.)
235 self, line: Line, _features: Collection[Feature], _mode: Mode
238 StringTransformer instances have a call signature that mirrors that of
239 the Transformer type.
242 CannotTransform(...) if the concrete StringTransformer class is unable
245 # Optimization to avoid calling `self.do_match(...)` when the line does
246 # not contain any string.
247 if not any(leaf.type == token.STRING for leaf in line.leaves):
248 raise CannotTransform("There are no strings in this line.")
250 match_result = self.do_match(line)
252 if isinstance(match_result, Err):
253 cant_transform = match_result.err()
254 raise CannotTransform(
255 f"The string transformer {self.__class__.__name__} does not recognize"
256 " this line as one that it can transform."
257 ) from cant_transform
259 string_indices = match_result.ok()
261 for line_result in self.do_transform(line, string_indices):
262 if isinstance(line_result, Err):
263 cant_transform = line_result.err()
264 raise CannotTransform(
265 "StringTransformer failed while attempting to transform string."
266 ) from cant_transform
267 line = line_result.ok()
273 """A custom (i.e. manual) string split.
275 A single CustomSplit instance represents a single substring.
278 Consider the following string:
285 This string will correspond to the following three CustomSplit instances:
287 CustomSplit(False, 16)
288 CustomSplit(False, 17)
289 CustomSplit(True, 16)
298 class CustomSplitMapMixin:
300 This mixin class is used to map merged strings to a sequence of
301 CustomSplits, which will then be used to re-split the strings iff none of
302 the resultant substrings go over the configured max line length.
305 _Key: ClassVar = Tuple[StringID, str]
306 _CUSTOM_SPLIT_MAP: ClassVar[Dict[_Key, Tuple[CustomSplit, ...]]] = defaultdict(
311 def _get_key(string: str) -> "CustomSplitMapMixin._Key":
314 A unique identifier that is used internally to map @string to a
315 group of custom splits.
317 return (id(string), string)
319 def add_custom_splits(
320 self, string: str, custom_splits: Iterable[CustomSplit]
322 """Custom Split Map Setter Method
325 Adds a mapping from @string to the custom splits @custom_splits.
327 key = self._get_key(string)
328 self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)
330 def pop_custom_splits(self, string: str) -> List[CustomSplit]:
331 """Custom Split Map Getter Method
334 * A list of the custom splits that are mapped to @string, if any
340 Deletes the mapping between @string and its associated custom
341 splits (which are returned to the caller).
343 key = self._get_key(string)
345 custom_splits = self._CUSTOM_SPLIT_MAP[key]
346 del self._CUSTOM_SPLIT_MAP[key]
348 return list(custom_splits)
350 def has_custom_splits(self, string: str) -> bool:
353 True iff @string is associated with a set of custom splits.
355 key = self._get_key(string)
356 return key in self._CUSTOM_SPLIT_MAP
359 class StringMerger(StringTransformer, CustomSplitMapMixin):
360 """StringTransformer that merges strings together.
363 (A) The line contains adjacent strings such that ALL of the validation checks
364 listed in StringMerger._validate_msg(...)'s docstring pass.
366 (B) The line contains a string which uses line continuation backslashes.
369 Depending on which of the two requirements above where met, either:
371 (A) The string group associated with the target string is merged.
373 (B) All line-continuation backslashes are removed from the target string.
376 StringMerger provides custom split information to StringSplitter.
379 def do_match(self, line: Line) -> TMatchResult:
382 is_valid_index = is_valid_index_factory(LL)
386 while is_valid_index(idx):
389 leaf.type == token.STRING
390 and is_valid_index(idx + 1)
391 and LL[idx + 1].type == token.STRING
393 # Let's check if the string group contains an inline comment
394 # If we have a comment inline, we don't merge the strings
395 contains_comment = False
397 while is_valid_index(i):
398 if LL[i].type != token.STRING:
400 if line.comments_after(LL[i]):
401 contains_comment = True
405 if not is_part_of_annotation(leaf) and not contains_comment:
406 string_indices.append(idx)
408 # Advance to the next non-STRING leaf.
410 while is_valid_index(idx) and LL[idx].type == token.STRING:
413 elif leaf.type == token.STRING and "\\\n" in leaf.value:
414 string_indices.append(idx)
415 # Advance to the next non-STRING leaf.
417 while is_valid_index(idx) and LL[idx].type == token.STRING:
424 return Ok(string_indices)
426 return TErr("This line has no strings that need merging.")
429 self, line: Line, string_indices: List[int]
430 ) -> Iterator[TResult[Line]]:
433 rblc_result = self._remove_backslash_line_continuation_chars(
434 new_line, string_indices
436 if isinstance(rblc_result, Ok):
437 new_line = rblc_result.ok()
439 msg_result = self._merge_string_group(new_line, string_indices)
440 if isinstance(msg_result, Ok):
441 new_line = msg_result.ok()
443 if isinstance(rblc_result, Err) and isinstance(msg_result, Err):
444 msg_cant_transform = msg_result.err()
445 rblc_cant_transform = rblc_result.err()
446 cant_transform = CannotTransform(
447 "StringMerger failed to merge any strings in this line."
450 # Chain the errors together using `__cause__`.
451 msg_cant_transform.__cause__ = rblc_cant_transform
452 cant_transform.__cause__ = msg_cant_transform
454 yield Err(cant_transform)
459 def _remove_backslash_line_continuation_chars(
460 line: Line, string_indices: List[int]
463 Merge strings that were split across multiple lines using
464 line-continuation backslashes.
467 Ok(new_line), if @line contains backslash line-continuation
470 Err(CannotTransform), otherwise.
474 indices_to_transform = []
475 for string_idx in string_indices:
476 string_leaf = LL[string_idx]
478 string_leaf.type == token.STRING
479 and "\\\n" in string_leaf.value
480 and not has_triple_quotes(string_leaf.value)
482 indices_to_transform.append(string_idx)
484 if not indices_to_transform:
486 "Found no string leaves that contain backslash line continuation"
490 new_line = line.clone()
491 new_line.comments = line.comments.copy()
492 append_leaves(new_line, line, LL)
494 for string_idx in indices_to_transform:
495 new_string_leaf = new_line.leaves[string_idx]
496 new_string_leaf.value = new_string_leaf.value.replace("\\\n", "")
500 def _merge_string_group(
501 self, line: Line, string_indices: List[int]
504 Merges string groups (i.e. set of adjacent strings).
506 Each index from `string_indices` designates one string group's first
507 leaf in `line.leaves`.
510 Ok(new_line), if ALL of the validation checks found in
511 _validate_msg(...) pass.
513 Err(CannotTransform), otherwise.
517 is_valid_index = is_valid_index_factory(LL)
519 # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.
520 merged_string_idx_dict: Dict[int, Tuple[int, Leaf]] = {}
521 for string_idx in string_indices:
522 vresult = self._validate_msg(line, string_idx)
523 if isinstance(vresult, Err):
525 merged_string_idx_dict[string_idx] = self._merge_one_string_group(
526 LL, string_idx, is_valid_index
529 if not merged_string_idx_dict:
530 return TErr("No string group is merged")
532 # Build the final line ('new_line') that this method will later return.
533 new_line = line.clone()
534 previous_merged_string_idx = -1
535 previous_merged_num_of_strings = -1
536 for i, leaf in enumerate(LL):
537 if i in merged_string_idx_dict:
538 previous_merged_string_idx = i
539 previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]
540 new_line.append(string_leaf)
543 previous_merged_string_idx
545 < previous_merged_string_idx + previous_merged_num_of_strings
547 for comment_leaf in line.comments_after(LL[i]):
548 new_line.append(comment_leaf, preformatted=True)
551 append_leaves(new_line, line, [leaf])
555 def _merge_one_string_group(
556 self, LL: List[Leaf], string_idx: int, is_valid_index: Callable[[int], bool]
557 ) -> Tuple[int, Leaf]:
559 Merges one string group where the first string in the group is
563 A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the
564 number of strings merged and `leaf` is the newly merged string
565 to be replaced in the new line.
567 # If the string group is wrapped inside an Atom node, we must make sure
568 # to later replace that Atom with our new (merged) string leaf.
569 atom_node = LL[string_idx].parent
571 # We will place BREAK_MARK in between every two substrings that we
572 # merge. We will then later go through our final result and use the
573 # various instances of BREAK_MARK we find to add the right values to
574 # the custom split map.
575 BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@"
577 QUOTE = LL[string_idx].value[-1]
579 def make_naked(string: str, string_prefix: str) -> str:
580 """Strip @string (i.e. make it a "naked" string)
583 * assert_is_leaf_string(@string)
586 A string that is identical to @string except that
587 @string_prefix has been stripped, the surrounding QUOTE
588 characters have been removed, and any remaining QUOTE
589 characters have been escaped.
591 assert_is_leaf_string(string)
592 if "f" in string_prefix:
593 string = _toggle_fexpr_quotes(string, QUOTE)
594 # After quotes toggling, quotes in expressions won't be escaped
595 # because quotes can't be reused in f-strings. So we can simply
596 # let the escaping logic below run without knowing f-string
599 RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)"
600 naked_string = string[len(string_prefix) + 1 : -1]
601 naked_string = re.sub(
602 "(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string
606 # Holds the CustomSplit objects that will later be added to the custom
610 # Temporary storage for the 'has_prefix' part of the CustomSplit objects.
613 # Sets the 'prefix' variable. This is the prefix that the final merged
615 next_str_idx = string_idx
619 and is_valid_index(next_str_idx)
620 and LL[next_str_idx].type == token.STRING
622 prefix = get_string_prefix(LL[next_str_idx].value).lower()
625 # The next loop merges the string group. The final string will be
628 # The following convenience variables are used:
633 # NSS: naked next string
637 next_str_idx = string_idx
638 while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:
641 SS = LL[next_str_idx].value
642 next_prefix = get_string_prefix(SS).lower()
644 # If this is an f-string group but this substring is not prefixed
646 if "f" in prefix and "f" not in next_prefix:
647 # Then we must escape any braces contained in this substring.
648 SS = re.sub(r"(\{|\})", r"\1\1", SS)
650 NSS = make_naked(SS, next_prefix)
652 has_prefix = bool(next_prefix)
653 prefix_tracker.append(has_prefix)
655 S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE
656 NS = make_naked(S, prefix)
660 # Take a note on the index of the non-STRING leaf.
661 non_string_idx = next_str_idx
663 S_leaf = Leaf(token.STRING, S)
664 if self.normalize_strings:
665 S_leaf.value = normalize_string_quotes(S_leaf.value)
667 # Fill the 'custom_splits' list with the appropriate CustomSplit objects.
668 temp_string = S_leaf.value[len(prefix) + 1 : -1]
669 for has_prefix in prefix_tracker:
670 mark_idx = temp_string.find(BREAK_MARK)
673 ), "Logic error while filling the custom string breakpoint cache."
675 temp_string = temp_string[mark_idx + len(BREAK_MARK) :]
676 breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1
677 custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))
679 string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
681 if atom_node is not None:
682 # If not all children of the atom node are merged (this can happen
683 # when there is a standalone comment in the middle) ...
684 if non_string_idx - string_idx < len(atom_node.children):
685 # We need to replace the old STRING leaves with the new string leaf.
686 first_child_idx = LL[string_idx].remove()
687 for idx in range(string_idx + 1, non_string_idx):
689 if first_child_idx is not None:
690 atom_node.insert_child(first_child_idx, string_leaf)
692 # Else replace the atom node with the new string leaf.
693 replace_child(atom_node, string_leaf)
695 self.add_custom_splits(string_leaf.value, custom_splits)
696 return num_of_strings, string_leaf
699 def _validate_msg(line: Line, string_idx: int) -> TResult[None]:
700 """Validate (M)erge (S)tring (G)roup
702 Transform-time string validation logic for _merge_string_group(...).
705 * Ok(None), if ALL validation checks (listed below) pass.
707 * Err(CannotTransform), if any of the following are true:
708 - The target string group does not contain ANY stand-alone comments.
709 - The target string is not in a string group (i.e. it has no
711 - The string group has more than one inline comment.
712 - The string group has an inline comment that appears to be a pragma.
713 - The set of all string prefixes in the string group is of
714 length greater than one and is not equal to {"", "f"}.
715 - The string group consists of raw strings.
716 - The string group is stringified type annotations. We don't want to
717 process stringified type annotations since pyright doesn't support
718 them spanning multiple string values. (NOTE: mypy, pytype, pyre do
719 support them, so we can change if pyright also gains support in the
720 future. See https://github.com/microsoft/pyright/issues/4359.)
722 # We first check for "inner" stand-alone comments (i.e. stand-alone
723 # comments that have a string leaf before them AND after them).
726 found_sa_comment = False
727 is_valid_index = is_valid_index_factory(line.leaves)
728 while is_valid_index(i) and line.leaves[i].type in [
732 if line.leaves[i].type == STANDALONE_COMMENT:
733 found_sa_comment = True
734 elif found_sa_comment:
736 "StringMerger does NOT merge string groups which contain "
737 "stand-alone comments."
742 num_of_inline_string_comments = 0
743 set_of_prefixes = set()
745 for leaf in line.leaves[string_idx:]:
746 if leaf.type != token.STRING:
747 # If the string group is trailed by a comma, we count the
748 # comments trailing the comma to be one of the string group's
750 if leaf.type == token.COMMA and id(leaf) in line.comments:
751 num_of_inline_string_comments += 1
754 if has_triple_quotes(leaf.value):
755 return TErr("StringMerger does NOT merge multiline strings.")
758 prefix = get_string_prefix(leaf.value).lower()
760 return TErr("StringMerger does NOT merge raw strings.")
762 set_of_prefixes.add(prefix)
764 if id(leaf) in line.comments:
765 num_of_inline_string_comments += 1
766 if contains_pragma_comment(line.comments[id(leaf)]):
767 return TErr("Cannot merge strings which have pragma comments.")
769 if num_of_strings < 2:
771 f"Not enough strings to merge (num_of_strings={num_of_strings})."
774 if num_of_inline_string_comments > 1:
776 f"Too many inline string comments ({num_of_inline_string_comments})."
779 if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}:
780 return TErr(f"Too many different prefixes ({set_of_prefixes}).")
785 class StringParenStripper(StringTransformer):
786 """StringTransformer that strips surrounding parentheses from strings.
789 The line contains a string which is surrounded by parentheses and:
790 - The target string is NOT the only argument to a function call.
791 - The target string is NOT a "pointless" string.
792 - If the target string contains a PERCENT, the brackets are not
793 preceded or followed by an operator with higher precedence than
797 The parentheses mentioned in the 'Requirements' section are stripped.
800 StringParenStripper has its own inherent usefulness, but it is also
801 relied on to clean up the parentheses created by StringParenWrapper (in
802 the event that they are no longer needed).
805 def do_match(self, line: Line) -> TMatchResult:
808 is_valid_index = is_valid_index_factory(LL)
819 # Should be a string...
820 if leaf.type != token.STRING:
823 # If this is a "pointless" string...
826 and leaf.parent.parent
827 and leaf.parent.parent.type == syms.simple_stmt
831 # Should be preceded by a non-empty LPAR...
833 not is_valid_index(idx - 1)
834 or LL[idx - 1].type != token.LPAR
835 or is_empty_lpar(LL[idx - 1])
839 # That LPAR should NOT be preceded by a function name or a closing
840 # bracket (which could be a function which returns a function or a
841 # list/dictionary that contains a function)...
842 if is_valid_index(idx - 2) and (
843 LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS
849 # Skip the string trailer, if one exists.
850 string_parser = StringParser()
851 next_idx = string_parser.parse(LL, string_idx)
853 # if the leaves in the parsed string include a PERCENT, we need to
854 # make sure the initial LPAR is NOT preceded by an operator with
855 # higher or equal precedence to PERCENT
856 if is_valid_index(idx - 2):
857 # mypy can't quite follow unless we name this
858 before_lpar = LL[idx - 2]
859 if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and (
876 # only unary PLUS/MINUS
878 and before_lpar.parent.type == syms.factor
879 and (before_lpar.type in {token.PLUS, token.MINUS})
884 # Should be followed by a non-empty RPAR...
886 is_valid_index(next_idx)
887 and LL[next_idx].type == token.RPAR
888 and not is_empty_rpar(LL[next_idx])
890 # That RPAR should NOT be followed by anything with higher
891 # precedence than PERCENT
892 if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in {
900 string_indices.append(string_idx)
902 while idx < len(LL) - 1 and LL[idx + 1].type == token.STRING:
906 return Ok(string_indices)
907 return TErr("This line has no strings wrapped in parens.")
910 self, line: Line, string_indices: List[int]
911 ) -> Iterator[TResult[Line]]:
914 string_and_rpar_indices: List[int] = []
915 for string_idx in string_indices:
916 string_parser = StringParser()
917 rpar_idx = string_parser.parse(LL, string_idx)
919 should_transform = True
920 for leaf in (LL[string_idx - 1], LL[rpar_idx]):
921 if line.comments_after(leaf):
922 # Should not strip parentheses which have comments attached
924 should_transform = False
927 string_and_rpar_indices.extend((string_idx, rpar_idx))
929 if string_and_rpar_indices:
930 yield Ok(self._transform_to_new_line(line, string_and_rpar_indices))
933 CannotTransform("All string groups have comments attached to them.")
936 def _transform_to_new_line(
937 self, line: Line, string_and_rpar_indices: List[int]
941 new_line = line.clone()
942 new_line.comments = line.comments.copy()
945 # We need to sort the indices, since string_idx and its matching
946 # rpar_idx may not come in order, e.g. in
947 # `("outer" % ("inner".join(items)))`, the "inner" string's
948 # string_idx is smaller than "outer" string's rpar_idx.
949 for idx in sorted(string_and_rpar_indices):
951 lpar_or_rpar_idx = idx - 1 if leaf.type == token.STRING else idx
952 append_leaves(new_line, line, LL[previous_idx + 1 : lpar_or_rpar_idx])
953 if leaf.type == token.STRING:
954 string_leaf = Leaf(token.STRING, LL[idx].value)
955 LL[lpar_or_rpar_idx].remove() # Remove lpar.
956 replace_child(LL[idx], string_leaf)
957 new_line.append(string_leaf)
959 old_comments = new_line.comments.pop(id(LL[idx]), [])
960 new_line.comments.setdefault(id(string_leaf), []).extend(old_comments)
962 LL[lpar_or_rpar_idx].remove() # This is a rpar.
966 # Append the leaves after the last idx:
967 append_leaves(new_line, line, LL[idx + 1 :])
972 class BaseStringSplitter(StringTransformer):
974 Abstract class for StringTransformers which transform a Line's strings by splitting
975 them or placing them on their own lines where necessary to avoid going over
976 the configured line length.
979 * The target string value is responsible for the line going over the
980 line length limit. It follows that after all of black's other line
981 split methods have been exhausted, this line (or one of the resulting
982 lines after all line splits are performed) would still be over the
983 line_length limit unless we split this string.
986 * The target string is NOT a "pointless" string (i.e. a string that has
987 no parent or siblings).
990 * The target string is not followed by an inline comment that appears
994 * The target string is not a multiline (i.e. triple-quote) string.
997 STRING_OPERATORS: Final = [
1010 def do_splitter_match(self, line: Line) -> TMatchResult:
1012 BaseStringSplitter asks its clients to override this method instead of
1013 `StringTransformer.do_match(...)`.
1015 Follows the same protocol as `StringTransformer.do_match(...)`.
1017 Refer to `help(StringTransformer.do_match)` for more information.
1020 def do_match(self, line: Line) -> TMatchResult:
1021 match_result = self.do_splitter_match(line)
1022 if isinstance(match_result, Err):
1025 string_indices = match_result.ok()
1026 assert len(string_indices) == 1, (
1027 f"{self.__class__.__name__} should only find one match at a time, found"
1028 f" {len(string_indices)}"
1030 string_idx = string_indices[0]
1031 vresult = self._validate(line, string_idx)
1032 if isinstance(vresult, Err):
1037 def _validate(self, line: Line, string_idx: int) -> TResult[None]:
1039 Checks that @line meets all of the requirements listed in this classes'
1040 docstring. Refer to `help(BaseStringSplitter)` for a detailed
1041 description of those requirements.
1044 * Ok(None), if ALL of the requirements are met.
1046 * Err(CannotTransform), if ANY of the requirements are NOT met.
1050 string_leaf = LL[string_idx]
1052 max_string_length = self._get_max_string_length(line, string_idx)
1053 if len(string_leaf.value) <= max_string_length:
1055 "The string itself is not what is causing this line to be too long."
1058 if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [
1063 f"This string ({string_leaf.value}) appears to be pointless (i.e. has"
1067 if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(
1068 line.comments[id(line.leaves[string_idx])]
1071 "Line appears to end with an inline pragma comment. Splitting the line"
1072 " could modify the pragma's behavior."
1075 if has_triple_quotes(string_leaf.value):
1076 return TErr("We cannot split multiline strings.")
1080 def _get_max_string_length(self, line: Line, string_idx: int) -> int:
1082 Calculates the max string length used when attempting to determine
1083 whether or not the target string is responsible for causing the line to
1084 go over the line length limit.
1086 WARNING: This method is tightly coupled to both StringSplitter and
1087 (especially) StringParenWrapper. There is probably a better way to
1088 accomplish what is being done here.
1091 max_string_length: such that `line.leaves[string_idx].value >
1092 max_string_length` implies that the target string IS responsible
1093 for causing this line to exceed the line length limit.
1097 is_valid_index = is_valid_index_factory(LL)
1099 # We use the shorthand "WMA4" in comments to abbreviate "We must
1100 # account for". When giving examples, we use STRING to mean some/any
1103 # Finally, we use the following convenience variables:
1105 # P: The leaf that is before the target string leaf.
1106 # N: The leaf that is after the target string leaf.
1107 # NN: The leaf that is after N.
1109 # WMA4 the whitespace at the beginning of the line.
1110 offset = line.depth * 4
1112 if is_valid_index(string_idx - 1):
1113 p_idx = string_idx - 1
1115 LL[string_idx - 1].type == token.LPAR
1116 and LL[string_idx - 1].value == ""
1119 # If the previous leaf is an empty LPAR placeholder, we should skip it.
1123 if P.type in self.STRING_OPERATORS:
1124 # WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`).
1125 offset += len(str(P)) + 1
1127 if P.type == token.COMMA:
1128 # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].
1131 if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]:
1132 # This conditional branch is meant to handle dictionary keys,
1133 # variable assignments, 'return STRING' statement lines, and
1134 # 'else STRING' ternary expression lines.
1136 # WMA4 a single space.
1139 # WMA4 the lengths of any leaves that came before that space,
1140 # but after any closing bracket before that space.
1141 for leaf in reversed(LL[: p_idx + 1]):
1142 offset += len(str(leaf))
1143 if leaf.type in CLOSING_BRACKETS:
1146 if is_valid_index(string_idx + 1):
1147 N = LL[string_idx + 1]
1148 if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2:
1149 # If the next leaf is an empty RPAR placeholder, we should skip it.
1150 N = LL[string_idx + 2]
1152 if N.type == token.COMMA:
1153 # WMA4 a single comma at the end of the string (e.g `STRING,`).
1156 if is_valid_index(string_idx + 2):
1157 NN = LL[string_idx + 2]
1159 if N.type == token.DOT and NN.type == token.NAME:
1160 # This conditional branch is meant to handle method calls invoked
1161 # off of a string literal up to and including the LPAR character.
1163 # WMA4 the '.' character.
1167 is_valid_index(string_idx + 3)
1168 and LL[string_idx + 3].type == token.LPAR
1170 # WMA4 the left parenthesis character.
1173 # WMA4 the length of the method's name.
1174 offset += len(NN.value)
1176 has_comments = False
1177 for comment_leaf in line.comments_after(LL[string_idx]):
1178 if not has_comments:
1180 # WMA4 two spaces before the '#' character.
1183 # WMA4 the length of the inline comment.
1184 offset += len(comment_leaf.value)
1186 max_string_length = count_chars_in_width(str(line), self.line_length - offset)
1187 return max_string_length
1190 def _prefer_paren_wrap_match(LL: List[Leaf]) -> Optional[int]:
1193 string_idx such that @LL[string_idx] is equal to our target (i.e.
1194 matched) string, if this line matches the "prefer paren wrap" statement
1195 requirements listed in the 'Requirements' section of the StringParenWrapper
1200 # The line must start with a string.
1201 if LL[0].type != token.STRING:
1209 # If the string is an immediate child of a list/set/tuple literal...
1211 parent_type(LL[0]) in matching_nodes
1212 or parent_type(LL[0].parent) in matching_nodes
1214 # And the string is surrounded by commas (or is the first/last child)...
1215 prev_sibling = LL[0].prev_sibling
1216 next_sibling = LL[0].next_sibling
1219 and not next_sibling
1220 and parent_type(LL[0]) == syms.atom
1222 # If it's an atom string, we need to check the parent atom's siblings.
1223 parent = LL[0].parent
1224 assert parent is not None # For type checkers.
1225 prev_sibling = parent.prev_sibling
1226 next_sibling = parent.next_sibling
1227 if (not prev_sibling or prev_sibling.type == token.COMMA) and (
1228 not next_sibling or next_sibling.type == token.COMMA
1235 def iter_fexpr_spans(s: str) -> Iterator[Tuple[int, int]]:
1237 Yields spans corresponding to expressions in a given f-string.
1238 Spans are half-open ranges (left inclusive, right exclusive).
1239 Assumes the input string is a valid f-string, but will not crash if the input
1242 stack: List[int] = [] # our curly paren stack
1246 # if we're in a string part of the f-string, ignore escaped curly braces
1247 if not stack and i + 1 < len(s) and s[i + 1] == "{":
1259 # we've made it back out of the expression! yield the span
1265 # if we're in an expression part of the f-string, fast forward through strings
1266 # note that backslashes are not legal in the expression portion of f-strings
1269 if s[i : i + 3] in ("'''", '"""'):
1270 delim = s[i : i + 3]
1271 elif s[i] in ("'", '"'):
1275 while i < len(s) and s[i : i + len(delim)] != delim:
1282 def fstring_contains_expr(s: str) -> bool:
1283 return any(iter_fexpr_spans(s))
1286 def _toggle_fexpr_quotes(fstring: str, old_quote: str) -> str:
1288 Toggles quotes used in f-string expressions that are `old_quote`.
1290 f-string expressions can't contain backslashes, so we need to toggle the
1291 quotes if the f-string itself will end up using the same quote. We can
1292 simply toggle without escaping because, quotes can't be reused in f-string
1293 expressions. They will fail to parse.
1295 NOTE: If PEP 701 is accepted, above statement will no longer be true.
1296 Though if quotes can be reused, we can simply reuse them without updates or
1297 escaping, once Black figures out how to parse the new grammar.
1299 new_quote = "'" if old_quote == '"' else '"'
1302 for start, end in iter_fexpr_spans(fstring):
1303 parts.append(fstring[previous_index:start])
1304 parts.append(fstring[start:end].replace(old_quote, new_quote))
1305 previous_index = end
1306 parts.append(fstring[previous_index:])
1307 return "".join(parts)
1310 class StringSplitter(BaseStringSplitter, CustomSplitMapMixin):
1312 StringTransformer that splits "atom" strings (i.e. strings which exist on
1313 lines by themselves).
1316 * The line consists ONLY of a single string (possibly prefixed by a
1317 string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE
1320 * All of the requirements listed in BaseStringSplitter's docstring.
1323 The string mentioned in the 'Requirements' section is split into as
1324 many substrings as necessary to adhere to the configured line length.
1326 In the final set of substrings, no substring should be smaller than
1327 MIN_SUBSTR_SIZE characters.
1329 The string will ONLY be split on spaces (i.e. each new substring should
1330 start with a space). Note that the string will NOT be split on a space
1331 which is escaped with a backslash.
1333 If the string is an f-string, it will NOT be split in the middle of an
1334 f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x
1335 else bar()} is an f-expression).
1337 If the string that is being split has an associated set of custom split
1338 records and those custom splits will NOT result in any line going over
1339 the configured line length, those custom splits are used. Otherwise the
1340 string is split as late as possible (from left-to-right) while still
1341 adhering to the transformation rules listed above.
1344 StringSplitter relies on StringMerger to construct the appropriate
1345 CustomSplit objects and add them to the custom split map.
1348 MIN_SUBSTR_SIZE: Final = 6
1350 def do_splitter_match(self, line: Line) -> TMatchResult:
1353 if self._prefer_paren_wrap_match(LL) is not None:
1354 return TErr("Line needs to be wrapped in parens first.")
1356 is_valid_index = is_valid_index_factory(LL)
1360 # The first two leaves MAY be the 'not in' keywords...
1363 and is_valid_index(idx + 1)
1364 and [LL[idx].type, LL[idx + 1].type] == [token.NAME, token.NAME]
1365 and str(LL[idx]) + str(LL[idx + 1]) == "not in"
1368 # Else the first leaf MAY be a string operator symbol or the 'in' keyword...
1369 elif is_valid_index(idx) and (
1370 LL[idx].type in self.STRING_OPERATORS
1371 or LL[idx].type == token.NAME
1372 and str(LL[idx]) == "in"
1376 # The next/first leaf MAY be an empty LPAR...
1377 if is_valid_index(idx) and is_empty_lpar(LL[idx]):
1380 # The next/first leaf MUST be a string...
1381 if not is_valid_index(idx) or LL[idx].type != token.STRING:
1382 return TErr("Line does not start with a string.")
1386 # Skip the string trailer, if one exists.
1387 string_parser = StringParser()
1388 idx = string_parser.parse(LL, string_idx)
1390 # That string MAY be followed by an empty RPAR...
1391 if is_valid_index(idx) and is_empty_rpar(LL[idx]):
1394 # That string / empty RPAR leaf MAY be followed by a comma...
1395 if is_valid_index(idx) and LL[idx].type == token.COMMA:
1398 # But no more leaves are allowed...
1399 if is_valid_index(idx):
1400 return TErr("This line does not end with a string.")
1402 return Ok([string_idx])
1405 self, line: Line, string_indices: List[int]
1406 ) -> Iterator[TResult[Line]]:
1408 assert len(string_indices) == 1, (
1409 f"{self.__class__.__name__} should only find one match at a time, found"
1410 f" {len(string_indices)}"
1412 string_idx = string_indices[0]
1414 QUOTE = LL[string_idx].value[-1]
1416 is_valid_index = is_valid_index_factory(LL)
1417 insert_str_child = insert_str_child_factory(LL[string_idx])
1419 prefix = get_string_prefix(LL[string_idx].value).lower()
1421 # We MAY choose to drop the 'f' prefix from substrings that don't
1422 # contain any f-expressions, but ONLY if the original f-string
1423 # contains at least one f-expression. Otherwise, we will alter the AST
1425 drop_pointless_f_prefix = ("f" in prefix) and fstring_contains_expr(
1426 LL[string_idx].value
1429 first_string_line = True
1431 string_op_leaves = self._get_string_operator_leaves(LL)
1432 string_op_leaves_length = (
1433 sum(len(str(prefix_leaf)) for prefix_leaf in string_op_leaves) + 1
1438 def maybe_append_string_operators(new_line: Line) -> None:
1441 If @line starts with a string operator and this is the first
1442 line we are constructing, this function appends the string
1443 operator to @new_line and replaces the old string operator leaf
1444 in the node structure. Otherwise this function does nothing.
1446 maybe_prefix_leaves = string_op_leaves if first_string_line else []
1447 for i, prefix_leaf in enumerate(maybe_prefix_leaves):
1448 replace_child(LL[i], prefix_leaf)
1449 new_line.append(prefix_leaf)
1452 is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA
1455 def max_last_string_column() -> int:
1458 The max allowed width of the string value used for the last
1459 line we will construct. Note that this value means the width
1460 rather than the number of characters (e.g., many East Asian
1461 characters expand to two columns).
1463 result = self.line_length
1464 result -= line.depth * 4
1465 result -= 1 if ends_with_comma else 0
1466 result -= string_op_leaves_length
1469 # --- Calculate Max Break Width (for string value)
1470 # We start with the line length limit
1471 max_break_width = self.line_length
1472 # The last index of a string of length N is N-1.
1473 max_break_width -= 1
1474 # Leading whitespace is not present in the string value (e.g. Leaf.value).
1475 max_break_width -= line.depth * 4
1476 if max_break_width < 0:
1478 f"Unable to split {LL[string_idx].value} at such high of a line depth:"
1483 # Check if StringMerger registered any custom splits.
1484 custom_splits = self.pop_custom_splits(LL[string_idx].value)
1485 # We use them ONLY if none of them would produce lines that exceed the
1487 use_custom_breakpoints = bool(
1489 and all(csplit.break_idx <= max_break_width for csplit in custom_splits)
1492 # Temporary storage for the remaining chunk of the string line that
1493 # can't fit onto the line currently being constructed.
1494 rest_value = LL[string_idx].value
1496 def more_splits_should_be_made() -> bool:
1499 True iff `rest_value` (the remaining string value from the last
1500 split), should be split again.
1502 if use_custom_breakpoints:
1503 return len(custom_splits) > 1
1505 return str_width(rest_value) > max_last_string_column()
1507 string_line_results: List[Ok[Line]] = []
1508 while more_splits_should_be_made():
1509 if use_custom_breakpoints:
1510 # Custom User Split (manual)
1511 csplit = custom_splits.pop(0)
1512 break_idx = csplit.break_idx
1514 # Algorithmic Split (automatic)
1516 count_chars_in_width(rest_value, max_break_width)
1517 - string_op_leaves_length
1519 maybe_break_idx = self._get_break_idx(rest_value, max_bidx)
1520 if maybe_break_idx is None:
1521 # If we are unable to algorithmically determine a good split
1522 # and this string has custom splits registered to it, we
1523 # fall back to using them--which means we have to start
1524 # over from the beginning.
1526 rest_value = LL[string_idx].value
1527 string_line_results = []
1528 first_string_line = True
1529 use_custom_breakpoints = True
1532 # Otherwise, we stop splitting here.
1535 break_idx = maybe_break_idx
1537 # --- Construct `next_value`
1538 next_value = rest_value[:break_idx] + QUOTE
1540 # HACK: The following 'if' statement is a hack to fix the custom
1541 # breakpoint index in the case of either: (a) substrings that were
1542 # f-strings but will have the 'f' prefix removed OR (b) substrings
1543 # that were not f-strings but will now become f-strings because of
1544 # redundant use of the 'f' prefix (i.e. none of the substrings
1545 # contain f-expressions but one or more of them had the 'f' prefix
1546 # anyway; in which case, we will prepend 'f' to _all_ substrings).
1548 # There is probably a better way to accomplish what is being done
1551 # If this substring is an f-string, we _could_ remove the 'f'
1552 # prefix, and the current custom split did NOT originally use a
1555 use_custom_breakpoints
1556 and not csplit.has_prefix
1558 # `next_value == prefix + QUOTE` happens when the custom
1559 # split is an empty string.
1560 next_value == prefix + QUOTE
1561 or next_value != self._normalize_f_string(next_value, prefix)
1564 # Then `csplit.break_idx` will be off by one after removing
1567 next_value = rest_value[:break_idx] + QUOTE
1569 if drop_pointless_f_prefix:
1570 next_value = self._normalize_f_string(next_value, prefix)
1572 # --- Construct `next_leaf`
1573 next_leaf = Leaf(token.STRING, next_value)
1574 insert_str_child(next_leaf)
1575 self._maybe_normalize_string_quotes(next_leaf)
1577 # --- Construct `next_line`
1578 next_line = line.clone()
1579 maybe_append_string_operators(next_line)
1580 next_line.append(next_leaf)
1581 string_line_results.append(Ok(next_line))
1583 rest_value = prefix + QUOTE + rest_value[break_idx:]
1584 first_string_line = False
1586 yield from string_line_results
1588 if drop_pointless_f_prefix:
1589 rest_value = self._normalize_f_string(rest_value, prefix)
1591 rest_leaf = Leaf(token.STRING, rest_value)
1592 insert_str_child(rest_leaf)
1594 # NOTE: I could not find a test case that verifies that the following
1595 # line is actually necessary, but it seems to be. Otherwise we risk
1596 # not normalizing the last substring, right?
1597 self._maybe_normalize_string_quotes(rest_leaf)
1599 last_line = line.clone()
1600 maybe_append_string_operators(last_line)
1602 # If there are any leaves to the right of the target string...
1603 if is_valid_index(string_idx + 1):
1604 # We use `temp_value` here to determine how long the last line
1605 # would be if we were to append all the leaves to the right of the
1606 # target string to the last string line.
1607 temp_value = rest_value
1608 for leaf in LL[string_idx + 1 :]:
1609 temp_value += str(leaf)
1610 if leaf.type == token.LPAR:
1613 # Try to fit them all on the same line with the last substring...
1615 str_width(temp_value) <= max_last_string_column()
1616 or LL[string_idx + 1].type == token.COMMA
1618 last_line.append(rest_leaf)
1619 append_leaves(last_line, line, LL[string_idx + 1 :])
1621 # Otherwise, place the last substring on one line and everything
1622 # else on a line below that...
1624 last_line.append(rest_leaf)
1627 non_string_line = line.clone()
1628 append_leaves(non_string_line, line, LL[string_idx + 1 :])
1629 yield Ok(non_string_line)
1630 # Else the target string was the last leaf...
1632 last_line.append(rest_leaf)
1633 last_line.comments = line.comments.copy()
1636 def _iter_nameescape_slices(self, string: str) -> Iterator[Tuple[Index, Index]]:
1639 All ranges of @string which, if @string were to be split there,
1640 would result in the splitting of an \\N{...} expression (which is NOT
1643 # True - the previous backslash was unescaped
1644 # False - the previous backslash was escaped *or* there was no backslash
1645 previous_was_unescaped_backslash = False
1646 it = iter(enumerate(string))
1649 previous_was_unescaped_backslash = not previous_was_unescaped_backslash
1651 if not previous_was_unescaped_backslash or c != "N":
1652 previous_was_unescaped_backslash = False
1654 previous_was_unescaped_backslash = False
1656 begin = idx - 1 # the position of backslash before \N{...}
1662 # malformed nameescape expression?
1663 # should have been detected by AST parsing earlier...
1664 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
1667 def _iter_fexpr_slices(self, string: str) -> Iterator[Tuple[Index, Index]]:
1670 All ranges of @string which, if @string were to be split there,
1671 would result in the splitting of an f-expression (which is NOT
1674 if "f" not in get_string_prefix(string).lower():
1676 yield from iter_fexpr_spans(string)
1678 def _get_illegal_split_indices(self, string: str) -> Set[Index]:
1679 illegal_indices: Set[Index] = set()
1681 self._iter_fexpr_slices(string),
1682 self._iter_nameescape_slices(string),
1684 for it in iterators:
1685 for begin, end in it:
1686 illegal_indices.update(range(begin, end + 1))
1687 return illegal_indices
1689 def _get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]:
1691 This method contains the algorithm that StringSplitter uses to
1692 determine which character to split each string at.
1695 @string: The substring that we are attempting to split.
1696 @max_break_idx: The ideal break index. We will return this value if it
1697 meets all the necessary conditions. In the likely event that it
1698 doesn't we will try to find the closest index BELOW @max_break_idx
1699 that does. If that fails, we will expand our search by also
1700 considering all valid indices ABOVE @max_break_idx.
1703 * assert_is_leaf_string(@string)
1704 * 0 <= @max_break_idx < len(@string)
1707 break_idx, if an index is able to be found that meets all of the
1708 conditions listed in the 'Transformations' section of this classes'
1713 is_valid_index = is_valid_index_factory(string)
1715 assert is_valid_index(max_break_idx)
1716 assert_is_leaf_string(string)
1718 _illegal_split_indices = self._get_illegal_split_indices(string)
1720 def breaks_unsplittable_expression(i: Index) -> bool:
1723 True iff returning @i would result in the splitting of an
1724 unsplittable expression (which is NOT allowed).
1726 return i in _illegal_split_indices
1728 def passes_all_checks(i: Index) -> bool:
1731 True iff ALL of the conditions listed in the 'Transformations'
1732 section of this classes' docstring would be be met by returning @i.
1734 is_space = string[i] == " "
1735 is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS
1737 is_not_escaped = True
1739 while is_valid_index(j) and string[j] == "\\":
1740 is_not_escaped = not is_not_escaped
1744 len(string[i:]) >= self.MIN_SUBSTR_SIZE
1745 and len(string[:i]) >= self.MIN_SUBSTR_SIZE
1748 (is_space or is_split_safe)
1751 and not breaks_unsplittable_expression(i)
1754 # First, we check all indices BELOW @max_break_idx.
1755 break_idx = max_break_idx
1756 while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx):
1759 if not passes_all_checks(break_idx):
1760 # If that fails, we check all indices ABOVE @max_break_idx.
1762 # If we are able to find a valid index here, the next line is going
1763 # to be longer than the specified line length, but it's probably
1764 # better than doing nothing at all.
1765 break_idx = max_break_idx + 1
1766 while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx):
1769 if not is_valid_index(break_idx) or not passes_all_checks(break_idx):
1774 def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None:
1775 if self.normalize_strings:
1776 leaf.value = normalize_string_quotes(leaf.value)
1778 def _normalize_f_string(self, string: str, prefix: str) -> str:
1781 * assert_is_leaf_string(@string)
1784 * If @string is an f-string that contains no f-expressions, we
1785 return a string identical to @string except that the 'f' prefix
1786 has been stripped and all double braces (i.e. '{{' or '}}') have
1787 been normalized (i.e. turned into '{' or '}').
1789 * Otherwise, we return @string.
1791 assert_is_leaf_string(string)
1793 if "f" in prefix and not fstring_contains_expr(string):
1794 new_prefix = prefix.replace("f", "")
1796 temp = string[len(prefix) :]
1797 temp = re.sub(r"\{\{", "{", temp)
1798 temp = re.sub(r"\}\}", "}", temp)
1801 return f"{new_prefix}{new_string}"
1805 def _get_string_operator_leaves(self, leaves: Iterable[Leaf]) -> List[Leaf]:
1808 string_op_leaves = []
1810 while LL[i].type in self.STRING_OPERATORS + [token.NAME]:
1811 prefix_leaf = Leaf(LL[i].type, str(LL[i]).strip())
1812 string_op_leaves.append(prefix_leaf)
1814 return string_op_leaves
1817 class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin):
1819 StringTransformer that wraps strings in parens and then splits at the LPAR.
1822 All of the requirements listed in BaseStringSplitter's docstring in
1823 addition to the requirements listed below:
1825 * The line is a return/yield statement, which returns/yields a string.
1827 * The line is part of a ternary expression (e.g. `x = y if cond else
1828 z`) such that the line starts with `else <string>`, where <string> is
1831 * The line is an assert statement, which ends with a string.
1833 * The line is an assignment statement (e.g. `x = <string>` or `x +=
1834 <string>`) such that the variable is being assigned the value of some
1837 * The line is a dictionary key assignment where some valid key is being
1838 assigned the value of some string.
1840 * The line is an lambda expression and the value is a string.
1842 * The line starts with an "atom" string that prefers to be wrapped in
1843 parens. It's preferred to be wrapped when it's is an immediate child of
1844 a list/set/tuple literal, AND the string is surrounded by commas (or is
1845 the first/last child).
1848 The chosen string is wrapped in parentheses and then split at the LPAR.
1850 We then have one line which ends with an LPAR and another line that
1851 starts with the chosen string. The latter line is then split again at
1852 the RPAR. This results in the RPAR (and possibly a trailing comma)
1853 being placed on its own line.
1855 NOTE: If any leaves exist to the right of the chosen string (except
1856 for a trailing comma, which would be placed after the RPAR), those
1857 leaves are placed inside the parentheses. In effect, the chosen
1858 string is not necessarily being "wrapped" by parentheses. We can,
1859 however, count on the LPAR being placed directly before the chosen
1862 In other words, StringParenWrapper creates "atom" strings. These
1863 can then be split again by StringSplitter, if necessary.
1866 In the event that a string line split by StringParenWrapper is
1867 changed such that it no longer needs to be given its own line,
1868 StringParenWrapper relies on StringParenStripper to clean up the
1869 parentheses it created.
1871 For "atom" strings that prefers to be wrapped in parens, it requires
1872 StringSplitter to hold the split until the string is wrapped in parens.
1875 def do_splitter_match(self, line: Line) -> TMatchResult:
1878 if line.leaves[-1].type in OPENING_BRACKETS:
1880 "Cannot wrap parens around a line that ends in an opening bracket."
1884 self._return_match(LL)
1885 or self._else_match(LL)
1886 or self._assert_match(LL)
1887 or self._assign_match(LL)
1888 or self._dict_or_lambda_match(LL)
1889 or self._prefer_paren_wrap_match(LL)
1892 if string_idx is not None:
1893 string_value = line.leaves[string_idx].value
1894 # If the string has neither spaces nor East Asian stops...
1896 char == " " or char in SPLIT_SAFE_CHARS for char in string_value
1898 # And will still violate the line length limit when split...
1899 max_string_width = self.line_length - ((line.depth + 1) * 4)
1900 if str_width(string_value) > max_string_width:
1901 # And has no associated custom splits...
1902 if not self.has_custom_splits(string_value):
1903 # Then we should NOT put this string on its own line.
1905 "We do not wrap long strings in parentheses when the"
1906 " resultant line would still be over the specified line"
1907 " length and can't be split further by StringSplitter."
1909 return Ok([string_idx])
1911 return TErr("This line does not contain any non-atomic strings.")
1914 def _return_match(LL: List[Leaf]) -> Optional[int]:
1917 string_idx such that @LL[string_idx] is equal to our target (i.e.
1918 matched) string, if this line matches the return/yield statement
1919 requirements listed in the 'Requirements' section of this classes'
1924 # If this line is apart of a return/yield statement and the first leaf
1925 # contains either the "return" or "yield" keywords...
1926 if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[
1928 ].value in ["return", "yield"]:
1929 is_valid_index = is_valid_index_factory(LL)
1931 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
1932 # The next visible leaf MUST contain a string...
1933 if is_valid_index(idx) and LL[idx].type == token.STRING:
1939 def _else_match(LL: List[Leaf]) -> Optional[int]:
1942 string_idx such that @LL[string_idx] is equal to our target (i.e.
1943 matched) string, if this line matches the ternary expression
1944 requirements listed in the 'Requirements' section of this classes'
1949 # If this line is apart of a ternary expression and the first leaf
1950 # contains the "else" keyword...
1952 parent_type(LL[0]) == syms.test
1953 and LL[0].type == token.NAME
1954 and LL[0].value == "else"
1956 is_valid_index = is_valid_index_factory(LL)
1958 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
1959 # The next visible leaf MUST contain a string...
1960 if is_valid_index(idx) and LL[idx].type == token.STRING:
1966 def _assert_match(LL: List[Leaf]) -> Optional[int]:
1969 string_idx such that @LL[string_idx] is equal to our target (i.e.
1970 matched) string, if this line matches the assert statement
1971 requirements listed in the 'Requirements' section of this classes'
1976 # If this line is apart of an assert statement and the first leaf
1977 # contains the "assert" keyword...
1978 if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert":
1979 is_valid_index = is_valid_index_factory(LL)
1981 for i, leaf in enumerate(LL):
1982 # We MUST find a comma...
1983 if leaf.type == token.COMMA:
1984 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
1986 # That comma MUST be followed by a string...
1987 if is_valid_index(idx) and LL[idx].type == token.STRING:
1990 # Skip the string trailer, if one exists.
1991 string_parser = StringParser()
1992 idx = string_parser.parse(LL, string_idx)
1994 # But no more leaves are allowed...
1995 if not is_valid_index(idx):
2001 def _assign_match(LL: List[Leaf]) -> Optional[int]:
2004 string_idx such that @LL[string_idx] is equal to our target (i.e.
2005 matched) string, if this line matches the assignment statement
2006 requirements listed in the 'Requirements' section of this classes'
2011 # If this line is apart of an expression statement or is a function
2012 # argument AND the first leaf contains a variable name...
2014 parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power]
2015 and LL[0].type == token.NAME
2017 is_valid_index = is_valid_index_factory(LL)
2019 for i, leaf in enumerate(LL):
2020 # We MUST find either an '=' or '+=' symbol...
2021 if leaf.type in [token.EQUAL, token.PLUSEQUAL]:
2022 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2024 # That symbol MUST be followed by a string...
2025 if is_valid_index(idx) and LL[idx].type == token.STRING:
2028 # Skip the string trailer, if one exists.
2029 string_parser = StringParser()
2030 idx = string_parser.parse(LL, string_idx)
2032 # The next leaf MAY be a comma iff this line is apart
2033 # of a function argument...
2035 parent_type(LL[0]) == syms.argument
2036 and is_valid_index(idx)
2037 and LL[idx].type == token.COMMA
2041 # But no more leaves are allowed...
2042 if not is_valid_index(idx):
2048 def _dict_or_lambda_match(LL: List[Leaf]) -> Optional[int]:
2051 string_idx such that @LL[string_idx] is equal to our target (i.e.
2052 matched) string, if this line matches the dictionary key assignment
2053 statement or lambda expression requirements listed in the
2054 'Requirements' section of this classes' docstring.
2058 # If this line is a part of a dictionary key assignment or lambda expression...
2059 parent_types = [parent_type(LL[0]), parent_type(LL[0].parent)]
2060 if syms.dictsetmaker in parent_types or syms.lambdef in parent_types:
2061 is_valid_index = is_valid_index_factory(LL)
2063 for i, leaf in enumerate(LL):
2064 # We MUST find a colon, it can either be dict's or lambda's colon...
2065 if leaf.type == token.COLON and i < len(LL) - 1:
2066 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
2068 # That colon MUST be followed by a string...
2069 if is_valid_index(idx) and LL[idx].type == token.STRING:
2072 # Skip the string trailer, if one exists.
2073 string_parser = StringParser()
2074 idx = string_parser.parse(LL, string_idx)
2076 # That string MAY be followed by a comma...
2077 if is_valid_index(idx) and LL[idx].type == token.COMMA:
2080 # But no more leaves are allowed...
2081 if not is_valid_index(idx):
2087 self, line: Line, string_indices: List[int]
2088 ) -> Iterator[TResult[Line]]:
2090 assert len(string_indices) == 1, (
2091 f"{self.__class__.__name__} should only find one match at a time, found"
2092 f" {len(string_indices)}"
2094 string_idx = string_indices[0]
2096 is_valid_index = is_valid_index_factory(LL)
2097 insert_str_child = insert_str_child_factory(LL[string_idx])
2100 ends_with_comma = False
2101 if LL[comma_idx].type == token.COMMA:
2102 ends_with_comma = True
2104 leaves_to_steal_comments_from = [LL[string_idx]]
2106 leaves_to_steal_comments_from.append(LL[comma_idx])
2109 first_line = line.clone()
2110 left_leaves = LL[:string_idx]
2112 # We have to remember to account for (possibly invisible) LPAR and RPAR
2113 # leaves that already wrapped the target string. If these leaves do
2114 # exist, we will replace them with our own LPAR and RPAR leaves.
2115 old_parens_exist = False
2116 if left_leaves and left_leaves[-1].type == token.LPAR:
2117 old_parens_exist = True
2118 leaves_to_steal_comments_from.append(left_leaves[-1])
2121 append_leaves(first_line, line, left_leaves)
2123 lpar_leaf = Leaf(token.LPAR, "(")
2124 if old_parens_exist:
2125 replace_child(LL[string_idx - 1], lpar_leaf)
2127 insert_str_child(lpar_leaf)
2128 first_line.append(lpar_leaf)
2130 # We throw inline comments that were originally to the right of the
2131 # target string to the top line. They will now be shown to the right of
2133 for leaf in leaves_to_steal_comments_from:
2134 for comment_leaf in line.comments_after(leaf):
2135 first_line.append(comment_leaf, preformatted=True)
2137 yield Ok(first_line)
2139 # --- Middle (String) Line
2140 # We only need to yield one (possibly too long) string line, since the
2141 # `StringSplitter` will break it down further if necessary.
2142 string_value = LL[string_idx].value
2145 depth=line.depth + 1,
2146 inside_brackets=True,
2147 should_split_rhs=line.should_split_rhs,
2148 magic_trailing_comma=line.magic_trailing_comma,
2150 string_leaf = Leaf(token.STRING, string_value)
2151 insert_str_child(string_leaf)
2152 string_line.append(string_leaf)
2154 old_rpar_leaf = None
2155 if is_valid_index(string_idx + 1):
2156 right_leaves = LL[string_idx + 1 :]
2160 if old_parens_exist:
2161 assert right_leaves and right_leaves[-1].type == token.RPAR, (
2162 "Apparently, old parentheses do NOT exist?!"
2163 f" (left_leaves={left_leaves}, right_leaves={right_leaves})"
2165 old_rpar_leaf = right_leaves.pop()
2166 elif right_leaves and right_leaves[-1].type == token.RPAR:
2167 # Special case for lambda expressions as dict's value, e.g.:
2169 # "key": lambda x: f"formatted: {x},
2171 # After wrapping the dict's value with parentheses, the string is
2172 # followed by a RPAR but its opening bracket is lambda's, not
2174 # "key": (lambda x: f"formatted: {x}),
2175 opening_bracket = right_leaves[-1].opening_bracket
2176 if opening_bracket is not None and opening_bracket in left_leaves:
2177 index = left_leaves.index(opening_bracket)
2180 and index < len(left_leaves) - 1
2181 and left_leaves[index - 1].type == token.COLON
2182 and left_leaves[index + 1].value == "lambda"
2186 append_leaves(string_line, line, right_leaves)
2188 yield Ok(string_line)
2191 last_line = line.clone()
2192 last_line.bracket_tracker = first_line.bracket_tracker
2194 new_rpar_leaf = Leaf(token.RPAR, ")")
2195 if old_rpar_leaf is not None:
2196 replace_child(old_rpar_leaf, new_rpar_leaf)
2198 insert_str_child(new_rpar_leaf)
2199 last_line.append(new_rpar_leaf)
2201 # If the target string ended with a comma, we place this comma to the
2202 # right of the RPAR on the last line.
2204 comma_leaf = Leaf(token.COMMA, ",")
2205 replace_child(LL[comma_idx], comma_leaf)
2206 last_line.append(comma_leaf)
2213 A state machine that aids in parsing a string's "trailer", which can be
2214 either non-existent, an old-style formatting sequence (e.g. `% varX` or `%
2215 (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX,
2218 NOTE: A new StringParser object MUST be instantiated for each string
2219 trailer we need to parse.
2222 We shall assume that `line` equals the `Line` object that corresponds
2223 to the following line of python code:
2225 x = "Some {}.".format("String") + some_other_string
2228 Furthermore, we will assume that `string_idx` is some index such that:
2230 assert line.leaves[string_idx].value == "Some {}."
2233 The following code snippet then holds:
2235 string_parser = StringParser()
2236 idx = string_parser.parse(line.leaves, string_idx)
2237 assert line.leaves[idx].type == token.PLUS
2241 DEFAULT_TOKEN: Final = 20210605
2243 # String Parser States
2248 SINGLE_FMT_ARG: Final = 5
2253 # Lookup Table for Next State
2254 _goto: Final[Dict[Tuple[ParserState, NodeType], ParserState]] = {
2255 # A string trailer may start with '.' OR '%'.
2256 (START, token.DOT): DOT,
2257 (START, token.PERCENT): PERCENT,
2258 (START, DEFAULT_TOKEN): DONE,
2259 # A '.' MUST be followed by an attribute or method name.
2260 (DOT, token.NAME): NAME,
2261 # A method name MUST be followed by an '(', whereas an attribute name
2262 # is the last symbol in the string trailer.
2263 (NAME, token.LPAR): LPAR,
2264 (NAME, DEFAULT_TOKEN): DONE,
2265 # A '%' symbol can be followed by an '(' or a single argument (e.g. a
2266 # string or variable name).
2267 (PERCENT, token.LPAR): LPAR,
2268 (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG,
2269 # If a '%' symbol is followed by a single argument, that argument is
2270 # the last leaf in the string trailer.
2271 (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE,
2272 # If present, a ')' symbol is the last symbol in a string trailer.
2273 # (NOTE: LPARS and nested RPARS are not included in this lookup table,
2274 # since they are treated as a special case by the parsing logic in this
2275 # classes' implementation.)
2276 (RPAR, DEFAULT_TOKEN): DONE,
2279 def __init__(self) -> None:
2280 self._state = self.START
2281 self._unmatched_lpars = 0
2283 def parse(self, leaves: List[Leaf], string_idx: int) -> int:
2286 * @leaves[@string_idx].type == token.STRING
2289 The index directly after the last leaf which is apart of the string
2290 trailer, if a "trailer" exists.
2292 @string_idx + 1, if no string "trailer" exists.
2294 assert leaves[string_idx].type == token.STRING
2296 idx = string_idx + 1
2297 while idx < len(leaves) and self._next_state(leaves[idx]):
2301 def _next_state(self, leaf: Leaf) -> bool:
2304 * On the first call to this function, @leaf MUST be the leaf that
2305 was directly after the string leaf in question (e.g. if our target
2306 string is `line.leaves[i]` then the first call to this method must
2307 be `line.leaves[i + 1]`).
2308 * On the next call to this function, the leaf parameter passed in
2309 MUST be the leaf directly following @leaf.
2312 True iff @leaf is apart of the string's trailer.
2314 # We ignore empty LPAR or RPAR leaves.
2315 if is_empty_par(leaf):
2318 next_token = leaf.type
2319 if next_token == token.LPAR:
2320 self._unmatched_lpars += 1
2322 current_state = self._state
2324 # The LPAR parser state is a special case. We will return True until we
2325 # find the matching RPAR token.
2326 if current_state == self.LPAR:
2327 if next_token == token.RPAR:
2328 self._unmatched_lpars -= 1
2329 if self._unmatched_lpars == 0:
2330 self._state = self.RPAR
2331 # Otherwise, we use a lookup table to determine the next state.
2333 # If the lookup table matches the current state to the next
2334 # token, we use the lookup table.
2335 if (current_state, next_token) in self._goto:
2336 self._state = self._goto[current_state, next_token]
2338 # Otherwise, we check if a the current state was assigned a
2340 if (current_state, self.DEFAULT_TOKEN) in self._goto:
2341 self._state = self._goto[current_state, self.DEFAULT_TOKEN]
2342 # If no default has been assigned, then this parser has a logic
2345 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
2347 if self._state == self.DONE:
2353 def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]:
2355 Factory for a convenience function that is used to orphan @string_leaf
2356 and then insert multiple new leaves into the same part of the node
2357 structure that @string_leaf had originally occupied.
2360 Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N =
2361 string_leaf.parent`. Assume the node `N` has the following
2368 Leaf(STRING, '"foo"'),
2372 We then run the code snippet shown below.
2374 insert_str_child = insert_str_child_factory(string_leaf)
2376 lpar = Leaf(token.LPAR, '(')
2377 insert_str_child(lpar)
2379 bar = Leaf(token.STRING, '"bar"')
2380 insert_str_child(bar)
2382 rpar = Leaf(token.RPAR, ')')
2383 insert_str_child(rpar)
2386 After which point, it follows that `string_leaf.parent is None` and
2387 the node `N` now has the following structure:
2394 Leaf(STRING, '"bar"'),
2399 string_parent = string_leaf.parent
2400 string_child_idx = string_leaf.remove()
2402 def insert_str_child(child: LN) -> None:
2403 nonlocal string_child_idx
2405 assert string_parent is not None
2406 assert string_child_idx is not None
2408 string_parent.insert_child(string_child_idx, child)
2409 string_child_idx += 1
2411 return insert_str_child
2414 def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]:
2420 is_valid_index = is_valid_index_factory(my_list)
2422 assert is_valid_index(0)
2423 assert is_valid_index(2)
2425 assert not is_valid_index(3)
2426 assert not is_valid_index(-1)
2430 def is_valid_index(idx: int) -> bool:
2433 True iff @idx is positive AND seq[@idx] does NOT raise an
2436 return 0 <= idx < len(seq)
2438 return is_valid_index