All patches and comments are welcome. Please squash your changes to logical
commits before using git-format-patch and git-send-email to
patches@git.madduck.net.
If you'd read over the Git project's submission guidelines and adhered to them,
I'd be especially grateful.
2 String transformers that can split and merge strings.
6 from abc import ABC, abstractmethod
7 from collections import defaultdict
8 from dataclasses import dataclass
26 if sys.version_info < (3, 8):
27 from typing_extensions import Final, Literal
29 from typing import Literal, Final
31 from mypy_extensions import trait
33 from black.brackets import BracketMatchError
34 from black.comments import contains_pragma_comment
35 from black.lines import Line, append_leaves
36 from black.mode import Feature
37 from black.nodes import (
48 from black.rusty import Err, Ok, Result
49 from black.strings import (
50 assert_is_leaf_string,
53 normalize_string_quotes,
55 from blib2to3.pgen2 import token
56 from blib2to3.pytree import Leaf, Node
59 class CannotTransform(Exception):
60 """Base class for errors raised by Transformers."""
65 LN = Union[Leaf, Node]
66 Transformer = Callable[[Line, Collection[Feature]], Iterator[Line]]
71 TResult = Result[T, CannotTransform] # (T)ransform Result
72 TMatchResult = TResult[Index]
75 def TErr(err_msg: str) -> Err[CannotTransform]:
78 Convenience function used when working with the TResult type.
80 cant_transform = CannotTransform(err_msg)
81 return Err(cant_transform)
84 def hug_power_op(line: Line, features: Collection[Feature]) -> Iterator[Line]:
85 """A transformer which normalizes spacing around power operators."""
87 # Performance optimization to avoid unnecessary Leaf clones and other ops.
88 for leaf in line.leaves:
89 if leaf.type == token.DOUBLESTAR:
92 raise CannotTransform("No doublestar token was found in the line.")
94 def is_simple_lookup(index: int, step: Literal[1, -1]) -> bool:
95 # Brackets and parentheses indicate calls, subscripts, etc. ...
96 # basically stuff that doesn't count as "simple". Only a NAME lookup
97 # or dotted lookup (eg. NAME.NAME) is OK.
99 disallowed = {token.RPAR, token.RSQB}
101 disallowed = {token.LPAR, token.LSQB}
103 while 0 <= index < len(line.leaves):
104 current = line.leaves[index]
105 if current.type in disallowed:
107 if current.type not in {token.NAME, token.DOT} or current.value == "for":
108 # If the current token isn't disallowed, we'll assume this is simple as
109 # only the disallowed tokens are semantically attached to this lookup
110 # expression we're checking. Also, stop early if we hit the 'for' bit
111 # of a comprehension.
118 def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool:
119 # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple
120 # lookup (see above), with or without a preceding unary operator.
121 start = line.leaves[index]
122 if start.type in {token.NAME, token.NUMBER}:
123 return is_simple_lookup(index, step=(1 if kind == "exponent" else -1))
125 if start.type in {token.PLUS, token.MINUS, token.TILDE}:
126 if line.leaves[index + 1].type in {token.NAME, token.NUMBER}:
127 # step is always one as bases with a preceding unary op will be checked
128 # for simplicity starting from the next token (so it'll hit the check
130 return is_simple_lookup(index + 1, step=1)
134 new_line = line.clone()
136 for idx, leaf in enumerate(line.leaves):
137 new_leaf = leaf.clone()
143 (0 < idx < len(line.leaves) - 1)
144 and leaf.type == token.DOUBLESTAR
145 and is_simple_operand(idx - 1, kind="base")
146 and line.leaves[idx - 1].value != "lambda"
147 and is_simple_operand(idx + 1, kind="exponent")
152 # We have to be careful to make a new line properly:
153 # - bracket related metadata must be maintained (handled by Line.append)
154 # - comments need to copied over, updating the leaf IDs they're attached to
155 new_line.append(new_leaf, preformatted=True)
156 for comment_leaf in line.comments_after(leaf):
157 new_line.append(comment_leaf, preformatted=True)
162 class StringTransformer(ABC):
164 An implementation of the Transformer protocol that relies on its
165 subclasses overriding the template methods `do_match(...)` and
168 This Transformer works exclusively on strings (for example, by merging
171 The following sections can be found among the docstrings of each concrete
172 StringTransformer subclass.
175 Which requirements must be met of the given Line for this
176 StringTransformer to be applied?
179 If the given Line meets all of the above requirements, which string
180 transformations can you expect to be applied to it by this
184 What contractual agreements does this StringTransformer have with other
185 StringTransfomers? Such collaborations should be eliminated/minimized
189 __name__: Final = "StringTransformer"
191 # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with
193 def __init__(self, line_length: int, normalize_strings: bool) -> None:
194 self.line_length = line_length
195 self.normalize_strings = normalize_strings
198 def do_match(self, line: Line) -> TMatchResult:
201 * Ok(string_idx) such that `line.leaves[string_idx]` is our target
202 string, if a match was able to be made.
204 * Err(CannotTransform), if a match was not able to be made.
208 def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
211 * Ok(new_line) where new_line is the new transformed line.
213 * Err(CannotTransform) if the transformation failed for some reason. The
214 `do_match(...)` template method should usually be used to reject
215 the form of the given Line, but in some cases it is difficult to
216 know whether or not a Line meets the StringTransformer's
217 requirements until the transformation is already midway.
220 This method should NOT mutate @line directly, but it MAY mutate the
221 Line's underlying Node structure. (WARNING: If the underlying Node
222 structure IS altered, then this method should NOT be allowed to
223 yield an CannotTransform after that point.)
226 def __call__(self, line: Line, _features: Collection[Feature]) -> Iterator[Line]:
228 StringTransformer instances have a call signature that mirrors that of
229 the Transformer type.
232 CannotTransform(...) if the concrete StringTransformer class is unable
235 # Optimization to avoid calling `self.do_match(...)` when the line does
236 # not contain any string.
237 if not any(leaf.type == token.STRING for leaf in line.leaves):
238 raise CannotTransform("There are no strings in this line.")
240 match_result = self.do_match(line)
242 if isinstance(match_result, Err):
243 cant_transform = match_result.err()
244 raise CannotTransform(
245 f"The string transformer {self.__class__.__name__} does not recognize"
246 " this line as one that it can transform."
247 ) from cant_transform
249 string_idx = match_result.ok()
251 for line_result in self.do_transform(line, string_idx):
252 if isinstance(line_result, Err):
253 cant_transform = line_result.err()
254 raise CannotTransform(
255 "StringTransformer failed while attempting to transform string."
256 ) from cant_transform
257 line = line_result.ok()
263 """A custom (i.e. manual) string split.
265 A single CustomSplit instance represents a single substring.
268 Consider the following string:
275 This string will correspond to the following three CustomSplit instances:
277 CustomSplit(False, 16)
278 CustomSplit(False, 17)
279 CustomSplit(True, 16)
288 class CustomSplitMapMixin:
290 This mixin class is used to map merged strings to a sequence of
291 CustomSplits, which will then be used to re-split the strings iff none of
292 the resultant substrings go over the configured max line length.
295 _Key: ClassVar = Tuple[StringID, str]
296 _CUSTOM_SPLIT_MAP: ClassVar[Dict[_Key, Tuple[CustomSplit, ...]]] = defaultdict(
301 def _get_key(string: str) -> "CustomSplitMapMixin._Key":
304 A unique identifier that is used internally to map @string to a
305 group of custom splits.
307 return (id(string), string)
309 def add_custom_splits(
310 self, string: str, custom_splits: Iterable[CustomSplit]
312 """Custom Split Map Setter Method
315 Adds a mapping from @string to the custom splits @custom_splits.
317 key = self._get_key(string)
318 self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)
320 def pop_custom_splits(self, string: str) -> List[CustomSplit]:
321 """Custom Split Map Getter Method
324 * A list of the custom splits that are mapped to @string, if any
330 Deletes the mapping between @string and its associated custom
331 splits (which are returned to the caller).
333 key = self._get_key(string)
335 custom_splits = self._CUSTOM_SPLIT_MAP[key]
336 del self._CUSTOM_SPLIT_MAP[key]
338 return list(custom_splits)
340 def has_custom_splits(self, string: str) -> bool:
343 True iff @string is associated with a set of custom splits.
345 key = self._get_key(string)
346 return key in self._CUSTOM_SPLIT_MAP
349 class StringMerger(StringTransformer, CustomSplitMapMixin):
350 """StringTransformer that merges strings together.
353 (A) The line contains adjacent strings such that ALL of the validation checks
354 listed in StringMerger.__validate_msg(...)'s docstring pass.
356 (B) The line contains a string which uses line continuation backslashes.
359 Depending on which of the two requirements above where met, either:
361 (A) The string group associated with the target string is merged.
363 (B) All line-continuation backslashes are removed from the target string.
366 StringMerger provides custom split information to StringSplitter.
369 def do_match(self, line: Line) -> TMatchResult:
372 is_valid_index = is_valid_index_factory(LL)
374 for i, leaf in enumerate(LL):
376 leaf.type == token.STRING
377 and is_valid_index(i + 1)
378 and LL[i + 1].type == token.STRING
382 if leaf.type == token.STRING and "\\\n" in leaf.value:
385 return TErr("This line has no strings that need merging.")
387 def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
389 rblc_result = self._remove_backslash_line_continuation_chars(
392 if isinstance(rblc_result, Ok):
393 new_line = rblc_result.ok()
395 msg_result = self._merge_string_group(new_line, string_idx)
396 if isinstance(msg_result, Ok):
397 new_line = msg_result.ok()
399 if isinstance(rblc_result, Err) and isinstance(msg_result, Err):
400 msg_cant_transform = msg_result.err()
401 rblc_cant_transform = rblc_result.err()
402 cant_transform = CannotTransform(
403 "StringMerger failed to merge any strings in this line."
406 # Chain the errors together using `__cause__`.
407 msg_cant_transform.__cause__ = rblc_cant_transform
408 cant_transform.__cause__ = msg_cant_transform
410 yield Err(cant_transform)
415 def _remove_backslash_line_continuation_chars(
416 line: Line, string_idx: int
419 Merge strings that were split across multiple lines using
420 line-continuation backslashes.
423 Ok(new_line), if @line contains backslash line-continuation
426 Err(CannotTransform), otherwise.
430 string_leaf = LL[string_idx]
432 string_leaf.type == token.STRING
433 and "\\\n" in string_leaf.value
434 and not has_triple_quotes(string_leaf.value)
437 f"String leaf {string_leaf} does not contain any backslash line"
438 " continuation characters."
441 new_line = line.clone()
442 new_line.comments = line.comments.copy()
443 append_leaves(new_line, line, LL)
445 new_string_leaf = new_line.leaves[string_idx]
446 new_string_leaf.value = new_string_leaf.value.replace("\\\n", "")
450 def _merge_string_group(self, line: Line, string_idx: int) -> TResult[Line]:
452 Merges string group (i.e. set of adjacent strings) where the first
453 string in the group is `line.leaves[string_idx]`.
456 Ok(new_line), if ALL of the validation checks found in
457 __validate_msg(...) pass.
459 Err(CannotTransform), otherwise.
463 is_valid_index = is_valid_index_factory(LL)
465 vresult = self._validate_msg(line, string_idx)
466 if isinstance(vresult, Err):
469 # If the string group is wrapped inside an Atom node, we must make sure
470 # to later replace that Atom with our new (merged) string leaf.
471 atom_node = LL[string_idx].parent
473 # We will place BREAK_MARK in between every two substrings that we
474 # merge. We will then later go through our final result and use the
475 # various instances of BREAK_MARK we find to add the right values to
476 # the custom split map.
477 BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@"
479 QUOTE = LL[string_idx].value[-1]
481 def make_naked(string: str, string_prefix: str) -> str:
482 """Strip @string (i.e. make it a "naked" string)
485 * assert_is_leaf_string(@string)
488 A string that is identical to @string except that
489 @string_prefix has been stripped, the surrounding QUOTE
490 characters have been removed, and any remaining QUOTE
491 characters have been escaped.
493 assert_is_leaf_string(string)
495 RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)"
496 naked_string = string[len(string_prefix) + 1 : -1]
497 naked_string = re.sub(
498 "(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string
502 # Holds the CustomSplit objects that will later be added to the custom
506 # Temporary storage for the 'has_prefix' part of the CustomSplit objects.
509 # Sets the 'prefix' variable. This is the prefix that the final merged
511 next_str_idx = string_idx
515 and is_valid_index(next_str_idx)
516 and LL[next_str_idx].type == token.STRING
518 prefix = get_string_prefix(LL[next_str_idx].value).lower()
521 # The next loop merges the string group. The final string will be
524 # The following convenience variables are used:
529 # NSS: naked next string
533 next_str_idx = string_idx
534 while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:
537 SS = LL[next_str_idx].value
538 next_prefix = get_string_prefix(SS).lower()
540 # If this is an f-string group but this substring is not prefixed
542 if "f" in prefix and "f" not in next_prefix:
543 # Then we must escape any braces contained in this substring.
544 SS = re.sub(r"(\{|\})", r"\1\1", SS)
546 NSS = make_naked(SS, next_prefix)
548 has_prefix = bool(next_prefix)
549 prefix_tracker.append(has_prefix)
551 S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE
552 NS = make_naked(S, prefix)
556 # Take a note on the index of the non-STRING leaf.
557 non_string_idx = next_str_idx
559 S_leaf = Leaf(token.STRING, S)
560 if self.normalize_strings:
561 S_leaf.value = normalize_string_quotes(S_leaf.value)
563 # Fill the 'custom_splits' list with the appropriate CustomSplit objects.
564 temp_string = S_leaf.value[len(prefix) + 1 : -1]
565 for has_prefix in prefix_tracker:
566 mark_idx = temp_string.find(BREAK_MARK)
569 ), "Logic error while filling the custom string breakpoint cache."
571 temp_string = temp_string[mark_idx + len(BREAK_MARK) :]
572 breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1
573 custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))
575 string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
577 if atom_node is not None:
578 # If not all children of the atom node are merged (this can happen
579 # when there is a standalone comment in the middle) ...
580 if non_string_idx - string_idx < len(atom_node.children):
581 # We need to replace the old STRING leaves with the new string leaf.
582 first_child_idx = LL[string_idx].remove()
583 for idx in range(string_idx + 1, non_string_idx):
585 if first_child_idx is not None:
586 atom_node.insert_child(first_child_idx, string_leaf)
588 # Else replace the atom node with the new string leaf.
589 replace_child(atom_node, string_leaf)
591 # Build the final line ('new_line') that this method will later return.
592 new_line = line.clone()
593 for i, leaf in enumerate(LL):
595 new_line.append(string_leaf)
597 if string_idx <= i < string_idx + num_of_strings:
598 for comment_leaf in line.comments_after(LL[i]):
599 new_line.append(comment_leaf, preformatted=True)
602 append_leaves(new_line, line, [leaf])
604 self.add_custom_splits(string_leaf.value, custom_splits)
608 def _validate_msg(line: Line, string_idx: int) -> TResult[None]:
609 """Validate (M)erge (S)tring (G)roup
611 Transform-time string validation logic for __merge_string_group(...).
614 * Ok(None), if ALL validation checks (listed below) pass.
616 * Err(CannotTransform), if any of the following are true:
617 - The target string group does not contain ANY stand-alone comments.
618 - The target string is not in a string group (i.e. it has no
620 - The string group has more than one inline comment.
621 - The string group has an inline comment that appears to be a pragma.
622 - The set of all string prefixes in the string group is of
623 length greater than one and is not equal to {"", "f"}.
624 - The string group consists of raw strings.
626 # We first check for "inner" stand-alone comments (i.e. stand-alone
627 # comments that have a string leaf before them AND after them).
630 found_sa_comment = False
631 is_valid_index = is_valid_index_factory(line.leaves)
632 while is_valid_index(i) and line.leaves[i].type in [
636 if line.leaves[i].type == STANDALONE_COMMENT:
637 found_sa_comment = True
638 elif found_sa_comment:
640 "StringMerger does NOT merge string groups which contain "
641 "stand-alone comments."
646 num_of_inline_string_comments = 0
647 set_of_prefixes = set()
649 for leaf in line.leaves[string_idx:]:
650 if leaf.type != token.STRING:
651 # If the string group is trailed by a comma, we count the
652 # comments trailing the comma to be one of the string group's
654 if leaf.type == token.COMMA and id(leaf) in line.comments:
655 num_of_inline_string_comments += 1
658 if has_triple_quotes(leaf.value):
659 return TErr("StringMerger does NOT merge multiline strings.")
662 prefix = get_string_prefix(leaf.value).lower()
664 return TErr("StringMerger does NOT merge raw strings.")
666 set_of_prefixes.add(prefix)
668 if id(leaf) in line.comments:
669 num_of_inline_string_comments += 1
670 if contains_pragma_comment(line.comments[id(leaf)]):
671 return TErr("Cannot merge strings which have pragma comments.")
673 if num_of_strings < 2:
675 f"Not enough strings to merge (num_of_strings={num_of_strings})."
678 if num_of_inline_string_comments > 1:
680 f"Too many inline string comments ({num_of_inline_string_comments})."
683 if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}:
684 return TErr(f"Too many different prefixes ({set_of_prefixes}).")
689 class StringParenStripper(StringTransformer):
690 """StringTransformer that strips surrounding parentheses from strings.
693 The line contains a string which is surrounded by parentheses and:
694 - The target string is NOT the only argument to a function call.
695 - The target string is NOT a "pointless" string.
696 - If the target string contains a PERCENT, the brackets are not
697 preceded or followed by an operator with higher precedence than
701 The parentheses mentioned in the 'Requirements' section are stripped.
704 StringParenStripper has its own inherent usefulness, but it is also
705 relied on to clean up the parentheses created by StringParenWrapper (in
706 the event that they are no longer needed).
709 def do_match(self, line: Line) -> TMatchResult:
712 is_valid_index = is_valid_index_factory(LL)
714 for idx, leaf in enumerate(LL):
715 # Should be a string...
716 if leaf.type != token.STRING:
719 # If this is a "pointless" string...
722 and leaf.parent.parent
723 and leaf.parent.parent.type == syms.simple_stmt
727 # Should be preceded by a non-empty LPAR...
729 not is_valid_index(idx - 1)
730 or LL[idx - 1].type != token.LPAR
731 or is_empty_lpar(LL[idx - 1])
735 # That LPAR should NOT be preceded by a function name or a closing
736 # bracket (which could be a function which returns a function or a
737 # list/dictionary that contains a function)...
738 if is_valid_index(idx - 2) and (
739 LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS
745 # Skip the string trailer, if one exists.
746 string_parser = StringParser()
747 next_idx = string_parser.parse(LL, string_idx)
749 # if the leaves in the parsed string include a PERCENT, we need to
750 # make sure the initial LPAR is NOT preceded by an operator with
751 # higher or equal precedence to PERCENT
752 if is_valid_index(idx - 2):
753 # mypy can't quite follow unless we name this
754 before_lpar = LL[idx - 2]
755 if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and (
772 # only unary PLUS/MINUS
774 and before_lpar.parent.type == syms.factor
775 and (before_lpar.type in {token.PLUS, token.MINUS})
780 # Should be followed by a non-empty RPAR...
782 is_valid_index(next_idx)
783 and LL[next_idx].type == token.RPAR
784 and not is_empty_rpar(LL[next_idx])
786 # That RPAR should NOT be followed by anything with higher
787 # precedence than PERCENT
788 if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in {
796 return Ok(string_idx)
798 return TErr("This line has no strings wrapped in parens.")
800 def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
803 string_parser = StringParser()
804 rpar_idx = string_parser.parse(LL, string_idx)
806 for leaf in (LL[string_idx - 1], LL[rpar_idx]):
807 if line.comments_after(leaf):
809 "Will not strip parentheses which have comments attached to them."
813 new_line = line.clone()
814 new_line.comments = line.comments.copy()
816 append_leaves(new_line, line, LL[: string_idx - 1])
817 except BracketMatchError:
818 # HACK: I believe there is currently a bug somewhere in
819 # right_hand_split() that is causing brackets to not be tracked
820 # properly by a shared BracketTracker.
821 append_leaves(new_line, line, LL[: string_idx - 1], preformatted=True)
823 string_leaf = Leaf(token.STRING, LL[string_idx].value)
824 LL[string_idx - 1].remove()
825 replace_child(LL[string_idx], string_leaf)
826 new_line.append(string_leaf)
829 new_line, line, LL[string_idx + 1 : rpar_idx] + LL[rpar_idx + 1 :]
832 LL[rpar_idx].remove()
837 class BaseStringSplitter(StringTransformer):
839 Abstract class for StringTransformers which transform a Line's strings by splitting
840 them or placing them on their own lines where necessary to avoid going over
841 the configured line length.
844 * The target string value is responsible for the line going over the
845 line length limit. It follows that after all of black's other line
846 split methods have been exhausted, this line (or one of the resulting
847 lines after all line splits are performed) would still be over the
848 line_length limit unless we split this string.
850 * The target string is NOT a "pointless" string (i.e. a string that has
851 no parent or siblings).
853 * The target string is not followed by an inline comment that appears
856 * The target string is not a multiline (i.e. triple-quote) string.
859 STRING_OPERATORS: Final = [
872 def do_splitter_match(self, line: Line) -> TMatchResult:
874 BaseStringSplitter asks its clients to override this method instead of
875 `StringTransformer.do_match(...)`.
877 Follows the same protocol as `StringTransformer.do_match(...)`.
879 Refer to `help(StringTransformer.do_match)` for more information.
882 def do_match(self, line: Line) -> TMatchResult:
883 match_result = self.do_splitter_match(line)
884 if isinstance(match_result, Err):
887 string_idx = match_result.ok()
888 vresult = self._validate(line, string_idx)
889 if isinstance(vresult, Err):
894 def _validate(self, line: Line, string_idx: int) -> TResult[None]:
896 Checks that @line meets all of the requirements listed in this classes'
897 docstring. Refer to `help(BaseStringSplitter)` for a detailed
898 description of those requirements.
901 * Ok(None), if ALL of the requirements are met.
903 * Err(CannotTransform), if ANY of the requirements are NOT met.
907 string_leaf = LL[string_idx]
909 max_string_length = self._get_max_string_length(line, string_idx)
910 if len(string_leaf.value) <= max_string_length:
912 "The string itself is not what is causing this line to be too long."
915 if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [
920 f"This string ({string_leaf.value}) appears to be pointless (i.e. has"
924 if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(
925 line.comments[id(line.leaves[string_idx])]
928 "Line appears to end with an inline pragma comment. Splitting the line"
929 " could modify the pragma's behavior."
932 if has_triple_quotes(string_leaf.value):
933 return TErr("We cannot split multiline strings.")
937 def _get_max_string_length(self, line: Line, string_idx: int) -> int:
939 Calculates the max string length used when attempting to determine
940 whether or not the target string is responsible for causing the line to
941 go over the line length limit.
943 WARNING: This method is tightly coupled to both StringSplitter and
944 (especially) StringParenWrapper. There is probably a better way to
945 accomplish what is being done here.
948 max_string_length: such that `line.leaves[string_idx].value >
949 max_string_length` implies that the target string IS responsible
950 for causing this line to exceed the line length limit.
954 is_valid_index = is_valid_index_factory(LL)
956 # We use the shorthand "WMA4" in comments to abbreviate "We must
957 # account for". When giving examples, we use STRING to mean some/any
960 # Finally, we use the following convenience variables:
962 # P: The leaf that is before the target string leaf.
963 # N: The leaf that is after the target string leaf.
964 # NN: The leaf that is after N.
966 # WMA4 the whitespace at the beginning of the line.
967 offset = line.depth * 4
969 if is_valid_index(string_idx - 1):
970 p_idx = string_idx - 1
972 LL[string_idx - 1].type == token.LPAR
973 and LL[string_idx - 1].value == ""
976 # If the previous leaf is an empty LPAR placeholder, we should skip it.
980 if P.type in self.STRING_OPERATORS:
981 # WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`).
982 offset += len(str(P)) + 1
984 if P.type == token.COMMA:
985 # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].
988 if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]:
989 # This conditional branch is meant to handle dictionary keys,
990 # variable assignments, 'return STRING' statement lines, and
991 # 'else STRING' ternary expression lines.
993 # WMA4 a single space.
996 # WMA4 the lengths of any leaves that came before that space,
997 # but after any closing bracket before that space.
998 for leaf in reversed(LL[: p_idx + 1]):
999 offset += len(str(leaf))
1000 if leaf.type in CLOSING_BRACKETS:
1003 if is_valid_index(string_idx + 1):
1004 N = LL[string_idx + 1]
1005 if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2:
1006 # If the next leaf is an empty RPAR placeholder, we should skip it.
1007 N = LL[string_idx + 2]
1009 if N.type == token.COMMA:
1010 # WMA4 a single comma at the end of the string (e.g `STRING,`).
1013 if is_valid_index(string_idx + 2):
1014 NN = LL[string_idx + 2]
1016 if N.type == token.DOT and NN.type == token.NAME:
1017 # This conditional branch is meant to handle method calls invoked
1018 # off of a string literal up to and including the LPAR character.
1020 # WMA4 the '.' character.
1024 is_valid_index(string_idx + 3)
1025 and LL[string_idx + 3].type == token.LPAR
1027 # WMA4 the left parenthesis character.
1030 # WMA4 the length of the method's name.
1031 offset += len(NN.value)
1033 has_comments = False
1034 for comment_leaf in line.comments_after(LL[string_idx]):
1035 if not has_comments:
1037 # WMA4 two spaces before the '#' character.
1040 # WMA4 the length of the inline comment.
1041 offset += len(comment_leaf.value)
1043 max_string_length = self.line_length - offset
1044 return max_string_length
1047 def _prefer_paren_wrap_match(LL: List[Leaf]) -> Optional[int]:
1050 string_idx such that @LL[string_idx] is equal to our target (i.e.
1051 matched) string, if this line matches the "prefer paren wrap" statement
1052 requirements listed in the 'Requirements' section of the StringParenWrapper
1057 # The line must start with a string.
1058 if LL[0].type != token.STRING:
1061 # If the string is surrounded by commas (or is the first/last child)...
1062 prev_sibling = LL[0].prev_sibling
1063 next_sibling = LL[0].next_sibling
1064 if not prev_sibling and not next_sibling and parent_type(LL[0]) == syms.atom:
1065 # If it's an atom string, we need to check the parent atom's siblings.
1066 parent = LL[0].parent
1067 assert parent is not None # For type checkers.
1068 prev_sibling = parent.prev_sibling
1069 next_sibling = parent.next_sibling
1070 if (not prev_sibling or prev_sibling.type == token.COMMA) and (
1071 not next_sibling or next_sibling.type == token.COMMA
1078 def iter_fexpr_spans(s: str) -> Iterator[Tuple[int, int]]:
1080 Yields spans corresponding to expressions in a given f-string.
1081 Spans are half-open ranges (left inclusive, right exclusive).
1082 Assumes the input string is a valid f-string, but will not crash if the input
1085 stack: List[int] = [] # our curly paren stack
1089 # if we're in a string part of the f-string, ignore escaped curly braces
1090 if not stack and i + 1 < len(s) and s[i + 1] == "{":
1102 # we've made it back out of the expression! yield the span
1108 # if we're in an expression part of the f-string, fast forward through strings
1109 # note that backslashes are not legal in the expression portion of f-strings
1112 if s[i : i + 3] in ("'''", '"""'):
1113 delim = s[i : i + 3]
1114 elif s[i] in ("'", '"'):
1118 while i < len(s) and s[i : i + len(delim)] != delim:
1125 def fstring_contains_expr(s: str) -> bool:
1126 return any(iter_fexpr_spans(s))
1129 class StringSplitter(BaseStringSplitter, CustomSplitMapMixin):
1131 StringTransformer that splits "atom" strings (i.e. strings which exist on
1132 lines by themselves).
1135 * The line consists ONLY of a single string (possibly prefixed by a
1136 string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE
1139 * All of the requirements listed in BaseStringSplitter's docstring.
1142 The string mentioned in the 'Requirements' section is split into as
1143 many substrings as necessary to adhere to the configured line length.
1145 In the final set of substrings, no substring should be smaller than
1146 MIN_SUBSTR_SIZE characters.
1148 The string will ONLY be split on spaces (i.e. each new substring should
1149 start with a space). Note that the string will NOT be split on a space
1150 which is escaped with a backslash.
1152 If the string is an f-string, it will NOT be split in the middle of an
1153 f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x
1154 else bar()} is an f-expression).
1156 If the string that is being split has an associated set of custom split
1157 records and those custom splits will NOT result in any line going over
1158 the configured line length, those custom splits are used. Otherwise the
1159 string is split as late as possible (from left-to-right) while still
1160 adhering to the transformation rules listed above.
1163 StringSplitter relies on StringMerger to construct the appropriate
1164 CustomSplit objects and add them to the custom split map.
1167 MIN_SUBSTR_SIZE: Final = 6
1169 def do_splitter_match(self, line: Line) -> TMatchResult:
1172 if self._prefer_paren_wrap_match(LL) is not None:
1173 return TErr("Line needs to be wrapped in parens first.")
1175 is_valid_index = is_valid_index_factory(LL)
1179 # The first two leaves MAY be the 'not in' keywords...
1182 and is_valid_index(idx + 1)
1183 and [LL[idx].type, LL[idx + 1].type] == [token.NAME, token.NAME]
1184 and str(LL[idx]) + str(LL[idx + 1]) == "not in"
1187 # Else the first leaf MAY be a string operator symbol or the 'in' keyword...
1188 elif is_valid_index(idx) and (
1189 LL[idx].type in self.STRING_OPERATORS
1190 or LL[idx].type == token.NAME
1191 and str(LL[idx]) == "in"
1195 # The next/first leaf MAY be an empty LPAR...
1196 if is_valid_index(idx) and is_empty_lpar(LL[idx]):
1199 # The next/first leaf MUST be a string...
1200 if not is_valid_index(idx) or LL[idx].type != token.STRING:
1201 return TErr("Line does not start with a string.")
1205 # Skip the string trailer, if one exists.
1206 string_parser = StringParser()
1207 idx = string_parser.parse(LL, string_idx)
1209 # That string MAY be followed by an empty RPAR...
1210 if is_valid_index(idx) and is_empty_rpar(LL[idx]):
1213 # That string / empty RPAR leaf MAY be followed by a comma...
1214 if is_valid_index(idx) and LL[idx].type == token.COMMA:
1217 # But no more leaves are allowed...
1218 if is_valid_index(idx):
1219 return TErr("This line does not end with a string.")
1221 return Ok(string_idx)
1223 def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
1226 QUOTE = LL[string_idx].value[-1]
1228 is_valid_index = is_valid_index_factory(LL)
1229 insert_str_child = insert_str_child_factory(LL[string_idx])
1231 prefix = get_string_prefix(LL[string_idx].value).lower()
1233 # We MAY choose to drop the 'f' prefix from substrings that don't
1234 # contain any f-expressions, but ONLY if the original f-string
1235 # contains at least one f-expression. Otherwise, we will alter the AST
1237 drop_pointless_f_prefix = ("f" in prefix) and fstring_contains_expr(
1238 LL[string_idx].value
1241 first_string_line = True
1243 string_op_leaves = self._get_string_operator_leaves(LL)
1244 string_op_leaves_length = (
1245 sum(len(str(prefix_leaf)) for prefix_leaf in string_op_leaves) + 1
1250 def maybe_append_string_operators(new_line: Line) -> None:
1253 If @line starts with a string operator and this is the first
1254 line we are constructing, this function appends the string
1255 operator to @new_line and replaces the old string operator leaf
1256 in the node structure. Otherwise this function does nothing.
1258 maybe_prefix_leaves = string_op_leaves if first_string_line else []
1259 for i, prefix_leaf in enumerate(maybe_prefix_leaves):
1260 replace_child(LL[i], prefix_leaf)
1261 new_line.append(prefix_leaf)
1264 is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA
1267 def max_last_string() -> int:
1270 The max allowed length of the string value used for the last
1271 line we will construct.
1273 result = self.line_length
1274 result -= line.depth * 4
1275 result -= 1 if ends_with_comma else 0
1276 result -= string_op_leaves_length
1279 # --- Calculate Max Break Index (for string value)
1280 # We start with the line length limit
1281 max_break_idx = self.line_length
1282 # The last index of a string of length N is N-1.
1284 # Leading whitespace is not present in the string value (e.g. Leaf.value).
1285 max_break_idx -= line.depth * 4
1286 if max_break_idx < 0:
1288 f"Unable to split {LL[string_idx].value} at such high of a line depth:"
1293 # Check if StringMerger registered any custom splits.
1294 custom_splits = self.pop_custom_splits(LL[string_idx].value)
1295 # We use them ONLY if none of them would produce lines that exceed the
1297 use_custom_breakpoints = bool(
1299 and all(csplit.break_idx <= max_break_idx for csplit in custom_splits)
1302 # Temporary storage for the remaining chunk of the string line that
1303 # can't fit onto the line currently being constructed.
1304 rest_value = LL[string_idx].value
1306 def more_splits_should_be_made() -> bool:
1309 True iff `rest_value` (the remaining string value from the last
1310 split), should be split again.
1312 if use_custom_breakpoints:
1313 return len(custom_splits) > 1
1315 return len(rest_value) > max_last_string()
1317 string_line_results: List[Ok[Line]] = []
1318 while more_splits_should_be_made():
1319 if use_custom_breakpoints:
1320 # Custom User Split (manual)
1321 csplit = custom_splits.pop(0)
1322 break_idx = csplit.break_idx
1324 # Algorithmic Split (automatic)
1325 max_bidx = max_break_idx - string_op_leaves_length
1326 maybe_break_idx = self._get_break_idx(rest_value, max_bidx)
1327 if maybe_break_idx is None:
1328 # If we are unable to algorithmically determine a good split
1329 # and this string has custom splits registered to it, we
1330 # fall back to using them--which means we have to start
1331 # over from the beginning.
1333 rest_value = LL[string_idx].value
1334 string_line_results = []
1335 first_string_line = True
1336 use_custom_breakpoints = True
1339 # Otherwise, we stop splitting here.
1342 break_idx = maybe_break_idx
1344 # --- Construct `next_value`
1345 next_value = rest_value[:break_idx] + QUOTE
1347 # HACK: The following 'if' statement is a hack to fix the custom
1348 # breakpoint index in the case of either: (a) substrings that were
1349 # f-strings but will have the 'f' prefix removed OR (b) substrings
1350 # that were not f-strings but will now become f-strings because of
1351 # redundant use of the 'f' prefix (i.e. none of the substrings
1352 # contain f-expressions but one or more of them had the 'f' prefix
1353 # anyway; in which case, we will prepend 'f' to _all_ substrings).
1355 # There is probably a better way to accomplish what is being done
1358 # If this substring is an f-string, we _could_ remove the 'f'
1359 # prefix, and the current custom split did NOT originally use a
1362 use_custom_breakpoints
1363 and not csplit.has_prefix
1365 # `next_value == prefix + QUOTE` happens when the custom
1366 # split is an empty string.
1367 next_value == prefix + QUOTE
1368 or next_value != self._normalize_f_string(next_value, prefix)
1371 # Then `csplit.break_idx` will be off by one after removing
1374 next_value = rest_value[:break_idx] + QUOTE
1376 if drop_pointless_f_prefix:
1377 next_value = self._normalize_f_string(next_value, prefix)
1379 # --- Construct `next_leaf`
1380 next_leaf = Leaf(token.STRING, next_value)
1381 insert_str_child(next_leaf)
1382 self._maybe_normalize_string_quotes(next_leaf)
1384 # --- Construct `next_line`
1385 next_line = line.clone()
1386 maybe_append_string_operators(next_line)
1387 next_line.append(next_leaf)
1388 string_line_results.append(Ok(next_line))
1390 rest_value = prefix + QUOTE + rest_value[break_idx:]
1391 first_string_line = False
1393 yield from string_line_results
1395 if drop_pointless_f_prefix:
1396 rest_value = self._normalize_f_string(rest_value, prefix)
1398 rest_leaf = Leaf(token.STRING, rest_value)
1399 insert_str_child(rest_leaf)
1401 # NOTE: I could not find a test case that verifies that the following
1402 # line is actually necessary, but it seems to be. Otherwise we risk
1403 # not normalizing the last substring, right?
1404 self._maybe_normalize_string_quotes(rest_leaf)
1406 last_line = line.clone()
1407 maybe_append_string_operators(last_line)
1409 # If there are any leaves to the right of the target string...
1410 if is_valid_index(string_idx + 1):
1411 # We use `temp_value` here to determine how long the last line
1412 # would be if we were to append all the leaves to the right of the
1413 # target string to the last string line.
1414 temp_value = rest_value
1415 for leaf in LL[string_idx + 1 :]:
1416 temp_value += str(leaf)
1417 if leaf.type == token.LPAR:
1420 # Try to fit them all on the same line with the last substring...
1422 len(temp_value) <= max_last_string()
1423 or LL[string_idx + 1].type == token.COMMA
1425 last_line.append(rest_leaf)
1426 append_leaves(last_line, line, LL[string_idx + 1 :])
1428 # Otherwise, place the last substring on one line and everything
1429 # else on a line below that...
1431 last_line.append(rest_leaf)
1434 non_string_line = line.clone()
1435 append_leaves(non_string_line, line, LL[string_idx + 1 :])
1436 yield Ok(non_string_line)
1437 # Else the target string was the last leaf...
1439 last_line.append(rest_leaf)
1440 last_line.comments = line.comments.copy()
1443 def _iter_nameescape_slices(self, string: str) -> Iterator[Tuple[Index, Index]]:
1446 All ranges of @string which, if @string were to be split there,
1447 would result in the splitting of an \\N{...} expression (which is NOT
1450 # True - the previous backslash was unescaped
1451 # False - the previous backslash was escaped *or* there was no backslash
1452 previous_was_unescaped_backslash = False
1453 it = iter(enumerate(string))
1456 previous_was_unescaped_backslash = not previous_was_unescaped_backslash
1458 if not previous_was_unescaped_backslash or c != "N":
1459 previous_was_unescaped_backslash = False
1461 previous_was_unescaped_backslash = False
1463 begin = idx - 1 # the position of backslash before \N{...}
1469 # malformed nameescape expression?
1470 # should have been detected by AST parsing earlier...
1471 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
1474 def _iter_fexpr_slices(self, string: str) -> Iterator[Tuple[Index, Index]]:
1477 All ranges of @string which, if @string were to be split there,
1478 would result in the splitting of an f-expression (which is NOT
1481 if "f" not in get_string_prefix(string).lower():
1483 yield from iter_fexpr_spans(string)
1485 def _get_illegal_split_indices(self, string: str) -> Set[Index]:
1486 illegal_indices: Set[Index] = set()
1488 self._iter_fexpr_slices(string),
1489 self._iter_nameescape_slices(string),
1491 for it in iterators:
1492 for begin, end in it:
1493 illegal_indices.update(range(begin, end + 1))
1494 return illegal_indices
1496 def _get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]:
1498 This method contains the algorithm that StringSplitter uses to
1499 determine which character to split each string at.
1502 @string: The substring that we are attempting to split.
1503 @max_break_idx: The ideal break index. We will return this value if it
1504 meets all the necessary conditions. In the likely event that it
1505 doesn't we will try to find the closest index BELOW @max_break_idx
1506 that does. If that fails, we will expand our search by also
1507 considering all valid indices ABOVE @max_break_idx.
1510 * assert_is_leaf_string(@string)
1511 * 0 <= @max_break_idx < len(@string)
1514 break_idx, if an index is able to be found that meets all of the
1515 conditions listed in the 'Transformations' section of this classes'
1520 is_valid_index = is_valid_index_factory(string)
1522 assert is_valid_index(max_break_idx)
1523 assert_is_leaf_string(string)
1525 _illegal_split_indices = self._get_illegal_split_indices(string)
1527 def breaks_unsplittable_expression(i: Index) -> bool:
1530 True iff returning @i would result in the splitting of an
1531 unsplittable expression (which is NOT allowed).
1533 return i in _illegal_split_indices
1535 def passes_all_checks(i: Index) -> bool:
1538 True iff ALL of the conditions listed in the 'Transformations'
1539 section of this classes' docstring would be be met by returning @i.
1541 is_space = string[i] == " "
1543 is_not_escaped = True
1545 while is_valid_index(j) and string[j] == "\\":
1546 is_not_escaped = not is_not_escaped
1550 len(string[i:]) >= self.MIN_SUBSTR_SIZE
1551 and len(string[:i]) >= self.MIN_SUBSTR_SIZE
1557 and not breaks_unsplittable_expression(i)
1560 # First, we check all indices BELOW @max_break_idx.
1561 break_idx = max_break_idx
1562 while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx):
1565 if not passes_all_checks(break_idx):
1566 # If that fails, we check all indices ABOVE @max_break_idx.
1568 # If we are able to find a valid index here, the next line is going
1569 # to be longer than the specified line length, but it's probably
1570 # better than doing nothing at all.
1571 break_idx = max_break_idx + 1
1572 while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx):
1575 if not is_valid_index(break_idx) or not passes_all_checks(break_idx):
1580 def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None:
1581 if self.normalize_strings:
1582 leaf.value = normalize_string_quotes(leaf.value)
1584 def _normalize_f_string(self, string: str, prefix: str) -> str:
1587 * assert_is_leaf_string(@string)
1590 * If @string is an f-string that contains no f-expressions, we
1591 return a string identical to @string except that the 'f' prefix
1592 has been stripped and all double braces (i.e. '{{' or '}}') have
1593 been normalized (i.e. turned into '{' or '}').
1595 * Otherwise, we return @string.
1597 assert_is_leaf_string(string)
1599 if "f" in prefix and not fstring_contains_expr(string):
1600 new_prefix = prefix.replace("f", "")
1602 temp = string[len(prefix) :]
1603 temp = re.sub(r"\{\{", "{", temp)
1604 temp = re.sub(r"\}\}", "}", temp)
1607 return f"{new_prefix}{new_string}"
1611 def _get_string_operator_leaves(self, leaves: Iterable[Leaf]) -> List[Leaf]:
1614 string_op_leaves = []
1616 while LL[i].type in self.STRING_OPERATORS + [token.NAME]:
1617 prefix_leaf = Leaf(LL[i].type, str(LL[i]).strip())
1618 string_op_leaves.append(prefix_leaf)
1620 return string_op_leaves
1623 class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin):
1625 StringTransformer that wraps strings in parens and then splits at the LPAR.
1628 All of the requirements listed in BaseStringSplitter's docstring in
1629 addition to the requirements listed below:
1631 * The line is a return/yield statement, which returns/yields a string.
1633 * The line is part of a ternary expression (e.g. `x = y if cond else
1634 z`) such that the line starts with `else <string>`, where <string> is
1637 * The line is an assert statement, which ends with a string.
1639 * The line is an assignment statement (e.g. `x = <string>` or `x +=
1640 <string>`) such that the variable is being assigned the value of some
1643 * The line is a dictionary key assignment where some valid key is being
1644 assigned the value of some string.
1646 * The line is an lambda expression and the value is a string.
1648 * The line starts with an "atom" string that prefers to be wrapped in
1649 parens. It's preferred to be wrapped when the string is surrounded by
1650 commas (or is the first/last child).
1653 The chosen string is wrapped in parentheses and then split at the LPAR.
1655 We then have one line which ends with an LPAR and another line that
1656 starts with the chosen string. The latter line is then split again at
1657 the RPAR. This results in the RPAR (and possibly a trailing comma)
1658 being placed on its own line.
1660 NOTE: If any leaves exist to the right of the chosen string (except
1661 for a trailing comma, which would be placed after the RPAR), those
1662 leaves are placed inside the parentheses. In effect, the chosen
1663 string is not necessarily being "wrapped" by parentheses. We can,
1664 however, count on the LPAR being placed directly before the chosen
1667 In other words, StringParenWrapper creates "atom" strings. These
1668 can then be split again by StringSplitter, if necessary.
1671 In the event that a string line split by StringParenWrapper is
1672 changed such that it no longer needs to be given its own line,
1673 StringParenWrapper relies on StringParenStripper to clean up the
1674 parentheses it created.
1676 For "atom" strings that prefers to be wrapped in parens, it requires
1677 StringSplitter to hold the split until the string is wrapped in parens.
1680 def do_splitter_match(self, line: Line) -> TMatchResult:
1683 if line.leaves[-1].type in OPENING_BRACKETS:
1685 "Cannot wrap parens around a line that ends in an opening bracket."
1689 self._return_match(LL)
1690 or self._else_match(LL)
1691 or self._assert_match(LL)
1692 or self._assign_match(LL)
1693 or self._dict_or_lambda_match(LL)
1694 or self._prefer_paren_wrap_match(LL)
1697 if string_idx is not None:
1698 string_value = line.leaves[string_idx].value
1699 # If the string has no spaces...
1700 if " " not in string_value:
1701 # And will still violate the line length limit when split...
1702 max_string_length = self.line_length - ((line.depth + 1) * 4)
1703 if len(string_value) > max_string_length:
1704 # And has no associated custom splits...
1705 if not self.has_custom_splits(string_value):
1706 # Then we should NOT put this string on its own line.
1708 "We do not wrap long strings in parentheses when the"
1709 " resultant line would still be over the specified line"
1710 " length and can't be split further by StringSplitter."
1712 return Ok(string_idx)
1714 return TErr("This line does not contain any non-atomic strings.")
1717 def _return_match(LL: List[Leaf]) -> Optional[int]:
1720 string_idx such that @LL[string_idx] is equal to our target (i.e.
1721 matched) string, if this line matches the return/yield statement
1722 requirements listed in the 'Requirements' section of this classes'
1727 # If this line is apart of a return/yield statement and the first leaf
1728 # contains either the "return" or "yield" keywords...
1729 if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[
1731 ].value in ["return", "yield"]:
1732 is_valid_index = is_valid_index_factory(LL)
1734 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
1735 # The next visible leaf MUST contain a string...
1736 if is_valid_index(idx) and LL[idx].type == token.STRING:
1742 def _else_match(LL: List[Leaf]) -> Optional[int]:
1745 string_idx such that @LL[string_idx] is equal to our target (i.e.
1746 matched) string, if this line matches the ternary expression
1747 requirements listed in the 'Requirements' section of this classes'
1752 # If this line is apart of a ternary expression and the first leaf
1753 # contains the "else" keyword...
1755 parent_type(LL[0]) == syms.test
1756 and LL[0].type == token.NAME
1757 and LL[0].value == "else"
1759 is_valid_index = is_valid_index_factory(LL)
1761 idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
1762 # The next visible leaf MUST contain a string...
1763 if is_valid_index(idx) and LL[idx].type == token.STRING:
1769 def _assert_match(LL: List[Leaf]) -> Optional[int]:
1772 string_idx such that @LL[string_idx] is equal to our target (i.e.
1773 matched) string, if this line matches the assert statement
1774 requirements listed in the 'Requirements' section of this classes'
1779 # If this line is apart of an assert statement and the first leaf
1780 # contains the "assert" keyword...
1781 if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert":
1782 is_valid_index = is_valid_index_factory(LL)
1784 for i, leaf in enumerate(LL):
1785 # We MUST find a comma...
1786 if leaf.type == token.COMMA:
1787 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
1789 # That comma MUST be followed by a string...
1790 if is_valid_index(idx) and LL[idx].type == token.STRING:
1793 # Skip the string trailer, if one exists.
1794 string_parser = StringParser()
1795 idx = string_parser.parse(LL, string_idx)
1797 # But no more leaves are allowed...
1798 if not is_valid_index(idx):
1804 def _assign_match(LL: List[Leaf]) -> Optional[int]:
1807 string_idx such that @LL[string_idx] is equal to our target (i.e.
1808 matched) string, if this line matches the assignment statement
1809 requirements listed in the 'Requirements' section of this classes'
1814 # If this line is apart of an expression statement or is a function
1815 # argument AND the first leaf contains a variable name...
1817 parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power]
1818 and LL[0].type == token.NAME
1820 is_valid_index = is_valid_index_factory(LL)
1822 for i, leaf in enumerate(LL):
1823 # We MUST find either an '=' or '+=' symbol...
1824 if leaf.type in [token.EQUAL, token.PLUSEQUAL]:
1825 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
1827 # That symbol MUST be followed by a string...
1828 if is_valid_index(idx) and LL[idx].type == token.STRING:
1831 # Skip the string trailer, if one exists.
1832 string_parser = StringParser()
1833 idx = string_parser.parse(LL, string_idx)
1835 # The next leaf MAY be a comma iff this line is apart
1836 # of a function argument...
1838 parent_type(LL[0]) == syms.argument
1839 and is_valid_index(idx)
1840 and LL[idx].type == token.COMMA
1844 # But no more leaves are allowed...
1845 if not is_valid_index(idx):
1851 def _dict_or_lambda_match(LL: List[Leaf]) -> Optional[int]:
1854 string_idx such that @LL[string_idx] is equal to our target (i.e.
1855 matched) string, if this line matches the dictionary key assignment
1856 statement or lambda expression requirements listed in the
1857 'Requirements' section of this classes' docstring.
1861 # If this line is a part of a dictionary key assignment or lambda expression...
1862 parent_types = [parent_type(LL[0]), parent_type(LL[0].parent)]
1863 if syms.dictsetmaker in parent_types or syms.lambdef in parent_types:
1864 is_valid_index = is_valid_index_factory(LL)
1866 for i, leaf in enumerate(LL):
1867 # We MUST find a colon, it can either be dict's or lambda's colon...
1868 if leaf.type == token.COLON:
1869 idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
1871 # That colon MUST be followed by a string...
1872 if is_valid_index(idx) and LL[idx].type == token.STRING:
1875 # Skip the string trailer, if one exists.
1876 string_parser = StringParser()
1877 idx = string_parser.parse(LL, string_idx)
1879 # That string MAY be followed by a comma...
1880 if is_valid_index(idx) and LL[idx].type == token.COMMA:
1883 # But no more leaves are allowed...
1884 if not is_valid_index(idx):
1889 def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
1892 is_valid_index = is_valid_index_factory(LL)
1893 insert_str_child = insert_str_child_factory(LL[string_idx])
1896 ends_with_comma = False
1897 if LL[comma_idx].type == token.COMMA:
1898 ends_with_comma = True
1900 leaves_to_steal_comments_from = [LL[string_idx]]
1902 leaves_to_steal_comments_from.append(LL[comma_idx])
1905 first_line = line.clone()
1906 left_leaves = LL[:string_idx]
1908 # We have to remember to account for (possibly invisible) LPAR and RPAR
1909 # leaves that already wrapped the target string. If these leaves do
1910 # exist, we will replace them with our own LPAR and RPAR leaves.
1911 old_parens_exist = False
1912 if left_leaves and left_leaves[-1].type == token.LPAR:
1913 old_parens_exist = True
1914 leaves_to_steal_comments_from.append(left_leaves[-1])
1917 append_leaves(first_line, line, left_leaves)
1919 lpar_leaf = Leaf(token.LPAR, "(")
1920 if old_parens_exist:
1921 replace_child(LL[string_idx - 1], lpar_leaf)
1923 insert_str_child(lpar_leaf)
1924 first_line.append(lpar_leaf)
1926 # We throw inline comments that were originally to the right of the
1927 # target string to the top line. They will now be shown to the right of
1929 for leaf in leaves_to_steal_comments_from:
1930 for comment_leaf in line.comments_after(leaf):
1931 first_line.append(comment_leaf, preformatted=True)
1933 yield Ok(first_line)
1935 # --- Middle (String) Line
1936 # We only need to yield one (possibly too long) string line, since the
1937 # `StringSplitter` will break it down further if necessary.
1938 string_value = LL[string_idx].value
1941 depth=line.depth + 1,
1942 inside_brackets=True,
1943 should_split_rhs=line.should_split_rhs,
1944 magic_trailing_comma=line.magic_trailing_comma,
1946 string_leaf = Leaf(token.STRING, string_value)
1947 insert_str_child(string_leaf)
1948 string_line.append(string_leaf)
1950 old_rpar_leaf = None
1951 if is_valid_index(string_idx + 1):
1952 right_leaves = LL[string_idx + 1 :]
1956 if old_parens_exist:
1957 assert right_leaves and right_leaves[-1].type == token.RPAR, (
1958 "Apparently, old parentheses do NOT exist?!"
1959 f" (left_leaves={left_leaves}, right_leaves={right_leaves})"
1961 old_rpar_leaf = right_leaves.pop()
1962 elif right_leaves and right_leaves[-1].type == token.RPAR:
1963 # Special case for lambda expressions as dict's value, e.g.:
1965 # "key": lambda x: f"formatted: {x},
1967 # After wrapping the dict's value with parentheses, the string is
1968 # followed by a RPAR but its opening bracket is lambda's, not
1970 # "key": (lambda x: f"formatted: {x}),
1971 opening_bracket = right_leaves[-1].opening_bracket
1972 if opening_bracket is not None and opening_bracket in left_leaves:
1973 index = left_leaves.index(opening_bracket)
1976 and index < len(left_leaves) - 1
1977 and left_leaves[index - 1].type == token.COLON
1978 and left_leaves[index + 1].value == "lambda"
1982 append_leaves(string_line, line, right_leaves)
1984 yield Ok(string_line)
1987 last_line = line.clone()
1988 last_line.bracket_tracker = first_line.bracket_tracker
1990 new_rpar_leaf = Leaf(token.RPAR, ")")
1991 if old_rpar_leaf is not None:
1992 replace_child(old_rpar_leaf, new_rpar_leaf)
1994 insert_str_child(new_rpar_leaf)
1995 last_line.append(new_rpar_leaf)
1997 # If the target string ended with a comma, we place this comma to the
1998 # right of the RPAR on the last line.
2000 comma_leaf = Leaf(token.COMMA, ",")
2001 replace_child(LL[comma_idx], comma_leaf)
2002 last_line.append(comma_leaf)
2009 A state machine that aids in parsing a string's "trailer", which can be
2010 either non-existent, an old-style formatting sequence (e.g. `% varX` or `%
2011 (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX,
2014 NOTE: A new StringParser object MUST be instantiated for each string
2015 trailer we need to parse.
2018 We shall assume that `line` equals the `Line` object that corresponds
2019 to the following line of python code:
2021 x = "Some {}.".format("String") + some_other_string
2024 Furthermore, we will assume that `string_idx` is some index such that:
2026 assert line.leaves[string_idx].value == "Some {}."
2029 The following code snippet then holds:
2031 string_parser = StringParser()
2032 idx = string_parser.parse(line.leaves, string_idx)
2033 assert line.leaves[idx].type == token.PLUS
2037 DEFAULT_TOKEN: Final = 20210605
2039 # String Parser States
2044 SINGLE_FMT_ARG: Final = 5
2049 # Lookup Table for Next State
2050 _goto: Final[Dict[Tuple[ParserState, NodeType], ParserState]] = {
2051 # A string trailer may start with '.' OR '%'.
2052 (START, token.DOT): DOT,
2053 (START, token.PERCENT): PERCENT,
2054 (START, DEFAULT_TOKEN): DONE,
2055 # A '.' MUST be followed by an attribute or method name.
2056 (DOT, token.NAME): NAME,
2057 # A method name MUST be followed by an '(', whereas an attribute name
2058 # is the last symbol in the string trailer.
2059 (NAME, token.LPAR): LPAR,
2060 (NAME, DEFAULT_TOKEN): DONE,
2061 # A '%' symbol can be followed by an '(' or a single argument (e.g. a
2062 # string or variable name).
2063 (PERCENT, token.LPAR): LPAR,
2064 (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG,
2065 # If a '%' symbol is followed by a single argument, that argument is
2066 # the last leaf in the string trailer.
2067 (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE,
2068 # If present, a ')' symbol is the last symbol in a string trailer.
2069 # (NOTE: LPARS and nested RPARS are not included in this lookup table,
2070 # since they are treated as a special case by the parsing logic in this
2071 # classes' implementation.)
2072 (RPAR, DEFAULT_TOKEN): DONE,
2075 def __init__(self) -> None:
2076 self._state = self.START
2077 self._unmatched_lpars = 0
2079 def parse(self, leaves: List[Leaf], string_idx: int) -> int:
2082 * @leaves[@string_idx].type == token.STRING
2085 The index directly after the last leaf which is apart of the string
2086 trailer, if a "trailer" exists.
2088 @string_idx + 1, if no string "trailer" exists.
2090 assert leaves[string_idx].type == token.STRING
2092 idx = string_idx + 1
2093 while idx < len(leaves) and self._next_state(leaves[idx]):
2097 def _next_state(self, leaf: Leaf) -> bool:
2100 * On the first call to this function, @leaf MUST be the leaf that
2101 was directly after the string leaf in question (e.g. if our target
2102 string is `line.leaves[i]` then the first call to this method must
2103 be `line.leaves[i + 1]`).
2104 * On the next call to this function, the leaf parameter passed in
2105 MUST be the leaf directly following @leaf.
2108 True iff @leaf is apart of the string's trailer.
2110 # We ignore empty LPAR or RPAR leaves.
2111 if is_empty_par(leaf):
2114 next_token = leaf.type
2115 if next_token == token.LPAR:
2116 self._unmatched_lpars += 1
2118 current_state = self._state
2120 # The LPAR parser state is a special case. We will return True until we
2121 # find the matching RPAR token.
2122 if current_state == self.LPAR:
2123 if next_token == token.RPAR:
2124 self._unmatched_lpars -= 1
2125 if self._unmatched_lpars == 0:
2126 self._state = self.RPAR
2127 # Otherwise, we use a lookup table to determine the next state.
2129 # If the lookup table matches the current state to the next
2130 # token, we use the lookup table.
2131 if (current_state, next_token) in self._goto:
2132 self._state = self._goto[current_state, next_token]
2134 # Otherwise, we check if a the current state was assigned a
2136 if (current_state, self.DEFAULT_TOKEN) in self._goto:
2137 self._state = self._goto[current_state, self.DEFAULT_TOKEN]
2138 # If no default has been assigned, then this parser has a logic
2141 raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
2143 if self._state == self.DONE:
2149 def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]:
2151 Factory for a convenience function that is used to orphan @string_leaf
2152 and then insert multiple new leaves into the same part of the node
2153 structure that @string_leaf had originally occupied.
2156 Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N =
2157 string_leaf.parent`. Assume the node `N` has the following
2164 Leaf(STRING, '"foo"'),
2168 We then run the code snippet shown below.
2170 insert_str_child = insert_str_child_factory(string_leaf)
2172 lpar = Leaf(token.LPAR, '(')
2173 insert_str_child(lpar)
2175 bar = Leaf(token.STRING, '"bar"')
2176 insert_str_child(bar)
2178 rpar = Leaf(token.RPAR, ')')
2179 insert_str_child(rpar)
2182 After which point, it follows that `string_leaf.parent is None` and
2183 the node `N` now has the following structure:
2190 Leaf(STRING, '"bar"'),
2195 string_parent = string_leaf.parent
2196 string_child_idx = string_leaf.remove()
2198 def insert_str_child(child: LN) -> None:
2199 nonlocal string_child_idx
2201 assert string_parent is not None
2202 assert string_child_idx is not None
2204 string_parent.insert_child(string_child_idx, child)
2205 string_child_idx += 1
2207 return insert_str_child
2210 def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]:
2216 is_valid_index = is_valid_index_factory(my_list)
2218 assert is_valid_index(0)
2219 assert is_valid_index(2)
2221 assert not is_valid_index(3)
2222 assert not is_valid_index(-1)
2226 def is_valid_index(idx: int) -> bool:
2229 True iff @idx is positive AND seq[@idx] does NOT raise an
2232 return 0 <= idx < len(seq)
2234 return is_valid_index