+ else:
+ yield line
+
+
+@dataclass # type: ignore
+class StringTransformer(ABC):
+ """
+ An implementation of the Transformer protocol that relies on its
+ subclasses overriding the template methods `do_match(...)` and
+ `do_transform(...)`.
+
+ This Transformer works exclusively on strings (for example, by merging
+ or splitting them).
+
+ The following sections can be found among the docstrings of each concrete
+ StringTransformer subclass.
+
+ Requirements:
+ Which requirements must be met of the given Line for this
+ StringTransformer to be applied?
+
+ Transformations:
+ If the given Line meets all of the above requirments, which string
+ transformations can you expect to be applied to it by this
+ StringTransformer?
+
+ Collaborations:
+ What contractual agreements does this StringTransformer have with other
+ StringTransfomers? Such collaborations should be eliminated/minimized
+ as much as possible.
+ """
+
+ line_length: int
+ normalize_strings: bool
+
+ @abstractmethod
+ def do_match(self, line: Line) -> TMatchResult:
+ """
+ Returns:
+ * Ok(string_idx) such that `line.leaves[string_idx]` is our target
+ string, if a match was able to be made.
+ OR
+ * Err(CannotTransform), if a match was not able to be made.
+ """
+
+ @abstractmethod
+ def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
+ """
+ Yields:
+ * Ok(new_line) where new_line is the new transformed line.
+ OR
+ * Err(CannotTransform) if the transformation failed for some reason. The
+ `do_match(...)` template method should usually be used to reject
+ the form of the given Line, but in some cases it is difficult to
+ know whether or not a Line meets the StringTransformer's
+ requirements until the transformation is already midway.
+
+ Side Effects:
+ This method should NOT mutate @line directly, but it MAY mutate the
+ Line's underlying Node structure. (WARNING: If the underlying Node
+ structure IS altered, then this method should NOT be allowed to
+ yield an CannotTransform after that point.)
+ """
+
+ def __call__(self, line: Line, _features: Collection[Feature]) -> Iterator[Line]:
+ """
+ StringTransformer instances have a call signature that mirrors that of
+ the Transformer type.
+
+ Raises:
+ CannotTransform(...) if the concrete StringTransformer class is unable
+ to transform @line.
+ """
+ # Optimization to avoid calling `self.do_match(...)` when the line does
+ # not contain any string.
+ if not any(leaf.type == token.STRING for leaf in line.leaves):
+ raise CannotTransform("There are no strings in this line.")
+
+ match_result = self.do_match(line)
+
+ if isinstance(match_result, Err):
+ cant_transform = match_result.err()
+ raise CannotTransform(
+ f"The string transformer {self.__class__.__name__} does not recognize"
+ " this line as one that it can transform."
+ ) from cant_transform
+
+ string_idx = match_result.ok()
+
+ for line_result in self.do_transform(line, string_idx):
+ if isinstance(line_result, Err):
+ cant_transform = line_result.err()
+ raise CannotTransform(
+ "StringTransformer failed while attempting to transform string."
+ ) from cant_transform
+ line = line_result.ok()
+ yield line
+
+
+@dataclass
+class CustomSplit:
+ """A custom (i.e. manual) string split.
+
+ A single CustomSplit instance represents a single substring.
+
+ Examples:
+ Consider the following string:
+ ```
+ "Hi there friend."
+ " This is a custom"
+ f" string {split}."
+ ```
+
+ This string will correspond to the following three CustomSplit instances:
+ ```
+ CustomSplit(False, 16)
+ CustomSplit(False, 17)
+ CustomSplit(True, 16)
+ ```
+ """
+
+ has_prefix: bool
+ break_idx: int
+
+
+class CustomSplitMapMixin:
+ """
+ This mixin class is used to map merged strings to a sequence of
+ CustomSplits, which will then be used to re-split the strings iff none of
+ the resultant substrings go over the configured max line length.
+ """
+
+ _Key = Tuple[StringID, str]
+ _CUSTOM_SPLIT_MAP: Dict[_Key, Tuple[CustomSplit, ...]] = defaultdict(tuple)
+
+ @staticmethod
+ def _get_key(string: str) -> "CustomSplitMapMixin._Key":
+ """
+ Returns:
+ A unique identifier that is used internally to map @string to a
+ group of custom splits.
+ """
+ return (id(string), string)
+
+ def add_custom_splits(
+ self, string: str, custom_splits: Iterable[CustomSplit]
+ ) -> None:
+ """Custom Split Map Setter Method
+
+ Side Effects:
+ Adds a mapping from @string to the custom splits @custom_splits.
+ """
+ key = self._get_key(string)
+ self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)
+
+ def pop_custom_splits(self, string: str) -> List[CustomSplit]:
+ """Custom Split Map Getter Method
+
+ Returns:
+ * A list of the custom splits that are mapped to @string, if any
+ exist.
+ OR
+ * [], otherwise.
+
+ Side Effects:
+ Deletes the mapping between @string and its associated custom
+ splits (which are returned to the caller).
+ """
+ key = self._get_key(string)
+
+ custom_splits = self._CUSTOM_SPLIT_MAP[key]
+ del self._CUSTOM_SPLIT_MAP[key]
+
+ return list(custom_splits)
+
+ def has_custom_splits(self, string: str) -> bool:
+ """
+ Returns:
+ True iff @string is associated with a set of custom splits.
+ """
+ key = self._get_key(string)
+ return key in self._CUSTOM_SPLIT_MAP
+
+
+class StringMerger(CustomSplitMapMixin, StringTransformer):
+ """StringTransformer that merges strings together.
+
+ Requirements:
+ (A) The line contains adjacent strings such that at most one substring
+ has inline comments AND none of those inline comments are pragmas AND
+ the set of all substring prefixes is either of length 1 or equal to
+ {"", "f"} AND none of the substrings are raw strings (i.e. are prefixed
+ with 'r').
+ OR
+ (B) The line contains a string which uses line continuation backslashes.
+
+ Transformations:
+ Depending on which of the two requirements above where met, either:
+
+ (A) The string group associated with the target string is merged.
+ OR
+ (B) All line-continuation backslashes are removed from the target string.
+
+ Collaborations:
+ StringMerger provides custom split information to StringSplitter.
+ """
+
+ def do_match(self, line: Line) -> TMatchResult:
+ LL = line.leaves
+
+ is_valid_index = is_valid_index_factory(LL)
+
+ for (i, leaf) in enumerate(LL):
+ if (
+ leaf.type == token.STRING
+ and is_valid_index(i + 1)
+ and LL[i + 1].type == token.STRING
+ ):
+ return Ok(i)
+
+ if leaf.type == token.STRING and "\\\n" in leaf.value:
+ return Ok(i)
+
+ return TErr("This line has no strings that need merging.")
+
+ def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
+ new_line = line
+ rblc_result = self.__remove_backslash_line_continuation_chars(
+ new_line, string_idx
+ )
+ if isinstance(rblc_result, Ok):
+ new_line = rblc_result.ok()
+
+ msg_result = self.__merge_string_group(new_line, string_idx)
+ if isinstance(msg_result, Ok):
+ new_line = msg_result.ok()
+
+ if isinstance(rblc_result, Err) and isinstance(msg_result, Err):
+ msg_cant_transform = msg_result.err()
+ rblc_cant_transform = rblc_result.err()
+ cant_transform = CannotTransform(
+ "StringMerger failed to merge any strings in this line."
+ )
+
+ # Chain the errors together using `__cause__`.
+ msg_cant_transform.__cause__ = rblc_cant_transform
+ cant_transform.__cause__ = msg_cant_transform
+
+ yield Err(cant_transform)
+ else:
+ yield Ok(new_line)
+
+ @staticmethod
+ def __remove_backslash_line_continuation_chars(
+ line: Line, string_idx: int
+ ) -> TResult[Line]:
+ """
+ Merge strings that were split across multiple lines using
+ line-continuation backslashes.
+
+ Returns:
+ Ok(new_line), if @line contains backslash line-continuation
+ characters.
+ OR
+ Err(CannotTransform), otherwise.
+ """
+ LL = line.leaves
+
+ string_leaf = LL[string_idx]
+ if not (
+ string_leaf.type == token.STRING
+ and "\\\n" in string_leaf.value
+ and not has_triple_quotes(string_leaf.value)
+ ):
+ return TErr(
+ f"String leaf {string_leaf} does not contain any backslash line"
+ " continuation characters."
+ )
+
+ new_line = line.clone()
+ new_line.comments = line.comments
+ append_leaves(new_line, line, LL)
+
+ new_string_leaf = new_line.leaves[string_idx]
+ new_string_leaf.value = new_string_leaf.value.replace("\\\n", "")
+
+ return Ok(new_line)
+
+ def __merge_string_group(self, line: Line, string_idx: int) -> TResult[Line]:
+ """
+ Merges string group (i.e. set of adjacent strings) where the first
+ string in the group is `line.leaves[string_idx]`.
+
+ Returns:
+ Ok(new_line), if ALL of the validation checks found in
+ __validate_msg(...) pass.
+ OR
+ Err(CannotTransform), otherwise.
+ """
+ LL = line.leaves
+
+ is_valid_index = is_valid_index_factory(LL)
+
+ vresult = self.__validate_msg(line, string_idx)
+ if isinstance(vresult, Err):
+ return vresult
+
+ # If the string group is wrapped inside an Atom node, we must make sure
+ # to later replace that Atom with our new (merged) string leaf.
+ atom_node = LL[string_idx].parent
+
+ # We will place BREAK_MARK in between every two substrings that we
+ # merge. We will then later go through our final result and use the
+ # various instances of BREAK_MARK we find to add the right values to
+ # the custom split map.
+ BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@"
+
+ QUOTE = LL[string_idx].value[-1]
+
+ def make_naked(string: str, string_prefix: str) -> str:
+ """Strip @string (i.e. make it a "naked" string)
+
+ Pre-conditions:
+ * assert_is_leaf_string(@string)
+
+ Returns:
+ A string that is identical to @string except that
+ @string_prefix has been stripped, the surrounding QUOTE
+ characters have been removed, and any remaining QUOTE
+ characters have been escaped.
+ """
+ assert_is_leaf_string(string)
+
+ RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)"
+ naked_string = string[len(string_prefix) + 1 : -1]
+ naked_string = re.sub(
+ "(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string
+ )
+ return naked_string
+
+ # Holds the CustomSplit objects that will later be added to the custom
+ # split map.
+ custom_splits = []
+
+ # Temporary storage for the 'has_prefix' part of the CustomSplit objects.
+ prefix_tracker = []
+
+ # Sets the 'prefix' variable. This is the prefix that the final merged
+ # string will have.
+ next_str_idx = string_idx
+ prefix = ""
+ while (
+ not prefix
+ and is_valid_index(next_str_idx)
+ and LL[next_str_idx].type == token.STRING
+ ):
+ prefix = get_string_prefix(LL[next_str_idx].value)
+ next_str_idx += 1
+
+ # The next loop merges the string group. The final string will be
+ # contained in 'S'.
+ #
+ # The following convenience variables are used:
+ #
+ # S: string
+ # NS: naked string
+ # SS: next string
+ # NSS: naked next string
+ S = ""
+ NS = ""
+ num_of_strings = 0
+ next_str_idx = string_idx
+ while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:
+ num_of_strings += 1
+
+ SS = LL[next_str_idx].value
+ next_prefix = get_string_prefix(SS)
+
+ # If this is an f-string group but this substring is not prefixed
+ # with 'f'...
+ if "f" in prefix and "f" not in next_prefix:
+ # Then we must escape any braces contained in this substring.
+ SS = re.subf(r"(\{|\})", "{1}{1}", SS)
+
+ NSS = make_naked(SS, next_prefix)
+
+ has_prefix = bool(next_prefix)
+ prefix_tracker.append(has_prefix)
+
+ S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE
+ NS = make_naked(S, prefix)
+
+ next_str_idx += 1
+
+ S_leaf = Leaf(token.STRING, S)
+ if self.normalize_strings:
+ normalize_string_quotes(S_leaf)
+
+ # Fill the 'custom_splits' list with the appropriate CustomSplit objects.
+ temp_string = S_leaf.value[len(prefix) + 1 : -1]
+ for has_prefix in prefix_tracker:
+ mark_idx = temp_string.find(BREAK_MARK)
+ assert (
+ mark_idx >= 0
+ ), "Logic error while filling the custom string breakpoint cache."
+
+ temp_string = temp_string[mark_idx + len(BREAK_MARK) :]
+ breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1
+ custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))
+
+ string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
+
+ if atom_node is not None:
+ replace_child(atom_node, string_leaf)
+
+ # Build the final line ('new_line') that this method will later return.
+ new_line = line.clone()
+ for (i, leaf) in enumerate(LL):
+ if i == string_idx:
+ new_line.append(string_leaf)
+
+ if string_idx <= i < string_idx + num_of_strings:
+ for comment_leaf in line.comments_after(LL[i]):
+ new_line.append(comment_leaf, preformatted=True)
+ continue
+
+ append_leaves(new_line, line, [leaf])
+
+ self.add_custom_splits(string_leaf.value, custom_splits)
+ return Ok(new_line)
+
+ @staticmethod
+ def __validate_msg(line: Line, string_idx: int) -> TResult[None]:
+ """Validate (M)erge (S)tring (G)roup
+
+ Transform-time string validation logic for __merge_string_group(...).
+
+ Returns:
+ * Ok(None), if ALL validation checks (listed below) pass.
+ OR
+ * Err(CannotTransform), if any of the following are true:
+ - The target string is not in a string group (i.e. it has no
+ adjacent strings).
+ - The string group has more than one inline comment.
+ - The string group has an inline comment that appears to be a pragma.
+ - The set of all string prefixes in the string group is of
+ length greater than one and is not equal to {"", "f"}.
+ - The string group consists of raw strings.
+ """
+ num_of_inline_string_comments = 0
+ set_of_prefixes = set()
+ num_of_strings = 0
+ for leaf in line.leaves[string_idx:]:
+ if leaf.type != token.STRING:
+ # If the string group is trailed by a comma, we count the
+ # comments trailing the comma to be one of the string group's
+ # comments.
+ if leaf.type == token.COMMA and id(leaf) in line.comments:
+ num_of_inline_string_comments += 1
+ break
+
+ if has_triple_quotes(leaf.value):
+ return TErr("StringMerger does NOT merge multiline strings.")
+
+ num_of_strings += 1
+ prefix = get_string_prefix(leaf.value)
+ if "r" in prefix:
+ return TErr("StringMerger does NOT merge raw strings.")
+
+ set_of_prefixes.add(prefix)
+
+ if id(leaf) in line.comments:
+ num_of_inline_string_comments += 1
+ if contains_pragma_comment(line.comments[id(leaf)]):
+ return TErr("Cannot merge strings which have pragma comments.")
+
+ if num_of_strings < 2:
+ return TErr(
+ f"Not enough strings to merge (num_of_strings={num_of_strings})."
+ )
+
+ if num_of_inline_string_comments > 1:
+ return TErr(
+ f"Too many inline string comments ({num_of_inline_string_comments})."
+ )
+
+ if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}:
+ return TErr(f"Too many different prefixes ({set_of_prefixes}).")
+
+ return Ok(None)
+
+
+class StringParenStripper(StringTransformer):
+ """StringTransformer that strips surrounding parentheses from strings.
+
+ Requirements:
+ The line contains a string which is surrounded by parentheses and:
+ - The target string is NOT the only argument to a function call).
+ - The RPAR is NOT followed by an attribute access (i.e. a dot).
+
+ Transformations:
+ The parentheses mentioned in the 'Requirements' section are stripped.
+
+ Collaborations:
+ StringParenStripper has its own inherent usefulness, but it is also
+ relied on to clean up the parentheses created by StringParenWrapper (in
+ the event that they are no longer needed).
+ """
+
+ def do_match(self, line: Line) -> TMatchResult:
+ LL = line.leaves
+
+ is_valid_index = is_valid_index_factory(LL)
+
+ for (idx, leaf) in enumerate(LL):
+ # Should be a string...
+ if leaf.type != token.STRING:
+ continue
+
+ # Should be preceded by a non-empty LPAR...
+ if (
+ not is_valid_index(idx - 1)
+ or LL[idx - 1].type != token.LPAR
+ or is_empty_lpar(LL[idx - 1])
+ ):
+ continue
+
+ # That LPAR should NOT be preceded by a function name or a closing
+ # bracket (which could be a function which returns a function or a
+ # list/dictionary that contains a function)...
+ if is_valid_index(idx - 2) and (
+ LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS
+ ):
+ continue
+
+ string_idx = idx
+
+ # Skip the string trailer, if one exists.
+ string_parser = StringParser()
+ next_idx = string_parser.parse(LL, string_idx)
+
+ # Should be followed by a non-empty RPAR...
+ if (
+ is_valid_index(next_idx)
+ and LL[next_idx].type == token.RPAR
+ and not is_empty_rpar(LL[next_idx])
+ ):
+ # That RPAR should NOT be followed by a '.' symbol.
+ if is_valid_index(next_idx + 1) and LL[next_idx + 1].type == token.DOT:
+ continue
+
+ return Ok(string_idx)
+
+ return TErr("This line has no strings wrapped in parens.")
+
+ def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
+ LL = line.leaves
+
+ string_parser = StringParser()
+ rpar_idx = string_parser.parse(LL, string_idx)
+
+ for leaf in (LL[string_idx - 1], LL[rpar_idx]):
+ if line.comments_after(leaf):
+ yield TErr(
+ "Will not strip parentheses which have comments attached to them."
+ )
+
+ new_line = line.clone()
+ new_line.comments = line.comments.copy()
+
+ append_leaves(new_line, line, LL[: string_idx - 1])
+
+ string_leaf = Leaf(token.STRING, LL[string_idx].value)
+ LL[string_idx - 1].remove()
+ replace_child(LL[string_idx], string_leaf)
+ new_line.append(string_leaf)
+
+ append_leaves(
+ new_line, line, LL[string_idx + 1 : rpar_idx] + LL[rpar_idx + 1 :],
+ )
+
+ LL[rpar_idx].remove()
+
+ yield Ok(new_line)
+
+
+class BaseStringSplitter(StringTransformer):
+ """
+ Abstract class for StringTransformers which transform a Line's strings by splitting
+ them or placing them on their own lines where necessary to avoid going over
+ the configured line length.
+
+ Requirements:
+ * The target string value is responsible for the line going over the
+ line length limit. It follows that after all of black's other line
+ split methods have been exhausted, this line (or one of the resulting
+ lines after all line splits are performed) would still be over the
+ line_length limit unless we split this string.
+ AND
+ * The target string is NOT a "pointless" string (i.e. a string that has
+ no parent or siblings).
+ AND
+ * The target string is not followed by an inline comment that appears
+ to be a pragma.
+ AND
+ * The target string is not a multiline (i.e. triple-quote) string.
+ """
+
+ @abstractmethod
+ def do_splitter_match(self, line: Line) -> TMatchResult:
+ """
+ BaseStringSplitter asks its clients to override this method instead of
+ `StringTransformer.do_match(...)`.
+
+ Follows the same protocol as `StringTransformer.do_match(...)`.
+
+ Refer to `help(StringTransformer.do_match)` for more information.
+ """
+
+ def do_match(self, line: Line) -> TMatchResult:
+ match_result = self.do_splitter_match(line)
+ if isinstance(match_result, Err):
+ return match_result
+
+ string_idx = match_result.ok()
+ vresult = self.__validate(line, string_idx)
+ if isinstance(vresult, Err):
+ return vresult
+
+ return match_result
+
+ def __validate(self, line: Line, string_idx: int) -> TResult[None]:
+ """
+ Checks that @line meets all of the requirements listed in this classes'
+ docstring. Refer to `help(BaseStringSplitter)` for a detailed
+ description of those requirements.
+
+ Returns:
+ * Ok(None), if ALL of the requirements are met.
+ OR
+ * Err(CannotTransform), if ANY of the requirements are NOT met.
+ """
+ LL = line.leaves
+
+ string_leaf = LL[string_idx]
+
+ max_string_length = self.__get_max_string_length(line, string_idx)
+ if len(string_leaf.value) <= max_string_length:
+ return TErr(
+ "The string itself is not what is causing this line to be too long."
+ )
+
+ if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [
+ token.STRING,
+ token.NEWLINE,
+ ]:
+ return TErr(
+ f"This string ({string_leaf.value}) appears to be pointless (i.e. has"
+ " no parent)."
+ )
+
+ if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(
+ line.comments[id(line.leaves[string_idx])]
+ ):
+ return TErr(
+ "Line appears to end with an inline pragma comment. Splitting the line"
+ " could modify the pragma's behavior."
+ )
+
+ if has_triple_quotes(string_leaf.value):
+ return TErr("We cannot split multiline strings.")
+
+ return Ok(None)
+
+ def __get_max_string_length(self, line: Line, string_idx: int) -> int:
+ """
+ Calculates the max string length used when attempting to determine
+ whether or not the target string is responsible for causing the line to
+ go over the line length limit.
+
+ WARNING: This method is tightly coupled to both StringSplitter and
+ (especially) StringParenWrapper. There is probably a better way to
+ accomplish what is being done here.
+
+ Returns:
+ max_string_length: such that `line.leaves[string_idx].value >
+ max_string_length` implies that the target string IS responsible
+ for causing this line to exceed the line length limit.
+ """
+ LL = line.leaves
+
+ is_valid_index = is_valid_index_factory(LL)
+
+ # We use the shorthand "WMA4" in comments to abbreviate "We must
+ # account for". When giving examples, we use STRING to mean some/any
+ # valid string.
+ #
+ # Finally, we use the following convenience variables:
+ #
+ # P: The leaf that is before the target string leaf.
+ # N: The leaf that is after the target string leaf.
+ # NN: The leaf that is after N.
+
+ # WMA4 the whitespace at the beginning of the line.
+ offset = line.depth * 4
+
+ if is_valid_index(string_idx - 1):
+ p_idx = string_idx - 1
+ if (
+ LL[string_idx - 1].type == token.LPAR
+ and LL[string_idx - 1].value == ""
+ and string_idx >= 2
+ ):
+ # If the previous leaf is an empty LPAR placeholder, we should skip it.
+ p_idx -= 1
+
+ P = LL[p_idx]
+ if P.type == token.PLUS:
+ # WMA4 a space and a '+' character (e.g. `+ STRING`).
+ offset += 2
+
+ if P.type == token.COMMA:
+ # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].
+ offset += 3
+
+ if P.type in [token.COLON, token.EQUAL, token.NAME]:
+ # This conditional branch is meant to handle dictionary keys,
+ # variable assignments, 'return STRING' statement lines, and
+ # 'else STRING' ternary expression lines.
+
+ # WMA4 a single space.
+ offset += 1
+
+ # WMA4 the lengths of any leaves that came before that space.
+ for leaf in LL[: p_idx + 1]:
+ offset += len(str(leaf))
+
+ if is_valid_index(string_idx + 1):
+ N = LL[string_idx + 1]
+ if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2:
+ # If the next leaf is an empty RPAR placeholder, we should skip it.
+ N = LL[string_idx + 2]
+
+ if N.type == token.COMMA:
+ # WMA4 a single comma at the end of the string (e.g `STRING,`).
+ offset += 1
+
+ if is_valid_index(string_idx + 2):
+ NN = LL[string_idx + 2]
+
+ if N.type == token.DOT and NN.type == token.NAME:
+ # This conditional branch is meant to handle method calls invoked
+ # off of a string literal up to and including the LPAR character.
+
+ # WMA4 the '.' character.
+ offset += 1
+
+ if (
+ is_valid_index(string_idx + 3)
+ and LL[string_idx + 3].type == token.LPAR
+ ):
+ # WMA4 the left parenthesis character.
+ offset += 1
+
+ # WMA4 the length of the method's name.
+ offset += len(NN.value)
+
+ has_comments = False
+ for comment_leaf in line.comments_after(LL[string_idx]):
+ if not has_comments:
+ has_comments = True
+ # WMA4 two spaces before the '#' character.
+ offset += 2
+
+ # WMA4 the length of the inline comment.
+ offset += len(comment_leaf.value)
+
+ max_string_length = self.line_length - offset
+ return max_string_length
+
+
+class StringSplitter(CustomSplitMapMixin, BaseStringSplitter):
+ """
+ StringTransformer that splits "atom" strings (i.e. strings which exist on
+ lines by themselves).
+
+ Requirements:
+ * The line consists ONLY of a single string (with the exception of a
+ '+' symbol which MAY exist at the start of the line), MAYBE a string
+ trailer, and MAYBE a trailing comma.
+ AND
+ * All of the requirements listed in BaseStringSplitter's docstring.
+
+ Transformations:
+ The string mentioned in the 'Requirements' section is split into as
+ many substrings as necessary to adhere to the configured line length.
+
+ In the final set of substrings, no substring should be smaller than
+ MIN_SUBSTR_SIZE characters.
+
+ The string will ONLY be split on spaces (i.e. each new substring should
+ start with a space).
+
+ If the string is an f-string, it will NOT be split in the middle of an
+ f-expression (e.g. in f"FooBar: {foo() if x else bar()}", {foo() if x
+ else bar()} is an f-expression).
+
+ If the string that is being split has an associated set of custom split
+ records and those custom splits will NOT result in any line going over
+ the configured line length, those custom splits are used. Otherwise the
+ string is split as late as possible (from left-to-right) while still
+ adhering to the transformation rules listed above.
+
+ Collaborations:
+ StringSplitter relies on StringMerger to construct the appropriate
+ CustomSplit objects and add them to the custom split map.
+ """
+
+ MIN_SUBSTR_SIZE = 6
+ # Matches an "f-expression" (e.g. {var}) that might be found in an f-string.
+ RE_FEXPR = r"""
+ (?<!\{)\{
+ (?:
+ [^\{\}]
+ | \{\{
+ | \}\}
+ )+?
+ (?<!\})(?:\}\})*\}(?!\})
+ """
+
+ def do_splitter_match(self, line: Line) -> TMatchResult:
+ LL = line.leaves
+
+ is_valid_index = is_valid_index_factory(LL)
+
+ idx = 0
+
+ # The first leaf MAY be a '+' symbol...
+ if is_valid_index(idx) and LL[idx].type == token.PLUS:
+ idx += 1
+
+ # The next/first leaf MAY be an empty LPAR...
+ if is_valid_index(idx) and is_empty_lpar(LL[idx]):
+ idx += 1
+
+ # The next/first leaf MUST be a string...
+ if not is_valid_index(idx) or LL[idx].type != token.STRING:
+ return TErr("Line does not start with a string.")
+
+ string_idx = idx
+
+ # Skip the string trailer, if one exists.
+ string_parser = StringParser()
+ idx = string_parser.parse(LL, string_idx)
+
+ # That string MAY be followed by an empty RPAR...
+ if is_valid_index(idx) and is_empty_rpar(LL[idx]):
+ idx += 1
+
+ # That string / empty RPAR leaf MAY be followed by a comma...
+ if is_valid_index(idx) and LL[idx].type == token.COMMA:
+ idx += 1
+
+ # But no more leaves are allowed...
+ if is_valid_index(idx):
+ return TErr("This line does not end with a string.")
+
+ return Ok(string_idx)
+
+ def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
+ LL = line.leaves
+
+ QUOTE = LL[string_idx].value[-1]
+
+ is_valid_index = is_valid_index_factory(LL)
+ insert_str_child = insert_str_child_factory(LL[string_idx])
+
+ prefix = get_string_prefix(LL[string_idx].value)
+
+ # We MAY choose to drop the 'f' prefix from substrings that don't
+ # contain any f-expressions, but ONLY if the original f-string
+ # containes at least one f-expression. Otherwise, we will alter the AST
+ # of the program.
+ drop_pointless_f_prefix = ("f" in prefix) and re.search(
+ self.RE_FEXPR, LL[string_idx].value, re.VERBOSE
+ )
+
+ first_string_line = True
+ starts_with_plus = LL[0].type == token.PLUS
+
+ def line_needs_plus() -> bool:
+ return first_string_line and starts_with_plus
+
+ def maybe_append_plus(new_line: Line) -> None:
+ """
+ Side Effects:
+ If @line starts with a plus and this is the first line we are
+ constructing, this function appends a PLUS leaf to @new_line
+ and replaces the old PLUS leaf in the node structure. Otherwise
+ this function does nothing.
+ """
+ if line_needs_plus():
+ plus_leaf = Leaf(token.PLUS, "+")
+ replace_child(LL[0], plus_leaf)
+ new_line.append(plus_leaf)
+
+ ends_with_comma = (
+ is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA
+ )
+
+ def max_last_string() -> int:
+ """
+ Returns:
+ The max allowed length of the string value used for the last
+ line we will construct.
+ """
+ result = self.line_length
+ result -= line.depth * 4
+ result -= 1 if ends_with_comma else 0
+ result -= 2 if line_needs_plus() else 0
+ return result
+
+ # --- Calculate Max Break Index (for string value)
+ # We start with the line length limit
+ max_break_idx = self.line_length
+ # The last index of a string of length N is N-1.
+ max_break_idx -= 1
+ # Leading whitespace is not present in the string value (e.g. Leaf.value).
+ max_break_idx -= line.depth * 4
+ if max_break_idx < 0:
+ yield TErr(
+ f"Unable to split {LL[string_idx].value} at such high of a line depth:"
+ f" {line.depth}"
+ )
+ return
+
+ # Check if StringMerger registered any custom splits.
+ custom_splits = self.pop_custom_splits(LL[string_idx].value)
+ # We use them ONLY if none of them would produce lines that exceed the
+ # line limit.
+ use_custom_breakpoints = bool(
+ custom_splits
+ and all(csplit.break_idx <= max_break_idx for csplit in custom_splits)
+ )
+
+ # Temporary storage for the remaining chunk of the string line that
+ # can't fit onto the line currently being constructed.
+ rest_value = LL[string_idx].value
+
+ def more_splits_should_be_made() -> bool:
+ """
+ Returns:
+ True iff `rest_value` (the remaining string value from the last
+ split), should be split again.
+ """
+ if use_custom_breakpoints:
+ return len(custom_splits) > 1
+ else:
+ return len(rest_value) > max_last_string()
+
+ string_line_results: List[Ok[Line]] = []
+ while more_splits_should_be_made():
+ if use_custom_breakpoints:
+ # Custom User Split (manual)
+ csplit = custom_splits.pop(0)
+ break_idx = csplit.break_idx
+ else:
+ # Algorithmic Split (automatic)
+ max_bidx = max_break_idx - 2 if line_needs_plus() else max_break_idx
+ maybe_break_idx = self.__get_break_idx(rest_value, max_bidx)
+ if maybe_break_idx is None:
+ # If we are unable to algorthmically determine a good split
+ # and this string has custom splits registered to it, we
+ # fall back to using them--which means we have to start
+ # over from the beginning.
+ if custom_splits:
+ rest_value = LL[string_idx].value
+ string_line_results = []
+ first_string_line = True
+ use_custom_breakpoints = True
+ continue
+
+ # Otherwise, we stop splitting here.
+ break
+
+ break_idx = maybe_break_idx
+
+ # --- Construct `next_value`
+ next_value = rest_value[:break_idx] + QUOTE
+ if (
+ # Are we allowed to try to drop a pointless 'f' prefix?
+ drop_pointless_f_prefix
+ # If we are, will we be successful?
+ and next_value != self.__normalize_f_string(next_value, prefix)
+ ):
+ # If the current custom split did NOT originally use a prefix,
+ # then `csplit.break_idx` will be off by one after removing
+ # the 'f' prefix.
+ break_idx = (
+ break_idx + 1
+ if use_custom_breakpoints and not csplit.has_prefix
+ else break_idx
+ )
+ next_value = rest_value[:break_idx] + QUOTE
+ next_value = self.__normalize_f_string(next_value, prefix)
+
+ # --- Construct `next_leaf`
+ next_leaf = Leaf(token.STRING, next_value)
+ insert_str_child(next_leaf)
+ self.__maybe_normalize_string_quotes(next_leaf)
+
+ # --- Construct `next_line`
+ next_line = line.clone()
+ maybe_append_plus(next_line)
+ next_line.append(next_leaf)
+ string_line_results.append(Ok(next_line))
+
+ rest_value = prefix + QUOTE + rest_value[break_idx:]
+ first_string_line = False
+
+ yield from string_line_results
+
+ if drop_pointless_f_prefix:
+ rest_value = self.__normalize_f_string(rest_value, prefix)
+
+ rest_leaf = Leaf(token.STRING, rest_value)
+ insert_str_child(rest_leaf)
+
+ # NOTE: I could not find a test case that verifies that the following
+ # line is actually necessary, but it seems to be. Otherwise we risk
+ # not normalizing the last substring, right?
+ self.__maybe_normalize_string_quotes(rest_leaf)
+
+ last_line = line.clone()
+ maybe_append_plus(last_line)
+
+ # If there are any leaves to the right of the target string...
+ if is_valid_index(string_idx + 1):
+ # We use `temp_value` here to determine how long the last line
+ # would be if we were to append all the leaves to the right of the
+ # target string to the last string line.
+ temp_value = rest_value
+ for leaf in LL[string_idx + 1 :]:
+ temp_value += str(leaf)
+ if leaf.type == token.LPAR:
+ break
+
+ # Try to fit them all on the same line with the last substring...
+ if (
+ len(temp_value) <= max_last_string()
+ or LL[string_idx + 1].type == token.COMMA
+ ):
+ last_line.append(rest_leaf)
+ append_leaves(last_line, line, LL[string_idx + 1 :])
+ yield Ok(last_line)
+ # Otherwise, place the last substring on one line and everything
+ # else on a line below that...
+ else:
+ last_line.append(rest_leaf)
+ yield Ok(last_line)
+
+ non_string_line = line.clone()
+ append_leaves(non_string_line, line, LL[string_idx + 1 :])
+ yield Ok(non_string_line)
+ # Else the target string was the last leaf...
+ else:
+ last_line.append(rest_leaf)
+ last_line.comments = line.comments.copy()
+ yield Ok(last_line)
+
+ def __get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]:
+ """
+ This method contains the algorithm that StringSplitter uses to
+ determine which character to split each string at.
+
+ Args:
+ @string: The substring that we are attempting to split.
+ @max_break_idx: The ideal break index. We will return this value if it
+ meets all the necessary conditions. In the likely event that it
+ doesn't we will try to find the closest index BELOW @max_break_idx
+ that does. If that fails, we will expand our search by also
+ considering all valid indices ABOVE @max_break_idx.
+
+ Pre-Conditions:
+ * assert_is_leaf_string(@string)
+ * 0 <= @max_break_idx < len(@string)
+
+ Returns:
+ break_idx, if an index is able to be found that meets all of the
+ conditions listed in the 'Transformations' section of this classes'
+ docstring.
+ OR
+ None, otherwise.
+ """
+ is_valid_index = is_valid_index_factory(string)
+
+ assert is_valid_index(max_break_idx)
+ assert_is_leaf_string(string)
+
+ _fexpr_slices: Optional[List[Tuple[Index, Index]]] = None
+
+ def fexpr_slices() -> Iterator[Tuple[Index, Index]]:
+ """
+ Yields:
+ All ranges of @string which, if @string were to be split there,
+ would result in the splitting of an f-expression (which is NOT
+ allowed).
+ """
+ nonlocal _fexpr_slices
+
+ if _fexpr_slices is None:
+ _fexpr_slices = []
+ for match in re.finditer(self.RE_FEXPR, string, re.VERBOSE):
+ _fexpr_slices.append(match.span())
+
+ yield from _fexpr_slices
+
+ is_fstring = "f" in get_string_prefix(string)
+
+ def breaks_fstring_expression(i: Index) -> bool:
+ """
+ Returns:
+ True iff returning @i would result in the splitting of an
+ f-expression (which is NOT allowed).
+ """
+ if not is_fstring:
+ return False
+
+ for (start, end) in fexpr_slices():
+ if start <= i < end:
+ return True
+
+ return False
+
+ def passes_all_checks(i: Index) -> bool:
+ """
+ Returns:
+ True iff ALL of the conditions listed in the 'Transformations'
+ section of this classes' docstring would be be met by returning @i.
+ """
+ is_space = string[i] == " "
+ is_big_enough = (
+ len(string[i:]) >= self.MIN_SUBSTR_SIZE
+ and len(string[:i]) >= self.MIN_SUBSTR_SIZE
+ )
+ return is_space and is_big_enough and not breaks_fstring_expression(i)
+
+ # First, we check all indices BELOW @max_break_idx.
+ break_idx = max_break_idx
+ while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx):
+ break_idx -= 1
+
+ if not passes_all_checks(break_idx):
+ # If that fails, we check all indices ABOVE @max_break_idx.
+ #
+ # If we are able to find a valid index here, the next line is going
+ # to be longer than the specified line length, but it's probably
+ # better than doing nothing at all.
+ break_idx = max_break_idx + 1
+ while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx):
+ break_idx += 1
+
+ if not is_valid_index(break_idx) or not passes_all_checks(break_idx):
+ return None
+
+ return break_idx
+
+ def __maybe_normalize_string_quotes(self, leaf: Leaf) -> None:
+ if self.normalize_strings:
+ normalize_string_quotes(leaf)
+
+ def __normalize_f_string(self, string: str, prefix: str) -> str:
+ """
+ Pre-Conditions:
+ * assert_is_leaf_string(@string)
+
+ Returns:
+ * If @string is an f-string that contains no f-expressions, we
+ return a string identical to @string except that the 'f' prefix
+ has been stripped and all double braces (i.e. '{{' or '}}') have
+ been normalized (i.e. turned into '{' or '}').
+ OR
+ * Otherwise, we return @string.
+ """
+ assert_is_leaf_string(string)
+
+ if "f" in prefix and not re.search(self.RE_FEXPR, string, re.VERBOSE):
+ new_prefix = prefix.replace("f", "")
+
+ temp = string[len(prefix) :]
+ temp = re.sub(r"\{\{", "{", temp)
+ temp = re.sub(r"\}\}", "}", temp)
+ new_string = temp
+
+ return f"{new_prefix}{new_string}"
+ else:
+ return string
+
+
+class StringParenWrapper(CustomSplitMapMixin, BaseStringSplitter):
+ """
+ StringTransformer that splits non-"atom" strings (i.e. strings that do not
+ exist on lines by themselves).
+
+ Requirements:
+ All of the requirements listed in BaseStringSplitter's docstring in
+ addition to the requirements listed below:
+
+ * The line is a return/yield statement, which returns/yields a string.
+ OR
+ * The line is part of a ternary expression (e.g. `x = y if cond else
+ z`) such that the line starts with `else <string>`, where <string> is
+ some string.
+ OR
+ * The line is an assert statement, which ends with a string.
+ OR
+ * The line is an assignment statement (e.g. `x = <string>` or `x +=
+ <string>`) such that the variable is being assigned the value of some
+ string.
+ OR
+ * The line is a dictionary key assignment where some valid key is being
+ assigned the value of some string.
+
+ Transformations:
+ The chosen string is wrapped in parentheses and then split at the LPAR.
+
+ We then have one line which ends with an LPAR and another line that
+ starts with the chosen string. The latter line is then split again at
+ the RPAR. This results in the RPAR (and possibly a trailing comma)
+ being placed on its own line.
+
+ NOTE: If any leaves exist to the right of the chosen string (except
+ for a trailing comma, which would be placed after the RPAR), those
+ leaves are placed inside the parentheses. In effect, the chosen
+ string is not necessarily being "wrapped" by parentheses. We can,
+ however, count on the LPAR being placed directly before the chosen
+ string.
+
+ In other words, StringParenWrapper creates "atom" strings. These
+ can then be split again by StringSplitter, if necessary.
+
+ Collaborations:
+ In the event that a string line split by StringParenWrapper is
+ changed such that it no longer needs to be given its own line,
+ StringParenWrapper relies on StringParenStripper to clean up the
+ parentheses it created.
+ """
+
+ def do_splitter_match(self, line: Line) -> TMatchResult:
+ LL = line.leaves
+
+ string_idx = None
+ string_idx = string_idx or self._return_match(LL)
+ string_idx = string_idx or self._else_match(LL)
+ string_idx = string_idx or self._assert_match(LL)
+ string_idx = string_idx or self._assign_match(LL)
+ string_idx = string_idx or self._dict_match(LL)
+
+ if string_idx is not None:
+ string_value = line.leaves[string_idx].value
+ # If the string has no spaces...
+ if " " not in string_value:
+ # And will still violate the line length limit when split...
+ max_string_length = self.line_length - ((line.depth + 1) * 4)
+ if len(string_value) > max_string_length:
+ # And has no associated custom splits...
+ if not self.has_custom_splits(string_value):
+ # Then we should NOT put this string on its own line.
+ return TErr(
+ "We do not wrap long strings in parentheses when the"
+ " resultant line would still be over the specified line"
+ " length and can't be split further by StringSplitter."
+ )
+ return Ok(string_idx)
+
+ return TErr("This line does not contain any non-atomic strings.")
+
+ @staticmethod
+ def _return_match(LL: List[Leaf]) -> Optional[int]:
+ """
+ Returns:
+ string_idx such that @LL[string_idx] is equal to our target (i.e.
+ matched) string, if this line matches the return/yield statement
+ requirements listed in the 'Requirements' section of this classes'
+ docstring.
+ OR
+ None, otherwise.
+ """
+ # If this line is apart of a return/yield statement and the first leaf
+ # contains either the "return" or "yield" keywords...
+ if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[
+ 0
+ ].value in ["return", "yield"]:
+ is_valid_index = is_valid_index_factory(LL)
+
+ idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
+ # The next visible leaf MUST contain a string...
+ if is_valid_index(idx) and LL[idx].type == token.STRING:
+ return idx
+
+ return None
+
+ @staticmethod
+ def _else_match(LL: List[Leaf]) -> Optional[int]:
+ """
+ Returns:
+ string_idx such that @LL[string_idx] is equal to our target (i.e.
+ matched) string, if this line matches the ternary expression
+ requirements listed in the 'Requirements' section of this classes'
+ docstring.
+ OR
+ None, otherwise.
+ """
+ # If this line is apart of a ternary expression and the first leaf
+ # contains the "else" keyword...
+ if (
+ parent_type(LL[0]) == syms.test
+ and LL[0].type == token.NAME
+ and LL[0].value == "else"
+ ):
+ is_valid_index = is_valid_index_factory(LL)
+
+ idx = 2 if is_valid_index(1) and is_empty_par(LL[1]) else 1
+ # The next visible leaf MUST contain a string...
+ if is_valid_index(idx) and LL[idx].type == token.STRING:
+ return idx
+
+ return None
+
+ @staticmethod
+ def _assert_match(LL: List[Leaf]) -> Optional[int]:
+ """
+ Returns:
+ string_idx such that @LL[string_idx] is equal to our target (i.e.
+ matched) string, if this line matches the assert statement
+ requirements listed in the 'Requirements' section of this classes'
+ docstring.
+ OR
+ None, otherwise.
+ """
+ # If this line is apart of an assert statement and the first leaf
+ # contains the "assert" keyword...
+ if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert":
+ is_valid_index = is_valid_index_factory(LL)
+
+ for (i, leaf) in enumerate(LL):
+ # We MUST find a comma...
+ if leaf.type == token.COMMA:
+ idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
+
+ # That comma MUST be followed by a string...
+ if is_valid_index(idx) and LL[idx].type == token.STRING:
+ string_idx = idx
+
+ # Skip the string trailer, if one exists.
+ string_parser = StringParser()
+ idx = string_parser.parse(LL, string_idx)
+
+ # But no more leaves are allowed...
+ if not is_valid_index(idx):
+ return string_idx
+
+ return None
+
+ @staticmethod
+ def _assign_match(LL: List[Leaf]) -> Optional[int]:
+ """
+ Returns:
+ string_idx such that @LL[string_idx] is equal to our target (i.e.
+ matched) string, if this line matches the assignment statement
+ requirements listed in the 'Requirements' section of this classes'
+ docstring.
+ OR
+ None, otherwise.
+ """
+ # If this line is apart of an expression statement or is a function
+ # argument AND the first leaf contains a variable name...
+ if (
+ parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power]
+ and LL[0].type == token.NAME
+ ):
+ is_valid_index = is_valid_index_factory(LL)
+
+ for (i, leaf) in enumerate(LL):
+ # We MUST find either an '=' or '+=' symbol...
+ if leaf.type in [token.EQUAL, token.PLUSEQUAL]:
+ idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
+
+ # That symbol MUST be followed by a string...
+ if is_valid_index(idx) and LL[idx].type == token.STRING:
+ string_idx = idx
+
+ # Skip the string trailer, if one exists.
+ string_parser = StringParser()
+ idx = string_parser.parse(LL, string_idx)
+
+ # The next leaf MAY be a comma iff this line is apart
+ # of a function argument...
+ if (
+ parent_type(LL[0]) == syms.argument
+ and is_valid_index(idx)
+ and LL[idx].type == token.COMMA
+ ):
+ idx += 1
+
+ # But no more leaves are allowed...
+ if not is_valid_index(idx):
+ return string_idx
+
+ return None
+
+ @staticmethod
+ def _dict_match(LL: List[Leaf]) -> Optional[int]:
+ """
+ Returns:
+ string_idx such that @LL[string_idx] is equal to our target (i.e.
+ matched) string, if this line matches the dictionary key assignment
+ statement requirements listed in the 'Requirements' section of this
+ classes' docstring.
+ OR
+ None, otherwise.
+ """
+ # If this line is apart of a dictionary key assignment...
+ if syms.dictsetmaker in [parent_type(LL[0]), parent_type(LL[0].parent)]:
+ is_valid_index = is_valid_index_factory(LL)
+
+ for (i, leaf) in enumerate(LL):
+ # We MUST find a colon...
+ if leaf.type == token.COLON:
+ idx = i + 2 if is_empty_par(LL[i + 1]) else i + 1
+
+ # That colon MUST be followed by a string...
+ if is_valid_index(idx) and LL[idx].type == token.STRING:
+ string_idx = idx
+
+ # Skip the string trailer, if one exists.
+ string_parser = StringParser()
+ idx = string_parser.parse(LL, string_idx)
+
+ # That string MAY be followed by a comma...
+ if is_valid_index(idx) and LL[idx].type == token.COMMA:
+ idx += 1
+
+ # But no more leaves are allowed...
+ if not is_valid_index(idx):
+ return string_idx
+
+ return None
+
+ def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
+ LL = line.leaves
+
+ is_valid_index = is_valid_index_factory(LL)
+ insert_str_child = insert_str_child_factory(LL[string_idx])
+
+ comma_idx = len(LL) - 1
+ ends_with_comma = False
+ if LL[comma_idx].type == token.COMMA:
+ ends_with_comma = True
+
+ leaves_to_steal_comments_from = [LL[string_idx]]
+ if ends_with_comma:
+ leaves_to_steal_comments_from.append(LL[comma_idx])
+
+ # --- First Line
+ first_line = line.clone()
+ left_leaves = LL[:string_idx]
+
+ # We have to remember to account for (possibly invisible) LPAR and RPAR
+ # leaves that already wrapped the target string. If these leaves do
+ # exist, we will replace them with our own LPAR and RPAR leaves.
+ old_parens_exist = False
+ if left_leaves and left_leaves[-1].type == token.LPAR:
+ old_parens_exist = True
+ leaves_to_steal_comments_from.append(left_leaves[-1])
+ left_leaves.pop()
+
+ append_leaves(first_line, line, left_leaves)
+
+ lpar_leaf = Leaf(token.LPAR, "(")
+ if old_parens_exist:
+ replace_child(LL[string_idx - 1], lpar_leaf)
+ else:
+ insert_str_child(lpar_leaf)
+ first_line.append(lpar_leaf)
+
+ # We throw inline comments that were originally to the right of the
+ # target string to the top line. They will now be shown to the right of
+ # the LPAR.
+ for leaf in leaves_to_steal_comments_from:
+ for comment_leaf in line.comments_after(leaf):
+ first_line.append(comment_leaf, preformatted=True)
+
+ yield Ok(first_line)
+
+ # --- Middle (String) Line
+ # We only need to yield one (possibly too long) string line, since the
+ # `StringSplitter` will break it down further if necessary.
+ string_value = LL[string_idx].value
+ string_line = Line(
+ depth=line.depth + 1,
+ inside_brackets=True,
+ should_explode=line.should_explode,
+ )
+ string_leaf = Leaf(token.STRING, string_value)
+ insert_str_child(string_leaf)
+ string_line.append(string_leaf)
+
+ old_rpar_leaf = None
+ if is_valid_index(string_idx + 1):
+ right_leaves = LL[string_idx + 1 :]
+ if ends_with_comma:
+ right_leaves.pop()
+
+ if old_parens_exist:
+ assert (
+ right_leaves and right_leaves[-1].type == token.RPAR
+ ), "Apparently, old parentheses do NOT exist?!"
+ old_rpar_leaf = right_leaves.pop()
+
+ append_leaves(string_line, line, right_leaves)
+
+ yield Ok(string_line)
+
+ # --- Last Line
+ last_line = line.clone()
+ last_line.bracket_tracker = first_line.bracket_tracker
+
+ new_rpar_leaf = Leaf(token.RPAR, ")")
+ if old_rpar_leaf is not None:
+ replace_child(old_rpar_leaf, new_rpar_leaf)
+ else:
+ insert_str_child(new_rpar_leaf)
+ last_line.append(new_rpar_leaf)
+
+ # If the target string ended with a comma, we place this comma to the
+ # right of the RPAR on the last line.
+ if ends_with_comma:
+ comma_leaf = Leaf(token.COMMA, ",")
+ replace_child(LL[comma_idx], comma_leaf)
+ last_line.append(comma_leaf)
+
+ yield Ok(last_line)
+
+
+class StringParser:
+ """
+ A state machine that aids in parsing a string's "trailer", which can be
+ either non-existant, an old-style formatting sequence (e.g. `% varX` or `%
+ (varX, varY)`), or a method-call / attribute access (e.g. `.format(varX,
+ varY)`).
+
+ NOTE: A new StringParser object MUST be instantiated for each string
+ trailer we need to parse.
+
+ Examples:
+ We shall assume that `line` equals the `Line` object that corresponds
+ to the following line of python code:
+ ```
+ x = "Some {}.".format("String") + some_other_string
+ ```
+
+ Furthermore, we will assume that `string_idx` is some index such that:
+ ```
+ assert line.leaves[string_idx].value == "Some {}."
+ ```
+
+ The following code snippet then holds:
+ ```
+ string_parser = StringParser()
+ idx = string_parser.parse(line.leaves, string_idx)
+ assert line.leaves[idx].type == token.PLUS
+ ```
+ """
+
+ DEFAULT_TOKEN = -1
+
+ # String Parser States
+ START = 1
+ DOT = 2
+ NAME = 3
+ PERCENT = 4
+ SINGLE_FMT_ARG = 5
+ LPAR = 6
+ RPAR = 7
+ DONE = 8
+
+ # Lookup Table for Next State
+ _goto: Dict[Tuple[ParserState, NodeType], ParserState] = {
+ # A string trailer may start with '.' OR '%'.
+ (START, token.DOT): DOT,
+ (START, token.PERCENT): PERCENT,
+ (START, DEFAULT_TOKEN): DONE,
+ # A '.' MUST be followed by an attribute or method name.
+ (DOT, token.NAME): NAME,
+ # A method name MUST be followed by an '(', whereas an attribute name
+ # is the last symbol in the string trailer.
+ (NAME, token.LPAR): LPAR,
+ (NAME, DEFAULT_TOKEN): DONE,
+ # A '%' symbol can be followed by an '(' or a single argument (e.g. a
+ # string or variable name).
+ (PERCENT, token.LPAR): LPAR,
+ (PERCENT, DEFAULT_TOKEN): SINGLE_FMT_ARG,
+ # If a '%' symbol is followed by a single argument, that argument is
+ # the last leaf in the string trailer.
+ (SINGLE_FMT_ARG, DEFAULT_TOKEN): DONE,
+ # If present, a ')' symbol is the last symbol in a string trailer.
+ # (NOTE: LPARS and nested RPARS are not included in this lookup table,
+ # since they are treated as a special case by the parsing logic in this
+ # classes' implementation.)
+ (RPAR, DEFAULT_TOKEN): DONE,
+ }
+
+ def __init__(self) -> None:
+ self._state = self.START
+ self._unmatched_lpars = 0
+
+ def parse(self, leaves: List[Leaf], string_idx: int) -> int:
+ """
+ Pre-conditions:
+ * @leaves[@string_idx].type == token.STRING
+
+ Returns:
+ The index directly after the last leaf which is apart of the string
+ trailer, if a "trailer" exists.
+ OR
+ @string_idx + 1, if no string "trailer" exists.
+ """
+ assert leaves[string_idx].type == token.STRING
+
+ idx = string_idx + 1
+ while idx < len(leaves) and self._next_state(leaves[idx]):
+ idx += 1
+ return idx
+
+ def _next_state(self, leaf: Leaf) -> bool:
+ """
+ Pre-conditions:
+ * On the first call to this function, @leaf MUST be the leaf that
+ was directly after the string leaf in question (e.g. if our target
+ string is `line.leaves[i]` then the first call to this method must
+ be `line.leaves[i + 1]`).
+ * On the next call to this function, the leaf paramater passed in
+ MUST be the leaf directly following @leaf.
+
+ Returns:
+ True iff @leaf is apart of the string's trailer.
+ """
+ # We ignore empty LPAR or RPAR leaves.
+ if is_empty_par(leaf):
+ return True
+
+ next_token = leaf.type
+ if next_token == token.LPAR:
+ self._unmatched_lpars += 1
+
+ current_state = self._state
+
+ # The LPAR parser state is a special case. We will return True until we
+ # find the matching RPAR token.
+ if current_state == self.LPAR:
+ if next_token == token.RPAR:
+ self._unmatched_lpars -= 1
+ if self._unmatched_lpars == 0:
+ self._state = self.RPAR
+ # Otherwise, we use a lookup table to determine the next state.
+ else:
+ # If the lookup table matches the current state to the next
+ # token, we use the lookup table.
+ if (current_state, next_token) in self._goto:
+ self._state = self._goto[current_state, next_token]
+ else:
+ # Otherwise, we check if a the current state was assigned a
+ # default.
+ if (current_state, self.DEFAULT_TOKEN) in self._goto:
+ self._state = self._goto[current_state, self.DEFAULT_TOKEN]
+ # If no default has been assigned, then this parser has a logic
+ # error.
+ else:
+ raise RuntimeError(f"{self.__class__.__name__} LOGIC ERROR!")
+
+ if self._state == self.DONE:
+ return False
+
+ return True
+
+
+def TErr(err_msg: str) -> Err[CannotTransform]:
+ """(T)ransform Err
+
+ Convenience function used when working with the TResult type.
+ """
+ cant_transform = CannotTransform(err_msg)
+ return Err(cant_transform)
+
+
+def contains_pragma_comment(comment_list: List[Leaf]) -> bool:
+ """
+ Returns:
+ True iff one of the comments in @comment_list is a pragma used by one
+ of the more common static analysis tools for python (e.g. mypy, flake8,
+ pylint).
+ """
+ for comment in comment_list:
+ if comment.value.startswith(("# type:", "# noqa", "# pylint:")):
+ return True
+
+ return False
+
+
+def insert_str_child_factory(string_leaf: Leaf) -> Callable[[LN], None]:
+ """
+ Factory for a convenience function that is used to orphan @string_leaf
+ and then insert multiple new leaves into the same part of the node
+ structure that @string_leaf had originally occupied.
+
+ Examples:
+ Let `string_leaf = Leaf(token.STRING, '"foo"')` and `N =
+ string_leaf.parent`. Assume the node `N` has the following
+ original structure:
+
+ Node(
+ expr_stmt, [
+ Leaf(NAME, 'x'),
+ Leaf(EQUAL, '='),
+ Leaf(STRING, '"foo"'),
+ ]
+ )
+
+ We then run the code snippet shown below.
+ ```
+ insert_str_child = insert_str_child_factory(string_leaf)
+
+ lpar = Leaf(token.LPAR, '(')
+ insert_str_child(lpar)
+
+ bar = Leaf(token.STRING, '"bar"')
+ insert_str_child(bar)
+
+ rpar = Leaf(token.RPAR, ')')
+ insert_str_child(rpar)
+ ```
+
+ After which point, it follows that `string_leaf.parent is None` and
+ the node `N` now has the following structure:
+
+ Node(
+ expr_stmt, [
+ Leaf(NAME, 'x'),
+ Leaf(EQUAL, '='),
+ Leaf(LPAR, '('),
+ Leaf(STRING, '"bar"'),
+ Leaf(RPAR, ')'),
+ ]
+ )
+ """
+ string_parent = string_leaf.parent
+ string_child_idx = string_leaf.remove()
+
+ def insert_str_child(child: LN) -> None:
+ nonlocal string_child_idx
+
+ assert string_parent is not None
+ assert string_child_idx is not None
+
+ string_parent.insert_child(string_child_idx, child)
+ string_child_idx += 1
+
+ return insert_str_child
+
+
+def has_triple_quotes(string: str) -> bool:
+ """
+ Returns:
+ True iff @string starts with three quotation characters.
+ """
+ raw_string = string.lstrip(STRING_PREFIX_CHARS)
+ return raw_string[:3] in {'"""', "'''"}
+
+
+def parent_type(node: Optional[LN]) -> Optional[NodeType]:
+ """
+ Returns:
+ @node.parent.type, if @node is not None and has a parent.
+ OR
+ None, otherwise.
+ """
+ if node is None or node.parent is None:
+ return None
+
+ return node.parent.type
+
+
+def is_empty_par(leaf: Leaf) -> bool:
+ return is_empty_lpar(leaf) or is_empty_rpar(leaf)
+
+
+def is_empty_lpar(leaf: Leaf) -> bool:
+ return leaf.type == token.LPAR and leaf.value == ""
+
+
+def is_empty_rpar(leaf: Leaf) -> bool:
+ return leaf.type == token.RPAR and leaf.value == ""
+
+
+def is_valid_index_factory(seq: Sequence[Any]) -> Callable[[int], bool]:
+ """
+ Examples:
+ ```
+ my_list = [1, 2, 3]
+
+ is_valid_index = is_valid_index_factory(my_list)
+
+ assert is_valid_index(0)
+ assert is_valid_index(2)
+
+ assert not is_valid_index(3)
+ assert not is_valid_index(-1)
+ ```
+ """
+
+ def is_valid_index(idx: int) -> bool:
+ """
+ Returns:
+ True iff @idx is positive AND seq[@idx] does NOT raise an
+ IndexError.
+ """
+ return 0 <= idx < len(seq)
+
+ return is_valid_index
+
+
+def line_to_string(line: Line) -> str:
+ """Returns the string representation of @line.
+
+ WARNING: This is known to be computationally expensive.
+ """
+ return str(line).strip("\n")
+
+
+def append_leaves(new_line: Line, old_line: Line, leaves: List[Leaf]) -> None:
+ """
+ Append leaves (taken from @old_line) to @new_line, making sure to fix the
+ underlying Node structure where appropriate.
+
+ All of the leaves in @leaves are duplicated. The duplicates are then
+ appended to @new_line and used to replace their originals in the underlying
+ Node structure. Any comments attatched to the old leaves are reattached to
+ the new leaves.
+
+ Pre-conditions:
+ set(@leaves) is a subset of set(@old_line.leaves).
+ """
+ for old_leaf in leaves:
+ assert old_leaf in old_line.leaves
+
+ new_leaf = Leaf(old_leaf.type, old_leaf.value)
+ replace_child(old_leaf, new_leaf)
+ new_line.append(new_leaf)
+
+ for comment_leaf in old_line.comments_after(old_leaf):
+ new_line.append(comment_leaf, preformatted=True)
+
+
+def replace_child(old_child: LN, new_child: LN) -> None:
+ """
+ Side Effects:
+ * If @old_child.parent is set, replace @old_child with @new_child in
+ @old_child's underlying Node structure.
+ OR
+ * Otherwise, this function does nothing.
+ """
+ parent = old_child.parent
+ if not parent:
+ return
+
+ child_idx = old_child.remove()
+ if child_idx is not None:
+ parent.insert_child(child_idx, new_child)
+
+
+def get_string_prefix(string: str) -> str:
+ """
+ Pre-conditions:
+ * assert_is_leaf_string(@string)
+
+ Returns:
+ @string's prefix (e.g. '', 'r', 'f', or 'rf').
+ """
+ assert_is_leaf_string(string)
+
+ prefix = ""
+ prefix_idx = 0
+ while string[prefix_idx] in STRING_PREFIX_CHARS:
+ prefix += string[prefix_idx].lower()
+ prefix_idx += 1
+
+ return prefix
+
+
+def assert_is_leaf_string(string: str) -> None:
+ """
+ Checks the pre-condition that @string has the format that you would expect
+ of `leaf.value` where `leaf` is some Leaf such that `leaf.type ==
+ token.STRING`. A more precise description of the pre-conditions that are
+ checked are listed below.
+
+ Pre-conditions:
+ * @string starts with either ', ", <prefix>', or <prefix>" where
+ `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`.
+ * @string ends with a quote character (' or ").
+
+ Raises:
+ AssertionError(...) if the pre-conditions listed above are not
+ satisfied.
+ """
+ dquote_idx = string.find('"')
+ squote_idx = string.find("'")
+ if -1 in [dquote_idx, squote_idx]:
+ quote_idx = max(dquote_idx, squote_idx)
+ else:
+ quote_idx = min(squote_idx, dquote_idx)
+
+ assert (
+ 0 <= quote_idx < len(string) - 1
+ ), f"{string!r} is missing a starting quote character (' or \")."
+ assert string[-1] in (
+ "'",
+ '"',
+ ), f"{string!r} is missing an ending quote character (' or \")."
+ assert set(string[:quote_idx]).issubset(
+ set(STRING_PREFIX_CHARS)
+ ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}."
+
+
+def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]:
+ """Split line into many lines, starting with the first matching bracket pair.
+
+ Note: this usually looks weird, only use this for function definitions.
+ Prefer RHS otherwise. This is why this function is not symmetrical with
+ :func:`right_hand_split` which also handles optional parentheses.
+ """
+ tail_leaves: List[Leaf] = []
+ body_leaves: List[Leaf] = []
+ head_leaves: List[Leaf] = []
+ current_leaves = head_leaves
+ matching_bracket: Optional[Leaf] = None
+ for leaf in line.leaves:
+ if (
+ current_leaves is body_leaves
+ and leaf.type in CLOSING_BRACKETS
+ and leaf.opening_bracket is matching_bracket
+ ):
+ current_leaves = tail_leaves if body_leaves else head_leaves
+ current_leaves.append(leaf)
+ if current_leaves is head_leaves:
+ if leaf.type in OPENING_BRACKETS:
+ matching_bracket = leaf
+ current_leaves = body_leaves
+ if not matching_bracket:
+ raise CannotSplit("No brackets found")
+
+ head = bracket_split_build_line(head_leaves, line, matching_bracket)
+ body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
+ tail = bracket_split_build_line(tail_leaves, line, matching_bracket)
+ bracket_split_succeeded_or_raise(head, body, tail)
+ for result in (head, body, tail):
+ if result:
+ yield result
+
+
+def right_hand_split(
+ line: Line,
+ line_length: int,
+ features: Collection[Feature] = (),
+ omit: Collection[LeafID] = (),
+) -> Iterator[Line]:
+ """Split line into many lines, starting with the last matching bracket pair.
+
+ If the split was by optional parentheses, attempt splitting without them, too.
+ `omit` is a collection of closing bracket IDs that shouldn't be considered for
+ this split.
+
+ Note: running this function modifies `bracket_depth` on the leaves of `line`.
+ """
+ tail_leaves: List[Leaf] = []
+ body_leaves: List[Leaf] = []
+ head_leaves: List[Leaf] = []
+ current_leaves = tail_leaves
+ opening_bracket: Optional[Leaf] = None
+ closing_bracket: Optional[Leaf] = None
+ for leaf in reversed(line.leaves):
+ if current_leaves is body_leaves:
+ if leaf is opening_bracket:
+ current_leaves = head_leaves if body_leaves else tail_leaves
+ current_leaves.append(leaf)
+ if current_leaves is tail_leaves:
+ if leaf.type in CLOSING_BRACKETS and id(leaf) not in omit:
+ opening_bracket = leaf.opening_bracket
+ closing_bracket = leaf
+ current_leaves = body_leaves
+ if not (opening_bracket and closing_bracket and head_leaves):
+ # If there is no opening or closing_bracket that means the split failed and
+ # all content is in the tail. Otherwise, if `head_leaves` are empty, it means
+ # the matching `opening_bracket` wasn't available on `line` anymore.
+ raise CannotSplit("No brackets found")
+
+ tail_leaves.reverse()
+ body_leaves.reverse()
+ head_leaves.reverse()
+ head = bracket_split_build_line(head_leaves, line, opening_bracket)
+ body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
+ tail = bracket_split_build_line(tail_leaves, line, opening_bracket)
+ bracket_split_succeeded_or_raise(head, body, tail)
+ if (
+ # the body shouldn't be exploded
+ not body.should_explode
+ # the opening bracket is an optional paren
+ and opening_bracket.type == token.LPAR
+ and not opening_bracket.value
+ # the closing bracket is an optional paren
+ and closing_bracket.type == token.RPAR
+ and not closing_bracket.value
+ # it's not an import (optional parens are the only thing we can split on
+ # in this case; attempting a split without them is a waste of time)
+ and not line.is_import
+ # there are no standalone comments in the body
+ and not body.contains_standalone_comments(0)
+ # and we can actually remove the parens
+ and can_omit_invisible_parens(body, line_length)
+ ):
+ omit = {id(closing_bracket), *omit}
+ try:
+ yield from right_hand_split(line, line_length, features=features, omit=omit)
+ return
+
+ except CannotSplit:
+ if not (
+ can_be_split(body)
+ or is_line_short_enough(body, line_length=line_length)
+ ):
+ raise CannotSplit(
+ "Splitting failed, body is still too long and can't be split."
+ )
+
+ elif head.contains_multiline_strings() or tail.contains_multiline_strings():
+ raise CannotSplit(
+ "The current optional pair of parentheses is bound to fail to"
+ " satisfy the splitting algorithm because the head or the tail"
+ " contains multiline strings which by definition never fit one"
+ " line."
+ )
+
+ ensure_visible(opening_bracket)
+ ensure_visible(closing_bracket)
+ for result in (head, body, tail):
+ if result:
+ yield result
+
+
+def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
+ """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
+
+ Do nothing otherwise.
+
+ A left- or right-hand split is based on a pair of brackets. Content before
+ (and including) the opening bracket is left on one line, content inside the
+ brackets is put on a separate line, and finally content starting with and
+ following the closing bracket is put on a separate line.
+
+ Those are called `head`, `body`, and `tail`, respectively. If the split
+ produced the same line (all content in `head`) or ended up with an empty `body`
+ and the `tail` is just the closing bracket, then it's considered failed.
+ """
+ tail_len = len(str(tail).strip())
+ if not body:
+ if tail_len == 0:
+ raise CannotSplit("Splitting brackets produced the same line")
+
+ elif tail_len < 3:
+ raise CannotSplit(
+ f"Splitting brackets on an empty body to save {tail_len} characters is"
+ " not worth it"
+ )
+
+
+def bracket_split_build_line(
+ leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
+) -> Line:
+ """Return a new line with given `leaves` and respective comments from `original`.
+
+ If `is_body` is True, the result line is one-indented inside brackets and as such
+ has its first leaf's prefix normalized and a trailing comma added when expected.
+ """
+ result = Line(depth=original.depth)
+ if is_body:
+ result.inside_brackets = True
+ result.depth += 1
+ if leaves:
+ # Since body is a new indent level, remove spurious leading whitespace.
+ normalize_prefix(leaves[0], inside_brackets=True)
+ # Ensure a trailing comma for imports and standalone function arguments, but
+ # be careful not to add one after any comments or within type annotations.
+ no_commas = (
+ original.is_def
+ and opening_bracket.value == "("
+ and not any(l.type == token.COMMA for l in leaves)
+ )
+
+ if original.is_import or no_commas:
+ for i in range(len(leaves) - 1, -1, -1):
+ if leaves[i].type == STANDALONE_COMMENT:
+ continue
+
+ if leaves[i].type != token.COMMA:
+ leaves.insert(i + 1, Leaf(token.COMMA, ","))
+ break
+
+ # Populate the line
+ for leaf in leaves:
+ result.append(leaf, preformatted=True)
+ for comment_after in original.comments_after(leaf):
+ result.append(comment_after, preformatted=True)
+ if is_body:
+ result.should_explode = should_explode(result, opening_bracket)
+ return result
+
+
+def dont_increase_indentation(split_func: Transformer) -> Transformer:
+ """Normalize prefix of the first leaf in every line returned by `split_func`.
+
+ This is a decorator over relevant split functions.
+ """
+
+ @wraps(split_func)
+ def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
+ for l in split_func(line, features):
+ normalize_prefix(l.leaves[0], inside_brackets=True)
+ yield l
+
+ return split_wrapper
+
+
+@dont_increase_indentation
+def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
+ """Split according to delimiters of the highest priority.
+
+ If the appropriate Features are given, the split will add trailing commas
+ also in function signatures and calls that contain `*` and `**`.
+ """
+ try:
+ last_leaf = line.leaves[-1]
+ except IndexError:
+ raise CannotSplit("Line empty")
+
+ bt = line.bracket_tracker
+ try:
+ delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})
+ except ValueError:
+ raise CannotSplit("No delimiters found")
+
+ if delimiter_priority == DOT_PRIORITY:
+ if bt.delimiter_count_with_priority(delimiter_priority) == 1:
+ raise CannotSplit("Splitting a single attribute from its owner looks wrong")
+
+ current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+ lowest_depth = sys.maxsize
+ trailing_comma_safe = True
+
+ def append_to_line(leaf: Leaf) -> Iterator[Line]:
+ """Append `leaf` to current line or to new line if appending impossible."""
+ nonlocal current_line
+ try:
+ current_line.append_safe(leaf, preformatted=True)
+ except ValueError:
+ yield current_line
+
+ current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+ current_line.append(leaf)
+
+ for leaf in line.leaves:
+ yield from append_to_line(leaf)
+
+ for comment_after in line.comments_after(leaf):
+ yield from append_to_line(comment_after)
+
+ lowest_depth = min(lowest_depth, leaf.bracket_depth)
+ if leaf.bracket_depth == lowest_depth:
+ if is_vararg(leaf, within={syms.typedargslist}):
+ trailing_comma_safe = (
+ trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
+ )
+ elif is_vararg(leaf, within={syms.arglist, syms.argument}):
+ trailing_comma_safe = (
+ trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
+ )
+
+ leaf_priority = bt.delimiters.get(id(leaf))
+ if leaf_priority == delimiter_priority:
+ yield current_line
+
+ current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+ if current_line:
+ if (
+ trailing_comma_safe
+ and delimiter_priority == COMMA_PRIORITY
+ and current_line.leaves[-1].type != token.COMMA
+ and current_line.leaves[-1].type != STANDALONE_COMMENT
+ ):
+ current_line.append(Leaf(token.COMMA, ","))
+ yield current_line
+
+
+@dont_increase_indentation
+def standalone_comment_split(
+ line: Line, features: Collection[Feature] = ()
+) -> Iterator[Line]:
+ """Split standalone comments from the rest of the line."""
+ if not line.contains_standalone_comments(0):
+ raise CannotSplit("Line does not have any standalone comments")
+
+ current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+
+ def append_to_line(leaf: Leaf) -> Iterator[Line]:
+ """Append `leaf` to current line or to new line if appending impossible."""
+ nonlocal current_line
+ try:
+ current_line.append_safe(leaf, preformatted=True)
+ except ValueError:
+ yield current_line
+
+ current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+ current_line.append(leaf)
+
+ for leaf in line.leaves:
+ yield from append_to_line(leaf)
+
+ for comment_after in line.comments_after(leaf):
+ yield from append_to_line(comment_after)
+
+ if current_line:
+ yield current_line
+
+
+def is_import(leaf: Leaf) -> bool:
+ """Return True if the given leaf starts an import statement."""
+ p = leaf.parent
+ t = leaf.type
+ v = leaf.value
+ return bool(
+ t == token.NAME
+ and (
+ (v == "import" and p and p.type == syms.import_name)
+ or (v == "from" and p and p.type == syms.import_from)
+ )
+ )
+
+
+def is_type_comment(leaf: Leaf, suffix: str = "") -> bool:
+ """Return True if the given leaf is a special comment.
+ Only returns true for type comments for now."""
+ t = leaf.type
+ v = leaf.value
+ return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix)
+
+
+def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
+ """Leave existing extra newlines if not `inside_brackets`. Remove everything
+ else.
+
+ Note: don't use backslashes for formatting or you'll lose your voting rights.
+ """
+ if not inside_brackets:
+ spl = leaf.prefix.split("#")
+ if "\\" not in spl[0]:
+ nl_count = spl[-1].count("\n")
+ if len(spl) > 1:
+ nl_count -= 1
+ leaf.prefix = "\n" * nl_count
+ return
+
+ leaf.prefix = ""
+
+
+def normalize_string_prefix(leaf: Leaf, remove_u_prefix: bool = False) -> None:
+ """Make all string prefixes lowercase.
+
+ If remove_u_prefix is given, also removes any u prefix from the string.
+
+ Note: Mutates its argument.
+ """
+ match = re.match(r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", leaf.value, re.DOTALL)
+ assert match is not None, f"failed to match string {leaf.value!r}"
+ orig_prefix = match.group(1)
+ new_prefix = orig_prefix.replace("F", "f").replace("B", "b").replace("U", "u")
+ if remove_u_prefix:
+ new_prefix = new_prefix.replace("u", "")
+ leaf.value = f"{new_prefix}{match.group(2)}"
+
+
+def normalize_string_quotes(leaf: Leaf) -> None:
+ """Prefer double quotes but only if it doesn't cause more escaping.
+
+ Adds or removes backslashes as appropriate. Doesn't parse and fix
+ strings nested in f-strings (yet).
+
+ Note: Mutates its argument.
+ """
+ value = leaf.value.lstrip(STRING_PREFIX_CHARS)
+ if value[:3] == '"""':
+ return
+
+ elif value[:3] == "'''":
+ orig_quote = "'''"
+ new_quote = '"""'
+ elif value[0] == '"':
+ orig_quote = '"'
+ new_quote = "'"
+ else:
+ orig_quote = "'"
+ new_quote = '"'
+ first_quote_pos = leaf.value.find(orig_quote)
+ if first_quote_pos == -1:
+ return # There's an internal error
+
+ prefix = leaf.value[:first_quote_pos]
+ unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
+ escaped_new_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
+ escaped_orig_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
+ body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)]
+ if "r" in prefix.casefold():
+ if unescaped_new_quote.search(body):
+ # There's at least one unescaped new_quote in this raw string
+ # so converting is impossible
+ return
+
+ # Do not introduce or remove backslashes in raw strings
+ new_body = body
+ else:
+ # remove unnecessary escapes
+ new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
+ if body != new_body:
+ # Consider the string without unnecessary escapes as the original
+ body = new_body
+ leaf.value = f"{prefix}{orig_quote}{body}{orig_quote}"
+ new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
+ new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
+ if "f" in prefix.casefold():
+ matches = re.findall(
+ r"""
+ (?:[^{]|^)\{ # start of the string or a non-{ followed by a single {
+ ([^{].*?) # contents of the brackets except if begins with {{
+ \}(?:[^}]|$) # A } followed by end of the string or a non-}
+ """,
+ new_body,
+ re.VERBOSE,
+ )
+ for m in matches:
+ if "\\" in str(m):
+ # Do not introduce backslashes in interpolated expressions
+ return
+
+ if new_quote == '"""' and new_body[-1:] == '"':
+ # edge case:
+ new_body = new_body[:-1] + '\\"'
+ orig_escape_count = body.count("\\")
+ new_escape_count = new_body.count("\\")
+ if new_escape_count > orig_escape_count:
+ return # Do not introduce more escaping
+
+ if new_escape_count == orig_escape_count and orig_quote == '"':
+ return # Prefer double quotes
+
+ leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
+
+
+def normalize_numeric_literal(leaf: Leaf) -> None:
+ """Normalizes numeric (float, int, and complex) literals.
+
+ All letters used in the representation are normalized to lowercase (except
+ in Python 2 long literals).
+ """
+ text = leaf.value.lower()
+ if text.startswith(("0o", "0b")):
+ # Leave octal and binary literals alone.
+ pass
+ elif text.startswith("0x"):
+ # Change hex literals to upper case.
+ before, after = text[:2], text[2:]
+ text = f"{before}{after.upper()}"
+ elif "e" in text:
+ before, after = text.split("e")
+ sign = ""
+ if after.startswith("-"):
+ after = after[1:]
+ sign = "-"
+ elif after.startswith("+"):
+ after = after[1:]
+ before = format_float_or_int_string(before)
+ text = f"{before}e{sign}{after}"
+ elif text.endswith(("j", "l")):
+ number = text[:-1]
+ suffix = text[-1]
+ # Capitalize in "2L" because "l" looks too similar to "1".
+ if suffix == "l":
+ suffix = "L"
+ text = f"{format_float_or_int_string(number)}{suffix}"
+ else:
+ text = format_float_or_int_string(text)
+ leaf.value = text
+
+
+def format_float_or_int_string(text: str) -> str:
+ """Formats a float string like "1.0"."""
+ if "." not in text:
+ return text
+
+ before, after = text.split(".")
+ return f"{before or 0}.{after or 0}"
+
+
+def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
+ """Make existing optional parentheses invisible or create new ones.
+
+ `parens_after` is a set of string leaf values immediately after which parens
+ should be put.
+
+ Standardizes on visible parentheses for single-element tuples, and keeps
+ existing visible parentheses for other tuples and generator expressions.
+ """
+ for pc in list_comments(node.prefix, is_endmarker=False):
+ if pc.value in FMT_OFF:
+ # This `node` has a prefix with `# fmt: off`, don't mess with parens.
+ return
+ check_lpar = False
+ for index, child in enumerate(list(node.children)):
+ # Fixes a bug where invisible parens are not properly stripped from
+ # assignment statements that contain type annotations.
+ if isinstance(child, Node) and child.type == syms.annassign:
+ normalize_invisible_parens(child, parens_after=parens_after)
+
+ # Add parentheses around long tuple unpacking in assignments.
+ if (
+ index == 0
+ and isinstance(child, Node)
+ and child.type == syms.testlist_star_expr
+ ):
+ check_lpar = True
+
+ if check_lpar:
+ if is_walrus_assignment(child):
+ continue
+
+ if child.type == syms.atom:
+ if maybe_make_parens_invisible_in_atom(child, parent=node):
+ wrap_in_parentheses(node, child, visible=False)
+ elif is_one_tuple(child):
+ wrap_in_parentheses(node, child, visible=True)
+ elif node.type == syms.import_from:
+ # "import from" nodes store parentheses directly as part of
+ # the statement
+ if child.type == token.LPAR:
+ # make parentheses invisible
+ child.value = "" # type: ignore
+ node.children[-1].value = "" # type: ignore
+ elif child.type != token.STAR:
+ # insert invisible parentheses
+ node.insert_child(index, Leaf(token.LPAR, ""))
+ node.append_child(Leaf(token.RPAR, ""))
+ break
+
+ elif not (isinstance(child, Leaf) and is_multiline_string(child)):
+ wrap_in_parentheses(node, child, visible=False)
+
+ check_lpar = isinstance(child, Leaf) and child.value in parens_after
+
+
+def normalize_fmt_off(node: Node) -> None:
+ """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
+ try_again = True
+ while try_again:
+ try_again = convert_one_fmt_off_pair(node)
+
+
+def convert_one_fmt_off_pair(node: Node) -> bool:
+ """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
+
+ Returns True if a pair was converted.
+ """
+ for leaf in node.leaves():
+ previous_consumed = 0
+ for comment in list_comments(leaf.prefix, is_endmarker=False):
+ if comment.value in FMT_OFF:
+ # We only want standalone comments. If there's no previous leaf or
+ # the previous leaf is indentation, it's a standalone comment in
+ # disguise.
+ if comment.type != STANDALONE_COMMENT:
+ prev = preceding_leaf(leaf)
+ if prev and prev.type not in WHITESPACE:
+ continue
+
+ ignored_nodes = list(generate_ignored_nodes(leaf))
+ if not ignored_nodes:
+ continue
+
+ first = ignored_nodes[0] # Can be a container node with the `leaf`.
+ parent = first.parent
+ prefix = first.prefix
+ first.prefix = prefix[comment.consumed :]
+ hidden_value = (
+ comment.value + "\n" + "".join(str(n) for n in ignored_nodes)
+ )
+ if hidden_value.endswith("\n"):
+ # That happens when one of the `ignored_nodes` ended with a NEWLINE
+ # leaf (possibly followed by a DEDENT).
+ hidden_value = hidden_value[:-1]
+ first_idx: Optional[int] = None
+ for ignored in ignored_nodes:
+ index = ignored.remove()
+ if first_idx is None:
+ first_idx = index
+ assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
+ assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
+ parent.insert_child(
+ first_idx,
+ Leaf(
+ STANDALONE_COMMENT,
+ hidden_value,
+ prefix=prefix[:previous_consumed] + "\n" * comment.newlines,
+ ),
+ )
+ return True
+
+ previous_consumed = comment.consumed
+
+ return False
+
+
+def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
+ """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
+
+ Stops at the end of the block.
+ """
+ container: Optional[LN] = container_of(leaf)
+ while container is not None and container.type != token.ENDMARKER:
+ if fmt_on(container):
+ return
+
+ # fix for fmt: on in children
+ if contains_fmt_on_at_column(container, leaf.column):
+ for child in container.children:
+ if contains_fmt_on_at_column(child, leaf.column):
+ return
+ yield child
+ else:
+ yield container
+ container = container.next_sibling
+
+
+def fmt_on(container: LN) -> bool:
+ is_fmt_on = False
+ for comment in list_comments(container.prefix, is_endmarker=False):
+ if comment.value in FMT_ON:
+ is_fmt_on = True
+ elif comment.value in FMT_OFF:
+ is_fmt_on = False
+ return is_fmt_on
+
+
+def contains_fmt_on_at_column(container: LN, column: int) -> bool:
+ for child in container.children:
+ if (
+ isinstance(child, Node)
+ and first_leaf_column(child) == column
+ or isinstance(child, Leaf)
+ and child.column == column
+ ):
+ if fmt_on(child):
+ return True
+
+ return False
+
+
+def first_leaf_column(node: Node) -> Optional[int]:
+ for child in node.children:
+ if isinstance(child, Leaf):
+ return child.column
+ return None
+
+
+def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
+ """If it's safe, make the parens in the atom `node` invisible, recursively.
+ Additionally, remove repeated, adjacent invisible parens from the atom `node`
+ as they are redundant.
+
+ Returns whether the node should itself be wrapped in invisible parentheses.
+
+ """
+ if (
+ node.type != syms.atom
+ or is_empty_tuple(node)
+ or is_one_tuple(node)
+ or (is_yield(node) and parent.type != syms.expr_stmt)
+ or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
+ ):
+ return False
+
+ first = node.children[0]
+ last = node.children[-1]
+ if first.type == token.LPAR and last.type == token.RPAR:
+ middle = node.children[1]
+ # make parentheses invisible
+ first.value = "" # type: ignore
+ last.value = "" # type: ignore
+ maybe_make_parens_invisible_in_atom(middle, parent=parent)
+
+ if is_atom_with_invisible_parens(middle):
+ # Strip the invisible parens from `middle` by replacing
+ # it with the child in-between the invisible parens
+ middle.replace(middle.children[1])
+
+ return False
+
+ return True
+
+
+def is_atom_with_invisible_parens(node: LN) -> bool:
+ """Given a `LN`, determines whether it's an atom `node` with invisible
+ parens. Useful in dedupe-ing and normalizing parens.
+ """
+ if isinstance(node, Leaf) or node.type != syms.atom:
+ return False
+
+ first, last = node.children[0], node.children[-1]
+ return (
+ isinstance(first, Leaf)
+ and first.type == token.LPAR
+ and first.value == ""
+ and isinstance(last, Leaf)
+ and last.type == token.RPAR
+ and last.value == ""
+ )
+
+
+def is_empty_tuple(node: LN) -> bool:
+ """Return True if `node` holds an empty tuple."""
+ return (
+ node.type == syms.atom
+ and len(node.children) == 2
+ and node.children[0].type == token.LPAR
+ and node.children[1].type == token.RPAR
+ )
+
+
+def unwrap_singleton_parenthesis(node: LN) -> Optional[LN]:
+ """Returns `wrapped` if `node` is of the shape ( wrapped ).
+
+ Parenthesis can be optional. Returns None otherwise"""
+ if len(node.children) != 3:
+ return None
+
+ lpar, wrapped, rpar = node.children
+ if not (lpar.type == token.LPAR and rpar.type == token.RPAR):
+ return None
+
+ return wrapped
+
+
+def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None:
+ """Wrap `child` in parentheses.
+
+ This replaces `child` with an atom holding the parentheses and the old
+ child. That requires moving the prefix.
+
+ If `visible` is False, the leaves will be valueless (and thus invisible).
+ """
+ lpar = Leaf(token.LPAR, "(" if visible else "")
+ rpar = Leaf(token.RPAR, ")" if visible else "")
+ prefix = child.prefix
+ child.prefix = ""
+ index = child.remove() or 0
+ new_child = Node(syms.atom, [lpar, child, rpar])
+ new_child.prefix = prefix
+ parent.insert_child(index, new_child)
+
+
+def is_one_tuple(node: LN) -> bool:
+ """Return True if `node` holds a tuple with one element, with or without parens."""
+ if node.type == syms.atom:
+ gexp = unwrap_singleton_parenthesis(node)
+ if gexp is None or gexp.type != syms.testlist_gexp:
+ return False
+
+ return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA
+
+ return (
+ node.type in IMPLICIT_TUPLE
+ and len(node.children) == 2
+ and node.children[1].type == token.COMMA
+ )
+
+
+def is_walrus_assignment(node: LN) -> bool:
+ """Return True iff `node` is of the shape ( test := test )"""
+ inner = unwrap_singleton_parenthesis(node)
+ return inner is not None and inner.type == syms.namedexpr_test
+
+
+def is_yield(node: LN) -> bool:
+ """Return True if `node` holds a `yield` or `yield from` expression."""
+ if node.type == syms.yield_expr:
+ return True
+
+ if node.type == token.NAME and node.value == "yield": # type: ignore
+ return True
+
+ if node.type != syms.atom:
+ return False
+
+ if len(node.children) != 3:
+ return False
+
+ lpar, expr, rpar = node.children
+ if lpar.type == token.LPAR and rpar.type == token.RPAR:
+ return is_yield(expr)
+
+ return False
+
+
+def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
+ """Return True if `leaf` is a star or double star in a vararg or kwarg.
+
+ If `within` includes VARARGS_PARENTS, this applies to function signatures.
+ If `within` includes UNPACKING_PARENTS, it applies to right hand-side
+ extended iterable unpacking (PEP 3132) and additional unpacking
+ generalizations (PEP 448).
+ """
+ if leaf.type not in VARARGS_SPECIALS or not leaf.parent:
+ return False
+
+ p = leaf.parent
+ if p.type == syms.star_expr:
+ # Star expressions are also used as assignment targets in extended
+ # iterable unpacking (PEP 3132). See what its parent is instead.
+ if not p.parent:
+ return False
+
+ p = p.parent
+
+ return p.type in within
+
+
+def is_multiline_string(leaf: Leaf) -> bool:
+ """Return True if `leaf` is a multiline string that actually spans many lines."""
+ return has_triple_quotes(leaf.value) and "\n" in leaf.value
+
+
+def is_stub_suite(node: Node) -> bool:
+ """Return True if `node` is a suite with a stub body."""
+ if (
+ len(node.children) != 4
+ or node.children[0].type != token.NEWLINE
+ or node.children[1].type != token.INDENT
+ or node.children[3].type != token.DEDENT
+ ):
+ return False
+
+ return is_stub_body(node.children[2])
+
+
+def is_stub_body(node: LN) -> bool:
+ """Return True if `node` is a simple statement containing an ellipsis."""
+ if not isinstance(node, Node) or node.type != syms.simple_stmt:
+ return False
+
+ if len(node.children) != 2:
+ return False
+
+ child = node.children[0]
+ return (
+ child.type == syms.atom
+ and len(child.children) == 3
+ and all(leaf == Leaf(token.DOT, ".") for leaf in child.children)
+ )
+
+
+def max_delimiter_priority_in_atom(node: LN) -> Priority:
+ """Return maximum delimiter priority inside `node`.
+
+ This is specific to atoms with contents contained in a pair of parentheses.
+ If `node` isn't an atom or there are no enclosing parentheses, returns 0.
+ """
+ if node.type != syms.atom:
+ return 0
+
+ first = node.children[0]
+ last = node.children[-1]
+ if not (first.type == token.LPAR and last.type == token.RPAR):
+ return 0
+
+ bt = BracketTracker()
+ for c in node.children[1:-1]:
+ if isinstance(c, Leaf):
+ bt.mark(c)
+ else:
+ for leaf in c.leaves():
+ bt.mark(leaf)
+ try:
+ return bt.max_delimiter_priority()
+
+ except ValueError:
+ return 0
+
+
+def ensure_visible(leaf: Leaf) -> None:
+ """Make sure parentheses are visible.
+
+ They could be invisible as part of some statements (see
+ :func:`normalize_invisible_parens` and :func:`visit_import_from`).
+ """
+ if leaf.type == token.LPAR:
+ leaf.value = "("
+ elif leaf.type == token.RPAR:
+ leaf.value = ")"
+
+
+def should_explode(line: Line, opening_bracket: Leaf) -> bool:
+ """Should `line` immediately be split with `delimiter_split()` after RHS?"""
+
+ if not (
+ opening_bracket.parent
+ and opening_bracket.parent.type in {syms.atom, syms.import_from}
+ and opening_bracket.value in "[{("
+ ):
+ return False
+
+ try:
+ last_leaf = line.leaves[-1]
+ exclude = {id(last_leaf)} if last_leaf.type == token.COMMA else set()
+ max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
+ except (IndexError, ValueError):
+ return False
+
+ return max_priority == COMMA_PRIORITY
+
+
+def get_features_used(node: Node) -> Set[Feature]:
+ """Return a set of (relatively) new Python features used in this file.
+
+ Currently looking for:
+ - f-strings;
+ - underscores in numeric literals;
+ - trailing commas after * or ** in function signatures and calls;
+ - positional only arguments in function signatures and lambdas;
+ """
+ features: Set[Feature] = set()
+ for n in node.pre_order():
+ if n.type == token.STRING:
+ value_head = n.value[:2] # type: ignore
+ if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
+ features.add(Feature.F_STRINGS)
+
+ elif n.type == token.NUMBER:
+ if "_" in n.value: # type: ignore
+ features.add(Feature.NUMERIC_UNDERSCORES)
+
+ elif n.type == token.SLASH:
+ if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}:
+ features.add(Feature.POS_ONLY_ARGUMENTS)
+
+ elif n.type == token.COLONEQUAL:
+ features.add(Feature.ASSIGNMENT_EXPRESSIONS)
+
+ elif (
+ n.type in {syms.typedargslist, syms.arglist}
+ and n.children
+ and n.children[-1].type == token.COMMA
+ ):
+ if n.type == syms.typedargslist:
+ feature = Feature.TRAILING_COMMA_IN_DEF
+ else:
+ feature = Feature.TRAILING_COMMA_IN_CALL
+
+ for ch in n.children:
+ if ch.type in STARS:
+ features.add(feature)
+
+ if ch.type == syms.argument:
+ for argch in ch.children:
+ if argch.type in STARS:
+ features.add(feature)
+
+ return features
+
+
+def detect_target_versions(node: Node) -> Set[TargetVersion]:
+ """Detect the version to target based on the nodes used."""
+ features = get_features_used(node)
+ return {
+ version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]
+ }
projects / etc / vim.git / blobdiff