+ def append_to_line(leaf: Leaf) -> Iterator[Line]:
+ """Append `leaf` to current line or to new line if appending impossible."""
+ nonlocal current_line
+ try:
+ current_line.append_safe(leaf, preformatted=True)
+ except ValueError:
+ yield current_line
+
+ current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+ current_line.append(leaf)
+
+ for leaf in line.leaves:
+ yield from append_to_line(leaf)
+
+ for comment_after in line.comments_after(leaf):
+ yield from append_to_line(comment_after)
+
+ if current_line:
+ yield current_line
+
+
+def is_import(leaf: Leaf) -> bool:
+ """Return True if the given leaf starts an import statement."""
+ p = leaf.parent
+ t = leaf.type
+ v = leaf.value
+ return bool(
+ t == token.NAME
+ and (
+ (v == "import" and p and p.type == syms.import_name)
+ or (v == "from" and p and p.type == syms.import_from)
+ )
+ )
+
+
+def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
+ """Leave existing extra newlines if not `inside_brackets`. Remove everything
+ else.
+
+ Note: don't use backslashes for formatting or you'll lose your voting rights.
+ """
+ if not inside_brackets:
+ spl = leaf.prefix.split("#")
+ if "\\" not in spl[0]:
+ nl_count = spl[-1].count("\n")
+ if len(spl) > 1:
+ nl_count -= 1
+ leaf.prefix = "\n" * nl_count
+ return
+
+ leaf.prefix = ""
+
+
+def normalize_string_prefix(leaf: Leaf, remove_u_prefix: bool = False) -> None:
+ """Make all string prefixes lowercase.
+
+ If remove_u_prefix is given, also removes any u prefix from the string.
+
+ Note: Mutates its argument.
+ """
+ match = re.match(r"^([furbFURB]*)(.*)$", leaf.value, re.DOTALL)
+ assert match is not None, f"failed to match string {leaf.value!r}"
+ orig_prefix = match.group(1)
+ new_prefix = orig_prefix.lower()
+ if remove_u_prefix:
+ new_prefix = new_prefix.replace("u", "")
+ leaf.value = f"{new_prefix}{match.group(2)}"
+
+
+def normalize_string_quotes(leaf: Leaf) -> None:
+ """Prefer double quotes but only if it doesn't cause more escaping.
+
+ Adds or removes backslashes as appropriate. Doesn't parse and fix
+ strings nested in f-strings (yet).
+
+ Note: Mutates its argument.
+ """
+ value = leaf.value.lstrip("furbFURB")
+ if value[:3] == '"""':
+ return
+
+ elif value[:3] == "'''":
+ orig_quote = "'''"
+ new_quote = '"""'
+ elif value[0] == '"':
+ orig_quote = '"'
+ new_quote = "'"
+ else:
+ orig_quote = "'"
+ new_quote = '"'
+ first_quote_pos = leaf.value.find(orig_quote)
+ if first_quote_pos == -1:
+ return # There's an internal error
+
+ prefix = leaf.value[:first_quote_pos]
+ unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
+ escaped_new_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
+ escaped_orig_quote = re.compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
+ body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)]
+ if "r" in prefix.casefold():
+ if unescaped_new_quote.search(body):
+ # There's at least one unescaped new_quote in this raw string
+ # so converting is impossible
+ return
+
+ # Do not introduce or remove backslashes in raw strings
+ new_body = body
+ else:
+ # remove unnecessary escapes
+ new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
+ if body != new_body:
+ # Consider the string without unnecessary escapes as the original
+ body = new_body
+ leaf.value = f"{prefix}{orig_quote}{body}{orig_quote}"
+ new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
+ new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
+ if "f" in prefix.casefold():
+ matches = re.findall(r"[^{]\{(.*?)\}[^}]", new_body)
+ for m in matches:
+ if "\\" in str(m):
+ # Do not introduce backslashes in interpolated expressions
+ return
+ if new_quote == '"""' and new_body[-1:] == '"':
+ # edge case:
+ new_body = new_body[:-1] + '\\"'
+ orig_escape_count = body.count("\\")
+ new_escape_count = new_body.count("\\")
+ if new_escape_count > orig_escape_count:
+ return # Do not introduce more escaping
+
+ if new_escape_count == orig_escape_count and orig_quote == '"':
+ return # Prefer double quotes
+
+ leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
+
+
+def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
+ """Normalizes numeric (float, int, and complex) literals.
+
+ All letters used in the representation are normalized to lowercase (except
+ in Python 2 long literals), and long number literals are split using underscores.
+ """
+ text = leaf.value.lower()
+ if text.startswith(("0o", "0b")):
+ # Leave octal and binary literals alone.
+ pass
+ elif text.startswith("0x"):
+ # Change hex literals to upper case.
+ before, after = text[:2], text[2:]
+ text = f"{before}{after.upper()}"
+ elif "e" in text:
+ before, after = text.split("e")
+ sign = ""
+ if after.startswith("-"):
+ after = after[1:]
+ sign = "-"
+ elif after.startswith("+"):
+ after = after[1:]
+ before = format_float_or_int_string(before, allow_underscores)
+ after = format_int_string(after, allow_underscores)
+ text = f"{before}e{sign}{after}"
+ elif text.endswith(("j", "l")):
+ number = text[:-1]
+ suffix = text[-1]
+ # Capitalize in "2L" because "l" looks too similar to "1".
+ if suffix == "l":
+ suffix = "L"
+ text = f"{format_float_or_int_string(number, allow_underscores)}{suffix}"
+ else:
+ text = format_float_or_int_string(text, allow_underscores)
+ leaf.value = text
+
+
+def format_float_or_int_string(text: str, allow_underscores: bool) -> str:
+ """Formats a float string like "1.0"."""
+ if "." not in text:
+ return format_int_string(text, allow_underscores)
+
+ before, after = text.split(".")
+ before = format_int_string(before, allow_underscores) if before else "0"
+ if after:
+ after = format_int_string(after, allow_underscores, count_from_end=False)
+ else:
+ after = "0"
+ return f"{before}.{after}"
+
+
+def format_int_string(
+ text: str, allow_underscores: bool, count_from_end: bool = True
+) -> str:
+ """Normalizes underscores in a string to e.g. 1_000_000.
+
+ Input must be a string of digits and optional underscores.
+ If count_from_end is False, we add underscores after groups of three digits
+ counting from the beginning instead of the end of the strings. This is used
+ for the fractional part of float literals.
+ """
+ if not allow_underscores:
+ return text
+
+ text = text.replace("_", "")
+ if len(text) <= 5:
+ # No underscores for numbers <= 5 digits long.
+ return text
+
+ if count_from_end:
+ # Avoid removing leading zeros, which are important if we're formatting
+ # part of a number like "0.001".
+ return format(int("1" + text), "3_")[1:].lstrip("_")
+ else:
+ return "_".join(text[i : i + 3] for i in range(0, len(text), 3))
+
+
+def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
+ """Make existing optional parentheses invisible or create new ones.
+
+ `parens_after` is a set of string leaf values immeditely after which parens
+ should be put.
+
+ Standardizes on visible parentheses for single-element tuples, and keeps
+ existing visible parentheses for other tuples and generator expressions.
+ """
+ for pc in list_comments(node.prefix, is_endmarker=False):
+ if pc.value in FMT_OFF:
+ # This `node` has a prefix with `# fmt: off`, don't mess with parens.
+ return
+
+ check_lpar = False
+ for index, child in enumerate(list(node.children)):
+ if check_lpar:
+ if child.type == syms.atom:
+ if maybe_make_parens_invisible_in_atom(child):
+ lpar = Leaf(token.LPAR, "")
+ rpar = Leaf(token.RPAR, "")
+ index = child.remove() or 0
+ node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+ elif is_one_tuple(child):
+ # wrap child in visible parentheses
+ lpar = Leaf(token.LPAR, "(")
+ rpar = Leaf(token.RPAR, ")")
+ child.remove()
+ node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+ elif node.type == syms.import_from:
+ # "import from" nodes store parentheses directly as part of
+ # the statement
+ if child.type == token.LPAR:
+ # make parentheses invisible
+ child.value = "" # type: ignore
+ node.children[-1].value = "" # type: ignore
+ elif child.type != token.STAR:
+ # insert invisible parentheses
+ node.insert_child(index, Leaf(token.LPAR, ""))
+ node.append_child(Leaf(token.RPAR, ""))
+ break
+
+ elif not (isinstance(child, Leaf) and is_multiline_string(child)):
+ # wrap child in invisible parentheses
+ lpar = Leaf(token.LPAR, "")
+ rpar = Leaf(token.RPAR, "")
+ index = child.remove() or 0
+ node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+
+ check_lpar = isinstance(child, Leaf) and child.value in parens_after
+
+
+def normalize_fmt_off(node: Node) -> None:
+ """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
+ try_again = True
+ while try_again:
+ try_again = convert_one_fmt_off_pair(node)
+
+
+def convert_one_fmt_off_pair(node: Node) -> bool:
+ """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
+
+ Returns True if a pair was converted.
+ """
+ for leaf in node.leaves():
+ previous_consumed = 0
+ for comment in list_comments(leaf.prefix, is_endmarker=False):
+ if comment.value in FMT_OFF:
+ # We only want standalone comments. If there's no previous leaf or
+ # the previous leaf is indentation, it's a standalone comment in
+ # disguise.
+ if comment.type != STANDALONE_COMMENT:
+ prev = preceding_leaf(leaf)
+ if prev and prev.type not in WHITESPACE:
+ continue
+
+ ignored_nodes = list(generate_ignored_nodes(leaf))
+ if not ignored_nodes:
+ continue
+
+ first = ignored_nodes[0] # Can be a container node with the `leaf`.
+ parent = first.parent
+ prefix = first.prefix
+ first.prefix = prefix[comment.consumed :]
+ hidden_value = (
+ comment.value + "\n" + "".join(str(n) for n in ignored_nodes)
+ )
+ if hidden_value.endswith("\n"):
+ # That happens when one of the `ignored_nodes` ended with a NEWLINE
+ # leaf (possibly followed by a DEDENT).
+ hidden_value = hidden_value[:-1]
+ first_idx = None
+ for ignored in ignored_nodes:
+ index = ignored.remove()
+ if first_idx is None:
+ first_idx = index
+ assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
+ assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
+ parent.insert_child(
+ first_idx,
+ Leaf(
+ STANDALONE_COMMENT,
+ hidden_value,
+ prefix=prefix[:previous_consumed] + "\n" * comment.newlines,
+ ),
+ )
+ return True
+
+ previous_consumed = comment.consumed
+
+ return False
+
+
+def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
+ """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
projects / etc / vim.git / blobdiff