+ All letters used in the representation are normalized to lowercase (except
+ in Python 2 long literals), and long number literals are split using underscores.
+ """
+ text = leaf.value.lower()
+ if text.startswith(("0o", "0x", "0b")):
+ # Leave octal, hex, and binary literals alone.
+ pass
+ elif "e" in text:
+ before, after = text.split("e")
+ sign = ""
+ if after.startswith("-"):
+ after = after[1:]
+ sign = "-"
+ elif after.startswith("+"):
+ after = after[1:]
+ before = format_float_or_int_string(before, allow_underscores)
+ after = format_int_string(after, allow_underscores)
+ text = f"{before}e{sign}{after}"
+ elif text.endswith(("j", "l")):
+ number = text[:-1]
+ suffix = text[-1]
+ # Capitalize in "2L" because "l" looks too similar to "1".
+ if suffix == "l":
+ suffix = "L"
+ text = f"{format_float_or_int_string(number, allow_underscores)}{suffix}"
+ else:
+ text = format_float_or_int_string(text, allow_underscores)
+ leaf.value = text
+
+
+def format_float_or_int_string(text: str, allow_underscores: bool) -> str:
+ """Formats a float string like "1.0"."""
+ if "." not in text:
+ return format_int_string(text, allow_underscores)
+
+ before, after = text.split(".")
+ before = format_int_string(before, allow_underscores) if before else "0"
+ if after:
+ after = format_int_string(after, allow_underscores, count_from_end=False)
+ else:
+ after = "0"
+ return f"{before}.{after}"
+
+
+def format_int_string(
+ text: str, allow_underscores: bool, count_from_end: bool = True
+) -> str:
+ """Normalizes underscores in a string to e.g. 1_000_000.
+
+ Input must be a string of digits and optional underscores.
+ If count_from_end is False, we add underscores after groups of three digits
+ counting from the beginning instead of the end of the strings. This is used
+ for the fractional part of float literals.
+ """
+ if not allow_underscores:
+ return text
+
+ text = text.replace("_", "")
+ if len(text) <= 6:
+ # No underscores for numbers <= 6 digits long.
+ return text
+
+ if count_from_end:
+ # Avoid removing leading zeros, which are important if we're formatting
+ # part of a number like "0.001".
+ return format(int("1" + text), "3_")[1:].lstrip("_")
+ else:
+ return "_".join(text[i : i + 3] for i in range(0, len(text), 3))
+
+
+def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
+ """Make existing optional parentheses invisible or create new ones.
+
+ `parens_after` is a set of string leaf values immeditely after which parens
+ should be put.
+
+ Standardizes on visible parentheses for single-element tuples, and keeps
+ existing visible parentheses for other tuples and generator expressions.
+ """
+ for pc in list_comments(node.prefix, is_endmarker=False):
+ if pc.value in FMT_OFF:
+ # This `node` has a prefix with `# fmt: off`, don't mess with parens.
+ return
+
+ check_lpar = False
+ for index, child in enumerate(list(node.children)):
+ if check_lpar:
+ if child.type == syms.atom:
+ if maybe_make_parens_invisible_in_atom(child):
+ lpar = Leaf(token.LPAR, "")
+ rpar = Leaf(token.RPAR, "")
+ index = child.remove() or 0
+ node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+ elif is_one_tuple(child):
+ # wrap child in visible parentheses
+ lpar = Leaf(token.LPAR, "(")
+ rpar = Leaf(token.RPAR, ")")
+ child.remove()
+ node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+ elif node.type == syms.import_from:
+ # "import from" nodes store parentheses directly as part of
+ # the statement
+ if child.type == token.LPAR:
+ # make parentheses invisible
+ child.value = "" # type: ignore
+ node.children[-1].value = "" # type: ignore
+ elif child.type != token.STAR:
+ # insert invisible parentheses
+ node.insert_child(index, Leaf(token.LPAR, ""))
+ node.append_child(Leaf(token.RPAR, ""))
+ break
+
+ elif not (isinstance(child, Leaf) and is_multiline_string(child)):
+ # wrap child in invisible parentheses
+ lpar = Leaf(token.LPAR, "")
+ rpar = Leaf(token.RPAR, "")
+ index = child.remove() or 0
+ node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+
+ check_lpar = isinstance(child, Leaf) and child.value in parens_after
+
+
+def normalize_fmt_off(node: Node) -> None:
+ """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
+ try_again = True
+ while try_again:
+ try_again = convert_one_fmt_off_pair(node)
+
+
+def convert_one_fmt_off_pair(node: Node) -> bool:
+ """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
+
+ Returns True if a pair was converted.
+ """
+ for leaf in node.leaves():
+ previous_consumed = 0
+ for comment in list_comments(leaf.prefix, is_endmarker=False):
+ if comment.value in FMT_OFF:
+ # We only want standalone comments. If there's no previous leaf or
+ # the previous leaf is indentation, it's a standalone comment in
+ # disguise.
+ if comment.type != STANDALONE_COMMENT:
+ prev = preceding_leaf(leaf)
+ if prev and prev.type not in WHITESPACE:
+ continue
+
+ ignored_nodes = list(generate_ignored_nodes(leaf))
+ if not ignored_nodes:
+ continue
+
+ first = ignored_nodes[0] # Can be a container node with the `leaf`.
+ parent = first.parent
+ prefix = first.prefix
+ first.prefix = prefix[comment.consumed :]
+ hidden_value = (
+ comment.value + "\n" + "".join(str(n) for n in ignored_nodes)
+ )
+ if hidden_value.endswith("\n"):
+ # That happens when one of the `ignored_nodes` ended with a NEWLINE
+ # leaf (possibly followed by a DEDENT).
+ hidden_value = hidden_value[:-1]
+ first_idx = None
+ for ignored in ignored_nodes:
+ index = ignored.remove()
+ if first_idx is None:
+ first_idx = index
+ assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
+ assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
+ parent.insert_child(
+ first_idx,
+ Leaf(
+ STANDALONE_COMMENT,
+ hidden_value,
+ prefix=prefix[:previous_consumed] + "\n" * comment.newlines,
+ ),
+ )
+ return True
+
+ previous_consumed = comment.consumed
+
+ return False
+
+
+def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
+ """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
+
+ Stops at the end of the block.
+ """
+ container: Optional[LN] = container_of(leaf)
+ while container is not None and container.type != token.ENDMARKER:
+ for comment in list_comments(container.prefix, is_endmarker=False):
+ if comment.value in FMT_ON:
+ return
+
+ yield container
+
+ container = container.next_sibling
+
+
+def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
+ """If it's safe, make the parens in the atom `node` invisible, recursively.
+
+ Returns whether the node should itself be wrapped in invisible parentheses.
+
+ """
+ if (
+ node.type != syms.atom
+ or is_empty_tuple(node)
+ or is_one_tuple(node)
+ or is_yield(node)
+ or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
+ ):
+ return False
+
+ first = node.children[0]
+ last = node.children[-1]
+ if first.type == token.LPAR and last.type == token.RPAR:
+ # make parentheses invisible
+ first.value = "" # type: ignore
+ last.value = "" # type: ignore
+ if len(node.children) > 1:
+ maybe_make_parens_invisible_in_atom(node.children[1])
+ return False
+
+ return True
+
+
+def is_empty_tuple(node: LN) -> bool:
+ """Return True if `node` holds an empty tuple."""
+ return (
+ node.type == syms.atom
+ and len(node.children) == 2
+ and node.children[0].type == token.LPAR
+ and node.children[1].type == token.RPAR
+ )
+
+
+def is_one_tuple(node: LN) -> bool:
+ """Return True if `node` holds a tuple with one element, with or without parens."""
+ if node.type == syms.atom:
+ if len(node.children) != 3:
+ return False
+
+ lpar, gexp, rpar = node.children
+ if not (
+ lpar.type == token.LPAR
+ and gexp.type == syms.testlist_gexp
+ and rpar.type == token.RPAR
+ ):
+ return False
+
+ return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA
+
+ return (
+ node.type in IMPLICIT_TUPLE
+ and len(node.children) == 2
+ and node.children[1].type == token.COMMA
+ )
+
+
+def is_yield(node: LN) -> bool:
+ """Return True if `node` holds a `yield` or `yield from` expression."""
+ if node.type == syms.yield_expr:
+ return True
+
+ if node.type == token.NAME and node.value == "yield": # type: ignore
+ return True
+
+ if node.type != syms.atom:
+ return False
+
+ if len(node.children) != 3:
+ return False
+
+ lpar, expr, rpar = node.children
+ if lpar.type == token.LPAR and rpar.type == token.RPAR:
+ return is_yield(expr)
+
+ return False
+
+
+def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
+ """Return True if `leaf` is a star or double star in a vararg or kwarg.
+
+ If `within` includes VARARGS_PARENTS, this applies to function signatures.
+ If `within` includes UNPACKING_PARENTS, it applies to right hand-side
+ extended iterable unpacking (PEP 3132) and additional unpacking
+ generalizations (PEP 448).
+ """
+ if leaf.type not in STARS or not leaf.parent:
+ return False
+
+ p = leaf.parent
+ if p.type == syms.star_expr:
+ # Star expressions are also used as assignment targets in extended
+ # iterable unpacking (PEP 3132). See what its parent is instead.
+ if not p.parent:
+ return False
+
+ p = p.parent
+
+ return p.type in within
+
+
+def is_multiline_string(leaf: Leaf) -> bool:
+ """Return True if `leaf` is a multiline string that actually spans many lines."""
+ value = leaf.value.lstrip("furbFURB")
+ return value[:3] in {'"""', "'''"} and "\n" in value
+
+
+def is_stub_suite(node: Node) -> bool:
+ """Return True if `node` is a suite with a stub body."""
+ if (
+ len(node.children) != 4
+ or node.children[0].type != token.NEWLINE
+ or node.children[1].type != token.INDENT
+ or node.children[3].type != token.DEDENT
+ ):
+ return False
+
+ return is_stub_body(node.children[2])
+
+
+def is_stub_body(node: LN) -> bool:
+ """Return True if `node` is a simple statement containing an ellipsis."""
+ if not isinstance(node, Node) or node.type != syms.simple_stmt:
+ return False
+
+ if len(node.children) != 2:
+ return False
+
+ child = node.children[0]
+ return (
+ child.type == syms.atom
+ and len(child.children) == 3
+ and all(leaf == Leaf(token.DOT, ".") for leaf in child.children)
+ )
+
+
+def max_delimiter_priority_in_atom(node: LN) -> int:
+ """Return maximum delimiter priority inside `node`.
+
+ This is specific to atoms with contents contained in a pair of parentheses.
+ If `node` isn't an atom or there are no enclosing parentheses, returns 0.
+ """
+ if node.type != syms.atom:
+ return 0
+
+ first = node.children[0]
+ last = node.children[-1]
+ if not (first.type == token.LPAR and last.type == token.RPAR):
+ return 0
+
+ bt = BracketTracker()
+ for c in node.children[1:-1]:
+ if isinstance(c, Leaf):
+ bt.mark(c)
+ else:
+ for leaf in c.leaves():
+ bt.mark(leaf)
+ try:
+ return bt.max_delimiter_priority()
+
+ except ValueError:
+ return 0
+
+
+def ensure_visible(leaf: Leaf) -> None:
+ """Make sure parentheses are visible.
+
+ They could be invisible as part of some statements (see
+ :func:`normalize_invible_parens` and :func:`visit_import_from`).
+ """
+ if leaf.type == token.LPAR:
+ leaf.value = "("
+ elif leaf.type == token.RPAR:
+ leaf.value = ")"
+
+
+def should_explode(line: Line, opening_bracket: Leaf) -> bool:
+ """Should `line` immediately be split with `delimiter_split()` after RHS?"""
+ if not (
+ opening_bracket.parent
+ and opening_bracket.parent.type in {syms.atom, syms.import_from}
+ and opening_bracket.value in "[{("
+ ):
+ return False
+
+ try:
+ last_leaf = line.leaves[-1]
+ exclude = {id(last_leaf)} if last_leaf.type == token.COMMA else set()
+ max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
+ except (IndexError, ValueError):
+ return False
+
+ return max_priority == COMMA_PRIORITY
+
+
+def is_python36(node: Node) -> bool:
+ """Return True if the current file is using Python 3.6+ features.
+
+ Currently looking for:
+ - f-strings;
+ - underscores in numeric literals; and
+ - trailing commas after * or ** in function signatures and calls.
+ """
+ for n in node.pre_order():
+ if n.type == token.STRING:
+ value_head = n.value[:2] # type: ignore
+ if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
+ return True
+
+ elif n.type == token.NUMBER:
+ if "_" in n.value: # type: ignore
+ return True
+
+ elif (
+ n.type in {syms.typedargslist, syms.arglist}
+ and n.children
+ and n.children[-1].type == token.COMMA
+ ):
+ for ch in n.children:
+ if ch.type in STARS:
+ return True
+
+ if ch.type == syms.argument:
+ for argch in ch.children:
+ if argch.type in STARS:
+ return True
+
+ return False
+
+
+def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
+ """Generate sets of closing bracket IDs that should be omitted in a RHS.
+
+ Brackets can be omitted if the entire trailer up to and including
+ a preceding closing bracket fits in one line.
+
+ Yielded sets are cumulative (contain results of previous yields, too). First
+ set is empty.
+ """
+
+ omit: Set[LeafID] = set()
+ yield omit
+
+ length = 4 * line.depth
+ opening_bracket = None
+ closing_bracket = None
+ inner_brackets: Set[LeafID] = set()
+ for index, leaf, leaf_length in enumerate_with_length(line, reversed=True):
+ length += leaf_length
+ if length > line_length:
+ break
+
+ has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
+ if leaf.type == STANDALONE_COMMENT or has_inline_comment:
+ break
+
+ if opening_bracket:
+ if leaf is opening_bracket:
+ opening_bracket = None
+ elif leaf.type in CLOSING_BRACKETS:
+ inner_brackets.add(id(leaf))
+ elif leaf.type in CLOSING_BRACKETS:
+ if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS:
+ # Empty brackets would fail a split so treat them as "inner"
+ # brackets (e.g. only add them to the `omit` set if another
+ # pair of brackets was good enough.
+ inner_brackets.add(id(leaf))
+ continue
+
+ if closing_bracket:
+ omit.add(id(closing_bracket))
+ omit.update(inner_brackets)
+ inner_brackets.clear()
+ yield omit
+
+ if leaf.value:
+ opening_bracket = leaf.opening_bracket
+ closing_bracket = leaf
+
+
+def get_future_imports(node: Node) -> Set[str]:
+ """Return a set of __future__ imports in the file."""
+ imports: Set[str] = set()
+
+ def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
+ for child in children:
+ if isinstance(child, Leaf):
+ if child.type == token.NAME:
+ yield child.value
+ elif child.type == syms.import_as_name:
+ orig_name = child.children[0]
+ assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
+ assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
+ yield orig_name.value
+ elif child.type == syms.import_as_names:
+ yield from get_imports_from_children(child.children)
+ else:
+ assert False, "Invalid syntax parsing imports"
+
+ for child in node.children:
+ if child.type != syms.simple_stmt:
+ break
+ first_child = child.children[0]
+ if isinstance(first_child, Leaf):
+ # Continue looking if we see a docstring; otherwise stop.
+ if (
+ len(child.children) == 2
+ and first_child.type == token.STRING
+ and child.children[1].type == token.NEWLINE
+ ):
+ continue
+ else:
+ break
+ elif first_child.type == syms.import_from:
+ module_name = first_child.children[1]
+ if not isinstance(module_name, Leaf) or module_name.value != "__future__":
+ break
+ imports |= set(get_imports_from_children(first_child.children[3:]))
+ else:
+ break
+ return imports
+
+
+def gen_python_files_in_dir(
+ path: Path,
+ root: Path,
+ include: Pattern[str],
+ exclude: Pattern[str],
+ report: "Report",
+) -> Iterator[Path]:
+ """Generate all files under `path` whose paths are not excluded by the
+ `exclude` regex, but are included by the `include` regex.
+
+ Symbolic links pointing outside of the `root` directory are ignored.
+
+ `report` is where output about exclusions goes.
+ """
+ assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
+ for child in path.iterdir():
+ try:
+ normalized_path = "/" + child.resolve().relative_to(root).as_posix()
+ except ValueError:
+ if child.is_symlink():
+ report.path_ignored(
+ child, f"is a symbolic link that points outside {root}"
+ )
+ continue
+
+ raise
+
+ if child.is_dir():
+ normalized_path += "/"
+ exclude_match = exclude.search(normalized_path)
+ if exclude_match and exclude_match.group(0):
+ report.path_ignored(child, f"matches the --exclude regular expression")
+ continue
+
+ if child.is_dir():
+ yield from gen_python_files_in_dir(child, root, include, exclude, report)
+
+ elif child.is_file():
+ include_match = include.search(normalized_path)
+ if include_match:
+ yield child
+
+
+@lru_cache()
+def find_project_root(srcs: Iterable[str]) -> Path:
+ """Return a directory containing .git, .hg, or pyproject.toml.
+
+ That directory can be one of the directories passed in `srcs` or their
+ common parent.
+
+ If no directory in the tree contains a marker that would specify it's the
+ project root, the root of the file system is returned.
+ """
+ if not srcs:
+ return Path("/").resolve()
+
+ common_base = min(Path(src).resolve() for src in srcs)
+ if common_base.is_dir():
+ # Append a fake file so `parents` below returns `common_base_dir`, too.
+ common_base /= "fake-file"
+ for directory in common_base.parents:
+ if (directory / ".git").is_dir():
+ return directory
+
+ if (directory / ".hg").is_dir():
+ return directory
+
+ if (directory / "pyproject.toml").is_file():
+ return directory
+
+ return directory
+
+
+@dataclass
+class Report:
+ """Provides a reformatting counter. Can be rendered with `str(report)`."""
+
+ check: bool = False
+ quiet: bool = False
+ verbose: bool = False
+ change_count: int = 0
+ same_count: int = 0
+ failure_count: int = 0
+
+ def done(self, src: Path, changed: Changed) -> None:
+ """Increment the counter for successful reformatting. Write out a message."""
+ if changed is Changed.YES:
+ reformatted = "would reformat" if self.check else "reformatted"
+ if self.verbose or not self.quiet:
+ out(f"{reformatted} {src}")
+ self.change_count += 1
+ else:
+ if self.verbose:
+ if changed is Changed.NO:
+ msg = f"{src} already well formatted, good job."
+ else:
+ msg = f"{src} wasn't modified on disk since last run."
+ out(msg, bold=False)
+ self.same_count += 1
projects / etc / vim.git / blobdiff