X-Git-Url: https://git.madduck.net/etc/vim.git/blobdiff_plain/2f3fa1f6d0cbc2a3f31c7440c422da173b068e7b:/src/black/comments.py..dc71922c768e543c9c3bbb1db5ea6d7fa801a814:/.vim/bundle/black/src/black/static/gitweb.js diff --git a/src/black/comments.py b/src/black/comments.py deleted file mode 100644 index c7513c2..0000000 --- a/src/black/comments.py +++ /dev/null @@ -1,272 +0,0 @@ -from dataclasses import dataclass -from functools import lru_cache -import regex as re -from typing import Iterator, List, Optional, Union - -from blib2to3.pytree import Node, Leaf -from blib2to3.pgen2 import token - -from black.nodes import first_leaf_column, preceding_leaf, container_of -from black.nodes import STANDALONE_COMMENT, WHITESPACE - -# types -LN = Union[Leaf, Node] - - -FMT_OFF = {"# fmt: off", "# fmt:off", "# yapf: disable"} -FMT_SKIP = {"# fmt: skip", "# fmt:skip"} -FMT_PASS = {*FMT_OFF, *FMT_SKIP} -FMT_ON = {"# fmt: on", "# fmt:on", "# yapf: enable"} - - -@dataclass -class ProtoComment: - """Describes a piece of syntax that is a comment. - - It's not a :class:`blib2to3.pytree.Leaf` so that: - - * it can be cached (`Leaf` objects should not be reused more than once as - they store their lineno, column, prefix, and parent information); - * `newlines` and `consumed` fields are kept separate from the `value`. This - simplifies handling of special marker comments like ``# fmt: off/on``. - """ - - type: int # token.COMMENT or STANDALONE_COMMENT - value: str # content of the comment - newlines: int # how many newlines before the comment - consumed: int # how many characters of the original leaf's prefix did we consume - - -def generate_comments(leaf: LN) -> Iterator[Leaf]: - """Clean the prefix of the `leaf` and generate comments from it, if any. - - Comments in lib2to3 are shoved into the whitespace prefix. This happens - in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation - move because it does away with modifying the grammar to include all the - possible places in which comments can be placed. - - The sad consequence for us though is that comments don't "belong" anywhere. - This is why this function generates simple parentless Leaf objects for - comments. We simply don't know what the correct parent should be. - - No matter though, we can live without this. We really only need to - differentiate between inline and standalone comments. The latter don't - share the line with any code. - - Inline comments are emitted as regular token.COMMENT leaves. Standalone - are emitted with a fake STANDALONE_COMMENT token identifier. - """ - for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): - yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines) - - -@lru_cache(maxsize=4096) -def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: - """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.""" - result: List[ProtoComment] = [] - if not prefix or "#" not in prefix: - return result - - consumed = 0 - nlines = 0 - ignored_lines = 0 - for index, line in enumerate(re.split("\r?\n", prefix)): - consumed += len(line) + 1 # adding the length of the split '\n' - line = line.lstrip() - if not line: - nlines += 1 - if not line.startswith("#"): - # Escaped newlines outside of a comment are not really newlines at - # all. We treat a single-line comment following an escaped newline - # as a simple trailing comment. - if line.endswith("\\"): - ignored_lines += 1 - continue - - if index == ignored_lines and not is_endmarker: - comment_type = token.COMMENT # simple trailing comment - else: - comment_type = STANDALONE_COMMENT - comment = make_comment(line) - result.append( - ProtoComment( - type=comment_type, value=comment, newlines=nlines, consumed=consumed - ) - ) - nlines = 0 - return result - - -def make_comment(content: str) -> str: - """Return a consistently formatted comment from the given `content` string. - - All comments (except for "##", "#!", "#:", '#'", "#%%") should have a single - space between the hash sign and the content. - - If `content` didn't start with a hash sign, one is provided. - """ - content = content.rstrip() - if not content: - return "#" - - if content[0] == "#": - content = content[1:] - NON_BREAKING_SPACE = " " - if ( - content - and content[0] == NON_BREAKING_SPACE - and not content.lstrip().startswith("type:") - ): - content = " " + content[1:] # Replace NBSP by a simple space - if content and content[0] not in " !:#'%": - content = " " + content - return "#" + content - - -def normalize_fmt_off(node: Node) -> None: - """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" - try_again = True - while try_again: - try_again = convert_one_fmt_off_pair(node) - - -def convert_one_fmt_off_pair(node: Node) -> bool: - """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. - - Returns True if a pair was converted. - """ - for leaf in node.leaves(): - previous_consumed = 0 - for comment in list_comments(leaf.prefix, is_endmarker=False): - if comment.value not in FMT_PASS: - previous_consumed = comment.consumed - continue - # We only want standalone comments. If there's no previous leaf or - # the previous leaf is indentation, it's a standalone comment in - # disguise. - if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT: - prev = preceding_leaf(leaf) - if prev: - if comment.value in FMT_OFF and prev.type not in WHITESPACE: - continue - if comment.value in FMT_SKIP and prev.type in WHITESPACE: - continue - - ignored_nodes = list(generate_ignored_nodes(leaf, comment)) - if not ignored_nodes: - continue - - first = ignored_nodes[0] # Can be a container node with the `leaf`. - parent = first.parent - prefix = first.prefix - if comment.value in FMT_OFF: - first.prefix = prefix[comment.consumed :] - if comment.value in FMT_SKIP: - first.prefix = "" - hidden_value = "".join(str(n) for n in ignored_nodes) - if comment.value in FMT_OFF: - hidden_value = comment.value + "\n" + hidden_value - if comment.value in FMT_SKIP: - hidden_value += " " + comment.value - if hidden_value.endswith("\n"): - # That happens when one of the `ignored_nodes` ended with a NEWLINE - # leaf (possibly followed by a DEDENT). - hidden_value = hidden_value[:-1] - first_idx: Optional[int] = None - for ignored in ignored_nodes: - index = ignored.remove() - if first_idx is None: - first_idx = index - assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)" - assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)" - parent.insert_child( - first_idx, - Leaf( - STANDALONE_COMMENT, - hidden_value, - prefix=prefix[:previous_consumed] + "\n" * comment.newlines, - ), - ) - return True - - return False - - -def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]: - """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. - - If comment is skip, returns leaf only. - Stops at the end of the block. - """ - container: Optional[LN] = container_of(leaf) - if comment.value in FMT_SKIP: - prev_sibling = leaf.prev_sibling - if comment.value in leaf.prefix and prev_sibling is not None: - leaf.prefix = leaf.prefix.replace(comment.value, "") - siblings = [prev_sibling] - while ( - "\n" not in prev_sibling.prefix - and prev_sibling.prev_sibling is not None - ): - prev_sibling = prev_sibling.prev_sibling - siblings.insert(0, prev_sibling) - for sibling in siblings: - yield sibling - elif leaf.parent is not None: - yield leaf.parent - return - while container is not None and container.type != token.ENDMARKER: - if is_fmt_on(container): - return - - # fix for fmt: on in children - if contains_fmt_on_at_column(container, leaf.column): - for child in container.children: - if contains_fmt_on_at_column(child, leaf.column): - return - yield child - else: - yield container - container = container.next_sibling - - -def is_fmt_on(container: LN) -> bool: - """Determine whether formatting is switched on within a container. - Determined by whether the last `# fmt:` comment is `on` or `off`. - """ - fmt_on = False - for comment in list_comments(container.prefix, is_endmarker=False): - if comment.value in FMT_ON: - fmt_on = True - elif comment.value in FMT_OFF: - fmt_on = False - return fmt_on - - -def contains_fmt_on_at_column(container: LN, column: int) -> bool: - """Determine if children at a given column have formatting switched on.""" - for child in container.children: - if ( - isinstance(child, Node) - and first_leaf_column(child) == column - or isinstance(child, Leaf) - and child.column == column - ): - if is_fmt_on(child): - return True - - return False - - -def contains_pragma_comment(comment_list: List[Leaf]) -> bool: - """ - Returns: - True iff one of the comments in @comment_list is a pragma used by one - of the more common static analysis tools for python (e.g. mypy, flake8, - pylint). - """ - for comment in comment_list: - if comment.value.startswith(("# type:", "# noqa", "# pylint:")): - return True - - return False