acks += asottile

[etc/vim.git] / black.py

diff --git a/black.py b/black.py
index 9ea46944b1e042260b57db4382cb6cfd0826f6ab..4995b53d51b0b4b4cc4270b9595a22abd94ead17 100644 (file)
--- a/black.py
+++ b/black.py
@@ -3,14 +3,30 @@
 import asyncio
 from asyncio.base_events import BaseEventLoop
 from concurrent.futures import Executor, ProcessPoolExecutor
 import asyncio
 from asyncio.base_events import BaseEventLoop
 from concurrent.futures import Executor, ProcessPoolExecutor

-from functools import partial
+from enum import Enum
+from functools import partial, wraps

 import keyword
 import keyword

+import logging
+from multiprocessing import Manager

 import os
 from pathlib import Path
 import tokenize
 import os
 from pathlib import Path
 import tokenize

+import signal

 import sys
 from typing import (
 import sys
 from typing import (

-    Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, Type, TypeVar, Union
+    Any,
+    Callable,
+    Dict,
+    Generic,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,

 )
 
 from attr import dataclass, Factory
 )
 
 from attr import dataclass, Factory


@@ -22,7 +38,7 @@ from blib2to3 import pygram, pytree
 from blib2to3.pgen2 import driver, token
 from blib2to3.pgen2.parse import ParseError
 
 from blib2to3.pgen2 import driver, token
 from blib2to3.pgen2.parse import ParseError
 

-__version__ = "18.3a4"
+__version__ = "18.4a0"

 DEFAULT_LINE_LENGTH = 88
 # types
 syms = pygram.python_symbols
 DEFAULT_LINE_LENGTH = 88
 # types
 syms = pygram.python_symbols


@@ -32,24 +48,27 @@ Depth = int
 NodeType = int
 LeafID = int
 Priority = int
 NodeType = int
 LeafID = int
 Priority = int

+Index = int

 LN = Union[Leaf, Node]
 LN = Union[Leaf, Node]

+SplitFunc = Callable[["Line", bool], Iterator["Line"]]

 out = partial(click.secho, bold=True, err=True)
 out = partial(click.secho, bold=True, err=True)

-err = partial(click.secho, fg='red', err=True)
+err = partial(click.secho, fg="red", err=True)

 
 
 class NothingChanged(UserWarning):
 
 
 class NothingChanged(UserWarning):

-    """Raised by `format_file` when the reformatted code is the same as source."""
+    """Raised by :func:`format_file` when reformatted code is the same as source."""

 
 
 class CannotSplit(Exception):
     """A readable split that fits the allotted line length is impossible.
 
 
 
 class CannotSplit(Exception):
     """A readable split that fits the allotted line length is impossible.
 

-    Raised by `left_hand_split()`, `right_hand_split()`, and `delimiter_split()`.
+    Raised by :func:`left_hand_split`, :func:`right_hand_split`, and
+    :func:`delimiter_split`.

     """
 
 
 class FormatError(Exception):
     """
 
 
 class FormatError(Exception):

-    """Base fmt: on/off error.
+    """Base exception for `# fmt: on` and `# fmt: off` handling.

 
     It holds the number of bytes of the prefix consumed before the format
     control comment appeared.
 
     It holds the number of bytes of the prefix consumed before the format
     control comment appeared.


@@ -76,32 +95,52 @@ class FormatOff(FormatError):
     """Found a comment like `# fmt: off` in the file."""
 
 
     """Found a comment like `# fmt: off` in the file."""
 
 

+class WriteBack(Enum):
+    NO = 0
+    YES = 1
+    DIFF = 2
+
+

 @click.command()
 @click.option(
 @click.command()
 @click.option(

-    '-l',
-    '--line-length',
+    "-l",
+    "--line-length",

     type=int,
     default=DEFAULT_LINE_LENGTH,
     type=int,
     default=DEFAULT_LINE_LENGTH,

-    help='How many character per line to allow.',
+    help="How many character per line to allow.",

     show_default=True,
 )
 @click.option(
     show_default=True,
 )
 @click.option(

-    '--check',
+    "--check",

     is_flag=True,
     help=(
     is_flag=True,
     help=(

-        "Don't write back the files, just return the status.  Return code 0 "
+        "Don't write the files back, just return the status.  Return code 0 "

         "means nothing would change.  Return code 1 means some files would be "
         "reformatted.  Return code 123 means there was an internal error."
     ),
 )
 @click.option(
         "means nothing would change.  Return code 1 means some files would be "
         "reformatted.  Return code 123 means there was an internal error."
     ),
 )
 @click.option(

-    '--fast/--safe',
+    "--diff",

     is_flag=True,
     is_flag=True,

-    help='If --fast given, skip temporary sanity checks. [default: --safe]',
+    help="Don't write the files back, just output a diff for each file on stdout.",
+)
+@click.option(
+    "--fast/--safe",
+    is_flag=True,
+    help="If --fast given, skip temporary sanity checks. [default: --safe]",
+)
+@click.option(
+    "-q",
+    "--quiet",
+    is_flag=True,
+    help=(
+        "Don't emit non-error messages to stderr. Errors are still emitted, "
+        "silence those with 2>/dev/null."
+    ),

 )
 @click.version_option(version=__version__)
 @click.argument(
 )
 @click.version_option(version=__version__)
 @click.argument(

-    'src',
+    "src",

     nargs=-1,
     type=click.Path(
         exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True
     nargs=-1,
     type=click.Path(
         exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True


@@ -109,7 +148,13 @@ class FormatOff(FormatError):
 )
 @click.pass_context
 def main(
 )
 @click.pass_context
 def main(

-    ctx: click.Context, line_length: int, check: bool, fast: bool, src: List[str]
+    ctx: click.Context,
+    line_length: int,
+    check: bool,
+    diff: bool,
+    fast: bool,
+    quiet: bool,
+    src: List[str],

 ) -> None:
     """The uncompromising code formatter."""
     sources: List[Path] = []
 ) -> None:
     """The uncompromising code formatter."""
     sources: List[Path] = []


@@ -120,23 +165,34 @@ def main(
         elif p.is_file():
             # if a file was explicitly given, we don't care about its extension
             sources.append(p)
         elif p.is_file():
             # if a file was explicitly given, we don't care about its extension
             sources.append(p)

-        elif s == '-':
-            sources.append(Path('-'))
+        elif s == "-":
+            sources.append(Path("-"))

         else:
         else:

-            err(f'invalid path: {s}')
+            err(f"invalid path: {s}")
+    if check and diff:
+        exc = click.ClickException("Options --check and --diff are mutually exclusive")
+        exc.exit_code = 2
+        raise exc
+
+    if check:
+        write_back = WriteBack.NO
+    elif diff:
+        write_back = WriteBack.DIFF
+    else:
+        write_back = WriteBack.YES

     if len(sources) == 0:
         ctx.exit(0)
     elif len(sources) == 1:
         p = sources[0]
     if len(sources) == 0:
         ctx.exit(0)
     elif len(sources) == 1:
         p = sources[0]

-        report = Report(check=check)
+        report = Report(check=check, quiet=quiet)

         try:
         try:

-            if not p.is_file() and str(p) == '-':
+            if not p.is_file() and str(p) == "-":

                 changed = format_stdin_to_stdout(
                 changed = format_stdin_to_stdout(

-                    line_length=line_length, fast=fast, write_back=not check
+                    line_length=line_length, fast=fast, write_back=write_back

                 )
             else:
                 changed = format_file_in_place(
                 )
             else:
                 changed = format_file_in_place(

-                    p, line_length=line_length, fast=fast, write_back=not check
+                    p, line_length=line_length, fast=fast, write_back=write_back

                 )
             report.done(p, changed)
         except Exception as exc:
                 )
             report.done(p, changed)
         except Exception as exc:


@@ -149,88 +205,142 @@ def main(
         try:
             return_code = loop.run_until_complete(
                 schedule_formatting(
         try:
             return_code = loop.run_until_complete(
                 schedule_formatting(

-                    sources, line_length, not check, fast, loop, executor
+                    sources, line_length, write_back, fast, quiet, loop, executor

                 )
             )
         finally:
                 )
             )
         finally:

-            loop.close()
+            shutdown(loop)

             ctx.exit(return_code)
 
 
 async def schedule_formatting(
     sources: List[Path],
     line_length: int,
             ctx.exit(return_code)
 
 
 async def schedule_formatting(
     sources: List[Path],
     line_length: int,

-    write_back: bool,
+    write_back: WriteBack,

     fast: bool,
     fast: bool,

+    quiet: bool,

     loop: BaseEventLoop,
     executor: Executor,
 ) -> int:
     loop: BaseEventLoop,
     executor: Executor,
 ) -> int:

+    """Run formatting of `sources` in parallel using the provided `executor`.
+
+    (Use ProcessPoolExecutors for actual parallelism.)
+
+    `line_length`, `write_back`, and `fast` options are passed to
+    :func:`format_file_in_place`.
+    """
+    lock = None
+    if write_back == WriteBack.DIFF:
+        # For diff output, we need locks to ensure we don't interleave output
+        # from different processes.
+        manager = Manager()
+        lock = manager.Lock()

     tasks = {
         src: loop.run_in_executor(
     tasks = {
         src: loop.run_in_executor(

-            executor, format_file_in_place, src, line_length, fast, write_back
+            executor, format_file_in_place, src, line_length, fast, write_back, lock

         )
         for src in sources
     }
         )
         for src in sources
     }

+    _task_values = list(tasks.values())
+    loop.add_signal_handler(signal.SIGINT, cancel, _task_values)
+    loop.add_signal_handler(signal.SIGTERM, cancel, _task_values)

     await asyncio.wait(tasks.values())
     cancelled = []
     await asyncio.wait(tasks.values())
     cancelled = []

-    report = Report()
+    report = Report(check=write_back is WriteBack.NO, quiet=quiet)

     for src, task in tasks.items():
         if not task.done():
     for src, task in tasks.items():
         if not task.done():

-            report.failed(src, 'timed out, cancelling')
+            report.failed(src, "timed out, cancelling")

             task.cancel()
             cancelled.append(task)
             task.cancel()
             cancelled.append(task)

+        elif task.cancelled():
+            cancelled.append(task)

         elif task.exception():
             report.failed(src, str(task.exception()))
         else:
             report.done(src, task.result())
     if cancelled:
         elif task.exception():
             report.failed(src, str(task.exception()))
         else:
             report.done(src, task.result())
     if cancelled:

-        await asyncio.wait(cancelled, timeout=2)
-    out('All done! ✨ 🍰 ✨')
-    click.echo(str(report))
+        await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
+    elif not quiet:
+        out("All done! ✨ 🍰 ✨")
+    if not quiet:
+        click.echo(str(report))

     return report.return_code
 
 
 def format_file_in_place(
     return report.return_code
 
 
 def format_file_in_place(

-    src: Path, line_length: int, fast: bool, write_back: bool = False
+    src: Path,
+    line_length: int,
+    fast: bool,
+    write_back: WriteBack = WriteBack.NO,
+    lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy

 ) -> bool:
 ) -> bool:

-    """Format the file and rewrite if changed. Return True if changed."""
+    """Format file under `src` path. Return True if changed.
+
+    If `write_back` is True, write reformatted code back to stdout.
+    `line_length` and `fast` options are passed to :func:`format_file_contents`.
+    """

     with tokenize.open(src) as src_buffer:
         src_contents = src_buffer.read()
     try:
     with tokenize.open(src) as src_buffer:
         src_contents = src_buffer.read()
     try:

-        contents = format_file_contents(
+        dst_contents = format_file_contents(

             src_contents, line_length=line_length, fast=fast
         )
     except NothingChanged:
         return False
 
             src_contents, line_length=line_length, fast=fast
         )
     except NothingChanged:
         return False
 

-    if write_back:
+    if write_back == write_back.YES:

         with open(src, "w", encoding=src_buffer.encoding) as f:
         with open(src, "w", encoding=src_buffer.encoding) as f:

-            f.write(contents)
+            f.write(dst_contents)
+    elif write_back == write_back.DIFF:
+        src_name = f"{src.name}  (original)"
+        dst_name = f"{src.name}  (formatted)"
+        diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
+        if lock:
+            lock.acquire()
+        try:
+            sys.stdout.write(diff_contents)
+        finally:
+            if lock:
+                lock.release()

     return True
 
 
 def format_stdin_to_stdout(
     return True
 
 
 def format_stdin_to_stdout(

-    line_length: int, fast: bool, write_back: bool = False
+    line_length: int, fast: bool, write_back: WriteBack = WriteBack.NO

 ) -> bool:
 ) -> bool:

-    """Format file on stdin and pipe output to stdout. Return True if changed."""
-    contents = sys.stdin.read()
+    """Format file on stdin. Return True if changed.
+
+    If `write_back` is True, write reformatted code back to stdout.
+    `line_length` and `fast` arguments are passed to :func:`format_file_contents`.
+    """
+    src = sys.stdin.read()

     try:
     try:

-        contents = format_file_contents(contents, line_length=line_length, fast=fast)
+        dst = format_file_contents(src, line_length=line_length, fast=fast)

         return True
 
     except NothingChanged:
         return True
 
     except NothingChanged:

+        dst = src

         return False
 
     finally:
         return False
 
     finally:

-        if write_back:
-            sys.stdout.write(contents)
+        if write_back == WriteBack.YES:
+            sys.stdout.write(dst)
+        elif write_back == WriteBack.DIFF:
+            src_name = "<stdin>  (original)"
+            dst_name = "<stdin>  (formatted)"
+            sys.stdout.write(diff(src, dst, src_name, dst_name))

 
 
 def format_file_contents(
     src_contents: str, line_length: int, fast: bool
 ) -> FileContent:
 
 
 def format_file_contents(
     src_contents: str, line_length: int, fast: bool
 ) -> FileContent:

-    """Reformats a file and returns its contents and encoding."""
-    if src_contents.strip() == '':
+    """Reformat contents a file and return new contents.
+
+    If `fast` is False, additionally confirm that the reformatted code is
+    valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
+    `line_length` is passed to :func:`format_str`.
+    """
+    if src_contents.strip() == "":

         raise NothingChanged
 
     dst_contents = format_str(src_contents, line_length=line_length)
         raise NothingChanged
 
     dst_contents = format_str(src_contents, line_length=line_length)


@@ -244,7 +354,10 @@ def format_file_contents(
 
 
 def format_str(src_contents: str, line_length: int) -> FileContent:
 
 
 def format_str(src_contents: str, line_length: int) -> FileContent:

-    """Reformats a string and returns new contents."""
+    """Reformat a string and return new contents.
+
+    `line_length` determines how many characters per line are allowed.
+    """

     src_node = lib2to3_parse(src_contents)
     dst_contents = ""
     lines = LineGenerator()
     src_node = lib2to3_parse(src_contents)
     dst_contents = ""
     lines = LineGenerator()


@@ -274,8 +387,8 @@ GRAMMARS = [
 def lib2to3_parse(src_txt: str) -> Node:
     """Given a string with source, return the lib2to3 Node."""
     grammar = pygram.python_grammar_no_print_statement
 def lib2to3_parse(src_txt: str) -> Node:
     """Given a string with source, return the lib2to3 Node."""
     grammar = pygram.python_grammar_no_print_statement

-    if src_txt[-1] != '\n':
-        nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n'
+    if src_txt[-1] != "\n":
+        nl = "\r\n" if "\r\n" in src_txt[:1024] else "\n"

         src_txt += nl
     for grammar in GRAMMARS:
         drv = driver.Driver(grammar, pytree.convert)
         src_txt += nl
     for grammar in GRAMMARS:
         drv = driver.Driver(grammar, pytree.convert)


@@ -305,20 +418,30 @@ def lib2to3_unparse(node: Node) -> str:
     return code
 
 
     return code
 
 

-T = TypeVar('T')
+T = TypeVar("T")

 
 
 class Visitor(Generic[T]):
 
 
 class Visitor(Generic[T]):

-    """Basic lib2to3 visitor that yields things on visiting."""
+    """Basic lib2to3 visitor that yields things of type `T` on `visit()`."""

 
     def visit(self, node: LN) -> Iterator[T]:
 
     def visit(self, node: LN) -> Iterator[T]:

+        """Main method to visit `node` and its children.
+
+        It tries to find a `visit_*()` method for the given `node.type`, like
+        `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.
+        If no dedicated `visit_*()` method is found, chooses `visit_default()`
+        instead.
+
+        Then yields objects of type `T` from the selected visitor.
+        """

         if node.type < 256:
             name = token.tok_name[node.type]
         else:
             name = type_repr(node.type)
         if node.type < 256:
             name = token.tok_name[node.type]
         else:
             name = type_repr(node.type)

-        yield from getattr(self, f'visit_{name}', self.visit_default)(node)
+        yield from getattr(self, f"visit_{name}", self.visit_default)(node)

 
     def visit_default(self, node: LN) -> Iterator[T]:
 
     def visit_default(self, node: LN) -> Iterator[T]:

+        """Default `visit_*()` implementation. Recurses to children of `node`."""

         if isinstance(node, Node):
             for child in node.children:
                 yield from self.visit(child)
         if isinstance(node, Node):
             for child in node.children:
                 yield from self.visit(child)


@@ -329,28 +452,28 @@ class DebugVisitor(Visitor[T]):
     tree_depth: int = 0
 
     def visit_default(self, node: LN) -> Iterator[T]:
     tree_depth: int = 0
 
     def visit_default(self, node: LN) -> Iterator[T]:

-        indent = ' ' * (2 * self.tree_depth)
+        indent = " " * (2 * self.tree_depth)

         if isinstance(node, Node):
             _type = type_repr(node.type)
         if isinstance(node, Node):
             _type = type_repr(node.type)

-            out(f'{indent}{_type}', fg='yellow')
+            out(f"{indent}{_type}", fg="yellow")

             self.tree_depth += 1
             for child in node.children:
                 yield from self.visit(child)
 
             self.tree_depth -= 1
             self.tree_depth += 1
             for child in node.children:
                 yield from self.visit(child)
 
             self.tree_depth -= 1

-            out(f'{indent}/{_type}', fg='yellow', bold=False)
+            out(f"{indent}/{_type}", fg="yellow", bold=False)

         else:
             _type = token.tok_name.get(node.type, str(node.type))
         else:
             _type = token.tok_name.get(node.type, str(node.type))

-            out(f'{indent}{_type}', fg='blue', nl=False)
+            out(f"{indent}{_type}", fg="blue", nl=False)

             if node.prefix:
                 # We don't have to handle prefixes for `Node` objects since
                 # that delegates to the first child anyway.
             if node.prefix:
                 # We don't have to handle prefixes for `Node` objects since
                 # that delegates to the first child anyway.

-                out(f' {node.prefix!r}', fg='green', bold=False, nl=False)
-            out(f' {node.value!r}', fg='blue', bold=False)
+                out(f" {node.prefix!r}", fg="green", bold=False, nl=False)
+            out(f" {node.value!r}", fg="blue", bold=False)

 
     @classmethod
     def show(cls, code: str) -> None:
 
     @classmethod
     def show(cls, code: str) -> None:

-        """Pretty-prints a given string of `code`.
+        """Pretty-print the lib2to3 AST of a given string of `code`.

 
         Convenience method for debugging.
         """
 
         Convenience method for debugging.
         """


@@ -360,7 +483,7 @@ class DebugVisitor(Visitor[T]):
 
 KEYWORDS = set(keyword.kwlist)
 WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE}
 
 KEYWORDS = set(keyword.kwlist)
 WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE}

-FLOW_CONTROL = {'return', 'raise', 'break', 'continue'}
+FLOW_CONTROL = {"return", "raise", "break", "continue"}

 STATEMENT = {
     syms.if_stmt,
     syms.while_stmt,
 STATEMENT = {
     syms.if_stmt,
     syms.while_stmt,


@@ -372,7 +495,7 @@ STATEMENT = {
     syms.classdef,
 }
 STANDALONE_COMMENT = 153
     syms.classdef,
 }
 STANDALONE_COMMENT = 153

-LOGIC_OPERATORS = {'and', 'or'}
+LOGIC_OPERATORS = {"and", "or"}

 COMPARATORS = {
     token.LESS,
     token.GREATER,
 COMPARATORS = {
     token.LESS,
     token.GREATER,


@@ -396,6 +519,7 @@ MATH_OPERATORS = {
     token.DOUBLESTAR,
     token.DOUBLESLASH,
 }
     token.DOUBLESTAR,
     token.DOUBLESLASH,
 }

+VARARGS = {token.STAR, token.DOUBLESTAR}

 COMPREHENSION_PRIORITY = 20
 COMMA_PRIORITY = 10
 LOGIC_PRIORITY = 5
 COMPREHENSION_PRIORITY = 20
 COMMA_PRIORITY = 10
 LOGIC_PRIORITY = 5


@@ -406,12 +530,28 @@ MATH_PRIORITY = 1
 
 @dataclass
 class BracketTracker:
 
 @dataclass
 class BracketTracker:

+    """Keeps track of brackets on a line."""
+

     depth: int = 0
     bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
     delimiters: Dict[LeafID, Priority] = Factory(dict)
     previous: Optional[Leaf] = None
 
     def mark(self, leaf: Leaf) -> None:
     depth: int = 0
     bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
     delimiters: Dict[LeafID, Priority] = Factory(dict)
     previous: Optional[Leaf] = None
 
     def mark(self, leaf: Leaf) -> None:

+        """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
+
+        All leaves receive an int `bracket_depth` field that stores how deep
+        within brackets a given leaf is. 0 means there are no enclosing brackets
+        that started on this line.
+
+        If a leaf is itself a closing bracket, it receives an `opening_bracket`
+        field that it forms a pair with. This is a one-directional link to
+        avoid reference cycles.
+
+        If a leaf is a delimiter (a token on which Black can split the line if
+        needed) and it's on depth 0, its `id()` is stored in the tracker's
+        `delimiters` field.
+        """

         if leaf.type == token.COMMENT:
             return
 
         if leaf.type == token.COMMENT:
             return
 


@@ -421,43 +561,23 @@ class BracketTracker:
             leaf.opening_bracket = opening_bracket
         leaf.bracket_depth = self.depth
         if self.depth == 0:
             leaf.opening_bracket = opening_bracket
         leaf.bracket_depth = self.depth
         if self.depth == 0:

-            delim = is_delimiter(leaf)
-            if delim:
-                self.delimiters[id(leaf)] = delim
-            elif self.previous is not None:
-                if leaf.type == token.STRING and self.previous.type == token.STRING:
-                    self.delimiters[id(self.previous)] = STRING_PRIORITY
-                elif (
-                    leaf.type == token.NAME
-                    and leaf.value == 'for'
-                    and leaf.parent
-                    and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
-                ):
-                    self.delimiters[id(self.previous)] = COMPREHENSION_PRIORITY
-                elif (
-                    leaf.type == token.NAME
-                    and leaf.value == 'if'
-                    and leaf.parent
-                    and leaf.parent.type in {syms.comp_if, syms.old_comp_if}
-                ):
-                    self.delimiters[id(self.previous)] = COMPREHENSION_PRIORITY
-                elif (
-                    leaf.type == token.NAME
-                    and leaf.value in LOGIC_OPERATORS
-                    and leaf.parent
-                ):
-                    self.delimiters[id(self.previous)] = LOGIC_PRIORITY
+            after_delim = is_split_after_delimiter(leaf, self.previous)
+            before_delim = is_split_before_delimiter(leaf, self.previous)
+            if after_delim > before_delim:
+                self.delimiters[id(leaf)] = after_delim
+            elif before_delim > after_delim and self.previous is not None:
+                self.delimiters[id(self.previous)] = before_delim

         if leaf.type in OPENING_BRACKETS:
             self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf
             self.depth += 1
         self.previous = leaf
 
     def any_open_brackets(self) -> bool:
         if leaf.type in OPENING_BRACKETS:
             self.bracket_match[self.depth, BRACKET[leaf.type]] = leaf
             self.depth += 1
         self.previous = leaf
 
     def any_open_brackets(self) -> bool:

-        """Returns True if there is an yet unmatched open bracket on the line."""
+        """Return True if there is an yet unmatched open bracket on the line."""

         return bool(self.bracket_match)
 
         return bool(self.bracket_match)
 

-    def max_priority(self, exclude: Iterable[LeafID] = ()) -> int:
-        """Returns the highest priority of a delimiter found on the line.
+    def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int:
+        """Return the highest priority of a delimiter found on the line.

 
         Values are consistent with what `is_delimiter()` returns.
         """
 
         Values are consistent with what `is_delimiter()` returns.
         """


@@ -466,15 +586,26 @@ class BracketTracker:
 
 @dataclass
 class Line:
 
 @dataclass
 class Line:

+    """Holds leaves and comments. Can be printed with `str(line)`."""
+

     depth: int = 0
     leaves: List[Leaf] = Factory(list)
     depth: int = 0
     leaves: List[Leaf] = Factory(list)

-    comments: Dict[LeafID, Leaf] = Factory(dict)
+    comments: List[Tuple[Index, Leaf]] = Factory(list)

     bracket_tracker: BracketTracker = Factory(BracketTracker)
     inside_brackets: bool = False
     has_for: bool = False
     _for_loop_variable: bool = False
 
     def append(self, leaf: Leaf, preformatted: bool = False) -> None:
     bracket_tracker: BracketTracker = Factory(BracketTracker)
     inside_brackets: bool = False
     has_for: bool = False
     _for_loop_variable: bool = False
 
     def append(self, leaf: Leaf, preformatted: bool = False) -> None:

+        """Add a new `leaf` to the end of the line.
+
+        Unless `preformatted` is True, the `leaf` will receive a new consistent
+        whitespace prefix and metadata applied by :class:`BracketTracker`.
+        Trailing commas are maybe removed, unpacked for loop variables are
+        demoted from being delimiters.
+
+        Inline comments are put aside.
+        """

         has_value = leaf.value.strip()
         if not has_value:
             return
         has_value = leaf.value.strip()
         if not has_value:
             return


@@ -488,35 +619,54 @@ class Line:
             self.bracket_tracker.mark(leaf)
             self.maybe_remove_trailing_comma(leaf)
             self.maybe_increment_for_loop_variable(leaf)
             self.bracket_tracker.mark(leaf)
             self.maybe_remove_trailing_comma(leaf)
             self.maybe_increment_for_loop_variable(leaf)

-            if self.maybe_adapt_standalone_comment(leaf):
-                return

 
         if not self.append_comment(leaf):
             self.leaves.append(leaf)
 
 
         if not self.append_comment(leaf):
             self.leaves.append(leaf)
 

+    def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None:
+        """Like :func:`append()` but disallow invalid standalone comment structure.
+
+        Raises ValueError when any `leaf` is appended after a standalone comment
+        or when a standalone comment is not the first leaf on the line.
+        """
+        if self.bracket_tracker.depth == 0:
+            if self.is_comment:
+                raise ValueError("cannot append to standalone comments")
+
+            if self.leaves and leaf.type == STANDALONE_COMMENT:
+                raise ValueError(
+                    "cannot append standalone comments to a populated line"
+                )
+
+        self.append(leaf, preformatted=preformatted)
+

     @property
     def is_comment(self) -> bool:
     @property
     def is_comment(self) -> bool:

-        return bool(self) and self.leaves[0].type == STANDALONE_COMMENT
+        """Is this line a standalone comment?"""
+        return len(self.leaves) == 1 and self.leaves[0].type == STANDALONE_COMMENT

 
     @property
     def is_decorator(self) -> bool:
 
     @property
     def is_decorator(self) -> bool:

+        """Is this line a decorator?"""

         return bool(self) and self.leaves[0].type == token.AT
 
     @property
     def is_import(self) -> bool:
         return bool(self) and self.leaves[0].type == token.AT
 
     @property
     def is_import(self) -> bool:

+        """Is this an import line?"""

         return bool(self) and is_import(self.leaves[0])
 
     @property
     def is_class(self) -> bool:
         return bool(self) and is_import(self.leaves[0])
 
     @property
     def is_class(self) -> bool:

+        """Is this line a class definition?"""

         return (
             bool(self)
             and self.leaves[0].type == token.NAME
         return (
             bool(self)
             and self.leaves[0].type == token.NAME

-            and self.leaves[0].value == 'class'
+            and self.leaves[0].value == "class"

         )
 
     @property
     def is_def(self) -> bool:
         )
 
     @property
     def is_def(self) -> bool:

-        """Also returns True for async defs."""
+        """Is this a function definition? (Also returns True for async defs.)"""

         try:
             first_leaf = self.leaves[0]
         except IndexError:
         try:
             first_leaf = self.leaves[0]
         except IndexError:


@@ -527,17 +677,21 @@ class Line:
         except IndexError:
             second_leaf = None
         return (
         except IndexError:
             second_leaf = None
         return (

-            (first_leaf.type == token.NAME and first_leaf.value == 'def')
+            (first_leaf.type == token.NAME and first_leaf.value == "def")

             or (
                 first_leaf.type == token.ASYNC
                 and second_leaf is not None
                 and second_leaf.type == token.NAME
             or (
                 first_leaf.type == token.ASYNC
                 and second_leaf is not None
                 and second_leaf.type == token.NAME

-                and second_leaf.value == 'def'
+                and second_leaf.value == "def"

             )
         )
 
     @property
     def is_flow_control(self) -> bool:
             )
         )
 
     @property
     def is_flow_control(self) -> bool:

+        """Is this line a flow control statement?
+
+        Those are `return`, `raise`, `break`, and `continue`.
+        """

         return (
             bool(self)
             and self.leaves[0].type == token.NAME
         return (
             bool(self)
             and self.leaves[0].type == token.NAME


@@ -546,13 +700,24 @@ class Line:
 
     @property
     def is_yield(self) -> bool:
 
     @property
     def is_yield(self) -> bool:

+        """Is this line a yield statement?"""

         return (
             bool(self)
             and self.leaves[0].type == token.NAME
         return (
             bool(self)
             and self.leaves[0].type == token.NAME

-            and self.leaves[0].value == 'yield'
+            and self.leaves[0].value == "yield"

         )
 
         )
 

+    @property
+    def contains_standalone_comments(self) -> bool:
+        """If so, needs to be split before emitting."""
+        for leaf in self.leaves:
+            if leaf.type == STANDALONE_COMMENT:
+                return True
+
+        return False
+

     def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
     def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:

+        """Remove trailing comma if there is one and it's safe."""

         if not (
             self.leaves
             and self.leaves[-1].type == token.COMMA
         if not (
             self.leaves
             and self.leaves[-1].type == token.COMMA


@@ -561,13 +726,13 @@ class Line:
             return False
 
         if closing.type == token.RBRACE:
             return False
 
         if closing.type == token.RBRACE:

-            self.leaves.pop()
+            self.remove_trailing_comma()

             return True
 
         if closing.type == token.RSQB:
             comma = self.leaves[-1]
             if comma.parent and comma.parent.type == syms.listmaker:
             return True
 
         if closing.type == token.RSQB:
             comma = self.leaves[-1]
             if comma.parent and comma.parent.type == syms.listmaker:

-                self.leaves.pop()
+                self.remove_trailing_comma()

                 return True
 
         # For parens let's check if it's safe to remove the comma.  If the
                 return True
 
         # For parens let's check if it's safe to remove the comma.  If the


@@ -595,7 +760,7 @@ class Line:
                     break
 
         if commas > 1:
                     break
 
         if commas > 1:

-            self.leaves.pop()
+            self.remove_trailing_comma()

             return True
 
         return False
             return True
 
         return False


@@ -603,10 +768,10 @@ class Line:
     def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
         """In a for loop, or comprehension, the variables are often unpacks.
 
     def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
         """In a for loop, or comprehension, the variables are often unpacks.
 

-        To avoid splitting on the comma in this situation, we will increase
-        the depth of tokens between `for` and `in`.
+        To avoid splitting on the comma in this situation, increase the depth of
+        tokens between `for` and `in`.

         """
         """

-        if leaf.type == token.NAME and leaf.value == 'for':
+        if leaf.type == token.NAME and leaf.value == "for":

             self.has_for = True
             self.bracket_tracker.depth += 1
             self._for_loop_variable = True
             self.has_for = True
             self.bracket_tracker.depth += 1
             self._for_loop_variable = True


@@ -615,80 +780,89 @@ class Line:
         return False
 
     def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
         return False
 
     def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:

-        # See `maybe_increment_for_loop_variable` above for explanation.
-        if self._for_loop_variable and leaf.type == token.NAME and leaf.value == 'in':
+        """See `maybe_increment_for_loop_variable` above for explanation."""
+        if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":

             self.bracket_tracker.depth -= 1
             self._for_loop_variable = False
             return True
 
         return False
 
             self.bracket_tracker.depth -= 1
             self._for_loop_variable = False
             return True
 
         return False
 

-    def maybe_adapt_standalone_comment(self, comment: Leaf) -> bool:
-        """Hack a standalone comment to act as a trailing comment for line splitting.
-
-        If this line has brackets and a standalone `comment`, we need to adapt
-        it to be able to still reformat the line.
-
-        This is not perfect, the line to which the standalone comment gets
-        appended will appear "too long" when splitting.
-        """
-        if not (
+    def append_comment(self, comment: Leaf) -> bool:
+        """Add an inline or standalone comment to the line."""
+        if (

             comment.type == STANDALONE_COMMENT
             and self.bracket_tracker.any_open_brackets()
         ):
             comment.type == STANDALONE_COMMENT
             and self.bracket_tracker.any_open_brackets()
         ):

+            comment.prefix = ""

             return False
 
             return False
 

-        comment.type = token.COMMENT
-        comment.prefix = '\n' + '    ' * (self.depth + 1)
-        return self.append_comment(comment)
-
-    def append_comment(self, comment: Leaf) -> bool:

         if comment.type != token.COMMENT:
             return False
 
         if comment.type != token.COMMENT:
             return False
 

-        try:
-            after = id(self.last_non_delimiter())
-        except LookupError:
+        after = len(self.leaves) - 1
+        if after == -1:

             comment.type = STANDALONE_COMMENT
             comment.type = STANDALONE_COMMENT

-            comment.prefix = ''
+            comment.prefix = ""

             return False
 
         else:
             return False
 
         else:

-            if after in self.comments:
-                self.comments[after].value += str(comment)
-            else:
-                self.comments[after] = comment
+            self.comments.append((after, comment))

             return True
 
             return True
 

-    def last_non_delimiter(self) -> Leaf:
-        for i in range(len(self.leaves)):
-            last = self.leaves[-i - 1]
-            if not is_delimiter(last):
-                return last
+    def comments_after(self, leaf: Leaf) -> Iterator[Leaf]:
+        """Generate comments that should appear directly after `leaf`."""
+        for _leaf_index, _leaf in enumerate(self.leaves):
+            if leaf is _leaf:
+                break
+
+        else:
+            return

 
 

-        raise LookupError("No non-delimiters found")
+        for index, comment_after in self.comments:
+            if _leaf_index == index:
+                yield comment_after
+
+    def remove_trailing_comma(self) -> None:
+        """Remove the trailing comma and moves the comments attached to it."""
+        comma_index = len(self.leaves) - 1
+        for i in range(len(self.comments)):
+            comment_index, comment = self.comments[i]
+            if comment_index == comma_index:
+                self.comments[i] = (comma_index - 1, comment)
+        self.leaves.pop()

 
     def __str__(self) -> str:
 
     def __str__(self) -> str:

+        """Render the line."""

         if not self:
         if not self:

-            return '\n'
+            return "\n"

 
 

-        indent = '    ' * self.depth
+        indent = "    " * self.depth

         leaves = iter(self.leaves)
         first = next(leaves)
         leaves = iter(self.leaves)
         first = next(leaves)

-        res = f'{first.prefix}{indent}{first.value}'
+        res = f"{first.prefix}{indent}{first.value}"

         for leaf in leaves:
             res += str(leaf)
         for leaf in leaves:
             res += str(leaf)

-        for comment in self.comments.values():
+        for _, comment in self.comments:

             res += str(comment)
             res += str(comment)

-        return res + '\n'
+        return res + "\n"

 
     def __bool__(self) -> bool:
 
     def __bool__(self) -> bool:

+        """Return True if the line has leaves or comments."""

         return bool(self.leaves or self.comments)
 
 
 class UnformattedLines(Line):
         return bool(self.leaves or self.comments)
 
 
 class UnformattedLines(Line):

+    """Just like :class:`Line` but stores lines which aren't reformatted."""

 
 

-    def append(self, leaf: Leaf, preformatted: bool = False) -> None:
+    def append(self, leaf: Leaf, preformatted: bool = True) -> None:
+        """Just add a new `leaf` to the end of the lines.
+
+        The `preformatted` argument is ignored.
+
+        Keeps track of indentation `depth`, which is useful when the user
+        says `# fmt: on`. Otherwise, doesn't do anything with the `leaf`.
+        """

         try:
             list(generate_comments(leaf))
         except FormatOn as f_on:
         try:
             list(generate_comments(leaf))
         except FormatOn as f_on:


@@ -701,27 +875,31 @@ class UnformattedLines(Line):
         elif leaf.type == token.DEDENT:
             self.depth -= 1
 
         elif leaf.type == token.DEDENT:
             self.depth -= 1
 

+    def __str__(self) -> str:
+        """Render unformatted lines from leaves which were added with `append()`.
+
+        `depth` is not used for indentation in this case.
+        """
+        if not self:
+            return "\n"
+
+        res = ""
+        for leaf in self.leaves:
+            res += str(leaf)
+        return res
+

     def append_comment(self, comment: Leaf) -> bool:
     def append_comment(self, comment: Leaf) -> bool:

+        """Not implemented in this class. Raises `NotImplementedError`."""

         raise NotImplementedError("Unformatted lines don't store comments separately.")
 
     def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
         raise NotImplementedError("Unformatted lines don't store comments separately.")
 
     def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:

+        """Does nothing and returns False."""

         return False
 
     def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
         return False
 
     def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:

+        """Does nothing and returns False."""

         return False
 
         return False
 

-    def maybe_adapt_standalone_comment(self, comment: Leaf) -> bool:
-        return False
-
-    def __str__(self) -> str:
-        if not self:
-            return '\n'
-
-        res = ''
-        for leaf in self.leaves:
-            res += str(leaf)
-        return res
-

 
 @dataclass
 class EmptyLineTracker:
 
 @dataclass
 class EmptyLineTracker:


@@ -737,11 +915,11 @@ class EmptyLineTracker:
     previous_defs: List[int] = Factory(list)
 
     def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
     previous_defs: List[int] = Factory(list)
 
     def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:

-        """Returns the number of extra empty lines before and after the `current_line`.
+        """Return the number of extra empty lines before and after the `current_line`.

 
 

-        This is for separating `def`, `async def` and `class` with extra empty lines
-        (two on module-level), as well as providing an extra empty line after flow
-        control keywords to make them more prominent.
+        This is for separating `def`, `async def` and `class` with extra empty
+        lines (two on module-level), as well as providing an extra empty line
+        after flow control keywords to make them more prominent.

         """
         if isinstance(current_line, UnformattedLines):
             return 0, 0
         """
         if isinstance(current_line, UnformattedLines):
             return 0, 0


@@ -754,14 +932,14 @@ class EmptyLineTracker:
 
     def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
         max_allowed = 1
 
     def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
         max_allowed = 1

-        if current_line.is_comment and current_line.depth == 0:
+        if current_line.depth == 0:

             max_allowed = 2
         if current_line.leaves:
             # Consume the first leaf's extra newlines.
             first_leaf = current_line.leaves[0]
             max_allowed = 2
         if current_line.leaves:
             # Consume the first leaf's extra newlines.
             first_leaf = current_line.leaves[0]

-            before = first_leaf.prefix.count('\n')
-            before = min(before, max(before, max_allowed))
-            first_leaf.prefix = ''
+            before = first_leaf.prefix.count("\n")
+            before = min(before, max_allowed)
+            first_leaf.prefix = ""

         else:
             before = 0
         depth = current_line.depth
         else:
             before = 0
         depth = current_line.depth


@@ -835,7 +1013,10 @@ class LineGenerator(Visitor[Line]):
         yield complete_line
 
     def visit(self, node: LN) -> Iterator[Line]:
         yield complete_line
 
     def visit(self, node: LN) -> Iterator[Line]:

-        """High-level entry point to the visitor."""
+        """Main method to visit `node` and its children.
+
+        Yields :class:`Line` objects.
+        """

         if isinstance(self.current_line, UnformattedLines):
             # File contained `# fmt: off`
             yield from self.visit_unformatted(node)
         if isinstance(self.current_line, UnformattedLines):
             # File contained `# fmt: off`
             yield from self.visit_unformatted(node)


@@ -844,6 +1025,7 @@ class LineGenerator(Visitor[Line]):
             yield from super().visit(node)
 
     def visit_default(self, node: LN) -> Iterator[Line]:
             yield from super().visit(node)
 
     def visit_default(self, node: LN) -> Iterator[Line]:

+        """Default `visit_*()` implementation. Recurses to children of `node`."""

         if isinstance(node, Leaf):
             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
             try:
         if isinstance(node, Leaf):
             any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()
             try:


@@ -876,23 +1058,31 @@ class LineGenerator(Visitor[Line]):
 
             else:
                 normalize_prefix(node, inside_brackets=any_open_brackets)
 
             else:
                 normalize_prefix(node, inside_brackets=any_open_brackets)

+                if node.type == token.STRING:
+                    normalize_string_quotes(node)

                 if node.type not in WHITESPACE:
                     self.current_line.append(node)
         yield from super().visit_default(node)
 
     def visit_INDENT(self, node: Node) -> Iterator[Line]:
                 if node.type not in WHITESPACE:
                     self.current_line.append(node)
         yield from super().visit_default(node)
 
     def visit_INDENT(self, node: Node) -> Iterator[Line]:

+        """Increase indentation level, maybe yield a line."""
+        # In blib2to3 INDENT never holds comments.

         yield from self.line(+1)
         yield from self.visit_default(node)
 
     def visit_DEDENT(self, node: Node) -> Iterator[Line]:
         yield from self.line(+1)
         yield from self.visit_default(node)
 
     def visit_DEDENT(self, node: Node) -> Iterator[Line]:

+        """Decrease indentation level, maybe yield a line."""

         # DEDENT has no value. Additionally, in blib2to3 it never holds comments.
         yield from self.line(-1)
 
     def visit_stmt(self, node: Node, keywords: Set[str]) -> Iterator[Line]:
         """Visit a statement.
 
         # DEDENT has no value. Additionally, in blib2to3 it never holds comments.
         yield from self.line(-1)
 
     def visit_stmt(self, node: Node, keywords: Set[str]) -> Iterator[Line]:
         """Visit a statement.
 

-        The relevant Python language keywords for this statement are NAME leaves
-        within it.
+        This implementation is shared for `if`, `while`, `for`, `try`, `except`,
+        `def`, `with`, and `class`.
+
+        The relevant Python language `keywords` for a given statement will be NAME
+        leaves within it. This methods puts those on a separate line.

         """
         for child in node.children:
             if child.type == token.NAME and child.value in keywords:  # type: ignore
         """
         for child in node.children:
             if child.type == token.NAME and child.value in keywords:  # type: ignore


@@ -901,7 +1091,7 @@ class LineGenerator(Visitor[Line]):
             yield from self.visit(child)
 
     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
             yield from self.visit(child)
 
     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:

-        """A statement without nested statements."""
+        """Visit a statement without nested statements."""

         is_suite_like = node.parent and node.parent.type in STATEMENT
         if is_suite_like:
             yield from self.line(+1)
         is_suite_like = node.parent and node.parent.type in STATEMENT
         if is_suite_like:
             yield from self.line(+1)


@@ -913,6 +1103,7 @@ class LineGenerator(Visitor[Line]):
             yield from self.visit_default(node)
 
     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
             yield from self.visit_default(node)
 
     def visit_async_stmt(self, node: Node) -> Iterator[Line]:

+        """Visit `async def`, `async for`, `async with`."""

         yield from self.line()
 
         children = iter(node.children)
         yield from self.line()
 
         children = iter(node.children)


@@ -927,18 +1118,22 @@ class LineGenerator(Visitor[Line]):
             yield from self.visit(child)
 
     def visit_decorators(self, node: Node) -> Iterator[Line]:
             yield from self.visit(child)
 
     def visit_decorators(self, node: Node) -> Iterator[Line]:

+        """Visit decorators."""

         for child in node.children:
             yield from self.line()
             yield from self.visit(child)
 
     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
         for child in node.children:
             yield from self.line()
             yield from self.visit(child)
 
     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:

+        """Remove a semicolon and put the other statement on a separate line."""

         yield from self.line()
 
     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:
         yield from self.line()
 
     def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:

+        """End of file. Process outstanding comments and end with a newline."""

         yield from self.visit_default(leaf)
         yield from self.line()
 
     def visit_unformatted(self, node: LN) -> Iterator[Line]:
         yield from self.visit_default(leaf)
         yield from self.line()
 
     def visit_unformatted(self, node: LN) -> Iterator[Line]:

+        """Used when file contained a `# fmt: off`."""

         if isinstance(node, Node):
             for child in node.children:
                 yield from self.visit(child)
         if isinstance(node, Node):
             for child in node.children:
                 yield from self.visit(child)


@@ -951,17 +1146,21 @@ class LineGenerator(Visitor[Line]):
                 yield from self.line()
                 yield from self.visit(node)
 
                 yield from self.line()
                 yield from self.visit(node)
 

+            if node.type == token.ENDMARKER:
+                # somebody decided not to put a final `# fmt: on`
+                yield from self.line()
+

     def __attrs_post_init__(self) -> None:
         """You are in a twisty little maze of passages."""
         v = self.visit_stmt
     def __attrs_post_init__(self) -> None:
         """You are in a twisty little maze of passages."""
         v = self.visit_stmt

-        self.visit_if_stmt = partial(v, keywords={'if', 'else', 'elif'})
-        self.visit_while_stmt = partial(v, keywords={'while', 'else'})
-        self.visit_for_stmt = partial(v, keywords={'for', 'else'})
-        self.visit_try_stmt = partial(v, keywords={'try', 'except', 'else', 'finally'})
-        self.visit_except_clause = partial(v, keywords={'except'})
-        self.visit_funcdef = partial(v, keywords={'def'})
-        self.visit_with_stmt = partial(v, keywords={'with'})
-        self.visit_classdef = partial(v, keywords={'class'})
+        self.visit_if_stmt = partial(v, keywords={"if", "else", "elif"})
+        self.visit_while_stmt = partial(v, keywords={"while", "else"})
+        self.visit_for_stmt = partial(v, keywords={"for", "else"})
+        self.visit_try_stmt = partial(v, keywords={"try", "except", "else", "finally"})
+        self.visit_except_clause = partial(v, keywords={"except"})
+        self.visit_funcdef = partial(v, keywords={"def"})
+        self.visit_with_stmt = partial(v, keywords={"with"})
+        self.visit_classdef = partial(v, keywords={"class"})

         self.visit_async_funcdef = self.visit_async_stmt
         self.visit_decorated = self.visit_decorators
 
         self.visit_async_funcdef = self.visit_async_stmt
         self.visit_decorated = self.visit_decorators
 


@@ -975,9 +1174,9 @@ ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
 
 def whitespace(leaf: Leaf) -> str:  # noqa C901
     """Return whitespace prefix if needed for the given `leaf`."""
 
 def whitespace(leaf: Leaf) -> str:  # noqa C901
     """Return whitespace prefix if needed for the given `leaf`."""

-    NO = ''
-    SPACE = ' '
-    DOUBLESPACE = '  '
+    NO = ""
+    SPACE = " "
+    DOUBLESPACE = "  "

     t = leaf.type
     p = leaf.parent
     v = leaf.value
     t = leaf.type
     p = leaf.parent
     v = leaf.value


@@ -1041,7 +1240,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
             and prevp.parent.type == syms.shift_expr
             and prevp.prev_sibling
             and prevp.prev_sibling.type == token.NAME
             and prevp.parent.type == syms.shift_expr
             and prevp.prev_sibling
             and prevp.prev_sibling.type == token.NAME

-            and prevp.prev_sibling.value == 'print'  # type: ignore
+            and prevp.prev_sibling.value == "print"  # type: ignore

         ):
             # Python 2 print chevron
             return NO
         ):
             # Python 2 print chevron
             return NO


@@ -1198,7 +1397,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
                 return NO
 
         elif t == token.NAME:
                 return NO
 
         elif t == token.NAME:

-            if v == 'import':
+            if v == "import":

                 return SPACE
 
             if prev and prev.type == token.DOT:
                 return SPACE
 
             if prev and prev.type == token.DOT:


@@ -1211,7 +1410,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
 
 
 def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
 
 
 def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:

-    """Returns the first leaf that precedes `node`, if any."""
+    """Return the first leaf that precedes `node`, if any."""

     while node:
         res = node.prev_sibling
         if res:
     while node:
         res = node.prev_sibling
         if res:


@@ -1228,17 +1427,35 @@ def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
     return None
 
 
     return None
 
 

-def is_delimiter(leaf: Leaf) -> int:
-    """Returns the priority of the `leaf` delimiter. Returns 0 if not delimiter.
+def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
+    """Return the priority of the `leaf` delimiter, given a line break after it.
+
+    The delimiter priorities returned here are from those delimiters that would
+    cause a line break after themselves.

 
     Higher numbers are higher priority.
     """
     if leaf.type == token.COMMA:
         return COMMA_PRIORITY
 
 
     Higher numbers are higher priority.
     """
     if leaf.type == token.COMMA:
         return COMMA_PRIORITY
 

-    if leaf.type in COMPARATORS:
-        return COMPARATOR_PRIORITY
+    if (
+        leaf.type in VARARGS
+        and leaf.parent
+        and leaf.parent.type in {syms.argument, syms.typedargslist}
+    ):
+        return MATH_PRIORITY

 
 

+    return 0
+
+
+def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
+    """Return the priority of the `leaf` delimiter, given a line before after it.
+
+    The delimiter priorities returned here are from those delimiters that would
+    cause a line break before themselves.
+
+    Higher numbers are higher priority.
+    """

     if (
         leaf.type in MATH_OPERATORS
         and leaf.parent
     if (
         leaf.type in MATH_OPERATORS
         and leaf.parent


@@ -1246,11 +1463,51 @@ def is_delimiter(leaf: Leaf) -> int:
     ):
         return MATH_PRIORITY
 
     ):
         return MATH_PRIORITY
 

+    if leaf.type in COMPARATORS:
+        return COMPARATOR_PRIORITY
+
+    if (
+        leaf.type == token.STRING
+        and previous is not None
+        and previous.type == token.STRING
+    ):
+        return STRING_PRIORITY
+
+    if (
+        leaf.type == token.NAME
+        and leaf.value == "for"
+        and leaf.parent
+        and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
+    ):
+        return COMPREHENSION_PRIORITY
+
+    if (
+        leaf.type == token.NAME
+        and leaf.value == "if"
+        and leaf.parent
+        and leaf.parent.type in {syms.comp_if, syms.old_comp_if}
+    ):
+        return COMPREHENSION_PRIORITY
+
+    if leaf.type == token.NAME and leaf.value in LOGIC_OPERATORS and leaf.parent:
+        return LOGIC_PRIORITY
+

     return 0
 
 
     return 0
 
 

+def is_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
+    """Return the priority of the `leaf` delimiter. Return 0 if not delimiter.
+
+    Higher numbers are higher priority.
+    """
+    return max(
+        is_split_before_delimiter(leaf, previous),
+        is_split_after_delimiter(leaf, previous),
+    )
+
+

 def generate_comments(leaf: Leaf) -> Iterator[Leaf]:
 def generate_comments(leaf: Leaf) -> Iterator[Leaf]:

-    """Cleans the prefix of the `leaf` and generates comments from it, if any.
+    """Clean the prefix of the `leaf` and generate comments from it, if any.

 
     Comments in lib2to3 are shoved into the whitespace prefix.  This happens
     in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation
 
     Comments in lib2to3 are shoved into the whitespace prefix.  This happens
     in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation


@@ -1272,17 +1529,17 @@ def generate_comments(leaf: Leaf) -> Iterator[Leaf]:
     if not p:
         return
 
     if not p:
         return
 

-    if '#' not in p:
+    if "#" not in p:

         return
 
     consumed = 0
     nlines = 0
         return
 
     consumed = 0
     nlines = 0

-    for index, line in enumerate(p.split('\n')):
+    for index, line in enumerate(p.split("\n")):

         consumed += len(line) + 1  # adding the length of the split '\n'
         line = line.lstrip()
         if not line:
             nlines += 1
         consumed += len(line) + 1  # adding the length of the split '\n'
         line = line.lstrip()
         if not line:
             nlines += 1

-        if not line.startswith('#'):
+        if not line.startswith("#"):

             continue
 
         if index == 0 and leaf.type != token.ENDMARKER:
             continue
 
         if index == 0 and leaf.type != token.ENDMARKER:


@@ -1290,33 +1547,45 @@ def generate_comments(leaf: Leaf) -> Iterator[Leaf]:
         else:
             comment_type = STANDALONE_COMMENT
         comment = make_comment(line)
         else:
             comment_type = STANDALONE_COMMENT
         comment = make_comment(line)

-        yield Leaf(comment_type, comment, prefix='\n' * nlines)
+        yield Leaf(comment_type, comment, prefix="\n" * nlines)

 
 

-        if comment in {'# fmt: on', '# yapf: enable'}:
+        if comment in {"# fmt: on", "# yapf: enable"}:

             raise FormatOn(consumed)
 
             raise FormatOn(consumed)
 

-        if comment in {'# fmt: off', '# yapf: disable'}:
-            raise FormatOff(consumed)
+        if comment in {"# fmt: off", "# yapf: disable"}:
+            if comment_type == STANDALONE_COMMENT:
+                raise FormatOff(consumed)
+
+            prev = preceding_leaf(leaf)
+            if not prev or prev.type in WHITESPACE:  # standalone comment in disguise
+                raise FormatOff(consumed)

 
         nlines = 0
 
 
 def make_comment(content: str) -> str:
 
         nlines = 0
 
 
 def make_comment(content: str) -> str:

+    """Return a consistently formatted comment from the given `content` string.
+
+    All comments (except for "##", "#!", "#:") should have a single space between
+    the hash sign and the content.
+
+    If `content` didn't start with a hash sign, one is provided.
+    """

     content = content.rstrip()
     if not content:
     content = content.rstrip()
     if not content:

-        return '#'
+        return "#"

 
 

-    if content[0] == '#':
+    if content[0] == "#":

         content = content[1:]
         content = content[1:]

-    if content and content[0] not in ' !:#':
-        content = ' ' + content
-    return '#' + content
+    if content and content[0] not in " !:#":
+        content = " " + content
+    return "#" + content

 
 
 def split_line(
     line: Line, line_length: int, inner: bool = False, py36: bool = False
 ) -> Iterator[Line]:
 
 
 def split_line(
     line: Line, line_length: int, inner: bool = False, py36: bool = False
 ) -> Iterator[Line]:

-    """Splits a `line` into potentially many lines.
+    """Split a `line` into potentially many lines.

 
     They should fit in the allotted `line_length` but might not be able to.
     `inner` signifies that there were a pair of brackets somewhere around the
 
     They should fit in the allotted `line_length` but might not be able to.
     `inner` signifies that there were a pair of brackets somewhere around the


@@ -1326,23 +1595,24 @@ def split_line(
     If `py36` is True, splitting may generate syntax that is only compatible
     with Python 3.6 and later.
     """
     If `py36` is True, splitting may generate syntax that is only compatible
     with Python 3.6 and later.
     """

-    if isinstance(line, UnformattedLines):
+    if isinstance(line, UnformattedLines) or line.is_comment:

         yield line
         return
 
         yield line
         return
 

-    line_str = str(line).strip('\n')
-    if len(line_str) <= line_length and '\n' not in line_str:
+    line_str = str(line).strip("\n")
+    if (
+        len(line_str) <= line_length
+        and "\n" not in line_str  # multiline strings
+        and not line.contains_standalone_comments
+    ):

         yield line
         return
 
         yield line
         return
 

+    split_funcs: List[SplitFunc]

     if line.is_def:
         split_funcs = [left_hand_split]
     elif line.inside_brackets:
     if line.is_def:
         split_funcs = [left_hand_split]
     elif line.inside_brackets:

-        split_funcs = [delimiter_split]
-        if '\n' not in line_str:
-            # Only attempt RHS if we don't have multiline strings or comments
-            # on this line.
-            split_funcs.append(right_hand_split)
+        split_funcs = [delimiter_split, standalone_comment_split, right_hand_split]

     else:
         split_funcs = [right_hand_split]
     for split_func in split_funcs:
     else:
         split_funcs = [right_hand_split]
     for split_func in split_funcs:


@@ -1351,8 +1621,8 @@ def split_line(
         # split altogether.
         result: List[Line] = []
         try:
         # split altogether.
         result: List[Line] = []
         try:

-            for l in split_func(line, py36=py36):
-                if str(l).strip('\n') == line_str:
+            for l in split_func(line, py36):
+                if str(l).strip("\n") == line_str:

                     raise CannotSplit("Split function returned an unchanged result")
 
                 result.extend(
                     raise CannotSplit("Split function returned an unchanged result")
 
                 result.extend(


@@ -1404,10 +1674,9 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
     ):
         for leaf in leaves:
             result.append(leaf, preformatted=True)
     ):
         for leaf in leaves:
             result.append(leaf, preformatted=True)

-            comment_after = line.comments.get(id(leaf))
-            if comment_after:
+            for comment_after in line.comments_after(leaf):

                 result.append(comment_after, preformatted=True)
                 result.append(comment_after, preformatted=True)

-    split_succeeded_or_raise(head, body, tail)
+    bracket_split_succeeded_or_raise(head, body, tail)

     for result in (head, body, tail):
         if result:
             yield result
     for result in (head, body, tail):
         if result:
             yield result


@@ -1444,16 +1713,28 @@ def right_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
     ):
         for leaf in leaves:
             result.append(leaf, preformatted=True)
     ):
         for leaf in leaves:
             result.append(leaf, preformatted=True)

-            comment_after = line.comments.get(id(leaf))
-            if comment_after:
+            for comment_after in line.comments_after(leaf):

                 result.append(comment_after, preformatted=True)
                 result.append(comment_after, preformatted=True)

-    split_succeeded_or_raise(head, body, tail)
+    bracket_split_succeeded_or_raise(head, body, tail)

     for result in (head, body, tail):
         if result:
             yield result
 
 
     for result in (head, body, tail):
         if result:
             yield result
 
 

-def split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
+def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
+    """Raise :exc:`CannotSplit` if the last left- or right-hand split failed.
+
+    Do nothing otherwise.
+
+    A left- or right-hand split is based on a pair of brackets. Content before
+    (and including) the opening bracket is left on one line, content inside the
+    brackets is put on a separate line, and finally content starting with and
+    following the closing bracket is put on a separate line.
+
+    Those are called `head`, `body`, and `tail`, respectively. If the split
+    produced the same line (all content in `head`) or ended up with an empty `body`
+    and the `tail` is just the closing bracket, then it's considered failed.
+    """

     tail_len = len(str(tail).strip())
     if not body:
         if tail_len == 0:
     tail_len = len(str(tail).strip())
     if not body:
         if tail_len == 0:


@@ -1466,12 +1747,27 @@ def split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None:
             )
 
 
             )
 
 

+def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
+    """Normalize prefix of the first leaf in every line returned by `split_func`.
+
+    This is a decorator over relevant split functions.
+    """
+
+    @wraps(split_func)
+    def split_wrapper(line: Line, py36: bool = False) -> Iterator[Line]:
+        for l in split_func(line, py36):
+            normalize_prefix(l.leaves[0], inside_brackets=True)
+            yield l
+
+    return split_wrapper
+
+
+@dont_increase_indentation

 def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
     """Split according to delimiters of the highest priority.
 
 def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
     """Split according to delimiters of the highest priority.
 

-    This kind of split doesn't increase indentation.

     If `py36` is True, the split will add trailing commas also in function
     If `py36` is True, the split will add trailing commas also in function

-    signatures that contain * and **.
+    signatures that contain `*` and `**`.

     """
     try:
         last_leaf = line.leaves[-1]
     """
     try:
         last_leaf = line.leaves[-1]


@@ -1480,18 +1776,33 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
 
     delimiters = line.bracket_tracker.delimiters
     try:
 
     delimiters = line.bracket_tracker.delimiters
     try:

-        delimiter_priority = line.bracket_tracker.max_priority(exclude={id(last_leaf)})
+        delimiter_priority = line.bracket_tracker.max_delimiter_priority(
+            exclude={id(last_leaf)}
+        )

     except ValueError:
         raise CannotSplit("No delimiters found")
 
     current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
     lowest_depth = sys.maxsize
     trailing_comma_safe = True
     except ValueError:
         raise CannotSplit("No delimiters found")
 
     current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
     lowest_depth = sys.maxsize
     trailing_comma_safe = True

+
+    def append_to_line(leaf: Leaf) -> Iterator[Line]:
+        """Append `leaf` to current line or to new line if appending impossible."""
+        nonlocal current_line
+        try:
+            current_line.append_safe(leaf, preformatted=True)
+        except ValueError as ve:
+            yield current_line
+
+            current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+            current_line.append(leaf)
+

     for leaf in line.leaves:
     for leaf in line.leaves:

-        current_line.append(leaf, preformatted=True)
-        comment_after = line.comments.get(id(leaf))
-        if comment_after:
-            current_line.append(comment_after, preformatted=True)
+        yield from append_to_line(leaf)
+
+        for comment_after in line.comments_after(leaf):
+            yield from append_to_line(comment_after)
+

         lowest_depth = min(lowest_depth, leaf.bracket_depth)
         if (
             leaf.bracket_depth == lowest_depth
         lowest_depth = min(lowest_depth, leaf.bracket_depth)
         if (
             leaf.bracket_depth == lowest_depth


@@ -1501,55 +1812,141 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
             trailing_comma_safe = trailing_comma_safe and py36
         leaf_priority = delimiters.get(id(leaf))
         if leaf_priority == delimiter_priority:
             trailing_comma_safe = trailing_comma_safe and py36
         leaf_priority = delimiters.get(id(leaf))
         if leaf_priority == delimiter_priority:

-            normalize_prefix(current_line.leaves[0], inside_brackets=True)

             yield current_line
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
     if current_line:
         if (
             yield current_line
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
     if current_line:
         if (

-            delimiter_priority == COMMA_PRIORITY
+            trailing_comma_safe
+            and delimiter_priority == COMMA_PRIORITY

             and current_line.leaves[-1].type != token.COMMA
             and current_line.leaves[-1].type != token.COMMA

-            and trailing_comma_safe
+            and current_line.leaves[-1].type != STANDALONE_COMMENT

         ):
         ):

-            current_line.append(Leaf(token.COMMA, ','))
-        normalize_prefix(current_line.leaves[0], inside_brackets=True)
+            current_line.append(Leaf(token.COMMA, ","))
+        yield current_line
+
+
+@dont_increase_indentation
+def standalone_comment_split(line: Line, py36: bool = False) -> Iterator[Line]:
+    """Split standalone comments from the rest of the line."""
+    for leaf in line.leaves:
+        if leaf.type == STANDALONE_COMMENT:
+            if leaf.bracket_depth == 0:
+                break
+
+    else:
+        raise CannotSplit("Line does not have any standalone comments")
+
+    current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+
+    def append_to_line(leaf: Leaf) -> Iterator[Line]:
+        """Append `leaf` to current line or to new line if appending impossible."""
+        nonlocal current_line
+        try:
+            current_line.append_safe(leaf, preformatted=True)
+        except ValueError as ve:
+            yield current_line
+
+            current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
+            current_line.append(leaf)
+
+    for leaf in line.leaves:
+        yield from append_to_line(leaf)
+
+        for comment_after in line.comments_after(leaf):
+            yield from append_to_line(comment_after)
+
+    if current_line:

         yield current_line
 
 
 def is_import(leaf: Leaf) -> bool:
         yield current_line
 
 
 def is_import(leaf: Leaf) -> bool:

-    """Returns True if the given leaf starts an import statement."""
+    """Return True if the given leaf starts an import statement."""

     p = leaf.parent
     t = leaf.type
     v = leaf.value
     return bool(
         t == token.NAME
         and (
     p = leaf.parent
     t = leaf.type
     v = leaf.value
     return bool(
         t == token.NAME
         and (

-            (v == 'import' and p and p.type == syms.import_name)
-            or (v == 'from' and p and p.type == syms.import_from)
+            (v == "import" and p and p.type == syms.import_name)
+            or (v == "from" and p and p.type == syms.import_from)

         )
     )
 
 
 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
         )
     )
 
 
 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:

-    """Leave existing extra newlines if not `inside_brackets`.
+    """Leave existing extra newlines if not `inside_brackets`. Remove everything
+    else.

 
 

-    Remove everything else.  Note: don't use backslashes for formatting or
-    you'll lose your voting rights.
+    Note: don't use backslashes for formatting or you'll lose your voting rights.

     """
     if not inside_brackets:
     """
     if not inside_brackets:

-        spl = leaf.prefix.split('#')
-        if '\\' not in spl[0]:
-            nl_count = spl[-1].count('\n')
+        spl = leaf.prefix.split("#")
+        if "\\" not in spl[0]:
+            nl_count = spl[-1].count("\n")

             if len(spl) > 1:
                 nl_count -= 1
             if len(spl) > 1:
                 nl_count -= 1

-            leaf.prefix = '\n' * nl_count
+            leaf.prefix = "\n" * nl_count

             return
 
             return
 

-    leaf.prefix = ''
+    leaf.prefix = ""
+
+
+def normalize_string_quotes(leaf: Leaf) -> None:
+    """Prefer double quotes but only if it doesn't cause more escaping.
+
+    Adds or removes backslashes as appropriate. Doesn't parse and fix
+    strings nested in f-strings (yet).
+
+    Note: Mutates its argument.
+    """
+    value = leaf.value.lstrip("furbFURB")
+    if value[:3] == '"""':
+        return
+
+    elif value[:3] == "'''":
+        orig_quote = "'''"
+        new_quote = '"""'
+    elif value[0] == '"':
+        orig_quote = '"'
+        new_quote = "'"
+    else:
+        orig_quote = "'"
+        new_quote = '"'
+    first_quote_pos = leaf.value.find(orig_quote)
+    if first_quote_pos == -1:
+        return  # There's an internal error
+
+    prefix = leaf.value[:first_quote_pos]
+    body = leaf.value[first_quote_pos + len(orig_quote):-len(orig_quote)]
+    if "r" in prefix.casefold():
+        if body.count(new_quote) != body.count(f"\\{new_quote}"):
+            # There's at least one unescaped new_quote in this raw string
+            # so converting is impossible
+            return
+
+        # Do not introduce or remove backslashes in raw strings
+        new_body = body
+    else:
+        new_body = body.replace(f"\\{orig_quote}", orig_quote).replace(
+            new_quote, f"\\{new_quote}"
+        )
+    if new_quote == '"""' and new_body[-1] == '"':
+        # edge case:
+        new_body = new_body[:-1] + '\\"'
+    orig_escape_count = body.count("\\")
+    new_escape_count = new_body.count("\\")
+    if new_escape_count > orig_escape_count:
+        return  # Do not introduce more escaping
+
+    if new_escape_count == orig_escape_count and orig_quote == '"':
+        return  # Prefer double quotes
+
+    leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"

 
 
 def is_python36(node: Node) -> bool:
 
 
 def is_python36(node: Node) -> bool:

-    """Returns True if the current file is using Python 3.6+ features.
+    """Return True if the current file is using Python 3.6+ features.

 
     Currently looking for:
     - f-strings; and
 
     Currently looking for:
     - f-strings; and


@@ -1558,7 +1955,7 @@ def is_python36(node: Node) -> bool:
     for n in node.pre_order():
         if n.type == token.STRING:
             value_head = n.value[:2]  # type: ignore
     for n in node.pre_order():
         if n.type == token.STRING:
             value_head = n.value[:2]  # type: ignore

-            if value_head in {'f"', 'F"', "f'", "F'", 'rf', 'fr', 'RF', 'FR'}:
+            if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:

                 return True
 
         elif (
                 return True
 
         elif (


@@ -1573,13 +1970,16 @@ def is_python36(node: Node) -> bool:
     return False
 
 
     return False
 
 

-PYTHON_EXTENSIONS = {'.py'}
+PYTHON_EXTENSIONS = {".py"}

 BLACKLISTED_DIRECTORIES = {
 BLACKLISTED_DIRECTORIES = {

-    'build', 'buck-out', 'dist', '_build', '.git', '.hg', '.mypy_cache', '.tox', '.venv'
+    "build", "buck-out", "dist", "_build", ".git", ".hg", ".mypy_cache", ".tox", ".venv"

 }
 
 
 def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
 }
 
 
 def gen_python_files_in_dir(path: Path) -> Iterator[Path]:

+    """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES
+    and have one of the PYTHON_EXTENSIONS.
+    """

     for child in path.iterdir():
         if child.is_dir():
             if child.name in BLACKLISTED_DIRECTORIES:
     for child in path.iterdir():
         if child.is_dir():
             if child.name in BLACKLISTED_DIRECTORIES:


@@ -1593,8 +1993,9 @@ def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
 
 @dataclass
 class Report:
 
 @dataclass
 class Report:

-    """Provides a reformatting counter."""
+    """Provides a reformatting counter. Can be rendered with `str(report)`."""

     check: bool = False
     check: bool = False

+    quiet: bool = False

     change_count: int = 0
     same_count: int = 0
     failure_count: int = 0
     change_count: int = 0
     same_count: int = 0
     failure_count: int = 0


@@ -1602,21 +2003,29 @@ class Report:
     def done(self, src: Path, changed: bool) -> None:
         """Increment the counter for successful reformatting. Write out a message."""
         if changed:
     def done(self, src: Path, changed: bool) -> None:
         """Increment the counter for successful reformatting. Write out a message."""
         if changed:

-            reformatted = 'would reformat' if self.check else 'reformatted'
-            out(f'{reformatted} {src}')
+            reformatted = "would reformat" if self.check else "reformatted"
+            if not self.quiet:
+                out(f"{reformatted} {src}")

             self.change_count += 1
         else:
             self.change_count += 1
         else:

-            out(f'{src} already well formatted, good job.', bold=False)
+            if not self.quiet:
+                out(f"{src} already well formatted, good job.", bold=False)

             self.same_count += 1
 
     def failed(self, src: Path, message: str) -> None:
         """Increment the counter for failed reformatting. Write out a message."""
             self.same_count += 1
 
     def failed(self, src: Path, message: str) -> None:
         """Increment the counter for failed reformatting. Write out a message."""

-        err(f'error: cannot format {src}: {message}')
+        err(f"error: cannot format {src}: {message}")

         self.failure_count += 1
 
     @property
     def return_code(self) -> int:
         self.failure_count += 1
 
     @property
     def return_code(self) -> int:

-        """Which return code should the app use considering the current state."""
+        """Return the exit code that the app should use.
+
+        This considers the current state of changed files and failures:
+        - if there were any failures, return 123;
+        - if any files were changed and --check is being used, return 1;
+        - otherwise return 0.
+        """

         # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with
         # 126 we have special returncodes reserved by the shell.
         if self.failure_count:
         # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with
         # 126 we have special returncodes reserved by the shell.
         if self.failure_count:


@@ -1628,7 +2037,7 @@ class Report:
         return 0
 
     def __str__(self) -> str:
         return 0
 
     def __str__(self) -> str:

-        """A color report of the current state.
+        """Render a color report of the current state.

 
         Use `click.unstyle` to remove colors.
         """
 
         Use `click.unstyle` to remove colors.
         """


@@ -1642,26 +2051,23 @@ class Report:
             failed = "failed to reformat"
         report = []
         if self.change_count:
             failed = "failed to reformat"
         report = []
         if self.change_count:

-            s = 's' if self.change_count > 1 else ''
+            s = "s" if self.change_count > 1 else ""

             report.append(
             report.append(

-                click.style(f'{self.change_count} file{s} {reformatted}', bold=True)
+                click.style(f"{self.change_count} file{s} {reformatted}", bold=True)

             )
         if self.same_count:
             )
         if self.same_count:

-            s = 's' if self.same_count > 1 else ''
-            report.append(f'{self.same_count} file{s} {unchanged}')
+            s = "s" if self.same_count > 1 else ""
+            report.append(f"{self.same_count} file{s} {unchanged}")

         if self.failure_count:
         if self.failure_count:

-            s = 's' if self.failure_count > 1 else ''
+            s = "s" if self.failure_count > 1 else ""

             report.append(
             report.append(

-                click.style(f'{self.failure_count} file{s} {failed}', fg='red')
+                click.style(f"{self.failure_count} file{s} {failed}", fg="red")

             )
             )

-        return ', '.join(report) + '.'
+        return ", ".join(report) + "."

 
 
 def assert_equivalent(src: str, dst: str) -> None:
 
 
 def assert_equivalent(src: str, dst: str) -> None:

-    """Raises AssertionError if `src` and `dst` aren't equivalent.
-
-    This is a temporary sanity check until Black becomes stable.
-    """
+    """Raise AssertionError if `src` and `dst` aren't equivalent."""

 
     import ast
     import traceback
 
     import ast
     import traceback


@@ -1704,17 +2110,17 @@ def assert_equivalent(src: str, dst: str) -> None:
     try:
         dst_ast = ast.parse(dst)
     except Exception as exc:
     try:
         dst_ast = ast.parse(dst)
     except Exception as exc:

-        log = dump_to_file(''.join(traceback.format_tb(exc.__traceback__)), dst)
+        log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)

         raise AssertionError(
             f"INTERNAL ERROR: Black produced invalid code: {exc}. "
             f"Please report a bug on https://github.com/ambv/black/issues.  "
             f"This invalid output might be helpful: {log}"
         ) from None
 
         raise AssertionError(
             f"INTERNAL ERROR: Black produced invalid code: {exc}. "
             f"Please report a bug on https://github.com/ambv/black/issues.  "
             f"This invalid output might be helpful: {log}"
         ) from None
 

-    src_ast_str = '\n'.join(_v(src_ast))
-    dst_ast_str = '\n'.join(_v(dst_ast))
+    src_ast_str = "\n".join(_v(src_ast))
+    dst_ast_str = "\n".join(_v(dst_ast))

     if src_ast_str != dst_ast_str:
     if src_ast_str != dst_ast_str:

-        log = dump_to_file(diff(src_ast_str, dst_ast_str, 'src', 'dst'))
+        log = dump_to_file(diff(src_ast_str, dst_ast_str, "src", "dst"))

         raise AssertionError(
             f"INTERNAL ERROR: Black produced code that is not equivalent to "
             f"the source.  "
         raise AssertionError(
             f"INTERNAL ERROR: Black produced code that is not equivalent to "
             f"the source.  "


@@ -1724,15 +2130,12 @@ def assert_equivalent(src: str, dst: str) -> None:
 
 
 def assert_stable(src: str, dst: str, line_length: int) -> None:
 
 
 def assert_stable(src: str, dst: str, line_length: int) -> None:

-    """Raises AssertionError if `dst` reformats differently the second time.
-
-    This is a temporary sanity check until Black becomes stable.
-    """
+    """Raise AssertionError if `dst` reformats differently the second time."""

     newdst = format_str(dst, line_length=line_length)
     if dst != newdst:
         log = dump_to_file(
     newdst = format_str(dst, line_length=line_length)
     if dst != newdst:
         log = dump_to_file(

-            diff(src, dst, 'source', 'first pass'),
-            diff(dst, newdst, 'first pass', 'second pass'),
+            diff(src, dst, "source", "first pass"),
+            diff(dst, newdst, "first pass", "second pass"),

         )
         raise AssertionError(
             f"INTERNAL ERROR: Black produced different code on the second pass "
         )
         raise AssertionError(
             f"INTERNAL ERROR: Black produced different code on the second pass "


@@ -1743,28 +2146,58 @@ def assert_stable(src: str, dst: str, line_length: int) -> None:
 
 
 def dump_to_file(*output: str) -> str:
 
 
 def dump_to_file(*output: str) -> str:

-    """Dumps `output` to a temporary file. Returns path to the file."""
+    """Dump `output` to a temporary file. Return path to the file."""

     import tempfile
 
     with tempfile.NamedTemporaryFile(
     import tempfile
 
     with tempfile.NamedTemporaryFile(

-        mode='w', prefix='blk_', suffix='.log', delete=False
+        mode="w", prefix="blk_", suffix=".log", delete=False

     ) as f:
         for lines in output:
             f.write(lines)
     ) as f:
         for lines in output:
             f.write(lines)

-            f.write('\n')
+            if lines and lines[-1] != "\n":
+                f.write("\n")

     return f.name
 
 
 def diff(a: str, b: str, a_name: str, b_name: str) -> str:
     return f.name
 
 
 def diff(a: str, b: str, a_name: str, b_name: str) -> str:

-    """Returns a udiff string between strings `a` and `b`."""
+    """Return a unified diff string between strings `a` and `b`."""

     import difflib
 
     import difflib
 

-    a_lines = [line + '\n' for line in a.split('\n')]
-    b_lines = [line + '\n' for line in b.split('\n')]
-    return ''.join(
+    a_lines = [line + "\n" for line in a.split("\n")]
+    b_lines = [line + "\n" for line in b.split("\n")]
+    return "".join(

         difflib.unified_diff(a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5)
     )
 
 
         difflib.unified_diff(a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5)
     )
 
 

-if __name__ == '__main__':
+def cancel(tasks: List[asyncio.Task]) -> None:
+    """asyncio signal handler that cancels all `tasks` and reports to stderr."""
+    err("Aborted!")
+    for task in tasks:
+        task.cancel()
+
+
+def shutdown(loop: BaseEventLoop) -> None:
+    """Cancel all pending tasks on `loop`, wait for them, and close the loop."""
+    try:
+        # This part is borrowed from asyncio/runners.py in Python 3.7b2.
+        to_cancel = [task for task in asyncio.Task.all_tasks(loop) if not task.done()]
+        if not to_cancel:
+            return
+
+        for task in to_cancel:
+            task.cancel()
+        loop.run_until_complete(
+            asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)
+        )
+    finally:
+        # `concurrent.futures.Future` objects cannot be cancelled once they
+        # are already running. There might be some when the `shutdown()` happened.
+        # Silence their logger's spew about the event loop being closed.
+        cf_logger = logging.getLogger("concurrent.futures")
+        cf_logger.setLevel(logging.CRITICAL)
+        loop.close()
+
+
+if __name__ == "__main__":

     main()
     main()