Add `--verbose` and report excluded paths in it, too

[etc/vim.git] / black.py

diff --git a/black.py b/black.py
index 11386d23f9df76ad9b57129ea2bb28eba4792f5d..b3b37ef3295687460e646679c0683c753e89926a 100644 (file)
--- a/black.py
+++ b/black.py
@@ -1,10 +1,8 @@
-#!/usr/bin/env python3
-

 import asyncio
 import pickle
 from asyncio.base_events import BaseEventLoop
 from concurrent.futures import Executor, ProcessPoolExecutor
 import asyncio
 import pickle
 from asyncio.base_events import BaseEventLoop
 from concurrent.futures import Executor, ProcessPoolExecutor

-from enum import Enum
+from enum import Enum, Flag

 from functools import partial, wraps
 import keyword
 import logging
 from functools import partial, wraps
 import keyword
 import logging


@@ -26,11 +24,13 @@ from typing import (
     List,
     Optional,
     Pattern,
     List,
     Optional,
     Pattern,

+    Sequence,

     Set,
     Tuple,
     Type,
     TypeVar,
     Union,
     Set,
     Tuple,
     Type,
     TypeVar,
     Union,

+    cast,

 )
 
 from appdirs import user_cache_dir
 )
 
 from appdirs import user_cache_dir


@@ -43,10 +43,17 @@ from blib2to3 import pygram, pytree
 from blib2to3.pgen2 import driver, token
 from blib2to3.pgen2.parse import ParseError
 
 from blib2to3.pgen2 import driver, token
 from blib2to3.pgen2.parse import ParseError
 

-__version__ = "18.4a2"
+
+__version__ = "18.5b1"

 DEFAULT_LINE_LENGTH = 88
 DEFAULT_LINE_LENGTH = 88

+DEFAULT_EXCLUDES = (
+    r"/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist)/"
+)
+DEFAULT_INCLUDES = r"\.pyi?$"
+CACHE_DIR = Path(user_cache_dir("black", version=__version__))
+
+

 # types
 # types

-syms = pygram.python_symbols

 FileContent = str
 Encoding = str
 Depth = int
 FileContent = str
 Encoding = str
 Depth = int


@@ -63,6 +70,9 @@ Cache = Dict[Path, CacheInfo]
 out = partial(click.secho, bold=True, err=True)
 err = partial(click.secho, fg="red", err=True)
 
 out = partial(click.secho, bold=True, err=True)
 err = partial(click.secho, fg="red", err=True)
 

+pygram.initialize(CACHE_DIR)
+syms = pygram.python_symbols
+

 
 class NothingChanged(UserWarning):
     """Raised by :func:`format_file` when reformatted code is the same as source."""
 
 class NothingChanged(UserWarning):
     """Raised by :func:`format_file` when reformatted code is the same as source."""


@@ -88,11 +98,11 @@ class FormatError(Exception):
         self.consumed = consumed
 
     def trim_prefix(self, leaf: Leaf) -> None:
         self.consumed = consumed
 
     def trim_prefix(self, leaf: Leaf) -> None:

-        leaf.prefix = leaf.prefix[self.consumed:]
+        leaf.prefix = leaf.prefix[self.consumed :]

 
     def leaf_from_consumed(self, leaf: Leaf) -> Leaf:
         """Returns a new Leaf from the consumed part of the prefix."""
 
     def leaf_from_consumed(self, leaf: Leaf) -> Leaf:
         """Returns a new Leaf from the consumed part of the prefix."""

-        unformatted_prefix = leaf.prefix[:self.consumed]
+        unformatted_prefix = leaf.prefix[: self.consumed]

         return Leaf(token.NEWLINE, unformatted_prefix)
 
 
         return Leaf(token.NEWLINE, unformatted_prefix)
 
 


@@ -109,6 +119,13 @@ class WriteBack(Enum):
     YES = 1
     DIFF = 2
 
     YES = 1
     DIFF = 2
 

+    @classmethod
+    def from_configuration(cls, *, check: bool, diff: bool) -> "WriteBack":
+        if check and not diff:
+            return cls.NO
+
+        return cls.DIFF if diff else cls.YES
+

 
 class Changed(Enum):
     NO = 0
 
 class Changed(Enum):
     NO = 0


@@ -116,6 +133,26 @@ class Changed(Enum):
     YES = 2
 
 
     YES = 2
 
 

+class FileMode(Flag):
+    AUTO_DETECT = 0
+    PYTHON36 = 1
+    PYI = 2
+    NO_STRING_NORMALIZATION = 4
+
+    @classmethod
+    def from_configuration(
+        cls, *, py36: bool, pyi: bool, skip_string_normalization: bool
+    ) -> "FileMode":
+        mode = cls.AUTO_DETECT
+        if py36:
+            mode |= cls.PYTHON36
+        if pyi:
+            mode |= cls.PYI
+        if skip_string_normalization:
+            mode |= cls.NO_STRING_NORMALIZATION
+        return mode
+
+

 @click.command()
 @click.option(
     "-l",
 @click.command()
 @click.option(
     "-l",


@@ -125,6 +162,29 @@ class Changed(Enum):
     help="How many character per line to allow.",
     show_default=True,
 )
     help="How many character per line to allow.",
     show_default=True,
 )

+@click.option(
+    "--py36",
+    is_flag=True,
+    help=(
+        "Allow using Python 3.6-only syntax on all input files.  This will put "
+        "trailing commas in function signatures and calls also after *args and "
+        "**kwargs.  [default: per-file auto-detection]"
+    ),
+)
+@click.option(
+    "--pyi",
+    is_flag=True,
+    help=(
+        "Format all input files like typing stubs regardless of file extension "
+        "(useful when piping source on standard input)."
+    ),
+)
+@click.option(
+    "-S",
+    "--skip-string-normalization",
+    is_flag=True,
+    help="Don't normalize string quotes or prefixes.",
+)

 @click.option(
     "--check",
     is_flag=True,
 @click.option(
     "--check",
     is_flag=True,


@@ -144,6 +204,31 @@ class Changed(Enum):
     is_flag=True,
     help="If --fast given, skip temporary sanity checks. [default: --safe]",
 )
     is_flag=True,
     help="If --fast given, skip temporary sanity checks. [default: --safe]",
 )

+@click.option(
+    "--include",
+    type=str,
+    default=DEFAULT_INCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "included on recursive searches.  An empty value means all files are "
+        "included regardless of the name.  Use forward slashes for directories on "
+        "all platforms (Windows, too).  Exclusions are calculated first, inclusions "
+        "later."
+    ),
+    show_default=True,
+)
+@click.option(
+    "--exclude",
+    type=str,
+    default=DEFAULT_EXCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "excluded on recursive searches.  An empty value means no paths are excluded. "
+        "Use forward slashes for directories on all platforms (Windows, too).  "
+        "Exclusions are calculated first, inclusions later."
+    ),
+    show_default=True,
+)

 @click.option(
     "-q",
     "--quiet",
 @click.option(
     "-q",
     "--quiet",


@@ -153,6 +238,15 @@ class Changed(Enum):
         "silence those with 2>/dev/null."
     ),
 )
         "silence those with 2>/dev/null."
     ),
 )

+@click.option(
+    "-v",
+    "--verbose",
+    is_flag=True,
+    help=(
+        "Also emit messages to stderr about files that were not changed or were "
+        "ignored due to --exclude=."
+    ),
+)

 @click.version_option(version=__version__)
 @click.argument(
     "src",
 @click.version_option(version=__version__)
 @click.argument(
     "src",


@@ -168,43 +262,72 @@ def main(
     check: bool,
     diff: bool,
     fast: bool,
     check: bool,
     diff: bool,
     fast: bool,

+    pyi: bool,
+    py36: bool,
+    skip_string_normalization: bool,

     quiet: bool,
     quiet: bool,

+    verbose: bool,
+    include: str,
+    exclude: str,

     src: List[str],
 ) -> None:
     """The uncompromising code formatter."""
     src: List[str],
 ) -> None:
     """The uncompromising code formatter."""

+    write_back = WriteBack.from_configuration(check=check, diff=diff)
+    mode = FileMode.from_configuration(
+        py36=py36, pyi=pyi, skip_string_normalization=skip_string_normalization
+    )
+    report = Report(check=check, quiet=quiet, verbose=verbose)

     sources: List[Path] = []
     sources: List[Path] = []

+    try:
+        include_regex = re.compile(include)
+    except re.error:
+        err(f"Invalid regular expression for include given: {include!r}")
+        ctx.exit(2)
+    try:
+        exclude_regex = re.compile(exclude)
+    except re.error:
+        err(f"Invalid regular expression for exclude given: {exclude!r}")
+        ctx.exit(2)
+    root = find_project_root(src)

     for s in src:
         p = Path(s)
         if p.is_dir():
     for s in src:
         p = Path(s)
         if p.is_dir():

-            sources.extend(gen_python_files_in_dir(p))
-        elif p.is_file():
+            sources.extend(
+                gen_python_files_in_dir(p, root, include_regex, exclude_regex, report)
+            )
+        elif p.is_file() or s == "-":

             # if a file was explicitly given, we don't care about its extension
             sources.append(p)
             # if a file was explicitly given, we don't care about its extension
             sources.append(p)

-        elif s == "-":
-            sources.append(Path("-"))

         else:
             err(f"invalid path: {s}")
         else:
             err(f"invalid path: {s}")

-
-    if check and not diff:
-        write_back = WriteBack.NO
-    elif diff:
-        write_back = WriteBack.DIFF
-    else:
-        write_back = WriteBack.YES
-    report = Report(check=check, quiet=quiet)

     if len(sources) == 0:
     if len(sources) == 0:

+        out("No paths given. Nothing to do 😴")

         ctx.exit(0)
         return
 
     elif len(sources) == 1:
         ctx.exit(0)
         return
 
     elif len(sources) == 1:

-        reformat_one(sources[0], line_length, fast, write_back, report)
+        reformat_one(
+            src=sources[0],
+            line_length=line_length,
+            fast=fast,
+            write_back=write_back,
+            mode=mode,
+            report=report,
+        )

     else:
         loop = asyncio.get_event_loop()
         executor = ProcessPoolExecutor(max_workers=os.cpu_count())
         try:
             loop.run_until_complete(
                 schedule_formatting(
     else:
         loop = asyncio.get_event_loop()
         executor = ProcessPoolExecutor(max_workers=os.cpu_count())
         try:
             loop.run_until_complete(
                 schedule_formatting(

-                    sources, line_length, fast, write_back, report, loop, executor
+                    sources=sources,
+                    line_length=line_length,
+                    fast=fast,
+                    write_back=write_back,
+                    mode=mode,
+                    report=report,
+                    loop=loop,
+                    executor=executor,

                 )
             )
         finally:
                 )
             )
         finally:


@@ -216,36 +339,43 @@ def main(
 
 
 def reformat_one(
 
 
 def reformat_one(

-    src: Path, line_length: int, fast: bool, write_back: WriteBack, report: "Report"
+    src: Path,
+    line_length: int,
+    fast: bool,
+    write_back: WriteBack,
+    mode: FileMode,
+    report: "Report",

 ) -> None:
     """Reformat a single file under `src` without spawning child processes.
 
     If `quiet` is True, non-error messages are not output. `line_length`,
 ) -> None:
     """Reformat a single file under `src` without spawning child processes.
 
     If `quiet` is True, non-error messages are not output. `line_length`,

-    `write_back`, and `fast` options are passed to :func:`format_file_in_place`.
+    `write_back`, `fast` and `pyi` options are passed to
+    :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.

     """
     try:
         changed = Changed.NO
         if not src.is_file() and str(src) == "-":
             if format_stdin_to_stdout(
     """
     try:
         changed = Changed.NO
         if not src.is_file() and str(src) == "-":
             if format_stdin_to_stdout(

-                line_length=line_length, fast=fast, write_back=write_back
+                line_length=line_length, fast=fast, write_back=write_back, mode=mode

             ):
                 changed = Changed.YES
         else:
             cache: Cache = {}
             if write_back != WriteBack.DIFF:
             ):
                 changed = Changed.YES
         else:
             cache: Cache = {}
             if write_back != WriteBack.DIFF:

-                cache = read_cache()
-                src = src.resolve()
-                if src in cache and cache[src] == get_cache_info(src):
+                cache = read_cache(line_length, mode)
+                res_src = src.resolve()
+                if res_src in cache and cache[res_src] == get_cache_info(res_src):

                     changed = Changed.CACHED
                     changed = Changed.CACHED

-            if (
-                changed is not Changed.CACHED
-                and format_file_in_place(
-                    src, line_length=line_length, fast=fast, write_back=write_back
-                )
+            if changed is not Changed.CACHED and format_file_in_place(
+                src,
+                line_length=line_length,
+                fast=fast,
+                write_back=write_back,
+                mode=mode,

             ):
                 changed = Changed.YES
             ):
                 changed = Changed.YES

-            if write_back != WriteBack.DIFF and changed is not Changed.NO:
-                write_cache(cache, [src])
+            if write_back == WriteBack.YES and changed is not Changed.NO:
+                write_cache(cache, [src], line_length, mode)

         report.done(src, changed)
     except Exception as exc:
         report.failed(src, str(exc))
         report.done(src, changed)
     except Exception as exc:
         report.failed(src, str(exc))


@@ -256,6 +386,7 @@ async def schedule_formatting(
     line_length: int,
     fast: bool,
     write_back: WriteBack,
     line_length: int,
     fast: bool,
     write_back: WriteBack,

+    mode: FileMode,

     report: "Report",
     loop: BaseEventLoop,
     executor: Executor,
     report: "Report",
     loop: BaseEventLoop,
     executor: Executor,


@@ -264,12 +395,12 @@ async def schedule_formatting(
 
     (Use ProcessPoolExecutors for actual parallelism.)
 
 
     (Use ProcessPoolExecutors for actual parallelism.)
 

-    `line_length`, `write_back`, and `fast` options are passed to
+    `line_length`, `write_back`, `fast`, and `pyi` options are passed to

     :func:`format_file_in_place`.
     """
     cache: Cache = {}
     if write_back != WriteBack.DIFF:
     :func:`format_file_in_place`.
     """
     cache: Cache = {}
     if write_back != WriteBack.DIFF:

-        cache = read_cache()
+        cache = read_cache(line_length, mode)

         sources, cached = filter_cached(cache, sources)
         for src in cached:
             report.done(src, Changed.CACHED)
         sources, cached = filter_cached(cache, sources)
         for src in cached:
             report.done(src, Changed.CACHED)


@@ -283,36 +414,40 @@ async def schedule_formatting(
             manager = Manager()
             lock = manager.Lock()
         tasks = {
             manager = Manager()
             lock = manager.Lock()
         tasks = {

-            src: loop.run_in_executor(
-                executor, format_file_in_place, src, line_length, fast, write_back, lock
-            )
-            for src in sources
+            loop.run_in_executor(
+                executor,
+                format_file_in_place,
+                src,
+                line_length,
+                fast,
+                write_back,
+                mode,
+                lock,
+            ): src
+            for src in sorted(sources)

         }
         }

-        _task_values = list(tasks.values())
+        pending: Iterable[asyncio.Task] = tasks.keys()

         try:
         try:

-            loop.add_signal_handler(signal.SIGINT, cancel, _task_values)
-            loop.add_signal_handler(signal.SIGTERM, cancel, _task_values)
+            loop.add_signal_handler(signal.SIGINT, cancel, pending)
+            loop.add_signal_handler(signal.SIGTERM, cancel, pending)

         except NotImplementedError:
             # There are no good alternatives for these on Windows
             pass
         except NotImplementedError:
             # There are no good alternatives for these on Windows
             pass

-        await asyncio.wait(_task_values)
-        for src, task in tasks.items():
-            if not task.done():
-                report.failed(src, "timed out, cancelling")
-                task.cancel()
-                cancelled.append(task)
-            elif task.cancelled():
-                cancelled.append(task)
-            elif task.exception():
-                report.failed(src, str(task.exception()))
-            else:
-                formatted.append(src)
-                report.done(src, Changed.YES if task.result() else Changed.NO)
-
+        while pending:
+            done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
+            for task in done:
+                src = tasks.pop(task)
+                if task.cancelled():
+                    cancelled.append(task)
+                elif task.exception():
+                    report.failed(src, str(task.exception()))
+                else:
+                    formatted.append(src)
+                    report.done(src, Changed.YES if task.result() else Changed.NO)

     if cancelled:
         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
     if cancelled:
         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)

-    if write_back != WriteBack.DIFF and formatted:
-        write_cache(cache, formatted)
+    if write_back == WriteBack.YES and formatted:
+        write_cache(cache, formatted, line_length, mode)

 
 
 def format_file_in_place(
 
 
 def format_file_in_place(


@@ -320,6 +455,7 @@ def format_file_in_place(
     line_length: int,
     fast: bool,
     write_back: WriteBack = WriteBack.NO,
     line_length: int,
     fast: bool,
     write_back: WriteBack = WriteBack.NO,

+    mode: FileMode = FileMode.AUTO_DETECT,

     lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy
 ) -> bool:
     """Format file under `src` path. Return True if changed.
     lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy
 ) -> bool:
     """Format file under `src` path. Return True if changed.


@@ -327,12 +463,13 @@ def format_file_in_place(
     If `write_back` is True, write reformatted code back to stdout.
     `line_length` and `fast` options are passed to :func:`format_file_contents`.
     """
     If `write_back` is True, write reformatted code back to stdout.
     `line_length` and `fast` options are passed to :func:`format_file_contents`.
     """

-
+    if src.suffix == ".pyi":
+        mode |= FileMode.PYI

     with tokenize.open(src) as src_buffer:
         src_contents = src_buffer.read()
     try:
         dst_contents = format_file_contents(
     with tokenize.open(src) as src_buffer:
         src_contents = src_buffer.read()
     try:
         dst_contents = format_file_contents(

-            src_contents, line_length=line_length, fast=fast
+            src_contents, line_length=line_length, fast=fast, mode=mode

         )
     except NothingChanged:
         return False
         )
     except NothingChanged:
         return False


@@ -355,17 +492,21 @@ def format_file_in_place(
 
 
 def format_stdin_to_stdout(
 
 
 def format_stdin_to_stdout(

-    line_length: int, fast: bool, write_back: WriteBack = WriteBack.NO
+    line_length: int,
+    fast: bool,
+    write_back: WriteBack = WriteBack.NO,
+    mode: FileMode = FileMode.AUTO_DETECT,

 ) -> bool:
     """Format file on stdin. Return True if changed.
 
     If `write_back` is True, write reformatted code back to stdout.
 ) -> bool:
     """Format file on stdin. Return True if changed.
 
     If `write_back` is True, write reformatted code back to stdout.

-    `line_length` and `fast` arguments are passed to :func:`format_file_contents`.
+    `line_length`, `fast`, `is_pyi`, and `force_py36` arguments are passed to
+    :func:`format_file_contents`.

     """
     src = sys.stdin.read()
     dst = src
     try:
     """
     src = sys.stdin.read()
     dst = src
     try:

-        dst = format_file_contents(src, line_length=line_length, fast=fast)
+        dst = format_file_contents(src, line_length=line_length, fast=fast, mode=mode)

         return True
 
     except NothingChanged:
         return True
 
     except NothingChanged:


@@ -381,7 +522,11 @@ def format_stdin_to_stdout(
 
 
 def format_file_contents(
 
 
 def format_file_contents(

-    src_contents: str, line_length: int, fast: bool
+    src_contents: str,
+    *,
+    line_length: int,
+    fast: bool,
+    mode: FileMode = FileMode.AUTO_DETECT,

 ) -> FileContent:
     """Reformat contents a file and return new contents.
 
 ) -> FileContent:
     """Reformat contents a file and return new contents.
 


@@ -392,26 +537,35 @@ def format_file_contents(
     if src_contents.strip() == "":
         raise NothingChanged
 
     if src_contents.strip() == "":
         raise NothingChanged
 

-    dst_contents = format_str(src_contents, line_length=line_length)
+    dst_contents = format_str(src_contents, line_length=line_length, mode=mode)

     if src_contents == dst_contents:
         raise NothingChanged
 
     if not fast:
         assert_equivalent(src_contents, dst_contents)
     if src_contents == dst_contents:
         raise NothingChanged
 
     if not fast:
         assert_equivalent(src_contents, dst_contents)

-        assert_stable(src_contents, dst_contents, line_length=line_length)
+        assert_stable(src_contents, dst_contents, line_length=line_length, mode=mode)

     return dst_contents
 
 
     return dst_contents
 
 

-def format_str(src_contents: str, line_length: int) -> FileContent:
+def format_str(
+    src_contents: str, line_length: int, *, mode: FileMode = FileMode.AUTO_DETECT
+) -> FileContent:

     """Reformat a string and return new contents.
 
     `line_length` determines how many characters per line are allowed.
     """
     src_node = lib2to3_parse(src_contents)
     dst_contents = ""
     """Reformat a string and return new contents.
 
     `line_length` determines how many characters per line are allowed.
     """
     src_node = lib2to3_parse(src_contents)
     dst_contents = ""

-    lines = LineGenerator()
-    elt = EmptyLineTracker()
-    py36 = is_python36(src_node)
+    future_imports = get_future_imports(src_node)
+    is_pyi = bool(mode & FileMode.PYI)
+    py36 = bool(mode & FileMode.PYTHON36) or is_python36(src_node)
+    normalize_strings = not bool(mode & FileMode.NO_STRING_NORMALIZATION)
+    lines = LineGenerator(
+        remove_u_prefix=py36 or "unicode_literals" in future_imports,
+        is_pyi=is_pyi,
+        normalize_strings=normalize_strings,
+    )
+    elt = EmptyLineTracker(is_pyi=is_pyi)

     empty_line = Line()
     after = 0
     for current_line in lines.visit(src_node):
     empty_line = Line()
     after = 0
     for current_line in lines.visit(src_node):


@@ -553,19 +707,20 @@ COMPARATORS = {
     token.GREATEREQUAL,
 }
 MATH_OPERATORS = {
     token.GREATEREQUAL,
 }
 MATH_OPERATORS = {

+    token.VBAR,
+    token.CIRCUMFLEX,
+    token.AMPER,
+    token.LEFTSHIFT,
+    token.RIGHTSHIFT,

     token.PLUS,
     token.MINUS,
     token.STAR,
     token.SLASH,
     token.PLUS,
     token.MINUS,
     token.STAR,
     token.SLASH,

-    token.VBAR,
-    token.AMPER,
+    token.DOUBLESLASH,

     token.PERCENT,
     token.PERCENT,

-    token.CIRCUMFLEX,
+    token.AT,

     token.TILDE,
     token.TILDE,

-    token.LEFTSHIFT,
-    token.RIGHTSHIFT,

     token.DOUBLESTAR,
     token.DOUBLESTAR,

-    token.DOUBLESLASH,

 }
 STARS = {token.STAR, token.DOUBLESTAR}
 VARARGS_PARENTS = {
 }
 STARS = {token.STAR, token.DOUBLESTAR}
 VARARGS_PARENTS = {


@@ -581,12 +736,62 @@ UNPACKING_PARENTS = {
     syms.listmaker,
     syms.testlist_gexp,
 }
     syms.listmaker,
     syms.testlist_gexp,
 }

+TEST_DESCENDANTS = {
+    syms.test,
+    syms.lambdef,
+    syms.or_test,
+    syms.and_test,
+    syms.not_test,
+    syms.comparison,
+    syms.star_expr,
+    syms.expr,
+    syms.xor_expr,
+    syms.and_expr,
+    syms.shift_expr,
+    syms.arith_expr,
+    syms.trailer,
+    syms.term,
+    syms.power,
+}
+ASSIGNMENTS = {
+    "=",
+    "+=",
+    "-=",
+    "*=",
+    "@=",
+    "/=",
+    "%=",
+    "&=",
+    "|=",
+    "^=",
+    "<<=",
+    ">>=",
+    "**=",
+    "//=",
+}

 COMPREHENSION_PRIORITY = 20
 COMPREHENSION_PRIORITY = 20

-COMMA_PRIORITY = 10
-LOGIC_PRIORITY = 5
-STRING_PRIORITY = 4
-COMPARATOR_PRIORITY = 3
-MATH_PRIORITY = 1
+COMMA_PRIORITY = 18
+TERNARY_PRIORITY = 16
+LOGIC_PRIORITY = 14
+STRING_PRIORITY = 12
+COMPARATOR_PRIORITY = 10
+MATH_PRIORITIES = {
+    token.VBAR: 9,
+    token.CIRCUMFLEX: 8,
+    token.AMPER: 7,
+    token.LEFTSHIFT: 6,
+    token.RIGHTSHIFT: 6,
+    token.PLUS: 5,
+    token.MINUS: 5,
+    token.STAR: 4,
+    token.SLASH: 4,
+    token.DOUBLESLASH: 4,
+    token.PERCENT: 4,
+    token.AT: 4,
+    token.TILDE: 3,
+    token.DOUBLESTAR: 2,
+}
+DOT_PRIORITY = 1

 
 
 @dataclass
 
 
 @dataclass


@@ -597,8 +802,8 @@ class BracketTracker:
     bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
     delimiters: Dict[LeafID, Priority] = Factory(dict)
     previous: Optional[Leaf] = None
     bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
     delimiters: Dict[LeafID, Priority] = Factory(dict)
     previous: Optional[Leaf] = None

-    _for_loop_variable: bool = False
-    _lambda_arguments: bool = False
+    _for_loop_variable: int = 0
+    _lambda_arguments: int = 0

 
     def mark(self, leaf: Leaf) -> None:
         """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
 
     def mark(self, leaf: Leaf) -> None:
         """Mark `leaf` with bracket-related metadata. Keep track of delimiters.


@@ -652,6 +857,17 @@ class BracketTracker:
         """
         return max(v for k, v in self.delimiters.items() if k not in exclude)
 
         """
         return max(v for k, v in self.delimiters.items() if k not in exclude)
 

+    def delimiter_count_with_priority(self, priority: int = 0) -> int:
+        """Return the number of delimiters with the given `priority`.
+
+        If no `priority` is passed, defaults to max priority on the line.
+        """
+        if not self.delimiters:
+            return 0
+
+        priority = priority or self.max_delimiter_priority()
+        return sum(1 for p in self.delimiters.values() if p == priority)
+

     def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
         """In a for loop, or comprehension, the variables are often unpacks.
 
     def maybe_increment_for_loop_variable(self, leaf: Leaf) -> bool:
         """In a for loop, or comprehension, the variables are often unpacks.
 


@@ -660,7 +876,7 @@ class BracketTracker:
         """
         if leaf.type == token.NAME and leaf.value == "for":
             self.depth += 1
         """
         if leaf.type == token.NAME and leaf.value == "for":
             self.depth += 1

-            self._for_loop_variable = True
+            self._for_loop_variable += 1

             return True
 
         return False
             return True
 
         return False


@@ -669,7 +885,7 @@ class BracketTracker:
         """See `maybe_increment_for_loop_variable` above for explanation."""
         if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":
             self.depth -= 1
         """See `maybe_increment_for_loop_variable` above for explanation."""
         if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":
             self.depth -= 1

-            self._for_loop_variable = False
+            self._for_loop_variable -= 1

             return True
 
         return False
             return True
 
         return False


@@ -682,7 +898,7 @@ class BracketTracker:
         """
         if leaf.type == token.NAME and leaf.value == "lambda":
             self.depth += 1
         """
         if leaf.type == token.NAME and leaf.value == "lambda":
             self.depth += 1

-            self._lambda_arguments = True
+            self._lambda_arguments += 1

             return True
 
         return False
             return True
 
         return False


@@ -691,11 +907,15 @@ class BracketTracker:
         """See `maybe_increment_lambda_arguments` above for explanation."""
         if self._lambda_arguments and leaf.type == token.COLON:
             self.depth -= 1
         """See `maybe_increment_lambda_arguments` above for explanation."""
         if self._lambda_arguments and leaf.type == token.COLON:
             self.depth -= 1

-            self._lambda_arguments = False
+            self._lambda_arguments -= 1

             return True
 
         return False
 
             return True
 
         return False
 

+    def get_open_lsqb(self) -> Optional[Leaf]:
+        """Return the most recent opening square bracket (if any)."""
+        return self.bracket_match.get((self.depth - 1, token.RSQB))
+

 
 @dataclass
 class Line:
 
 @dataclass
 class Line:


@@ -706,6 +926,7 @@ class Line:
     comments: List[Tuple[Index, Leaf]] = Factory(list)
     bracket_tracker: BracketTracker = Factory(BracketTracker)
     inside_brackets: bool = False
     comments: List[Tuple[Index, Leaf]] = Factory(list)
     bracket_tracker: BracketTracker = Factory(BracketTracker)
     inside_brackets: bool = False

+    should_explode: bool = False

 
     def append(self, leaf: Leaf, preformatted: bool = False) -> None:
         """Add a new `leaf` to the end of the line.
 
     def append(self, leaf: Leaf, preformatted: bool = False) -> None:
         """Add a new `leaf` to the end of the line.


@@ -721,14 +942,17 @@ class Line:
         if not has_value:
             return
 
         if not has_value:
             return
 

+        if token.COLON == leaf.type and self.is_class_paren_empty:
+            del self.leaves[-2:]

         if self.leaves and not preformatted:
             # Note: at this point leaf.prefix should be empty except for
             # imports, for which we only preserve newlines.
         if self.leaves and not preformatted:
             # Note: at this point leaf.prefix should be empty except for
             # imports, for which we only preserve newlines.

-            leaf.prefix += whitespace(leaf)
+            leaf.prefix += whitespace(
+                leaf, complex_subscript=self.is_complex_subscript(leaf)
+            )

         if self.inside_brackets or not preformatted:
             self.bracket_tracker.mark(leaf)
             self.maybe_remove_trailing_comma(leaf)
         if self.inside_brackets or not preformatted:
             self.bracket_tracker.mark(leaf)
             self.maybe_remove_trailing_comma(leaf)

-

         if not self.append_comment(leaf):
             self.leaves.append(leaf)
 
         if not self.append_comment(leaf):
             self.leaves.append(leaf)
 


@@ -773,6 +997,13 @@ class Line:
             and self.leaves[0].value == "class"
         )
 
             and self.leaves[0].value == "class"
         )
 

+    @property
+    def is_stub_class(self) -> bool:
+        """Is this line a class definition with a body consisting only of "..."?"""
+        return self.is_class and self.leaves[-3:] == [
+            Leaf(token.DOT, ".") for _ in range(3)
+        ]
+

     @property
     def is_def(self) -> bool:
         """Is this a function definition? (Also returns True for async defs.)"""
     @property
     def is_def(self) -> bool:
         """Is this a function definition? (Also returns True for async defs.)"""


@@ -785,35 +1016,36 @@ class Line:
             second_leaf: Optional[Leaf] = self.leaves[1]
         except IndexError:
             second_leaf = None
             second_leaf: Optional[Leaf] = self.leaves[1]
         except IndexError:
             second_leaf = None

-        return (
-            (first_leaf.type == token.NAME and first_leaf.value == "def")
-            or (
-                first_leaf.type == token.ASYNC
-                and second_leaf is not None
-                and second_leaf.type == token.NAME
-                and second_leaf.value == "def"
-            )
+        return (first_leaf.type == token.NAME and first_leaf.value == "def") or (
+            first_leaf.type == token.ASYNC
+            and second_leaf is not None
+            and second_leaf.type == token.NAME
+            and second_leaf.value == "def"

         )
 
     @property
         )
 
     @property

-    def is_flow_control(self) -> bool:
-        """Is this line a flow control statement?
+    def is_class_paren_empty(self) -> bool:
+        """Is this a class with no base classes but using parentheses?

 
 

-        Those are `return`, `raise`, `break`, and `continue`.
+        Those are unnecessary and should be removed.

         """
         return (
             bool(self)
         """
         return (
             bool(self)

-            and self.leaves[0].type == token.NAME
-            and self.leaves[0].value in FLOW_CONTROL
+            and len(self.leaves) == 4
+            and self.is_class
+            and self.leaves[2].type == token.LPAR
+            and self.leaves[2].value == "("
+            and self.leaves[3].type == token.RPAR
+            and self.leaves[3].value == ")"

         )
 
     @property
         )
 
     @property

-    def is_yield(self) -> bool:
-        """Is this line a yield statement?"""
+    def is_triple_quoted_string(self) -> bool:
+        """Is the line a triple quoted string?"""

         return (
             bool(self)
         return (
             bool(self)

-            and self.leaves[0].type == token.NAME
-            and self.leaves[0].value == "yield"
+            and self.leaves[0].type == token.STRING
+            and self.leaves[0].value.startswith(('"""', "'''"))

         )
 
     def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
         )
 
     def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:


@@ -844,9 +1076,14 @@ class Line:
                 self.remove_trailing_comma()
                 return True
 
                 self.remove_trailing_comma()
                 return True
 

-        # For parens let's check if it's safe to remove the comma.  If the
-        # trailing one is the only one, we might mistakenly change a tuple
-        # into a different type by removing the comma.
+        # For parens let's check if it's safe to remove the comma.
+        # Imports are always safe.
+        if self.is_import:
+            self.remove_trailing_comma()
+            return True
+
+        # Otheriwsse, if the trailing one is the only one, we might mistakenly
+        # change a tuple into a different type by removing the comma.

         depth = closing.bracket_depth + 1
         commas = 0
         opening = closing.opening_bracket
         depth = closing.bracket_depth + 1
         commas = 0
         opening = closing.opening_bracket


@@ -857,7 +1094,7 @@ class Line:
         else:
             return False
 
         else:
             return False
 

-        for leaf in self.leaves[_opening_index + 1:]:
+        for leaf in self.leaves[_opening_index + 1 :]:

             if leaf is closing:
                 break
 
             if leaf is closing:
                 break
 


@@ -896,17 +1133,21 @@ class Line:
             self.comments.append((after, comment))
             return True
 
             self.comments.append((after, comment))
             return True
 

-    def comments_after(self, leaf: Leaf) -> Iterator[Leaf]:
-        """Generate comments that should appear directly after `leaf`."""
-        for _leaf_index, _leaf in enumerate(self.leaves):
-            if leaf is _leaf:
-                break
+    def comments_after(self, leaf: Leaf, _index: int = -1) -> Iterator[Leaf]:
+        """Generate comments that should appear directly after `leaf`.

 
 

-        else:
-            return
+        Provide a non-negative leaf `_index` to speed up the function.
+        """
+        if _index == -1:
+            for _index, _leaf in enumerate(self.leaves):
+                if leaf is _leaf:
+                    break
+
+            else:
+                return

 
         for index, comment_after in self.comments:
 
         for index, comment_after in self.comments:

-            if _leaf_index == index:
+            if _index == index:

                 yield comment_after
 
     def remove_trailing_comma(self) -> None:
                 yield comment_after
 
     def remove_trailing_comma(self) -> None:


@@ -918,6 +1159,24 @@ class Line:
                 self.comments[i] = (comma_index - 1, comment)
         self.leaves.pop()
 
                 self.comments[i] = (comma_index - 1, comment)
         self.leaves.pop()
 

+    def is_complex_subscript(self, leaf: Leaf) -> bool:
+        """Return True iff `leaf` is part of a slice with non-trivial exprs."""
+        open_lsqb = (
+            leaf if leaf.type == token.LSQB else self.bracket_tracker.get_open_lsqb()
+        )
+        if open_lsqb is None:
+            return False
+
+        subscript_start = open_lsqb.next_sibling
+        if (
+            isinstance(subscript_start, Node)
+            and subscript_start.type == syms.subscriptlist
+        ):
+            subscript_start = child_towards(subscript_start, leaf)
+        return subscript_start is not None and any(
+            n.type in TEST_DESCENDANTS for n in subscript_start.pre_order()
+        )
+

     def __str__(self) -> str:
         """Render the line."""
         if not self:
     def __str__(self) -> str:
         """Render the line."""
         if not self:


@@ -996,6 +1255,8 @@ class EmptyLineTracker:
     the prefix of the first leaf consists of optional newlines.  Those newlines
     are consumed by `maybe_empty_lines()` and included in the computation.
     """
     the prefix of the first leaf consists of optional newlines.  Those newlines
     are consumed by `maybe_empty_lines()` and included in the computation.
     """

+
+    is_pyi: bool = False

     previous_line: Optional[Line] = None
     previous_after: int = 0
     previous_defs: List[int] = Factory(list)
     previous_line: Optional[Line] = None
     previous_after: int = 0
     previous_defs: List[int] = Factory(list)


@@ -1004,8 +1265,7 @@ class EmptyLineTracker:
         """Return the number of extra empty lines before and after the `current_line`.
 
         This is for separating `def`, `async def` and `class` with extra empty
         """Return the number of extra empty lines before and after the `current_line`.
 
         This is for separating `def`, `async def` and `class` with extra empty

-        lines (two on module-level), as well as providing an extra empty line
-        after flow control keywords to make them more prominent.
+        lines (two on module-level).

         """
         if isinstance(current_line, UnformattedLines):
             return 0, 0
         """
         if isinstance(current_line, UnformattedLines):
             return 0, 0


@@ -1019,7 +1279,7 @@ class EmptyLineTracker:
     def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
         max_allowed = 1
         if current_line.depth == 0:
     def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
         max_allowed = 1
         if current_line.depth == 0:

-            max_allowed = 2
+            max_allowed = 1 if self.is_pyi else 2

         if current_line.leaves:
             # Consume the first leaf's extra newlines.
             first_leaf = current_line.leaves[0]
         if current_line.leaves:
             # Consume the first leaf's extra newlines.
             first_leaf = current_line.leaves[0]


@@ -1031,7 +1291,10 @@ class EmptyLineTracker:
         depth = current_line.depth
         while self.previous_defs and self.previous_defs[-1] >= depth:
             self.previous_defs.pop()
         depth = current_line.depth
         while self.previous_defs and self.previous_defs[-1] >= depth:
             self.previous_defs.pop()

-            before = 1 if depth else 2
+            if self.is_pyi:
+                before = 0 if depth else 1
+            else:
+                before = 1 if depth else 2

         is_decorator = current_line.is_decorator
         if is_decorator or current_line.is_def or current_line.is_class:
             if not is_decorator:
         is_decorator = current_line.is_decorator
         if is_decorator or current_line.is_def or current_line.is_class:
             if not is_decorator:


@@ -1043,6 +1306,11 @@ class EmptyLineTracker:
             if self.previous_line.is_decorator:
                 return 0, 0
 
             if self.previous_line.is_decorator:
                 return 0, 0
 

+            if self.previous_line.depth < current_line.depth and (
+                self.previous_line.is_class or self.previous_line.is_def
+            ):
+                return 0, 0
+

             if (
                 self.previous_line.is_comment
                 and self.previous_line.depth == current_line.depth
             if (
                 self.previous_line.is_comment
                 and self.previous_line.depth == current_line.depth


@@ -1050,14 +1318,22 @@ class EmptyLineTracker:
             ):
                 return 0, 0
 
             ):
                 return 0, 0
 

-            newlines = 2
-            if current_line.depth:
+            if self.is_pyi:
+                if self.previous_line.depth > current_line.depth:
+                    newlines = 1
+                elif current_line.is_class or self.previous_line.is_class:
+                    if current_line.is_stub_class and self.previous_line.is_stub_class:
+                        newlines = 0
+                    else:
+                        newlines = 1
+                else:
+                    newlines = 0
+            else:
+                newlines = 2
+            if current_line.depth and newlines:

                 newlines -= 1
             return newlines, 0
 
                 newlines -= 1
             return newlines, 0
 

-        if current_line.is_flow_control:
-            return before, 1
-

         if (
             self.previous_line
             and self.previous_line.is_import
         if (
             self.previous_line
             and self.previous_line.is_import


@@ -1068,10 +1344,10 @@ class EmptyLineTracker:
 
         if (
             self.previous_line
 
         if (
             self.previous_line

-            and self.previous_line.is_yield
-            and (not current_line.is_yield or depth != self.previous_line.depth)
+            and self.previous_line.is_class
+            and current_line.is_triple_quoted_string

         ):
         ):

-            return (before or 1), 0
+            return before, 1

 
         return before, 0
 
 
         return before, 0
 


@@ -1083,7 +1359,11 @@ class LineGenerator(Visitor[Line]):
     Note: destroys the tree it's visiting by mutating prefixes of its leaves
     in ways that will no longer stringify to valid Python code on the tree.
     """
     Note: destroys the tree it's visiting by mutating prefixes of its leaves
     in ways that will no longer stringify to valid Python code on the tree.
     """

+
+    is_pyi: bool = False
+    normalize_strings: bool = True

     current_line: Line = Factory(Line)
     current_line: Line = Factory(Line)

+    remove_u_prefix: bool = False

 
     def line(self, indent: int = 0, type: Type[Line] = Line) -> Iterator[Line]:
         """Generate a line.
 
     def line(self, indent: int = 0, type: Type[Line] = Line) -> Iterator[Line]:
         """Generate a line.


@@ -1150,7 +1430,8 @@ class LineGenerator(Visitor[Line]):
 
             else:
                 normalize_prefix(node, inside_brackets=any_open_brackets)
 
             else:
                 normalize_prefix(node, inside_brackets=any_open_brackets)

-                if node.type == token.STRING:
+                if self.normalize_strings and node.type == token.STRING:
+                    normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)

                     normalize_string_quotes(node)
                 if node.type not in WHITESPACE:
                     self.current_line.append(node)
                     normalize_string_quotes(node)
                 if node.type not in WHITESPACE:
                     self.current_line.append(node)


@@ -1182,14 +1463,13 @@ class LineGenerator(Visitor[Line]):
         """Visit a statement.
 
         This implementation is shared for `if`, `while`, `for`, `try`, `except`,
         """Visit a statement.
 
         This implementation is shared for `if`, `while`, `for`, `try`, `except`,

-        `def`, `with`, `class`, and `assert`.
+        `def`, `with`, `class`, `assert` and assignments.

 
         The relevant Python language `keywords` for a given statement will be
         NAME leaves within it. This methods puts those on a separate line.
 
 
         The relevant Python language `keywords` for a given statement will be
         NAME leaves within it. This methods puts those on a separate line.
 

-        `parens` holds pairs of nodes where invisible parentheses should be put.
-        Keys hold nodes after which opening parentheses should be put, values
-        hold nodes before which closing parentheses should be put.
+        `parens` holds a set of string leaf values immediately after which
+        invisible parens should be put.

         """
         normalize_invisible_parens(node, parens_after=parens)
         for child in node.children:
         """
         normalize_invisible_parens(node, parens_after=parens)
         for child in node.children:


@@ -1198,16 +1478,27 @@ class LineGenerator(Visitor[Line]):
 
             yield from self.visit(child)
 
 
             yield from self.visit(child)
 

+    def visit_suite(self, node: Node) -> Iterator[Line]:
+        """Visit a suite."""
+        if self.is_pyi and is_stub_suite(node):
+            yield from self.visit(node.children[2])
+        else:
+            yield from self.visit_default(node)
+

     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
         """Visit a statement without nested statements."""
         is_suite_like = node.parent and node.parent.type in STATEMENT
         if is_suite_like:
     def visit_simple_stmt(self, node: Node) -> Iterator[Line]:
         """Visit a statement without nested statements."""
         is_suite_like = node.parent and node.parent.type in STATEMENT
         if is_suite_like:

-            yield from self.line(+1)
-            yield from self.visit_default(node)
-            yield from self.line(-1)
+            if self.is_pyi and is_stub_body(node):
+                yield from self.visit_default(node)
+            else:
+                yield from self.line(+1)
+                yield from self.visit_default(node)
+                yield from self.line(-1)

 
         else:
 
         else:

-            yield from self.line()
+            if not self.is_pyi or not node.parent or not is_stub_suite(node.parent):
+                yield from self.line()

             yield from self.visit_default(node)
 
     def visit_async_stmt(self, node: Node) -> Iterator[Line]:
             yield from self.visit_default(node)
 
     def visit_async_stmt(self, node: Node) -> Iterator[Line]:


@@ -1231,32 +1522,6 @@ class LineGenerator(Visitor[Line]):
             yield from self.line()
             yield from self.visit(child)
 
             yield from self.line()
             yield from self.visit(child)
 

-    def visit_import_from(self, node: Node) -> Iterator[Line]:
-        """Visit import_from and maybe put invisible parentheses.
-
-        This is separate from `visit_stmt` because import statements don't
-        support arbitrary atoms and thus handling of parentheses is custom.
-        """
-        check_lpar = False
-        for index, child in enumerate(node.children):
-            if check_lpar:
-                if child.type == token.LPAR:
-                    # make parentheses invisible
-                    child.value = ""  # type: ignore
-                    node.children[-1].value = ""  # type: ignore
-                else:
-                    # insert invisible parentheses
-                    node.insert_child(index, Leaf(token.LPAR, ""))
-                    node.append_child(Leaf(token.RPAR, ""))
-                break
-
-            check_lpar = (
-                child.type == token.NAME and child.value == "import"  # type: ignore
-            )
-
-        for child in node.children:
-            yield from self.visit(child)
-

     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
         """Remove a semicolon and put the other statement on a separate line."""
         yield from self.line()
     def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:
         """Remove a semicolon and put the other statement on a separate line."""
         yield from self.line()


@@ -1289,7 +1554,9 @@ class LineGenerator(Visitor[Line]):
         v = self.visit_stmt
         Ø: Set[str] = set()
         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})
         v = self.visit_stmt
         Ø: Set[str] = set()
         self.visit_assert_stmt = partial(v, keywords={"assert"}, parens={"assert", ","})

-        self.visit_if_stmt = partial(v, keywords={"if", "else", "elif"}, parens={"if"})
+        self.visit_if_stmt = partial(
+            v, keywords={"if", "else", "elif"}, parens={"if", "elif"}
+        )

         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
         self.visit_try_stmt = partial(
         self.visit_while_stmt = partial(v, keywords={"while", "else"}, parens={"while"})
         self.visit_for_stmt = partial(v, keywords={"for", "else"}, parens={"for", "in"})
         self.visit_try_stmt = partial(


@@ -1299,6 +1566,9 @@ class LineGenerator(Visitor[Line]):
         self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
         self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)
         self.visit_with_stmt = partial(v, keywords={"with"}, parens=Ø)
         self.visit_funcdef = partial(v, keywords={"def"}, parens=Ø)
         self.visit_classdef = partial(v, keywords={"class"}, parens=Ø)

+        self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS)
+        self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"})
+        self.visit_import_from = partial(v, keywords=Ø, parens={"import"})

         self.visit_async_funcdef = self.visit_async_stmt
         self.visit_decorated = self.visit_decorators
 
         self.visit_async_funcdef = self.visit_async_stmt
         self.visit_decorated = self.visit_decorators
 


@@ -1311,8 +1581,12 @@ BRACKETS = OPENING_BRACKETS | CLOSING_BRACKETS
 ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
 
 
 ALWAYS_NO_SPACE = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}
 
 

-def whitespace(leaf: Leaf) -> str:  # noqa C901
-    """Return whitespace prefix if needed for the given `leaf`."""
+def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str:  # noqa C901
+    """Return whitespace prefix if needed for the given `leaf`.
+
+    `complex_subscript` signals whether the given leaf is part of a subscription
+    which has non-trivial arguments, like arithmetic expressions or function calls.
+    """

     NO = ""
     SPACE = " "
     DOUBLESPACE = "  "
     NO = ""
     SPACE = " "
     DOUBLESPACE = "  "


@@ -1326,7 +1600,11 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
         return DOUBLESPACE
 
     assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}"
         return DOUBLESPACE
 
     assert p is not None, f"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}"

-    if t == token.COLON and p.type not in {syms.subscript, syms.subscriptlist}:
+    if t == token.COLON and p.type not in {
+        syms.subscript,
+        syms.subscriptlist,
+        syms.sliceop,
+    }:

         return NO
 
     prev = leaf.prev_sibling
         return NO
 
     prev = leaf.prev_sibling


@@ -1336,12 +1614,21 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
             return NO
 
         if t == token.COLON:
             return NO
 
         if t == token.COLON:

-            return SPACE if prevp.type == token.COMMA else NO
+            if prevp.type == token.COLON:
+                return NO
+
+            elif prevp.type != token.COMMA and not complex_subscript:
+                return NO
+
+            return SPACE

 
         if prevp.type == token.EQUAL:
             if prevp.parent:
                 if prevp.parent.type in {
 
         if prevp.type == token.EQUAL:
             if prevp.parent:
                 if prevp.parent.type in {

-                    syms.arglist, syms.argument, syms.parameters, syms.varargslist
+                    syms.arglist,
+                    syms.argument,
+                    syms.parameters,
+                    syms.varargslist,

                 }:
                     return NO
 
                 }:
                     return NO
 


@@ -1357,7 +1644,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
 
         elif prevp.type == token.COLON:
             if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:
 
         elif prevp.type == token.COLON:
             if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:

-                return NO
+                return SPACE if complex_subscript else NO

 
         elif (
             prevp.parent
 
         elif (
             prevp.parent


@@ -1463,7 +1750,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
         if prev and prev.type == token.LPAR:
             return NO
 
         if prev and prev.type == token.LPAR:
             return NO
 

-    elif p.type == syms.subscript:
+    elif p.type in {syms.subscript, syms.sliceop}:

         # indexing
         if not prev:
             assert p.parent is not None, "subscripts are always parented"
         # indexing
         if not prev:
             assert p.parent is not None, "subscripts are always parented"


@@ -1472,7 +1759,7 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
 
             return NO
 
 
             return NO
 

-        else:
+        elif not complex_subscript:

             return NO
 
     elif p.type == syms.atom:
             return NO
 
     elif p.type == syms.atom:


@@ -1494,10 +1781,10 @@ def whitespace(leaf: Leaf) -> str:  # noqa C901
 
             prevp_parent = prevp.parent
             assert prevp_parent is not None
 
             prevp_parent = prevp.parent
             assert prevp_parent is not None

-            if (
-                prevp.type == token.COLON
-                and prevp_parent.type in {syms.subscript, syms.sliceop}
-            ):
+            if prevp.type == token.COLON and prevp_parent.type in {
+                syms.subscript,
+                syms.sliceop,
+            }:

                 return NO
 
             elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:
                 return NO
 
             elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:


@@ -1542,6 +1829,14 @@ def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:
     return None
 
 
     return None
 
 

+def child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:
+    """Return the child of `ancestor` that contains `descendant`."""
+    node: Optional[LN] = descendant
+    while node and node.parent != ancestor:
+        node = node.parent
+    return node
+
+

 def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
     """Return the priority of the `leaf` delimiter, given a line break after it.
 
 def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
     """Return the priority of the `leaf` delimiter, given a line break after it.
 


@@ -1569,12 +1864,20 @@ def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
         # Don't treat them as a delimiter.
         return 0
 
         # Don't treat them as a delimiter.
         return 0
 

+    if (
+        leaf.type == token.DOT
+        and leaf.parent
+        and leaf.parent.type not in {syms.import_from, syms.dotted_name}
+        and (previous is None or previous.type in CLOSING_BRACKETS)
+    ):
+        return DOT_PRIORITY
+

     if (
         leaf.type in MATH_OPERATORS
         and leaf.parent
         and leaf.parent.type not in {syms.factor, syms.star_expr}
     ):
     if (
         leaf.type in MATH_OPERATORS
         and leaf.parent
         and leaf.parent.type not in {syms.factor, syms.star_expr}
     ):

-        return MATH_PRIORITY
+        return MATH_PRIORITIES[leaf.type]

 
     if leaf.type in COMPARATORS:
         return COMPARATOR_PRIORITY
 
     if leaf.type in COMPARATORS:
         return COMPARATOR_PRIORITY


@@ -1586,29 +1889,60 @@ def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
     ):
         return STRING_PRIORITY
 
     ):
         return STRING_PRIORITY
 

+    if leaf.type != token.NAME:
+        return 0
+

     if (
     if (

-        leaf.type == token.NAME
-        and leaf.value == "for"
+        leaf.value == "for"

         and leaf.parent
         and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
     ):
         return COMPREHENSION_PRIORITY
 
     if (
         and leaf.parent
         and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
     ):
         return COMPREHENSION_PRIORITY
 
     if (

-        leaf.type == token.NAME
-        and leaf.value == "if"
+        leaf.value == "if"

         and leaf.parent
         and leaf.parent.type in {syms.comp_if, syms.old_comp_if}
     ):
         return COMPREHENSION_PRIORITY
 
         and leaf.parent
         and leaf.parent.type in {syms.comp_if, syms.old_comp_if}
     ):
         return COMPREHENSION_PRIORITY
 

-    if leaf.type == token.NAME and leaf.value in LOGIC_OPERATORS and leaf.parent:
+    if leaf.value in {"if", "else"} and leaf.parent and leaf.parent.type == syms.test:
+        return TERNARY_PRIORITY
+
+    if leaf.value == "is":
+        return COMPARATOR_PRIORITY
+
+    if (
+        leaf.value == "in"
+        and leaf.parent
+        and leaf.parent.type in {syms.comp_op, syms.comparison}
+        and not (
+            previous is not None
+            and previous.type == token.NAME
+            and previous.value == "not"
+        )
+    ):
+        return COMPARATOR_PRIORITY
+
+    if (
+        leaf.value == "not"
+        and leaf.parent
+        and leaf.parent.type == syms.comp_op
+        and not (
+            previous is not None
+            and previous.type == token.NAME
+            and previous.value == "is"
+        )
+    ):
+        return COMPARATOR_PRIORITY
+
+    if leaf.value in LOGIC_OPERATORS and leaf.parent:

         return LOGIC_PRIORITY
 
     return 0
 
 
         return LOGIC_PRIORITY
 
     return 0
 
 

-def generate_comments(leaf: Leaf) -> Iterator[Leaf]:
+def generate_comments(leaf: LN) -> Iterator[Leaf]:

     """Clean the prefix of the `leaf` and generate comments from it, if any.
 
     Comments in lib2to3 are shoved into the whitespace prefix.  This happens
     """Clean the prefix of the `leaf` and generate comments from it, if any.
 
     Comments in lib2to3 are shoved into the whitespace prefix.  This happens


@@ -1702,10 +2036,8 @@ def split_line(
         return
 
     line_str = str(line).strip("\n")
         return
 
     line_str = str(line).strip("\n")

-    if (
-        len(line_str) <= line_length
-        and "\n" not in line_str  # multiline strings
-        and not line.contains_standalone_comments()
+    if not line.should_explode and is_line_short_enough(
+        line, line_length=line_length, line_str=line_str

     ):
         yield line
         return
     ):
         yield line
         return


@@ -1713,10 +2045,24 @@ def split_line(
     split_funcs: List[SplitFunc]
     if line.is_def:
         split_funcs = [left_hand_split]
     split_funcs: List[SplitFunc]
     if line.is_def:
         split_funcs = [left_hand_split]

-    elif line.inside_brackets:
-        split_funcs = [delimiter_split, standalone_comment_split, right_hand_split]

     else:
     else:

-        split_funcs = [right_hand_split]
+
+        def rhs(line: Line, py36: bool = False) -> Iterator[Line]:
+            for omit in generate_trailers_to_omit(line, line_length):
+                lines = list(right_hand_split(line, line_length, py36, omit=omit))
+                if is_line_short_enough(lines[0], line_length=line_length):
+                    yield from lines
+                    return
+
+            # All splits failed, best effort split with no omits.
+            # This mostly happens to multiline strings that are by definition
+            # reported as not fitting a single line.
+            yield from right_hand_split(line, py36)
+
+        if line.inside_brackets:
+            split_funcs = [delimiter_split, standalone_comment_split, rhs]
+        else:
+            split_funcs = [rhs]

     for split_func in split_funcs:
         # We are accumulating lines in `result` because we might want to abort
         # mission and return the original line in the end, or attempt a different
     for split_func in split_funcs:
         # We are accumulating lines in `result` because we might want to abort
         # mission and return the original line in the end, or attempt a different


@@ -1745,7 +2091,8 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
     """Split line into many lines, starting with the first matching bracket pair.
 
     Note: this usually looks weird, only use this for function definitions.
     """Split line into many lines, starting with the first matching bracket pair.
 
     Note: this usually looks weird, only use this for function definitions.

-    Prefer RHS otherwise.
+    Prefer RHS otherwise.  This is why this function is not symmetrical with
+    :func:`right_hand_split` which also handles optional parentheses.

     """
     head = Line(depth=line.depth)
     body = Line(depth=line.depth + 1, inside_brackets=True)
     """
     head = Line(depth=line.depth)
     body = Line(depth=line.depth + 1, inside_brackets=True)


@@ -1783,9 +2130,16 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
 
 
 def right_hand_split(
 
 
 def right_hand_split(

-    line: Line, py36: bool = False, omit: Collection[LeafID] = ()
+    line: Line, line_length: int, py36: bool = False, omit: Collection[LeafID] = ()

 ) -> Iterator[Line]:
 ) -> Iterator[Line]:

-    """Split line into many lines, starting with the last matching bracket pair."""
+    """Split line into many lines, starting with the last matching bracket pair.
+
+    If the split was by optional parentheses, attempt splitting without them, too.
+    `omit` is a collection of closing bracket IDs that shouldn't be considered for
+    this split.
+
+    Note: running this function modifies `bracket_depth` on the leaves of `line`.
+    """

     head = Line(depth=line.depth)
     body = Line(depth=line.depth + 1, inside_brackets=True)
     tail = Line(depth=line.depth)
     head = Line(depth=line.depth)
     body = Line(depth=line.depth + 1, inside_brackets=True)
     tail = Line(depth=line.depth)


@@ -1811,8 +2165,9 @@ def right_hand_split(
     # Since body is a new indent level, remove spurious leading whitespace.
     if body_leaves:
         normalize_prefix(body_leaves[0], inside_brackets=True)
     # Since body is a new indent level, remove spurious leading whitespace.
     if body_leaves:
         normalize_prefix(body_leaves[0], inside_brackets=True)

-    elif not head_leaves:
-        # No `head` and no `body` means the split failed. `tail` has all content.
+    if not head_leaves:
+        # No `head` means the split failed. Either `tail` has all content or
+        # the matching `opening_bracket` wasn't available on `line` anymore.

         raise CannotSplit("No brackets found")
 
     # Build the new lines.
         raise CannotSplit("No brackets found")
 
     # Build the new lines.


@@ -1824,23 +2179,29 @@ def right_hand_split(
     bracket_split_succeeded_or_raise(head, body, tail)
     assert opening_bracket and closing_bracket
     if (
     bracket_split_succeeded_or_raise(head, body, tail)
     assert opening_bracket and closing_bracket
     if (

+        # the opening bracket is an optional paren

         opening_bracket.type == token.LPAR
         and not opening_bracket.value
         opening_bracket.type == token.LPAR
         and not opening_bracket.value

+        # the closing bracket is an optional paren

         and closing_bracket.type == token.RPAR
         and not closing_bracket.value
         and closing_bracket.type == token.RPAR
         and not closing_bracket.value

+        # there are no standalone comments in the body
+        and not line.contains_standalone_comments(0)
+        # and it's not an import (optional parens are the only thing we can split
+        # on in this case; attempting a split without them is a waste of time)
+        and not line.is_import

     ):
     ):

-        # These parens were optional. If there aren't any delimiters or standalone
-        # comments in the body, they were unnecessary and another split without
-        # them should be attempted.
-        if not (
-            body.bracket_tracker.delimiters or line.contains_standalone_comments(0)
-        ):
-            omit = {id(closing_bracket), *omit}
-            yield from right_hand_split(line, py36=py36, omit=omit)
-            return
+        omit = {id(closing_bracket), *omit}
+        if can_omit_invisible_parens(body, line_length):
+            try:
+                yield from right_hand_split(line, line_length, py36=py36, omit=omit)
+                return
+            except CannotSplit:
+                pass

 
     ensure_visible(opening_bracket)
     ensure_visible(closing_bracket)
 
     ensure_visible(opening_bracket)
     ensure_visible(closing_bracket)

+    body.should_explode = should_explode(body, opening_bracket)

     for result in (head, body, tail):
         if result:
             yield result
     for result in (head, body, tail):
         if result:
             yield result


@@ -1899,14 +2260,16 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
     except IndexError:
         raise CannotSplit("Line empty")
 
     except IndexError:
         raise CannotSplit("Line empty")
 

-    delimiters = line.bracket_tracker.delimiters
+    bt = line.bracket_tracker

     try:
     try:

-        delimiter_priority = line.bracket_tracker.max_delimiter_priority(
-            exclude={id(last_leaf)}
-        )
+        delimiter_priority = bt.max_delimiter_priority(exclude={id(last_leaf)})

     except ValueError:
         raise CannotSplit("No delimiters found")
 
     except ValueError:
         raise CannotSplit("No delimiters found")
 

+    if delimiter_priority == DOT_PRIORITY:
+        if bt.delimiter_count_with_priority(delimiter_priority) == 1:
+            raise CannotSplit("Splitting a single attribute from its owner looks wrong")
+

     current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
     lowest_depth = sys.maxsize
     trailing_comma_safe = True
     current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
     lowest_depth = sys.maxsize
     trailing_comma_safe = True


@@ -1922,19 +2285,18 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 

-    for leaf in line.leaves:
+    for index, leaf in enumerate(line.leaves):

         yield from append_to_line(leaf)
 
         yield from append_to_line(leaf)
 

-        for comment_after in line.comments_after(leaf):
+        for comment_after in line.comments_after(leaf, index):

             yield from append_to_line(comment_after)
 
         lowest_depth = min(lowest_depth, leaf.bracket_depth)
             yield from append_to_line(comment_after)
 
         lowest_depth = min(lowest_depth, leaf.bracket_depth)

-        if (
-            leaf.bracket_depth == lowest_depth
-            and is_vararg(leaf, within=VARARGS_PARENTS)
+        if leaf.bracket_depth == lowest_depth and is_vararg(
+            leaf, within=VARARGS_PARENTS

         ):
             trailing_comma_safe = trailing_comma_safe and py36
         ):
             trailing_comma_safe = trailing_comma_safe and py36

-        leaf_priority = delimiters.get(id(leaf))
+        leaf_priority = bt.delimiters.get(id(leaf))

         if leaf_priority == delimiter_priority:
             yield current_line
 
         if leaf_priority == delimiter_priority:
             yield current_line
 


@@ -1969,10 +2331,10 @@ def standalone_comment_split(line: Line, py36: bool = False) -> Iterator[Line]:
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 

-    for leaf in line.leaves:
+    for index, leaf in enumerate(line.leaves):

         yield from append_to_line(leaf)
 
         yield from append_to_line(leaf)
 

-        for comment_after in line.comments_after(leaf):
+        for comment_after in line.comments_after(leaf, index):

             yield from append_to_line(comment_after)
 
     if current_line:
             yield from append_to_line(comment_after)
 
     if current_line:


@@ -2011,6 +2373,22 @@ def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
     leaf.prefix = ""
 
 
     leaf.prefix = ""
 
 

+def normalize_string_prefix(leaf: Leaf, remove_u_prefix: bool = False) -> None:
+    """Make all string prefixes lowercase.
+
+    If remove_u_prefix is given, also removes any u prefix from the string.
+
+    Note: Mutates its argument.
+    """
+    match = re.match(r"^([furbFURB]*)(.*)$", leaf.value, re.DOTALL)
+    assert match is not None, f"failed to match string {leaf.value!r}"
+    orig_prefix = match.group(1)
+    new_prefix = orig_prefix.lower()
+    if remove_u_prefix:
+        new_prefix = new_prefix.replace("u", "")
+    leaf.value = f"{new_prefix}{match.group(2)}"
+
+

 def normalize_string_quotes(leaf: Leaf) -> None:
     """Prefer double quotes but only if it doesn't cause more escaping.
 
 def normalize_string_quotes(leaf: Leaf) -> None:
     """Prefer double quotes but only if it doesn't cause more escaping.
 


@@ -2040,7 +2418,7 @@ def normalize_string_quotes(leaf: Leaf) -> None:
     unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
     escaped_new_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{new_quote}")
     escaped_orig_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{orig_quote}")
     unescaped_new_quote = re.compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
     escaped_new_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{new_quote}")
     escaped_orig_quote = re.compile(rf"([^\\]|^)\\(\\\\)*{orig_quote}")

-    body = leaf.value[first_quote_pos + len(orig_quote):-len(orig_quote)]
+    body = leaf.value[first_quote_pos + len(orig_quote) : -len(orig_quote)]

     if "r" in prefix.casefold():
         if unescaped_new_quote.search(body):
             # There's at least one unescaped new_quote in this raw string
     if "r" in prefix.casefold():
         if unescaped_new_quote.search(body):
             # There's at least one unescaped new_quote in this raw string


@@ -2075,31 +2453,42 @@ def normalize_string_quotes(leaf: Leaf) -> None:
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 

+    `parens_after` is a set of string leaf values immeditely after which parens
+    should be put.
+

     Standardizes on visible parentheses for single-element tuples, and keeps
     existing visible parentheses for other tuples and generator expressions.
     """
     Standardizes on visible parentheses for single-element tuples, and keeps
     existing visible parentheses for other tuples and generator expressions.
     """

+    try:
+        list(generate_comments(node))
+    except FormatOff:
+        return  # This `node` has a prefix with `# fmt: off`, don't mess with parens.
+

     check_lpar = False
     check_lpar = False

-    for child in list(node.children):
+    for index, child in enumerate(list(node.children)):

         if check_lpar:
             if child.type == syms.atom:
         if check_lpar:
             if child.type == syms.atom:

-                if not (
-                    is_empty_tuple(child)
-                    or is_one_tuple(child)
-                    or max_delimiter_priority_in_atom(child) >= COMMA_PRIORITY
-                ):
-                    first = child.children[0]
-                    last = child.children[-1]
-                    if first.type == token.LPAR and last.type == token.RPAR:
-                        # make parentheses invisible
-                        first.value = ""  # type: ignore
-                        last.value = ""  # type: ignore
+                maybe_make_parens_invisible_in_atom(child)

             elif is_one_tuple(child):
                 # wrap child in visible parentheses
                 lpar = Leaf(token.LPAR, "(")
                 rpar = Leaf(token.RPAR, ")")
             elif is_one_tuple(child):
                 # wrap child in visible parentheses
                 lpar = Leaf(token.LPAR, "(")
                 rpar = Leaf(token.RPAR, ")")

-                index = child.remove() or 0
+                child.remove()

                 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
                 node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))

-            else:
+            elif node.type == syms.import_from:
+                # "import from" nodes store parentheses directly as part of
+                # the statement
+                if child.type == token.LPAR:
+                    # make parentheses invisible
+                    child.value = ""  # type: ignore
+                    node.children[-1].value = ""  # type: ignore
+                elif child.type != token.STAR:
+                    # insert invisible parentheses
+                    node.insert_child(index, Leaf(token.LPAR, ""))
+                    node.append_child(Leaf(token.RPAR, ""))
+                break
+
+            elif not (isinstance(child, Leaf) and is_multiline_string(child)):

                 # wrap child in invisible parentheses
                 lpar = Leaf(token.LPAR, "")
                 rpar = Leaf(token.RPAR, "")
                 # wrap child in invisible parentheses
                 lpar = Leaf(token.LPAR, "")
                 rpar = Leaf(token.RPAR, "")


@@ -2109,6 +2498,30 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
         check_lpar = isinstance(child, Leaf) and child.value in parens_after
 
 
         check_lpar = isinstance(child, Leaf) and child.value in parens_after
 
 

+def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
+    """If it's safe, make the parens in the atom `node` invisible, recusively."""
+    if (
+        node.type != syms.atom
+        or is_empty_tuple(node)
+        or is_one_tuple(node)
+        or is_yield(node)
+        or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
+    ):
+        return False
+
+    first = node.children[0]
+    last = node.children[-1]
+    if first.type == token.LPAR and last.type == token.RPAR:
+        # make parentheses invisible
+        first.value = ""  # type: ignore
+        last.value = ""  # type: ignore
+        if len(node.children) > 1:
+            maybe_make_parens_invisible_in_atom(node.children[1])
+        return True
+
+    return False
+
+

 def is_empty_tuple(node: LN) -> bool:
     """Return True if `node` holds an empty tuple."""
     return (
 def is_empty_tuple(node: LN) -> bool:
     """Return True if `node` holds an empty tuple."""
     return (


@@ -2142,12 +2555,33 @@ def is_one_tuple(node: LN) -> bool:
     )
 
 
     )
 
 

+def is_yield(node: LN) -> bool:
+    """Return True if `node` holds a `yield` or `yield from` expression."""
+    if node.type == syms.yield_expr:
+        return True
+
+    if node.type == token.NAME and node.value == "yield":  # type: ignore
+        return True
+
+    if node.type != syms.atom:
+        return False
+
+    if len(node.children) != 3:
+        return False
+
+    lpar, expr, rpar = node.children
+    if lpar.type == token.LPAR and rpar.type == token.RPAR:
+        return is_yield(expr)
+
+    return False
+
+

 def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
     """Return True if `leaf` is a star or double star in a vararg or kwarg.
 
     If `within` includes VARARGS_PARENTS, this applies to function signatures.
 def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
     """Return True if `leaf` is a star or double star in a vararg or kwarg.
 
     If `within` includes VARARGS_PARENTS, this applies to function signatures.

-    If `within` includes COLLECTION_LIBERALS_PARENTS, it applies to right
-    hand-side extended iterable unpacking (PEP 3132) and additional unpacking
+    If `within` includes UNPACKING_PARENTS, it applies to right hand-side
+    extended iterable unpacking (PEP 3132) and additional unpacking

     generalizations (PEP 448).
     """
     if leaf.type not in STARS or not leaf.parent:
     generalizations (PEP 448).
     """
     if leaf.type not in STARS or not leaf.parent:


@@ -2165,6 +2599,41 @@ def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
     return p.type in within
 
 
     return p.type in within
 
 

+def is_multiline_string(leaf: Leaf) -> bool:
+    """Return True if `leaf` is a multiline string that actually spans many lines."""
+    value = leaf.value.lstrip("furbFURB")
+    return value[:3] in {'"""', "'''"} and "\n" in value
+
+
+def is_stub_suite(node: Node) -> bool:
+    """Return True if `node` is a suite with a stub body."""
+    if (
+        len(node.children) != 4
+        or node.children[0].type != token.NEWLINE
+        or node.children[1].type != token.INDENT
+        or node.children[3].type != token.DEDENT
+    ):
+        return False
+
+    return is_stub_body(node.children[2])
+
+
+def is_stub_body(node: LN) -> bool:
+    """Return True if `node` is a simple statement containing an ellipsis."""
+    if not isinstance(node, Node) or node.type != syms.simple_stmt:
+        return False
+
+    if len(node.children) != 2:
+        return False
+
+    child = node.children[0]
+    return (
+        child.type == syms.atom
+        and len(child.children) == 3
+        and all(leaf == Leaf(token.DOT, ".") for leaf in child.children)
+    )
+
+

 def max_delimiter_priority_in_atom(node: LN) -> int:
     """Return maximum delimiter priority inside `node`.
 
 def max_delimiter_priority_in_atom(node: LN) -> int:
     """Return maximum delimiter priority inside `node`.
 


@@ -2205,12 +2674,31 @@ def ensure_visible(leaf: Leaf) -> None:
         leaf.value = ")"
 
 
         leaf.value = ")"
 
 

+def should_explode(line: Line, opening_bracket: Leaf) -> bool:
+    """Should `line` immediately be split with `delimiter_split()` after RHS?"""
+    if not (
+        opening_bracket.parent
+        and opening_bracket.parent.type in {syms.atom, syms.import_from}
+        and opening_bracket.value in "[{("
+    ):
+        return False
+
+    try:
+        last_leaf = line.leaves[-1]
+        exclude = {id(last_leaf)} if last_leaf.type == token.COMMA else set()
+        max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)
+    except (IndexError, ValueError):
+        return False
+
+    return max_priority == COMMA_PRIORITY
+
+

 def is_python36(node: Node) -> bool:
     """Return True if the current file is using Python 3.6+ features.
 
     Currently looking for:
     - f-strings; and
 def is_python36(node: Node) -> bool:
     """Return True if the current file is using Python 3.6+ features.
 
     Currently looking for:
     - f-strings; and

-    - trailing commas after * or ** in function signatures.
+    - trailing commas after * or ** in function signatures and calls.

     """
     for n in node.pre_order():
         if n.type == token.STRING:
     """
     for n in node.pre_order():
         if n.type == token.STRING:


@@ -2219,7 +2707,7 @@ def is_python36(node: Node) -> bool:
                 return True
 
         elif (
                 return True
 
         elif (

-            n.type == syms.typedargslist
+            n.type in {syms.typedargslist, syms.arglist}

             and n.children
             and n.children[-1].type == token.COMMA
         ):
             and n.children
             and n.children[-1].type == token.COMMA
         ):


@@ -2227,35 +2715,170 @@ def is_python36(node: Node) -> bool:
                 if ch.type in STARS:
                     return True
 
                 if ch.type in STARS:
                     return True
 

+                if ch.type == syms.argument:
+                    for argch in ch.children:
+                        if argch.type in STARS:
+                            return True
+

     return False
 
 
     return False
 
 

-PYTHON_EXTENSIONS = {".py"}
-BLACKLISTED_DIRECTORIES = {
-    "build", "buck-out", "dist", "_build", ".git", ".hg", ".mypy_cache", ".tox", ".venv"
-}
+def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:
+    """Generate sets of closing bracket IDs that should be omitted in a RHS.
+
+    Brackets can be omitted if the entire trailer up to and including
+    a preceding closing bracket fits in one line.
+
+    Yielded sets are cumulative (contain results of previous yields, too).  First
+    set is empty.
+    """
+
+    omit: Set[LeafID] = set()
+    yield omit
+
+    length = 4 * line.depth
+    opening_bracket = None
+    closing_bracket = None
+    optional_brackets: Set[LeafID] = set()
+    inner_brackets: Set[LeafID] = set()
+    for index, leaf, leaf_length in enumerate_with_length(line, reversed=True):
+        length += leaf_length
+        if length > line_length:
+            break
+
+        has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)
+        if leaf.type == STANDALONE_COMMENT or has_inline_comment:
+            break
+
+        optional_brackets.discard(id(leaf))
+        if opening_bracket:
+            if leaf is opening_bracket:
+                opening_bracket = None
+            elif leaf.type in CLOSING_BRACKETS:
+                inner_brackets.add(id(leaf))
+        elif leaf.type in CLOSING_BRACKETS:
+            if not leaf.value:
+                optional_brackets.add(id(opening_bracket))
+                continue
+
+            if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS:
+                # Empty brackets would fail a split so treat them as "inner"
+                # brackets (e.g. only add them to the `omit` set if another
+                # pair of brackets was good enough.
+                inner_brackets.add(id(leaf))
+                continue
+
+            opening_bracket = leaf.opening_bracket
+            if closing_bracket:
+                omit.add(id(closing_bracket))
+                omit.update(inner_brackets)
+                inner_brackets.clear()
+                yield omit
+            closing_bracket = leaf

 
 
 
 

-def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
-    """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES
-    and have one of the PYTHON_EXTENSIONS.
+def get_future_imports(node: Node) -> Set[str]:
+    """Return a set of __future__ imports in the file."""
+    imports = set()
+    for child in node.children:
+        if child.type != syms.simple_stmt:
+            break
+        first_child = child.children[0]
+        if isinstance(first_child, Leaf):
+            # Continue looking if we see a docstring; otherwise stop.
+            if (
+                len(child.children) == 2
+                and first_child.type == token.STRING
+                and child.children[1].type == token.NEWLINE
+            ):
+                continue
+            else:
+                break
+        elif first_child.type == syms.import_from:
+            module_name = first_child.children[1]
+            if not isinstance(module_name, Leaf) or module_name.value != "__future__":
+                break
+            for import_from_child in first_child.children[3:]:
+                if isinstance(import_from_child, Leaf):
+                    if import_from_child.type == token.NAME:
+                        imports.add(import_from_child.value)
+                else:
+                    assert import_from_child.type == syms.import_as_names
+                    for leaf in import_from_child.children:
+                        if isinstance(leaf, Leaf) and leaf.type == token.NAME:
+                            imports.add(leaf.value)
+        else:
+            break
+    return imports
+
+
+def gen_python_files_in_dir(
+    path: Path,
+    root: Path,
+    include: Pattern[str],
+    exclude: Pattern[str],
+    report: "Report",
+) -> Iterator[Path]:
+    """Generate all files under `path` whose paths are not excluded by the
+    `exclude` regex, but are included by the `include` regex.
+
+    `report` is where output about exclusions goes.

     """
     """

+    assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"

     for child in path.iterdir():
     for child in path.iterdir():

+        normalized_path = child.resolve().relative_to(root).as_posix()

         if child.is_dir():
         if child.is_dir():

-            if child.name in BLACKLISTED_DIRECTORIES:
-                continue
+            normalized_path += "/"
+        exclude_match = exclude.search(normalized_path)
+        if exclude_match and exclude_match.group(0):
+            report.path_ignored(child, f"matches --exclude={exclude.pattern}")
+            continue
+
+        if child.is_dir():
+            yield from gen_python_files_in_dir(child, root, include, exclude, report)

 
 

-            yield from gen_python_files_in_dir(child)
+        elif child.is_file():
+            include_match = include.search(normalized_path)
+            if include_match:
+                yield child

 
 

-        elif child.suffix in PYTHON_EXTENSIONS:
-            yield child
+
+def find_project_root(srcs: List[str]) -> Path:
+    """Return a directory containing .git, .hg, or pyproject.toml.
+
+    That directory can be one of the directories passed in `srcs` or their
+    common parent.
+
+    If no directory in the tree contains a marker that would specify it's the
+    project root, the root of the file system is returned.
+    """
+    if not srcs:
+        return Path("/").resolve()
+
+    common_base = min(Path(src).resolve() for src in srcs)
+    if common_base.is_dir():
+        # Append a fake file so `parents` below returns `common_base_dir`, too.
+        common_base /= "fake-file"
+    for directory in common_base.parents:
+        if (directory / ".git").is_dir():
+            return directory
+
+        if (directory / ".hg").is_dir():
+            return directory
+
+        if (directory / "pyproject.toml").is_file():
+            return directory
+
+    return directory

 
 
 @dataclass
 class Report:
     """Provides a reformatting counter. Can be rendered with `str(report)`."""
 
 
 @dataclass
 class Report:
     """Provides a reformatting counter. Can be rendered with `str(report)`."""

+

     check: bool = False
     quiet: bool = False
     check: bool = False
     quiet: bool = False

+    verbose: bool = False

     change_count: int = 0
     same_count: int = 0
     failure_count: int = 0
     change_count: int = 0
     same_count: int = 0
     failure_count: int = 0


@@ -2264,11 +2887,11 @@ class Report:
         """Increment the counter for successful reformatting. Write out a message."""
         if changed is Changed.YES:
             reformatted = "would reformat" if self.check else "reformatted"
         """Increment the counter for successful reformatting. Write out a message."""
         if changed is Changed.YES:
             reformatted = "would reformat" if self.check else "reformatted"

-            if not self.quiet:
+            if self.verbose or not self.quiet:

                 out(f"{reformatted} {src}")
             self.change_count += 1
         else:
                 out(f"{reformatted} {src}")
             self.change_count += 1
         else:

-            if not self.quiet:
+            if self.verbose:

                 if changed is Changed.NO:
                     msg = f"{src} already well formatted, good job."
                 else:
                 if changed is Changed.NO:
                     msg = f"{src} already well formatted, good job."
                 else:


@@ -2281,6 +2904,10 @@ class Report:
         err(f"error: cannot format {src}: {message}")
         self.failure_count += 1
 
         err(f"error: cannot format {src}: {message}")
         self.failure_count += 1
 

+    def path_ignored(self, path: Path, message: str) -> None:
+        if self.verbose:
+            out(f"{path} ignored: {message}", bold=False)
+

     @property
     def return_code(self) -> int:
         """Return the exit code that the app should use.
     @property
     def return_code(self) -> int:
         """Return the exit code that the app should use.


@@ -2393,9 +3020,11 @@ def assert_equivalent(src: str, dst: str) -> None:
         ) from None
 
 
         ) from None
 
 

-def assert_stable(src: str, dst: str, line_length: int) -> None:
+def assert_stable(
+    src: str, dst: str, line_length: int, mode: FileMode = FileMode.AUTO_DETECT
+) -> None:

     """Raise AssertionError if `dst` reformats differently the second time."""
     """Raise AssertionError if `dst` reformats differently the second time."""

-    newdst = format_str(dst, line_length=line_length)
+    newdst = format_str(dst, line_length=line_length, mode=mode)

     if dst != newdst:
         log = dump_to_file(
             diff(src, dst, "source", "first pass"),
     if dst != newdst:
         log = dump_to_file(
             diff(src, dst, "source", "first pass"),


@@ -2434,7 +3063,7 @@ def diff(a: str, b: str, a_name: str, b_name: str) -> str:
     )
 
 
     )
 
 

-def cancel(tasks: List[asyncio.Task]) -> None:
+def cancel(tasks: Iterable[asyncio.Task]) -> None:

     """asyncio signal handler that cancels all `tasks` and reports to stderr."""
     err("Aborted!")
     for task in tasks:
     """asyncio signal handler that cancels all `tasks` and reports to stderr."""
     err("Aborted!")
     for task in tasks:


@@ -2472,19 +3101,159 @@ def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
     return regex.sub(replacement, regex.sub(replacement, original))
 
 
     return regex.sub(replacement, regex.sub(replacement, original))
 
 

-CACHE_DIR = Path(user_cache_dir("black", version=__version__))
-CACHE_FILE = CACHE_DIR / "cache.pickle"
+def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:
+    """Like `reversed(enumerate(sequence))` if that were possible."""
+    index = len(sequence) - 1
+    for element in reversed(sequence):
+        yield (index, element)
+        index -= 1
+
+
+def enumerate_with_length(
+    line: Line, reversed: bool = False
+) -> Iterator[Tuple[Index, Leaf, int]]:
+    """Return an enumeration of leaves with their length.

 
 

+    Stops prematurely on multiline strings and standalone comments.
+    """
+    op = cast(
+        Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]],
+        enumerate_reversed if reversed else enumerate,
+    )
+    for index, leaf in op(line.leaves):
+        length = len(leaf.prefix) + len(leaf.value)
+        if "\n" in leaf.value:
+            return  # Multiline strings, we can't continue.
+
+        comment: Optional[Leaf]
+        for comment in line.comments_after(leaf, index):
+            length += len(comment.value)
+
+        yield index, leaf, length
+
+
+def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool:
+    """Return True if `line` is no longer than `line_length`.
+
+    Uses the provided `line_str` rendering, if any, otherwise computes a new one.
+    """
+    if not line_str:
+        line_str = str(line).strip("\n")
+    return (
+        len(line_str) <= line_length
+        and "\n" not in line_str  # multiline strings
+        and not line.contains_standalone_comments()
+    )
+
+
+def can_omit_invisible_parens(line: Line, line_length: int) -> bool:
+    """Does `line` have a shape safe to reformat without optional parens around it?
+
+    Returns True for only a subset of potentially nice looking formattings but
+    the point is to not return false positives that end up producing lines that
+    are too long.
+    """
+    bt = line.bracket_tracker
+    if not bt.delimiters:
+        # Without delimiters the optional parentheses are useless.
+        return True
+
+    max_priority = bt.max_delimiter_priority()
+    if bt.delimiter_count_with_priority(max_priority) > 1:
+        # With more than one delimiter of a kind the optional parentheses read better.
+        return False
+
+    if max_priority == DOT_PRIORITY:
+        # A single stranded method call doesn't require optional parentheses.
+        return True
+
+    assert len(line.leaves) >= 2, "Stranded delimiter"
+
+    first = line.leaves[0]
+    second = line.leaves[1]
+    penultimate = line.leaves[-2]
+    last = line.leaves[-1]
+
+    # With a single delimiter, omit if the expression starts or ends with
+    # a bracket.
+    if first.type in OPENING_BRACKETS and second.type not in CLOSING_BRACKETS:
+        remainder = False
+        length = 4 * line.depth
+        for _index, leaf, leaf_length in enumerate_with_length(line):
+            if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first:
+                remainder = True
+            if remainder:
+                length += leaf_length
+                if length > line_length:
+                    break

 
 

-def read_cache() -> Cache:
+                if leaf.type in OPENING_BRACKETS:
+                    # There are brackets we can further split on.
+                    remainder = False
+
+        else:
+            # checked the entire string and line length wasn't exceeded
+            if len(line.leaves) == _index + 1:
+                return True
+
+        # Note: we are not returning False here because a line might have *both*
+        # a leading opening bracket and a trailing closing bracket.  If the
+        # opening bracket doesn't match our rule, maybe the closing will.
+
+    if (
+        last.type == token.RPAR
+        or last.type == token.RBRACE
+        or (
+            # don't use indexing for omitting optional parentheses;
+            # it looks weird
+            last.type == token.RSQB
+            and last.parent
+            and last.parent.type != syms.trailer
+        )
+    ):
+        if penultimate.type in OPENING_BRACKETS:
+            # Empty brackets don't help.
+            return False
+
+        if is_multiline_string(first):
+            # Additional wrapping of a multiline string in this situation is
+            # unnecessary.
+            return True
+
+        length = 4 * line.depth
+        seen_other_brackets = False
+        for _index, leaf, leaf_length in enumerate_with_length(line):
+            length += leaf_length
+            if leaf is last.opening_bracket:
+                if seen_other_brackets or length <= line_length:
+                    return True
+
+            elif leaf.type in OPENING_BRACKETS:
+                # There are brackets we can further split on.
+                seen_other_brackets = True
+
+    return False
+
+
+def get_cache_file(line_length: int, mode: FileMode) -> Path:
+    pyi = bool(mode & FileMode.PYI)
+    py36 = bool(mode & FileMode.PYTHON36)
+    return (
+        CACHE_DIR
+        / f"cache.{line_length}{'.pyi' if pyi else ''}{'.py36' if py36 else ''}.pickle"
+    )
+
+
+def read_cache(line_length: int, mode: FileMode) -> Cache:

     """Read the cache if it exists and is well formed.
 
     If it is not well formed, the call to write_cache later should resolve the issue.
     """
     """Read the cache if it exists and is well formed.
 
     If it is not well formed, the call to write_cache later should resolve the issue.
     """

-    if not CACHE_FILE.exists():
+    cache_file = get_cache_file(line_length, mode)
+    if not cache_file.exists():

         return {}
 
         return {}
 

-    with CACHE_FILE.open("rb") as fobj:
+    with cache_file.open("rb") as fobj:

         try:
             cache: Cache = pickle.load(fobj)
         except pickle.UnpicklingError:
         try:
             cache: Cache = pickle.load(fobj)
         except pickle.UnpicklingError:


@@ -2517,13 +3286,16 @@ def filter_cached(
     return todo, done
 
 
     return todo, done
 
 

-def write_cache(cache: Cache, sources: List[Path]) -> None:
+def write_cache(
+    cache: Cache, sources: List[Path], line_length: int, mode: FileMode
+) -> None:

     """Update the cache file."""
     """Update the cache file."""

+    cache_file = get_cache_file(line_length, mode)

     try:
         if not CACHE_DIR.exists():
             CACHE_DIR.mkdir(parents=True)
         new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
     try:
         if not CACHE_DIR.exists():
             CACHE_DIR.mkdir(parents=True)
         new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}

-        with CACHE_FILE.open("wb") as fobj:
+        with cache_file.open("wb") as fobj:

             pickle.dump(new_cache, fobj, protocol=pickle.HIGHEST_PROTOCOL)
     except OSError:
         pass
             pickle.dump(new_cache, fobj, protocol=pickle.HIGHEST_PROTOCOL)
     except OSError:
         pass