Add GitHub Actions badge to README.md (#1134)

[etc/vim.git] / black.py

diff --git a/black.py b/black.py
index 29d56c923f6fa65c5e8b3d5bc83097e570137f2f..1c35643725c62acf5a6e83ad13f91afbf70ad8f1 100644 (file)
--- a/black.py
+++ b/black.py
@@ -1,5 +1,7 @@
+import ast

 import asyncio
 from concurrent.futures import Executor, ProcessPoolExecutor
 import asyncio
 from concurrent.futures import Executor, ProcessPoolExecutor

+from contextlib import contextmanager

 from datetime import datetime
 from enum import Enum
 from functools import lru_cache, partial, wraps
 from datetime import datetime
 from enum import Enum
 from functools import lru_cache, partial, wraps


@@ -10,7 +12,7 @@ from multiprocessing import Manager, freeze_support
 import os
 from pathlib import Path
 import pickle
 import os
 from pathlib import Path
 import pickle

-import re
+import regex as re

 import signal
 import sys
 import tempfile
 import signal
 import sys
 import tempfile


@@ -35,12 +37,15 @@ from typing import (
     Union,
     cast,
 )
     Union,
     cast,
 )

+from typing_extensions import Final
+from mypy_extensions import mypyc_attr

 
 from appdirs import user_cache_dir
 
 from appdirs import user_cache_dir

-from attr import dataclass, evolve, Factory
+from dataclasses import dataclass, field, replace

 import click
 import toml
 from typed_ast import ast3, ast27
 import click
 import toml
 from typed_ast import ast3, ast27

+from pathspec import PathSpec

 
 # lib2to3 fork
 from blib2to3.pytree import Node, Leaf, type_repr
 
 # lib2to3 fork
 from blib2to3.pytree import Node, Leaf, type_repr


@@ -49,12 +54,10 @@ from blib2to3.pgen2 import driver, token
 from blib2to3.pgen2.grammar import Grammar
 from blib2to3.pgen2.parse import ParseError
 
 from blib2to3.pgen2.grammar import Grammar
 from blib2to3.pgen2.parse import ParseError
 

+from _black_version import version as __version__

 
 

-__version__ = "19.3b0"

 DEFAULT_LINE_LENGTH = 88
 DEFAULT_LINE_LENGTH = 88

-DEFAULT_EXCLUDES = (
-    r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)/"
-)
+DEFAULT_EXCLUDES = r"/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist)/"  # noqa: B950

 DEFAULT_INCLUDES = r"\.pyi?$"
 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
 
 DEFAULT_INCLUDES = r"\.pyi?$"
 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
 


@@ -140,6 +143,8 @@ class Feature(Enum):
     # set for every version of python.
     ASYNC_IDENTIFIERS = 6
     ASYNC_KEYWORDS = 7
     # set for every version of python.
     ASYNC_IDENTIFIERS = 6
     ASYNC_KEYWORDS = 7

+    ASSIGNMENT_EXPRESSIONS = 8
+    POS_ONLY_ARGUMENTS = 9

 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {


@@ -174,13 +179,15 @@ VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
         Feature.TRAILING_COMMA_IN_CALL,
         Feature.TRAILING_COMMA_IN_DEF,
         Feature.ASYNC_KEYWORDS,
         Feature.TRAILING_COMMA_IN_CALL,
         Feature.TRAILING_COMMA_IN_DEF,
         Feature.ASYNC_KEYWORDS,

+        Feature.ASSIGNMENT_EXPRESSIONS,
+        Feature.POS_ONLY_ARGUMENTS,

     },
 }
 
 
 @dataclass
 class FileMode:
     },
 }
 
 
 @dataclass
 class FileMode:

-    target_versions: Set[TargetVersion] = Factory(set)
+    target_versions: Set[TargetVersion] = field(default_factory=set)

     line_length: int = DEFAULT_LINE_LENGTH
     string_normalization: bool = True
     is_pyi: bool = False
     line_length: int = DEFAULT_LINE_LENGTH
     string_normalization: bool = True
     is_pyi: bool = False


@@ -242,6 +249,17 @@ def read_pyproject_toml(
     return value
 
 
     return value
 
 

+def target_version_option_callback(
+    c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]
+) -> List[TargetVersion]:
+    """Compute the target versions from a --target-version flag.
+
+    This is its own function because mypy couldn't infer the type correctly
+    when it was a lambda, causing mypyc trouble.
+    """
+    return [TargetVersion[val.upper()] for val in v]
+
+

 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
 @click.option(
 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
 @click.option("-c", "--code", type=str, help="Format the code passed in as a string.")
 @click.option(


@@ -256,7 +274,7 @@ def read_pyproject_toml(
     "-t",
     "--target-version",
     type=click.Choice([v.name.lower() for v in TargetVersion]),
     "-t",
     "--target-version",
     type=click.Choice([v.name.lower() for v in TargetVersion]),

-    callback=lambda c, p, v: [TargetVersion[val.upper()] for val in v],
+    callback=target_version_option_callback,

     multiple=True,
     help=(
         "Python versions that should be supported by Black's output. [default: "
     multiple=True,
     help=(
         "Python versions that should be supported by Black's output. [default: "


@@ -383,7 +401,7 @@ def main(
     verbose: bool,
     include: str,
     exclude: str,
     verbose: bool,
     include: str,
     exclude: str,

-    src: Tuple[str],
+    src: Tuple[str, ...],

     config: Optional[str],
 ) -> None:
     """The uncompromising code formatter."""
     config: Optional[str],
 ) -> None:
     """The uncompromising code formatter."""


@@ -427,11 +445,14 @@ def main(
     report = Report(check=check, quiet=quiet, verbose=verbose)
     root = find_project_root(src)
     sources: Set[Path] = set()
     report = Report(check=check, quiet=quiet, verbose=verbose)
     root = find_project_root(src)
     sources: Set[Path] = set()

+    path_empty(src, quiet, verbose, ctx)

     for s in src:
         p = Path(s)
         if p.is_dir():
             sources.update(
     for s in src:
         p = Path(s)
         if p.is_dir():
             sources.update(

-                gen_python_files_in_dir(p, root, include_regex, exclude_regex, report)
+                gen_python_files_in_dir(
+                    p, root, include_regex, exclude_regex, report, get_gitignore(root)
+                )

             )
         elif p.is_file() or s == "-":
             # if a file was explicitly given, we don't care about its extension
             )
         elif p.is_file() or s == "-":
             # if a file was explicitly given, we don't care about its extension


@@ -440,7 +461,7 @@ def main(
             err(f"invalid path: {s}")
     if len(sources) == 0:
         if verbose or not quiet:
             err(f"invalid path: {s}")
     if len(sources) == 0:
         if verbose or not quiet:

-            out("No paths given. Nothing to do 😴")
+            out("No Python files are present to be formatted. Nothing to do 😴")

         ctx.exit(0)
 
     if len(sources) == 1:
         ctx.exit(0)
 
     if len(sources) == 1:


@@ -462,6 +483,18 @@ def main(
     ctx.exit(report.return_code)
 
 
     ctx.exit(report.return_code)
 
 

+def path_empty(
+    src: Tuple[str, ...], quiet: bool, verbose: bool, ctx: click.Context
+) -> None:
+    """
+    Exit if there is no `src` provided for formatting
+    """
+    if not src:
+        if verbose or not quiet:
+            out("No Path provided. Nothing to do 😴")
+            ctx.exit(0)
+
+

 def reformat_one(
     src: Path, fast: bool, write_back: WriteBack, mode: FileMode, report: "Report"
 ) -> None:
 def reformat_one(
     src: Path, fast: bool, write_back: WriteBack, mode: FileMode, report: "Report"
 ) -> None:


@@ -523,6 +556,7 @@ def reformat_many(
         )
     finally:
         shutdown(loop)
         )
     finally:
         shutdown(loop)

+        executor.shutdown()

 
 
 async def schedule_formatting(
 
 
 async def schedule_formatting(


@@ -566,7 +600,7 @@ async def schedule_formatting(
         ): src
         for src in sorted(sources)
     }
         ): src
         for src in sorted(sources)
     }

-    pending: Iterable[asyncio.Future] = tasks.keys()
+    pending: Iterable["asyncio.Future[bool]"] = tasks.keys()

     try:
         loop.add_signal_handler(signal.SIGINT, cancel, pending)
         loop.add_signal_handler(signal.SIGTERM, cancel, pending)
     try:
         loop.add_signal_handler(signal.SIGINT, cancel, pending)
         loop.add_signal_handler(signal.SIGTERM, cancel, pending)


@@ -610,7 +644,7 @@ def format_file_in_place(
     `mode` and `fast` options are passed to :func:`format_file_contents`.
     """
     if src.suffix == ".pyi":
     `mode` and `fast` options are passed to :func:`format_file_contents`.
     """
     if src.suffix == ".pyi":

-        mode = evolve(mode, is_pyi=True)
+        mode = replace(mode, is_pyi=True)

 
     then = datetime.utcfromtimestamp(src.stat().st_mtime)
     with open(src, "rb") as buf:
 
     then = datetime.utcfromtimestamp(src.stat().st_mtime)
     with open(src, "rb") as buf:


@@ -620,17 +654,16 @@ def format_file_in_place(
     except NothingChanged:
         return False
 
     except NothingChanged:
         return False
 

-    if write_back == write_back.YES:
+    if write_back == WriteBack.YES:

         with open(src, "w", encoding=encoding, newline=newline) as f:
             f.write(dst_contents)
         with open(src, "w", encoding=encoding, newline=newline) as f:
             f.write(dst_contents)

-    elif write_back == write_back.DIFF:
+    elif write_back == WriteBack.DIFF:

         now = datetime.utcnow()
         src_name = f"{src}\t{then} +0000"
         dst_name = f"{src}\t{now} +0000"
         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)
         now = datetime.utcnow()
         src_name = f"{src}\t{then} +0000"
         dst_name = f"{src}\t{now} +0000"
         diff_contents = diff(src_contents, dst_contents, src_name, dst_name)

-        if lock:
-            lock.acquire()
-        try:
+
+        with lock or nullcontext():

             f = io.TextIOWrapper(
                 sys.stdout.buffer,
                 encoding=encoding,
             f = io.TextIOWrapper(
                 sys.stdout.buffer,
                 encoding=encoding,


@@ -639,9 +672,7 @@ def format_file_in_place(
             )
             f.write(diff_contents)
             f.detach()
             )
             f.write(diff_contents)
             f.detach()

-        finally:
-            if lock:
-                lock.release()
+

     return True
 
 
     return True
 
 


@@ -771,7 +802,8 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
             # Python 2.7
             pygram.python_grammar,
         ]
             # Python 2.7
             pygram.python_grammar,
         ]

-    elif all(version.is_python2() for version in target_versions):
+
+    if all(version.is_python2() for version in target_versions):

         # Python 2-only code, so try Python 2 grammars.
         return [
             # Python 2.7 with future print_function import
         # Python 2-only code, so try Python 2 grammars.
         return [
             # Python 2.7 with future print_function import


@@ -779,21 +811,21 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
             # Python 2.7
             pygram.python_grammar,
         ]
             # Python 2.7
             pygram.python_grammar,
         ]

-    else:
-        # Python 3-compatible code, so only try Python 3 grammar.
-        grammars = []
-        # If we have to parse both, try to parse async as a keyword first
-        if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS):
-            # Python 3.7+
-            grammars.append(
-                pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords  # noqa: B950
-            )
-        if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
-            # Python 3.0-3.6
-            grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
-        # At least one of the above branches must have been taken, because every Python
-        # version has exactly one of the two 'ASYNC_*' flags
-        return grammars
+
+    # Python 3-compatible code, so only try Python 3 grammar.
+    grammars = []
+    # If we have to parse both, try to parse async as a keyword first
+    if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS):
+        # Python 3.7+
+        grammars.append(
+            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
+        )
+    if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
+        # Python 3.0-3.6
+        grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
+    # At least one of the above branches must have been taken, because every Python
+    # version has exactly one of the two 'ASYNC_*' flags
+    return grammars

 
 
 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
 
 
 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:


@@ -848,8 +880,16 @@ class Visitor(Generic[T]):
         if node.type < 256:
             name = token.tok_name[node.type]
         else:
         if node.type < 256:
             name = token.tok_name[node.type]
         else:

-            name = type_repr(node.type)
-        yield from getattr(self, f"visit_{name}", self.visit_default)(node)
+            name = str(type_repr(node.type))
+        # We explicitly branch on whether a visitor exists (instead of
+        # using self.visit_default as the default arg to getattr) in order
+        # to save needing to create a bound method object and so mypyc can
+        # generate a native call to visit_default.
+        visitf = getattr(self, f"visit_{name}", None)
+        if visitf:
+            yield from visitf(node)
+        else:
+            yield from self.visit_default(node)

 
     def visit_default(self, node: LN) -> Iterator[T]:
         """Default `visit_*()` implementation. Recurses to children of `node`."""
 
     def visit_default(self, node: LN) -> Iterator[T]:
         """Default `visit_*()` implementation. Recurses to children of `node`."""


@@ -894,8 +934,8 @@ class DebugVisitor(Visitor[T]):
         list(v.visit(code))
 
 
         list(v.visit(code))
 
 

-WHITESPACE = {token.DEDENT, token.INDENT, token.NEWLINE}
-STATEMENT = {
+WHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE}
+STATEMENT: Final = {

     syms.if_stmt,
     syms.while_stmt,
     syms.for_stmt,
     syms.if_stmt,
     syms.while_stmt,
     syms.for_stmt,


@@ -905,10 +945,10 @@ STATEMENT = {
     syms.funcdef,
     syms.classdef,
 }
     syms.funcdef,
     syms.classdef,
 }

-STANDALONE_COMMENT = 153
+STANDALONE_COMMENT: Final = 153

 token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT"
 token.tok_name[STANDALONE_COMMENT] = "STANDALONE_COMMENT"

-LOGIC_OPERATORS = {"and", "or"}
-COMPARATORS = {
+LOGIC_OPERATORS: Final = {"and", "or"}
+COMPARATORS: Final = {

     token.LESS,
     token.GREATER,
     token.EQEQUAL,
     token.LESS,
     token.GREATER,
     token.EQEQUAL,


@@ -916,7 +956,7 @@ COMPARATORS = {
     token.LESSEQUAL,
     token.GREATEREQUAL,
 }
     token.LESSEQUAL,
     token.GREATEREQUAL,
 }

-MATH_OPERATORS = {
+MATH_OPERATORS: Final = {

     token.VBAR,
     token.CIRCUMFLEX,
     token.AMPER,
     token.VBAR,
     token.CIRCUMFLEX,
     token.AMPER,


@@ -932,22 +972,23 @@ MATH_OPERATORS = {
     token.TILDE,
     token.DOUBLESTAR,
 }
     token.TILDE,
     token.DOUBLESTAR,
 }

-STARS = {token.STAR, token.DOUBLESTAR}
-VARARGS_PARENTS = {
+STARS: Final = {token.STAR, token.DOUBLESTAR}
+VARARGS_SPECIALS: Final = STARS | {token.SLASH}
+VARARGS_PARENTS: Final = {

     syms.arglist,
     syms.argument,  # double star in arglist
     syms.trailer,  # single argument to call
     syms.typedargslist,
     syms.varargslist,  # lambdas
 }
     syms.arglist,
     syms.argument,  # double star in arglist
     syms.trailer,  # single argument to call
     syms.typedargslist,
     syms.varargslist,  # lambdas
 }

-UNPACKING_PARENTS = {
+UNPACKING_PARENTS: Final = {

     syms.atom,  # single element of a list or set literal
     syms.dictsetmaker,
     syms.listmaker,
     syms.testlist_gexp,
     syms.testlist_star_expr,
 }
     syms.atom,  # single element of a list or set literal
     syms.dictsetmaker,
     syms.listmaker,
     syms.testlist_gexp,
     syms.testlist_star_expr,
 }

-TEST_DESCENDANTS = {
+TEST_DESCENDANTS: Final = {

     syms.test,
     syms.lambdef,
     syms.or_test,
     syms.test,
     syms.lambdef,
     syms.or_test,


@@ -964,7 +1005,7 @@ TEST_DESCENDANTS = {
     syms.term,
     syms.power,
 }
     syms.term,
     syms.power,
 }

-ASSIGNMENTS = {
+ASSIGNMENTS: Final = {

     "=",
     "+=",
     "-=",
     "=",
     "+=",
     "-=",


@@ -980,13 +1021,13 @@ ASSIGNMENTS = {
     "**=",
     "//=",
 }
     "**=",
     "//=",
 }

-COMPREHENSION_PRIORITY = 20
-COMMA_PRIORITY = 18
-TERNARY_PRIORITY = 16
-LOGIC_PRIORITY = 14
-STRING_PRIORITY = 12
-COMPARATOR_PRIORITY = 10
-MATH_PRIORITIES = {
+COMPREHENSION_PRIORITY: Final = 20
+COMMA_PRIORITY: Final = 18
+TERNARY_PRIORITY: Final = 16
+LOGIC_PRIORITY: Final = 14
+STRING_PRIORITY: Final = 12
+COMPARATOR_PRIORITY: Final = 10
+MATH_PRIORITIES: Final = {

     token.VBAR: 9,
     token.CIRCUMFLEX: 8,
     token.AMPER: 7,
     token.VBAR: 9,
     token.CIRCUMFLEX: 8,
     token.AMPER: 7,


@@ -1002,7 +1043,7 @@ MATH_PRIORITIES = {
     token.TILDE: 3,
     token.DOUBLESTAR: 2,
 }
     token.TILDE: 3,
     token.DOUBLESTAR: 2,
 }

-DOT_PRIORITY = 1
+DOT_PRIORITY: Final = 1

 
 
 @dataclass
 
 
 @dataclass


@@ -1010,11 +1051,11 @@ class BracketTracker:
     """Keeps track of brackets on a line."""
 
     depth: int = 0
     """Keeps track of brackets on a line."""
 
     depth: int = 0

-    bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
-    delimiters: Dict[LeafID, Priority] = Factory(dict)
+    bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = field(default_factory=dict)
+    delimiters: Dict[LeafID, Priority] = field(default_factory=dict)

     previous: Optional[Leaf] = None
     previous: Optional[Leaf] = None

-    _for_loop_depths: List[int] = Factory(list)
-    _lambda_argument_depths: List[int] = Factory(list)
+    _for_loop_depths: List[int] = field(default_factory=list)
+    _lambda_argument_depths: List[int] = field(default_factory=list)

 
     def mark(self, leaf: Leaf) -> None:
         """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
 
     def mark(self, leaf: Leaf) -> None:
         """Mark `leaf` with bracket-related metadata. Keep track of delimiters.


@@ -1142,9 +1183,10 @@ class Line:
     """Holds leaves and comments. Can be printed with `str(line)`."""
 
     depth: int = 0
     """Holds leaves and comments. Can be printed with `str(line)`."""
 
     depth: int = 0

-    leaves: List[Leaf] = Factory(list)
-    comments: Dict[LeafID, List[Leaf]] = Factory(dict)  # keys ordered like `leaves`
-    bracket_tracker: BracketTracker = Factory(BracketTracker)
+    leaves: List[Leaf] = field(default_factory=list)
+    # keys ordered like `leaves`
+    comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict)
+    bracket_tracker: BracketTracker = field(default_factory=BracketTracker)

     inside_brackets: bool = False
     should_explode: bool = False
 
     inside_brackets: bool = False
     should_explode: bool = False
 


@@ -1224,6 +1266,69 @@ class Line:
             Leaf(token.DOT, ".") for _ in range(3)
         ]
 
             Leaf(token.DOT, ".") for _ in range(3)
         ]
 

+    @property
+    def is_collection_with_optional_trailing_comma(self) -> bool:
+        """Is this line a collection literal with a trailing comma that's optional?
+
+        Note that the trailing comma in a 1-tuple is not optional.
+        """
+        if not self.leaves or len(self.leaves) < 4:
+            return False
+
+        # Look for and address a trailing colon.
+        if self.leaves[-1].type == token.COLON:
+            closer = self.leaves[-2]
+            close_index = -2
+        else:
+            closer = self.leaves[-1]
+            close_index = -1
+        if closer.type not in CLOSING_BRACKETS or self.inside_brackets:
+            return False
+
+        if closer.type == token.RPAR:
+            # Tuples require an extra check, because if there's only
+            # one element in the tuple removing the comma unmakes the
+            # tuple.
+            #
+            # We also check for parens before looking for the trailing
+            # comma because in some cases (eg assigning a dict
+            # literal) the literal gets wrapped in temporary parens
+            # during parsing. This case is covered by the
+            # collections.py test data.
+            opener = closer.opening_bracket
+            for _open_index, leaf in enumerate(self.leaves):
+                if leaf is opener:
+                    break
+
+            else:
+                # Couldn't find the matching opening paren, play it safe.
+                return False
+
+            commas = 0
+            comma_depth = self.leaves[close_index - 1].bracket_depth
+            for leaf in self.leaves[_open_index + 1 : close_index]:
+                if leaf.bracket_depth == comma_depth and leaf.type == token.COMMA:
+                    commas += 1
+            if commas > 1:
+                # We haven't looked yet for the trailing comma because
+                # we might also have caught noop parens.
+                return self.leaves[close_index - 1].type == token.COMMA
+
+            elif commas == 1:
+                return False  # it's either a one-tuple or didn't have a trailing comma
+
+            if self.leaves[close_index - 1].type in CLOSING_BRACKETS:
+                close_index -= 1
+                closer = self.leaves[close_index]
+                if closer.type == token.RPAR:
+                    # TODO: this is a gut feeling. Will we ever see this?
+                    return False
+
+        if self.leaves[close_index - 1].type != token.COMMA:
+            return False
+
+        return True
+

     @property
     def is_def(self) -> bool:
         """Is this a function definition? (Also returns True for async defs.)"""
     @property
     def is_def(self) -> bool:
         """Is this a function definition? (Also returns True for async defs.)"""


@@ -1271,12 +1376,12 @@ class Line:
     def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
         """If so, needs to be split before emitting."""
         for leaf in self.leaves:
     def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool:
         """If so, needs to be split before emitting."""
         for leaf in self.leaves:

-            if leaf.type == STANDALONE_COMMENT:
-                if leaf.bracket_depth <= depth_limit:
-                    return True
+            if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit:
+                return True
+

         return False
 
         return False
 

-    def contains_inner_type_comments(self) -> bool:
+    def contains_uncollapsable_type_comments(self) -> bool:

         ignored_ids = set()
         try:
             last_leaf = self.leaves[-1]
         ignored_ids = set()
         try:
             last_leaf = self.leaves[-1]


@@ -1293,13 +1398,53 @@ class Line:
         except IndexError:
             return False
 
         except IndexError:
             return False
 

+        # A type comment is uncollapsable if it is attached to a leaf
+        # that isn't at the end of the line (since that could cause it
+        # to get associated to a different argument) or if there are
+        # comments before it (since that could cause it to get hidden
+        # behind a comment.
+        comment_seen = False

         for leaf_id, comments in self.comments.items():
         for leaf_id, comments in self.comments.items():

-            if leaf_id in ignored_ids:
-                continue
-

             for comment in comments:
                 if is_type_comment(comment):
             for comment in comments:
                 if is_type_comment(comment):

-                    return True
+                    if comment_seen or (
+                        not is_type_comment(comment, " ignore")
+                        and leaf_id not in ignored_ids
+                    ):
+                        return True
+
+                comment_seen = True
+
+        return False
+
+    def contains_unsplittable_type_ignore(self) -> bool:
+        if not self.leaves:
+            return False
+
+        # If a 'type: ignore' is attached to the end of a line, we
+        # can't split the line, because we can't know which of the
+        # subexpressions the ignore was meant to apply to.
+        #
+        # We only want this to apply to actual physical lines from the
+        # original source, though: we don't want the presence of a
+        # 'type: ignore' at the end of a multiline expression to
+        # justify pushing it all onto one line. Thus we
+        # (unfortunately) need to check the actual source lines and
+        # only report an unsplittable 'type: ignore' if this line was
+        # one line in the original code.
+
+        # Grab the first and last line numbers, skipping generated leaves
+        first_line = next((l.lineno for l in self.leaves if l.lineno != 0), 0)
+        last_line = next((l.lineno for l in reversed(self.leaves) if l.lineno != 0), 0)
+
+        if first_line == last_line:
+            # We look at the last two leaves since a comma or an
+            # invisible paren could have been added at the end of the
+            # line.
+            for node in self.leaves[-2:]:
+                for comment in self.comments.get(id(node), []):
+                    if is_type_comment(comment, " ignore"):
+                        return True

 
         return False
 
 
         return False
 


@@ -1312,60 +1457,40 @@ class Line:
 
     def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
         """Remove trailing comma if there is one and it's safe."""
 
     def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
         """Remove trailing comma if there is one and it's safe."""

+        if not (self.leaves and self.leaves[-1].type == token.COMMA):
+            return False
+
+        # We remove trailing commas only in the case of importing a
+        # single name from a module.

         if not (
             self.leaves
         if not (
             self.leaves

+            and self.is_import
+            and len(self.leaves) > 4

             and self.leaves[-1].type == token.COMMA
             and closing.type in CLOSING_BRACKETS
             and self.leaves[-1].type == token.COMMA
             and closing.type in CLOSING_BRACKETS

+            and self.leaves[-4].type == token.NAME
+            and (
+                # regular `from foo import bar,`
+                self.leaves[-4].value == "import"
+                # `from foo import (bar as baz,)
+                or (
+                    len(self.leaves) > 6
+                    and self.leaves[-6].value == "import"
+                    and self.leaves[-3].value == "as"
+                )
+                # `from foo import bar as baz,`
+                or (
+                    len(self.leaves) > 5
+                    and self.leaves[-5].value == "import"
+                    and self.leaves[-3].value == "as"
+                )
+            )
+            and closing.type == token.RPAR

         ):
             return False
 
         ):
             return False
 

-        if closing.type == token.RBRACE:
-            self.remove_trailing_comma()
-            return True
-
-        if closing.type == token.RSQB:
-            comma = self.leaves[-1]
-            if comma.parent and comma.parent.type == syms.listmaker:
-                self.remove_trailing_comma()
-                return True
-
-        # For parens let's check if it's safe to remove the comma.
-        # Imports are always safe.
-        if self.is_import:
-            self.remove_trailing_comma()
-            return True
-
-        # Otherwise, if the trailing one is the only one, we might mistakenly
-        # change a tuple into a different type by removing the comma.
-        depth = closing.bracket_depth + 1
-        commas = 0
-        opening = closing.opening_bracket
-        for _opening_index, leaf in enumerate(self.leaves):
-            if leaf is opening:
-                break
-
-        else:
-            return False
-
-        for leaf in self.leaves[_opening_index + 1 :]:
-            if leaf is closing:
-                break
-
-            bracket_depth = leaf.bracket_depth
-            if bracket_depth == depth and leaf.type == token.COMMA:
-                commas += 1
-                if leaf.parent and leaf.parent.type in {
-                    syms.arglist,
-                    syms.typedargslist,
-                }:
-                    commas += 1
-                    break
-
-        if commas > 1:
-            self.remove_trailing_comma()
-            return True
-
-        return False
+        self.remove_trailing_comma()
+        return True

 
     def append_comment(self, comment: Leaf) -> bool:
         """Add an inline or standalone comment to the line."""
 
     def append_comment(self, comment: Leaf) -> bool:
         """Add an inline or standalone comment to the line."""


@@ -1399,6 +1524,7 @@ class Line:
                 comment.type = STANDALONE_COMMENT
                 comment.prefix = ""
                 return False
                 comment.type = STANDALONE_COMMENT
                 comment.prefix = ""
                 return False

+

             last_leaf = self.leaves[-2]
         self.comments.setdefault(id(last_leaf), []).append(comment)
         return True
             last_leaf = self.leaves[-2]
         self.comments.setdefault(id(last_leaf), []).append(comment)
         return True


@@ -1466,7 +1592,7 @@ class EmptyLineTracker:
     is_pyi: bool = False
     previous_line: Optional[Line] = None
     previous_after: int = 0
     is_pyi: bool = False
     previous_line: Optional[Line] = None
     previous_after: int = 0

-    previous_defs: List[int] = Factory(list)
+    previous_defs: List[int] = field(default_factory=list)

 
     def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
         """Return the number of extra empty lines before and after the `current_line`.
 
     def maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]:
         """Return the number of extra empty lines before and after the `current_line`.


@@ -1475,7 +1601,13 @@ class EmptyLineTracker:
         lines (two on module-level).
         """
         before, after = self._maybe_empty_lines(current_line)
         lines (two on module-level).
         """
         before, after = self._maybe_empty_lines(current_line)

-        before -= self.previous_after
+        before = (
+            # Black should not insert empty lines at the beginning
+            # of the file
+            0
+            if self.previous_line is None
+            else before - self.previous_after
+        )

         self.previous_after = after
         self.previous_line = current_line
         return before, after
         self.previous_after = after
         self.previous_line = current_line
         return before, after


@@ -1574,7 +1706,7 @@ class LineGenerator(Visitor[Line]):
 
     is_pyi: bool = False
     normalize_strings: bool = True
 
     is_pyi: bool = False
     normalize_strings: bool = True

-    current_line: Line = Factory(Line)
+    current_line: Line = field(default_factory=Line)

     remove_u_prefix: bool = False
 
     def line(self, indent: int = 0) -> Iterator[Line]:
     remove_u_prefix: bool = False
 
     def line(self, indent: int = 0) -> Iterator[Line]:


@@ -1623,46 +1755,13 @@ class LineGenerator(Visitor[Line]):
                 self.current_line.append(node)
         yield from super().visit_default(node)
 
                 self.current_line.append(node)
         yield from super().visit_default(node)
 

-    def visit_atom(self, node: Node) -> Iterator[Line]:
-        # Always make parentheses invisible around a single node, because it should
-        # not be needed (except in the case of yield, where removing the parentheses
-        # produces a SyntaxError).
-        if (
-            len(node.children) == 3
-            and isinstance(node.children[0], Leaf)
-            and node.children[0].type == token.LPAR
-            and isinstance(node.children[2], Leaf)
-            and node.children[2].type == token.RPAR
-            and isinstance(node.children[1], Leaf)
-            and not (
-                node.children[1].type == token.NAME
-                and node.children[1].value == "yield"
-            )
-        ):
-            node.children[0].value = ""
-            node.children[2].value = ""
-        yield from super().visit_default(node)
-
-    def visit_factor(self, node: Node) -> Iterator[Line]:
-        """Force parentheses between a unary op and a binary power:
-
-        -2 ** 8 -> -(2 ** 8)
-        """
-        child = node.children[1]
-        if child.type == syms.power and len(child.children) == 3:
-            lpar = Leaf(token.LPAR, "(")
-            rpar = Leaf(token.RPAR, ")")
-            index = child.remove() or 0
-            node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
-        yield from self.visit_default(node)
-
-    def visit_INDENT(self, node: Node) -> Iterator[Line]:
+    def visit_INDENT(self, node: Leaf) -> Iterator[Line]:

         """Increase indentation level, maybe yield a line."""
         # In blib2to3 INDENT never holds comments.
         yield from self.line(+1)
         yield from self.visit_default(node)
 
         """Increase indentation level, maybe yield a line."""
         # In blib2to3 INDENT never holds comments.
         yield from self.line(+1)
         yield from self.visit_default(node)
 

-    def visit_DEDENT(self, node: Node) -> Iterator[Line]:
+    def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:

         """Decrease indentation level, maybe yield a line."""
         # The current line might still wait for trailing comments.  At DEDENT time
         # there won't be any (they would be prefixes on the preceding NEWLINE).
         """Decrease indentation level, maybe yield a line."""
         # The current line might still wait for trailing comments.  At DEDENT time
         # there won't be any (they would be prefixes on the preceding NEWLINE).


@@ -1755,7 +1854,24 @@ class LineGenerator(Visitor[Line]):
             yield from self.line()
         yield from self.visit_default(leaf)
 
             yield from self.line()
         yield from self.visit_default(leaf)
 

-    def __attrs_post_init__(self) -> None:
+    def visit_factor(self, node: Node) -> Iterator[Line]:
+        """Force parentheses between a unary op and a binary power:
+
+        -2 ** 8 -> -(2 ** 8)
+        """
+        _operator, operand = node.children
+        if (
+            operand.type == syms.power
+            and len(operand.children) == 3
+            and operand.children[1].type == token.DOUBLESTAR
+        ):
+            lpar = Leaf(token.LPAR, "(")
+            rpar = Leaf(token.RPAR, ")")
+            index = operand.remove() or 0
+            node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))
+        yield from self.visit_default(node)
+
+    def __post_init__(self) -> None:

         """You are in a twisty little maze of passages."""
         v = self.visit_stmt
         Ø: Set[str] = set()
         """You are in a twisty little maze of passages."""
         v = self.visit_stmt
         Ø: Set[str] = set()


@@ -1845,7 +1961,7 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str:  # noqa: C901
                     # that, too.
                     return prevp.prefix
 
                     # that, too.
                     return prevp.prefix
 

-        elif prevp.type in STARS:
+        elif prevp.type in VARARGS_SPECIALS:

             if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):
                 return NO
 
             if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):
                 return NO
 


@@ -1935,7 +2051,7 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str:  # noqa: C901
             if not prevp or prevp.type == token.LPAR:
                 return NO
 
             if not prevp or prevp.type == token.LPAR:
                 return NO
 

-        elif prev.type in {token.EQUAL} | STARS:
+        elif prev.type in {token.EQUAL} | VARARGS_SPECIALS:

             return NO
 
     elif p.type == syms.decorator:
             return NO
 
     elif p.type == syms.decorator:


@@ -2302,9 +2418,13 @@ def split_line(
     line_str = str(line).strip("\n")
 
     if (
     line_str = str(line).strip("\n")
 
     if (

-        not line.contains_inner_type_comments()
+        not line.contains_uncollapsable_type_comments()

         and not line.should_explode
         and not line.should_explode

-        and is_line_short_enough(line, line_length=line_length, line_str=line_str)
+        and not line.is_collection_with_optional_trailing_comma
+        and (
+            is_line_short_enough(line, line_length=line_length, line_str=line_str)
+            or line.contains_unsplittable_type_ignore()
+        )

     ):
         yield line
         return
     ):
         yield line
         return


@@ -2324,7 +2444,9 @@ def split_line(
             # All splits failed, best effort split with no omits.
             # This mostly happens to multiline strings that are by definition
             # reported as not fitting a single line.
             # All splits failed, best effort split with no omits.
             # This mostly happens to multiline strings that are by definition
             # reported as not fitting a single line.

-            yield from right_hand_split(line, line_length, features=features)
+            # line_length=1 here was historically a bug that somehow became a feature.
+            # See #762 and #781 for the full story.
+            yield from right_hand_split(line, line_length=1, features=features)

 
         if line.inside_brackets:
             split_funcs = [delimiter_split, standalone_comment_split, rhs]
 
         if line.inside_brackets:
             split_funcs = [delimiter_split, standalone_comment_split, rhs]


@@ -2367,7 +2489,7 @@ def left_hand_split(line: Line, features: Collection[Feature] = ()) -> Iterator[
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []
     current_leaves = head_leaves
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []
     current_leaves = head_leaves

-    matching_bracket = None
+    matching_bracket: Optional[Leaf] = None

     for leaf in line.leaves:
         if (
             current_leaves is body_leaves
     for leaf in line.leaves:
         if (
             current_leaves is body_leaves


@@ -2410,8 +2532,8 @@ def right_hand_split(
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []
     current_leaves = tail_leaves
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []
     current_leaves = tail_leaves

-    opening_bracket = None
-    closing_bracket = None
+    opening_bracket: Optional[Leaf] = None
+    closing_bracket: Optional[Leaf] = None

     for leaf in reversed(line.leaves):
         if current_leaves is body_leaves:
             if leaf is opening_bracket:
     for leaf in reversed(line.leaves):
         if current_leaves is body_leaves:
             if leaf is opening_bracket:


@@ -2523,20 +2645,22 @@ def bracket_split_build_line(
             # Since body is a new indent level, remove spurious leading whitespace.
             normalize_prefix(leaves[0], inside_brackets=True)
             # Ensure a trailing comma for imports and standalone function arguments, but
             # Since body is a new indent level, remove spurious leading whitespace.
             normalize_prefix(leaves[0], inside_brackets=True)
             # Ensure a trailing comma for imports and standalone function arguments, but

-            # be careful not to add one after any comments.
-            no_commas = original.is_def and not any(
-                l.type == token.COMMA for l in leaves
+            # be careful not to add one after any comments or within type annotations.
+            no_commas = (
+                original.is_def
+                and opening_bracket.value == "("
+                and not any(l.type == token.COMMA for l in leaves)

             )
 
             if original.is_import or no_commas:
                 for i in range(len(leaves) - 1, -1, -1):
                     if leaves[i].type == STANDALONE_COMMENT:
                         continue
             )
 
             if original.is_import or no_commas:
                 for i in range(len(leaves) - 1, -1, -1):
                     if leaves[i].type == STANDALONE_COMMENT:
                         continue

-                    elif leaves[i].type == token.COMMA:
-                        break
-                    else:
+
+                    if leaves[i].type != token.COMMA:

                         leaves.insert(i + 1, Leaf(token.COMMA, ","))
                         leaves.insert(i + 1, Leaf(token.COMMA, ","))

-                        break
+                    break
+

     # Populate the line
     for leaf in leaves:
         result.append(leaf, preformatted=True)
     # Populate the line
     for leaf in leaves:
         result.append(leaf, preformatted=True)


@@ -2677,12 +2801,12 @@ def is_import(leaf: Leaf) -> bool:
     )
 
 
     )
 
 

-def is_type_comment(leaf: Leaf) -> bool:
+def is_type_comment(leaf: Leaf, suffix: str = "") -> bool:

     """Return True if the given leaf is a special comment.
     Only returns true for type comments for now."""
     t = leaf.type
     v = leaf.value
     """Return True if the given leaf is a special comment.
     Only returns true for type comments for now."""
     t = leaf.type
     v = leaf.value

-    return t in {token.COMMENT, t == STANDALONE_COMMENT} and v.startswith("# type:")
+    return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix)

 
 
 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:
 
 
 def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None:


@@ -2780,6 +2904,7 @@ def normalize_string_quotes(leaf: Leaf) -> None:
             if "\\" in str(m):
                 # Do not introduce backslashes in interpolated expressions
                 return
             if "\\" in str(m):
                 # Do not introduce backslashes in interpolated expressions
                 return

+

     if new_quote == '"""' and new_body[-1:] == '"':
         # edge case:
         new_body = new_body[:-1] + '\\"'
     if new_quote == '"""' and new_body[-1:] == '"':
         # edge case:
         new_body = new_body[:-1] + '\\"'


@@ -2852,7 +2977,6 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
         if pc.value in FMT_OFF:
             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
             return
         if pc.value in FMT_OFF:
             # This `node` has a prefix with `# fmt: off`, don't mess with parens.
             return

-

     check_lpar = False
     for index, child in enumerate(list(node.children)):
         # Add parentheses around long tuple unpacking in assignments.
     check_lpar = False
     for index, child in enumerate(list(node.children)):
         # Add parentheses around long tuple unpacking in assignments.


@@ -2864,18 +2988,14 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
             check_lpar = True
 
         if check_lpar:
             check_lpar = True
 
         if check_lpar:

+            if is_walrus_assignment(child):
+                continue
+

             if child.type == syms.atom:
                 if maybe_make_parens_invisible_in_atom(child, parent=node):
             if child.type == syms.atom:
                 if maybe_make_parens_invisible_in_atom(child, parent=node):

-                    lpar = Leaf(token.LPAR, "")
-                    rpar = Leaf(token.RPAR, "")
-                    index = child.remove() or 0
-                    node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+                    wrap_in_parentheses(node, child, visible=False)

             elif is_one_tuple(child):
             elif is_one_tuple(child):

-                # wrap child in visible parentheses
-                lpar = Leaf(token.LPAR, "(")
-                rpar = Leaf(token.RPAR, ")")
-                child.remove()
-                node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+                wrap_in_parentheses(node, child, visible=True)

             elif node.type == syms.import_from:
                 # "import from" nodes store parentheses directly as part of
                 # the statement
             elif node.type == syms.import_from:
                 # "import from" nodes store parentheses directly as part of
                 # the statement


@@ -2890,15 +3010,7 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
                 break
 
             elif not (isinstance(child, Leaf) and is_multiline_string(child)):
                 break
 
             elif not (isinstance(child, Leaf) and is_multiline_string(child)):

-                # wrap child in invisible parentheses
-                lpar = Leaf(token.LPAR, "")
-                rpar = Leaf(token.RPAR, "")
-                index = child.remove() or 0
-                prefix = child.prefix
-                child.prefix = ""
-                new_child = Node(syms.atom, [lpar, child, rpar])
-                new_child.prefix = prefix
-                node.insert_child(index, new_child)
+                wrap_in_parentheses(node, child, visible=False)

 
         check_lpar = isinstance(child, Leaf) and child.value in parens_after
 
 
         check_lpar = isinstance(child, Leaf) and child.value in parens_after
 


@@ -2942,7 +3054,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool:
                     # That happens when one of the `ignored_nodes` ended with a NEWLINE
                     # leaf (possibly followed by a DEDENT).
                     hidden_value = hidden_value[:-1]
                     # That happens when one of the `ignored_nodes` ended with a NEWLINE
                     # leaf (possibly followed by a DEDENT).
                     hidden_value = hidden_value[:-1]

-                first_idx = None
+                first_idx: Optional[int] = None

                 for ignored in ignored_nodes:
                     index = ignored.remove()
                     if first_idx is None:
                 for ignored in ignored_nodes:
                     index = ignored.remove()
                     if first_idx is None:


@@ -2971,9 +3083,14 @@ def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
     """
     container: Optional[LN] = container_of(leaf)
     while container is not None and container.type != token.ENDMARKER:
     """
     container: Optional[LN] = container_of(leaf)
     while container is not None and container.type != token.ENDMARKER:

+        is_fmt_on = False

         for comment in list_comments(container.prefix, is_endmarker=False):
             if comment.value in FMT_ON:
         for comment in list_comments(container.prefix, is_endmarker=False):
             if comment.value in FMT_ON:

-                return
+                is_fmt_on = True
+            elif comment.value in FMT_OFF:
+                is_fmt_on = False
+        if is_fmt_on:
+            return

 
         yield container
 
 
         yield container
 


@@ -2982,6 +3099,8 @@ def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
 
 def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
     """If it's safe, make the parens in the atom `node` invisible, recursively.
 
 def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
     """If it's safe, make the parens in the atom `node` invisible, recursively.

+    Additionally, remove repeated, adjacent invisible parens from the atom `node`
+    as they are redundant.

 
     Returns whether the node should itself be wrapped in invisible parentheses.
 
 
     Returns whether the node should itself be wrapped in invisible parentheses.
 


@@ -2998,16 +3117,40 @@ def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:
     first = node.children[0]
     last = node.children[-1]
     if first.type == token.LPAR and last.type == token.RPAR:
     first = node.children[0]
     last = node.children[-1]
     if first.type == token.LPAR and last.type == token.RPAR:

+        middle = node.children[1]

         # make parentheses invisible
         first.value = ""  # type: ignore
         last.value = ""  # type: ignore
         # make parentheses invisible
         first.value = ""  # type: ignore
         last.value = ""  # type: ignore

-        if len(node.children) > 1:
-            maybe_make_parens_invisible_in_atom(node.children[1], parent=parent)
+        maybe_make_parens_invisible_in_atom(middle, parent=parent)
+
+        if is_atom_with_invisible_parens(middle):
+            # Strip the invisible parens from `middle` by replacing
+            # it with the child in-between the invisible parens
+            middle.replace(middle.children[1])
+

         return False
 
     return True
 
 
         return False
 
     return True
 
 

+def is_atom_with_invisible_parens(node: LN) -> bool:
+    """Given a `LN`, determines whether it's an atom `node` with invisible
+    parens. Useful in dedupe-ing and normalizing parens.
+    """
+    if isinstance(node, Leaf) or node.type != syms.atom:
+        return False
+
+    first, last = node.children[0], node.children[-1]
+    return (
+        isinstance(first, Leaf)
+        and first.type == token.LPAR
+        and first.value == ""
+        and isinstance(last, Leaf)
+        and last.type == token.RPAR
+        and last.value == ""
+    )
+
+

 def is_empty_tuple(node: LN) -> bool:
     """Return True if `node` holds an empty tuple."""
     return (
 def is_empty_tuple(node: LN) -> bool:
     """Return True if `node` holds an empty tuple."""
     return (


@@ -3018,18 +3161,43 @@ def is_empty_tuple(node: LN) -> bool:
     )
 
 
     )
 
 

+def unwrap_singleton_parenthesis(node: LN) -> Optional[LN]:
+    """Returns `wrapped` if `node` is of the shape ( wrapped ).
+
+    Parenthesis can be optional. Returns None otherwise"""
+    if len(node.children) != 3:
+        return None
+
+    lpar, wrapped, rpar = node.children
+    if not (lpar.type == token.LPAR and rpar.type == token.RPAR):
+        return None
+
+    return wrapped
+
+
+def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None:
+    """Wrap `child` in parentheses.
+
+    This replaces `child` with an atom holding the parentheses and the old
+    child.  That requires moving the prefix.
+
+    If `visible` is False, the leaves will be valueless (and thus invisible).
+    """
+    lpar = Leaf(token.LPAR, "(" if visible else "")
+    rpar = Leaf(token.RPAR, ")" if visible else "")
+    prefix = child.prefix
+    child.prefix = ""
+    index = child.remove() or 0
+    new_child = Node(syms.atom, [lpar, child, rpar])
+    new_child.prefix = prefix
+    parent.insert_child(index, new_child)
+
+

 def is_one_tuple(node: LN) -> bool:
     """Return True if `node` holds a tuple with one element, with or without parens."""
     if node.type == syms.atom:
 def is_one_tuple(node: LN) -> bool:
     """Return True if `node` holds a tuple with one element, with or without parens."""
     if node.type == syms.atom:

-        if len(node.children) != 3:
-            return False
-
-        lpar, gexp, rpar = node.children
-        if not (
-            lpar.type == token.LPAR
-            and gexp.type == syms.testlist_gexp
-            and rpar.type == token.RPAR
-        ):
+        gexp = unwrap_singleton_parenthesis(node)
+        if gexp is None or gexp.type != syms.testlist_gexp:

             return False
 
         return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA
             return False
 
         return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA


@@ -3041,6 +3209,12 @@ def is_one_tuple(node: LN) -> bool:
     )
 
 
     )
 
 

+def is_walrus_assignment(node: LN) -> bool:
+    """Return True iff `node` is of the shape ( test := test )"""
+    inner = unwrap_singleton_parenthesis(node)
+    return inner is not None and inner.type == syms.namedexpr_test
+
+

 def is_yield(node: LN) -> bool:
     """Return True if `node` holds a `yield` or `yield from` expression."""
     if node.type == syms.yield_expr:
 def is_yield(node: LN) -> bool:
     """Return True if `node` holds a `yield` or `yield from` expression."""
     if node.type == syms.yield_expr:


@@ -3070,7 +3244,7 @@ def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:
     extended iterable unpacking (PEP 3132) and additional unpacking
     generalizations (PEP 448).
     """
     extended iterable unpacking (PEP 3132) and additional unpacking
     generalizations (PEP 448).
     """

-    if leaf.type not in STARS or not leaf.parent:
+    if leaf.type not in VARARGS_SPECIALS or not leaf.parent:

         return False
 
     p = leaf.parent
         return False
 
     p = leaf.parent


@@ -3185,8 +3359,9 @@ def get_features_used(node: Node) -> Set[Feature]:
 
     Currently looking for:
     - f-strings;
 
     Currently looking for:
     - f-strings;

-    - underscores in numeric literals; and
-    - trailing commas after * or ** in function signatures and calls.
+    - underscores in numeric literals;
+    - trailing commas after * or ** in function signatures and calls;
+    - positional only arguments in function signatures and lambdas;

     """
     features: Set[Feature] = set()
     for n in node.pre_order():
     """
     features: Set[Feature] = set()
     for n in node.pre_order():


@@ -3199,6 +3374,13 @@ def get_features_used(node: Node) -> Set[Feature]:
             if "_" in n.value:  # type: ignore
                 features.add(Feature.NUMERIC_UNDERSCORES)
 
             if "_" in n.value:  # type: ignore
                 features.add(Feature.NUMERIC_UNDERSCORES)
 

+        elif n.type == token.SLASH:
+            if n.parent and n.parent.type in {syms.typedargslist, syms.arglist}:
+                features.add(Feature.POS_ONLY_ARGUMENTS)
+
+        elif n.type == token.COLONEQUAL:
+            features.add(Feature.ASSIGNMENT_EXPRESSIONS)
+

         elif (
             n.type in {syms.typedargslist, syms.arglist}
             and n.children
         elif (
             n.type in {syms.typedargslist, syms.arglist}
             and n.children


@@ -3243,8 +3425,8 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf
     yield omit
 
     length = 4 * line.depth
     yield omit
 
     length = 4 * line.depth

-    opening_bracket = None
-    closing_bracket = None
+    opening_bracket: Optional[Leaf] = None
+    closing_bracket: Optional[Leaf] = None

     inner_brackets: Set[LeafID] = set()
     for index, leaf, leaf_length in enumerate_with_length(line, reversed=True):
         length += leaf_length
     inner_brackets: Set[LeafID] = set()
     for index, leaf, leaf_length in enumerate_with_length(line, reversed=True):
         length += leaf_length


@@ -3288,19 +3470,23 @@ def get_future_imports(node: Node) -> Set[str]:
             if isinstance(child, Leaf):
                 if child.type == token.NAME:
                     yield child.value
             if isinstance(child, Leaf):
                 if child.type == token.NAME:
                     yield child.value

+

             elif child.type == syms.import_as_name:
                 orig_name = child.children[0]
                 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
                 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
                 yield orig_name.value
             elif child.type == syms.import_as_name:
                 orig_name = child.children[0]
                 assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
                 assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
                 yield orig_name.value

+

             elif child.type == syms.import_as_names:
                 yield from get_imports_from_children(child.children)
             elif child.type == syms.import_as_names:
                 yield from get_imports_from_children(child.children)

+

             else:
                 raise AssertionError("Invalid syntax parsing imports")
 
     for child in node.children:
         if child.type != syms.simple_stmt:
             break
             else:
                 raise AssertionError("Invalid syntax parsing imports")
 
     for child in node.children:
         if child.type != syms.simple_stmt:
             break

+

         first_child = child.children[0]
         if isinstance(first_child, Leaf):
             # Continue looking if we see a docstring; otherwise stop.
         first_child = child.children[0]
         if isinstance(first_child, Leaf):
             # Continue looking if we see a docstring; otherwise stop.


@@ -3310,24 +3496,39 @@ def get_future_imports(node: Node) -> Set[str]:
                 and child.children[1].type == token.NEWLINE
             ):
                 continue
                 and child.children[1].type == token.NEWLINE
             ):
                 continue

-            else:
-                break
+
+            break
+

         elif first_child.type == syms.import_from:
             module_name = first_child.children[1]
             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
                 break
         elif first_child.type == syms.import_from:
             module_name = first_child.children[1]
             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
                 break

+

             imports |= set(get_imports_from_children(first_child.children[3:]))
         else:
             break
             imports |= set(get_imports_from_children(first_child.children[3:]))
         else:
             break

+

     return imports
 
 
     return imports
 
 

+@lru_cache()
+def get_gitignore(root: Path) -> PathSpec:
+    """ Return a PathSpec matching gitignore content if present."""
+    gitignore = root / ".gitignore"
+    lines: List[str] = []
+    if gitignore.is_file():
+        with gitignore.open() as gf:
+            lines = gf.readlines()
+    return PathSpec.from_lines("gitwildmatch", lines)
+
+

 def gen_python_files_in_dir(
     path: Path,
     root: Path,
     include: Pattern[str],
     exclude: Pattern[str],
     report: "Report",
 def gen_python_files_in_dir(
     path: Path,
     root: Path,
     include: Pattern[str],
     exclude: Pattern[str],
     report: "Report",

+    gitignore: PathSpec,

 ) -> Iterator[Path]:
     """Generate all files under `path` whose paths are not excluded by the
     `exclude` regex, but are included by the `include` regex.
 ) -> Iterator[Path]:
     """Generate all files under `path` whose paths are not excluded by the
     `exclude` regex, but are included by the `include` regex.


@@ -3338,8 +3539,18 @@ def gen_python_files_in_dir(
     """
     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
     for child in path.iterdir():
     """
     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
     for child in path.iterdir():

+        # First ignore files matching .gitignore
+        if gitignore.match_file(child.as_posix()):
+            report.path_ignored(child, f"matches the .gitignore file content")
+            continue
+
+        # Then ignore with `exclude` option.

         try:
             normalized_path = "/" + child.resolve().relative_to(root).as_posix()
         try:
             normalized_path = "/" + child.resolve().relative_to(root).as_posix()

+        except OSError as e:
+            report.path_ignored(child, f"cannot be read because {e}")
+            continue
+

         except ValueError:
             if child.is_symlink():
                 report.path_ignored(
         except ValueError:
             if child.is_symlink():
                 report.path_ignored(


@@ -3351,13 +3562,16 @@ def gen_python_files_in_dir(
 
         if child.is_dir():
             normalized_path += "/"
 
         if child.is_dir():
             normalized_path += "/"

+

         exclude_match = exclude.search(normalized_path)
         if exclude_match and exclude_match.group(0):
             report.path_ignored(child, f"matches the --exclude regular expression")
             continue
 
         if child.is_dir():
         exclude_match = exclude.search(normalized_path)
         if exclude_match and exclude_match.group(0):
             report.path_ignored(child, f"matches the --exclude regular expression")
             continue
 
         if child.is_dir():

-            yield from gen_python_files_in_dir(child, root, include, exclude, report)
+            yield from gen_python_files_in_dir(
+                child, root, include, exclude, report, gitignore
+            )

 
         elif child.is_file():
             include_match = include.search(normalized_path)
 
         elif child.is_file():
             include_match = include.search(normalized_path)


@@ -3480,32 +3694,59 @@ class Report:
         return ", ".join(report) + "."
 
 
         return ", ".join(report) + "."
 
 

-def parse_ast(src: str) -> Union[ast3.AST, ast27.AST]:
-    for feature_version in (7, 6):
-        try:
-            return ast3.parse(src, feature_version=feature_version)
-        except SyntaxError:
-            continue
+def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]:
+    filename = "<unknown>"
+    if sys.version_info >= (3, 8):
+        # TODO: support Python 4+ ;)
+        for minor_version in range(sys.version_info[1], 4, -1):
+            try:
+                return ast.parse(src, filename, feature_version=(3, minor_version))
+            except SyntaxError:
+                continue
+    else:
+        for feature_version in (7, 6):
+            try:
+                return ast3.parse(src, filename, feature_version=feature_version)
+            except SyntaxError:
+                continue

 
     return ast27.parse(src)
 
 
 
     return ast27.parse(src)
 
 

+def _fixup_ast_constants(
+    node: Union[ast.AST, ast3.AST, ast27.AST]
+) -> Union[ast.AST, ast3.AST, ast27.AST]:
+    """Map ast nodes deprecated in 3.8 to Constant."""
+    if isinstance(node, (ast.Str, ast3.Str, ast27.Str, ast.Bytes, ast3.Bytes)):
+        return ast.Constant(value=node.s)
+
+    if isinstance(node, (ast.Num, ast3.Num, ast27.Num)):
+        return ast.Constant(value=node.n)
+
+    if isinstance(node, (ast.NameConstant, ast3.NameConstant)):
+        return ast.Constant(value=node.value)
+
+    return node
+
+

 def assert_equivalent(src: str, dst: str) -> None:
     """Raise AssertionError if `src` and `dst` aren't equivalent."""
 
 def assert_equivalent(src: str, dst: str) -> None:
     """Raise AssertionError if `src` and `dst` aren't equivalent."""
 

-    def _v(node: Union[ast3.AST, ast27.AST], depth: int = 0) -> Iterator[str]:
+    def _v(node: Union[ast.AST, ast3.AST, ast27.AST], depth: int = 0) -> Iterator[str]:

         """Simple visitor generating strings to compare ASTs by content."""
         """Simple visitor generating strings to compare ASTs by content."""

+
+        node = _fixup_ast_constants(node)
+

         yield f"{'  ' * depth}{node.__class__.__name__}("
 
         yield f"{'  ' * depth}{node.__class__.__name__}("
 

-        for field in sorted(node._fields):
+        for field in sorted(node._fields):  # noqa: F402

             # TypeIgnore has only one field 'lineno' which breaks this comparison
             # TypeIgnore has only one field 'lineno' which breaks this comparison

-            if isinstance(node, (ast3.TypeIgnore, ast27.TypeIgnore)):
+            type_ignore_classes = (ast3.TypeIgnore, ast27.TypeIgnore)
+            if sys.version_info >= (3, 8):
+                type_ignore_classes += (ast.TypeIgnore,)
+            if isinstance(node, type_ignore_classes):

                 break
 
                 break
 

-            # Ignore str kind which is case sensitive / and ignores unicode_literals
-            if isinstance(node, (ast3.Str, ast27.Str, ast3.Bytes)) and field == "kind":
-                continue
-

             try:
                 value = getattr(node, field)
             except AttributeError:
             try:
                 value = getattr(node, field)
             except AttributeError:


@@ -3519,15 +3760,16 @@ def assert_equivalent(src: str, dst: str) -> None:
                     # parentheses and they change the AST.
                     if (
                         field == "targets"
                     # parentheses and they change the AST.
                     if (
                         field == "targets"

-                        and isinstance(node, (ast3.Delete, ast27.Delete))
-                        and isinstance(item, (ast3.Tuple, ast27.Tuple))
+                        and isinstance(node, (ast.Delete, ast3.Delete, ast27.Delete))
+                        and isinstance(item, (ast.Tuple, ast3.Tuple, ast27.Tuple))

                     ):
                         for item in item.elts:
                             yield from _v(item, depth + 2)
                     ):
                         for item in item.elts:
                             yield from _v(item, depth + 2)

-                    elif isinstance(item, (ast3.AST, ast27.AST)):
+
+                    elif isinstance(item, (ast.AST, ast3.AST, ast27.AST)):

                         yield from _v(item, depth + 2)
 
                         yield from _v(item, depth + 2)
 

-            elif isinstance(value, (ast3.AST, ast27.AST)):
+            elif isinstance(value, (ast.AST, ast3.AST, ast27.AST)):

                 yield from _v(value, depth + 2)
 
             else:
                 yield from _v(value, depth + 2)
 
             else:


@@ -3549,7 +3791,7 @@ def assert_equivalent(src: str, dst: str) -> None:
         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
         raise AssertionError(
             f"INTERNAL ERROR: Black produced invalid code: {exc}. "
         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
         raise AssertionError(
             f"INTERNAL ERROR: Black produced invalid code: {exc}. "

-            f"Please report a bug on https://github.com/python/black/issues.  "
+            f"Please report a bug on https://github.com/psf/black/issues.  "

             f"This invalid output might be helpful: {log}"
         ) from None
 
             f"This invalid output might be helpful: {log}"
         ) from None
 


@@ -3560,7 +3802,7 @@ def assert_equivalent(src: str, dst: str) -> None:
         raise AssertionError(
             f"INTERNAL ERROR: Black produced code that is not equivalent to "
             f"the source.  "
         raise AssertionError(
             f"INTERNAL ERROR: Black produced code that is not equivalent to "
             f"the source.  "

-            f"Please report a bug on https://github.com/python/black/issues.  "
+            f"Please report a bug on https://github.com/psf/black/issues.  "

             f"This diff might be helpful: {log}"
         ) from None
 
             f"This diff might be helpful: {log}"
         ) from None
 


@@ -3576,11 +3818,12 @@ def assert_stable(src: str, dst: str, mode: FileMode) -> None:
         raise AssertionError(
             f"INTERNAL ERROR: Black produced different code on the second pass "
             f"of the formatter.  "
         raise AssertionError(
             f"INTERNAL ERROR: Black produced different code on the second pass "
             f"of the formatter.  "

-            f"Please report a bug on https://github.com/python/black/issues.  "
+            f"Please report a bug on https://github.com/psf/black/issues.  "

             f"This diff might be helpful: {log}"
         ) from None
 
 
             f"This diff might be helpful: {log}"
         ) from None
 
 

+@mypyc_attr(patchable=True)

 def dump_to_file(*output: str) -> str:
     """Dump `output` to a temporary file. Return path to the file."""
     with tempfile.NamedTemporaryFile(
 def dump_to_file(*output: str) -> str:
     """Dump `output` to a temporary file. Return path to the file."""
     with tempfile.NamedTemporaryFile(


@@ -3593,6 +3836,15 @@ def dump_to_file(*output: str) -> str:
     return f.name
 
 
     return f.name
 
 

+@contextmanager
+def nullcontext() -> Iterator[None]:
+    """Return an empty context manager.
+
+    To be used like `nullcontext` in Python 3.7.
+    """
+    yield
+
+

 def diff(a: str, b: str, a_name: str, b_name: str) -> str:
     """Return a unified diff string between strings `a` and `b`."""
     import difflib
 def diff(a: str, b: str, a_name: str, b_name: str) -> str:
     """Return a unified diff string between strings `a` and `b`."""
     import difflib


@@ -3604,7 +3856,7 @@ def diff(a: str, b: str, a_name: str, b_name: str) -> str:
     )
 
 
     )
 
 

-def cancel(tasks: Iterable[asyncio.Task]) -> None:
+def cancel(tasks: Iterable["asyncio.Task[Any]"]) -> None:

     """asyncio signal handler that cancels all `tasks` and reports to stderr."""
     err("Aborted!")
     for task in tasks:
     """asyncio signal handler that cancels all `tasks` and reports to stderr."""
     err("Aborted!")
     for task in tasks:


@@ -3653,7 +3905,8 @@ def re_compile_maybe_verbose(regex: str) -> Pattern[str]:
     """
     if "\n" in regex:
         regex = "(?x)" + regex
     """
     if "\n" in regex:
         regex = "(?x)" + regex

-    return re.compile(regex)
+    compiled: Pattern[str] = re.compile(regex)
+    return compiled

 
 
 def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:
 
 
 def enumerate_reversed(sequence: Sequence[T]) -> Iterator[Tuple[Index, T]]:


@@ -3841,7 +4094,7 @@ def read_cache(mode: FileMode) -> Cache:
     with cache_file.open("rb") as fobj:
         try:
             cache: Cache = pickle.load(fobj)
     with cache_file.open("rb") as fobj:
         try:
             cache: Cache = pickle.load(fobj)

-        except pickle.UnpicklingError:
+        except (pickle.UnpicklingError, ValueError):

             return {}
 
     return cache
             return {}
 
     return cache


@@ -3876,7 +4129,7 @@ def write_cache(cache: Cache, sources: Iterable[Path], mode: FileMode) -> None:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
         new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
         with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
         new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
         with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:

-            pickle.dump(new_cache, f, protocol=pickle.HIGHEST_PROTOCOL)
+            pickle.dump(new_cache, f, protocol=4)

         os.replace(f.name, cache_file)
     except OSError:
         pass
         os.replace(f.name, cache_file)
     except OSError:
         pass