Add link to the pyproject.toml for setting up pre-commit hook (#885)

[etc/vim.git] / black.py

diff --git a/black.py b/black.py
index b7276663679737cdae4a77a11df7a2c683e2b800..635eba207cd0c2a695cab27d68326216a90325f5 100644 (file)
--- a/black.py
+++ b/black.py
@@ -16,6 +16,7 @@ import signal
 import sys
 import tempfile
 import tokenize
 import sys
 import tempfile
 import tokenize

+import traceback

 from typing import (
     Any,
     Callable,
 from typing import (
     Any,
     Callable,


@@ -40,6 +41,7 @@ from appdirs import user_cache_dir
 from attr import dataclass, evolve, Factory
 import click
 import toml
 from attr import dataclass, evolve, Factory
 import click
 import toml

+from typed_ast import ast3, ast27

 
 # lib2to3 fork
 from blib2to3.pytree import Node, Leaf, type_repr
 
 # lib2to3 fork
 from blib2to3.pytree import Node, Leaf, type_repr


@@ -68,7 +70,7 @@ LeafID = int
 Priority = int
 Index = int
 LN = Union[Leaf, Node]
 Priority = int
 Index = int
 LN = Union[Leaf, Node]

-SplitFunc = Callable[["Line", bool], Iterator["Line"]]
+SplitFunc = Callable[["Line", Collection["Feature"]], Iterator["Line"]]

 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]
 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]


@@ -133,31 +135,46 @@ class Feature(Enum):
     UNICODE_LITERALS = 1
     F_STRINGS = 2
     NUMERIC_UNDERSCORES = 3
     UNICODE_LITERALS = 1
     F_STRINGS = 2
     NUMERIC_UNDERSCORES = 3

-    TRAILING_COMMA = 4
+    TRAILING_COMMA_IN_CALL = 4
+    TRAILING_COMMA_IN_DEF = 5
+    # The following two feature-flags are mutually exclusive, and exactly one should be
+    # set for every version of python.
+    ASYNC_IDENTIFIERS = 6
+    ASYNC_KEYWORDS = 7

 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {

-    TargetVersion.PY27: set(),
-    TargetVersion.PY33: {Feature.UNICODE_LITERALS},
-    TargetVersion.PY34: {Feature.UNICODE_LITERALS},
-    TargetVersion.PY35: {Feature.UNICODE_LITERALS, Feature.TRAILING_COMMA},
+    TargetVersion.PY27: {Feature.ASYNC_IDENTIFIERS},
+    TargetVersion.PY33: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
+    TargetVersion.PY34: {Feature.UNICODE_LITERALS, Feature.ASYNC_IDENTIFIERS},
+    TargetVersion.PY35: {
+        Feature.UNICODE_LITERALS,
+        Feature.TRAILING_COMMA_IN_CALL,
+        Feature.ASYNC_IDENTIFIERS,
+    },

     TargetVersion.PY36: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,
     TargetVersion.PY36: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,

-        Feature.TRAILING_COMMA,
+        Feature.TRAILING_COMMA_IN_CALL,
+        Feature.TRAILING_COMMA_IN_DEF,
+        Feature.ASYNC_IDENTIFIERS,

     },
     TargetVersion.PY37: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,
     },
     TargetVersion.PY37: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,

-        Feature.TRAILING_COMMA,
+        Feature.TRAILING_COMMA_IN_CALL,
+        Feature.TRAILING_COMMA_IN_DEF,
+        Feature.ASYNC_KEYWORDS,

     },
     TargetVersion.PY38: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,
     },
     TargetVersion.PY38: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,

-        Feature.TRAILING_COMMA,
+        Feature.TRAILING_COMMA_IN_CALL,
+        Feature.TRAILING_COMMA_IN_DEF,
+        Feature.ASYNC_KEYWORDS,

     },
 }
 
     },
 }
 


@@ -227,6 +244,7 @@ def read_pyproject_toml(
 
 
 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))
 
 
 @click.command(context_settings=dict(help_option_names=["-h", "--help"]))

+@click.option("-c", "--code", type=str, help="Format the code passed in as a string.")

 @click.option(
     "-l",
     "--line-length",
 @click.option(
     "-l",
     "--line-length",


@@ -319,7 +337,7 @@ def read_pyproject_toml(
     "--quiet",
     is_flag=True,
     help=(
     "--quiet",
     is_flag=True,
     help=(

-        "Don't emit non-error messages to stderr. Errors are still emitted, "
+        "Don't emit non-error messages to stderr. Errors are still emitted; "

         "silence those with 2>/dev/null."
     ),
 )
         "silence those with 2>/dev/null."
     ),
 )


@@ -353,6 +371,7 @@ def read_pyproject_toml(
 @click.pass_context
 def main(
     ctx: click.Context,
 @click.pass_context
 def main(
     ctx: click.Context,

+    code: Optional[str],

     line_length: int,
     target_version: List[TargetVersion],
     check: bool,
     line_length: int,
     target_version: List[TargetVersion],
     check: bool,


@@ -393,6 +412,9 @@ def main(
     )
     if config and verbose:
         out(f"Using configuration from {config}.", bold=False, fg="blue")
     )
     if config and verbose:
         out(f"Using configuration from {config}.", bold=False, fg="blue")

+    if code is not None:
+        print(format_str(code, mode=mode))
+        ctx.exit(0)

     try:
         include_regex = re_compile_maybe_verbose(include)
     except re.error:
     try:
         include_regex = re_compile_maybe_verbose(include)
     except re.error:


@@ -431,25 +453,12 @@ def main(
             report=report,
         )
     else:
             report=report,
         )
     else:

-        loop = asyncio.get_event_loop()
-        executor = ProcessPoolExecutor(max_workers=os.cpu_count())
-        try:
-            loop.run_until_complete(
-                schedule_formatting(
-                    sources=sources,
-                    fast=fast,
-                    write_back=write_back,
-                    mode=mode,
-                    report=report,
-                    loop=loop,
-                    executor=executor,
-                )
-            )
-        finally:
-            shutdown(loop)
+        reformat_many(
+            sources=sources, fast=fast, write_back=write_back, mode=mode, report=report
+        )
+

     if verbose or not quiet:
     if verbose or not quiet:

-        bang = "💥 💔 💥" if report.return_code else "✨ 🍰 ✨"
-        out(f"All done! {bang}")
+        out("Oh no! 💥 💔 💥" if report.return_code else "All done! ✨ 🍰 ✨")

         click.secho(str(report), err=True)
     ctx.exit(report.return_code)
 
         click.secho(str(report), err=True)
     ctx.exit(report.return_code)
 


@@ -459,8 +468,7 @@ def reformat_one(
 ) -> None:
     """Reformat a single file under `src` without spawning child processes.
 
 ) -> None:
     """Reformat a single file under `src` without spawning child processes.
 

-    If `quiet` is True, non-error messages are not output. `line_length`,
-    `write_back`, `fast` and `pyi` options are passed to
+    `fast`, `write_back`, and `mode` options are passed to

     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
     """
     try:
     :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.
     """
     try:


@@ -488,6 +496,36 @@ def reformat_one(
         report.failed(src, str(exc))
 
 
         report.failed(src, str(exc))
 
 

+def reformat_many(
+    sources: Set[Path],
+    fast: bool,
+    write_back: WriteBack,
+    mode: FileMode,
+    report: "Report",
+) -> None:
+    """Reformat multiple files using a ProcessPoolExecutor."""
+    loop = asyncio.get_event_loop()
+    worker_count = os.cpu_count()
+    if sys.platform == "win32":
+        # Work around https://bugs.python.org/issue26903
+        worker_count = min(worker_count, 61)
+    executor = ProcessPoolExecutor(max_workers=worker_count)
+    try:
+        loop.run_until_complete(
+            schedule_formatting(
+                sources=sources,
+                fast=fast,
+                write_back=write_back,
+                mode=mode,
+                report=report,
+                loop=loop,
+                executor=executor,
+            )
+        )
+    finally:
+        shutdown(loop)
+
+

 async def schedule_formatting(
     sources: Set[Path],
     fast: bool,
 async def schedule_formatting(
     sources: Set[Path],
     fast: bool,


@@ -522,12 +560,14 @@ async def schedule_formatting(
         manager = Manager()
         lock = manager.Lock()
     tasks = {
         manager = Manager()
         lock = manager.Lock()
     tasks = {

-        loop.run_in_executor(
-            executor, format_file_in_place, src, fast, mode, write_back, lock
+        asyncio.ensure_future(
+            loop.run_in_executor(
+                executor, format_file_in_place, src, fast, mode, write_back, lock
+            )

         ): src
         for src in sorted(sources)
     }
         ): src
         for src in sorted(sources)
     }

-    pending: Iterable[asyncio.Task] = tasks.keys()
+    pending: Iterable[asyncio.Future] = tasks.keys()

     try:
         loop.add_signal_handler(signal.SIGINT, cancel, pending)
         loop.add_signal_handler(signal.SIGTERM, cancel, pending)
     try:
         loop.add_signal_handler(signal.SIGINT, cancel, pending)
         loop.add_signal_handler(signal.SIGTERM, cancel, pending)


@@ -568,7 +608,7 @@ def format_file_in_place(
 
     If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
     code to the file.
 
     If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
     code to the file.

-    `line_length` and `fast` options are passed to :func:`format_file_contents`.
+    `mode` and `fast` options are passed to :func:`format_file_contents`.

     """
     if src.suffix == ".pyi":
         mode = evolve(mode, is_pyi=True)
     """
     if src.suffix == ".pyi":
         mode = evolve(mode, is_pyi=True)


@@ -646,7 +686,7 @@ def format_file_contents(
 
     If `fast` is False, additionally confirm that the reformatted code is
     valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.
 
     If `fast` is False, additionally confirm that the reformatted code is
     valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.

-    `line_length` is passed to :func:`format_str`.
+    `mode` is passed to :func:`format_str`.

     """
     if src_contents.strip() == "":
         raise NothingChanged
     """
     if src_contents.strip() == "":
         raise NothingChanged


@@ -664,10 +704,11 @@ def format_file_contents(
 def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
     """Reformat a string and return new contents.
 
 def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
     """Reformat a string and return new contents.
 

-    `line_length` determines how many characters per line are allowed.
+    `mode` determines formatting options, such as how many characters per line are
+    allowed.

     """
     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
     """
     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)

-    dst_contents = ""
+    dst_contents = []

     future_imports = get_future_imports(src_node)
     if mode.target_versions:
         versions = mode.target_versions
     future_imports = get_future_imports(src_node)
     if mode.target_versions:
         versions = mode.target_versions


@@ -683,19 +724,22 @@ def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
     empty_line = Line()
     after = 0
     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
     empty_line = Line()
     after = 0

+    split_line_features = {
+        feature
+        for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
+        if supports_feature(versions, feature)
+    }

     for current_line in lines.visit(src_node):
         for _ in range(after):
     for current_line in lines.visit(src_node):
         for _ in range(after):

-            dst_contents += str(empty_line)
+            dst_contents.append(str(empty_line))

         before, after = elt.maybe_empty_lines(current_line)
         for _ in range(before):
         before, after = elt.maybe_empty_lines(current_line)
         for _ in range(before):

-            dst_contents += str(empty_line)
+            dst_contents.append(str(empty_line))

         for line in split_line(
         for line in split_line(

-            current_line,
-            line_length=mode.line_length,
-            supports_trailing_commas=supports_feature(versions, Feature.TRAILING_COMMA),
+            current_line, line_length=mode.line_length, features=split_line_features

         ):
         ):

-            dst_contents += str(line)
-    return dst_contents
+            dst_contents.append(str(line))
+    return "".join(dst_contents)

 
 
 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
 
 
 def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:


@@ -715,24 +759,42 @@ def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
         return tiow.read(), encoding, newline
 
 
         return tiow.read(), encoding, newline
 
 

-GRAMMARS = [
-    pygram.python_grammar_no_print_statement_no_exec_statement,
-    pygram.python_grammar_no_print_statement,
-    pygram.python_grammar,
-]
-
-

 def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
     if not target_versions:
 def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
     if not target_versions:

-        return GRAMMARS
-    elif all(not version.is_python2() for version in target_versions):
-        # Python 2-compatible code, so don't try Python 3 grammar.
+        # No target_version specified, so try all grammars.

         return [
         return [

+            # Python 3.7+
+            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
+            # Python 3.0-3.6

             pygram.python_grammar_no_print_statement_no_exec_statement,
             pygram.python_grammar_no_print_statement_no_exec_statement,

+            # Python 2.7 with future print_function import
+            pygram.python_grammar_no_print_statement,
+            # Python 2.7
+            pygram.python_grammar,
+        ]
+    elif all(version.is_python2() for version in target_versions):
+        # Python 2-only code, so try Python 2 grammars.
+        return [
+            # Python 2.7 with future print_function import

             pygram.python_grammar_no_print_statement,
             pygram.python_grammar_no_print_statement,

+            # Python 2.7
+            pygram.python_grammar,

         ]
     else:
         ]
     else:

-        return [pygram.python_grammar]
+        # Python 3-compatible code, so only try Python 3 grammar.
+        grammars = []
+        # If we have to parse both, try to parse async as a keyword first
+        if not supports_feature(target_versions, Feature.ASYNC_IDENTIFIERS):
+            # Python 3.7+
+            grammars.append(
+                pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords  # noqa: B950
+            )
+        if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
+            # Python 3.0-3.6
+            grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
+        # At least one of the above branches must have been taken, because every Python
+        # version has exactly one of the two 'ASYNC_*' flags
+        return grammars

 
 
 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
 
 
 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:


@@ -999,7 +1061,7 @@ class BracketTracker:
         """Return True if there is an yet unmatched open bracket on the line."""
         return bool(self.bracket_match)
 
         """Return True if there is an yet unmatched open bracket on the line."""
         return bool(self.bracket_match)
 

-    def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> int:
+    def max_delimiter_priority(self, exclude: Iterable[LeafID] = ()) -> Priority:

         """Return the highest priority of a delimiter found on the line.
 
         Values are consistent with what `is_split_*_delimiter()` return.
         """Return the highest priority of a delimiter found on the line.
 
         Values are consistent with what `is_split_*_delimiter()` return.


@@ -1007,7 +1069,7 @@ class BracketTracker:
         """
         return max(v for k, v in self.delimiters.items() if k not in exclude)
 
         """
         return max(v for k, v in self.delimiters.items() if k not in exclude)
 

-    def delimiter_count_with_priority(self, priority: int = 0) -> int:
+    def delimiter_count_with_priority(self, priority: Priority = 0) -> int:

         """Return the number of delimiters with the given `priority`.
 
         If no `priority` is passed, defaults to max priority on the line.
         """Return the number of delimiters with the given `priority`.
 
         If no `priority` is passed, defaults to max priority on the line.


@@ -1540,6 +1602,26 @@ class LineGenerator(Visitor[Line]):
                 self.current_line.append(node)
         yield from super().visit_default(node)
 
                 self.current_line.append(node)
         yield from super().visit_default(node)
 

+    def visit_atom(self, node: Node) -> Iterator[Line]:
+        # Always make parentheses invisible around a single node, because it should
+        # not be needed (except in the case of yield, where removing the parentheses
+        # produces a SyntaxError).
+        if (
+            len(node.children) == 3
+            and isinstance(node.children[0], Leaf)
+            and node.children[0].type == token.LPAR
+            and isinstance(node.children[2], Leaf)
+            and node.children[2].type == token.RPAR
+            and isinstance(node.children[1], Leaf)
+            and not (
+                node.children[1].type == token.NAME
+                and node.children[1].value == "yield"
+            )
+        ):
+            node.children[0].value = ""
+            node.children[2].value = ""
+        yield from super().visit_default(node)
+

     def visit_INDENT(self, node: Node) -> Iterator[Line]:
         """Increase indentation level, maybe yield a line."""
         # In blib2to3 INDENT never holds comments.
     def visit_INDENT(self, node: Node) -> Iterator[Line]:
         """Increase indentation level, maybe yield a line."""
         # In blib2to3 INDENT never holds comments.


@@ -1953,7 +2035,7 @@ def container_of(leaf: Leaf) -> LN:
     return container
 
 
     return container
 
 

-def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int:
+def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority:

     """Return the priority of the `leaf` delimiter, given a line break after it.
 
     The delimiter priorities returned here are from those delimiters that would
     """Return the priority of the `leaf` delimiter, given a line break after it.
 
     The delimiter priorities returned here are from those delimiters that would


@@ -1967,7 +2049,7 @@ def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int
     return 0
 
 
     return 0
 
 

-def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int:
+def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority:

     """Return the priority of the `leaf` delimiter, given a line break before it.
 
     The delimiter priorities returned here are from those delimiters that would
     """Return the priority of the `leaf` delimiter, given a line break before it.
 
     The delimiter priorities returned here are from those delimiters that would


@@ -2117,15 +2199,21 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:
 
     consumed = 0
     nlines = 0
 
     consumed = 0
     nlines = 0

+    ignored_lines = 0

     for index, line in enumerate(prefix.split("\n")):
         consumed += len(line) + 1  # adding the length of the split '\n'
         line = line.lstrip()
         if not line:
             nlines += 1
         if not line.startswith("#"):
     for index, line in enumerate(prefix.split("\n")):
         consumed += len(line) + 1  # adding the length of the split '\n'
         line = line.lstrip()
         if not line:
             nlines += 1
         if not line.startswith("#"):

+            # Escaped newlines outside of a comment are not really newlines at
+            # all. We treat a single-line comment following an escaped newline
+            # as a simple trailing comment.
+            if line.endswith("\\"):
+                ignored_lines += 1

             continue
 
             continue
 

-        if index == 0 and not is_endmarker:
+        if index == ignored_lines and not is_endmarker:

             comment_type = token.COMMENT  # simple trailing comment
         else:
             comment_type = STANDALONE_COMMENT
             comment_type = token.COMMENT  # simple trailing comment
         else:
             comment_type = STANDALONE_COMMENT


@@ -2162,7 +2250,7 @@ def split_line(
     line: Line,
     line_length: int,
     inner: bool = False,
     line: Line,
     line_length: int,
     inner: bool = False,

-    supports_trailing_commas: bool = False,
+    features: Collection[Feature] = (),

 ) -> Iterator[Line]:
     """Split a `line` into potentially many lines.
 
 ) -> Iterator[Line]:
     """Split a `line` into potentially many lines.
 


@@ -2171,7 +2259,7 @@ def split_line(
     current `line`, possibly transitively. This means we can fallback to splitting
     by delimiters if the LHS/RHS don't yield any results.
 
     current `line`, possibly transitively. This means we can fallback to splitting
     by delimiters if the LHS/RHS don't yield any results.
 

-    If `supports_trailing_commas` is True, splitting may use the TRAILING_COMMA feature.
+    `features` are syntactical features that may be used in the output.

     """
     if line.is_comment:
         yield line
     """
     if line.is_comment:
         yield line


@@ -2192,13 +2280,9 @@ def split_line(
         split_funcs = [left_hand_split]
     else:
 
         split_funcs = [left_hand_split]
     else:
 

-        def rhs(line: Line, supports_trailing_commas: bool = False) -> Iterator[Line]:
+        def rhs(line: Line, features: Collection[Feature]) -> Iterator[Line]:

             for omit in generate_trailers_to_omit(line, line_length):
             for omit in generate_trailers_to_omit(line, line_length):

-                lines = list(
-                    right_hand_split(
-                        line, line_length, supports_trailing_commas, omit=omit
-                    )
-                )
+                lines = list(right_hand_split(line, line_length, features, omit=omit))

                 if is_line_short_enough(lines[0], line_length=line_length):
                     yield from lines
                     return
                 if is_line_short_enough(lines[0], line_length=line_length):
                     yield from lines
                     return


@@ -2206,7 +2290,7 @@ def split_line(
             # All splits failed, best effort split with no omits.
             # This mostly happens to multiline strings that are by definition
             # reported as not fitting a single line.
             # All splits failed, best effort split with no omits.
             # This mostly happens to multiline strings that are by definition
             # reported as not fitting a single line.

-            yield from right_hand_split(line, supports_trailing_commas)
+            yield from right_hand_split(line, line_length, features=features)

 
         if line.inside_brackets:
             split_funcs = [delimiter_split, standalone_comment_split, rhs]
 
         if line.inside_brackets:
             split_funcs = [delimiter_split, standalone_comment_split, rhs]


@@ -2218,16 +2302,13 @@ def split_line(
         # split altogether.
         result: List[Line] = []
         try:
         # split altogether.
         result: List[Line] = []
         try:

-            for l in split_func(line, supports_trailing_commas):
+            for l in split_func(line, features):

                 if str(l).strip("\n") == line_str:
                     raise CannotSplit("Split function returned an unchanged result")
 
                 result.extend(
                     split_line(
                 if str(l).strip("\n") == line_str:
                     raise CannotSplit("Split function returned an unchanged result")
 
                 result.extend(
                     split_line(

-                        l,
-                        line_length=line_length,
-                        inner=True,
-                        supports_trailing_commas=supports_trailing_commas,
+                        l, line_length=line_length, inner=True, features=features

                     )
                 )
         except CannotSplit:
                     )
                 )
         except CannotSplit:


@@ -2241,9 +2322,7 @@ def split_line(
         yield line
 
 
         yield line
 
 

-def left_hand_split(
-    line: Line, supports_trailing_commas: bool = False
-) -> Iterator[Line]:
+def left_hand_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:

     """Split line into many lines, starting with the first matching bracket pair.
 
     Note: this usually looks weird, only use this for function definitions.
     """Split line into many lines, starting with the first matching bracket pair.
 
     Note: this usually looks weird, only use this for function definitions.


@@ -2282,7 +2361,7 @@ def left_hand_split(
 def right_hand_split(
     line: Line,
     line_length: int,
 def right_hand_split(
     line: Line,
     line_length: int,

-    supports_trailing_commas: bool = False,
+    features: Collection[Feature] = (),

     omit: Collection[LeafID] = (),
 ) -> Iterator[Line]:
     """Split line into many lines, starting with the last matching bracket pair.
     omit: Collection[LeafID] = (),
 ) -> Iterator[Line]:
     """Split line into many lines, starting with the last matching bracket pair.


@@ -2341,12 +2420,7 @@ def right_hand_split(
     ):
         omit = {id(closing_bracket), *omit}
         try:
     ):
         omit = {id(closing_bracket), *omit}
         try:

-            yield from right_hand_split(
-                line,
-                line_length,
-                supports_trailing_commas=supports_trailing_commas,
-                omit=omit,
-            )
+            yield from right_hand_split(line, line_length, features=features, omit=omit)

             return
 
         except CannotSplit:
             return
 
         except CannotSplit:


@@ -2414,10 +2488,17 @@ def bracket_split_build_line(
         if leaves:
             # Since body is a new indent level, remove spurious leading whitespace.
             normalize_prefix(leaves[0], inside_brackets=True)
         if leaves:
             # Since body is a new indent level, remove spurious leading whitespace.
             normalize_prefix(leaves[0], inside_brackets=True)

-            # Ensure a trailing comma when expected.
+            # Ensure a trailing comma for imports, but be careful not to add one after
+            # any comments.

             if original.is_import:
             if original.is_import:

-                if leaves[-1].type != token.COMMA:
-                    leaves.append(Leaf(token.COMMA, ","))
+                for i in range(len(leaves) - 1, -1, -1):
+                    if leaves[i].type == STANDALONE_COMMENT:
+                        continue
+                    elif leaves[i].type == token.COMMA:
+                        break
+                    else:
+                        leaves.insert(i + 1, Leaf(token.COMMA, ","))
+                        break

     # Populate the line
     for leaf in leaves:
         result.append(leaf, preformatted=True)
     # Populate the line
     for leaf in leaves:
         result.append(leaf, preformatted=True)


@@ -2435,10 +2516,8 @@ def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
     """
 
     @wraps(split_func)
     """
 
     @wraps(split_func)

-    def split_wrapper(
-        line: Line, supports_trailing_commas: bool = False
-    ) -> Iterator[Line]:
-        for l in split_func(line, supports_trailing_commas):
+    def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
+        for l in split_func(line, features):

             normalize_prefix(l.leaves[0], inside_brackets=True)
             yield l
 
             normalize_prefix(l.leaves[0], inside_brackets=True)
             yield l
 


@@ -2446,13 +2525,11 @@ def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
 
 
 @dont_increase_indentation
 
 
 @dont_increase_indentation

-def delimiter_split(
-    line: Line, supports_trailing_commas: bool = False
-) -> Iterator[Line]:
+def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:

     """Split according to delimiters of the highest priority.
 
     """Split according to delimiters of the highest priority.
 

-    If `supports_trailing_commas` is True, the split will add trailing commas
-    also in function signatures that contain `*` and `**`.
+    If the appropriate Features are given, the split will add trailing commas
+    also in function signatures and calls that contain `*` and `**`.

     """
     try:
         last_leaf = line.leaves[-1]
     """
     try:
         last_leaf = line.leaves[-1]


@@ -2491,10 +2568,16 @@ def delimiter_split(
             yield from append_to_line(comment_after)
 
         lowest_depth = min(lowest_depth, leaf.bracket_depth)
             yield from append_to_line(comment_after)
 
         lowest_depth = min(lowest_depth, leaf.bracket_depth)

-        if leaf.bracket_depth == lowest_depth and is_vararg(
-            leaf, within=VARARGS_PARENTS
-        ):
-            trailing_comma_safe = trailing_comma_safe and supports_trailing_commas
+        if leaf.bracket_depth == lowest_depth:
+            if is_vararg(leaf, within={syms.typedargslist}):
+                trailing_comma_safe = (
+                    trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
+                )
+            elif is_vararg(leaf, within={syms.arglist, syms.argument}):
+                trailing_comma_safe = (
+                    trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
+                )
+

         leaf_priority = bt.delimiters.get(id(leaf))
         if leaf_priority == delimiter_priority:
             yield current_line
         leaf_priority = bt.delimiters.get(id(leaf))
         if leaf_priority == delimiter_priority:
             yield current_line


@@ -2513,7 +2596,7 @@ def delimiter_split(
 
 @dont_increase_indentation
 def standalone_comment_split(
 
 @dont_increase_indentation
 def standalone_comment_split(

-    line: Line, supports_trailing_commas: bool = False
+    line: Line, features: Collection[Feature] = ()

 ) -> Iterator[Line]:
     """Split standalone comments from the rest of the line."""
     if not line.contains_standalone_comments(0):
 ) -> Iterator[Line]:
     """Split standalone comments from the rest of the line."""
     if not line.contains_standalone_comments(0):


@@ -2646,7 +2729,15 @@ def normalize_string_quotes(leaf: Leaf) -> None:
         new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
         new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
     if "f" in prefix.casefold():
         new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
         new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
     if "f" in prefix.casefold():

-        matches = re.findall(r"[^{]\{(.*?)\}[^}]", new_body)
+        matches = re.findall(
+            r"""
+            (?:[^{]|^)\{  # start of the string or a non-{ followed by a single {
+                ([^{].*?)  # contents of the brackets except if begins with {{
+            \}(?:[^}]|$)  # A } followed by end of the string or a non-}
+            """,
+            new_body,
+            re.VERBOSE,
+        )

         for m in matches:
             if "\\" in str(m):
                 # Do not introduce backslashes in interpolated expressions
         for m in matches:
             if "\\" in str(m):
                 # Do not introduce backslashes in interpolated expressions


@@ -2713,7 +2804,7 @@ def format_float_or_int_string(text: str) -> str:
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 

-    `parens_after` is a set of string leaf values immeditely after which parens
+    `parens_after` is a set of string leaf values immediately after which parens

     should be put.
 
     Standardizes on visible parentheses for single-element tuples, and keeps
     should be put.
 
     Standardizes on visible parentheses for single-element tuples, and keeps


@@ -2726,9 +2817,17 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
 
     check_lpar = False
     for index, child in enumerate(list(node.children)):
 
     check_lpar = False
     for index, child in enumerate(list(node.children)):

+        # Add parentheses around long tuple unpacking in assignments.
+        if (
+            index == 0
+            and isinstance(child, Node)
+            and child.type == syms.testlist_star_expr
+        ):
+            check_lpar = True
+

         if check_lpar:
             if child.type == syms.atom:
         if check_lpar:
             if child.type == syms.atom:

-                if maybe_make_parens_invisible_in_atom(child):
+                if maybe_make_parens_invisible_in_atom(child, parent=node):

                     lpar = Leaf(token.LPAR, "")
                     rpar = Leaf(token.RPAR, "")
                     index = child.remove() or 0
                     lpar = Leaf(token.LPAR, "")
                     rpar = Leaf(token.RPAR, "")
                     index = child.remove() or 0


@@ -2757,7 +2856,11 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
                 lpar = Leaf(token.LPAR, "")
                 rpar = Leaf(token.RPAR, "")
                 index = child.remove() or 0
                 lpar = Leaf(token.LPAR, "")
                 rpar = Leaf(token.RPAR, "")
                 index = child.remove() or 0

-                node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))
+                prefix = child.prefix
+                child.prefix = ""
+                new_child = Node(syms.atom, [lpar, child, rpar])
+                new_child.prefix = prefix
+                node.insert_child(index, new_child)

 
         check_lpar = isinstance(child, Leaf) and child.value in parens_after
 
 
         check_lpar = isinstance(child, Leaf) and child.value in parens_after
 


@@ -2839,7 +2942,7 @@ def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
         container = container.next_sibling
 
 
         container = container.next_sibling
 
 

-def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
+def maybe_make_parens_invisible_in_atom(node: LN, parent: LN) -> bool:

     """If it's safe, make the parens in the atom `node` invisible, recursively.
 
     Returns whether the node should itself be wrapped in invisible parentheses.
     """If it's safe, make the parens in the atom `node` invisible, recursively.
 
     Returns whether the node should itself be wrapped in invisible parentheses.


@@ -2849,7 +2952,7 @@ def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
         node.type != syms.atom
         or is_empty_tuple(node)
         or is_one_tuple(node)
         node.type != syms.atom
         or is_empty_tuple(node)
         or is_one_tuple(node)

-        or is_yield(node)
+        or (is_yield(node) and parent.type != syms.expr_stmt)

         or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
     ):
         return False
         or max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY
     ):
         return False


@@ -2861,7 +2964,7 @@ def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
         first.value = ""  # type: ignore
         last.value = ""  # type: ignore
         if len(node.children) > 1:
         first.value = ""  # type: ignore
         last.value = ""  # type: ignore
         if len(node.children) > 1:

-            maybe_make_parens_invisible_in_atom(node.children[1])
+            maybe_make_parens_invisible_in_atom(node.children[1], parent=parent)

         return False
 
     return True
         return False
 
     return True


@@ -2979,7 +3082,7 @@ def is_stub_body(node: LN) -> bool:
     )
 
 
     )
 
 

-def max_delimiter_priority_in_atom(node: LN) -> int:
+def max_delimiter_priority_in_atom(node: LN) -> Priority:

     """Return maximum delimiter priority inside `node`.
 
     This is specific to atoms with contents contained in a pair of parentheses.
     """Return maximum delimiter priority inside `node`.
 
     This is specific to atoms with contents contained in a pair of parentheses.


@@ -3063,14 +3166,19 @@ def get_features_used(node: Node) -> Set[Feature]:
             and n.children
             and n.children[-1].type == token.COMMA
         ):
             and n.children
             and n.children[-1].type == token.COMMA
         ):

+            if n.type == syms.typedargslist:
+                feature = Feature.TRAILING_COMMA_IN_DEF
+            else:
+                feature = Feature.TRAILING_COMMA_IN_CALL
+

             for ch in n.children:
                 if ch.type in STARS:
             for ch in n.children:
                 if ch.type in STARS:

-                    features.add(Feature.TRAILING_COMMA)
+                    features.add(feature)

 
                 if ch.type == syms.argument:
                     for argch in ch.children:
                         if argch.type in STARS:
 
                 if ch.type == syms.argument:
                     for argch in ch.children:
                         if argch.type in STARS:

-                            features.add(Feature.TRAILING_COMMA)
+                            features.add(feature)

 
     return features
 
 
     return features
 


@@ -3150,7 +3258,7 @@ def get_future_imports(node: Node) -> Set[str]:
             elif child.type == syms.import_as_names:
                 yield from get_imports_from_children(child.children)
             else:
             elif child.type == syms.import_as_names:
                 yield from get_imports_from_children(child.children)
             else:

-                assert False, "Invalid syntax parsing imports"
+                raise AssertionError("Invalid syntax parsing imports")

 
     for child in node.children:
         if child.type != syms.simple_stmt:
 
     for child in node.children:
         if child.type != syms.simple_stmt:


@@ -3334,17 +3442,32 @@ class Report:
         return ", ".join(report) + "."
 
 
         return ", ".join(report) + "."
 
 

+def parse_ast(src: str) -> Union[ast3.AST, ast27.AST]:
+    for feature_version in (7, 6):
+        try:
+            return ast3.parse(src, feature_version=feature_version)
+        except SyntaxError:
+            continue
+
+    return ast27.parse(src)
+
+

 def assert_equivalent(src: str, dst: str) -> None:
     """Raise AssertionError if `src` and `dst` aren't equivalent."""
 
 def assert_equivalent(src: str, dst: str) -> None:
     """Raise AssertionError if `src` and `dst` aren't equivalent."""
 

-    import ast
-    import traceback
-
-    def _v(node: ast.AST, depth: int = 0) -> Iterator[str]:
+    def _v(node: Union[ast3.AST, ast27.AST], depth: int = 0) -> Iterator[str]:

         """Simple visitor generating strings to compare ASTs by content."""
         yield f"{'  ' * depth}{node.__class__.__name__}("
 
         for field in sorted(node._fields):
         """Simple visitor generating strings to compare ASTs by content."""
         yield f"{'  ' * depth}{node.__class__.__name__}("
 
         for field in sorted(node._fields):

+            # TypeIgnore has only one field 'lineno' which breaks this comparison
+            if isinstance(node, (ast3.TypeIgnore, ast27.TypeIgnore)):
+                break
+
+            # Ignore str kind which is case sensitive / and ignores unicode_literals
+            if isinstance(node, (ast3.Str, ast27.Str, ast3.Bytes)) and field == "kind":
+                continue
+

             try:
                 value = getattr(node, field)
             except AttributeError:
             try:
                 value = getattr(node, field)
             except AttributeError:


@@ -3358,15 +3481,15 @@ def assert_equivalent(src: str, dst: str) -> None:
                     # parentheses and they change the AST.
                     if (
                         field == "targets"
                     # parentheses and they change the AST.
                     if (
                         field == "targets"

-                        and isinstance(node, ast.Delete)
-                        and isinstance(item, ast.Tuple)
+                        and isinstance(node, (ast3.Delete, ast27.Delete))
+                        and isinstance(item, (ast3.Tuple, ast27.Tuple))

                     ):
                         for item in item.elts:
                             yield from _v(item, depth + 2)
                     ):
                         for item in item.elts:
                             yield from _v(item, depth + 2)

-                    elif isinstance(item, ast.AST):
+                    elif isinstance(item, (ast3.AST, ast27.AST)):

                         yield from _v(item, depth + 2)
 
                         yield from _v(item, depth + 2)
 

-            elif isinstance(value, ast.AST):
+            elif isinstance(value, (ast3.AST, ast27.AST)):

                 yield from _v(value, depth + 2)
 
             else:
                 yield from _v(value, depth + 2)
 
             else:


@@ -3375,22 +3498,20 @@ def assert_equivalent(src: str, dst: str) -> None:
         yield f"{'  ' * depth})  # /{node.__class__.__name__}"
 
     try:
         yield f"{'  ' * depth})  # /{node.__class__.__name__}"
 
     try:

-        src_ast = ast.parse(src)
+        src_ast = parse_ast(src)

     except Exception as exc:
     except Exception as exc:

-        major, minor = sys.version_info[:2]

         raise AssertionError(
         raise AssertionError(

-            f"cannot use --safe with this file; failed to parse source file "
-            f"with Python {major}.{minor}'s builtin AST. Re-run with --fast "
-            f"or stop using deprecated Python 2 syntax. AST error message: {exc}"
+            f"cannot use --safe with this file; failed to parse source file.  "
+            f"AST error message: {exc}"

         )
 
     try:
         )
 
     try:

-        dst_ast = ast.parse(dst)
+        dst_ast = parse_ast(dst)

     except Exception as exc:
         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
         raise AssertionError(
             f"INTERNAL ERROR: Black produced invalid code: {exc}. "
     except Exception as exc:
         log = dump_to_file("".join(traceback.format_tb(exc.__traceback__)), dst)
         raise AssertionError(
             f"INTERNAL ERROR: Black produced invalid code: {exc}. "

-            f"Please report a bug on https://github.com/ambv/black/issues.  "
+            f"Please report a bug on https://github.com/python/black/issues.  "

             f"This invalid output might be helpful: {log}"
         ) from None
 
             f"This invalid output might be helpful: {log}"
         ) from None
 


@@ -3401,7 +3522,7 @@ def assert_equivalent(src: str, dst: str) -> None:
         raise AssertionError(
             f"INTERNAL ERROR: Black produced code that is not equivalent to "
             f"the source.  "
         raise AssertionError(
             f"INTERNAL ERROR: Black produced code that is not equivalent to "
             f"the source.  "

-            f"Please report a bug on https://github.com/ambv/black/issues.  "
+            f"Please report a bug on https://github.com/python/black/issues.  "

             f"This diff might be helpful: {log}"
         ) from None
 
             f"This diff might be helpful: {log}"
         ) from None
 


@@ -3417,15 +3538,13 @@ def assert_stable(src: str, dst: str, mode: FileMode) -> None:
         raise AssertionError(
             f"INTERNAL ERROR: Black produced different code on the second pass "
             f"of the formatter.  "
         raise AssertionError(
             f"INTERNAL ERROR: Black produced different code on the second pass "
             f"of the formatter.  "

-            f"Please report a bug on https://github.com/ambv/black/issues.  "
+            f"Please report a bug on https://github.com/python/black/issues.  "

             f"This diff might be helpful: {log}"
         ) from None
 
 
 def dump_to_file(*output: str) -> str:
     """Dump `output` to a temporary file. Return path to the file."""
             f"This diff might be helpful: {log}"
         ) from None
 
 
 def dump_to_file(*output: str) -> str:
     """Dump `output` to a temporary file. Return path to the file."""

-    import tempfile
-

     with tempfile.NamedTemporaryFile(
         mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8"
     ) as f:
     with tempfile.NamedTemporaryFile(
         mode="w", prefix="blk_", suffix=".log", delete=False, encoding="utf8"
     ) as f:


@@ -3523,7 +3642,6 @@ def enumerate_with_length(
         if "\n" in leaf.value:
             return  # Multiline strings, we can't continue.
 
         if "\n" in leaf.value:
             return  # Multiline strings, we can't continue.
 

-        comment: Optional[Leaf]

         for comment in line.comments_after(leaf):
             length += len(comment.value)
 
         for comment in line.comments_after(leaf):
             length += len(comment.value)