Refactor Travis (#614)

[etc/vim.git] / black.py

diff --git a/black.py b/black.py
index 7e39c92009d2aefcf3b2910f9dce1fcca8b2cbe2..fd0364ba8d5422865f7047fbb2a6cb58f8bfd33f 100644 (file)
--- a/black.py
+++ b/black.py
@@ -5,6 +5,7 @@ from datetime import datetime
 from enum import Enum, Flag
 from functools import lru_cache, partial, wraps
 import io
 from enum import Enum, Flag
 from functools import lru_cache, partial, wraps
 import io

+import itertools

 import keyword
 import logging
 from multiprocessing import Manager
 import keyword
 import logging
 from multiprocessing import Manager


@@ -20,6 +21,7 @@ from typing import (
     Callable,
     Collection,
     Dict,
     Callable,
     Collection,
     Dict,

+    Generator,

     Generic,
     Iterable,
     Iterator,
     Generic,
     Iterable,
     Iterator,


@@ -46,10 +48,10 @@ from blib2to3.pgen2 import driver, token
 from blib2to3.pgen2.parse import ParseError
 
 
 from blib2to3.pgen2.parse import ParseError
 
 

-__version__ = "18.6b2"
+__version__ = "18.9b0"

 DEFAULT_LINE_LENGTH = 88
 DEFAULT_EXCLUDES = (
 DEFAULT_LINE_LENGTH = 88
 DEFAULT_EXCLUDES = (

-    r"/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist)/"
+    r"/(\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)/"

 )
 DEFAULT_INCLUDES = r"\.pyi?$"
 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
 )
 DEFAULT_INCLUDES = r"\.pyi?$"
 CACHE_DIR = Path(user_cache_dir("black", version=__version__))


@@ -78,26 +80,27 @@ syms = pygram.python_symbols
 
 
 class NothingChanged(UserWarning):
 
 
 class NothingChanged(UserWarning):

-    """Raised by :func:`format_file` when reformatted code is the same as source."""
+    """Raised when reformatted code is the same as source."""

 
 
 class CannotSplit(Exception):
 
 
 class CannotSplit(Exception):

-    """A readable split that fits the allotted line length is impossible.
+    """A readable split that fits the allotted line length is impossible."""

 
 

-    Raised by :func:`left_hand_split`, :func:`right_hand_split`, and
-    :func:`delimiter_split`.
-    """
+
+class InvalidInput(ValueError):
+    """Raised when input source code fails all parse attempts."""

 
 
 class WriteBack(Enum):
     NO = 0
     YES = 1
     DIFF = 2
 
 
 class WriteBack(Enum):
     NO = 0
     YES = 1
     DIFF = 2

+    CHECK = 3

 
     @classmethod
     def from_configuration(cls, *, check: bool, diff: bool) -> "WriteBack":
         if check and not diff:
 
     @classmethod
     def from_configuration(cls, *, check: bool, diff: bool) -> "WriteBack":
         if check and not diff:

-            return cls.NO
+            return cls.CHECK

 
         return cls.DIFF if diff else cls.YES
 
 
         return cls.DIFF if diff else cls.YES
 


@@ -113,10 +116,16 @@ class FileMode(Flag):
     PYTHON36 = 1
     PYI = 2
     NO_STRING_NORMALIZATION = 4
     PYTHON36 = 1
     PYI = 2
     NO_STRING_NORMALIZATION = 4

+    NO_NUMERIC_UNDERSCORE_NORMALIZATION = 8

 
     @classmethod
     def from_configuration(
 
     @classmethod
     def from_configuration(

-        cls, *, py36: bool, pyi: bool, skip_string_normalization: bool
+        cls,
+        *,
+        py36: bool,
+        pyi: bool,
+        skip_string_normalization: bool,
+        skip_numeric_underscore_normalization: bool,

     ) -> "FileMode":
         mode = cls.AUTO_DETECT
         if py36:
     ) -> "FileMode":
         mode = cls.AUTO_DETECT
         if py36:


@@ -125,6 +134,8 @@ class FileMode(Flag):
             mode |= cls.PYI
         if skip_string_normalization:
             mode |= cls.NO_STRING_NORMALIZATION
             mode |= cls.PYI
         if skip_string_normalization:
             mode |= cls.NO_STRING_NORMALIZATION

+        if skip_numeric_underscore_normalization:
+            mode |= cls.NO_NUMERIC_UNDERSCORE_NORMALIZATION

         return mode
 
 
         return mode
 
 


@@ -168,7 +179,7 @@ def read_pyproject_toml(
     "--line-length",
     type=int,
     default=DEFAULT_LINE_LENGTH,
     "--line-length",
     type=int,
     default=DEFAULT_LINE_LENGTH,

-    help="How many character per line to allow.",
+    help="How many characters per line to allow.",

     show_default=True,
 )
 @click.option(
     show_default=True,
 )
 @click.option(


@@ -194,6 +205,12 @@ def read_pyproject_toml(
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )

+@click.option(
+    "-N",
+    "--skip-numeric-underscore-normalization",
+    is_flag=True,
+    help="Don't normalize underscores in numeric literals.",
+)

 @click.option(
     "--check",
     is_flag=True,
 @click.option(
     "--check",
     is_flag=True,


@@ -284,6 +301,7 @@ def main(
     pyi: bool,
     py36: bool,
     skip_string_normalization: bool,
     pyi: bool,
     py36: bool,
     skip_string_normalization: bool,

+    skip_numeric_underscore_normalization: bool,

     quiet: bool,
     verbose: bool,
     include: str,
     quiet: bool,
     verbose: bool,
     include: str,


@@ -294,7 +312,10 @@ def main(
     """The uncompromising code formatter."""
     write_back = WriteBack.from_configuration(check=check, diff=diff)
     mode = FileMode.from_configuration(
     """The uncompromising code formatter."""
     write_back = WriteBack.from_configuration(check=check, diff=diff)
     mode = FileMode.from_configuration(

-        py36=py36, pyi=pyi, skip_string_normalization=skip_string_normalization
+        py36=py36,
+        pyi=pyi,
+        skip_string_normalization=skip_string_normalization,
+        skip_numeric_underscore_normalization=skip_numeric_underscore_normalization,

     )
     if config and verbose:
         out(f"Using configuration from {config}.", bold=False, fg="blue")
     )
     if config and verbose:
         out(f"Using configuration from {config}.", bold=False, fg="blue")


@@ -397,7 +418,9 @@ def reformat_one(
                 mode=mode,
             ):
                 changed = Changed.YES
                 mode=mode,
             ):
                 changed = Changed.YES

-            if write_back == WriteBack.YES and changed is not Changed.NO:
+            if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (
+                write_back is WriteBack.CHECK and changed is Changed.NO
+            ):

                 write_cache(cache, [src], line_length, mode)
         report.done(src, changed)
     except Exception as exc:
                 write_cache(cache, [src], line_length, mode)
         report.done(src, changed)
     except Exception as exc:


@@ -427,50 +450,58 @@ async def schedule_formatting(
         sources, cached = filter_cached(cache, sources)
         for src in sorted(cached):
             report.done(src, Changed.CACHED)
         sources, cached = filter_cached(cache, sources)
         for src in sorted(cached):
             report.done(src, Changed.CACHED)

+    if not sources:
+        return
+

     cancelled = []
     cancelled = []

-    formatted = []
-    if sources:
-        lock = None
-        if write_back == WriteBack.DIFF:
-            # For diff output, we need locks to ensure we don't interleave output
-            # from different processes.
-            manager = Manager()
-            lock = manager.Lock()
-        tasks = {
-            loop.run_in_executor(
-                executor,
-                format_file_in_place,
-                src,
-                line_length,
-                fast,
-                write_back,
-                mode,
-                lock,
-            ): src
-            for src in sorted(sources)
-        }
-        pending: Iterable[asyncio.Task] = tasks.keys()
-        try:
-            loop.add_signal_handler(signal.SIGINT, cancel, pending)
-            loop.add_signal_handler(signal.SIGTERM, cancel, pending)
-        except NotImplementedError:
-            # There are no good alternatives for these on Windows
-            pass
-        while pending:
-            done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
-            for task in done:
-                src = tasks.pop(task)
-                if task.cancelled():
-                    cancelled.append(task)
-                elif task.exception():
-                    report.failed(src, str(task.exception()))
-                else:
-                    formatted.append(src)
-                    report.done(src, Changed.YES if task.result() else Changed.NO)
+    sources_to_cache = []
+    lock = None
+    if write_back == WriteBack.DIFF:
+        # For diff output, we need locks to ensure we don't interleave output
+        # from different processes.
+        manager = Manager()
+        lock = manager.Lock()
+    tasks = {
+        loop.run_in_executor(
+            executor,
+            format_file_in_place,
+            src,
+            line_length,
+            fast,
+            write_back,
+            mode,
+            lock,
+        ): src
+        for src in sorted(sources)
+    }
+    pending: Iterable[asyncio.Task] = tasks.keys()
+    try:
+        loop.add_signal_handler(signal.SIGINT, cancel, pending)
+        loop.add_signal_handler(signal.SIGTERM, cancel, pending)
+    except NotImplementedError:
+        # There are no good alternatives for these on Windows.
+        pass
+    while pending:
+        done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
+        for task in done:
+            src = tasks.pop(task)
+            if task.cancelled():
+                cancelled.append(task)
+            elif task.exception():
+                report.failed(src, str(task.exception()))
+            else:
+                changed = Changed.YES if task.result() else Changed.NO
+                # If the file was written back or was successfully checked as
+                # well-formatted, store this information in the cache.
+                if write_back is WriteBack.YES or (
+                    write_back is WriteBack.CHECK and changed is Changed.NO
+                ):
+                    sources_to_cache.append(src)
+                report.done(src, changed)

     if cancelled:
         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
     if cancelled:
         await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)

-    if write_back == WriteBack.YES and formatted:
-        write_cache(cache, formatted, line_length, mode)
+    if sources_to_cache:
+        write_cache(cache, sources_to_cache, line_length, mode)

 
 
 def format_file_in_place(
 
 
 def format_file_in_place(


@@ -483,7 +514,8 @@ def format_file_in_place(
 ) -> bool:
     """Format file under `src` path. Return True if changed.
 
 ) -> bool:
     """Format file under `src` path. Return True if changed.
 

-    If `write_back` is True, write reformatted code back to stdout.
+    If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted
+    code to the file.

     `line_length` and `fast` options are passed to :func:`format_file_contents`.
     """
     if src.suffix == ".pyi":
     `line_length` and `fast` options are passed to :func:`format_file_contents`.
     """
     if src.suffix == ".pyi":


@@ -532,7 +564,8 @@ def format_stdin_to_stdout(
 ) -> bool:
     """Format file on stdin. Return True if changed.
 
 ) -> bool:
     """Format file on stdin. Return True if changed.
 

-    If `write_back` is True, write reformatted code back to stdout.
+    If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,
+    write a diff to stdout.

     `line_length`, `fast`, `is_pyi`, and `force_py36` arguments are passed to
     :func:`format_file_contents`.
     """
     `line_length`, `fast`, `is_pyi`, and `force_py36` arguments are passed to
     :func:`format_file_contents`.
     """


@@ -593,7 +626,7 @@ def format_str(
 
     `line_length` determines how many characters per line are allowed.
     """
 
     `line_length` determines how many characters per line are allowed.
     """

-    src_node = lib2to3_parse(src_contents)
+    src_node = lib2to3_parse(src_contents.lstrip())

     dst_contents = ""
     future_imports = get_future_imports(src_node)
     is_pyi = bool(mode & FileMode.PYI)
     dst_contents = ""
     future_imports = get_future_imports(src_node)
     is_pyi = bool(mode & FileMode.PYI)


@@ -604,6 +637,8 @@ def format_str(
         remove_u_prefix=py36 or "unicode_literals" in future_imports,
         is_pyi=is_pyi,
         normalize_strings=normalize_strings,
         remove_u_prefix=py36 or "unicode_literals" in future_imports,
         is_pyi=is_pyi,
         normalize_strings=normalize_strings,

+        allow_underscores=py36
+        and not bool(mode & FileMode.NO_NUMERIC_UNDERSCORE_NORMALIZATION),

     )
     elt = EmptyLineTracker(is_pyi=is_pyi)
     empty_line = Line()
     )
     elt = EmptyLineTracker(is_pyi=is_pyi)
     empty_line = Line()


@@ -645,7 +680,6 @@ GRAMMARS = [
 
 def lib2to3_parse(src_txt: str) -> Node:
     """Given a string with source, return the lib2to3 Node."""
 
 def lib2to3_parse(src_txt: str) -> Node:
     """Given a string with source, return the lib2to3 Node."""

-    grammar = pygram.python_grammar_no_print_statement

     if src_txt[-1:] != "\n":
         src_txt += "\n"
     for grammar in GRAMMARS:
     if src_txt[-1:] != "\n":
         src_txt += "\n"
     for grammar in GRAMMARS:


@@ -661,7 +695,7 @@ def lib2to3_parse(src_txt: str) -> Node:
                 faulty_line = lines[lineno - 1]
             except IndexError:
                 faulty_line = "<line number missing in source>"
                 faulty_line = lines[lineno - 1]
             except IndexError:
                 faulty_line = "<line number missing in source>"

-            exc = ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}")
+            exc = InvalidInput(f"Cannot parse: {lineno}:{column}: {faulty_line}")

     else:
         raise exc from None
 
     else:
         raise exc from None
 


@@ -796,18 +830,6 @@ UNPACKING_PARENTS = {
     syms.testlist_gexp,
     syms.testlist_star_expr,
 }
     syms.testlist_gexp,
     syms.testlist_star_expr,
 }

-SURROUNDED_BY_BRACKETS = {
-    syms.typedargslist,
-    syms.arglist,
-    syms.subscriptlist,
-    syms.vfplist,
-    syms.import_as_names,
-    syms.yield_expr,
-    syms.testlist_gexp,
-    syms.testlist_star_expr,
-    syms.listmaker,
-    syms.dictsetmaker,
-}

 TEST_DESCENDANTS = {
     syms.test,
     syms.lambdef,
 TEST_DESCENDANTS = {
     syms.test,
     syms.lambdef,


@@ -874,8 +896,8 @@ class BracketTracker:
     bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
     delimiters: Dict[LeafID, Priority] = Factory(dict)
     previous: Optional[Leaf] = None
     bracket_match: Dict[Tuple[Depth, NodeType], Leaf] = Factory(dict)
     delimiters: Dict[LeafID, Priority] = Factory(dict)
     previous: Optional[Leaf] = None

-    _for_loop_variable: int = 0
-    _lambda_arguments: int = 0
+    _for_loop_depths: List[int] = Factory(list)
+    _lambda_argument_depths: List[int] = Factory(list)

 
     def mark(self, leaf: Leaf) -> None:
         """Mark `leaf` with bracket-related metadata. Keep track of delimiters.
 
     def mark(self, leaf: Leaf) -> None:
         """Mark `leaf` with bracket-related metadata. Keep track of delimiters.


@@ -948,16 +970,21 @@ class BracketTracker:
         """
         if leaf.type == token.NAME and leaf.value == "for":
             self.depth += 1
         """
         if leaf.type == token.NAME and leaf.value == "for":
             self.depth += 1

-            self._for_loop_variable += 1
+            self._for_loop_depths.append(self.depth)

             return True
 
         return False
 
     def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
         """See `maybe_increment_for_loop_variable` above for explanation."""
             return True
 
         return False
 
     def maybe_decrement_after_for_loop_variable(self, leaf: Leaf) -> bool:
         """See `maybe_increment_for_loop_variable` above for explanation."""

-        if self._for_loop_variable and leaf.type == token.NAME and leaf.value == "in":
+        if (
+            self._for_loop_depths
+            and self._for_loop_depths[-1] == self.depth
+            and leaf.type == token.NAME
+            and leaf.value == "in"
+        ):

             self.depth -= 1
             self.depth -= 1

-            self._for_loop_variable -= 1
+            self._for_loop_depths.pop()

             return True
 
         return False
             return True
 
         return False


@@ -970,16 +997,20 @@ class BracketTracker:
         """
         if leaf.type == token.NAME and leaf.value == "lambda":
             self.depth += 1
         """
         if leaf.type == token.NAME and leaf.value == "lambda":
             self.depth += 1

-            self._lambda_arguments += 1
+            self._lambda_argument_depths.append(self.depth)

             return True
 
         return False
 
     def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool:
         """See `maybe_increment_lambda_arguments` above for explanation."""
             return True
 
         return False
 
     def maybe_decrement_after_lambda_arguments(self, leaf: Leaf) -> bool:
         """See `maybe_increment_lambda_arguments` above for explanation."""

-        if self._lambda_arguments and leaf.type == token.COLON:
+        if (
+            self._lambda_argument_depths
+            and self._lambda_argument_depths[-1] == self.depth
+            and leaf.type == token.COLON
+        ):

             self.depth -= 1
             self.depth -= 1

-            self._lambda_arguments -= 1
+            self._lambda_argument_depths.pop()

             return True
 
         return False
             return True
 
         return False


@@ -995,7 +1026,9 @@ class Line:
 
     depth: int = 0
     leaves: List[Leaf] = Factory(list)
 
     depth: int = 0
     leaves: List[Leaf] = Factory(list)

-    comments: List[Tuple[Index, Leaf]] = Factory(list)
+    # The LeafID keys of comments must remain ordered by the corresponding leaf's index
+    # in leaves
+    comments: Dict[LeafID, List[Leaf]] = Factory(dict)

     bracket_tracker: BracketTracker = Factory(BracketTracker)
     inside_brackets: bool = False
     should_explode: bool = False
     bracket_tracker: BracketTracker = Factory(BracketTracker)
     inside_brackets: bool = False
     should_explode: bool = False


@@ -1161,7 +1194,7 @@ class Line:
             self.remove_trailing_comma()
             return True
 
             self.remove_trailing_comma()
             return True
 

-        # Otheriwsse, if the trailing one is the only one, we might mistakenly
+        # Otherwise, if the trailing one is the only one, we might mistakenly

         # change a tuple into a different type by removing the comma.
         depth = closing.bracket_depth + 1
         commas = 0
         # change a tuple into a different type by removing the comma.
         depth = closing.bracket_depth + 1
         commas = 0


@@ -1202,43 +1235,35 @@ class Line:
         if comment.type != token.COMMENT:
             return False
 
         if comment.type != token.COMMENT:
             return False
 

-        after = len(self.leaves) - 1
-        if after == -1:
+        if not self.leaves:

             comment.type = STANDALONE_COMMENT
             comment.prefix = ""
             return False
 
         else:
             comment.type = STANDALONE_COMMENT
             comment.prefix = ""
             return False
 
         else:

-            self.comments.append((after, comment))
-            return True
-
-    def comments_after(self, leaf: Leaf, _index: int = -1) -> Iterator[Leaf]:
-        """Generate comments that should appear directly after `leaf`.
-
-        Provide a non-negative leaf `_index` to speed up the function.
-        """
-        if not self.comments:
-            return
-
-        if _index == -1:
-            for _index, _leaf in enumerate(self.leaves):
-                if leaf is _leaf:
-                    break
-
+            leaf_id = id(self.leaves[-1])
+            if leaf_id not in self.comments:
+                self.comments[leaf_id] = [comment]

             else:
             else:

-                return
+                self.comments[leaf_id].append(comment)
+            return True

 
 

-        for index, comment_after in self.comments:
-            if _index == index:
-                yield comment_after
+    def comments_after(self, leaf: Leaf) -> List[Leaf]:
+        """Generate comments that should appear directly after `leaf`."""
+        return self.comments.get(id(leaf), [])

 
     def remove_trailing_comma(self) -> None:
         """Remove the trailing comma and moves the comments attached to it."""
 
     def remove_trailing_comma(self) -> None:
         """Remove the trailing comma and moves the comments attached to it."""

-        comma_index = len(self.leaves) - 1
-        for i in range(len(self.comments)):
-            comment_index, comment = self.comments[i]
-            if comment_index == comma_index:
-                self.comments[i] = (comma_index - 1, comment)
+        # Remember, the LeafID keys of self.comments are ordered by the
+        # corresponding leaf's index in self.leaves
+        # If id(self.leaves[-2]) is in self.comments, the order doesn't change.
+        # Otherwise, we insert it into self.comments, and it becomes the last entry.
+        # However, since we delete id(self.leaves[-1]) from self.comments, the invariant
+        # is maintained
+        self.comments.setdefault(id(self.leaves[-2]), []).extend(
+            self.comments.get(id(self.leaves[-1]), [])
+        )
+        self.comments.pop(id(self.leaves[-1]), None)

         self.leaves.pop()
 
     def is_complex_subscript(self, leaf: Leaf) -> bool:
         self.leaves.pop()
 
     def is_complex_subscript(self, leaf: Leaf) -> bool:


@@ -1270,7 +1295,7 @@ class Line:
         res = f"{first.prefix}{indent}{first.value}"
         for leaf in leaves:
             res += str(leaf)
         res = f"{first.prefix}{indent}{first.value}"
         for leaf in leaves:
             res += str(leaf)

-        for _, comment in self.comments:
+        for comment in itertools.chain.from_iterable(self.comments.values()):

             res += str(comment)
         return res + "\n"
 
             res += str(comment)
         return res + "\n"
 


@@ -1374,7 +1399,7 @@ class EmptyLineTracker:
                 newlines = 1
             elif current_line.is_class or self.previous_line.is_class:
                 if current_line.is_stub_class and self.previous_line.is_stub_class:
                 newlines = 1
             elif current_line.is_class or self.previous_line.is_class:
                 if current_line.is_stub_class and self.previous_line.is_stub_class:

-                    # No blank line between classes with an emty body
+                    # No blank line between classes with an empty body

                     newlines = 0
                 else:
                     newlines = 1
                     newlines = 0
                 else:
                     newlines = 1


@@ -1402,6 +1427,7 @@ class LineGenerator(Visitor[Line]):
     normalize_strings: bool = True
     current_line: Line = Factory(Line)
     remove_u_prefix: bool = False
     normalize_strings: bool = True
     current_line: Line = Factory(Line)
     remove_u_prefix: bool = False

+    allow_underscores: bool = False

 
     def line(self, indent: int = 0) -> Iterator[Line]:
         """Generate a line.
 
     def line(self, indent: int = 0) -> Iterator[Line]:
         """Generate a line.


@@ -1443,6 +1469,8 @@ class LineGenerator(Visitor[Line]):
             if self.normalize_strings and node.type == token.STRING:
                 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
                 normalize_string_quotes(node)
             if self.normalize_strings and node.type == token.STRING:
                 normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
                 normalize_string_quotes(node)

+            if node.type == token.NUMBER:
+                normalize_numeric_literal(node, self.allow_underscores)

             if node.type not in WHITESPACE:
                 self.current_line.append(node)
         yield from super().visit_default(node)
             if node.type not in WHITESPACE:
                 self.current_line.append(node)
         yield from super().visit_default(node)


@@ -1852,14 +1880,14 @@ def container_of(leaf: Leaf) -> LN:
         if parent.type == syms.file_input:
             break
 
         if parent.type == syms.file_input:
             break
 

-        if parent.type in SURROUNDED_BY_BRACKETS:
+        if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS:

             break
 
         container = parent
     return container
 
 
             break
 
         container = parent
     return container
 
 

-def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
+def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int:

     """Return the priority of the `leaf` delimiter, given a line break after it.
 
     The delimiter priorities returned here are from those delimiters that would
     """Return the priority of the `leaf` delimiter, given a line break after it.
 
     The delimiter priorities returned here are from those delimiters that would


@@ -1873,8 +1901,8 @@ def is_split_after_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
     return 0
 
 
     return 0
 
 

-def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
-    """Return the priority of the `leaf` delimiter, given a line before after it.
+def is_split_before_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> int:
+    """Return the priority of the `leaf` delimiter, given a line break before it.

 
     The delimiter priorities returned here are from those delimiters that would
     cause a line break before themselves.
 
     The delimiter priorities returned here are from those delimiters that would
     cause a line break before themselves.


@@ -1911,15 +1939,20 @@ def is_split_before_delimiter(leaf: Leaf, previous: Leaf = None) -> int:
     ):
         return STRING_PRIORITY
 
     ):
         return STRING_PRIORITY
 

-    if leaf.type != token.NAME:
+    if leaf.type not in {token.NAME, token.ASYNC}:

         return 0
 
     if (
         leaf.value == "for"
         and leaf.parent
         and leaf.parent.type in {syms.comp_for, syms.old_comp_for}
         return 0
 
     if (
         leaf.value == "for"
         and leaf.parent
         and leaf.parent.type in {syms.comp_for, syms.old_comp_for}

+        or leaf.type == token.ASYNC

     ):
     ):

-        return COMPREHENSION_PRIORITY
+        if (
+            not isinstance(leaf.prev_sibling, Leaf)
+            or leaf.prev_sibling.value != "async"
+        ):
+            return COMPREHENSION_PRIORITY

 
     if (
         leaf.value == "if"
 
     if (
         leaf.value == "if"


@@ -1993,6 +2026,16 @@ def generate_comments(leaf: LN) -> Iterator[Leaf]:
 
 @dataclass
 class ProtoComment:
 
 @dataclass
 class ProtoComment:

+    """Describes a piece of syntax that is a comment.
+
+    It's not a :class:`blib2to3.pytree.Leaf` so that:
+
+    * it can be cached (`Leaf` objects should not be reused more than once as
+      they store their lineno, column, prefix, and parent information);
+    * `newlines` and `consumed` fields are kept separate from the `value`. This
+      simplifies handling of special marker comments like ``# fmt: off/on``.
+    """
+

     type: int  # token.COMMENT or STANDALONE_COMMENT
     value: str  # content of the comment
     newlines: int  # how many newlines before the comment
     type: int  # token.COMMENT or STANDALONE_COMMENT
     value: str  # content of the comment
     newlines: int  # how many newlines before the comment


@@ -2001,6 +2044,7 @@ class ProtoComment:
 
 @lru_cache(maxsize=4096)
 def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:
 
 @lru_cache(maxsize=4096)
 def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:

+    """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`."""

     result: List[ProtoComment] = []
     if not prefix or "#" not in prefix:
         return result
     result: List[ProtoComment] = []
     if not prefix or "#" not in prefix:
         return result


@@ -2032,8 +2076,8 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:
 def make_comment(content: str) -> str:
     """Return a consistently formatted comment from the given `content` string.
 
 def make_comment(content: str) -> str:
     """Return a consistently formatted comment from the given `content` string.
 

-    All comments (except for "##", "#!", "#:") should have a single space between
-    the hash sign and the content.
+    All comments (except for "##", "#!", "#:", '#'", "#%%") should have a single
+    space between the hash sign and the content.

 
     If `content` didn't start with a hash sign, one is provided.
     """
 
     If `content` didn't start with a hash sign, one is provided.
     """


@@ -2043,7 +2087,7 @@ def make_comment(content: str) -> str:
 
     if content[0] == "#":
         content = content[1:]
 
     if content[0] == "#":
         content = content[1:]

-    if content and content[0] not in " !:#":
+    if content and content[0] not in " !:#'%":

         content = " " + content
     return "#" + content
 
         content = " " + content
     return "#" + content
 


@@ -2106,7 +2150,7 @@ def split_line(
                 result.extend(
                     split_line(l, line_length=line_length, inner=True, py36=py36)
                 )
                 result.extend(
                     split_line(l, line_length=line_length, inner=True, py36=py36)
                 )

-        except CannotSplit as cs:
+        except CannotSplit:

             continue
 
         else:
             continue
 
         else:


@@ -2124,9 +2168,6 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
     Prefer RHS otherwise.  This is why this function is not symmetrical with
     :func:`right_hand_split` which also handles optional parentheses.
     """
     Prefer RHS otherwise.  This is why this function is not symmetrical with
     :func:`right_hand_split` which also handles optional parentheses.
     """

-    head = Line(depth=line.depth)
-    body = Line(depth=line.depth + 1, inside_brackets=True)
-    tail = Line(depth=line.depth)

     tail_leaves: List[Leaf] = []
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []
     tail_leaves: List[Leaf] = []
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []


@@ -2144,15 +2185,12 @@ def left_hand_split(line: Line, py36: bool = False) -> Iterator[Line]:
             if leaf.type in OPENING_BRACKETS:
                 matching_bracket = leaf
                 current_leaves = body_leaves
             if leaf.type in OPENING_BRACKETS:
                 matching_bracket = leaf
                 current_leaves = body_leaves

-    # Since body is a new indent level, remove spurious leading whitespace.
-    if body_leaves:
-        normalize_prefix(body_leaves[0], inside_brackets=True)
-    # Build the new lines.
-    for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves):
-        for leaf in leaves:
-            result.append(leaf, preformatted=True)
-            for comment_after in line.comments_after(leaf):
-                result.append(comment_after, preformatted=True)
+    if not matching_bracket:
+        raise CannotSplit("No brackets found")
+
+    head = bracket_split_build_line(head_leaves, line, matching_bracket)
+    body = bracket_split_build_line(body_leaves, line, matching_bracket, is_body=True)
+    tail = bracket_split_build_line(tail_leaves, line, matching_bracket)

     bracket_split_succeeded_or_raise(head, body, tail)
     for result in (head, body, tail):
         if result:
     bracket_split_succeeded_or_raise(head, body, tail)
     for result in (head, body, tail):
         if result:


@@ -2170,9 +2208,6 @@ def right_hand_split(
 
     Note: running this function modifies `bracket_depth` on the leaves of `line`.
     """
 
     Note: running this function modifies `bracket_depth` on the leaves of `line`.
     """

-    head = Line(depth=line.depth)
-    body = Line(depth=line.depth + 1, inside_brackets=True)
-    tail = Line(depth=line.depth)

     tail_leaves: List[Leaf] = []
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []
     tail_leaves: List[Leaf] = []
     body_leaves: List[Leaf] = []
     head_leaves: List[Leaf] = []


@@ -2189,25 +2224,18 @@ def right_hand_split(
                 opening_bracket = leaf.opening_bracket
                 closing_bracket = leaf
                 current_leaves = body_leaves
                 opening_bracket = leaf.opening_bracket
                 closing_bracket = leaf
                 current_leaves = body_leaves

-    tail_leaves.reverse()
-    body_leaves.reverse()
-    head_leaves.reverse()
-    # Since body is a new indent level, remove spurious leading whitespace.
-    if body_leaves:
-        normalize_prefix(body_leaves[0], inside_brackets=True)
-    if not head_leaves:
-        # No `head` means the split failed. Either `tail` has all content or
+    if not (opening_bracket and closing_bracket and head_leaves):
+        # If there is no opening or closing_bracket that means the split failed and
+        # all content is in the tail.  Otherwise, if `head_leaves` are empty, it means

         # the matching `opening_bracket` wasn't available on `line` anymore.
         raise CannotSplit("No brackets found")
 
         # the matching `opening_bracket` wasn't available on `line` anymore.
         raise CannotSplit("No brackets found")
 

-    # Build the new lines.
-    for result, leaves in (head, head_leaves), (body, body_leaves), (tail, tail_leaves):
-        for leaf in leaves:
-            result.append(leaf, preformatted=True)
-            for comment_after in line.comments_after(leaf):
-                result.append(comment_after, preformatted=True)
-    assert opening_bracket and closing_bracket
-    body.should_explode = should_explode(body, opening_bracket)
+    tail_leaves.reverse()
+    body_leaves.reverse()
+    head_leaves.reverse()
+    head = bracket_split_build_line(head_leaves, line, opening_bracket)
+    body = bracket_split_build_line(body_leaves, line, opening_bracket, is_body=True)
+    tail = bracket_split_build_line(tail_leaves, line, opening_bracket)

     bracket_split_succeeded_or_raise(head, body, tail)
     if (
         # the body shouldn't be exploded
     bracket_split_succeeded_or_raise(head, body, tail)
     if (
         # the body shouldn't be exploded


@@ -2281,6 +2309,35 @@ def bracket_split_succeeded_or_raise(head: Line, body: Line, tail: Line) -> None
             )
 
 
             )
 
 

+def bracket_split_build_line(
+    leaves: List[Leaf], original: Line, opening_bracket: Leaf, *, is_body: bool = False
+) -> Line:
+    """Return a new line with given `leaves` and respective comments from `original`.
+
+    If `is_body` is True, the result line is one-indented inside brackets and as such
+    has its first leaf's prefix normalized and a trailing comma added when expected.
+    """
+    result = Line(depth=original.depth)
+    if is_body:
+        result.inside_brackets = True
+        result.depth += 1
+        if leaves:
+            # Since body is a new indent level, remove spurious leading whitespace.
+            normalize_prefix(leaves[0], inside_brackets=True)
+            # Ensure a trailing comma when expected.
+            if original.is_import:
+                if leaves[-1].type != token.COMMA:
+                    leaves.append(Leaf(token.COMMA, ","))
+    # Populate the line
+    for leaf in leaves:
+        result.append(leaf, preformatted=True)
+        for comment_after in original.comments_after(leaf):
+            result.append(comment_after, preformatted=True)
+    if is_body:
+        result.should_explode = should_explode(result, opening_bracket)
+    return result
+
+

 def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
     """Normalize prefix of the first leaf in every line returned by `split_func`.
 
 def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
     """Normalize prefix of the first leaf in every line returned by `split_func`.
 


@@ -2327,16 +2384,16 @@ def delimiter_split(line: Line, py36: bool = False) -> Iterator[Line]:
         nonlocal current_line
         try:
             current_line.append_safe(leaf, preformatted=True)
         nonlocal current_line
         try:
             current_line.append_safe(leaf, preformatted=True)

-        except ValueError as ve:
+        except ValueError:

             yield current_line
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 
             yield current_line
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 

-    for index, leaf in enumerate(line.leaves):
+    for leaf in line.leaves:

         yield from append_to_line(leaf)
 
         yield from append_to_line(leaf)
 

-        for comment_after in line.comments_after(leaf, index):
+        for comment_after in line.comments_after(leaf):

             yield from append_to_line(comment_after)
 
         lowest_depth = min(lowest_depth, leaf.bracket_depth)
             yield from append_to_line(comment_after)
 
         lowest_depth = min(lowest_depth, leaf.bracket_depth)


@@ -2373,16 +2430,16 @@ def standalone_comment_split(line: Line, py36: bool = False) -> Iterator[Line]:
         nonlocal current_line
         try:
             current_line.append_safe(leaf, preformatted=True)
         nonlocal current_line
         try:
             current_line.append_safe(leaf, preformatted=True)

-        except ValueError as ve:
+        except ValueError:

             yield current_line
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 
             yield current_line
 
             current_line = Line(depth=line.depth, inside_brackets=line.inside_brackets)
             current_line.append(leaf)
 

-    for index, leaf in enumerate(line.leaves):
+    for leaf in line.leaves:

         yield from append_to_line(leaf)
 
         yield from append_to_line(leaf)
 

-        for comment_after in line.comments_after(leaf, index):
+        for comment_after in line.comments_after(leaf):

             yield from append_to_line(comment_after)
 
     if current_line:
             yield from append_to_line(comment_after)
 
     if current_line:


@@ -2504,6 +2561,83 @@ def normalize_string_quotes(leaf: Leaf) -> None:
     leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
 
 
     leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"
 
 

+def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
+    """Normalizes numeric (float, int, and complex) literals.
+
+    All letters used in the representation are normalized to lowercase (except
+    in Python 2 long literals), and long number literals are split using underscores.
+    """
+    text = leaf.value.lower()
+    if text.startswith(("0o", "0b")):
+        # Leave octal and binary literals alone.
+        pass
+    elif text.startswith("0x"):
+        # Change hex literals to upper case.
+        before, after = text[:2], text[2:]
+        text = f"{before}{after.upper()}"
+    elif "e" in text:
+        before, after = text.split("e")
+        sign = ""
+        if after.startswith("-"):
+            after = after[1:]
+            sign = "-"
+        elif after.startswith("+"):
+            after = after[1:]
+        before = format_float_or_int_string(before, allow_underscores)
+        after = format_int_string(after, allow_underscores)
+        text = f"{before}e{sign}{after}"
+    elif text.endswith(("j", "l")):
+        number = text[:-1]
+        suffix = text[-1]
+        # Capitalize in "2L" because "l" looks too similar to "1".
+        if suffix == "l":
+            suffix = "L"
+        text = f"{format_float_or_int_string(number, allow_underscores)}{suffix}"
+    else:
+        text = format_float_or_int_string(text, allow_underscores)
+    leaf.value = text
+
+
+def format_float_or_int_string(text: str, allow_underscores: bool) -> str:
+    """Formats a float string like "1.0"."""
+    if "." not in text:
+        return format_int_string(text, allow_underscores)
+
+    before, after = text.split(".")
+    before = format_int_string(before, allow_underscores) if before else "0"
+    if after:
+        after = format_int_string(after, allow_underscores, count_from_end=False)
+    else:
+        after = "0"
+    return f"{before}.{after}"
+
+
+def format_int_string(
+    text: str, allow_underscores: bool, count_from_end: bool = True
+) -> str:
+    """Normalizes underscores in a string to e.g. 1_000_000.
+
+    Input must be a string of digits and optional underscores.
+    If count_from_end is False, we add underscores after groups of three digits
+    counting from the beginning instead of the end of the strings. This is used
+    for the fractional part of float literals.
+    """
+    if not allow_underscores:
+        return text
+
+    text = text.replace("_", "")
+    if len(text) <= 5:
+        # No underscores for numbers <= 5 digits long.
+        return text
+
+    if count_from_end:
+        # Avoid removing leading zeros, which are important if we're formatting
+        # part of a number like "0.001".
+        return format(int("1" + text), "3_")[1:].lstrip("_")
+    else:
+        return "_".join(text[i : i + 3] for i in range(0, len(text), 3))
+
+

 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 
 def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     """Make existing optional parentheses invisible or create new ones.
 


@@ -2522,7 +2656,11 @@ def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
     for index, child in enumerate(list(node.children)):
         if check_lpar:
             if child.type == syms.atom:
     for index, child in enumerate(list(node.children)):
         if check_lpar:
             if child.type == syms.atom:

-                maybe_make_parens_invisible_in_atom(child)
+                if maybe_make_parens_invisible_in_atom(child):
+                    lpar = Leaf(token.LPAR, "")
+                    rpar = Leaf(token.RPAR, "")
+                    index = child.remove() or 0
+                    node.insert_child(index, Node(syms.atom, [lpar, child, rpar]))

             elif is_one_tuple(child):
                 # wrap child in visible parentheses
                 lpar = Leaf(token.LPAR, "(")
             elif is_one_tuple(child):
                 # wrap child in visible parentheses
                 lpar = Leaf(token.LPAR, "(")


@@ -2577,6 +2715,9 @@ def convert_one_fmt_off_pair(node: Node) -> bool:
                         continue
 
                 ignored_nodes = list(generate_ignored_nodes(leaf))
                         continue
 
                 ignored_nodes = list(generate_ignored_nodes(leaf))

+                if not ignored_nodes:
+                    continue
+

                 first = ignored_nodes[0]  # Can be a container node with the `leaf`.
                 parent = first.parent
                 prefix = first.prefix
                 first = ignored_nodes[0]  # Can be a container node with the `leaf`.
                 parent = first.parent
                 prefix = first.prefix


@@ -2605,7 +2746,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool:
                 )
                 return True
 
                 )
                 return True
 

-            previous_consumed += comment.consumed
+            previous_consumed = comment.consumed

 
     return False
 
 
     return False
 


@@ -2627,7 +2768,11 @@ def generate_ignored_nodes(leaf: Leaf) -> Iterator[LN]:
 
 
 def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
 
 
 def maybe_make_parens_invisible_in_atom(node: LN) -> bool:

-    """If it's safe, make the parens in the atom `node` invisible, recursively."""
+    """If it's safe, make the parens in the atom `node` invisible, recursively.
+
+    Returns whether the node should itself be wrapped in invisible parentheses.
+
+    """

     if (
         node.type != syms.atom
         or is_empty_tuple(node)
     if (
         node.type != syms.atom
         or is_empty_tuple(node)


@@ -2645,9 +2790,9 @@ def maybe_make_parens_invisible_in_atom(node: LN) -> bool:
         last.value = ""  # type: ignore
         if len(node.children) > 1:
             maybe_make_parens_invisible_in_atom(node.children[1])
         last.value = ""  # type: ignore
         if len(node.children) > 1:
             maybe_make_parens_invisible_in_atom(node.children[1])

-        return True
+        return False

 
 

-    return False
+    return True

 
 
 def is_empty_tuple(node: LN) -> bool:
 
 
 def is_empty_tuple(node: LN) -> bool:


@@ -2825,7 +2970,8 @@ def is_python36(node: Node) -> bool:
     """Return True if the current file is using Python 3.6+ features.
 
     Currently looking for:
     """Return True if the current file is using Python 3.6+ features.
 
     Currently looking for:

-    - f-strings; and
+    - f-strings;
+    - underscores in numeric literals; and

     - trailing commas after * or ** in function signatures and calls.
     """
     for n in node.pre_order():
     - trailing commas after * or ** in function signatures and calls.
     """
     for n in node.pre_order():


@@ -2834,6 +2980,10 @@ def is_python36(node: Node) -> bool:
             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
                 return True
 
             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
                 return True
 

+        elif n.type == token.NUMBER:
+            if "_" in n.value:  # type: ignore
+                return True
+

         elif (
             n.type in {syms.typedargslist, syms.arglist}
             and n.children
         elif (
             n.type in {syms.typedargslist, syms.arglist}
             and n.children


@@ -2867,7 +3017,6 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf
     length = 4 * line.depth
     opening_bracket = None
     closing_bracket = None
     length = 4 * line.depth
     opening_bracket = None
     closing_bracket = None

-    optional_brackets: Set[LeafID] = set()

     inner_brackets: Set[LeafID] = set()
     for index, leaf, leaf_length in enumerate_with_length(line, reversed=True):
         length += leaf_length
     inner_brackets: Set[LeafID] = set()
     for index, leaf, leaf_length in enumerate_with_length(line, reversed=True):
         length += leaf_length


@@ -2878,17 +3027,12 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf
         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
             break
 
         if leaf.type == STANDALONE_COMMENT or has_inline_comment:
             break
 

-        optional_brackets.discard(id(leaf))

         if opening_bracket:
             if leaf is opening_bracket:
                 opening_bracket = None
             elif leaf.type in CLOSING_BRACKETS:
                 inner_brackets.add(id(leaf))
         elif leaf.type in CLOSING_BRACKETS:
         if opening_bracket:
             if leaf is opening_bracket:
                 opening_bracket = None
             elif leaf.type in CLOSING_BRACKETS:
                 inner_brackets.add(id(leaf))
         elif leaf.type in CLOSING_BRACKETS:

-            if not leaf.value:
-                optional_brackets.add(id(opening_bracket))
-                continue
-

             if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS:
                 # Empty brackets would fail a split so treat them as "inner"
                 # brackets (e.g. only add them to the `omit` set if another
             if index > 0 and line.leaves[index - 1].type in OPENING_BRACKETS:
                 # Empty brackets would fail a split so treat them as "inner"
                 # brackets (e.g. only add them to the `omit` set if another


@@ -2896,18 +3040,36 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf
                 inner_brackets.add(id(leaf))
                 continue
 
                 inner_brackets.add(id(leaf))
                 continue
 

-            opening_bracket = leaf.opening_bracket

             if closing_bracket:
                 omit.add(id(closing_bracket))
                 omit.update(inner_brackets)
                 inner_brackets.clear()
                 yield omit
             if closing_bracket:
                 omit.add(id(closing_bracket))
                 omit.update(inner_brackets)
                 inner_brackets.clear()
                 yield omit

-            closing_bracket = leaf
+
+            if leaf.value:
+                opening_bracket = leaf.opening_bracket
+                closing_bracket = leaf

 
 
 def get_future_imports(node: Node) -> Set[str]:
     """Return a set of __future__ imports in the file."""
 
 
 def get_future_imports(node: Node) -> Set[str]:
     """Return a set of __future__ imports in the file."""

-    imports = set()
+    imports: Set[str] = set()
+
+    def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:
+        for child in children:
+            if isinstance(child, Leaf):
+                if child.type == token.NAME:
+                    yield child.value
+            elif child.type == syms.import_as_name:
+                orig_name = child.children[0]
+                assert isinstance(orig_name, Leaf), "Invalid syntax parsing imports"
+                assert orig_name.type == token.NAME, "Invalid syntax parsing imports"
+                yield orig_name.value
+            elif child.type == syms.import_as_names:
+                yield from get_imports_from_children(child.children)
+            else:
+                assert False, "Invalid syntax parsing imports"
+

     for child in node.children:
         if child.type != syms.simple_stmt:
             break
     for child in node.children:
         if child.type != syms.simple_stmt:
             break


@@ -2926,15 +3088,7 @@ def get_future_imports(node: Node) -> Set[str]:
             module_name = first_child.children[1]
             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
                 break
             module_name = first_child.children[1]
             if not isinstance(module_name, Leaf) or module_name.value != "__future__":
                 break

-            for import_from_child in first_child.children[3:]:
-                if isinstance(import_from_child, Leaf):
-                    if import_from_child.type == token.NAME:
-                        imports.add(import_from_child.value)
-                else:
-                    assert import_from_child.type == syms.import_as_names
-                    for leaf in import_from_child.children:
-                        if isinstance(leaf, Leaf) and leaf.type == token.NAME:
-                            imports.add(leaf.value)
+            imports |= set(get_imports_from_children(first_child.children[3:]))

         else:
             break
     return imports
         else:
             break
     return imports


@@ -2950,7 +3104,7 @@ def gen_python_files_in_dir(
     """Generate all files under `path` whose paths are not excluded by the
     `exclude` regex, but are included by the `include` regex.
 
     """Generate all files under `path` whose paths are not excluded by the
     `exclude` regex, but are included by the `include` regex.
 

-    Symbolic links pointing outside of the root directory are ignored.
+    Symbolic links pointing outside of the `root` directory are ignored.

 
     `report` is where output about exclusions goes.
     """
 
     `report` is where output about exclusions goes.
     """


@@ -2961,8 +3115,7 @@ def gen_python_files_in_dir(
         except ValueError:
             if child.is_symlink():
                 report.path_ignored(
         except ValueError:
             if child.is_symlink():
                 report.path_ignored(

-                    child,
-                    "is a symbolic link that points outside of the root directory",
+                    child, f"is a symbolic link that points outside {root}"

                 )
                 continue
 
                 )
                 continue
 


@@ -3060,7 +3213,7 @@ class Report:
         - otherwise return 0.
         """
         # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with
         - otherwise return 0.
         """
         # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with

-        # 126 we have special returncodes reserved by the shell.
+        # 126 we have special return codes reserved by the shell.

         if self.failure_count:
             return 123
 
         if self.failure_count:
             return 123
 


@@ -3278,7 +3431,7 @@ def enumerate_with_length(
             return  # Multiline strings, we can't continue.
 
         comment: Optional[Leaf]
             return  # Multiline strings, we can't continue.
 
         comment: Optional[Leaf]

-        for comment in line.comments_after(leaf, index):
+        for comment in line.comments_after(leaf):

             length += len(comment.value)
 
         yield index, leaf, length
             length += len(comment.value)
 
         yield index, leaf, length


@@ -3482,5 +3635,32 @@ def write_cache(
         pass
 
 
         pass
 
 

-if __name__ == "__main__":
+def patch_click() -> None:
+    """Make Click not crash.
+
+    On certain misconfigured environments, Python 3 selects the ASCII encoding as the
+    default which restricts paths that it can access during the lifetime of the
+    application.  Click refuses to work in this scenario by raising a RuntimeError.
+
+    In case of Black the likelihood that non-ASCII characters are going to be used in
+    file paths is minimal since it's Python source code.  Moreover, this crash was
+    spurious on Python 3.7 thanks to PEP 538 and PEP 540.
+    """
+    try:
+        from click import core
+        from click import _unicodefun  # type: ignore
+    except ModuleNotFoundError:
+        return
+
+    for module in (core, _unicodefun):
+        if hasattr(module, "_verify_python3_env"):
+            module._verify_python3_env = lambda: None
+
+
+def patched_main() -> None:
+    patch_click()

     main()
     main()

+
+
+if __name__ == "__main__":
+    patched_main()