Delay worker count determination

os.cpu_count() can return None (sounds like a super arcane edge case
though) so the type annotation for the `workers` parameter of
`black.main` is wrong. This *could* technically cause a runtime
TypeError since it'd trip one of mypyc's runtime type checks so we
might as well fix it.

Reading the documentation (and cross-checking with the source code),
you are actually allowed to pass None as `max_workers` to
`concurrent.futures.ProcessPoolExecutor`. If it is None, the pool
initializer will simply call os.cpu_count() [^1] (defaulting to 1 if it
returns None [^2]). It'll even round down the worker count to a level
that's safe for Windows.

... so theoretically we don't even need to call os.cpu_count()
ourselves, but our Windows limit is 60 (unlike the stdlib's 61) and I'd
prefer not accidentally reintroducing a crash on machines with many,
many CPU cores.

[^1]: https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ProcessPoolExecutor
[^2]: https://github.com/python/cpython/blob/a372a7d65320396d44e8beb976e3a6c382963d4e/Lib/concurrent/futures/process.py#L600

diff --git a/src/black/__init__.py b/src/black/__init__.py
index 117dc832c9891c61a6433b5fcdc6a077a4c3dfed..86a0b63744264c85dc23f8702768d333c047f221 100644 (file)
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -1,6 +1,5 @@
 import io
 import json
 import io
 import json

-import os

 import platform
 import re
 import sys
 import platform
 import re
 import sys


@@ -28,11 +27,6 @@ from typing import (
     Union,
 )
 
     Union,
 )
 

-if sys.version_info >= (3, 8):
-    from typing import Final
-else:
-    from typing_extensions import Final
-

 import click
 from click.core import ParameterSource
 from mypy_extensions import mypyc_attr
 import click
 from click.core import ParameterSource
 from mypy_extensions import mypyc_attr


@@ -92,7 +86,6 @@ from blib2to3.pgen2 import token
 from blib2to3.pytree import Leaf, Node
 
 COMPILED = Path(__file__).suffix in (".pyd", ".so")
 from blib2to3.pytree import Leaf, Node
 
 COMPILED = Path(__file__).suffix in (".pyd", ".so")

-DEFAULT_WORKERS: Final = os.cpu_count()

 
 # types
 FileContent = str
 
 # types
 FileContent = str


@@ -371,9 +364,8 @@ def validate_regex(
     "-W",
     "--workers",
     type=click.IntRange(min=1),
     "-W",
     "--workers",
     type=click.IntRange(min=1),

-    default=DEFAULT_WORKERS,
-    show_default=True,
-    help="Number of parallel workers",
+    default=None,
+    help="Number of parallel workers [default: number of CPUs in the system]",

 )
 @click.option(
     "-q",
 )
 @click.option(
     "-q",


@@ -448,7 +440,7 @@ def main(  # noqa: C901
     extend_exclude: Optional[Pattern[str]],
     force_exclude: Optional[Pattern[str]],
     stdin_filename: Optional[str],
     extend_exclude: Optional[Pattern[str]],
     force_exclude: Optional[Pattern[str]],
     stdin_filename: Optional[str],

-    workers: int,
+    workers: Optional[int],

     src: Tuple[str, ...],
     config: Optional[str],
 ) -> None:
     src: Tuple[str, ...],
     config: Optional[str],
 ) -> None:

diff --git a/src/black/concurrency.py b/src/black/concurrency.py
index d77ea40bd4675e1f89c4dd58554fec5ee77c6825..bdc368d5add243b17803fa7e6bb2ff23036c6411 100644 (file)
--- a/src/black/concurrency.py
+++ b/src/black/concurrency.py
@@ -6,6 +6,7 @@ NOTE: this module is only imported if we need to format several files at once.
 
 import asyncio
 import logging
 
 import asyncio
 import logging

+import os

 import signal
 import sys
 from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
 import signal
 import sys
 from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor


@@ -15,7 +16,7 @@ from typing import Any, Iterable, Optional, Set
 
 from mypy_extensions import mypyc_attr
 
 
 from mypy_extensions import mypyc_attr
 

-from black import DEFAULT_WORKERS, WriteBack, format_file_in_place
+from black import WriteBack, format_file_in_place

 from black.cache import Cache, filter_cached, read_cache, write_cache
 from black.mode import Mode
 from black.output import err
 from black.cache import Cache, filter_cached, read_cache, write_cache
 from black.mode import Mode
 from black.output import err


@@ -87,13 +88,13 @@ def reformat_many(
     maybe_install_uvloop()
 
     executor: Executor
     maybe_install_uvloop()
 
     executor: Executor

-    worker_count = workers if workers is not None else DEFAULT_WORKERS
+    if workers is None:
+        workers = os.cpu_count() or 1

     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903

-        assert worker_count is not None
-        worker_count = min(worker_count, 60)
+        workers = min(workers, 60)

     try:
     try:

-        executor = ProcessPoolExecutor(max_workers=worker_count)
+        executor = ProcessPoolExecutor(max_workers=workers)

     except (ImportError, NotImplementedError, OSError):
         # we arrive here if the underlying system does not support multi-processing
         # like in AWS Lambda or Termux, in which case we gracefully fallback to
     except (ImportError, NotImplementedError, OSError):
         # we arrive here if the underlying system does not support multi-processing
         # like in AWS Lambda or Termux, in which case we gracefully fallback to