exclude tests/data/.* from mypy (#3917)

[etc/vim.git] / src / black / concurrency.py

diff --git a/src/black/concurrency.py b/src/black/concurrency.py
index d77ea40bd4675e1f89c4dd58554fec5ee77c6825..ce01657839981794b5ee1a2995cf079ac74298c1 100644 (file)
--- a/src/black/concurrency.py
+++ b/src/black/concurrency.py
@@ -6,6 +6,7 @@ NOTE: this module is only imported if we need to format several files at once.
 
 import asyncio
 import logging
 
 import asyncio
 import logging

+import os

 import signal
 import sys
 from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
 import signal
 import sys
 from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor


@@ -15,8 +16,8 @@ from typing import Any, Iterable, Optional, Set
 
 from mypy_extensions import mypyc_attr
 
 
 from mypy_extensions import mypyc_attr
 

-from black import DEFAULT_WORKERS, WriteBack, format_file_in_place
-from black.cache import Cache, filter_cached, read_cache, write_cache
+from black import WriteBack, format_file_in_place
+from black.cache import Cache

 from black.mode import Mode
 from black.output import err
 from black.report import Changed, Report
 from black.mode import Mode
 from black.output import err
 from black.report import Changed, Report


@@ -46,23 +47,14 @@ def cancel(tasks: Iterable["asyncio.Task[Any]"]) -> None:
 def shutdown(loop: asyncio.AbstractEventLoop) -> None:
     """Cancel all pending tasks on `loop`, wait for them, and close the loop."""
     try:
 def shutdown(loop: asyncio.AbstractEventLoop) -> None:
     """Cancel all pending tasks on `loop`, wait for them, and close the loop."""
     try:

-        if sys.version_info[:2] >= (3, 7):
-            all_tasks = asyncio.all_tasks
-        else:
-            all_tasks = asyncio.Task.all_tasks

         # This part is borrowed from asyncio/runners.py in Python 3.7b2.
         # This part is borrowed from asyncio/runners.py in Python 3.7b2.

-        to_cancel = [task for task in all_tasks(loop) if not task.done()]
+        to_cancel = [task for task in asyncio.all_tasks(loop) if not task.done()]

         if not to_cancel:
             return
 
         for task in to_cancel:
             task.cancel()
         if not to_cancel:
             return
 
         for task in to_cancel:
             task.cancel()

-        if sys.version_info >= (3, 7):
-            loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
-        else:
-            loop.run_until_complete(
-                asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)
-            )
+        loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))

     finally:
         # `concurrent.futures.Future` objects cannot be cancelled once they
         # are already running. There might be some when the `shutdown()` happened.
     finally:
         # `concurrent.futures.Future` objects cannot be cancelled once they
         # are already running. There might be some when the `shutdown()` happened.


@@ -87,13 +79,14 @@ def reformat_many(
     maybe_install_uvloop()
 
     executor: Executor
     maybe_install_uvloop()
 
     executor: Executor

-    worker_count = workers if workers is not None else DEFAULT_WORKERS
+    if workers is None:
+        workers = int(os.environ.get("BLACK_NUM_WORKERS", 0))
+        workers = workers or os.cpu_count() or 1

     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903

-        assert worker_count is not None
-        worker_count = min(worker_count, 60)
+        workers = min(workers, 60)

     try:
     try:

-        executor = ProcessPoolExecutor(max_workers=worker_count)
+        executor = ProcessPoolExecutor(max_workers=workers)

     except (ImportError, NotImplementedError, OSError):
         # we arrive here if the underlying system does not support multi-processing
         # like in AWS Lambda or Termux, in which case we gracefully fallback to
     except (ImportError, NotImplementedError, OSError):
         # we arrive here if the underlying system does not support multi-processing
         # like in AWS Lambda or Termux, in which case we gracefully fallback to


@@ -140,10 +133,9 @@ async def schedule_formatting(
     `write_back`, `fast`, and `mode` options are passed to
     :func:`format_file_in_place`.
     """
     `write_back`, `fast`, and `mode` options are passed to
     :func:`format_file_in_place`.
     """

-    cache: Cache = {}
+    cache = Cache.read(mode)

     if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
     if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):

-        cache = read_cache(mode)
-        sources, cached = filter_cached(cache, sources)
+        sources, cached = cache.filtered_cached(sources)

         for src in sorted(cached):
             report.done(src, Changed.CACHED)
     if not sources:
         for src in sorted(cached):
             report.done(src, Changed.CACHED)
     if not sources:


@@ -190,9 +182,6 @@ async def schedule_formatting(
                     sources_to_cache.append(src)
                 report.done(src, changed)
     if cancelled:
                     sources_to_cache.append(src)
                 report.done(src, changed)
     if cancelled:

-        if sys.version_info >= (3, 7):
-            await asyncio.gather(*cancelled, return_exceptions=True)
-        else:
-            await asyncio.gather(*cancelled, loop=loop, return_exceptions=True)
+        await asyncio.gather(*cancelled, return_exceptions=True)

     if sources_to_cache:
     if sources_to_cache:

-        write_cache(cache, sources_to_cache, mode)
+        cache.write(sources_to_cache)