permits black to run in AWS Lambda: (#1141)

AWS Lambda and some other virtualized environment may not permit access
to /dev/shm on Linux and as such, trying to use ProcessPoolExecutor will
fail.

As using parallelism is only a 'nice to have' feature of black, if it fails
we gracefully fallback to a monoprocess implementation, which permits black
to finish normally.

Co-authored-by: Allan Simon <asimon@yolaw.fr>

diff --git a/black.py b/black.py
index 2a913fc5f2e1ba75ce476cec155e879de75c79c2..fc1597a44342d751fcd6f165ca3c7a59abddd3ce 100644 (file)
--- a/black.py
+++ b/black.py
@@ -618,7 +618,14 @@ def reformat_many(
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
         worker_count = min(worker_count, 61)
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
         worker_count = min(worker_count, 61)

-    executor = ProcessPoolExecutor(max_workers=worker_count)
+    try:
+        executor = ProcessPoolExecutor(max_workers=worker_count)
+    except OSError:
+        # we arrive here if the underlying system does not support multi-processing
+        # like in AWS Lambda, in which case we gracefully fallback to the default
+        # mono-process Executor by using None
+        executor = None
+

     try:
         loop.run_until_complete(
             schedule_formatting(
     try:
         loop.run_until_complete(
             schedule_formatting(


@@ -633,7 +640,8 @@ def reformat_many(
         )
     finally:
         shutdown(loop)
         )
     finally:
         shutdown(loop)

-        executor.shutdown()
+        if executor is not None:
+            executor.shutdown()

 
 
 async def schedule_formatting(
 
 
 async def schedule_formatting(


@@ -643,7 +651,7 @@ async def schedule_formatting(
     mode: Mode,
     report: "Report",
     loop: asyncio.AbstractEventLoop,
     mode: Mode,
     report: "Report",
     loop: asyncio.AbstractEventLoop,

-    executor: Executor,
+    executor: Optional[Executor],

 ) -> None:
     """Run formatting of `sources` in parallel using the provided `executor`.
 
 ) -> None:
     """Run formatting of `sources` in parallel using the provided `executor`.
 

diff --git a/tests/test_black.py b/tests/test_black.py
index 11bba1fea0759089a6caf9c9d56000c765362b9e..a0e57dc72d4a14e20b7a24162ed3ee3ec6a6072f 100644 (file)
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -1273,6 +1273,27 @@ class BlackTestCase(unittest.TestCase):
             self.assertIn(one, cache)
             self.assertIn(two, cache)
 
             self.assertIn(one, cache)
             self.assertIn(two, cache)
 

+    @patch("black.ProcessPoolExecutor", autospec=True)
+    def test_works_in_mono_process_only_environment(self, mock_executor) -> None:
+        mock_executor.side_effect = OSError()
+        mode = black.FileMode()
+        with cache_dir() as workspace:
+            one = (workspace / "one.py").resolve()
+            with one.open("w") as fobj:
+                fobj.write("print('hello')")
+            two = (workspace / "two.py").resolve()
+            with two.open("w") as fobj:
+                fobj.write("print('hello')")
+            black.write_cache({}, [one], mode)
+            self.invokeBlack([str(workspace)])
+            with one.open("r") as fobj:
+                self.assertEqual(fobj.read(), "print('hello')")
+            with two.open("r") as fobj:
+                self.assertEqual(fobj.read(), 'print("hello")\n')
+            cache = black.read_cache(mode)
+            self.assertIn(one, cache)
+            self.assertIn(two, cache)
+

     def test_no_cache_when_writeback_diff(self) -> None:
         mode = black.FileMode()
         with cache_dir() as workspace:
     def test_no_cache_when_writeback_diff(self) -> None:
         mode = black.FileMode()
         with cache_dir() as workspace: