]> git.madduck.net Git - etc/vim.git/commitdiff

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

permits black to run in AWS Lambda: (#1141)
authorAllan Simon <allan.simon@supinfo.com>
Fri, 8 May 2020 13:46:07 +0000 (15:46 +0200)
committerGitHub <noreply@github.com>
Fri, 8 May 2020 13:46:07 +0000 (15:46 +0200)
AWS Lambda and some other virtualized environment may not permit access
to /dev/shm on Linux and as such, trying to use ProcessPoolExecutor will
fail.

As using parallelism is only a 'nice to have' feature of black, if it fails
we gracefully fallback to a monoprocess implementation, which permits black
to finish normally.

Co-authored-by: Allan Simon <asimon@yolaw.fr>
black.py
tests/test_black.py

index 2a913fc5f2e1ba75ce476cec155e879de75c79c2..fc1597a44342d751fcd6f165ca3c7a59abddd3ce 100644 (file)
--- a/black.py
+++ b/black.py
@@ -618,7 +618,14 @@ def reformat_many(
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
         worker_count = min(worker_count, 61)
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
         worker_count = min(worker_count, 61)
-    executor = ProcessPoolExecutor(max_workers=worker_count)
+    try:
+        executor = ProcessPoolExecutor(max_workers=worker_count)
+    except OSError:
+        # we arrive here if the underlying system does not support multi-processing
+        # like in AWS Lambda, in which case we gracefully fallback to the default
+        # mono-process Executor by using None
+        executor = None
+
     try:
         loop.run_until_complete(
             schedule_formatting(
     try:
         loop.run_until_complete(
             schedule_formatting(
@@ -633,7 +640,8 @@ def reformat_many(
         )
     finally:
         shutdown(loop)
         )
     finally:
         shutdown(loop)
-        executor.shutdown()
+        if executor is not None:
+            executor.shutdown()
 
 
 async def schedule_formatting(
 
 
 async def schedule_formatting(
@@ -643,7 +651,7 @@ async def schedule_formatting(
     mode: Mode,
     report: "Report",
     loop: asyncio.AbstractEventLoop,
     mode: Mode,
     report: "Report",
     loop: asyncio.AbstractEventLoop,
-    executor: Executor,
+    executor: Optional[Executor],
 ) -> None:
     """Run formatting of `sources` in parallel using the provided `executor`.
 
 ) -> None:
     """Run formatting of `sources` in parallel using the provided `executor`.
 
index 11bba1fea0759089a6caf9c9d56000c765362b9e..a0e57dc72d4a14e20b7a24162ed3ee3ec6a6072f 100644 (file)
@@ -1273,6 +1273,27 @@ class BlackTestCase(unittest.TestCase):
             self.assertIn(one, cache)
             self.assertIn(two, cache)
 
             self.assertIn(one, cache)
             self.assertIn(two, cache)
 
+    @patch("black.ProcessPoolExecutor", autospec=True)
+    def test_works_in_mono_process_only_environment(self, mock_executor) -> None:
+        mock_executor.side_effect = OSError()
+        mode = black.FileMode()
+        with cache_dir() as workspace:
+            one = (workspace / "one.py").resolve()
+            with one.open("w") as fobj:
+                fobj.write("print('hello')")
+            two = (workspace / "two.py").resolve()
+            with two.open("w") as fobj:
+                fobj.write("print('hello')")
+            black.write_cache({}, [one], mode)
+            self.invokeBlack([str(workspace)])
+            with one.open("r") as fobj:
+                self.assertEqual(fobj.read(), "print('hello')")
+            with two.open("r") as fobj:
+                self.assertEqual(fobj.read(), 'print("hello")\n')
+            cache = black.read_cache(mode)
+            self.assertIn(one, cache)
+            self.assertIn(two, cache)
+
     def test_no_cache_when_writeback_diff(self) -> None:
         mode = black.FileMode()
         with cache_dir() as workspace:
     def test_no_cache_when_writeback_diff(self) -> None:
         mode = black.FileMode()
         with cache_dir() as workspace: