]> git.madduck.net Git - etc/vim.git/commitdiff

madduck's git repository

Every one of the projects in this repository is available at the canonical URL git://git.madduck.net/madduck/pub/<projectpath> — see each project's metadata for the exact URL.

All patches and comments are welcome. Please squash your changes to logical commits before using git-format-patch and git-send-email to patches@git.madduck.net. If you'd read over the Git project's submission guidelines and adhered to them, I'd be especially grateful.

SSH access, as well as push access can be individually arranged.

If you use my repositories frequently, consider adding the following snippet to ~/.gitconfig and using the third clone URL listed for each project:

[url "git://git.madduck.net/madduck/"]
  insteadOf = madduck:

speed up cache by approximately 42x by avoiding pathlib (#1953)
authorAnthony Sottile <asottile@umich.edu>
Thu, 4 Feb 2021 21:03:42 +0000 (13:03 -0800)
committerGitHub <noreply@github.com>
Thu, 4 Feb 2021 21:03:42 +0000 (13:03 -0800)
CHANGES.md
src/black/__init__.py
tests/test_black.py

index ca8a0472a3bbc6ccf2d55fe3e4851b256140b7f8..5f7ca4f4da7825f4b04794f0c32e40b6dbcae225 100644 (file)
@@ -26,6 +26,8 @@
 
 - use lowercase hex strings (#1692)
 
 
 - use lowercase hex strings (#1692)
 
+- speed up caching by avoiding pathlib (#1950)
+
 #### _Packaging_
 
 - Self-contained native _Black_ binaries are now provided for releases via GitHub
 #### _Packaging_
 
 - Self-contained native _Black_ binaries are now provided for releases via GitHub
index 9034bf6cf77226880e7e2fc05fd8f95a3a647573..7c1a013aae3663949767567e356acdb586a04e3d 100644 (file)
@@ -93,7 +93,7 @@ Transformer = Callable[["Line", Collection["Feature"]], Iterator["Line"]]
 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]
 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]
-Cache = Dict[Path, CacheInfo]
+Cache = Dict[str, CacheInfo]
 out = partial(click.secho, bold=True, err=True)
 err = partial(click.secho, fg="red", err=True)
 
 out = partial(click.secho, bold=True, err=True)
 err = partial(click.secho, fg="red", err=True)
 
@@ -724,7 +724,8 @@ def reformat_one(
             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
                 cache = read_cache(mode)
                 res_src = src.resolve()
             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
                 cache = read_cache(mode)
                 res_src = src.resolve()
-                if res_src in cache and cache[res_src] == get_cache_info(res_src):
+                res_src_s = str(res_src)
+                if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
                     changed = Changed.CACHED
             if changed is not Changed.CACHED and format_file_in_place(
                 src, fast=fast, write_back=write_back, mode=mode
                     changed = Changed.CACHED
             if changed is not Changed.CACHED and format_file_in_place(
                 src, fast=fast, write_back=write_back, mode=mode
@@ -6781,8 +6782,8 @@ def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set
     """
     todo, done = set(), set()
     for src in sources:
     """
     todo, done = set(), set()
     for src in sources:
-        src = src.resolve()
-        if cache.get(src) != get_cache_info(src):
+        res_src = src.resolve()
+        if cache.get(str(res_src)) != get_cache_info(res_src):
             todo.add(src)
         else:
             done.add(src)
             todo.add(src)
         else:
             done.add(src)
@@ -6794,7 +6795,10 @@ def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None:
     cache_file = get_cache_file(mode)
     try:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
     cache_file = get_cache_file(mode)
     try:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
-        new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
+        new_cache = {
+            **cache,
+            **{str(src.resolve()): get_cache_info(src) for src in sources},
+        }
         with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:
             pickle.dump(new_cache, f, protocol=4)
         os.replace(f.name, cache_file)
         with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:
             pickle.dump(new_cache, f, protocol=4)
         os.replace(f.name, cache_file)
index 28b75787bd6593d630c0812adfb2bbda340a6bc8..cfd3cbd91b3d59056eaa285557d0a38e9564399b 100644 (file)
@@ -1003,7 +1003,7 @@ class BlackTestCase(BlackBaseTestCase):
                 fobj.write("print('hello')")
             self.invokeBlack([str(src)])
             cache = black.read_cache(mode)
                 fobj.write("print('hello')")
             self.invokeBlack([str(src)])
             cache = black.read_cache(mode)
-            self.assertIn(src, cache)
+            self.assertIn(str(src), cache)
 
     def test_cache_single_file_already_cached(self) -> None:
         mode = DEFAULT_MODE
 
     def test_cache_single_file_already_cached(self) -> None:
         mode = DEFAULT_MODE
@@ -1035,8 +1035,8 @@ class BlackTestCase(BlackBaseTestCase):
             with two.open("r") as fobj:
                 self.assertEqual(fobj.read(), 'print("hello")\n')
             cache = black.read_cache(mode)
             with two.open("r") as fobj:
                 self.assertEqual(fobj.read(), 'print("hello")\n')
             cache = black.read_cache(mode)
-            self.assertIn(one, cache)
-            self.assertIn(two, cache)
+            self.assertIn(str(one), cache)
+            self.assertIn(str(two), cache)
 
     def test_no_cache_when_writeback_diff(self) -> None:
         mode = DEFAULT_MODE
 
     def test_no_cache_when_writeback_diff(self) -> None:
         mode = DEFAULT_MODE
@@ -1116,8 +1116,8 @@ class BlackTestCase(BlackBaseTestCase):
             src.touch()
             black.write_cache({}, [src], mode)
             cache = black.read_cache(mode)
             src.touch()
             black.write_cache({}, [src], mode)
             cache = black.read_cache(mode)
-            self.assertIn(src, cache)
-            self.assertEqual(cache[src], black.get_cache_info(src))
+            self.assertIn(str(src), cache)
+            self.assertEqual(cache[str(src)], black.get_cache_info(src))
 
     def test_filter_cached(self) -> None:
         with TemporaryDirectory() as workspace:
 
     def test_filter_cached(self) -> None:
         with TemporaryDirectory() as workspace:
@@ -1128,7 +1128,10 @@ class BlackTestCase(BlackBaseTestCase):
             uncached.touch()
             cached.touch()
             cached_but_changed.touch()
             uncached.touch()
             cached.touch()
             cached_but_changed.touch()
-            cache = {cached: black.get_cache_info(cached), cached_but_changed: (0.0, 0)}
+            cache = {
+                str(cached): black.get_cache_info(cached),
+                str(cached_but_changed): (0.0, 0),
+            }
             todo, done = black.filter_cached(
                 cache, {uncached, cached, cached_but_changed}
             )
             todo, done = black.filter_cached(
                 cache, {uncached, cached, cached_but_changed}
             )
@@ -1156,8 +1159,8 @@ class BlackTestCase(BlackBaseTestCase):
                 fobj.write('print("hello")\n')
             self.invokeBlack([str(workspace)], exit_code=123)
             cache = black.read_cache(mode)
                 fobj.write('print("hello")\n')
             self.invokeBlack([str(workspace)], exit_code=123)
             cache = black.read_cache(mode)
-            self.assertNotIn(failing, cache)
-            self.assertIn(clean, cache)
+            self.assertNotIn(str(failing), cache)
+            self.assertIn(str(clean), cache)
 
     def test_write_cache_write_fail(self) -> None:
         mode = DEFAULT_MODE
 
     def test_write_cache_write_fail(self) -> None:
         mode = DEFAULT_MODE
@@ -1210,9 +1213,9 @@ class BlackTestCase(BlackBaseTestCase):
             path.touch()
             black.write_cache({}, [path], mode)
             one = black.read_cache(mode)
             path.touch()
             black.write_cache({}, [path], mode)
             one = black.read_cache(mode)
-            self.assertIn(path, one)
+            self.assertIn(str(path), one)
             two = black.read_cache(short_mode)
             two = black.read_cache(short_mode)
-            self.assertNotIn(path, two)
+            self.assertNotIn(str(path), two)
 
     def test_single_file_force_pyi(self) -> None:
         pyi_mode = replace(DEFAULT_MODE, is_pyi=True)
 
     def test_single_file_force_pyi(self) -> None:
         pyi_mode = replace(DEFAULT_MODE, is_pyi=True)
@@ -1226,9 +1229,9 @@ class BlackTestCase(BlackBaseTestCase):
                 actual = fh.read()
             # verify cache with --pyi is separate
             pyi_cache = black.read_cache(pyi_mode)
                 actual = fh.read()
             # verify cache with --pyi is separate
             pyi_cache = black.read_cache(pyi_mode)
-            self.assertIn(path, pyi_cache)
+            self.assertIn(str(path), pyi_cache)
             normal_cache = black.read_cache(DEFAULT_MODE)
             normal_cache = black.read_cache(DEFAULT_MODE)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertFormatEqual(expected, actual)
         black.assert_equivalent(contents, actual)
         black.assert_stable(contents, actual, pyi_mode)
         self.assertFormatEqual(expected, actual)
         black.assert_equivalent(contents, actual)
         black.assert_stable(contents, actual, pyi_mode)
@@ -1255,8 +1258,8 @@ class BlackTestCase(BlackBaseTestCase):
             pyi_cache = black.read_cache(pyi_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
             pyi_cache = black.read_cache(pyi_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_pyi(self) -> None:
         source, expected = read_data("force_pyi")
 
     def test_pipe_force_pyi(self) -> None:
         source, expected = read_data("force_pyi")
@@ -1280,9 +1283,9 @@ class BlackTestCase(BlackBaseTestCase):
                 actual = fh.read()
             # verify cache with --target-version is separate
             py36_cache = black.read_cache(py36_mode)
                 actual = fh.read()
             # verify cache with --target-version is separate
             py36_cache = black.read_cache(py36_mode)
-            self.assertIn(path, py36_cache)
+            self.assertIn(str(path), py36_cache)
             normal_cache = black.read_cache(reg_mode)
             normal_cache = black.read_cache(reg_mode)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertEqual(actual, expected)
 
     @event_loop()
         self.assertEqual(actual, expected)
 
     @event_loop()
@@ -1307,8 +1310,8 @@ class BlackTestCase(BlackBaseTestCase):
             pyi_cache = black.read_cache(py36_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
             pyi_cache = black.read_cache(py36_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_py36(self) -> None:
         source, expected = read_data("force_py36")
 
     def test_pipe_force_py36(self) -> None:
         source, expected = read_data("force_py36")