Allow specifying `--workers` via environment variable (#3743)

[etc/vim.git] / gallery / gallery.py

diff --git a/gallery/gallery.py b/gallery/gallery.py
index eafa02f210c9b6140b431561b348171ad9980f30..38e52e3479571559715fa9ec9122945f91b10dfe 100755 (executable)
--- a/gallery/gallery.py
+++ b/gallery/gallery.py
@@ -10,26 +10,16 @@ from argparse import ArgumentParser, Namespace
 from concurrent.futures import ThreadPoolExecutor
 from functools import lru_cache, partial
 from pathlib import Path
-from typing import (  # type: ignore # typing can't see Literal
-    Generator,
-    List,
-    Literal,
-    NamedTuple,
-    Optional,
-    Tuple,
-    Union,
-    cast,
-)
+from typing import Generator, List, NamedTuple, Optional, Tuple, Union, cast
 from urllib.request import urlopen, urlretrieve
 
 PYPI_INSTANCE = "https://pypi.org/pypi"
 PYPI_TOP_PACKAGES = (
-    "https://hugovk.github.io/top-pypi-packages/top-pypi-packages-{days}-days.json"
+    "https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.min.json"
 )
 INTERNAL_BLACK_REPO = f"{tempfile.gettempdir()}/__black"
 
 ArchiveKind = Union[tarfile.TarFile, zipfile.ZipFile]
-Days = Union[Literal[30], Literal[365]]
 
 subprocess.run = partial(subprocess.run, check=True)  # type: ignore
 # https://github.com/python/mypy/issues/1484
@@ -51,7 +41,7 @@ def get_pypi_download_url(package: str, version: Optional[str]) -> str:
             sources = metadata["releases"][version]
         else:
             raise ValueError(
-                f"No releases found with given version ('{version}') tag. "
+                f"No releases found with version ('{version}') tag. "
                 f"Found releases: {metadata['releases'].keys()}"
             )
 
@@ -64,8 +54,8 @@ def get_pypi_download_url(package: str, version: Optional[str]) -> str:
     return cast(str, source["url"])
 
 
-def get_top_packages(days: Days) -> List[str]:
-    with urlopen(PYPI_TOP_PACKAGES.format(days=days)) as page:
+def get_top_packages() -> List[str]:
+    with urlopen(PYPI_TOP_PACKAGES) as page:
         result = json.load(page)
 
     return [package["project"] for package in result["rows"]]
@@ -74,7 +64,7 @@ def get_top_packages(days: Days) -> List[str]:
 def get_package_source(package: str, version: Optional[str]) -> str:
     if package == "cpython":
         if version is None:
-            version = "master"
+            version = "main"
         return f"https://github.com/python/cpython/archive/{version}.zip"
     elif package == "pypy":
         if version is None:
@@ -127,18 +117,20 @@ DEFAULT_SLICE = slice(None)  # for flake8
 
 
 def download_and_extract_top_packages(
-    directory: Path, days: Days = 365, workers: int = 8, limit: slice = DEFAULT_SLICE,
+    directory: Path,
+    workers: int = 8,
+    limit: slice = DEFAULT_SLICE,
 ) -> Generator[Path, None, None]:
     with ThreadPoolExecutor(max_workers=workers) as executor:
         bound_downloader = partial(get_package, version=None, directory=directory)
-        for package in executor.map(bound_downloader, get_top_packages(days)[limit]):
+        for package in executor.map(bound_downloader, get_top_packages()[limit]):
             if package is not None:
                 yield package
 
 
 def git_create_repository(repo: Path) -> None:
     subprocess.run(["git", "init"], cwd=repo)
-    git_add_and_commit(msg="Inital commit", repo=repo)
+    git_add_and_commit(msg="Initial commit", repo=repo)
 
 
 def git_add_and_commit(msg: str, repo: Path) -> None:
@@ -245,39 +237,38 @@ def format_repos(repos: Tuple[Path, ...], options: Namespace) -> None:
                 black_version=black_version,
                 input_directory=options.input,
             )
-        git_switch_branch("master", repo=repo)
+        git_switch_branch("main", repo=repo)
 
-    git_switch_branch("master", repo=options.black_repo)
+    git_switch_branch("main", repo=options.black_repo)
 
 
 def main() -> None:
     parser = ArgumentParser(
         description="""Black Gallery is a script that
-    automates process of applying different black versions to a selected
+    automates the process of applying different Black versions to a selected
     PyPI package and seeing the results between versions."""
     )
 
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument("-p", "--pypi-package", help="PyPI package to download.")
     group.add_argument(
-        "-t", "--top-packages", help="Top n PyPI package to download.", type=int
+        "-t", "--top-packages", help="Top n PyPI packages to download.", type=int
     )
 
-    parser.add_argument("-b", "--black-repo", help="Black's git repository.", type=Path)
+    parser.add_argument("-b", "--black-repo", help="Black's Git repository.", type=Path)
     parser.add_argument(
         "-v",
         "--version",
         help=(
-            "Version for PyPI given pypi package. "
-            "Will be discarded if used with -t option."
+            "Version for given PyPI package. Will be discarded if used with -t option."
         ),
     )
     parser.add_argument(
         "-w",
         "--workers",
         help=(
-            "Maximum amount of threads to download at the sametime. "
-            "Will be discard if used with -p option."
+            "Maximum number of threads to download with at the same time. "
+            "Will be discarded if used with -p option."
         ),
     )
     parser.add_argument(
@@ -285,7 +276,7 @@ def main() -> None:
         "--input",
         default=Path("/input"),
         type=Path,
-        help="Input directory to read configurations.",
+        help="Input directory to read configuration.",
     )
     parser.add_argument(
         "-o",
@@ -294,7 +285,7 @@ def main() -> None:
         type=Path,
         help="Output directory to download and put result artifacts.",
     )
-    parser.add_argument("versions", nargs="*", default=("master",), help="")
+    parser.add_argument("versions", nargs="*", default=("main",), help="")
 
     options = parser.parse_args()
     repos = init_repos(options)