From e6934fd8d9d749d8bab4e880d1e3f1f27537c2c1 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Thu, 21 May 2020 21:57:58 -0700 Subject: [PATCH] Enable primer on CI Runs + add all README listed black projects into primer.json (#1440) MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit * Add all listed by projects into primer.json + Enable on CI Runs - Change workers default to 2 as black uses system CPU count - Increase timeout to 5 mins for subprocess black runs - Takes about 120s for 13 (3 disabled) projects on my 2018 Macbook Pro - I was not removing directories tho ... Will open an issue to investigate the failing projects and make this run cleaner. - Once we get more stable we can expect more repos to be black formatted Run it: - `black-primer -k -w /tmp/primer_large_test --debug --rebase` ``` [2020-05-20 21:44:01,273] DEBUG: Starting /Users/cooper/venvs/b/bin/black-primer (cli.py:125) [2020-05-20 21:44:01,273] DEBUG: Using selector: KqueueSelector (selector_events.py:53) [2020-05-20 21:44:01,274] INFO: 16 projects to run Black over (lib.py:276) [2020-05-20 21:44:01,274] DEBUG: Using 2 parallel workers to run Black (lib.py:281) [2020-05-20 21:44:01,274] DEBUG: worker 0 workng on aioexabgp (lib.py:215) [2020-05-20 21:44:01,276] DEBUG: worker 1 workng on attrs (lib.py:215) [2020-05-20 21:44:02,443] INFO: Finished aioexabgp (lib.py:249) [2020-05-20 21:44:02,443] DEBUG: worker 0 workng on bandersnatch (lib.py:215) [2020-05-20 21:44:04,409] INFO: Finished bandersnatch (lib.py:249) [2020-05-20 21:44:04,409] DEBUG: worker 0 workng on channels (lib.py:215) [2020-05-20 21:44:04,702] INFO: Finished attrs (lib.py:249) [2020-05-20 21:44:04,702] DEBUG: worker 1 workng on django (lib.py:215) [2020-05-20 21:44:04,702] INFO: Skipping django as it's disabled via config (lib.py:222) [2020-05-20 21:44:04,702] DEBUG: worker 1 workng on flake8-bugbear (lib.py:215) [2020-05-20 21:44:05,813] INFO: Finished channels (lib.py:249) [2020-05-20 21:44:05,813] DEBUG: worker 0 workng on hypothesis (lib.py:215) [2020-05-20 21:44:06,071] INFO: Finished flake8-bugbear (lib.py:249) [2020-05-20 21:44:06,071] DEBUG: worker 1 workng on pandas (lib.py:215) [2020-05-20 21:44:06,071] INFO: Skipping pandas as it's disabled via config (lib.py:222) [2020-05-20 21:44:06,071] DEBUG: worker 1 workng on poetry (lib.py:215) [2020-05-20 21:44:16,207] INFO: Finished hypothesis (lib.py:249) [2020-05-20 21:44:16,207] DEBUG: worker 0 workng on ptr (lib.py:215) [2020-05-20 21:44:17,077] INFO: Finished poetry (lib.py:249) [2020-05-20 21:44:17,077] DEBUG: worker 1 workng on pyramid (lib.py:215) [2020-05-20 21:44:17,460] INFO: Finished ptr (lib.py:249) [2020-05-20 21:44:17,460] DEBUG: worker 0 workng on pytest (lib.py:215) [2020-05-20 21:44:17,460] INFO: Skipping pytest as it's disabled via config (lib.py:222) [2020-05-20 21:44:17,460] DEBUG: worker 0 workng on sqlalchemy (lib.py:215) [2020-05-20 21:44:33,319] INFO: Finished pyramid (lib.py:249) [2020-05-20 21:44:33,319] DEBUG: worker 1 workng on tox (lib.py:215) [2020-05-20 21:44:42,274] INFO: Finished tox (lib.py:249) [2020-05-20 21:44:42,275] DEBUG: worker 1 workng on virtualenv (lib.py:215) [2020-05-20 21:44:47,928] INFO: Finished virtualenv (lib.py:249) [2020-05-20 21:44:47,928] DEBUG: worker 1 workng on warehouse (lib.py:215) [2020-05-20 21:45:16,784] INFO: Finished warehouse (lib.py:249) [2020-05-20 21:45:16,784] DEBUG: project_runner 1 exiting (lib.py:213) [2020-05-20 21:45:45,700] INFO: Finished sqlalchemy (lib.py:249) [2020-05-20 21:45:45,700] DEBUG: project_runner 0 exiting (lib.py:213) [2020-05-20 21:45:45,701] INFO: Analyzing results (lib.py:292) -- primer results 📊 -- 13 / 16 succeeded (81.25%) ✅ 0 / 16 FAILED (0.0%) 💩 - 3 projects disabled by config - 0 projects skipped due to Python version - 0 skipped due to long checkout ``` * Move to partial for rmtree + specify a onerror handler for PermissionError on Windows for git * Set default coding to utf8 for very important emoji's on Windows * Set Python encoding to utf-8 for Windows * Appease the white space gods of Black! Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- .github/workflows/test.yml | 6 +++ src/black_primer/cli.py | 7 +-- src/black_primer/lib.py | 40 +++++++++++++-- src/black_primer/primer.json | 96 ++++++++++++++++++++++++++++++++++-- 4 files changed, 140 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd0af61..9fe99f5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,3 +28,9 @@ jobs: - name: Unit tests run: | coverage run -m unittest + + - name: primer run + env: + pythonioencoding: utf-8 + run: | + black-primer diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index 09ab03f..5903adc 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 +# coding=utf8 + import asyncio import logging import sys from datetime import datetime -from os import cpu_count from pathlib import Path from shutil import rmtree, which from tempfile import gettempdir @@ -61,7 +62,7 @@ async def async_main( finally: if not keep and work_path.exists(): LOG.debug(f"Removing {work_path}") - rmtree(work_path) + rmtree(work_path, onerror=lib.handle_PermissionError) return -2 @@ -114,7 +115,7 @@ async def async_main( @click.option( "-W", "--workers", - default=int((cpu_count() or 4) / 2) or 1, + default=2, type=int, show_default=True, help="Number of parallel worker coroutines", diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index 4f929f1..913f9d5 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -1,15 +1,19 @@ #!/usr/bin/env python3 import asyncio +import errno import json import logging +import os +import stat import sys +from functools import partial from pathlib import Path from platform import system from shutil import rmtree, which from subprocess import CalledProcessError from sys import version_info -from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, NamedTuple, Optional, Sequence, Tuple from urllib.parse import urlparse import click @@ -36,7 +40,7 @@ class Results(NamedTuple): async def _gen_check_output( cmd: Sequence[str], - timeout: float = 30, + timeout: float = 300, env: Optional[Dict[str, str]] = None, cwd: Optional[Path] = None, ) -> Tuple[bytes, bytes]: @@ -176,6 +180,30 @@ async def git_checkout_or_rebase( return repo_path +def handle_PermissionError( + func: Callable, path: Path, exc: Tuple[Any, Any, Any] +) -> None: + """ + Handle PermissionError during shutil.rmtree. + + This checks if the erroring function is either 'os.rmdir' or 'os.unlink', and that + the error was EACCES (i.e. Permission denied). If true, the path is set writable, + readable, and executable by everyone. Finally, it tries the error causing delete + operation again. + + If the check is false, then the original error will be reraised as this function + can't handle it. + """ + excvalue = exc[1] + LOG.debug(f"Handling {excvalue} from {func.__name__}... ") + if func in (os.rmdir, os.unlink) and excvalue.errno == errno.EACCES: + LOG.debug(f"Setting {path} writable, readable, and executable by everyone... ") + os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # chmod 0777 + func(path) # Try the error causing delete operation again + else: + raise + + async def load_projects_queue( config_path: Path, ) -> Tuple[Dict[str, Any], asyncio.Queue]: @@ -212,6 +240,7 @@ async def project_runner( except asyncio.QueueEmpty: LOG.debug(f"project_runner {idx} exiting") return + LOG.debug(f"worker {idx} working on {project_name}") project_config = config["projects"][project_name] @@ -243,7 +272,12 @@ async def project_runner( if not keep: LOG.debug(f"Removing {repo_path}") - await loop.run_in_executor(None, rmtree, repo_path) + rmtree_partial = partial( + rmtree, path=repo_path, onerror=handle_PermissionError + ) + await loop.run_in_executor(None, rmtree_partial) + + LOG.info(f"Finished {project_name}") async def process_queue( diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 678942c..f5cc3fd 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -3,7 +3,7 @@ "projects": { "aioexabgp": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", "long_checkout": false, "py_versions": ["all"] @@ -17,17 +17,107 @@ }, "bandersnatch": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/pypa/bandersnatch.git", "long_checkout": false, "py_versions": ["all"] }, - "flake8-bugbear": { + "channels": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/django/channels.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "django": { + "disabled_reason": "black --check --diff returned 123", + "disabled": true, "cli_arguments": [], "expect_formatting_changes": true, + "git_clone_url": "https://github.com/django/django.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "flake8-bugbear": { + "cli_arguments": [], + "expect_formatting_changes": false, "git_clone_url": "https://github.com/PyCQA/flake8-bugbear.git", "long_checkout": false, "py_versions": ["all"] + }, + "hypothesis": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/HypothesisWorks/hypothesis.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "pandas": { + "disabled_reason": "black --check --diff returned 123", + "disabled": true, + "cli_arguments": [], + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/pandas-dev/pandas.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "poetry": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/python-poetry/poetry.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "pyramid": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/Pylons/pyramid.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "ptr": { + "cli_arguments": [], + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/facebookincubator/ptr.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "pytest": { + "disabled_reason": "black --check --diff returned 123", + "disabled": true, + "cli_arguments": [], + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/pytest-dev/pytest.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "sqlalchemy": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/sqlalchemy/sqlalchemy.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "tox": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/tox-dev/tox.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "virtualenv": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/pypa/virtualenv.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "warehouse": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/pypa/warehouse.git", + "long_checkout": false, + "py_versions": ["all"] } } } -- 2.39.5