#!/usr/bin/env python3
import asyncio
+import errno
import json
import logging
+import os
+import stat
import sys
+from functools import partial
from pathlib import Path
from platform import system
from shutil import rmtree, which
from subprocess import CalledProcessError
from sys import version_info
-from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple
+from typing import Any, Callable, Dict, NamedTuple, Optional, Sequence, Tuple
from urllib.parse import urlparse
import click
WINDOWS = system() == "Windows"
BLACK_BINARY = "black.exe" if WINDOWS else "black"
-GIT_BIANRY = "git.exe" if WINDOWS else "git"
+GIT_BINARY = "git.exe" if WINDOWS else "git"
LOG = logging.getLogger(__name__)
async def _gen_check_output(
cmd: Sequence[str],
- timeout: float = 30,
+ timeout: float = 300,
env: Optional[Dict[str, str]] = None,
cwd: Optional[Path] = None,
) -> Tuple[bytes, bytes]:
raise
if process.returncode != 0:
+ returncode = process.returncode
+ if returncode is None:
+ returncode = 69
+
cmd_str = " ".join(cmd)
- raise CalledProcessError(
- process.returncode, cmd_str, output=stdout, stderr=stderr
- )
+ raise CalledProcessError(returncode, cmd_str, output=stdout, stderr=stderr)
return (stdout, stderr)
else:
results.stats["success"] += 1
return
+ elif cpe.returncode > 1:
+ results.stats["failed"] += 1
+ results.failed_projects[repo_path.name] = cpe
+ return
LOG.error(f"Unknown error with {repo_path}")
raise
depth: int = 1,
) -> Optional[Path]:
"""git Clone project or rebase"""
- git_bin = str(which(GIT_BIANRY))
+ git_bin = str(which(GIT_BINARY))
if not git_bin:
LOG.error("No git binary found")
return None
return repo_path
+def handle_PermissionError(
+ func: Callable, path: Path, exc: Tuple[Any, Any, Any]
+) -> None:
+ """
+ Handle PermissionError during shutil.rmtree.
+
+ This checks if the erroring function is either 'os.rmdir' or 'os.unlink', and that
+ the error was EACCES (i.e. Permission denied). If true, the path is set writable,
+ readable, and executable by everyone. Finally, it tries the error causing delete
+ operation again.
+
+ If the check is false, then the original error will be reraised as this function
+ can't handle it.
+ """
+ excvalue = exc[1]
+ LOG.debug(f"Handling {excvalue} from {func.__name__}... ")
+ if func in (os.rmdir, os.unlink) and excvalue.errno == errno.EACCES:
+ LOG.debug(f"Setting {path} writable, readable, and executable by everyone... ")
+ os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # chmod 0777
+ func(path) # Try the error causing delete operation again
+ else:
+ raise
+
+
async def load_projects_queue(
config_path: Path,
) -> Tuple[Dict[str, Any], asyncio.Queue]:
except asyncio.QueueEmpty:
LOG.debug(f"project_runner {idx} exiting")
return
+ LOG.debug(f"worker {idx} working on {project_name}")
project_config = config["projects"][project_name]
if not keep:
LOG.debug(f"Removing {repo_path}")
- await loop.run_in_executor(None, rmtree, repo_path)
+ rmtree_partial = partial(
+ rmtree, path=repo_path, onerror=handle_PermissionError
+ )
+ await loop.run_in_executor(None, rmtree_partial)
+
+ LOG.info(f"Finished {project_name}")
async def process_queue(