Update discussion of AST safety check in README (#2159)

[etc/vim.git] / src / black_primer / lib.py

diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py
index 4f929f128ce1fd7b949441317f4b89e22bc1b4af..7999f6a3e3652690bbcf3cda197685b7898c68bb 100644 (file)
--- a/src/black_primer/lib.py
+++ b/src/black_primer/lib.py
@@ -1,15 +1,20 @@
 #!/usr/bin/env python3
 
 import asyncio
 #!/usr/bin/env python3
 
 import asyncio

+import errno

 import json
 import logging
 import json
 import logging

+import os
+import stat

 import sys
 import sys

+from functools import partial

 from pathlib import Path
 from platform import system
 from shutil import rmtree, which
 from subprocess import CalledProcessError
 from sys import version_info
 from pathlib import Path
 from platform import system
 from shutil import rmtree, which
 from subprocess import CalledProcessError
 from sys import version_info

-from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple
+from tempfile import TemporaryDirectory
+from typing import Any, Callable, Dict, NamedTuple, Optional, Sequence, Tuple

 from urllib.parse import urlparse
 
 import click
 from urllib.parse import urlparse
 
 import click


@@ -17,7 +22,7 @@ import click
 
 WINDOWS = system() == "Windows"
 BLACK_BINARY = "black.exe" if WINDOWS else "black"
 
 WINDOWS = system() == "Windows"
 BLACK_BINARY = "black.exe" if WINDOWS else "black"

-GIT_BIANRY = "git.exe" if WINDOWS else "git"
+GIT_BINARY = "git.exe" if WINDOWS else "git"

 LOG = logging.getLogger(__name__)
 
 
 LOG = logging.getLogger(__name__)
 
 


@@ -36,7 +41,7 @@ class Results(NamedTuple):
 
 async def _gen_check_output(
     cmd: Sequence[str],
 
 async def _gen_check_output(
     cmd: Sequence[str],

-    timeout: float = 30,
+    timeout: float = 300,

     env: Optional[Dict[str, str]] = None,
     cwd: Optional[Path] = None,
 ) -> Tuple[bytes, bytes]:
     env: Optional[Dict[str, str]] = None,
     cwd: Optional[Path] = None,
 ) -> Tuple[bytes, bytes]:


@@ -54,6 +59,11 @@ async def _gen_check_output(
         await process.wait()
         raise
 
         await process.wait()
         raise
 

+    # A non-optional timeout was supplied to asyncio.wait_for, guaranteeing
+    # a timeout or completed process.  A terminated Python process will have a
+    # non-empty returncode value.
+    assert process.returncode is not None
+

     if process.returncode != 0:
         cmd_str = " ".join(cmd)
         raise CalledProcessError(
     if process.returncode != 0:
         cmd_str = " ".join(cmd)
         raise CalledProcessError(


@@ -112,24 +122,36 @@ async def black_run(
         cmd.extend(*project_config["cli_arguments"])
     cmd.extend(["--check", "--diff", "."])
 
         cmd.extend(*project_config["cli_arguments"])
     cmd.extend(["--check", "--diff", "."])
 

-    try:
-        _stdout, _stderr = await _gen_check_output(cmd, cwd=repo_path)
-    except asyncio.TimeoutError:
-        results.stats["failed"] += 1
-        LOG.error(f"Running black for {repo_path} timed out ({cmd})")
-    except CalledProcessError as cpe:
-        # TODO: Tune for smarter for higher signal
-        # If any other return value than 1 we raise - can disable project in config
-        if cpe.returncode == 1:
-            if not project_config["expect_formatting_changes"]:
+    with TemporaryDirectory() as tmp_path:
+        # Prevent reading top-level user configs by manipulating envionment variables
+        env = {
+            **os.environ,
+            "XDG_CONFIG_HOME": tmp_path,  # Unix-like
+            "USERPROFILE": tmp_path,  # Windows (changes `Path.home()` output)
+        }
+
+        try:
+            _stdout, _stderr = await _gen_check_output(cmd, cwd=repo_path, env=env)
+        except asyncio.TimeoutError:
+            results.stats["failed"] += 1
+            LOG.error(f"Running black for {repo_path} timed out ({cmd})")
+        except CalledProcessError as cpe:
+            # TODO: Tune for smarter for higher signal
+            # If any other return value than 1 we raise - can disable project in config
+            if cpe.returncode == 1:
+                if not project_config["expect_formatting_changes"]:
+                    results.stats["failed"] += 1
+                    results.failed_projects[repo_path.name] = cpe
+                else:
+                    results.stats["success"] += 1
+                return
+            elif cpe.returncode > 1:

                 results.stats["failed"] += 1
                 results.failed_projects[repo_path.name] = cpe
                 results.stats["failed"] += 1
                 results.failed_projects[repo_path.name] = cpe

-            else:
-                results.stats["success"] += 1
-            return
+                return

 
 

-        LOG.error(f"Unknown error with {repo_path}")
-        raise
+            LOG.error(f"Unknown error with {repo_path}")
+            raise

 
     # If we get here and expect formatting changes something is up
     if project_config["expect_formatting_changes"]:
 
     # If we get here and expect formatting changes something is up
     if project_config["expect_formatting_changes"]:


@@ -150,7 +172,7 @@ async def git_checkout_or_rebase(
     depth: int = 1,
 ) -> Optional[Path]:
     """git Clone project or rebase"""
     depth: int = 1,
 ) -> Optional[Path]:
     """git Clone project or rebase"""

-    git_bin = str(which(GIT_BIANRY))
+    git_bin = str(which(GIT_BINARY))

     if not git_bin:
         LOG.error("No git binary found")
         return None
     if not git_bin:
         LOG.error("No git binary found")
         return None


@@ -176,6 +198,30 @@ async def git_checkout_or_rebase(
     return repo_path
 
 
     return repo_path
 
 

+def handle_PermissionError(
+    func: Callable, path: Path, exc: Tuple[Any, Any, Any]
+) -> None:
+    """
+    Handle PermissionError during shutil.rmtree.
+
+    This checks if the erroring function is either 'os.rmdir' or 'os.unlink', and that
+    the error was EACCES (i.e. Permission denied). If true, the path is set writable,
+    readable, and executable by everyone. Finally, it tries the error causing delete
+    operation again.
+
+    If the check is false, then the original error will be reraised as this function
+    can't handle it.
+    """
+    excvalue = exc[1]
+    LOG.debug(f"Handling {excvalue} from {func.__name__}... ")
+    if func in (os.rmdir, os.unlink) and excvalue.errno == errno.EACCES:
+        LOG.debug(f"Setting {path} writable, readable, and executable by everyone... ")
+        os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)  # chmod 0777
+        func(path)  # Try the error causing delete operation again
+    else:
+        raise
+
+

 async def load_projects_queue(
     config_path: Path,
 ) -> Tuple[Dict[str, Any], asyncio.Queue]:
 async def load_projects_queue(
     config_path: Path,
 ) -> Tuple[Dict[str, Any], asyncio.Queue]:


@@ -212,6 +258,7 @@ async def project_runner(
         except asyncio.QueueEmpty:
             LOG.debug(f"project_runner {idx} exiting")
             return
         except asyncio.QueueEmpty:
             LOG.debug(f"project_runner {idx} exiting")
             return

+        LOG.debug(f"worker {idx} working on {project_name}")

 
         project_config = config["projects"][project_name]
 
 
         project_config = config["projects"][project_name]
 


@@ -243,7 +290,12 @@ async def project_runner(
 
         if not keep:
             LOG.debug(f"Removing {repo_path}")
 
         if not keep:
             LOG.debug(f"Removing {repo_path}")

-            await loop.run_in_executor(None, rmtree, repo_path)
+            rmtree_partial = partial(
+                rmtree, path=repo_path, onerror=handle_PermissionError
+            )
+            await loop.run_in_executor(None, rmtree_partial)
+
+        LOG.info(f"Finished {project_name}")

 
 
 async def process_queue(
 
 
 async def process_queue(