first commit

This commit is contained in:
Yura 2024-09-15 15:12:16 +03:00
commit 417e54da96
5696 changed files with 900003 additions and 0 deletions

View file

@ -0,0 +1,14 @@
"""Errors."""
from __future__ import annotations
class ProcessCallFailedError(RuntimeError):
"""Failed a process call."""
def __init__(self, code, out, err, cmd) -> None:
super().__init__(code, out, err, cmd)
self.code = code
self.out = out
self.err = err
self.cmd = cmd

View file

@ -0,0 +1,166 @@
"""holds locking functionality that works across processes."""
from __future__ import annotations
import logging
import os
from abc import ABC, abstractmethod
from contextlib import contextmanager, suppress
from pathlib import Path
from threading import Lock, RLock
from filelock import FileLock, Timeout
class _CountedFileLock(FileLock):
def __init__(self, lock_file) -> None:
parent = os.path.dirname(lock_file)
if not os.path.isdir(parent):
with suppress(OSError):
os.makedirs(parent)
super().__init__(lock_file)
self.count = 0
self.thread_safe = RLock()
def acquire(self, timeout=None, poll_interval=0.05):
if not self.thread_safe.acquire(timeout=-1 if timeout is None else timeout):
raise Timeout(self.lock_file)
if self.count == 0:
super().acquire(timeout, poll_interval)
self.count += 1
def release(self, force=False): # noqa: FBT002
with self.thread_safe:
if self.count > 0:
self.thread_safe.release()
if self.count == 1:
super().release(force=force)
self.count = max(self.count - 1, 0)
_lock_store = {}
_store_lock = Lock()
class PathLockBase(ABC):
def __init__(self, folder) -> None:
path = Path(folder)
self.path = path.resolve() if path.exists() else path
def __repr__(self) -> str:
return f"{self.__class__.__name__}({self.path})"
def __truediv__(self, other):
return type(self)(self.path / other)
@abstractmethod
def __enter__(self):
raise NotImplementedError
@abstractmethod
def __exit__(self, exc_type, exc_val, exc_tb):
raise NotImplementedError
@abstractmethod
@contextmanager
def lock_for_key(self, name, no_block=False): # noqa: FBT002
raise NotImplementedError
@abstractmethod
@contextmanager
def non_reentrant_lock_for_key(self, name):
raise NotImplementedError
class ReentrantFileLock(PathLockBase):
def __init__(self, folder) -> None:
super().__init__(folder)
self._lock = None
def _create_lock(self, name=""):
lock_file = str(self.path / f"{name}.lock")
with _store_lock:
if lock_file not in _lock_store:
_lock_store[lock_file] = _CountedFileLock(lock_file)
return _lock_store[lock_file]
@staticmethod
def _del_lock(lock):
if lock is not None:
with _store_lock, lock.thread_safe:
if lock.count == 0:
_lock_store.pop(lock.lock_file, None)
def __del__(self) -> None:
self._del_lock(self._lock)
def __enter__(self):
self._lock = self._create_lock()
self._lock_file(self._lock)
def __exit__(self, exc_type, exc_val, exc_tb):
self._release(self._lock)
self._del_lock(self._lock)
self._lock = None
def _lock_file(self, lock, no_block=False): # noqa: FBT002
# multiple processes might be trying to get a first lock... so we cannot check if this directory exist without
# a lock, but that lock might then become expensive, and it's not clear where that lock should live.
# Instead here we just ignore if we fail to create the directory.
with suppress(OSError):
os.makedirs(str(self.path))
try:
lock.acquire(0.0001)
except Timeout:
if no_block:
raise
logging.debug("lock file %s present, will block until released", lock.lock_file)
lock.release() # release the acquire try from above
lock.acquire()
@staticmethod
def _release(lock):
lock.release()
@contextmanager
def lock_for_key(self, name, no_block=False): # noqa: FBT002
lock = self._create_lock(name)
try:
try:
self._lock_file(lock, no_block)
yield
finally:
self._release(lock)
finally:
self._del_lock(lock)
lock = None
@contextmanager
def non_reentrant_lock_for_key(self, name):
with _CountedFileLock(str(self.path / f"{name}.lock")):
yield
class NoOpFileLock(PathLockBase):
def __enter__(self):
raise NotImplementedError
def __exit__(self, exc_type, exc_val, exc_tb):
raise NotImplementedError
@contextmanager
def lock_for_key(self, name, no_block=False): # noqa: ARG002, FBT002
yield
@contextmanager
def non_reentrant_lock_for_key(self, name): # noqa: ARG002
yield
__all__ = [
"NoOpFileLock",
"ReentrantFileLock",
"Timeout",
]

View file

@ -0,0 +1,16 @@
from __future__ import annotations
from ._permission import make_exe, set_tree
from ._sync import copy, copytree, ensure_dir, safe_delete, symlink
from ._win import get_short_path_name
__all__ = [
"copy",
"copytree",
"ensure_dir",
"get_short_path_name",
"make_exe",
"safe_delete",
"set_tree",
"symlink",
]

View file

@ -0,0 +1,30 @@
from __future__ import annotations
import os
from stat import S_IXGRP, S_IXOTH, S_IXUSR
def make_exe(filename):
original_mode = filename.stat().st_mode
levels = [S_IXUSR, S_IXGRP, S_IXOTH]
for at in range(len(levels), 0, -1):
try:
mode = original_mode
for level in levels[:at]:
mode |= level
filename.chmod(mode)
break
except OSError:
continue
def set_tree(folder, stat):
for root, _, files in os.walk(str(folder)):
for filename in files:
os.chmod(os.path.join(root, filename), stat)
__all__ = (
"make_exe",
"set_tree",
)

View file

@ -0,0 +1,83 @@
from __future__ import annotations
import logging
import os
import shutil
import sys
from stat import S_IWUSR
def ensure_dir(path):
if not path.exists():
logging.debug("create folder %s", str(path))
os.makedirs(str(path))
def ensure_safe_to_do(src, dest):
if src == dest:
msg = f"source and destination is the same {src}"
raise ValueError(msg)
if not dest.exists():
return
if dest.is_dir() and not dest.is_symlink():
logging.debug("remove directory %s", dest)
safe_delete(dest)
else:
logging.debug("remove file %s", dest)
dest.unlink()
def symlink(src, dest):
ensure_safe_to_do(src, dest)
logging.debug("symlink %s", _Debug(src, dest))
dest.symlink_to(src, target_is_directory=src.is_dir())
def copy(src, dest):
ensure_safe_to_do(src, dest)
is_dir = src.is_dir()
method = copytree if is_dir else shutil.copy
logging.debug("copy %s", _Debug(src, dest))
method(str(src), str(dest))
def copytree(src, dest):
for root, _, files in os.walk(src):
dest_dir = os.path.join(dest, os.path.relpath(root, src))
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
for name in files:
src_f = os.path.join(root, name)
dest_f = os.path.join(dest_dir, name)
shutil.copy(src_f, dest_f)
def safe_delete(dest):
def onerror(func, path, exc_info): # noqa: ARG001
if not os.access(path, os.W_OK):
os.chmod(path, S_IWUSR)
func(path)
else:
raise # noqa: PLE0704
kwargs = {"onexc" if sys.version_info >= (3, 12) else "onerror": onerror}
shutil.rmtree(str(dest), ignore_errors=True, **kwargs)
class _Debug:
def __init__(self, src, dest) -> None:
self.src = src
self.dest = dest
def __str__(self) -> str:
return f"{'directory ' if self.src.is_dir() else ''}{self.src!s} to {self.dest!s}"
__all__ = [
"copy",
"copytree",
"ensure_dir",
"safe_delete",
"symlink",
"symlink",
]

View file

@ -0,0 +1,23 @@
from __future__ import annotations
def get_short_path_name(long_name):
"""Gets the short path name of a given long path - http://stackoverflow.com/a/23598461/200291."""
import ctypes # noqa: PLC0415
from ctypes import wintypes # noqa: PLC0415
_GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW # noqa: N806
_GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD]
_GetShortPathNameW.restype = wintypes.DWORD
output_buf_size = 0
while True:
output_buf = ctypes.create_unicode_buffer(output_buf_size)
needed = _GetShortPathNameW(long_name, output_buf, output_buf_size)
if output_buf_size >= needed:
return output_buf.value
output_buf_size = needed
__all__ = [
"get_short_path_name",
]

View file

@ -0,0 +1,30 @@
from __future__ import annotations
import subprocess
CREATE_NO_WINDOW = 0x80000000
def run_cmd(cmd):
try:
process = subprocess.Popen(
cmd,
universal_newlines=True,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
encoding="utf-8",
)
out, err = process.communicate() # input disabled
code = process.returncode
except OSError as error:
code, out, err = error.errno, "", error.strerror
if code == 2 and "file" in err: # noqa: PLR2004
err = str(error) # FileNotFoundError in Python >= 3.3
return code, out, err
__all__ = (
"CREATE_NO_WINDOW",
"run_cmd",
)

View file

@ -0,0 +1,41 @@
from __future__ import annotations # noqa: A005
import logging
import os
import zipfile
from virtualenv.info import IS_WIN, ROOT
def read(full_path):
sub_file = _get_path_within_zip(full_path)
with zipfile.ZipFile(ROOT, "r") as zip_file, zip_file.open(sub_file) as file_handler:
return file_handler.read().decode("utf-8")
def extract(full_path, dest):
logging.debug("extract %s to %s", full_path, dest)
sub_file = _get_path_within_zip(full_path)
with zipfile.ZipFile(ROOT, "r") as zip_file:
info = zip_file.getinfo(sub_file)
info.filename = dest.name
zip_file.extract(info, str(dest.parent))
def _get_path_within_zip(full_path):
full_path = os.path.realpath(os.path.abspath(str(full_path)))
prefix = f"{ROOT}{os.sep}"
if not full_path.startswith(prefix):
msg = f"full_path={full_path} should start with prefix={prefix}."
raise RuntimeError(msg)
sub_file = full_path[len(prefix) :]
if IS_WIN:
# paths are always UNIX separators, even on Windows, though __file__ still follows platform default
sub_file = sub_file.replace(os.sep, "/")
return sub_file
__all__ = [
"extract",
"read",
]