first commit

This commit is contained in:
Yura 2024-09-15 15:12:16 +03:00
commit 417e54da96
5696 changed files with 900003 additions and 0 deletions

View file

@ -0,0 +1,56 @@
"""Application data stored by virtualenv."""
from __future__ import annotations
import logging
import os
from platformdirs import user_data_dir
from .na import AppDataDisabled
from .read_only import ReadOnlyAppData
from .via_disk_folder import AppDataDiskFolder
from .via_tempdir import TempAppData
def _default_app_data_dir(env):
key = "VIRTUALENV_OVERRIDE_APP_DATA"
if key in env:
return env[key]
return user_data_dir(appname="virtualenv", appauthor="pypa")
def make_app_data(folder, **kwargs):
is_read_only = kwargs.pop("read_only")
env = kwargs.pop("env")
if kwargs: # py3+ kwonly
msg = "unexpected keywords: {}"
raise TypeError(msg)
if folder is None:
folder = _default_app_data_dir(env)
folder = os.path.abspath(folder)
if is_read_only:
return ReadOnlyAppData(folder)
if not os.path.isdir(folder):
try:
os.makedirs(folder)
logging.debug("created app data folder %s", folder)
except OSError as exception:
logging.info("could not create app data folder %s due to %r", folder, exception)
if os.access(folder, os.W_OK):
return AppDataDiskFolder(folder)
logging.debug("app data folder %s has no write access", folder)
return TempAppData()
__all__ = (
"AppDataDisabled",
"AppDataDiskFolder",
"ReadOnlyAppData",
"TempAppData",
"make_app_data",
)

View file

@ -0,0 +1,96 @@
"""Application data stored by virtualenv."""
from __future__ import annotations
from abc import ABC, abstractmethod
from contextlib import contextmanager
from virtualenv.info import IS_ZIPAPP
class AppData(ABC):
"""Abstract storage interface for the virtualenv application."""
@abstractmethod
def close(self):
"""Called before virtualenv exits."""
@abstractmethod
def reset(self):
"""Called when the user passes in the reset app data."""
@abstractmethod
def py_info(self, path):
raise NotImplementedError
@abstractmethod
def py_info_clear(self):
raise NotImplementedError
@property
def can_update(self):
raise NotImplementedError
@abstractmethod
def embed_update_log(self, distribution, for_py_version):
raise NotImplementedError
@property
def house(self):
raise NotImplementedError
@property
def transient(self):
raise NotImplementedError
@abstractmethod
def wheel_image(self, for_py_version, name):
raise NotImplementedError
@contextmanager
def ensure_extracted(self, path, to_folder=None):
"""Some paths might be within the zipapp, unzip these to a path on the disk."""
if IS_ZIPAPP:
with self.extract(path, to_folder) as result:
yield result
else:
yield path
@abstractmethod
@contextmanager
def extract(self, path, to_folder):
raise NotImplementedError
@abstractmethod
@contextmanager
def locked(self, path):
raise NotImplementedError
class ContentStore(ABC):
@abstractmethod
def exists(self):
raise NotImplementedError
@abstractmethod
def read(self):
raise NotImplementedError
@abstractmethod
def write(self, content):
raise NotImplementedError
@abstractmethod
def remove(self):
raise NotImplementedError
@abstractmethod
@contextmanager
def locked(self):
pass
__all__ = [
"AppData",
"ContentStore",
]

View file

@ -0,0 +1,72 @@
from __future__ import annotations
from contextlib import contextmanager
from .base import AppData, ContentStore
class AppDataDisabled(AppData):
"""No application cache available (most likely as we don't have write permissions)."""
transient = True
can_update = False
def __init__(self) -> None:
pass
error = RuntimeError("no app data folder available, probably no write access to the folder")
def close(self):
"""Do nothing."""
def reset(self):
"""Do nothing."""
def py_info(self, path): # noqa: ARG002
return ContentStoreNA()
def embed_update_log(self, distribution, for_py_version): # noqa: ARG002
return ContentStoreNA()
def extract(self, path, to_folder): # noqa: ARG002
raise self.error
@contextmanager
def locked(self, path): # noqa: ARG002
"""Do nothing."""
yield
@property
def house(self):
raise self.error
def wheel_image(self, for_py_version, name): # noqa: ARG002
raise self.error
def py_info_clear(self):
"""Nothing to clear."""
class ContentStoreNA(ContentStore):
def exists(self):
return False
def read(self):
"""Nothing to read."""
return
def write(self, content):
"""Nothing to write."""
def remove(self):
"""Nothing to remove."""
@contextmanager
def locked(self):
yield
__all__ = [
"AppDataDisabled",
"ContentStoreNA",
]

View file

@ -0,0 +1,42 @@
from __future__ import annotations
import os.path
from virtualenv.util.lock import NoOpFileLock
from .via_disk_folder import AppDataDiskFolder, PyInfoStoreDisk
class ReadOnlyAppData(AppDataDiskFolder):
can_update = False
def __init__(self, folder: str) -> None:
if not os.path.isdir(folder):
msg = f"read-only app data directory {folder} does not exist"
raise RuntimeError(msg)
super().__init__(folder)
self.lock = NoOpFileLock(folder)
def reset(self) -> None:
msg = "read-only app data does not support reset"
raise RuntimeError(msg)
def py_info_clear(self) -> None:
raise NotImplementedError
def py_info(self, path):
return _PyInfoStoreDiskReadOnly(self.py_info_at, path)
def embed_update_log(self, distribution, for_py_version):
raise NotImplementedError
class _PyInfoStoreDiskReadOnly(PyInfoStoreDisk):
def write(self, content): # noqa: ARG002
msg = "read-only app data python info cannot be updated"
raise RuntimeError(msg)
__all__ = [
"ReadOnlyAppData",
]

View file

@ -0,0 +1,174 @@
"""
A rough layout of the current storage goes as:
virtualenv-app-data
py - <version> <cache information about python interpreters>
*.json/lock
wheel <cache wheels used for seeding>
house
*.whl <wheels downloaded go here>
<python major.minor> -> 3.9
img-<version>
image
<install class> -> CopyPipInstall / SymlinkPipInstall
<wheel name> -> pip-20.1.1-py2.py3-none-any
embed
3 -> json format versioning
*.json -> for every distribution contains data about newer embed versions and releases
unzip <in zip app we cannot refer to some internal files, so first extract them>
<virtualenv version>
py_info.py
debug.py
_virtualenv.py
""" # noqa: D415
from __future__ import annotations
import json
import logging
from abc import ABC
from contextlib import contextmanager, suppress
from hashlib import sha256
from virtualenv.util.lock import ReentrantFileLock
from virtualenv.util.path import safe_delete
from virtualenv.util.zipapp import extract
from virtualenv.version import __version__
from .base import AppData, ContentStore
class AppDataDiskFolder(AppData):
"""Store the application data on the disk within a folder layout."""
transient = False
can_update = True
def __init__(self, folder) -> None:
self.lock = ReentrantFileLock(folder)
def __repr__(self) -> str:
return f"{type(self).__name__}({self.lock.path})"
def __str__(self) -> str:
return str(self.lock.path)
def reset(self):
logging.debug("reset app data folder %s", self.lock.path)
safe_delete(self.lock.path)
def close(self):
"""Do nothing."""
@contextmanager
def locked(self, path):
path_lock = self.lock / path
with path_lock:
yield path_lock.path
@contextmanager
def extract(self, path, to_folder):
root = ReentrantFileLock(to_folder()) if to_folder is not None else self.lock / "unzip" / __version__
with root.lock_for_key(path.name):
dest = root.path / path.name
if not dest.exists():
extract(path, dest)
yield dest
@property
def py_info_at(self):
return self.lock / "py_info" / "1"
def py_info(self, path):
return PyInfoStoreDisk(self.py_info_at, path)
def py_info_clear(self):
"""clear py info."""
py_info_folder = self.py_info_at
with py_info_folder:
for filename in py_info_folder.path.iterdir():
if filename.suffix == ".json":
with py_info_folder.lock_for_key(filename.stem):
if filename.exists():
filename.unlink()
def embed_update_log(self, distribution, for_py_version):
return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "3", distribution)
@property
def house(self):
path = self.lock.path / "wheel" / "house"
path.mkdir(parents=True, exist_ok=True)
return path
def wheel_image(self, for_py_version, name):
return self.lock.path / "wheel" / for_py_version / "image" / "1" / name
class JSONStoreDisk(ContentStore, ABC):
def __init__(self, in_folder, key, msg, msg_args) -> None:
self.in_folder = in_folder
self.key = key
self.msg = msg
self.msg_args = (*msg_args, self.file)
@property
def file(self):
return self.in_folder.path / f"{self.key}.json"
def exists(self):
return self.file.exists()
def read(self):
data, bad_format = None, False
try:
data = json.loads(self.file.read_text(encoding="utf-8"))
except ValueError:
bad_format = True
except Exception: # noqa: BLE001, S110
pass
else:
logging.debug("got %s from %s", self.msg, self.msg_args)
return data
if bad_format:
with suppress(OSError): # reading and writing on the same file may cause race on multiple processes
self.remove()
return None
def remove(self):
self.file.unlink()
logging.debug("removed %s at %s", self.msg, self.msg_args)
@contextmanager
def locked(self):
with self.in_folder.lock_for_key(self.key):
yield
def write(self, content):
folder = self.file.parent
folder.mkdir(parents=True, exist_ok=True)
self.file.write_text(json.dumps(content, sort_keys=True, indent=2), encoding="utf-8")
logging.debug("wrote %s at %s", self.msg, self.msg_args)
class PyInfoStoreDisk(JSONStoreDisk):
def __init__(self, in_folder, path) -> None:
key = sha256(str(path).encode("utf-8")).hexdigest()
super().__init__(in_folder, key, "python info of %s", (path,))
class EmbedDistributionUpdateStoreDisk(JSONStoreDisk):
def __init__(self, in_folder, distribution) -> None:
super().__init__(
in_folder,
distribution,
"embed update of distribution %s",
(distribution,),
)
__all__ = [
"AppDataDiskFolder",
"JSONStoreDisk",
"PyInfoStoreDisk",
]

View file

@ -0,0 +1,32 @@
from __future__ import annotations
import logging
from tempfile import mkdtemp
from virtualenv.util.path import safe_delete
from .via_disk_folder import AppDataDiskFolder
class TempAppData(AppDataDiskFolder):
transient = True
can_update = False
def __init__(self) -> None:
super().__init__(folder=mkdtemp())
logging.debug("created temporary app data folder %s", self.lock.path)
def reset(self):
"""This is a temporary folder, is already empty to start with."""
def close(self):
logging.debug("remove temporary app data folder %s", self.lock.path)
safe_delete(self.lock.path)
def embed_update_log(self, distribution, for_py_version):
raise NotImplementedError
__all__ = [
"TempAppData",
]