| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277 |
- """Stdlib-only recovery and lifetime protection for dependency generations."""
- from __future__ import annotations
- import atexit
- import base64
- from collections.abc import Callable
- from contextlib import contextmanager, suppress
- import hashlib
- import json
- import logging
- import os
- from pathlib import Path
- import shutil
- import time
- import uuid
- from pm.environments import dependency_home_root, install_state_dir, runtime_facts_path
- # Private aliases: this module calls them through its globals (tests patch ``_atomic_bytes``
- # here) and updaters shipped before PM import them by these names mid-swap
- # (tests/compat/old_updater_surface.json). New code imports the pm.filesystem names.
- from pm.filesystem import (
- durable_write_bytes as _atomic_bytes,
- file_digest as _digest,
- lock_fd as _lock,
- lock_path,
- read_bytes_or_none as _bytes,
- )
- LOG = logging.getLogger(__name__)
- # How long a process that only needs to READ the install state waits for a writer. The holder can
- # be another profile's backend rebuilding the whole dependency environment (measured: tens of
- # seconds on a bundle, minutes when a sync falls back to an sdist build), and a backend that never
- # binds its port is worse than one that binds against the previous generation. Losers skip — the
- # same rule boot_bootstrap._RecordLock states for home maintenance.
- INSTALL_LOCK_TIMEOUT_SECONDS = 10.0
- # A lease file younger than this is never a prune candidate: its creator may still sit between
- # ``O_CREAT|O_EXCL`` and ``flock`` (two syscalls; the window is microseconds, the margin generous).
- @contextmanager
- def runtime_lock(project: Path, *, timeout: float | None = INSTALL_LOCK_TIMEOUT_SECONDS):
- """Hold the per-install dependency lock; yields True when held, False when the wait expired.
- Callers decide what a lost race means: readers skip the work the lock guards and carry on
- (``activate_dependencies`` still selects and leases the committed generation), writers that
- cannot be skipped pass ``timeout=None`` — an install the user asked for is theirs to wait on.
- """
- lock = install_state_dir(project) / ".install.lock"
- fd = lock_path(lock, timeout=timeout, create_dir=True)
- if fd is None:
- LOG.warning("dependency lock still held after %ss; continuing without it (%s)", timeout, lock)
- yield False
- return
- try:
- yield True
- finally:
- os.close(fd)
- def _recover_plugin_publication(project: Path, row: dict, journal: Path) -> None:
- from hermes_cli.fs_utils import rmtree_force
- target, backup, metadata = (Path(row[key]) for key in ("target", "backup", "metadata"))
- home = dependency_home_root().resolve()
- if (not target.resolve().is_relative_to(home) or target.parent.name != "plugins"
- or backup.parent != target.parent or not backup.name.startswith(".previous-")
- or metadata != target.parent / ".install-metadata.json"):
- raise ValueError("plugin publication paths escape their home")
- committed = row.get("committed") or _digest(runtime_facts_path(project)) != row["facts_before"]
- if committed:
- if backup.exists():
- rmtree_force(backup)
- else:
- old = base64.b64decode(row["metadata_before"], validate=True) if row["metadata_before"] is not None else None
- current = _bytes(metadata)
- new = base64.b64decode(row["metadata_after"], validate=True)
- if current not in (old, new):
- raise ValueError("plugin metadata changed after publication; preserve it for manual recovery")
- if backup.exists():
- if target.exists():
- rmtree_force(target)
- os.replace(backup, target)
- elif not row["target_existed"] and target.exists():
- rmtree_force(target)
- if old is None:
- metadata.unlink(missing_ok=True)
- else:
- _atomic_bytes(metadata, old)
- journal.unlink()
- def recover_publication(project: Path) -> None:
- """Recover while holding runtime_lock, before activation or another write."""
- journal = install_state_dir(project) / "publication.json"
- data = _bytes(journal)
- if data is None:
- return
- try:
- row = json.loads(data)
- if row.get("kind") == "plugin":
- _recover_plugin_publication(project, row, journal)
- return
- # One row may carry several configs (a plugin eviction edits every home that enables it).
- entries = row["configs"] if "configs" in row else [row]
- configs = []
- for entry in entries:
- config = Path(entry["config"])
- if config.name != "config.yaml" or not config.resolve().is_relative_to(dependency_home_root().resolve()):
- raise ValueError("config path is outside Hermes state")
- previous = base64.b64decode(entry["previous"], validate=True) if entry["previous"] is not None else None
- configs.append((config, previous, entry.get("config_after")))
- if not row.get("committed") and _digest(runtime_facts_path(project)) == row["facts_before"]:
- for config, previous, after in configs:
- prior = hashlib.sha256(previous).hexdigest() if previous is not None else None
- if _digest(config) not in (prior, after):
- raise ValueError("config changed after publication began; preserve it for manual recovery")
- # No selection was published. Roll back before any plugin is loaded.
- for config, previous, _after in configs:
- if previous is None:
- config.unlink(missing_ok=True)
- else:
- _atomic_bytes(config, previous)
- journal.unlink()
- except (ValueError, KeyError, TypeError, OSError) as exc:
- raise RuntimeError(f"cannot recover dependency publication: {journal}: {exc}") from exc
- def finish_publication(project: Path) -> None:
- """Persist a commit even when code/config changed without a new generation."""
- journal = install_state_dir(project) / "publication.json"
- row = json.loads(journal.read_bytes())
- row["committed"] = True
- _atomic_bytes(journal, json.dumps(row).encode())
- recover_publication(project)
- def lease_generation(environment: Path) -> Callable[[], None]:
- """Hold a kernel lock until process exit; the returned callable releases it early.
- Call under ``runtime_lock`` at boot. Without the lock (``runtime_lock`` timed out) the
- caller must re-read the selection after leasing: an installer may have moved it in between,
- and an unselected, unleased generation is exactly what the collector removes.
- """
- return lease_directory(environment.parent)
- def lease_directory(generation: Path) -> Callable[[], None]:
- """Pin a lease-managed generation directory for this process's lifetime."""
- if not (generation / ".lease-managed").is_file():
- return lambda: None # Generations produced before leases stay conservatively retained.
- leases = generation / ".leases"
- leases.mkdir(exist_ok=True)
- _prune_unlocked_leases(leases)
- while True:
- lease = leases / uuid.uuid4().hex
- fd = os.open(lease, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600)
- try:
- _lock(fd, wait=True)
- except BaseException:
- os.close(fd)
- raise
- # Create and flock are two syscalls; a peer's prune (pm workers and launch run outside
- # ``runtime_lock``) can lock-and-unlink the file in between, and we would then hold an
- # unlinked inode invisible to every collector. The pruner only unlinks while holding the
- # lock, so once we hold it the path is either still ours or already gone for good.
- if lease.exists():
- break
- os.close(fd)
- def release() -> None:
- atexit.unregister(release)
- os.close(fd)
- # Best effort: the collector ignores unlocked lease files, but one per
- # invocation would otherwise accumulate for the life of the generation.
- with suppress(OSError):
- lease.unlink()
- atexit.register(release)
- return release
- def collect_generations(project: Path, *, min_age_seconds: float = 86400) -> list[Path]:
- """Remove unselected lease-managed generations after their readers exit."""
- from pm.environments import selected_venv
- removed = []
- root = install_state_dir(project)
- if not root.exists():
- return removed
- with runtime_lock(project) as held:
- if not held:
- return removed # maintenance: another process owns the install, skip rather than queue
- recover_publication(project)
- selected = selected_venv(project).parent.resolve()
- generations = root / "environments"
- if not generations.is_dir():
- return removed
- for generation in generations.iterdir():
- if generation.is_symlink() or not generation.is_dir() or generation.resolve() == selected:
- continue
- marker = generation / ".lease-managed"
- if not marker.is_file() or time.time() - marker.stat().st_mtime < min_age_seconds:
- continue
- if not leases_held(generation):
- shutil.rmtree(generation)
- removed.append(generation)
- return removed
- def leases_held(generation: Path) -> bool:
- """True while any process still holds a lease taken by ``lease_directory``."""
- return _prune_unlocked_leases(generation / ".leases")
- def _open_lease_probe(lease: Path) -> int:
- """Let a Windows pruner unlink while still holding its exclusion lock.
- Creator handles deliberately keep their ordinary sharing: an open creator
- between O_EXCL and locking must still prevent a peer from deleting its lease.
- """
- if os.name != "nt":
- return os.open(lease, os.O_RDWR)
- import ctypes
- from ctypes import wintypes
- import msvcrt
- kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
- kernel32.CreateFileW.argtypes = [
- wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p,
- wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE,
- ]
- kernel32.CreateFileW.restype = wintypes.HANDLE
- kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
- kernel32.CloseHandle.restype = wintypes.BOOL
- # GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | WRITE | DELETE, OPEN_EXISTING.
- handle = kernel32.CreateFileW(str(lease), 0xC0000000, 0x7, None, 3, 0x80, None)
- if handle == ctypes.c_void_p(-1).value:
- raise ctypes.WinError(ctypes.get_last_error())
- try:
- # The CRT owns the handle after conversion; os.close releases it and its lock.
- return msvcrt.open_osfhandle(handle, os.O_RDWR | os.O_BINARY | os.O_NOINHERIT)
- except BaseException:
- kernel32.CloseHandle(handle)
- raise
- def _prune_unlocked_leases(leases: Path) -> bool:
- """Remove abandoned lease files and report whether any live lock remains.
- Kernel locks disappear even when ``execv``, ``os._exit`` or a crash bypasses
- ``atexit``. Cleaning those unlocked files whenever a reader arrives bounds leaks in
- the selected generation too, which generation GC intentionally never visits.
- Fail closed: a lease this user cannot open or remove is held, never a boot failure.
- """
- held = False
- for lease in leases.glob("*"):
- try:
- fd = _open_lease_probe(lease)
- except FileNotFoundError:
- continue
- except OSError:
- held = True
- continue
- try:
- if not _lock(fd, wait=False):
- held = True
- else:
- try:
- lease.unlink()
- except FileNotFoundError:
- pass # A peer already removed it.
- except OSError:
- held = True
- finally:
- os.close(fd)
- return held
|