runtime_state.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. """Stdlib-only recovery and lifetime protection for dependency generations."""
  2. from __future__ import annotations
  3. import atexit
  4. import base64
  5. from collections.abc import Callable
  6. from contextlib import contextmanager, suppress
  7. import hashlib
  8. import json
  9. import logging
  10. import os
  11. from pathlib import Path
  12. import shutil
  13. import time
  14. import uuid
  15. from pm.environments import dependency_home_root, install_state_dir, runtime_facts_path
  16. # Private aliases: this module calls them through its globals (tests patch ``_atomic_bytes``
  17. # here) and updaters shipped before PM import them by these names mid-swap
  18. # (tests/compat/old_updater_surface.json). New code imports the pm.filesystem names.
  19. from pm.filesystem import (
  20. durable_write_bytes as _atomic_bytes,
  21. file_digest as _digest,
  22. lock_fd as _lock,
  23. lock_path,
  24. read_bytes_or_none as _bytes,
  25. )
  26. LOG = logging.getLogger(__name__)
  27. # How long a process that only needs to READ the install state waits for a writer. The holder can
  28. # be another profile's backend rebuilding the whole dependency environment (measured: tens of
  29. # seconds on a bundle, minutes when a sync falls back to an sdist build), and a backend that never
  30. # binds its port is worse than one that binds against the previous generation. Losers skip — the
  31. # same rule boot_bootstrap._RecordLock states for home maintenance.
  32. INSTALL_LOCK_TIMEOUT_SECONDS = 10.0
  33. # A lease file younger than this is never a prune candidate: its creator may still sit between
  34. # ``O_CREAT|O_EXCL`` and ``flock`` (two syscalls; the window is microseconds, the margin generous).
  35. @contextmanager
  36. def runtime_lock(project: Path, *, timeout: float | None = INSTALL_LOCK_TIMEOUT_SECONDS):
  37. """Hold the per-install dependency lock; yields True when held, False when the wait expired.
  38. Callers decide what a lost race means: readers skip the work the lock guards and carry on
  39. (``activate_dependencies`` still selects and leases the committed generation), writers that
  40. cannot be skipped pass ``timeout=None`` — an install the user asked for is theirs to wait on.
  41. """
  42. lock = install_state_dir(project) / ".install.lock"
  43. fd = lock_path(lock, timeout=timeout, create_dir=True)
  44. if fd is None:
  45. LOG.warning("dependency lock still held after %ss; continuing without it (%s)", timeout, lock)
  46. yield False
  47. return
  48. try:
  49. yield True
  50. finally:
  51. os.close(fd)
  52. def _recover_plugin_publication(project: Path, row: dict, journal: Path) -> None:
  53. from hermes_cli.fs_utils import rmtree_force
  54. target, backup, metadata = (Path(row[key]) for key in ("target", "backup", "metadata"))
  55. home = dependency_home_root().resolve()
  56. if (not target.resolve().is_relative_to(home) or target.parent.name != "plugins"
  57. or backup.parent != target.parent or not backup.name.startswith(".previous-")
  58. or metadata != target.parent / ".install-metadata.json"):
  59. raise ValueError("plugin publication paths escape their home")
  60. committed = row.get("committed") or _digest(runtime_facts_path(project)) != row["facts_before"]
  61. if committed:
  62. if backup.exists():
  63. rmtree_force(backup)
  64. else:
  65. old = base64.b64decode(row["metadata_before"], validate=True) if row["metadata_before"] is not None else None
  66. current = _bytes(metadata)
  67. new = base64.b64decode(row["metadata_after"], validate=True)
  68. if current not in (old, new):
  69. raise ValueError("plugin metadata changed after publication; preserve it for manual recovery")
  70. if backup.exists():
  71. if target.exists():
  72. rmtree_force(target)
  73. os.replace(backup, target)
  74. elif not row["target_existed"] and target.exists():
  75. rmtree_force(target)
  76. if old is None:
  77. metadata.unlink(missing_ok=True)
  78. else:
  79. _atomic_bytes(metadata, old)
  80. journal.unlink()
  81. def recover_publication(project: Path) -> None:
  82. """Recover while holding runtime_lock, before activation or another write."""
  83. journal = install_state_dir(project) / "publication.json"
  84. data = _bytes(journal)
  85. if data is None:
  86. return
  87. try:
  88. row = json.loads(data)
  89. if row.get("kind") == "plugin":
  90. _recover_plugin_publication(project, row, journal)
  91. return
  92. # One row may carry several configs (a plugin eviction edits every home that enables it).
  93. entries = row["configs"] if "configs" in row else [row]
  94. configs = []
  95. for entry in entries:
  96. config = Path(entry["config"])
  97. if config.name != "config.yaml" or not config.resolve().is_relative_to(dependency_home_root().resolve()):
  98. raise ValueError("config path is outside Hermes state")
  99. previous = base64.b64decode(entry["previous"], validate=True) if entry["previous"] is not None else None
  100. configs.append((config, previous, entry.get("config_after")))
  101. if not row.get("committed") and _digest(runtime_facts_path(project)) == row["facts_before"]:
  102. for config, previous, after in configs:
  103. prior = hashlib.sha256(previous).hexdigest() if previous is not None else None
  104. if _digest(config) not in (prior, after):
  105. raise ValueError("config changed after publication began; preserve it for manual recovery")
  106. # No selection was published. Roll back before any plugin is loaded.
  107. for config, previous, _after in configs:
  108. if previous is None:
  109. config.unlink(missing_ok=True)
  110. else:
  111. _atomic_bytes(config, previous)
  112. journal.unlink()
  113. except (ValueError, KeyError, TypeError, OSError) as exc:
  114. raise RuntimeError(f"cannot recover dependency publication: {journal}: {exc}") from exc
  115. def finish_publication(project: Path) -> None:
  116. """Persist a commit even when code/config changed without a new generation."""
  117. journal = install_state_dir(project) / "publication.json"
  118. row = json.loads(journal.read_bytes())
  119. row["committed"] = True
  120. _atomic_bytes(journal, json.dumps(row).encode())
  121. recover_publication(project)
  122. def lease_generation(environment: Path) -> Callable[[], None]:
  123. """Hold a kernel lock until process exit; the returned callable releases it early.
  124. Call under ``runtime_lock`` at boot. Without the lock (``runtime_lock`` timed out) the
  125. caller must re-read the selection after leasing: an installer may have moved it in between,
  126. and an unselected, unleased generation is exactly what the collector removes.
  127. """
  128. return lease_directory(environment.parent)
  129. def lease_directory(generation: Path) -> Callable[[], None]:
  130. """Pin a lease-managed generation directory for this process's lifetime."""
  131. if not (generation / ".lease-managed").is_file():
  132. return lambda: None # Generations produced before leases stay conservatively retained.
  133. leases = generation / ".leases"
  134. leases.mkdir(exist_ok=True)
  135. _prune_unlocked_leases(leases)
  136. while True:
  137. lease = leases / uuid.uuid4().hex
  138. fd = os.open(lease, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600)
  139. try:
  140. _lock(fd, wait=True)
  141. except BaseException:
  142. os.close(fd)
  143. raise
  144. # Create and flock are two syscalls; a peer's prune (pm workers and launch run outside
  145. # ``runtime_lock``) can lock-and-unlink the file in between, and we would then hold an
  146. # unlinked inode invisible to every collector. The pruner only unlinks while holding the
  147. # lock, so once we hold it the path is either still ours or already gone for good.
  148. if lease.exists():
  149. break
  150. os.close(fd)
  151. def release() -> None:
  152. atexit.unregister(release)
  153. os.close(fd)
  154. # Best effort: the collector ignores unlocked lease files, but one per
  155. # invocation would otherwise accumulate for the life of the generation.
  156. with suppress(OSError):
  157. lease.unlink()
  158. atexit.register(release)
  159. return release
  160. def collect_generations(project: Path, *, min_age_seconds: float = 86400) -> list[Path]:
  161. """Remove unselected lease-managed generations after their readers exit."""
  162. from pm.environments import selected_venv
  163. removed = []
  164. root = install_state_dir(project)
  165. if not root.exists():
  166. return removed
  167. with runtime_lock(project) as held:
  168. if not held:
  169. return removed # maintenance: another process owns the install, skip rather than queue
  170. recover_publication(project)
  171. selected = selected_venv(project).parent.resolve()
  172. generations = root / "environments"
  173. if not generations.is_dir():
  174. return removed
  175. for generation in generations.iterdir():
  176. if generation.is_symlink() or not generation.is_dir() or generation.resolve() == selected:
  177. continue
  178. marker = generation / ".lease-managed"
  179. if not marker.is_file() or time.time() - marker.stat().st_mtime < min_age_seconds:
  180. continue
  181. if not leases_held(generation):
  182. shutil.rmtree(generation)
  183. removed.append(generation)
  184. return removed
  185. def leases_held(generation: Path) -> bool:
  186. """True while any process still holds a lease taken by ``lease_directory``."""
  187. return _prune_unlocked_leases(generation / ".leases")
  188. def _open_lease_probe(lease: Path) -> int:
  189. """Let a Windows pruner unlink while still holding its exclusion lock.
  190. Creator handles deliberately keep their ordinary sharing: an open creator
  191. between O_EXCL and locking must still prevent a peer from deleting its lease.
  192. """
  193. if os.name != "nt":
  194. return os.open(lease, os.O_RDWR)
  195. import ctypes
  196. from ctypes import wintypes
  197. import msvcrt
  198. kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
  199. kernel32.CreateFileW.argtypes = [
  200. wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p,
  201. wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE,
  202. ]
  203. kernel32.CreateFileW.restype = wintypes.HANDLE
  204. kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
  205. kernel32.CloseHandle.restype = wintypes.BOOL
  206. # GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | WRITE | DELETE, OPEN_EXISTING.
  207. handle = kernel32.CreateFileW(str(lease), 0xC0000000, 0x7, None, 3, 0x80, None)
  208. if handle == ctypes.c_void_p(-1).value:
  209. raise ctypes.WinError(ctypes.get_last_error())
  210. try:
  211. # The CRT owns the handle after conversion; os.close releases it and its lock.
  212. return msvcrt.open_osfhandle(handle, os.O_RDWR | os.O_BINARY | os.O_NOINHERIT)
  213. except BaseException:
  214. kernel32.CloseHandle(handle)
  215. raise
  216. def _prune_unlocked_leases(leases: Path) -> bool:
  217. """Remove abandoned lease files and report whether any live lock remains.
  218. Kernel locks disappear even when ``execv``, ``os._exit`` or a crash bypasses
  219. ``atexit``. Cleaning those unlocked files whenever a reader arrives bounds leaks in
  220. the selected generation too, which generation GC intentionally never visits.
  221. Fail closed: a lease this user cannot open or remove is held, never a boot failure.
  222. """
  223. held = False
  224. for lease in leases.glob("*"):
  225. try:
  226. fd = _open_lease_probe(lease)
  227. except FileNotFoundError:
  228. continue
  229. except OSError:
  230. held = True
  231. continue
  232. try:
  233. if not _lock(fd, wait=False):
  234. held = True
  235. else:
  236. try:
  237. lease.unlink()
  238. except FileNotFoundError:
  239. pass # A peer already removed it.
  240. except OSError:
  241. held = True
  242. finally:
  243. os.close(fd)
  244. return held