install_states.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185
  1. """Reclaim per-checkout dependency state whose checkout is gone.
  2. Dependency state lives under ``<home>/installs/<install_key>`` where the key hashes the
  3. checkout path (:func:`pm.environments.install_key`). Every ``hermes -w`` worktree, review tree
  4. or scratch clone that boots the agent therefore commits a full environment (venv, test venv,
  5. PM runtime, ~200 MB each), and nothing removed it when the tree was deleted: one developer
  6. host carried 402 such orphans (80 GB) after a fortnight of worktree campaigns.
  7. The state dir records the checkout it belongs to in ``inputs/.project-root`` (written by
  8. :func:`pm.environments.record_activation_inputs`). A missing path alone does not prove the
  9. checkout is gone: a data root shared with a container (``-v ~/.hermes:/opt/data``) or an
  10. unmounted volume records paths this process cannot see. So a dir is an orphan only when its
  11. checkout sat where this process can see that it was deleted -- under the data root that owns
  12. ``installs/`` (scratch clones, the runtime checkout) or directly in a ``.worktrees/`` dir that
  13. still exists. Removal still fails closed on anything a live process could hold: the install
  14. lock, or a dependency / PM runtime generation lease.
  15. A checkout elsewhere (``~/.cache/review-1/head``, a scratch clone outside the data root) can never
  16. be proved deleted, so its missing checkout alone would keep the state forever. It is reclaimed
  17. once the state has also gone unused for ``worktrees.install_state_retention_days``: every sync
  18. recreates ``inputs/`` and every launch takes a generation lease (moving its
  19. ``.leases`` dir), so a checkout that still exists behind a mount boundary and is used at all keeps
  20. its state. One unused for longer is rebuilt by ``hermes pm repair`` / the next activation.
  21. """
  22. from __future__ import annotations
  23. import contextlib
  24. import logging
  25. import os
  26. import shutil
  27. import time
  28. from pathlib import Path
  29. logger = logging.getLogger(__name__)
  30. def _provably_deleted(root: Path, data_root: Path) -> bool:
  31. if root.exists():
  32. return False
  33. return root.is_relative_to(data_root) or (root.parent.name == ".worktrees" and root.parent.is_dir())
  34. def _last_used(state: Path) -> float:
  35. """Newest mtime among what a sync (``inputs/`` is recreated, ``facts.json`` rewritten) or a
  36. launch (a generation's ``.leases`` dir) moves. Not the state dir itself: a part-way removal
  37. moves it, and that must not restart the clock for the retry."""
  38. paths = [state / "inputs", state / "facts.json"]
  39. for generations in (state / "environments", state / "pm-runtime" / "generations"):
  40. if generations.is_dir():
  41. paths += [generation / ".leases" for generation in generations.iterdir()]
  42. mtimes = [0.0]
  43. for path in paths:
  44. with contextlib.suppress(OSError):
  45. mtimes.append(path.lstat().st_mtime)
  46. return max(mtimes)
  47. def _retention_seconds(data_root: Path) -> float | None:
  48. """The store-wide retention, from the config of the data root that owns ``installs/``.
  49. Every profile of that root shares the store, so the active profile's config must not decide
  50. for the others: a named profile without the key would apply the default to a state the root
  51. opted out of reclaiming."""
  52. from hermes_cli.config import cfg_get, load_config_readonly
  53. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  54. token = set_hermes_home_override(data_root)
  55. try:
  56. days = cfg_get(load_config_readonly(), "worktrees", "install_state_retention_days", default=14)
  57. finally:
  58. reset_hermes_home_override(token)
  59. return days * 86400 if isinstance(days, (int, float)) and days > 0 else None
  60. def _is_orphan(state: Path, data_root: Path, retention: float | None) -> bool:
  61. record = state / "inputs" / ".project-root"
  62. if not state.is_dir() or not record.is_file():
  63. return False
  64. try:
  65. root = record.read_text(encoding="utf-8-sig").strip()
  66. except OSError:
  67. return False
  68. if not root or Path(root).exists():
  69. return False
  70. return _provably_deleted(Path(root), data_root) or (
  71. retention is not None and time.time() - _last_used(state) > retention)
  72. def orphan_install_states(installs: Path) -> list[Path]:
  73. """State dirs whose recorded checkout is provably deleted, or missing and unused past the
  74. retention period. Dirs without a record, and dirs whose checkout exists, are left alone."""
  75. if not installs.is_dir():
  76. return []
  77. data_root = installs.resolve().parent
  78. retention = _retention_seconds(installs.parent)
  79. return [state for state in sorted(installs.iterdir()) if _is_orphan(state, data_root, retention)]
  80. @contextlib.contextmanager
  81. def _install_lock(state: Path):
  82. """Hold the state's install lock (``runtime_lock``'s) without waiting: yields False while
  83. another process holds it, or when it cannot even be opened."""
  84. from pm.filesystem import lock_path
  85. try:
  86. fd = lock_path(state / ".install.lock", timeout=0)
  87. except OSError:
  88. fd = None
  89. if fd is None:
  90. yield False
  91. return
  92. try:
  93. yield True
  94. finally:
  95. os.close(fd)
  96. def _leased(state: Path) -> bool:
  97. """True while a generation lease (dependency env or PM runtime) is held."""
  98. from hermes_cli.runtime_state import leases_held
  99. for generations in (state / "environments", state / "pm-runtime" / "generations"):
  100. if generations.is_dir():
  101. for generation in generations.iterdir():
  102. if generation.is_dir() and (generation / ".leases").is_dir() and leases_held(generation):
  103. return True
  104. return False
  105. _KEPT_UNTIL_LAST = frozenset({"inputs", ".install.lock"})
  106. def _remove_contents(state: Path) -> bool:
  107. """Remove everything but the record and the held lock; True when nothing else is left.
  108. The record goes last, and only once everything else is gone: the startup prune runs on a
  109. daemon thread and a removal can fail part-way (an open file on Windows, a read-only dir), and
  110. a dir that lost its record first would never be recognised again."""
  111. for child in state.iterdir():
  112. if child.name in _KEPT_UNTIL_LAST:
  113. continue
  114. if child.is_dir() and not child.is_symlink():
  115. shutil.rmtree(child, ignore_errors=True)
  116. else:
  117. with contextlib.suppress(OSError):
  118. child.unlink()
  119. return not any(child.name not in _KEPT_UNTIL_LAST for child in state.iterdir())
  120. def collect_orphan_install_states(installs: Path) -> list[Path]:
  121. """Remove every unheld orphan state dir under *installs*; returns what was removed.
  122. Each candidate is re-checked under its install lock right before removal: removing a large
  123. orphan takes long enough for another state to be synced or launched (lease taken and
  124. released) since the candidates were listed, and that use must keep it."""
  125. if not installs.is_dir():
  126. return []
  127. data_root = installs.resolve().parent
  128. retention = _retention_seconds(installs.parent)
  129. removed: list[Path] = []
  130. for state in [s for s in sorted(installs.iterdir()) if _is_orphan(s, data_root, retention)]:
  131. with _install_lock(state) as locked:
  132. if not locked or not _is_orphan(state, data_root, retention) or _leased(state):
  133. logger.debug("orphan install state %s is held or back in use; skipped", state.name)
  134. continue
  135. emptied = _remove_contents(state)
  136. if emptied: # still under the lock: only the record is left, so it goes last
  137. shutil.rmtree(state / "inputs", ignore_errors=True)
  138. # POSIX: unlink while still holding it. A launcher already waiting on this file
  139. # then sees it gone once it gets the lock and retries on a fresh one (lock_path);
  140. # unlinked after release, the path could be a file that launcher just made.
  141. if os.name != "nt":
  142. with contextlib.suppress(OSError):
  143. (state / ".install.lock").unlink()
  144. if not emptied:
  145. logger.debug("orphan install state %s only partly removed; retried next pass", state.name)
  146. continue
  147. # Non-recursive: a launch from a checkout recreated at the same path may have taken the
  148. # lock the moment it was released and written here; its files make rmdir fail, not vanish.
  149. with contextlib.suppress(OSError):
  150. if os.name == "nt": # an open file cannot be unlinked: a waiting launcher keeps it
  151. (state / ".install.lock").unlink()
  152. state.rmdir()
  153. if not state.exists():
  154. removed.append(state)
  155. return removed