web_server_owner_exit.py 2.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657
  1. """Supersession retirement for Desktop-owned ``hermes serve --isolated`` backends reached over SSH.
  2. Such a backend is detached on purpose (#91668) and the remote Desktop decides ownership through
  3. ``desktop-ssh/<ownershipId>/backend.lock.json``. When a reconnect cannot prove the recorded pid is
  4. its own (dead-looking pid, foreign argv proof) it drops the lock without signalling that pid and
  5. spawns a new nonce (#132034), so the old backend keeps running until the idle watchdog fires — never,
  6. while a half-open tunnel still counts as a client — stacking writers on ``state.db``.
  7. This watchdog polls the lock and, once a VALID lock names another ``spawnNonce`` on consecutive
  8. polls and the retirement fence proves the backend idle (closing admission so no turn can start
  9. mid-teardown), exits cleanly. A missing, unreadable or skewed lock keeps the process up: those are
  10. the windows between a cleanup and the replacement's write, or another Desktop build's file.
  11. """
  12. from __future__ import annotations
  13. import threading
  14. import time
  15. from pathlib import Path
  16. from typing import Callable, Optional
  17. from hermes_cli.web_server_skew_exit import _run_retirement_watchdog
  18. DEFAULT_OWNER_POLL_S = 15.0
  19. def should_retire_superseded(*, lock: Optional[dict], my_nonce: str, age_s: float) -> bool:
  20. """Retire only when a valid lock provably names another spawn of this ownership slot. A young
  21. process may still see the previous spawn's lock, so it must first outlive the same settle window
  22. the orphan reaper gives the Desktop to write the lock."""
  23. from hermes_cli.dashboard_procs import _REAP_MIN_AGE_SECONDS
  24. return lock is not None and lock["spawnNonce"] != my_nonce and age_s >= _REAP_MIN_AGE_SECONDS
  25. def start_owner_watchdog(server, *, lock_path: Path, nonce: str,
  26. read_lock: Optional[Callable[[Path], Optional[dict]]] = None,
  27. fence=None, poll_s: float = DEFAULT_OWNER_POLL_S,
  28. now: Callable[[], float] = time.monotonic,
  29. max_polls: Optional[int] = None) -> threading.Thread:
  30. """Daemon thread that sets ``server.should_exit`` once this backend is provably superseded and
  31. provably idle. ``max_polls`` bounds the loop for tests only."""
  32. if read_lock is None:
  33. from hermes_cli.dashboard_procs import read_valid_backend_lock
  34. read_lock = read_valid_backend_lock
  35. reader: Callable[[Path], Optional[dict]] = read_lock
  36. started = now()
  37. def _observe() -> Optional[str]:
  38. lock = reader(lock_path)
  39. if lock is None or not should_retire_superseded(lock=lock, my_nonce=nonce, age_s=now() - started):
  40. return None
  41. return f"SSH-isolated backend {nonce} was superseded by spawn {lock['spawnNonce']}; retiring."
  42. return _run_retirement_watchdog(server, observe=_observe, fence=fence, poll_s=poll_s,
  43. max_polls=max_polls, thread_name="ssh-isolated-owner-watchdog")