web_server_skew_exit.py 4.4 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697
  1. """Code-skew retirement for Desktop-owned ``hermes serve --isolated`` backends reached over SSH.
  2. Such a backend belongs to a Desktop on another machine: the host's updater never restarts it (only
  3. the remote client holds its token and owner nonce), so after ``hermes update`` it keeps serving the
  4. code it imported at startup against a tree that has moved on, and lazily imports new modules into
  5. an old process. The idle watchdog (``web_server_idle_exit``) only fires once every client has left,
  6. which a connected Desktop never does.
  7. This watchdog polls ``gateway.code_skew`` (the boot revision the serve lifespan records, compared
  8. with the checkout on disk) and, once no update is in flight and the retirement fence proves the
  9. backend idle (and closes admission so no turn can start mid-teardown), exits cleanly. The client's
  10. reconnect path respawns it from the new code. Every uncertain read (unknown revision, busy or
  11. indeterminate ledgers, a live update) keeps the process up.
  12. """
  13. from __future__ import annotations
  14. import logging
  15. import threading
  16. import time
  17. from typing import Callable, Optional
  18. _log = logging.getLogger(__name__)
  19. DEFAULT_SKEW_POLL_S = 30.0
  20. # Two consecutive observations: a checkout mid-rewrite (ref file replaced, packed-refs repacked) can
  21. # briefly read differently; a real update reads the new revision on every poll after it finishes.
  22. _CONFIRMATIONS = 2
  23. # ``(boot_rev, disk_rev)`` when the checkout drifted since boot, else None (``detect_code_skew``).
  24. Skew = Optional[tuple[str, str]]
  25. def should_retire_for_skew(*, skew: Skew, update_in_progress: bool) -> bool:
  26. """Retire only on a proven code change, and never while the tree may still be mid-swap."""
  27. return bool(skew) and not update_in_progress
  28. def _update_in_progress() -> bool:
  29. try:
  30. from hermes_cli.update_lock import update_in_progress
  31. return update_in_progress()
  32. except Exception:
  33. return True # cannot prove the swap is over: wait
  34. def _run_retirement_watchdog(server, *, observe: Callable[[], Optional[str]], fence, poll_s: float,
  35. max_polls: Optional[int], thread_name: str) -> threading.Thread:
  36. """Daemon thread shared by the SSH-isolated retirement watchdogs: ``observe`` returns the log
  37. reason while retirement is warranted (else None); after ``_CONFIRMATIONS`` consecutive reasons
  38. the backend exits only through the retirement fence (proven idle, admission closed first)."""
  39. if fence is None:
  40. from hermes_cli.backend_retirement import retirement
  41. fence = retirement
  42. def _loop() -> None:
  43. seen = polls = 0
  44. while not getattr(server, "should_exit", False) and (max_polls is None or polls < max_polls):
  45. polls += 1
  46. reason = observe()
  47. seen = seen + 1 if reason else 0
  48. if seen >= _CONFIRMATIONS:
  49. permit = fence.prepare()
  50. if permit.get("ok") and fence.commit(permit.get("token")).get("ok"):
  51. _log.warning("%s", reason)
  52. server.should_exit = True
  53. return
  54. time.sleep(poll_s)
  55. thread = threading.Thread(target=_loop, daemon=True, name=thread_name)
  56. thread.start()
  57. return thread
  58. def start_code_skew_watchdog(server, *, skew_fn: Optional[Callable[[], Skew]] = None,
  59. update_probe: Callable[[], bool] = _update_in_progress,
  60. fence=None, poll_s: float = DEFAULT_SKEW_POLL_S,
  61. max_polls: Optional[int] = None) -> threading.Thread:
  62. """Daemon thread that sets ``server.should_exit`` once the loaded code is provably stale and
  63. the backend is provably idle. ``max_polls`` bounds the loop for tests only."""
  64. if skew_fn is None:
  65. from gateway.code_skew import detect_code_skew
  66. skew_fn = detect_code_skew
  67. read_skew: Callable[[], Skew] = skew_fn
  68. def _observe() -> Optional[str]:
  69. skew = read_skew()
  70. if not (skew and should_retire_for_skew(skew=skew, update_in_progress=update_probe())):
  71. return None
  72. return ("SSH-isolated backend loaded {} but the install is now at {}; "
  73. "retiring so its client respawns it on the new code.".format(*skew))
  74. return _run_retirement_watchdog(server, observe=_observe, fence=fence, poll_s=poll_s,
  75. max_polls=max_polls, thread_name="ssh-isolated-skew-watchdog")