web_server_idle_exit.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170
  1. """Idle-exit for Desktop-owned ``hermes serve --isolated`` backends reached over SSH (#101626).
  2. That backend is deliberately detached (``setsid``/``nohup``, PPID 1) so it survives the SSH channel
  3. closing, and every teardown path lives on the CLIENT. A laptop that sleeps mid-session (dark wake
  4. reconnects the tunnel, spawns a backend, sleeps again) therefore leaves a backend behind every
  5. cycle — each one an extra writer on ``state.db``. The server needs its own liveness signal.
  6. Two pieces, both scoped to the SSH-isolated case (a session token was handed over via
  7. ``--ssh-session-token-file``):
  8. * An ASGI wrapper counts accepted WebSocket connections (every dashboard WS route: /api/ws,
  9. /api/pty, /api/console, /api/pub, /api/events, /api/audio/speak-stream) without touching the
  10. handlers. When the count has been zero for the grace window and no agent turn is running, the
  11. watchdog asks uvicorn to exit gracefully (WAL checkpoint, exit 0). An indeterminate turn probe
  12. fails closed: the backend stays up.
  13. * Loopback normally disables uvicorn's WS ping (a dead local client sends FIN/RST). Across an SSH
  14. tunnel the local socket is healthy while the far end is asleep, so pings are the only way to notice
  15. a half-open tunnel; the isolated backend keeps a slow ping with a long timeout so a GIL-holding
  16. turn cannot trip it.
  17. Design and the client-count/turn-probe/fail-closed shape are from #101678 by @StanleyStetson; this
  18. is the slim redo on the decomposed web server.
  19. """
  20. from __future__ import annotations
  21. import logging
  22. import threading
  23. import time
  24. from typing import Callable, Optional
  25. _log = logging.getLogger(__name__)
  26. DEFAULT_IDLE_GRACE_S = 900.0
  27. # Slow enough that a long GIL-holding turn (minutes) cannot trip it, fast enough that a sleeping
  28. # laptop's half-open tunnel is noticed well inside the idle grace window.
  29. TUNNEL_WS_PING_INTERVAL_S = 60.0
  30. TUNNEL_WS_PING_TIMEOUT_S = 600.0
  31. class IdleClientTracker:
  32. """Live accepted-WebSocket count plus the moment the last client left."""
  33. def __init__(self, now: Callable[[], float] = time.monotonic) -> None:
  34. self._now = now
  35. self._lock = threading.Lock()
  36. self._live = 0
  37. self._last_client_at = now()
  38. def on_open(self) -> None:
  39. with self._lock:
  40. self._live += 1
  41. self._last_client_at = self._now()
  42. def on_close(self) -> None:
  43. with self._lock:
  44. self._live = max(0, self._live - 1)
  45. self._last_client_at = self._now()
  46. def live_count(self) -> int:
  47. with self._lock:
  48. return self._live
  49. def idle_for(self) -> float:
  50. with self._lock:
  51. return 0.0 if self._live else self._now() - self._last_client_at
  52. def wrap_asgi_with_ws_tracking(app, tracker: IdleClientTracker):
  53. """Count WebSocket sessions at the ASGI boundary: open on the ``websocket.accept`` send, close
  54. when the scope ends. Handlers stay untouched, so a new WS route is tracked automatically."""
  55. async def _app(scope, receive, send):
  56. if scope.get("type") != "websocket":
  57. return await app(scope, receive, send)
  58. accepted = False
  59. async def _send(message):
  60. nonlocal accepted
  61. if message.get("type") == "websocket.accept" and not accepted:
  62. accepted = True
  63. tracker.on_open()
  64. await send(message)
  65. try:
  66. await app(scope, receive, _send)
  67. finally:
  68. if accepted:
  69. tracker.on_close()
  70. return _app
  71. _probe_failure_logged = False
  72. def _session_work_in_flight(session: dict) -> bool:
  73. if any(session.get(key) for key in ("running", "queued_prompt", "queued_prompts", "_auto_continue_scheduled")):
  74. return True
  75. return any(thread.is_alive() for key in ("_run_thread", "_agent_build_thread")
  76. if (thread := session.get(key)) is not None)
  77. def busy_ledger() -> Optional[str]:
  78. """Name the ledger that holds work — ``"retirement_admission"``, ``"session:<id>"``,
  79. ``"delegation"``, ``"cron:<job ids>"`` — ``""`` when every ledger is empty, ``None`` when one
  80. cannot be read. The Desktop's idle probe reports this so a backend that will not retire says
  81. WHICH work it is protecting, instead of an opaque "turn in flight".
  82. Session work lives on ``tui_gateway.server`` (the voice mixin's helper is bound into that
  83. namespace). Cron runs live outside that table (``cron.scheduler.get_running_job_ids``, the same
  84. ledger the gateway shutdown drain reads): without it a daily job mid-run reported "no turn" and
  85. the exit killed it (#107485). None keeps the backend alive forever, so the cause is logged once —
  86. a silent never-exits would be the original bug with a new face."""
  87. global _probe_failure_logged
  88. try:
  89. import tui_gateway.server as gateway
  90. from hermes_cli.backend_retirement import retirement
  91. if retirement.active_count():
  92. return "retirement_admission"
  93. with gateway._sessions_lock:
  94. busy_sessions = [sid for sid, s in gateway._sessions.items() if _session_work_in_flight(s)]
  95. if busy_sessions:
  96. return "session:" + ",".join(str(sid) for sid in busy_sessions)
  97. from tools.async_delegation import active_count
  98. from cron.scheduler import get_running_job_ids
  99. if active_count():
  100. return "delegation"
  101. running_jobs = get_running_job_ids()
  102. return "cron:" + ",".join(sorted(running_jobs)) if running_jobs else ""
  103. except Exception:
  104. if not _probe_failure_logged:
  105. _probe_failure_logged = True
  106. _log.warning("idle-exit turn probe unavailable; this backend will not self-retire", exc_info=True)
  107. return None
  108. def turn_in_flight() -> Optional[bool]:
  109. """Bool view of :func:`busy_ledger`, kept so ``should_exit_idle`` / the idle watchdog and injected
  110. test probes keep their ``Optional[bool]`` contract; None when the ledgers cannot be read."""
  111. ledger = busy_ledger()
  112. return None if ledger is None else bool(ledger)
  113. def should_exit_idle(tracker: IdleClientTracker, grace_s: float,
  114. probe: Callable[[], Optional[bool]] = turn_in_flight) -> bool:
  115. """Exit only when no client has been connected for ``grace_s`` AND no turn is provably running.
  116. A probe that cannot answer keeps the process (fail closed)."""
  117. return tracker.idle_for() >= grace_s and probe() is False # idle_for() is 0 while a client is connected
  118. def start_idle_watchdog(server, tracker: IdleClientTracker, *, grace_s: float = DEFAULT_IDLE_GRACE_S,
  119. poll_s: float = 15.0, probe: Callable[[], Optional[bool]] = turn_in_flight) -> threading.Thread:
  120. """Daemon thread that sets ``server.should_exit`` once :func:`should_exit_idle` holds."""
  121. poll_s = min(poll_s, max(0.5, grace_s / 4))
  122. def _loop() -> None:
  123. while not getattr(server, "should_exit", False):
  124. if should_exit_idle(tracker, grace_s, probe):
  125. _log.warning("SSH-isolated backend idle for %.0fs with no client and no running turn; exiting.",
  126. tracker.idle_for())
  127. server.should_exit = True
  128. return
  129. time.sleep(poll_s)
  130. thread = threading.Thread(target=_loop, daemon=True, name="ssh-isolated-idle-watchdog")
  131. thread.start()
  132. return thread