run_idle_exit.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282
  1. """Idle exit for an auto-started (unmanaged) gateway.
  2. A client that finds no gateway for its home starts a detached ``gateway run --idle-exit``
  3. (``hermes_cli.gateway_runtime_start``). Nothing else ever stops that process, so every throwaway home
  4. (scripts, tests, one-shot ``chat -q``) used to leave a ~270 MB daemon behind. This watcher ends it once
  5. it has served nobody and has nothing to do for ``gateway.unmanaged_idle_exit_seconds`` (default 600,
  6. ``0`` disables):
  7. - no attached client (live ticketed WebSocket, outstanding ticket, or recent HTTP/WS activity);
  8. - no messaging adapter configured, connected or reconnecting (any profile);
  9. - no runnable cron job and no kanban task the dispatcher must act on;
  10. - no queued/started admission, live managed worker, in-flight agent/cron/API run, background
  11. process or async delegation, active heartbeat/loop, pending handoff or hosted-room task.
  12. Every probe fails CLOSED: an unreadable source counts as work, so the gateway stays up. A gateway with
  13. a supervisor (systemd/launchd/Desktop/external) or started without ``--idle-exit`` (installed services,
  14. ``hermes gateway run`` by hand, ``hermes gateway ensure`` from Desktop) never arms this. The exit
  15. publishes ``draining`` with ``drain_reason: idle_exit`` before anything else, so a client that races it
  16. waits for the process to go and starts a fresh one instead of failing.
  17. """
  18. from __future__ import annotations
  19. import asyncio
  20. import contextlib
  21. import logging
  22. import sqlite3
  23. import time
  24. from pathlib import Path
  25. logger = logging.getLogger("gateway.run")
  26. DEFAULT_IDLE_EXIT_SECONDS = 600.0
  27. IDLE_EXIT_REASON = "idle_exit"
  28. # Head of the ``exit_reason`` persisted with ``stopped``: a reconnecting client's ``--recover``
  29. # (ui-tui/scripts/gateway_bootstrap.py) tells this self-ended exit from an operator stop by it.
  30. IDLE_EXIT_STOP_PREFIX = "auto-started gateway idle"
  31. # Kanban statuses the embedded dispatcher acts on without a human (promotion, spawn, reclaim).
  32. _KANBAN_DUE_STATUSES = ("scheduled", "ready", "running", "review")
  33. def idle_exit_seconds(runner) -> float:
  34. """``gateway.unmanaged_idle_exit_seconds`` from the launch home; 0 disables, junk keeps the default."""
  35. from gateway.run import _load_gateway_config
  36. section = (_load_gateway_config() or {}).get("gateway")
  37. value = section.get("unmanaged_idle_exit_seconds") if isinstance(section, dict) else None
  38. if value is None:
  39. return DEFAULT_IDLE_EXIT_SECONDS
  40. try:
  41. seconds = float(value)
  42. except (TypeError, ValueError):
  43. logger.warning("gateway.unmanaged_idle_exit_seconds=%r is not a number; using %.0f", value,
  44. DEFAULT_IDLE_EXIT_SECONDS)
  45. return DEFAULT_IDLE_EXIT_SECONDS
  46. return max(0.0, seconds)
  47. def note_client_activity(runner) -> None:
  48. runner._idle_exit_last_activity = time.monotonic()
  49. @contextlib.contextmanager
  50. def attached_client(runner):
  51. """Count one live client connection for the idle check (and stamp activity on both ends)."""
  52. note_client_activity(runner)
  53. runner._idle_exit_clients = getattr(runner, "_idle_exit_clients", 0) + 1
  54. try:
  55. yield
  56. finally:
  57. runner._idle_exit_clients -= 1
  58. note_client_activity(runner)
  59. def _messaging_busy(runner) -> str | None:
  60. from gateway.config import Platform
  61. live = list(getattr(runner, "adapters", None) or {})
  62. for adapters in (getattr(runner, "_profile_adapters", None) or {}).values():
  63. live.extend(adapters)
  64. for pending in (getattr(runner, "_profile_failed_platforms", None) or {}).values():
  65. live.extend(pending or {})
  66. live.extend(getattr(runner, "_failed_platforms", None) or {})
  67. config = getattr(runner, "config", None)
  68. if config is not None:
  69. live.extend(p for p, cfg in config.platforms.items() if getattr(cfg, "enabled", False))
  70. live = {getattr(p, 'value', str(p)) for p in live if p != Platform.LOCAL} # LOCAL never holds the gateway
  71. return f"messaging {sorted(live)}" if live else None
  72. def _served_homes(runner) -> list[Path]:
  73. from gateway.run import _cron_tick_profile_homes
  74. from gateway.session_authorities import all_authorities
  75. homes = {Path(a.profile_id) for a in all_authorities(runner)}
  76. homes.update(Path(h) for _name, h in _cron_tick_profile_homes(runner.config))
  77. return sorted(homes)
  78. def _cron_busy(homes) -> str | None:
  79. from cron.jobs import is_job_runnable, is_terminal_job, load_jobs, use_cron_store
  80. from cron.scheduler import get_running_job_ids
  81. if get_running_job_ids():
  82. return "cron job running"
  83. for home in homes:
  84. if not (home / "cron" / "jobs.json").exists():
  85. continue
  86. with use_cron_store(home):
  87. jobs = load_jobs()
  88. if any(is_job_runnable(job) and not is_terminal_job(job) for job in jobs):
  89. return f"cron jobs scheduled in {home}"
  90. return None
  91. def _kanban_busy() -> str | None:
  92. from hermes_cli import kanban_db
  93. marks = ",".join("?" * len(_KANBAN_DUE_STATUSES))
  94. for meta in kanban_db.list_boards(include_archived=False):
  95. path = kanban_db.kanban_db_path(board=meta.get("slug"))
  96. if not path.exists():
  97. continue
  98. with contextlib.closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=2)) as conn:
  99. if conn.execute(f"SELECT 1 FROM tasks WHERE status IN ({marks}) LIMIT 1",
  100. _KANBAN_DUE_STATUSES).fetchone():
  101. return f"kanban work on board {meta.get('slug')}"
  102. return None
  103. def _ledger_busy(runner) -> str | None:
  104. from gateway.session_authorities import all_authorities
  105. for authority in all_authorities(runner):
  106. db = authority.db
  107. if db._read_all("SELECT 1 FROM session_admissions WHERE status IN ('queued','started') LIMIT 1"):
  108. return f"admissions pending in {authority.profile_id}"
  109. if db._read_all("SELECT 1 FROM worker_executions WHERE status != 'terminal' LIMIT 1"):
  110. return f"managed worker live in {authority.profile_id}"
  111. service = getattr(authority, "hosted_room_service", None)
  112. if service is not None:
  113. status = service.status()
  114. if status.get("current_tasks") or status.get("leased_rooms"):
  115. return f"hosted-room work in {authority.profile_id}"
  116. return None
  117. def _profile_start_busy(runner) -> str | None:
  118. """A profile hot-serve is still building has no authority to walk yet, but its queued admissions
  119. are recovered once it starts; exiting now would cancel that start. A filled or failed start
  120. leaves the registry's starting set, so the hold ends with it."""
  121. registry = getattr(runner, "session_authorities", None)
  122. starting = registry.starting_homes() if registry is not None else ()
  123. return f"profile starting in {starting[0]}" if starting else None
  124. def _store_gates_busy(homes) -> str | None:
  125. from gateway.run_idle_gates import (
  126. profile_has_active_heartbeat, profile_has_active_loop, profile_has_pending_handoff)
  127. for home in homes:
  128. for label, gate in (("heartbeat", profile_has_active_heartbeat), ("loop", profile_has_active_loop),
  129. ("handoff", profile_has_pending_handoff)):
  130. if gate(home):
  131. return f"active {label} in {home}"
  132. return None
  133. def _attaching(runner) -> bool:
  134. tickets = getattr(runner, "session_ticket_store", None)
  135. return tickets is not None and tickets.outstanding()
  136. def _client_busy(runner, window: float) -> str | None:
  137. clients = getattr(runner, "_idle_exit_clients", 0)
  138. if clients:
  139. return f"{clients} client(s) attached"
  140. if _attaching(runner):
  141. return "client attaching"
  142. quiet = time.monotonic() - getattr(runner, "_idle_exit_last_activity", 0.0)
  143. if quiet < window:
  144. return f"client activity {quiet:.0f}s ago"
  145. return None
  146. def busy_reason(runner, *, window: float, count_clients: bool = True) -> str | None:
  147. """Why the gateway must stay up, or None when it may exit. Fails closed on any error.
  148. ``count_clients=False``: a restart is pending, and an attached or returning client is exactly who
  149. needs the fresh gateway; every duty nothing would restart (adapters, cron, admissions) still counts.
  150. A client mid-attach (unredeemed ticket) still holds it: its first connect would be refused with
  151. no rediscovery, while an attached client waits out the drain and starts the fresh gateway.
  152. """
  153. try:
  154. if count_clients and (reason := _client_busy(runner, window)):
  155. return reason
  156. if not count_clients and _attaching(runner):
  157. return "client attaching"
  158. if runner._draining or not runner._running:
  159. return "already stopping"
  160. if runner._active_work_count():
  161. return "agent work in flight"
  162. if runner._scale_to_zero_has_live_background_work():
  163. return "background work in flight"
  164. homes = _served_homes(runner)
  165. return (_messaging_busy(runner) or _ledger_busy(runner) or _profile_start_busy(runner)
  166. or _cron_busy(homes) or _kanban_busy() or _store_gates_busy(homes))
  167. except Exception as exc:
  168. logger.debug("idle-exit probe failed; staying up", exc_info=True)
  169. return f"idle probe failed ({type(exc).__name__})"
  170. def _begin_idle_exit(runner, window: float, restart: str | None = None) -> None:
  171. """Withdraw admission synchronously (no await between the last idle check and here), then stop."""
  172. runner._draining = True
  173. runner.session_runtime_descriptor.update(state="draining", capabilities=[], drain_reason=IDLE_EXIT_REASON)
  174. runner._exit_reason = (f"{IDLE_EXIT_STOP_PREFIX}; restarting to load {restart}" if restart else
  175. f"{IDLE_EXIT_STOP_PREFIX} for {window:.0f}s (gateway.unmanaged_idle_exit_seconds)")
  176. logger.info("Exiting: %s; the next client starts a fresh gateway", runner._exit_reason)
  177. runner._idle_exit_stop_task = asyncio.ensure_future(runner.stop())
  178. async def _sleep_unless_restart_pending(interval: float) -> None:
  179. """Sleep ``interval``, waking within a second of a pending restart: a user retrying right after
  180. the failed turn must reach a fresh gateway, not this one. Always sleeps at least one slice, so a
  181. pending restart held up by other work is re-probed once a second, never in a spin."""
  182. from pm.environments_adopt import restart_pending
  183. deadline = time.monotonic() + interval
  184. while True:
  185. await asyncio.sleep(min(1.0, max(0.0, deadline - time.monotonic())))
  186. if restart_pending() or time.monotonic() >= deadline:
  187. return
  188. async def _retired_for_restart(runner, window: float) -> bool:
  189. """Retire as soon as nothing but clients holds the gateway when a dependency install this process
  190. ran can only load in a new one (``pm.environments_adopt.restart_pending``). Its clients wait out the
  191. ``idle_exit`` drain and start a fresh gateway, which boots the new generation."""
  192. from pm.environments_adopt import restart_pending
  193. if not (restart := restart_pending()):
  194. return False
  195. if await asyncio.to_thread(busy_reason, runner, window=window, count_clients=False) is not None:
  196. return False
  197. if busy_reason(runner, window=window, count_clients=False) is not None:
  198. return False
  199. _begin_idle_exit(runner, window, restart=restart)
  200. return True
  201. async def unmanaged_idle_exit_watcher(runner) -> None:
  202. window = idle_exit_seconds(runner)
  203. if window <= 0:
  204. logger.info("Auto-started gateway idle exit disabled (gateway.unmanaged_idle_exit_seconds=0)")
  205. return
  206. logger.info("Auto-started gateway: exits after %.0fs with no clients, adapters, cron/kanban work or "
  207. "admissions (gateway.unmanaged_idle_exit_seconds)", window)
  208. interval = min(30.0, max(0.5, window / 4))
  209. idle_since = None
  210. while runner._running and not runner._draining:
  211. await _sleep_unless_restart_pending(interval)
  212. window = idle_exit_seconds(runner)
  213. if window <= 0:
  214. idle_since = None
  215. continue
  216. if await _retired_for_restart(runner, window):
  217. return
  218. # The store/cron/kanban reads touch disk; keep them off the loop, then re-check the
  219. # loop-only facts (clients, tickets, draining) synchronously right before acting.
  220. reason = await asyncio.to_thread(busy_reason, runner, window=window)
  221. if reason is not None:
  222. idle_since = None
  223. continue
  224. idle_since = idle_since or time.monotonic()
  225. if time.monotonic() - idle_since < interval and window > interval:
  226. continue # two consecutive idle readings, so one racing write cannot slip between them
  227. if busy_reason(runner, window=window) is None:
  228. _begin_idle_exit(runner, window)
  229. return
  230. def arm_unmanaged_idle_exit(runner) -> bool:
  231. """Start the watcher for an ``--idle-exit`` launch that no supervisor owns. Returns whether it armed."""
  232. from gateway.control_socket import _detect_supervisor
  233. supervisor = _detect_supervisor()
  234. if supervisor != "manual":
  235. logger.info("Auto-started gateway idle exit not armed: supervised by %s", supervisor)
  236. return False
  237. note_client_activity(runner)
  238. runner._spawn_supervised(lambda: unmanaged_idle_exit_watcher(runner), "unmanaged_idle_exit_watcher")
  239. return True