cli_shutdown.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. """Classic-CLI shutdown and one-shot finalize helpers: process session-id sync, deferred agent startup, exit watchdog, cleanup steps, session-finalize notifications and the terminal input-mode reset.
  2. Split out of ``cli.py``; ``cli`` re-exports every public name and moved bodies late-bind
  3. cli-level names through ``from cli import ...`` at call time so facade monkeypatch seams hold.
  4. """
  5. from __future__ import annotations
  6. import logging
  7. import os
  8. import sys
  9. import threading
  10. import time
  11. from contextlib import suppress
  12. # Log-record parity with the origin module.
  13. logger = logging.getLogger("cli")
  14. def _cli():
  15. """Late import of the ``cli`` facade: mutable CLI module state (and its test seams) lives there."""
  16. import cli
  17. return cli
  18. def _sync_process_session_id(session_id: str) -> None:
  19. """Keep process-local session-id consumers aligned after CLI switches."""
  20. from gateway.session_context import set_current_session_id
  21. set_current_session_id(session_id)
  22. def _flush_logging_and_stdio() -> None:
  23. """Best-effort ``logging.shutdown()`` + stdout/stderr flush before ``os._exit``."""
  24. with suppress(Exception):
  25. logging.shutdown()
  26. for _stream in (sys.stdout, sys.stderr):
  27. with suppress(Exception):
  28. _stream.flush()
  29. def _float_env(name: str, default: float) -> float:
  30. """``float(os.getenv(name))``, or ``default`` when unset/unparseable."""
  31. try:
  32. return float(os.getenv(name, default))
  33. except (TypeError, ValueError):
  34. return default
  35. def _exit_watchdog_timeout() -> float:
  36. """``HERMES_EXIT_WATCHDOG_S`` as a float (default 30; ``0`` disables)."""
  37. from cli import _float_env
  38. return _float_env("HERMES_EXIT_WATCHDOG_S", 30.0)
  39. def _arm_exit_watchdog(timeout_s: float | None = None, *, from_signal: bool = False) -> None:
  40. """Daemon timer that ``os._exit(0)``s after ``timeout_s`` once shutdown has begun.
  41. Backstop for a cleanup step wedged on network I/O and for interpreter teardown
  42. blocked joining non-daemon threads (ThreadPoolExecutor's atexit join). The daemon
  43. timer survives ``Py_FinalizeEx``'s joins. ``HERMES_EXIT_WATCHDOG_S=0`` disables.
  44. 1. 2. Interpreter teardown blocked joining non-daemon threads — stdlib ``ThreadPoolExecutor`` workers
  45. are joined unconditionally by ``concurrent.futures``' atexit hook even after ``shutdown(wait=False)``,
  46. so one tool thread wedged on a socket held the process open forever (#27563 class).
  47. """
  48. from cli import _exit_watchdog_timeout, _flush_logging_and_stdio
  49. if timeout_s is None:
  50. timeout_s = _exit_watchdog_timeout()
  51. if timeout_s <= 0:
  52. return
  53. # Never under pytest: a delayed os._exit(0) would silently kill the test worker.
  54. if os.environ.get("PYTEST_CURRENT_TEST"):
  55. return
  56. def _watchdog():
  57. time.sleep(timeout_s)
  58. # The signal-armed watchdog yields to cleanup's own timer once cleanup is running.
  59. if from_signal and _cli()._cleanup_in_progress:
  60. return
  61. try:
  62. logger.warning(
  63. "Exit watchdog fired after %.0fs — forcing process exit "
  64. "(a cleanup step or non-daemon thread is wedged).",
  65. timeout_s,
  66. )
  67. except Exception:
  68. pass
  69. _flush_logging_and_stdio()
  70. # os._exit skips cleanup: a foreground command in its own process group would outlive us.
  71. with suppress(Exception):
  72. from tools.environments.base import kill_live_foreground_processes
  73. kill_live_foreground_processes(now=True)
  74. os._exit(0)
  75. with suppress(Exception): # never block shutdown on watchdog setup
  76. threading.Thread(target=_watchdog, daemon=True, name="exit-watchdog").start()
  77. def _shutdown_agent_memory_provider(agent) -> None:
  78. """Memory-provider shutdown (on_session_end + shutdown_all) at the real session boundary."""
  79. if not (agent and hasattr(agent, 'shutdown_memory_provider')):
  80. return
  81. # A /new shortly before exit leaves an LLM-bound boundary task queued; shutdown_all()'s
  82. # ~5s drain would cancel it, so give it a bounded head start (watchdog is the backstop).
  83. _mm = getattr(agent, '_memory_manager', None)
  84. if _mm is not None and hasattr(_mm, 'flush_pending'):
  85. with suppress(Exception):
  86. _mm.flush_pending(timeout=10)
  87. # Forward the agent's transcript so on_session_end hooks see the real conversation;
  88. # no-arg fallback for stubs / partially-initialised agents.
  89. _session_msgs = getattr(agent, '_session_messages', None)
  90. _sid = getattr(agent, "session_id", None) or "<unknown>"
  91. # ``_session_messages`` is set on ``AIAgent.__init__`` and refreshed every turn via
  92. # ``_persist_session``. Fall back to no-arg on test stubs / partially-initialised agents where the
  93. # attribute is missing. See #15165.
  94. if isinstance(_session_msgs, list):
  95. logger.info("CLI cleanup calling memory shutdown for session %s with %d message(s)", _sid, len(_session_msgs))
  96. agent.shutdown_memory_provider(_session_msgs)
  97. else:
  98. logger.info("CLI cleanup calling memory shutdown for session %s without session message list", _sid)
  99. agent.shutdown_memory_provider()
  100. def _stop_cli_wake_word() -> None:
  101. from tools.wake_word import stop_listening
  102. if _cli()._cli_wake_owner is not None:
  103. stop_listening(owner=_cli()._cli_wake_owner)
  104. def _interrupt_async_delegations() -> None:
  105. from tools.async_delegation import interrupt_all
  106. interrupt_all(reason="CLI shutdown")
  107. def _shutdown_mcp_servers() -> None:
  108. from tools.mcp_tool_lifecycle import shutdown_mcp_servers
  109. shutdown_mcp_servers()
  110. def _shutdown_cached_aux_clients() -> None:
  111. # Otherwise AsyncHttpxClientWrapper.__del__ fires on a closed loop ("Press ENTER to continue...").
  112. from agent.auxiliary_client import shutdown_cached_clients
  113. shutdown_cached_clients()
  114. # Ordered teardown steps (attribute names, resolved at call time so tests can patch them)
  115. # and the exception class each swallows.
  116. # Async delegations and MCP servers live in the gateway, which owns their lifetime: a CLI exit
  117. # neither interrupts the delegations nor tears down MCP servers.
  118. _CLEANUP_STEPS = (
  119. ("_stop_cli_wake_word", Exception), ("_cleanup_all_terminals", Exception),
  120. ("_cleanup_all_browsers", Exception), ("_shutdown_cached_aux_clients", Exception),
  121. )
  122. def _gateway_owns_session_end(agent) -> bool:
  123. """A gateway-backed session ends in the gateway: the viewer's ``end_session`` (``session.detach
  124. {end_reason}``) makes the owner run finalize and the memory commit (gateway/session_viewer_close)."""
  125. from hermes_cli.gateway_agent import GatewayAgent
  126. return isinstance(agent, GatewayAgent)
  127. def _should_emit_cleanup_session_finalize(session_id: str | None) -> bool:
  128. if _gateway_owns_session_end(_cli()._active_agent_ref):
  129. return False
  130. # A handed-off session is owned by the gateway process — never finalize it here.
  131. # The CLI must not finalize it on exit — that sets end_reason on a row the gateway reopened and is
  132. # actively writing to, causing the handoff leg to vanish from session history (#88234).
  133. if session_id is not None and session_id in _cli()._handed_off_session_ids:
  134. return False
  135. if not _cli()._single_query_finalize_attempted_session_ids:
  136. return True
  137. if session_id is None:
  138. return False
  139. return session_id not in _cli()._single_query_finalize_attempted_session_ids
  140. def _notify_session_finalize(*, session_id: str | None, platform: str = "cli", reason: str = "shutdown") -> list[str]:
  141. """Fire on_session_finalize; return the plugin messages meant for the user (never raises)."""
  142. with suppress(Exception):
  143. from hermes_cli.lifecycle import finalize_session, session_end_messages
  144. return session_end_messages(finalize_session(session_id=session_id, platform=platform, reason=reason))
  145. return []
  146. def _oneshot_agent_and_session(cli):
  147. """``(agent, session_id)`` for a one-shot run; the agent's id wins over the CLI's."""
  148. agent = getattr(cli, "agent", None)
  149. return agent, getattr(agent, "session_id", None) or getattr(cli, "session_id", None)
  150. def _invoke_interrupted_session_end(agent, session_id, reason: str, **extra) -> None:
  151. """Best-effort ``on_session_end`` hook for a turn cut short (never raises)."""
  152. with suppress(Exception):
  153. from hermes_cli.lifecycle import invoke_hook as _invoke_hook
  154. _invoke_hook(
  155. "on_session_end", session_id=session_id, completed=False, interrupted=True,
  156. model=getattr(agent, "model", None), platform=getattr(agent, "platform", None) or "cli",
  157. reason=reason, **extra,
  158. )
  159. def _emit_interrupted_session_end(cli, *, reason: str = "keyboard_interrupt") -> None:
  160. """Best-effort on_session_end hook for interrupted non-interactive runs."""
  161. from cli import _invoke_interrupted_session_end, _oneshot_agent_and_session
  162. agent, session_id = _oneshot_agent_and_session(cli)
  163. if agent is None:
  164. return
  165. with suppress(Exception):
  166. agent.interrupt(reason.replace("_", " "))
  167. # The gateway owns the lifecycle now; a gateway turn fires on_session_end itself when interrupted.
  168. if session_id in _cli()._handed_off_session_ids or _gateway_owns_session_end(agent):
  169. return
  170. if session_id:
  171. with suppress(Exception):
  172. cli.session_id = session_id
  173. _invoke_interrupted_session_end(
  174. agent, session_id, reason,
  175. task_id=getattr(agent, "_current_task_id", "") or "",
  176. turn_id=getattr(agent, "_current_turn_id", "") or "",
  177. api_request_id=getattr(agent, "_current_api_request_id", "") or "",
  178. )
  179. def _notify_single_query_session_finalize(cli, *, reason: str = "shutdown") -> None:
  180. from cli import _notify_session_finalize, _oneshot_agent_and_session
  181. agent, session_id = _oneshot_agent_and_session(cli)
  182. if session_id in _cli()._single_query_finalize_attempted_session_ids:
  183. return
  184. # The gateway owns the lifecycle now; for a gateway session the flush's end_session detached with
  185. # an end_reason, so the gateway finalizes it.
  186. if session_id in _cli()._handed_off_session_ids or _gateway_owns_session_end(agent):
  187. return
  188. try:
  189. # stderr: one-shot stdout is the answer scripts parse.
  190. for message in _notify_session_finalize(
  191. session_id=session_id, platform=getattr(agent, "platform", None) or "cli", reason=reason):
  192. print(message, file=sys.stderr, flush=True)
  193. finally:
  194. _cli()._single_query_finalize_attempted_session_ids.add(session_id)
  195. def _flush_one_shot_session_store(cli) -> None:
  196. """Durably flush + finalize the one-shot session row before exit (idempotent, best-effort).
  197. One-shot runs get a single turn, so nothing retries a transiently-failed transcript
  198. flush, closes the session row, or drains token deltas the kanban ``os._exit(0)``
  199. path skips. Handed-off sessions are left alone.
  200. - a turn whose in-loop ``_flush_messages_to_session_db`` failed under write-lock contention (e.g. a busy
  201. multiplex gateway sharing state.db) was silently lost — the reply reached stdout and agent.log but the
  202. resumed session's stored history never changed (#88583); - the resumed/created titled session row was
  203. left dangling open (``ended_at``/``end_reason`` NULL) on every one-shot exit; - queued async
  204. token-accounting deltas relied on interpreter-exit hooks, which the kanban SIGTERM path's
  205. ``os._exit(0)`` skips entirely.
  206. Idempotent and best-effort: ``_persist_session`` dedupes via the per-message ``_DB_PERSISTED_MARKER``
  207. stamps (already-written turns are not re-written) and ``end_session`` no-ops on an already-ended row.
  208. See #88234.
  209. """
  210. from cli import _oneshot_agent_and_session
  211. agent, session_id = _oneshot_agent_and_session(cli)
  212. if agent is None or not session_id or session_id in _cli()._handed_off_session_ids:
  213. return
  214. if getattr(agent, "_persist_disabled", False):
  215. return
  216. # Passing cli.conversation_history keeps resumed messages identity-skipped even when
  217. # the failed flush never stamped them.
  218. try:
  219. msgs = getattr(agent, "_session_messages", None)
  220. if isinstance(msgs, list) and msgs and hasattr(agent, "_persist_session"):
  221. agent._persist_session(msgs, getattr(cli, "conversation_history", None))
  222. except Exception:
  223. logger.debug("one-shot final session persist retry failed", exc_info=True)
  224. db = getattr(agent, "_session_db", None) or getattr(cli, "_session_db", None)
  225. if db is None:
  226. return
  227. try:
  228. db.flush_token_counts()
  229. except Exception:
  230. logger.debug("one-shot token-count drain failed", exc_info=True)
  231. try:
  232. db.end_session(session_id, "cli_close")
  233. except Exception:
  234. logger.debug("one-shot end_session failed", exc_info=True)
  235. def _wait_for_oneshot_background_completions(cli) -> None:
  236. """Bounded linger for notify_on_complete background processes (children write to our pipes).
  237. Waits on the whole registry: a one-shot process hosts one agent, and task_id
  238. filtering would skip processes registered before the session id settled.
  239. Skipped when the quiet -Q notify-resume loop already consumed the run's linger
  240. budget: it calls wait_for_pending_completions with a shared deadline, so a
  241. re-wait here would double-block on the same stuck notify_on_complete child.
  242. See #90879.
  243. """
  244. from cli import _oneshot_agent_and_session
  245. from tools.process_registry import process_registry
  246. if getattr(cli, "_quiet_notify_linger_done", False):
  247. return
  248. _agent, task_id = _oneshot_agent_and_session(cli)
  249. result = process_registry.wait_for_pending_completions(None)
  250. if result.get("waited"):
  251. logger.info(
  252. "One-shot exit linger for session %s: completed=%s timed_out=%s",
  253. task_id or "<unknown>",
  254. result.get("completed"),
  255. result.get("timed_out"),
  256. )
  257. def _finalize_single_query(cli) -> None:
  258. """Settle the session, then release its lease, then linger as a bystander.
  259. Three phases, in this order:
  260. 1. **Session-owned settlement** — the durable flush (transcript retry, token
  261. drain, ``end_session``), the finalize hook, and memory-provider session
  262. finalization. Providers finalize THIS session's remote state at
  263. ``on_session_end`` (OpenViking synchronously commits ``sessions/{sid}``,
  264. Supermemory flushes pending turns stamped with the session id), so this
  265. belongs inside ownership — post-release it would run while a successor
  266. may already be mid-turn on the session. The flush runs first: memory
  267. shutdown can issue aux-LLM calls and nothing after it may fail in a way
  268. that loses the turn (#88583). Settlement is best-effort by design:
  269. failures are logged and the release still happens, because the failure
  270. mode to avoid is an exited process pinning the lease — a dangling open
  271. row is recoverable (session_recovery reaps stale rows), a wedged lease
  272. is not.
  273. 2. **Ownership handoff** — release the active-session lease. Every turn of
  274. the run (main turn, kanban goal loop, notify-completion follow-ups) has
  275. finished and the success path has no session-owned writes left, so a
  276. waiting delivery may acquire and resume the session immediately:
  277. releasing only at process exit let an alive-but-idle one-shot refuse
  278. deliveries for the whole exit linger (bounded, minutes) after its turn
  279. had ended (#118826 / #122770).
  280. 3. **Process-only work, lease-free** — the bounded linger for
  281. notify_on_complete children (pipe drain, the boundary the one-shot path
  282. itself declares "NOT part of the spawner's delivery", #113608) and
  283. resource teardown. ``_run_cleanup``'s memory-shutdown call is idempotent
  284. (``_memory_provider_shutdown``), so its post-release repeat is a no-op;
  285. neither it nor the linger touches session rows.
  286. Releasing between settlement and the linger is what prevents the handoff race:
  287. a successor that acquires and reopens the session can never receive this
  288. process's stale ``cli_close`` end-stamp, because phase 1 has already run it.
  289. """
  290. import cli as cli_module
  291. from cli import _flush_one_shot_session_store, _notify_single_query_session_finalize, _run_cleanup, _shutdown_agent_memory_provider, _wait_for_oneshot_background_completions
  292. try:
  293. try:
  294. _flush_one_shot_session_store(cli)
  295. except Exception:
  296. logger.debug("one-shot session store flush failed", exc_info=True)
  297. _notify_single_query_session_finalize(cli)
  298. try:
  299. # A failed goal-loop re-init leaves cli.agent None while the agent
  300. # that ran the turns is still the module's active ref.
  301. _shutdown_agent_memory_provider(getattr(cli, "agent", None) or cli_module._active_agent_ref)
  302. except Exception:
  303. logger.debug("one-shot memory provider shutdown failed", exc_info=True)
  304. finally:
  305. # Even a failed settlement must not pin the lease: the failure mode to avoid
  306. # at all costs is a dead-ish process holding ownership, not a dangling row.
  307. cli._release_active_session()
  308. try:
  309. _wait_for_oneshot_background_completions(cli)
  310. except Exception:
  311. logger.debug("one-shot background completion wait failed", exc_info=True)
  312. _run_cleanup(notify_session_finalize=False)