wake.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. """Wake an existing agent session from a background completion event. Push-capable adapters
  2. (``supports_async_delivery``) get a synthetic ``MessageEvent(internal=True)`` via handle_message;
  3. stateless adapters (API server) would run that under a ``build_session_key()`` key that never
  4. matches the raw ``X-Hermes-Session-Id`` real turns use (invisible parallel session), so we self-POST
  5. ``/v1/chat/completions`` with the raw id header to resume the REAL session. Exception:
  6. async-delegation completions: the CLIENT owns the next turn, so they are never self-POSTed as a
  7. new ``role=user`` prompt (could cross a pending human-confirmation gate); instead
  8. ``persist_delegation_delivery`` writes a durable DELIVERY row (``display_kind=
  9. "async_delegation_complete"``, read by TUI/desktop pollers). Failures RAISE (after bounded retries
  10. on transient errors) so callers can rewind cursors / retry instead of silently losing the event."""
  11. from __future__ import annotations
  12. import asyncio
  13. import logging
  14. from typing import Any, Optional
  15. logger = logging.getLogger(__name__)
  16. # A wake self-post runs the whole agent turn synchronously (stream=false); generous ceiling so long
  17. # tool-using turns aren't killed mid-flight.
  18. WAKE_TURN_TIMEOUT_SECONDS = 600.0
  19. # Backoff between retries on transient failures. The API server has no per-session lock (concurrent
  20. # turns are last-writer-wins) but DOES enforce a global max_concurrent_runs cap via HTTP 429, which
  21. # is worth waiting out.
  22. _RETRY_DELAYS_SECONDS = (2.0, 5.0, 10.0)
  23. def adapter_supports_push(adapter: Any) -> bool:
  24. """Whether this adapter can push a message to the user after a turn ends. Reads
  25. ``supports_async_delivery`` off the adapter class rather than the request-scoped contextvar
  26. (background watchers run outside any bound session context). Adapters that don't declare
  27. the flag are push-capable."""
  28. return bool(getattr(adapter, "supports_async_delivery", True))
  29. # Store refusal codes (``RuntimeStoreError.reason``) that refuse a notice for good by themselves: the
  30. # route has no session (``not_found``) or the payload is malformed (``invalid_params``). A bare
  31. # ``admission_conflict`` proves nothing: an unowned completion, or one pinned to a live session the
  32. # route does not run, may still find its owner (d6635ffe03b), so it is permanent only when the
  33. # authority's state proves the pin gone (``refusal_proven_permanent``). Every other refusal
  34. # (``runtime_draining`` during a restart's after-turn wait or a parked profile, ``queue_full``,
  35. # ``session_busy``, ``stale_epoch``, ...) clears on its own: back off, never give the notice up.
  36. PERMANENT_REFUSAL_REASONS = frozenset({'not_found', 'invalid_params'})
  37. class WakeNotAccepted(RuntimeError):
  38. """No adapter admission: retry without treating a healthy chat as dead. ``reason`` is the
  39. store's refusal code when the session authority refused it, ``runtime_draining`` when the
  40. adapter path refused it during a drain (None: the adapter did not accept);
  41. ``permanent`` is set once the refusal is proven final (``refusal_proven_permanent``)."""
  42. def __init__(self, message: str = "internal wake not accepted by adapter", *, reason: Optional[str] = None):
  43. super().__init__(message)
  44. self.reason = reason
  45. self.permanent = False
  46. def session_owned_by_profile(config: Any, profile: Optional[str], session_id: Any) -> bool:
  47. """True when a stateless (``api_server``) destination's raw session id is canonically owned by
  48. served *profile*'s own session store.
  49. A shared-listener mirror platform has no chat/thread/guild anchor a ``profile_routes`` entry
  50. could match, so the session store itself is the ownership proof: the row must exist in that
  51. profile's ``state.db`` under its own home and carry that profile's stamp (a NULL legacy stamp
  52. belongs to the store's own profile — the same rule the dashboard's session routes apply). An
  53. unserved profile, a missing row, a row stamped for another profile, or an unreadable store all
  54. fail closed. Shared by the Kanban notifier and the background-process wake path.
  55. """
  56. import contextlib
  57. from pathlib import Path
  58. if not session_id or not profile:
  59. return False
  60. profile = str(profile)
  61. try:
  62. from gateway.run import _multiplex_profile_homes
  63. home = dict(_multiplex_profile_homes(config)).get(profile)
  64. if home is None:
  65. return False
  66. from hermes_state import SessionDB
  67. db = SessionDB(Path(home) / "state.db", read_only=True)
  68. except Exception as exc:
  69. logger.debug("wake: session ownership check unavailable for %s/%s: %s", profile, session_id, exc)
  70. return False
  71. try:
  72. row = db.get_session(str(session_id))
  73. except Exception as exc:
  74. logger.debug("wake: session ownership lookup failed for %s/%s: %s", profile, session_id, exc)
  75. return False
  76. finally:
  77. with contextlib.suppress(Exception):
  78. db.close()
  79. return bool(row) and (row.get("profile_name") or profile) == profile
  80. async def admit_internal_event(adapter: Any, event: Any) -> None:
  81. """Require a concrete adapter admission, not merely a handler returning None.
  82. The public handler return stays unchanged. This receipt means scheduled/queued,
  83. not model execution, authorization of a later turn, or successful outbound delivery.
  84. """
  85. event._gateway_accepted = False
  86. # A draining gateway answers new work with a drain notice and discards queued follow-ups,
  87. # so a wake admitted now would never run. Refuse it so the caller keeps it for a retry.
  88. if _runner_draining(adapter):
  89. raise WakeNotAccepted("gateway is draining; retry after restart", reason="runtime_draining")
  90. await adapter.handle_message(event)
  91. if event._gateway_accepted is not True:
  92. raise WakeNotAccepted()
  93. # handle_message can await before it claims or queues the event (Telegram topic recovery, the
  94. # busy handler's profile scope). A drain seen here dooms the stored event: the drain gate refuses
  95. # a claimed turn and the pending discard drops a queued follow-up. Refuse it so the producer
  96. # retries it after the restart.
  97. if _runner_draining(adapter):
  98. raise WakeNotAccepted("gateway began draining during admission; retry after restart",
  99. reason="runtime_draining")
  100. # No drain began before or during admission, so the producer takes this receipt as final and
  101. # will not retry: a drain that begins later must run the wake rather than drop it (#138123).
  102. event._admitted_before_drain = True
  103. def _runner_draining(adapter: Any) -> bool:
  104. return getattr(getattr(adapter, "gateway_runner", None), "_draining", False) is True
  105. async def deliver_wake(adapter: Any, *, text: str, session_id: str = "", source: Any = None,
  106. notification_category: str = "result", profile: Optional[str] = None) -> None:
  107. """Deliver a wake turn to the session behind ``adapter``. ``session_id`` is the RAW session id
  108. (``X-Hermes-Session-Id`` / state.db key) — required for non-push adapters. ``source`` is the
  109. ``SessionSource`` for the synthetic event — required for push-capable adapters. ``profile``
  110. names the served profile that canonically owns a non-push destination; a non-default value is
  111. delivered in-process under the caller's profile scope (see ``_self_post_chat_completion``).
  112. Raises on failure so the caller can rewind/retry."""
  113. if adapter_supports_push(adapter):
  114. if source is None:
  115. raise ValueError("deliver_wake: push-capable adapter requires a SessionSource")
  116. from gateway.platforms.event import MessageEvent, MessageType
  117. runner = getattr(adapter, "gateway_runner", None)
  118. if runner is not None:
  119. from gateway.session_automation import admit_route_wake
  120. from hermes_state_runtime import RuntimeStoreError
  121. category = {"notification_category": notification_category} if notification_category != "result" else {}
  122. try:
  123. if await admit_route_wake(runner, adapter, source, text, "kanban-wake", category):
  124. return # committed to the session FIFO: never interrupts the admitted turn
  125. except RuntimeStoreError as exc:
  126. raise WakeNotAccepted(str(exc), reason=exc.reason) from exc
  127. synth_event = MessageEvent(text=text, message_type=MessageType.TEXT, source=source, internal=True,
  128. metadata={"notification_category": notification_category})
  129. await admit_internal_event(adapter, synth_event)
  130. return
  131. if not session_id:
  132. raise ValueError("deliver_wake: non-push adapter (supports_async_delivery=False) "
  133. "requires the raw session id to self-post the wake turn")
  134. extra: dict = {}
  135. if profile:
  136. extra["profile"] = profile
  137. if notification_category == "diagnostic":
  138. extra["notification_category"] = notification_category
  139. await _self_post_chat_completion(adapter, text=text, session_id=session_id, **extra)
  140. def _delegation_display_metadata(evt: dict) -> dict:
  141. """Display-only metadata for a persisted delegation delivery row. Mirrors
  142. ``tui_gateway.server._async_delegation_display_metadata`` (same ``display_kind`` consumer
  143. contract) without importing the TUI stack."""
  144. raw_results = evt.get("results")
  145. results = [r for r in raw_results if isinstance(r, dict)] if isinstance(raw_results, list) else []
  146. task_count = len(results) or 1
  147. completed_count = sum(1 for r in results if r.get("status") in {"completed", "success"})
  148. failed_count = sum(1 for r in results if r.get("status") in {"failed", "error"})
  149. metadata = {"delegation_id": str(evt.get("delegation_id") or ""), "task_count": task_count,
  150. "completed_count": completed_count or task_count - failed_count,
  151. "failed_count": failed_count}
  152. if evt.get("task_failure_notice"):
  153. metadata["delivery_notice"] = f"task_failure:{results[0].get('task_index', '') if results else ''}"
  154. metadata["notification_category"] = "diagnostic"
  155. from gateway.warning_notifications import warning_notifications_enabled
  156. if not warning_notifications_enabled("api_server"):
  157. metadata["presentation_suppressed"] = True
  158. duration = evt.get("total_duration_seconds") or evt.get("duration_seconds")
  159. if isinstance(duration, (int, float)):
  160. metadata["duration_seconds"] = duration
  161. return metadata
  162. async def persist_delegation_delivery(adapter: Any, *, text: str, session_id: str, evt: Optional[dict] = None) -> None:
  163. """Persist an async-delegation completion as a durable DELIVERY row (see module docstring)
  164. WITHOUT running any agent turn. Raises on failure so the caller can release the durable claim
  165. and retry.
  166. 85957: on stateless api_server sessions the client owns the turn after ``event.complete`` — a completion
  167. must never become a new ``role=user`` prompt via the self-post (that starts an unauthorized agent turn
  168. and can cross a pending human-confirmation gate). Instead, append the completion to the session
  169. transcript as a timeline bookkeeping row (``display_kind="async_delegation_complete"`` + display
  170. metadata — the exact shape the TUI/desktop delivery path persists), WITHOUT running any agent turn.
  171. Clients polling ``GET /api/sessions/{id}/messages`` see it immediately; the pre-request repair belt
  172. folds it into the next real client turn as context. See #85957.
  173. """
  174. if not session_id:
  175. raise ValueError("persist_delegation_delivery: raw session id required to persist "
  176. "the completion on the api_server session transcript")
  177. ensure = getattr(adapter, "_ensure_session_db", None)
  178. db: Any = await asyncio.to_thread(ensure) if callable(ensure) else None
  179. if db is None:
  180. raise RuntimeError("persist_delegation_delivery: api_server SessionDB unavailable — "
  181. f"cannot persist completion for session {session_id}")
  182. # #98619: the parent run may have compressed/rotated between dispatch and this detached
  183. # completion — the captured origin id is then a closed parent and the append below is
  184. # rejected with CompressionSessionClosedError forever (the watcher retries the same stale
  185. # id). Adopt the live continuation tip first, the same canonical resolution
  186. # /api/sessions/{id}/messages reads use, so the delivery row lands where the next run and
  187. # the messages endpoint both resolve. Fails open to the original id.
  188. resolver = getattr(db, "resolve_resume_session_id", None)
  189. if callable(resolver):
  190. try:
  191. resolved = await asyncio.to_thread(resolver, session_id)
  192. if resolved:
  193. session_id = str(resolved)
  194. except Exception:
  195. logger.debug("delegation delivery continuation resolve failed for %s", session_id, exc_info=True)
  196. await asyncio.to_thread(
  197. db.append_delegation_delivery, session_id, text, _delegation_display_metadata(evt or {}),
  198. )
  199. logger.info(
  200. "async delegation completion persisted as delivery row for api_server session %s (no wake turn)", session_id
  201. )
  202. async def _self_post_chat_completion(adapter: Any, *, text: str, session_id: str,
  203. notification_category: str = "result",
  204. profile: Optional[str] = None) -> None:
  205. """POST the wake text to the in-pod API server as a normal session turn, using the adapter's
  206. own bind host/port/key. Session continuation via ``X-Hermes-Session-Id`` is 403-gated on
  207. ``API_SERVER_KEY``, so a missing key is a hard error rather than a wake in a fresh session
  208. nobody watches.
  209. ``profile`` (a served secondary under ``gateway.multiplex_profiles``) takes the IN-PROCESS
  210. route instead of the HTTP one: the shared listener's ``/p/<profile>/`` mirror authenticates
  211. with that profile's own ``API_SERVER_KEY`` — which a route-only profile legitimately does not
  212. have — and an unprefixed self-post would resume the session in the DEFAULT profile's store.
  213. The caller already holds the owner profile's runtime scope, so the turn lands in that profile's
  214. own session. A non-default profile whose adapter cannot run in-process fails closed.
  215. """
  216. if profile and str(profile) != "default":
  217. in_process: Any = getattr(adapter, "run_internal_session_turn", None)
  218. if not callable(in_process):
  219. raise RuntimeError(
  220. f"wake self-post for served profile {profile!r} requires in-process session "
  221. "delivery; refusing to self-post as the default profile")
  222. await in_process(session_id=session_id, text=text, profile=str(profile),
  223. notification_category=notification_category)
  224. return
  225. import aiohttp
  226. host = str(getattr(adapter, "_host", "") or "127.0.0.1")
  227. if host in ("0.0.0.0", "::", "*"):
  228. host = "127.0.0.1" # wildcard bind — connect over loopback
  229. port = int(getattr(adapter, "_port", 0) or 8642)
  230. api_key = str(getattr(adapter, "_api_key", "") or "")
  231. if not api_key:
  232. raise RuntimeError("wake self-post requires API_SERVER_KEY: session continuation via "
  233. "X-Hermes-Session-Id is rejected (403) on an unauthenticated API "
  234. "server, so the wake cannot reach the target session")
  235. if ":" in host and not host.startswith("["):
  236. host = f"[{host}]" # bare IPv6 literal
  237. url = f"http://{host}:{port}/v1/chat/completions"
  238. headers = {"Authorization": f"Bearer {api_key}", "X-Hermes-Session-Id": session_id}
  239. payload = {"model": str(getattr(adapter, "_model_name", "") or "hermes-agent"),
  240. "messages": [{"role": "user", "content": text}], "stream": False}
  241. if notification_category == "diagnostic":
  242. payload["hermes_notification_category"] = "diagnostic"
  243. last_err: Optional[BaseException] = None
  244. attempts = 1 + len(_RETRY_DELAYS_SECONDS)
  245. for attempt in range(attempts):
  246. if attempt:
  247. await asyncio.sleep(_RETRY_DELAYS_SECONDS[attempt - 1])
  248. try:
  249. timeout = aiohttp.ClientTimeout(total=WAKE_TURN_TIMEOUT_SECONDS)
  250. async with aiohttp.ClientSession(timeout=timeout) as http:
  251. async with http.post(url, json=payload, headers=headers) as resp:
  252. if resp.status == 429: # concurrency cap — transient; back off and retry
  253. last_err = RuntimeError(
  254. f"wake self-post got HTTP 429 (concurrency cap) for session {session_id}"
  255. )
  256. logger.warning("%s; attempt %d/%d", last_err, attempt + 1, attempts)
  257. continue
  258. if resp.status >= 400: # non-transient (auth/validation): fail immediately
  259. body = (await resp.text())[:300]
  260. raise RuntimeError(
  261. f"wake self-post failed for session {session_id}: HTTP {resp.status}: {body}"
  262. )
  263. await resp.read()
  264. logger.info("wake self-post delivered for session %s (attempt %d)", session_id, attempt + 1)
  265. return
  266. except (TimeoutError, aiohttp.ClientError, OSError) as exc:
  267. last_err = exc
  268. logger.warning("wake self-post transient failure for session %s (attempt %d/%d): %s",
  269. session_id, attempt + 1, attempts, exc)
  270. raise RuntimeError(
  271. f"wake self-post gave up for session {session_id} after {attempts} attempts: {last_err}"
  272. ) from last_err