run_goals.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580
  1. """Goal/heartbeat continuation, post-turn hooks and loop-wakeup watcher methods for GatewayRunner.
  2. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO.
  3. ``gateway.run`` internals are imported lazily inside method bodies (import cycle),
  4. so ``patch("gateway.run.X")`` keeps intercepting them at call time.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import dataclasses
  9. import logging
  10. import time
  11. from contextlib import nullcontext, suppress
  12. from typing import TYPE_CHECKING, Any, Optional
  13. from gateway.config import Platform
  14. from gateway.platforms.event import MessageEvent, MessageType
  15. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  16. from gateway.run import GatewayRunner
  17. from gateway.run_turn_runner import TurnRunner
  18. # Log-record parity with the origin module.
  19. logger = logging.getLogger("gateway.run")
  20. class GatewayGoalsMixin:
  21. """Goal/heartbeat continuation, post-turn hooks and loop-wakeup watcher methods for GatewayRunner."""
  22. # ── /goal — persistent cross-turn goals (Ralph-style loop) ──────────
  23. def _goal_max_turns_from_config(self) -> int:
  24. """Configured /goal turn budget. GatewayRunner.config is a GatewayConfig dataclass, so the
  25. top-level ``goals`` block is only reachable via hermes_cli.config.load_config()."""
  26. try:
  27. goals_cfg = (
  28. (self.config or {}).get("goals", {})
  29. if isinstance(self.config, dict)
  30. else getattr(self.config, "goals", {}) or {}
  31. )
  32. if not goals_cfg:
  33. from hermes_cli.config import load_config
  34. goals_cfg = (load_config() or {}).get("goals") or {}
  35. return int(goals_cfg.get("max_turns", 20) or 20)
  36. except Exception:
  37. return 20
  38. async def _warm_goals_session_db(self, label: str) -> None:
  39. """Warm the goals SessionDB cache off-loop (best-effort): a cold cache runs the state.db
  40. init on the loop thread and freezes the loop. The executor hop keeps the profile home
  41. override alive under multiplex; a failed warm-up is a bounded stall, never a crash."""
  42. try:
  43. from hermes_cli.goals import _get_session_db as _warm_goals_db
  44. await self._run_in_executor_with_context(_warm_goals_db)
  45. except Exception as exc:
  46. logger.warning("%s: session DB warm-up failed: %s", label, exc)
  47. async def _session_entry_for_manager(self, event: MessageEvent, label: str):
  48. """Session entry for a /goal or /heartbeat manager, or None when lookup fails. Warms the
  49. SessionDB cache first (a cold cache drops the first write while the reply claims it was
  50. set). Internal events never touch activity (idle/daily reset clock)."""
  51. await self._warm_goals_session_db(label)
  52. try:
  53. session_entry = await self.async_session_store.get_or_create_session(
  54. event.source, touch_activity=not bool(getattr(event, "internal", False)),
  55. )
  56. except Exception as exc:
  57. logger.debug("%s: session lookup failed: %s", label, exc)
  58. return None
  59. return session_entry if getattr(session_entry, "session_id", None) else None
  60. async def _manager_for_event(self, event: MessageEvent, kind: str, load):
  61. """``(manager, session_entry)`` for *kind* ("goal"/"heartbeat"), or ``(None, None)``.
  62. ``load()`` imports the manager class and returns a ``session_id -> manager`` factory."""
  63. try:
  64. factory = load()
  65. except Exception as exc:
  66. logger.debug("%s manager unavailable: %s", kind, exc)
  67. return None, None
  68. session_entry = await self._session_entry_for_manager(event, f"{kind} manager")
  69. if session_entry is None:
  70. return None, None
  71. return factory(session_entry.session_id), session_entry
  72. async def _get_goal_manager_for_event(self, event: MessageEvent):
  73. """Return ``(GoalManager, session_entry)`` for this event, or ``(None, None)``."""
  74. def _load():
  75. from hermes_cli.goals import GoalManager
  76. max_turns = self._goal_max_turns_from_config()
  77. return lambda sid: GoalManager(session_id=sid, default_max_turns=max_turns)
  78. return await self._manager_for_event(event, "goal", _load)
  79. async def _get_heartbeat_manager_for_event(self, event: MessageEvent):
  80. """Return ``(HeartbeatManager, session_entry)`` for this event, or ``(None, None)``."""
  81. def _load():
  82. from hermes_cli.heartbeat import HeartbeatManager
  83. return lambda sid: HeartbeatManager(session_id=sid)
  84. return await self._manager_for_event(event, "heartbeat", _load)
  85. @staticmethod
  86. def _synthetic_prompt_event(source: Any, text: str, *, internal: bool = False) -> MessageEvent:
  87. """Build the TEXT event used to inject a goal/heartbeat/loop prompt into a session.
  88. The stored source's ``message_id`` is the message that registered the watch; a synthetic
  89. prompt is not a reply to it, so it is dropped or every progress bubble and final reply
  90. would quote that stale message (Telegram DM topics route anchorless via the topic id).
  91. ``replace_source`` keeps the receiving-adapter provenance native admission reads.
  92. """
  93. from gateway.session_identity import replace_source
  94. source = replace_source(source, message_id=None) if getattr(source, "message_id", None) else source
  95. return MessageEvent(
  96. text=text, message_type=MessageType.TEXT, source=source, internal=internal,
  97. inherit_channel_pin=True,
  98. )
  99. def _register_heartbeat_watch(self, quick_key: str, source: Any, session_id: str) -> None:
  100. """Track the canonical route and start the restart-recoverable poller."""
  101. watch = getattr(self, "_heartbeat_watch", None)
  102. if watch is None:
  103. watch = self._heartbeat_watch = {}
  104. watch[quick_key] = (source, session_id)
  105. self._start_heartbeat_poller()
  106. def _unregister_heartbeat_watch(self, quick_key: str) -> None:
  107. (getattr(self, "_heartbeat_watch", None) or {}).pop(quick_key, None)
  108. async def _heartbeat_poll_once(self, watch: dict) -> None:
  109. """Wake each idle watched session once; leave busy sessions' ticks unclaimed."""
  110. for quick_key, (source, session_id) in list(watch.items()):
  111. try:
  112. with self._profile_scope_for_source(source):
  113. await self._heartbeat_poll_watch(watch, quick_key, source, session_id)
  114. except Exception as exc:
  115. logger.debug("heartbeat poll for %s failed: %s", quick_key, exc)
  116. async def _heartbeat_poll_watch(self, watch, quick_key, source, session_id):
  117. await self._warm_goals_session_db("heartbeat poll")
  118. store = getattr(self, "session_store", None)
  119. if store is not None:
  120. current = store.peek_session_id(quick_key)
  121. if not current:
  122. watch.pop(quick_key, None)
  123. return
  124. session_id = current
  125. watch[quick_key] = (source, session_id)
  126. adapter = self._delivery_adapter_for(source)
  127. from gateway.session_local import LocalSessionAdapter
  128. from gateway.session_viewer_automation import viewer_driven
  129. local = isinstance(adapter, LocalSessionAdapter)
  130. # A local chat has no message handler: its owner admits the tick (``admit_heartbeat``).
  131. if adapter is None or not (adapter._message_handler or local):
  132. return
  133. if local and viewer_driven(adapter.authority, source.chat_id, "viewer_heartbeat"):
  134. return # the classic CLI's watchdog fires it: that CLI paints only the turns it submits
  135. if getattr(self, 'session_authority', None) is not None:
  136. from gateway.run_heartbeat_acceptance import admit_heartbeat
  137. await admit_heartbeat(self, adapter, source, session_id, quick_key)
  138. return
  139. if (
  140. self._is_session_running(quick_key)
  141. or quick_key in adapter._active_sessions
  142. or self._queue_depth(quick_key, adapter=adapter) > 0
  143. ):
  144. return # keep missed intervals due until user work has drained
  145. from agent.estop import check_paused
  146. if check_paused("heartbeat", logger):
  147. return # `hermes pause`: leave the tick unclaimed so it fires after `hermes resume`
  148. from hermes_cli.heartbeat import HeartbeatManager
  149. mgr = HeartbeatManager(session_id=session_id)
  150. if not mgr.has_heartbeat():
  151. watch.pop(quick_key, None)
  152. return
  153. prompt = mgr.due_prompt()
  154. if not prompt:
  155. return
  156. event = self._synthetic_prompt_event(source, prompt)
  157. event.metadata["gateway_session_key"] = quick_key
  158. event._heartbeat_execution_started = False
  159. # Provenance read by display_kind_for_event / the turn's quiet surfaces; the event stays
  160. # non-internal so authorization and the emergency stop still apply.
  161. event._heartbeat_session_id = session_id
  162. # A pinned route skips topic recovery: no await between the idle
  163. # check and adapter claim. FIFO alone never wakes an idle session.
  164. try:
  165. await adapter.handle_message(event)
  166. finally:
  167. task = getattr(adapter, "_session_tasks", {}).get(quick_key)
  168. if task is not None:
  169. from gateway.run_heartbeat_acceptance import settle_heartbeat_attempt
  170. task.add_done_callback(lambda done: settle_heartbeat_attempt(event, mgr))
  171. elif quick_key not in adapter._active_sessions:
  172. mgr.abandon_fire()
  173. def _start_heartbeat_poller(self) -> None:
  174. """Start the single gateway-wide heartbeat poll task (idempotent)."""
  175. existing = getattr(self, "_heartbeat_poll_task", None)
  176. if existing is not None and not existing.done():
  177. return
  178. from hermes_cli.heartbeat import POLL_SECONDS
  179. async def _poll_loop():
  180. while True:
  181. await asyncio.sleep(POLL_SECONDS)
  182. from gateway.run_heartbeat_restore import restore_heartbeat_watches
  183. await restore_heartbeat_watches(self)
  184. watch = getattr(self, "_heartbeat_watch", None)
  185. if watch:
  186. await self._heartbeat_poll_once(watch)
  187. try:
  188. task = self._heartbeat_poll_task = asyncio.create_task(_poll_loop())
  189. # PERMANENT once started (infinite loop) — tag it like a _spawn_supervised watcher so
  190. # _scale_to_zero_has_live_background_work() doesn't treat the gateway as busy forever.
  191. task._hermes_supervised_watcher = True # type: ignore[attr-defined]
  192. _bg = getattr(self, "_background_tasks", None)
  193. if _bg is not None:
  194. _bg.add(task)
  195. task.add_done_callback(_bg.discard)
  196. except Exception:
  197. logger.debug("Failed to start heartbeat poller", exc_info=True)
  198. def _goal_notice_adapter(self, source: Any):
  199. adapter = self._delivery_adapter_for(source)
  200. if not adapter:
  201. logger.debug("goal continuation: no adapter for %s", getattr(source, "platform", None))
  202. return adapter
  203. async def _send_goal_status_notice(self, source: Any, message: str) -> None:
  204. """Send a /goal judge status line back to the originating chat/thread."""
  205. if getattr(source, "platform", None) == Platform.LOCAL:
  206. # The local adapter's send is a no-op (viewers read the event stream): main's TUI
  207. # painted the verdict from ``status.update {kind: goal}``.
  208. from gateway.session_authorities import active_authority
  209. authority = active_authority(self)
  210. live = authority.sessions.get(source.chat_id) if authority is not None else None
  211. if live is not None:
  212. live.event_stream.publish(source.chat_id, {"kind": "goal", "text": message},
  213. event_type="status.update")
  214. return
  215. adapter = self._goal_notice_adapter(source)
  216. if not adapter:
  217. return
  218. metadata = None
  219. with suppress(Exception):
  220. metadata = self._thread_metadata_for_source(source)
  221. result = await adapter.send(source.chat_id, message, metadata=metadata)
  222. if result is not None and not getattr(result, "success", True):
  223. logger.warning(
  224. "goal continuation: status send failed: %s", getattr(result, "error", "unknown error"),
  225. )
  226. async def _defer_goal_status_notice_after_delivery(self, source: Any, message: str) -> None:
  227. """Send a /goal status line after the main response is delivered.
  228. The adapter sends the agent response after this caller returns, so for reading order use
  229. its one-shot post-delivery callback when available, else deliver directly (never drop).
  230. """
  231. adapter = self._goal_notice_adapter(source)
  232. if not adapter:
  233. return
  234. async def _deliver() -> None:
  235. try:
  236. await self._send_goal_status_notice(source, message)
  237. except Exception as exc:
  238. logger.warning("goal continuation: status send failed: %s", exc, exc_info=True)
  239. session_key = None
  240. with suppress(Exception):
  241. session_key = self._session_key_for_source(source)
  242. if session_key and hasattr(adapter, "register_post_delivery_callback"):
  243. try:
  244. active = getattr(adapter, "_active_sessions", {}).get(session_key)
  245. generation = getattr(active, "_hermes_run_generation", None) if active is not None else None
  246. adapter.register_post_delivery_callback(session_key, _deliver, generation=generation)
  247. return
  248. except Exception as exc:
  249. logger.debug("goal continuation: post-delivery callback registration failed: %s", exc)
  250. await _deliver()
  251. async def _post_turn_manager(self, session_entry: Any, label: str, module: str, load):
  252. """Shared head of the post-turn hooks: ``load()`` imports the manager module and returns a
  253. ``session_id -> manager`` factory; None when unavailable / no session id. Warms the
  254. SessionDB cache first — a cold cache at the turn boundary drops the read/write."""
  255. try:
  256. factory = load()
  257. except Exception as exc:
  258. logger.debug("%s: %s module unavailable: %s", label, module, exc)
  259. return None
  260. sid = getattr(session_entry, "session_id", None) or ""
  261. if not sid:
  262. return None
  263. await self._warm_goals_session_db(label)
  264. # The manager loads its state from state.db on construction: off the loop, in this scope.
  265. return await asyncio.to_thread(factory, sid)
  266. async def _post_turn_goal_continuation(
  267. self, *, session_entry: Any, source: Any, final_response: str,
  268. ) -> None:
  269. """Run the goal judge after a gateway turn (AFTER delivery) and, if still active, enqueue a
  270. continuation through the adapter FIFO so a simultaneous real user message takes priority."""
  271. def _load():
  272. from hermes_cli.goals import GoalManager
  273. max_turns = self._goal_max_turns_from_config()
  274. return lambda sid: GoalManager(session_id=sid, default_max_turns=max_turns)
  275. mgr = await self._post_turn_manager(session_entry, "goal continuation", "goals", _load)
  276. if mgr is None or not mgr.is_active():
  277. return
  278. _bg_procs, _active_deleg = None, 0
  279. with suppress(Exception):
  280. from hermes_cli.goals import count_active_delegations, gather_background_processes as _gather_bg
  281. # Only THIS session's processes (gateway turns register under turn_ctx.session_id):
  282. # subagents' pollers must not park the parent's goal.
  283. _bg_procs = _gather_bg(owner_task_id=getattr(session_entry, "session_id", None) or None)
  284. _active_deleg = count_active_delegations(getattr(session_entry, "session_id", None))
  285. # judge_goal() is a synchronous aux-LLM HTTP call (10-40 s; would block Discord heartbeats).
  286. # _run_in_executor_with_context carries the profile secret scope / aux runtime contextvars
  287. # without which aux credential resolution fails under multiplexing.
  288. decision = await self._run_in_executor_with_context(
  289. lambda: mgr.evaluate_after_turn(
  290. final_response or "", user_initiated=True, background_processes=_bg_procs,
  291. active_delegations=_active_deleg,
  292. ),
  293. )
  294. msg = decision.get("message") or ""
  295. # Deferred until the visible final response is delivered, else "✓ Goal achieved" precedes it.
  296. if msg and source is not None:
  297. await self._defer_goal_status_notice_after_delivery(source, msg)
  298. prompt = decision.get("continuation_prompt") or ""
  299. if not decision.get("should_continue") or not prompt or source is None:
  300. return
  301. if getattr(source, "platform", None) == Platform.LOCAL:
  302. # A local session's adapter FIFO has no reader: its turns are authority admissions.
  303. # Commit the continuation there, as main's TUI chained it after the turn.
  304. from gateway.session_automation import admit_route_wake
  305. try:
  306. await admit_route_wake(self, self._delivery_adapter_for(source), source, prompt, "goal-continuation")
  307. except Exception: # health: allow BLE001 -- best-effort like the FIFO enqueue below; traceback logged
  308. logger.debug("goal continuation: local admission failed", exc_info=True)
  309. return
  310. # Enqueue via the adapter's FIFO so a user message already in flight preempts naturally.
  311. try:
  312. adapter = self._delivery_adapter_for(source)
  313. _quick_key = self._session_key_for_source(source)
  314. if adapter and _quick_key:
  315. self._enqueue_fifo(_quick_key, self._synthetic_prompt_event(source, prompt), adapter)
  316. except Exception as exc:
  317. logger.debug("goal continuation: enqueue failed: %s", exc)
  318. async def _run_post_turn_hooks(
  319. self, *, agent_result: Any, source: Any, is_internal: bool, event: Any = None,
  320. ) -> None:
  321. """Run goal and loop bookkeeping after an agent turn returns."""
  322. from gateway.session_authorities import active_authority
  323. from gateway.session_viewer_automation import viewer_driven
  324. if viewer_driven(active_authority(self), getattr(source, "chat_id", None)):
  325. return # the classic CLI viewer judges and continues through ``session.automation``
  326. final_text = self._final_text_for_post_turn_hooks(agent_result, event)
  327. try:
  328. session_entry = await self.async_session_store.get_or_create_session(
  329. source, touch_activity=not is_internal,
  330. )
  331. except Exception as exc:
  332. logger.debug("post-turn session resolution failed: %s", exc)
  333. return
  334. # Empty interrupted/errored responses must not drive /goal, but an in-flight /loop tick
  335. # still needs to be released and rescheduled. A failed turn with text (a refusal: its
  336. # folder is gone) is no evidence either (the TUI's hook skips it the same way): the tick is
  337. # released with no text, so a --until judge never grades it.
  338. failed = self._turn_failed()
  339. hooks = [("loop completion", self._post_turn_loop_completion)]
  340. if final_text.strip() and not failed:
  341. hooks.insert(0, ("goal continuation", self._post_turn_goal_continuation))
  342. for label, hook in hooks:
  343. try:
  344. await hook(session_entry=session_entry, source=source, final_response="" if failed else final_text)
  345. except Exception as exc:
  346. logger.debug("%s hook failed: %s", label, exc)
  347. await self._publish_local_control(source, only_if_present=True)
  348. async def _publish_local_control(self, source: Any, *, only_if_present: bool = False) -> None:
  349. """A local (Desktop/Ink) session's goal or loop state just moved here: repaint its viewers' cards,
  350. whose loop countdown runs off the snapshot's ``next_due_at``."""
  351. if getattr(source, "platform", None) != Platform.LOCAL:
  352. return
  353. from gateway.session_authorities import active_authority
  354. from gateway.session_control_actions import publish_async
  355. await publish_async(active_authority(self), source.chat_id, only_if_present=only_if_present)
  356. @staticmethod
  357. def _turn_failed() -> bool:
  358. """Whether the owner recorded this turn's result as failed (``session_ingress`` captures it
  359. around ``_handle_message``; turns outside an owner capture nothing)."""
  360. from gateway.session_results import execution_result
  361. captured = execution_result.get() or {}
  362. result = captured.get('terminal_result', captured.get('result'))
  363. return isinstance(result, dict) and bool(result.get('failed'))
  364. @staticmethod
  365. def _final_text_for_post_turn_hooks(agent_result, event=None) -> str:
  366. """Text for /goal and /loop after a gateway turn. Streamed turns return None from
  367. _handle_message_with_agent (already_sent); the delivered reply is stashed on the event."""
  368. text = ""
  369. if isinstance(agent_result, dict):
  370. text = str(agent_result.get("final_response") or "")
  371. elif isinstance(agent_result, str):
  372. text = agent_result
  373. if text.strip():
  374. return text
  375. streamed = getattr(event, "_streamed_final_response", None)
  376. return streamed if isinstance(streamed, str) and streamed.strip() else text
  377. async def _post_turn_loop_completion(
  378. self, *, session_entry: Any, source: Any, final_response: str,
  379. ) -> None:
  380. """Complete a /loop wakeup tick after a gateway turn. No-op unless a tick is in flight
  381. (``awaiting_response``, set when the wakeup was injected); applies the LOOP_COMPLETE marker
  382. / --until judge / caps and schedules the next tick for the idle wakeup watcher."""
  383. def _load():
  384. from hermes_cli.loops import LoopManager
  385. return lambda sid: LoopManager(session_id=sid)
  386. mgr = await self._post_turn_manager(session_entry, "loop completion", "loops", _load)
  387. state = mgr.state if mgr is not None else None
  388. if state is None or not state.awaiting_response:
  389. return
  390. # The --until judge is a sync aux-LLM call — keep it off the event loop, but carry the
  391. # contextvars: a bare executor hop drops the profile HERMES_HOME override and secret scope,
  392. # so a served secondary's tick would be written into the DEFAULT profile's state.db.
  393. decision = await self._run_in_executor_with_context(mgr.complete_tick, final_response or "")
  394. msg = decision.get("message") or ""
  395. if msg and source is not None:
  396. await self._defer_goal_status_notice_after_delivery(source, msg)
  397. async def _loop_wakeup_fire_one(
  398. self, sid: str, state: Any, now: float, warned_no_route: set, profile: Optional[str] = None,
  399. ) -> None:
  400. """Inject one due /loop wakeup into its session, applying every deferral rule. ``profile`` is
  401. the store being scanned (None = default); a ``profile`` persisted in the route wins."""
  402. from hermes_cli.loops import LoopManager, goal_blocks_loop_tick
  403. if state.awaiting_response or now < state.next_due_at:
  404. return
  405. route = state.route or {}
  406. platform_name = route.get("platform", "")
  407. chat_id = route.get("chat_id", "")
  408. if not platform_name or not chat_id:
  409. return # CLI / TUI-owned loop — their own schedulers drive it.
  410. profile = route.get("profile") or profile
  411. # The loop's OWN profile's adapter map, fail closed: ``self.adapters`` is the default profile's,
  412. # so a secondary session's wakeup would inject via the default bot on a bare chat_id (a
  413. # Telegram DM lands in the user's chat with the other bot).
  414. adapters = self._adapters_for_profile(profile)
  415. adapter = next((a for p, a in adapters.items() if p.value == platform_name), None)
  416. if adapter is None:
  417. if sid not in warned_no_route:
  418. warned_no_route.add(sid)
  419. logger.debug(
  420. "loop wakeup: no adapter for platform %r (session %s, profile %s)", platform_name, sid, profile,
  421. )
  422. return
  423. source = self._build_process_event_source({
  424. "session_key": "",
  425. "platform": platform_name,
  426. "chat_id": chat_id,
  427. **{k: route.get(k, "") for k in ("chat_type", "thread_id", "user_id", "user_name")},
  428. })
  429. if source is None:
  430. return
  431. if profile and not getattr(source, "profile", None):
  432. source.profile = profile # session key + runtime scope of the injected turn
  433. session_key = None
  434. with suppress(Exception):
  435. session_key = self._session_key_for_source(source)
  436. if session_key and session_key in self._running_agents:
  437. return # busy — stays due, next scan retries
  438. if goal_blocks_loop_tick(sid):
  439. return
  440. mgr = LoopManager(session_id=sid)
  441. if not mgr.is_due(now):
  442. return
  443. from agent.estop import check_paused
  444. # Loop wakeups are injected as internal events, which bypass the inbound estop gate; without
  445. # this a `hermes pause` would still start agent turns. Not claiming the tick keeps it due.
  446. if check_paused("loop", logger):
  447. return
  448. # fire_tick()/complete_tick() are writes (BEGIN IMMEDIATE) taking the SessionDB writer lock; a slow
  449. # writer elsewhere holding it while the loop thread blocked froze the gateway until the watchdog
  450. # fired. The context-preserving executor keeps the profile HERMES_HOME override under multiplex.
  451. wakeup = await self._run_in_executor_with_context(mgr.fire_tick)
  452. if not wakeup:
  453. return
  454. # #85957: after the parent turn's event.complete the CLIENT owns the next turn on this stateless
  455. # surface. Persist the completion as a durable delivery row — never self-post it as a new role=user
  456. # prompt.
  457. # #85957: same client-owns-the-turn rule as the raw-key branch above — persist the completion as a
  458. # delivery row, never self-post it as a new role=user prompt.
  459. try:
  460. logger.info(
  461. "loop wakeup #%s — injecting for %s chat=%s thread=%s",
  462. mgr.state.ticks_fired if mgr.state else "?",
  463. platform_name, source.chat_id, source.thread_id,
  464. )
  465. from gateway.session_automation import admit_route_wake
  466. # Under a session authority the tick commits to the FIFO (never a classic turn beside an
  467. # admitted one); a slash-command loop is a command and keeps the command path.
  468. if wakeup.lstrip().startswith("/") or not await admit_route_wake(self, adapter, source, wakeup, "loop-tick"):
  469. await adapter.handle_message(self._synthetic_prompt_event(source, wakeup, internal=True))
  470. # Slash-command loops dispatch through the command path and never hit the post-turn
  471. # completion hook — complete the tick immediately (caps + scheduling).
  472. if wakeup.lstrip().startswith("/"):
  473. await self._run_in_executor_with_context(mgr.complete_tick, "")
  474. except Exception as exc:
  475. logger.warning("loop wakeup injection failed for %s: %s", sid, exc)
  476. with suppress(Exception):
  477. mgr.abandon_tick()
  478. # Viewers swap the countdown for "running now" (or re-anchor it after a slash tick settled).
  479. await self._publish_local_control(source)
  480. async def _loop_wakeup_watcher(self, interval: float = 15.0) -> None:
  481. """Fire due /loop wakeups for idle gateway sessions: a coarse ticker scans persisted loops
  482. (SessionDB ``loop:*`` rows) and injects each due prompt via the synthetic-message path.
  483. Deferrals: session running a turn (FIFO would race the live turn); active non-parked /goal
  484. (goal owns the idle boundary); no routing metadata (one-time warning).
  485. Multiplex: one gateway-wide task, so ``list_active_loops`` alone reads only the launch home's
  486. store — a ``/loop`` set from a secondary profile's chat would never fire. Every served
  487. profile's store is scanned under its own runtime scope (same shape as ``_handoff_watcher``),
  488. and each hit is fired against that profile's adapters."""
  489. from gateway.run import _async_profile_runtime_scope, _resolve_handoff_watch_scopes
  490. from gateway.run_idle_gates import profile_has_active_loop
  491. await asyncio.sleep(5) # let platforms finish connecting
  492. warned_no_route: set = set()
  493. def _scope(profile_home):
  494. # profile_home None = the launch profile's own store; once the process multiplexes it
  495. # binds its own scope instead of running on ambient env (see _scope_or_null).
  496. if profile_home is not None:
  497. return _async_profile_runtime_scope(profile_home)
  498. from tui_gateway.launch_profile_policy import async_launch_profile_scope_if_multiplexed
  499. return async_launch_profile_scope_if_multiplexed()
  500. async def _scan_one_store(profile_name: Optional[str]) -> None:
  501. from hermes_cli.loops import list_active_loops
  502. # Warm once per scan: the scan reads every persisted loop and a cold cache would
  503. # run the state.db init on the loop thread before the first read.
  504. await self._warm_goals_session_db("loop wakeup")
  505. # Off-loop too: the read is lock-free under WAL but convoys on the writer lock without it.
  506. active_loops = await self._run_in_executor_with_context(list_active_loops)
  507. now = time.time()
  508. for sid, state in active_loops:
  509. await self._loop_wakeup_fire_one(sid, state, now, warned_no_route, profile_name)
  510. while self._running:
  511. try:
  512. # Multiplex resolution walks the filesystem off-loop; a stalled walk on the loop
  513. # trips the loop-liveness watchdog (exit 75).
  514. for profile_name, profile_home in await _resolve_handoff_watch_scopes(self):
  515. # Idle gate (run_idle_gates): skip the scope entry when the profile's store holds
  516. # no active loop. The root scan (None) is unscoped and stays cheap.
  517. if profile_home is not None and not await self._run_in_executor_with_context(
  518. profile_has_active_loop, profile_home):
  519. continue
  520. async with _scope(profile_home):
  521. await _scan_one_store(profile_name)
  522. except Exception as exc:
  523. logger.debug("loop wakeup watcher error: %s", exc)
  524. await asyncio.sleep(interval)