run_notifications_ownership.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. """Completion ownership for GatewayRunner: the session a pinned completion belongs to.
  2. Split out of ``gateway/run_notifications.py`` (itself split out of ``gateway/run.py``); bound onto
  3. ``GatewayRunner`` through ``GatewayNotificationsMixin``. A completion may be pinned to a delegate
  4. child's execution transcript: ownership follows only ``model_config._delegate_from`` to the
  5. human-facing session, shared by delivery preflight and route resolution, and no route moves until
  6. that owner is proven.
  7. """
  8. from __future__ import annotations
  9. import asyncio
  10. import json
  11. import logging
  12. import sqlite3
  13. from typing import Any, Optional, Sequence, cast
  14. from gateway.session import SessionEntry
  15. from hermes_state_errors import StateDbReplacedError
  16. # Log-record parity with the origin module.
  17. logger = logging.getLogger("gateway.run")
  18. # Upper bound on the ``model_config._delegate_from`` chain walked by _resolve_async_delegation_session
  19. # when canonicalizing a delegate child to its human-facing gateway parent (#92611). Real nesting is
  20. # bounded far lower by delegation.max_spawn_depth (default 2); this only stops a corrupt chain from
  21. # turning one completion event into an unbounded run of sequential session reads. Exceeding it is
  22. # fail-closed, like every other unresolved provenance.
  23. _MAX_DELEGATE_PROVENANCE_HOPS = 16
  24. class GatewayNotificationOwnershipMixin:
  25. """Ownership proof and route resolution for pinned async and process completions."""
  26. async def _resolve_compression_lineage_target(
  27. self, session_db: Any, session_entry: Optional[SessionEntry], pinned_session_id: str,
  28. ) -> tuple[str, Optional[str]]:
  29. """Distinguish uncertain compression ownership from a proven foreign route."""
  30. try:
  31. target_session_id = await session_db.get_compression_tip(pinned_session_id)
  32. if not target_session_id or target_session_id == pinned_session_id:
  33. return "retry", None # Rotation may not have published its continuation yet.
  34. tip_row = await session_db.get_session(target_session_id)
  35. if tip_row is None or tip_row.get("ended_at"):
  36. return "retry", None
  37. if session_entry is None or session_entry.session_id in {pinned_session_id, target_session_id}:
  38. return "deliver", target_session_id
  39. # Across several rotations, accept a stale route only when its own tip is the same live target.
  40. route_row = await session_db.get_session(session_entry.session_id)
  41. if route_row is None:
  42. return "retry", None
  43. if route_row.get("ended_at") and route_row.get("end_reason") == "compression":
  44. route_tip = await session_db.get_compression_tip(session_entry.session_id)
  45. if not route_tip or route_tip == session_entry.session_id:
  46. return "retry", None
  47. if route_tip == target_session_id:
  48. return "deliver", target_session_id
  49. except Exception:
  50. logger.debug("Async-delegation compression ownership lookup failed for %s", pinned_session_id, exc_info=True)
  51. return "retry", None
  52. logger.warning(
  53. "Async-delegation completion for compression lineage %s -> %s "
  54. "does not own current route %s; dropping injection.",
  55. pinned_session_id, target_session_id, session_entry.session_id,
  56. )
  57. return "terminal", None
  58. async def _closed_local_target(
  59. self, session_db: Any, entry: Optional[SessionEntry], pinned_session_id: str, pinned_row: Optional[dict],
  60. ) -> str:
  61. """The closed (/new, exit) physical session of a local conversation a completion is noted on
  62. (#134889 LC-7, ``session_completion_notes``), else ``""``. The pinned session, or the compression
  63. tip it rotated to, must be the one the conversation's route still names: a local /new opens a new
  64. route, so the replacement never matches, and messaging routes never qualify. Only a close ends a
  65. local tip (``session.detach {end_reason}`` stamps the client's own reason: ``cli_close``,
  66. ``tui_close``, ``tui_shutdown``...), so any end but compression is one."""
  67. from gateway.config import Platform
  68. if entry is None or entry.platform != Platform.LOCAL or pinned_row is None:
  69. return ""
  70. target, row = pinned_session_id, pinned_row
  71. if row.get("end_reason") == "compression":
  72. try:
  73. target = await session_db.get_compression_tip(pinned_session_id)
  74. row = await session_db.get_session(target) if target == entry.session_id else None
  75. except (sqlite3.Error, StateDbReplacedError):
  76. logger.debug("Closed local tip lookup failed for %s", pinned_session_id, exc_info=True)
  77. return "" # the compression path below defers it
  78. if (target != entry.session_id or row is None or not row.get("ended_at")
  79. or row.get("end_reason") == "compression"):
  80. return ""
  81. return target
  82. async def _lookup_completion_owner(self, session_id: str, current_session_id: str = ""):
  83. """Nonmutating delegation proof shared by preflight and route resolution.
  84. A current route is an explicit ownership boundary, not a route to repair.
  85. Generic parent pointers also describe compression and branches; only
  86. ``_delegate_from`` authorizes walking out of an execution transcript.
  87. """
  88. from gateway.run import _USER_BOUNDARY_END_REASONS
  89. session_db = cast(Any, self)._session_db
  90. if session_db is None:
  91. return "retry", session_id, None
  92. seen = set()
  93. for hops in range(_MAX_DELEGATE_PROVENANCE_HOPS + 1):
  94. if session_id in seen:
  95. return "terminal", session_id, None
  96. seen.add(session_id)
  97. try:
  98. row = await session_db.get_session(session_id)
  99. except Exception:
  100. logger.debug("Completion ownership lookup failed for %s", session_id, exc_info=True)
  101. return "retry", session_id, None
  102. if row is None:
  103. return "terminal", session_id, None
  104. # User closure is a boundary at EVERY hop, including execution rows.
  105. # Ordinary delegate completion may still follow provenance to its owner.
  106. if row.get("ended_at") and row.get("end_reason") in _USER_BOUNDARY_END_REASONS:
  107. return "terminal", session_id, row
  108. if session_id == current_session_id:
  109. return "deliver", session_id, row
  110. # A hop compressed into the current route is its conversation (a resumed delegate run), as
  111. # admission rules: climbing past it would reach the parent the run was spawned from.
  112. if current_session_id and row.get("end_reason") == "compression":
  113. try:
  114. if await session_db.get_compression_tip(session_id) == current_session_id:
  115. return "deliver", session_id, row
  116. except Exception:
  117. logger.debug("Completion compression lookup failed for %s", session_id, exc_info=True)
  118. return "retry", session_id, None
  119. config = row.get("model_config")
  120. try:
  121. if isinstance(config, str):
  122. config = json.loads(config)
  123. except (TypeError, ValueError):
  124. return "terminal", session_id, None
  125. if config is None:
  126. config = {}
  127. if not isinstance(config, dict):
  128. return "terminal", session_id, None
  129. if "_delegate_from" not in config:
  130. verdict = "terminal" if row.get("source") == "subagent" else "deliver"
  131. return verdict, session_id, row
  132. parent = config["_delegate_from"]
  133. if not isinstance(parent, str) or not parent.strip() or hops == _MAX_DELEGATE_PROVENANCE_HOPS:
  134. return "terminal", session_id, None
  135. session_id = parent
  136. return "terminal", session_id, None
  137. async def _watch_pin_gone(self, evt: dict, refusal: str, *, log: bool = True) -> bool:
  138. """True (logged once with ``log``) when a refused notice is pinned to a permanently-gone session.
  139. The session authority admits a notice only into its pinned session or that session's
  140. compression continuation, so after /new it refuses the notice on every retry and the drain
  141. would spin on it (#141562). This is the completion pre-flight's verdict, applied after the
  142. refusal. A dropped notice is gone for good; a /resume reopens the pinned session, so the
  143. process's later notices are admitted again.
  144. """
  145. pinned = str(evt.get("parent_session_id") or "").strip()
  146. if not pinned:
  147. return False
  148. session_key = str(evt.get("session_key") or "")
  149. verdict, _owner, _target = await cast(Any, self)._classify_completion_owner(pinned, session_key)
  150. if verdict != "terminal":
  151. return False
  152. if not log:
  153. return True
  154. logger.warning(
  155. "Dropping %s notification of process %s: pinned session %s is permanently gone (closed "
  156. "at a user boundary such as /new, or no longer an owner); the route now runs %s and "
  157. "refuses it (%s). Output remains available via process(action='log').",
  158. evt.get("type") or "watch", evt.get("session_id") or "<unknown>", pinned,
  159. cast(Any, self).session_store.peek_session_id(session_key) or "<none>", refusal,
  160. )
  161. return True
  162. async def _resolve_async_delegation_session(
  163. self, session_entry: SessionEntry, pinned_session_id: str, proven_session_id: str = "",
  164. completion_ids: Sequence[str] = (),
  165. ) -> Optional[SessionEntry]:
  166. """Resolve an async completion to its verified owning gateway session.
  167. Follow compression-rotation lineage (parent row ended, child continues), but never let a
  168. late completion override an unrelated /new or restored route. Unknown ownership fails
  169. closed; the result stays in the delegation records. An owner that cannot be confirmed right
  170. now (``retry``) sends the completions admitted with the event back to the delivery queue.
  171. A pinned *delegate* row is likewise canonicalized to its human-facing parent through
  172. ``model_config._delegate_from`` before any route check or mutation (#92611). Unresolvable
  173. provenance — a cycle, a missing parent, a malformed config, or a chain longer than
  174. ``_MAX_DELEGATE_PROVENANCE_HOPS`` — drops the injection rather than falling back to the
  175. current route entry. That is deliberate: routing an internal child's output into a session
  176. whose ownership was never verified is the same class of defect this canonicalization fixes,
  177. and the completion remains readable in the delegation records either way.
  178. """
  179. from gateway.run import _USER_BOUNDARY_END_REASONS
  180. session_db = cast(Any, self._session_db)
  181. if session_db is None:
  182. logger.warning(
  183. "Async-delegation completion has no session database; "
  184. "dropping injection (#55578 fail-closed)."
  185. )
  186. return None
  187. if proven_session_id and proven_session_id == session_entry.session_id:
  188. # Admission already settled the durable claim on this proof; another read could only lose it.
  189. return session_entry
  190. # Snapshot before ownership awaits: a concurrent /stop or /new wins.
  191. run_generation = self._current_session_run_generation(session_entry.session_key)
  192. verdict, pinned_session_id, pinned_row = await self._lookup_completion_owner(
  193. pinned_session_id, session_entry.session_id,
  194. )
  195. if verdict != "deliver" or pinned_row is None:
  196. return await self._requeue_uncertain_completion(verdict, completion_ids)
  197. target_session_id = pinned_session_id
  198. follows_compression = False
  199. if pinned_row.get("ended_at"):
  200. _end_reason = str(pinned_row.get("end_reason") or "")
  201. if _end_reason in _USER_BOUNDARY_END_REASONS:
  202. logger.warning(
  203. "Async-delegation completion pinned to user-closed session %s "
  204. "(end_reason=%r); dropping injection instead of resurrecting it "
  205. "(#55578 fail-closed).", pinned_session_id, _end_reason,
  206. )
  207. return None
  208. if _end_reason != "compression":
  209. # Idle/timeout end (scale-to-zero norm): the chat route is still valid, so deliver to its
  210. # current session rather than drop (the row would be acked then silently lost).
  211. logger.info(
  212. "Async-delegation completion pinned to %s-ended session %s; "
  213. "retargeting to the chat's current session %s.",
  214. _end_reason or "idle", pinned_session_id, session_entry.session_id,
  215. )
  216. return session_entry
  217. follows_compression = True
  218. verdict, target_session_id = await self._resolve_compression_lineage_target(
  219. session_db, session_entry, pinned_session_id,
  220. )
  221. if verdict != "deliver" or target_session_id is None:
  222. return await self._requeue_uncertain_completion(verdict, completion_ids)
  223. if target_session_id == session_entry.session_id:
  224. return session_entry
  225. prior_session_id = session_entry.session_id
  226. if not self._is_session_run_current(session_entry.session_key, run_generation):
  227. logger.warning(
  228. "Async-delegation completion for routing key %s was invalidated while resolving pinned "
  229. "session %s; leaving the route on %s and dropping injection.",
  230. session_entry.session_key, pinned_session_id, prior_session_id,
  231. )
  232. return None
  233. if follows_compression:
  234. switched = await self.async_session_store.advance_compression_session(
  235. session_entry.session_key, prior_session_id, target_session_id,
  236. )
  237. else:
  238. # CAS on the session this completion resolved against: a route replaced meanwhile
  239. # (/new, /resume) wins over the stale completion.
  240. switched = await self.async_session_store.switch_session(
  241. session_entry.session_key, target_session_id, expected_session_id=prior_session_id,
  242. )
  243. if switched is None:
  244. logger.warning(
  245. "Async-delegation completion could not bind routing key %s to "
  246. "owning session %s (route moved or unknown); dropping injection.",
  247. session_entry.session_key, target_session_id,
  248. )
  249. return None
  250. logger.info(
  251. "Pinned async-delegation completion to owning session %s (was %s) for routing key %s (#57498)",
  252. target_session_id, prior_session_id, session_entry.session_key,
  253. )
  254. return switched
  255. async def _requeue_uncertain_completion(self, verdict: str, completion_ids: Sequence[str]) -> None:
  256. """Offer admitted completions again when the turn could not confirm their owner (``retry``),
  257. so the next delivery re-runs the pre-flight; boundary verdicts stay dropped."""
  258. if verdict != "retry" or not completion_ids:
  259. return
  260. from tools.async_delegation import requeue_admitted_completion
  261. from tools.process_registry import process_registry
  262. runner = cast(Any, self)
  263. with runner._completion_delivery_lock:
  264. for delegation_id in completion_ids:
  265. runner._completion_deliveries_delivered.pop(("async_delegation", delegation_id, ""), None)
  266. for delegation_id in completion_ids:
  267. try:
  268. requeued = await asyncio.to_thread(
  269. requeue_admitted_completion, delegation_id, process_registry.completion_queue)
  270. except Exception:
  271. logger.debug("Could not requeue admitted completion %s", delegation_id, exc_info=True)
  272. requeued = False
  273. logger.warning(
  274. "Async-delegation completion %s: owner not confirmed after admission; %s.", delegation_id,
  275. "offered again" if requeued else "not offered again (result remains queryable)",
  276. )