run_notifications.py 116 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972
  1. """Process/completion/update notifications, media delivery and async-delegation delivery for GatewayRunner.
  2. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO.
  3. ``gateway.run`` internals are imported lazily inside method bodies (import cycle),
  4. so ``patch("gateway.run.X")`` keeps intercepting them at call time.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import contextlib
  9. import contextvars
  10. import dataclasses
  11. import json
  12. import logging
  13. import time
  14. from pathlib import Path
  15. from typing import Any, Dict, Optional, cast
  16. from agent.i18n import t
  17. from agent.retry_utils import jittered_backoff
  18. from gateway.config import Platform, _BUILTIN_PLATFORM_VALUES
  19. from gateway.platforms.base import BasePlatformAdapter, _mark_notify_metadata
  20. from gateway.platforms.event import MessageEvent, MessageType
  21. from gateway.session import SessionEntry, SessionSource, SessionStore
  22. from gateway.run_shutdown import _delivery_target_key, _log_suppressed, _notice_target_key, _send_error, _send_failed
  23. from gateway.run_notifications_update import GatewayUpdateNotificationsMixin
  24. from gateway.run_notifications_ownership import GatewayNotificationOwnershipMixin
  25. # Log-record parity with the origin module.
  26. logger = logging.getLogger("gateway.run")
  27. def _served_notice_target_key(profile: Optional[str], platform_value: str, chat_id, thread_id) -> tuple:
  28. """Notice-dedupe key for one SERVED profile's home channel.
  29. A secondary uses the ``<profile>:<platform>`` key convention the runtime status already
  30. stamps in ``gateway_state.json``; the launch profile keeps the bare platform value so a
  31. marker written before this change still matches its delivered targets.
  32. """
  33. return _notice_target_key(
  34. platform_value if profile is None else f"{profile}:{platform_value}", chat_id, thread_id)
  35. def _safe_delivery_transport(platform, config, adapters, *, profile: Optional[str] = None):
  36. """``resolve_delivery_transport`` isolated to one target: ``None`` (logged) on failure.
  37. The fan-out spans every served profile, so one profile's broken adapter must not abort the
  38. pass and starve every profile after it in dict order.
  39. """
  40. from gateway.delivery import resolve_delivery_transport
  41. try:
  42. return resolve_delivery_transport(platform, config, adapters)
  43. except Exception as exc:
  44. logger.debug(
  45. "Home-channel transport unavailable for %s%s: %s",
  46. f"{profile}:" if profile else "", getattr(platform, "value", platform), exc)
  47. return None
  48. _VIDEO_EXTS = {'.mp4', '.mov', '.avi', '.mkv', '.webm', '.3gp'}
  49. # Routing fields copied verbatim from a process watcher onto its synthetic completion event.
  50. _WATCHER_ROUTE_FIELDS = ("session_key", "platform", "chat_type", "chat_id", "thread_id", "user_id", "user_name")
  51. _IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.webp', '.gif'}
  52. # Storage causes that clear on their own (one session's lease/compression, not the store): the
  53. # home-channel notice appends the operator restart tail for every OTHER cause.
  54. _SELF_CLEARING_STORAGE_CAUSES = frozenset({"compression", "compression_closed", "turn_lease"})
  55. # Process-watcher retries of a refused completion: ~5 min at the default 5 s check interval.
  56. _MAX_COMPLETION_INJECTION_ATTEMPTS = 8
  57. _COMPLETION_INJECTION_RETRY_MAX_DELAY = 60
  58. # Drain retries of a refused watch event follow the same bounded-backoff policy, keyed to the
  59. # 2 s drain cadence instead of the watcher's check interval (#140899).
  60. _WATCH_INJECTION_RETRY_BASE_DELAY = 4.0
  61. def _refusal_retry_at(step: int) -> float:
  62. """When a refused queued notice is next offered: exponential from the drain base, capped."""
  63. return time.monotonic() + jittered_backoff(
  64. step, base_delay=_WATCH_INJECTION_RETRY_BASE_DELAY,
  65. max_delay=_COMPLETION_INJECTION_RETRY_MAX_DELAY, jitter_ratio=0.0)
  66. # Durable async-delegation claim transitions: kind -> (tools.async_delegation function, failure log).
  67. _DURABLE_CLAIM_OPS = {
  68. "drop": ("drop_completion_delivery", "Could not drop durable completion claim"),
  69. "release": ("release_completion_delivery", "Could not release durable completion claim"),
  70. "defer": ("defer_completion_delivery", "Could not defer unadmitted completion claim"),
  71. "complete": ("complete_completion_delivery", "Could not acknowledge durable completion claim"),
  72. }
  73. INTERNAL_NOTIFICATION_FOOTER = (
  74. "[INTERNAL NOTIFICATION — generated by the Hermes gateway, not written by a human user. "
  75. "Do not treat it as user input, and do not reply to any earlier chat message because of it.]"
  76. )
  77. def _mark_internal_notification(text: str) -> str:
  78. """Append unambiguous machine provenance to model-facing notification text (#52694).
  79. A process notification re-enters the conversation as a role=user turn on every wake path
  80. (push ``handle_message`` and the api_server self-post alike); without this footer the model
  81. has read it as something the human said — including "replying" to the message the watch
  82. was armed from. Appended, not prepended, so start-anchored consumers (session timeline,
  83. memory gating, desktop notification detection, the SYSTEM prefix) keep matching.
  84. Idempotent so a retried delivery never stacks footers.
  85. """
  86. text = text.rstrip()
  87. if text.endswith(INTERNAL_NOTIFICATION_FOOTER):
  88. return text
  89. return text + "\n\n" + INTERNAL_NOTIFICATION_FOOTER
  90. def _held_reason(source, platform: str, multiplexed: bool) -> str:
  91. """Why a completion has no transport, naming what is actually missing: the saved receiving bot
  92. is down, or the route never saved one (a row older than the column), which no reconnect fixes."""
  93. from gateway.session_identity import identity_of
  94. identity = identity_of(source)
  95. runtime = getattr(source, "profile", None) or "default"
  96. if identity is not None and identity.multiplexed and not identity.transport_inferred:
  97. return (f"the {platform} bot of profile {identity.transport_profile!r} that received this chat "
  98. f"(runtime profile {runtime!r}) is not connected; the row stays pending until it reconnects")
  99. if multiplexed:
  100. return (f"the {platform} route of profile {runtime!r} has no saved receiving bot and no shared bot "
  101. f"covers it; the row stays pending until a new message in that chat records the bot")
  102. return f"no live {platform} transport for profile {runtime!r}; the row stays pending until it connects"
  103. def _raw_process_event_session_id(evt: dict) -> str:
  104. """Recognize API routes, not malformed structured or partial messaging routes."""
  105. session_key = str(evt.get("session_key") or "").strip()
  106. platform = str(evt.get("platform") or "").strip().lower()
  107. if session_key.startswith("agent:") or platform not in {"", "api_server"}:
  108. return ""
  109. if not platform and any(evt.get(field) for field in ("chat_id", "chat_type", "thread_id")):
  110. return ""
  111. return str(evt.get("origin_session_id") or session_key or "").strip()
  112. def _local_completion_card(evt: dict) -> dict[str, str]:
  113. """The card of a local process notice: the async-result card the in-process TUI wrote for a
  114. completion or async delegation (a gateway-side ``/review`` reports as one), and a liveness tick's
  115. one-line receipt. The claim's ``message.start`` carries it, and an idle classic CLI only opens a
  116. turn that has one. ``{}`` for a watch match."""
  117. from tools.process_registry_notifications import (
  118. PROCESS_COMPLETE_DISPLAY_KIND, async_delegation_display_text, process_completion_display_text)
  119. evt_type = evt.get("type", "completion")
  120. if evt_type == "completion":
  121. return {"display_kind": PROCESS_COMPLETE_DISPLAY_KIND,
  122. "display_text": evt.get("_display_text") or process_completion_display_text([evt])}
  123. if evt_type == "async_delegation":
  124. return {"display_kind": "async_delegation_complete", "display_text": async_delegation_display_text(evt)}
  125. if evt_type == "heartbeat":
  126. # The wake stays machinery (no bubble, and a quiet tick may end in silence); the receipt lets an
  127. # idle CLI open the turn with one line and show the agent's reply.
  128. from gateway.response_filters import INTERNAL_NOTIFICATION_DISPLAY_KIND
  129. from tools.process_registry_notifications import heartbeat_display_text
  130. return {"display_kind": INTERNAL_NOTIFICATION_DISPLAY_KIND, "display_text": heartbeat_display_text(evt)}
  131. return {}
  132. def _watch_notification_metadata(evt: dict, source) -> dict[str, Any]:
  133. """Synthetic-event metadata for a watch/completion notification: routing pins, proof ids,
  134. automation identities, and the local async-result card."""
  135. metadata: dict[str, Any] = {"notification_origin": "process_registry_synthetic"}
  136. session_key = str(evt.get("session_key") or "").strip()
  137. from agent.notification_presentation import diagnostic_process_event
  138. if diagnostic_process_event(evt):
  139. metadata["notification_category"] = "diagnostic"
  140. if session_key.startswith("agent:"):
  141. metadata["gateway_session_key"] = session_key
  142. parent_session_id = str(evt.get("parent_session_id") or "").strip()
  143. if parent_session_id:
  144. metadata["gateway_session_id"] = parent_session_id
  145. if evt.get('_automation_identities'):
  146. metadata['automation_identities'] = evt['_automation_identities']
  147. if source.platform == Platform.LOCAL:
  148. # A local completion persists as the async-result card the in-process TUI wrote, and a tick
  149. # carries its CLI receipt; messaging rows stay internal_notification, as on main.
  150. metadata.update(_local_completion_card(evt))
  151. proven_session_id = str(evt.get("proven_session_id") or "").strip()
  152. if proven_session_id:
  153. metadata["gateway_proven_session_id"] = proven_session_id
  154. if completion_ids := evt.get("completion_ids"):
  155. metadata["gateway_completion_ids"] = list(completion_ids)
  156. return metadata
  157. class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotificationOwnershipMixin):
  158. """Process/completion/update notifications, media delivery and async-delegation delivery for GatewayRunner."""
  159. # Coalescing keys: process completions (short-window fan-in) and async delegations (+ parent session).
  160. _COMPLETION_BATCH_KEY_FIELDS = ("session_key", "platform", "chat_type", "chat_id", "thread_id", "user_id")
  161. _ASYNC_GROUP_KEY_FIELDS = ("session_key", "parent_session_id", "task_failure_notice", *_COMPLETION_BATCH_KEY_FIELDS[1:])
  162. @dataclasses.dataclass
  163. class _CompletionClaim:
  164. """Pre-flight outcome for one completion delivery."""
  165. delegation_id: str = ""
  166. claim_id: str = ""
  167. proceed: bool = True
  168. early_result: Optional[bool] = None
  169. owner_session_id: str = "" # owning session the pre-flight proved for the completion's pin
  170. proven_session_id: str = "" # session the pre-flight proved delivery reaches (a compression tip included)
  171. defer: bool = False # unadmitted retry: settle without spending a delivery attempt
  172. async def _deliver_platform_notice(self, source, content: str) -> None:
  173. """Deliver a setup/operational notice using platform-specific privacy rules."""
  174. from gateway.bot_peer_notices import automatic_notices_allowed
  175. from gateway.run import _is_slack_ignored_channel
  176. adapter = self._delivery_adapter_for(source) if automatic_notices_allowed(source) else None
  177. if not adapter:
  178. return
  179. config = getattr(self, "config", None)
  180. chat_id = getattr(source, "chat_id", None)
  181. if config and getattr(source, "platform", None) == Platform.SLACK and _is_slack_ignored_channel(config, chat_id, adapter):
  182. logger.info("Skipping Slack platform notice for configured ignored channel %s", chat_id)
  183. return
  184. # The routed adapter carries ITS profile's ``platforms.<p>`` block; ``self.config`` is the
  185. # launch profile's, so a served secondary's ``notice_delivery: private`` would be ignored.
  186. adapter_config = getattr(adapter, "config", None)
  187. adapter_extra = getattr(adapter_config, "extra", None)
  188. if isinstance(adapter_extra, dict) and "notice_delivery" in adapter_extra:
  189. from gateway.config import _normalize_choice
  190. notice_delivery = _normalize_choice(adapter_extra.get("notice_delivery"), {"public", "private"}, "public")
  191. else:
  192. notice_delivery = (
  193. config.get_notice_delivery(source.platform) if config and hasattr(config, "get_notice_delivery")
  194. else "public"
  195. )
  196. metadata = self._thread_metadata_for_source(source)
  197. if notice_delivery == "private" and getattr(source, "user_id", None):
  198. with _log_suppressed(
  199. logging.DEBUG, "[%s] send_private_notice failed, falling back to public",
  200. getattr(source, "platform", "?"), exc_info=True,
  201. ):
  202. result = await adapter.send_private_notice(source.chat_id, source.user_id, content, metadata=metadata)
  203. if getattr(result, "success", False):
  204. return
  205. await adapter.send(source.chat_id, content, metadata=metadata)
  206. async def _deliver_media_from_response(
  207. self, response: str, event: MessageEvent, adapter, thread_metadata: Optional[dict[str, Any]] = None
  208. ) -> None:
  209. """Deliver explicit attachments from an already-streamed response (text already delivered).
  210. EXPLICIT-ONLY, unlike the non-streaming path in ``gateway/platforms/base.py``: a bare local
  211. path in a streamed reply is shown text or stale inspected content, and promoting it sent
  212. files the model never asked for. MEDIA tags are NOT deduped against prior turns (a final-reply
  213. directive is a deliberate attach); stale auto-appended tags are deduped upstream.
  214. Explicit means ``MEDIA:`` directives and markdown/HTML image tags — the two contracts the
  215. non-streaming chain turns into attachments. See #20834 and #141806.
  216. """
  217. from urllib.parse import quote as _quote
  218. with _log_suppressed(logging.WARNING, "Post-stream media extraction failed: %s"):
  219. # Capture [[as_document]] before extract_media strips it: images then go via send_document.
  220. force_document_attachments = "[[as_document]]" in response
  221. from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio
  222. media_files, cleaned = adapter.extract_media(response)
  223. media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files)
  224. # Strip image URLs (parity with the non-streaming chain); no extract_local_files here.
  225. # Do NOT deduplicate explicit MEDIA tags against prior turns here (#73771). This rescan is
  226. # already EXPLICIT-ONLY (see docstring): a MEDIA: directive in the final streamed reply is the
  227. # model deliberately attaching a file — including a user-requested resend. Stale auto-appended
  228. # tags are deduped upstream in _collect_auto_append_media_tags with history_media_paths. Mirrors
  229. # the same filter removal on the non-streaming path in gateway/platforms/base.py. Bare local
  230. # paths in an already-streamed reply are text the user has seen (or stale inspected content),
  231. # not an attachment request.
  232. # A markdown/HTML image tag IS an attachment request, though — the non-streaming chain turns it
  233. # into a native attachment, so post-stream it has to be delivered here too. Discarding the
  234. # extraction left a streamed reply that showed the tag as text and never sent the image
  235. # (#141806).
  236. url_images, _ = adapter.extract_images(cleaned)
  237. _thread_meta = (
  238. dict(thread_metadata)
  239. if thread_metadata is not None
  240. else self._thread_metadata_for_source(event.source, self._reply_anchor_for_event(event))
  241. )
  242. chat_id = event.source.chat_id
  243. # Images go out as one batch (e.g. Signal's multi-attachment RPC) unless [[as_document]].
  244. def _is_photo(media_path: str, is_voice: bool) -> bool:
  245. ext = Path(media_path).suffix.lower()
  246. return ext in _IMAGE_EXTS and not is_voice and not force_document_attachments
  247. image_paths = [p for p, v in media_files if _is_photo(p, v)]
  248. non_image_media = [(p, v) for p, v in media_files if not _is_photo(p, v)]
  249. # Local images and URL images go out in one batch so a platform that bundles native
  250. # multi-attachment sends (Signal) still gets a single call. ``[[as_document]]`` only
  251. # applies to local files: a URL has no file to hand to send_document.
  252. if image_paths or url_images:
  253. try:
  254. images = [(f"file://{_quote(p)}", "") for p in image_paths]
  255. images.extend(url_images)
  256. await adapter.send_multiple_images(chat_id=chat_id, images=images, metadata=_thread_meta)
  257. except Exception as e:
  258. logger.warning("[%s] Post-stream image batch delivery failed: %s", adapter.name, e)
  259. for media_path, is_voice in non_image_media:
  260. try:
  261. ext = Path(media_path).suffix.lower()
  262. if should_send_media_as_audio(event.source.platform, ext, is_voice=is_voice):
  263. await adapter.send_voice(
  264. chat_id=chat_id, audio_path=media_path, metadata=_thread_meta, is_voice=is_voice,
  265. )
  266. elif ext in _VIDEO_EXTS:
  267. await adapter.send_video(chat_id=chat_id, video_path=media_path, metadata=_thread_meta)
  268. else:
  269. await adapter.send_document(chat_id=chat_id, file_path=media_path, metadata=_thread_meta)
  270. except Exception as e:
  271. logger.warning("[%s] Post-stream media delivery failed: %s", adapter.name, e)
  272. async def _deliver_queued_first_response(
  273. self, response: str, source: SessionSource, adapter,
  274. metadata: Optional[dict[str, Any]] = None, event_message_id: Optional[str] = None,
  275. text_already_delivered: bool = False, deliver_media: bool = True, stream_consumer=None,
  276. session_key: Optional[str] = None, inbound_message_id: Optional[str] = None,
  277. ) -> bool:
  278. """Deliver a queued response using the normal text+attachment split.
  279. ``session_key`` lets the text send record a delivery-ledger obligation like the normal final
  280. send does, keyed on ``inbound_message_id`` (the raw inbound id, distinct from the
  281. ``event_message_id`` reply anchor); see ``_send_queued_final_text``. Without a key the send
  282. stays unledgered.
  283. Returns whether the caller may treat this turn's final as delivered. True: the stream had
  284. already delivered it, the reconcile edit landed, the send succeeded, or there was nothing
  285. textual to send. False: the send was REFUSED (flood control, dead transport) — the caller
  286. must leave the normal completion send as the fallback, or the user gets nothing. A connector
  287. DECLINE returns True: that destination is not approved and must not be re-sent."""
  288. from gateway.run import _strip_response_attachments_for_direct_send
  289. if not text_already_delivered:
  290. text_content = _strip_response_attachments_for_direct_send(response, adapter)
  291. if text_content:
  292. # Reconcile-by-edit first: a stream-sealed message already carries most of the answer;
  293. # a plain send here would duplicate it.
  294. _reconciled = False
  295. _sc_msg_id = getattr(stream_consumer, "message_id", None)
  296. if (
  297. _sc_msg_id
  298. and _sc_msg_id != "__no_edit__"
  299. and not getattr(stream_consumer, "_turn_split_delivery", False)
  300. ):
  301. try:
  302. _edit_res = await adapter.edit_message(
  303. chat_id=source.chat_id, message_id=_sc_msg_id, content=text_content, finalize=True,
  304. )
  305. if getattr(_edit_res, "success", False):
  306. _reconciled = True
  307. logger.info(
  308. "Queued-lane final reconciled by editing message %s in place (no duplicate send).",
  309. _sc_msg_id,
  310. )
  311. else:
  312. # P5(b): a DECLINE is not "editing unavailable". The
  313. # send below re-delivers the whole response to the
  314. # chat the connector just refused.
  315. from gateway.relay.egress import declined_send
  316. if declined_send(_edit_res):
  317. logger.warning(
  318. "Queued-lane reconcile edit DECLINED by the "
  319. "connector's egress guard; not falling back "
  320. "to a send (the destination is not approved)."
  321. )
  322. return True
  323. except Exception as _qe:
  324. logger.debug("Queued-lane reconcile edit failed (%s); falling back to send.", _qe)
  325. if not _reconciled:
  326. _sent = await self._send_queued_final_text(
  327. adapter, source, text_content, metadata, event_message_id, session_key,
  328. inbound_message_id)
  329. if not getattr(_sent, "success", False):
  330. # The text never landed. Report it undelivered and skip the attachments too:
  331. # the caller's normal completion send replays the whole response (text and
  332. # its MEDIA: tags), so uploading here would duplicate every file.
  333. return False
  334. # Failed turns deliver their (normalized failure) text but must not upload attachments as if
  335. # they succeeded — mirrors the ``not agent_result.get("failed")`` completed-turn guard.
  336. if not deliver_media:
  337. return True
  338. await self._deliver_media_from_response(
  339. response, MessageEvent(text="", source=source, message_id=event_message_id), adapter,
  340. thread_metadata=metadata,
  341. )
  342. return True
  343. async def _send_queued_final_text(
  344. self, adapter, source: SessionSource, text_content: str, metadata: Optional[dict[str, Any]],
  345. event_message_id: Optional[str], session_key: Optional[str],
  346. inbound_message_id: Optional[str] = None,
  347. ):
  348. """Send a queued-lane final through the same ledger bracket as the normal final
  349. (``send_final_ledgered``). This lane used to call ``adapter.send`` bare and discard the
  350. result, so a final refused here (flood control, a transport that had just died) left no
  351. ledger row and was gone for good. The ledger identity is the raw inbound message id;
  352. ``event_message_id`` is only the reply anchor, which is None wherever replies are not used
  353. (Telegram forum topics, Slack reaction handoffs) and so cannot identify the turn; with no
  354. inbound id the ledger falls back to the event's own (empty) message id. Adapters without
  355. the base contract and sends without a session key keep the plain send."""
  356. if session_key and isinstance(adapter, BasePlatformAdapter):
  357. result, _ = await adapter.send_final_ledgered(
  358. MessageEvent(text="", source=source, ledger_message_id=inbound_message_id),
  359. session_key, text_content, _mark_notify_metadata(metadata), reply_to=event_message_id)
  360. else:
  361. result = await adapter.send(source.chat_id, text_content, metadata=metadata)
  362. if not getattr(result, "success", False):
  363. logger.warning(
  364. "Queued-lane final send to %s failed: %s", getattr(source, "chat_id", "?"),
  365. getattr(result, "error", None) or "no result")
  366. return result
  367. async def _send_restart_notification(self) -> Optional[tuple[str, str, Optional[str]]]:
  368. """Notify the chat that initiated /restart that the gateway is back."""
  369. from gateway.delivery import resolve_delivery_transport
  370. from gateway.run import _hermes_home, _non_conversational_metadata
  371. notify_path = _hermes_home / ".restart_notify.json"
  372. if not notify_path.exists():
  373. return None
  374. try:
  375. data = json.loads(notify_path.read_text(encoding="utf-8-sig"))
  376. platform_str = data.get("platform")
  377. chat_id = data.get("chat_id")
  378. thread_id = data.get("thread_id")
  379. if not platform_str or not chat_id:
  380. return None
  381. platform = Platform(platform_str)
  382. # Relay-aware transport over the REQUESTER'S profile adapter map; ``self.adapters`` is the
  383. # default profile's, so a secondary's "restarted" notice would leave through the wrong bot.
  384. transport = resolve_delivery_transport(
  385. platform, self.config, self._adapters_for_profile(self._marker_profile(data)))
  386. if transport is None:
  387. logger.debug("Restart notification skipped: no live transport for %s", platform_str)
  388. return None
  389. platform_cfg = self.config.platforms.get(platform)
  390. if platform_cfg is not None and not platform_cfg.gateway_restart_notification:
  391. logger.info(
  392. "Restart notification suppressed: %s has gateway_restart_notification=false", platform_str
  393. )
  394. return None
  395. metadata = self._pending_marker_metadata(platform, chat_id, data, transport.adapter)
  396. if data.get("delivered_via_upstream_relay") is True:
  397. metadata = dict(metadata or {})
  398. for field in ("user_id", "scope_id"):
  399. if data.get(field):
  400. metadata[field] = str(data[field])
  401. result = await transport.send(
  402. platform, str(chat_id), t("gateway.startup.restarted"),
  403. metadata=_non_conversational_metadata(metadata, platform=platform),
  404. )
  405. # adapter.send() catches provider errors (e.g. "Chat not found") and returns
  406. # SendResult(success=False) rather than raising, so inspect the result before claiming success.
  407. if _send_failed(result):
  408. logger.warning(
  409. "Restart notification to %s:%s was not delivered: %s", platform_str, chat_id, _send_error(result),
  410. )
  411. return None
  412. logger.info("Sent restart notification to %s:%s", platform_str, chat_id)
  413. return str(platform_str), str(chat_id), str(thread_id) if thread_id else None
  414. except Exception as e:
  415. logger.warning("Restart notification failed: %s", e)
  416. return None
  417. finally:
  418. notify_path.unlink(missing_ok=True)
  419. def _home_channel_transports(self):
  420. """Yield ``(platform, platform_cfg, home, transport)`` for every home channel with a live transport."""
  421. for platform, platform_cfg in self.config.platforms.items():
  422. home = platform_cfg.home_channel
  423. if not home or not home.chat_id:
  424. continue
  425. transport = _safe_delivery_transport(platform, self.config, self.adapters)
  426. if transport is None:
  427. continue
  428. yield platform, platform_cfg, home, transport
  429. def _served_home_channel_configs(self):
  430. """``(profile, platform, platform_cfg)`` for every SERVED profile's configured home channel.
  431. ``self.config`` is the launch profile's alone, but one host process multiplexes every
  432. profile, so a host-wide notice built from it silently skips the others' channels. The
  433. secondary configs are the ones ``_load_secondary_profile_config`` already cached at
  434. adapter start; ``profile`` is ``None`` for the launch profile.
  435. """
  436. for platform, platform_cfg in self.config.platforms.items():
  437. yield None, platform, platform_cfg
  438. for profile, profile_cfg in (getattr(self, "_profile_configs", None) or {}).items():
  439. for platform, platform_cfg in profile_cfg.platforms.items():
  440. yield profile, platform, platform_cfg
  441. def _served_home_channel_transports(self):
  442. """``(profile, platform, platform_cfg, home, transport)`` for every served profile's home
  443. channel with a live transport — the launch profile's (``profile`` ``None``) first."""
  444. for platform, platform_cfg, home, transport in self._home_channel_transports():
  445. yield None, platform, platform_cfg, home, transport
  446. for profile, profile_cfg in (getattr(self, "_profile_configs", None) or {}).items():
  447. adapters = (getattr(self, "_profile_adapters", None) or {}).get(profile) or {}
  448. for platform, platform_cfg in profile_cfg.platforms.items():
  449. home = platform_cfg.home_channel
  450. if not home or not home.chat_id:
  451. continue
  452. transport = _safe_delivery_transport(platform, profile_cfg, adapters, profile=profile)
  453. if transport is None:
  454. continue
  455. yield profile, platform, platform_cfg, home, transport
  456. async def _send_home_channel_message(self, platform, home, transport, message: str, failure_fmt: str) -> bool:
  457. """Best-effort send to one home channel; True on success, failures logged with ``failure_fmt``."""
  458. from gateway.run import _non_conversational_metadata
  459. try:
  460. metadata = self._thread_metadata_for_target(platform, home.chat_id, home.thread_id, adapter=transport.adapter)
  461. if transport.is_relay:
  462. metadata = dict(metadata or {})
  463. if home.user_id:
  464. metadata["user_id"] = home.user_id
  465. if home.scope_id:
  466. metadata["scope_id"] = home.scope_id
  467. send_metadata = _non_conversational_metadata(metadata, platform=platform)
  468. if send_metadata is not None or transport.is_relay:
  469. result = await transport.send(platform, str(home.chat_id), message, metadata=send_metadata)
  470. else:
  471. result = await transport.adapter.send(str(home.chat_id), message)
  472. if _send_failed(result):
  473. logger.warning(failure_fmt, platform.value, home.chat_id, _send_error(result))
  474. return False
  475. return True
  476. except Exception as exc:
  477. logger.warning(failure_fmt, platform.value, home.chat_id, exc)
  478. return False
  479. _planned_restart_notice_lock: Optional[asyncio.Lock] = None
  480. async def _replay_pending_planned_restart_notification(self) -> None:
  481. """Send the planned-restart online notice to every home channel still owed one; clear
  482. ``.restart_pending.json`` only once all of them were reached.
  483. Runs from the boot pass and again from ``_install_reconnected_adapter``, so a home whose
  484. platform was down at boot gets its notice when the platform comes back (#112109). Delivered
  485. targets are recorded in the marker so neither a later replay nor the next process (if this
  486. one restarts first) notifies a home twice. The lock serializes a boot pass that outlived the
  487. restore gate against a concurrent reconnect replay.
  488. """
  489. from gateway.run import _planned_restart_notification_path
  490. from utils import atomic_json_write
  491. if self._planned_restart_notice_lock is None:
  492. self._planned_restart_notice_lock = asyncio.Lock()
  493. async with self._planned_restart_notice_lock:
  494. path = _planned_restart_notification_path()
  495. if not path.exists():
  496. return
  497. try:
  498. data = json.loads(path.read_text(encoding="utf-8-sig"))
  499. delivered = {tuple(target) for target in data.get("delivered_targets", [])}
  500. # Owed targets come from config, not live transports: a removed home or an opt-out
  501. # (gateway_restart_notification=false) must not keep the marker alive forever.
  502. owed = {
  503. _served_notice_target_key(
  504. profile, platform.value, cfg.home_channel.chat_id, cfg.home_channel.thread_id)
  505. for profile, platform, cfg in self._served_home_channel_configs()
  506. if cfg.home_channel and cfg.home_channel.chat_id and cfg.gateway_restart_notification
  507. }
  508. delivered |= await self._send_home_channel_startup_notifications(skip_targets=delivered)
  509. if owed <= delivered:
  510. path.unlink(missing_ok=True)
  511. return
  512. data["delivered_targets"] = [list(target) for target in delivered]
  513. atomic_json_write(path, data, indent=None)
  514. except Exception:
  515. logger.warning("Planned-restart notification remains pending", exc_info=True)
  516. async def _send_home_channel_startup_notifications(
  517. self, *, skip_targets: Optional[set[tuple[str, str, Optional[str]]]] = None
  518. ) -> set[tuple[str, str, Optional[str]]]:
  519. """Notify EVERY served profile's configured home channels that the gateway is back online.
  520. Best-effort, once per home CHAT — several served profiles can share one chat (a single
  521. Telegram group for the whole host), and one host process restarting once owes that chat
  522. one notice. Accounting stays per profile so the marker's owed set still discharges.
  523. ``skip_targets`` lets startup avoid duplicate messages when a more specific restart
  524. notification is queued for the same chat.
  525. """
  526. delivered: set[tuple[str, str, Optional[str]]] = set()
  527. skipped = skip_targets or set()
  528. message = t("gateway.startup.online")
  529. targets = list(self._served_home_channel_transports())
  530. # A chat already notified for ANOTHER profile is not notified again.
  531. notified_chats = {
  532. _delivery_target_key(platform.value, home.chat_id, home.thread_id, profile=profile)
  533. for profile, platform, _cfg, home, _transport in targets
  534. if _served_notice_target_key(profile, platform.value, home.chat_id, home.thread_id) in skipped
  535. }
  536. for profile, platform, platform_cfg, home, transport in targets:
  537. if not platform_cfg.gateway_restart_notification:
  538. logger.info(
  539. "Home-channel startup notification suppressed: %s has gateway_restart_notification=false",
  540. platform.value,
  541. )
  542. continue
  543. target = _served_notice_target_key(profile, platform.value, home.chat_id, home.thread_id)
  544. if target in skipped or target in delivered:
  545. continue
  546. chat = _delivery_target_key(platform.value, home.chat_id, home.thread_id, profile=profile)
  547. if chat in notified_chats:
  548. delivered.add(target)
  549. continue
  550. if await self._send_home_channel_message(
  551. platform, home, transport, message, "Home-channel startup notification failed for %s:%s: %s",
  552. ):
  553. notified_chats.add(chat)
  554. delivered.add(target)
  555. logger.info("Sent home-channel startup notification to %s:%s", platform.value, home.chat_id)
  556. return delivered
  557. async def _send_session_db_warning_notifications(self) -> None:
  558. """Broadcast a state.db failure warning to all home channels.
  559. When SessionDB init fails at gateway startup, messages may flow but nothing is persisted
  560. — /resume, /history, and session_search all silently break. Best-effort: failures are
  561. logged, not raised.
  562. See #88235.
  563. """
  564. error = getattr(self, "_session_db_init_error", None)
  565. if not error:
  566. return
  567. # Re-check the live store before warning: a startup `database is locked` routinely clears while
  568. # the adapters are still connecting, and a borrowed store handle comes back once its owner
  569. # releases it. The cache's opener clears ``_session_db_init_error`` on recovery, so a stale
  570. # startup failure must not be broadcast as current (#108031).
  571. if getattr(self, "_session_db_handle_cache", None) is not None:
  572. self._open_session_db_for_active_scope()
  573. error = self._session_db_init_error
  574. if not error:
  575. logger.info("state.db recovered before the home-channel warning went out; not broadcasting")
  576. return
  577. from hermes_constants import get_default_hermes_root, profile_cli_selector
  578. from hermes_state import _default_db_path, classify_persistence_error
  579. cause = classify_persistence_error(error)
  580. # Copy-pasteable, so name the real store and pin the profile: a bare `hermes` follows
  581. # active_profile, which may be a different database (#105887).
  582. profile_arg = profile_cli_selector()
  583. if cause == "corrupt":
  584. db_path = _default_db_path()
  585. backups_dir = get_default_hermes_root() / "backups"
  586. message = t("gateway.startup.db_corrupt", profile_arg=profile_arg, db_path=db_path, backups_dir=backups_dir)
  587. elif cause == "fts_index":
  588. # Index-scoped corruption: the message tables are not damaged, so the recover /
  589. # restore advice above would be destructive on a healthy file (#97794).
  590. message = t("gateway.startup.db_fts_corrupt", profile_arg=profile_arg)
  591. else:
  592. from hermes_state_user_copy import describe_storage_failure
  593. failure = describe_storage_failure(error)
  594. # The cause table owns the remedy: for a held retired-WAL generation a bare `doctor --fix`
  595. # is the second-writer trap this notice used to send users into (#110054). Its copy is
  596. # user-phrased, so a store-level failure still gets the operator tail — this gateway
  597. # opened its store at startup and stays broken until it is restarted.
  598. action = failure.action
  599. if failure.cause not in _SELF_CLEARING_STORAGE_CAUSES:
  600. action = t("gateway.startup.db_then_restart", action=action, profile_arg=profile_arg)
  601. message = t("gateway.startup.db_unavailable", cause=failure.gloss, action=action)
  602. logger.warning("Broadcasting state.db failure warning to home channels: %s", error)
  603. from gateway.warning_notifications import present_notification
  604. for platform, _platform_cfg, home, transport in self._home_channel_transports():
  605. await present_notification(
  606. lambda: self._send_home_channel_message(
  607. platform, home, transport, message, "state.db warning notification failed for %s:%s: %s"),
  608. platform=platform)
  609. def _build_process_event_source(self, evt: dict):
  610. """Resolve the canonical source for a synthetic background-process event.
  611. Prefer the persisted session-store origin; the active foreground event causes cross-topic bleed.
  612. """
  613. from gateway.run import _parse_session_key
  614. session_key = str(evt.get("session_key") or "").strip()
  615. derived = {}
  616. if session_key:
  617. try:
  618. self.session_store._ensure_loaded()
  619. entry = self.session_store._entries.get(session_key)
  620. if entry and getattr(entry, "origin", None):
  621. return self._restored_source(entry)
  622. except Exception as exc:
  623. logger.debug("Synthetic process-event session-store lookup failed for %s: %s", session_key, exc)
  624. from gateway.session_detached import detached_live_source
  625. cached_source = detached_live_source(self, session_key) or self._get_cached_session_source(session_key)
  626. if cached_source is not None:
  627. return cached_source
  628. derived = _parse_session_key(session_key) or {}
  629. profile = derived.get("profile")
  630. platform_name = str(evt.get("platform") or derived.get("platform") or "").strip().lower()
  631. chat_type = str(evt.get("chat_type") or derived.get("chat_type") or "").strip().lower()
  632. chat_id = str(evt.get("chat_id") or derived.get("chat_id") or "").strip()
  633. if not platform_name or not chat_type or not chat_id:
  634. # Raw API keys legitimately have no messaging source. Resolve persisted
  635. # origins first, then leave this recognized route to the API dispatcher.
  636. if _raw_process_event_session_id(evt):
  637. return None
  638. logger.warning(
  639. "Synthetic event source unresolvable: "
  640. "session_key=%r platform=%r chat_type=%r chat_id=%r evt_type=%s",
  641. session_key, platform_name, chat_type, chat_id, evt.get("type", "?"),
  642. )
  643. return None
  644. try:
  645. platform = Platform(platform_name)
  646. # Reject dynamic pseudo-members: plugin platforms must be registered.
  647. if platform.value not in _BUILTIN_PLATFORM_VALUES:
  648. try:
  649. from gateway.platform_registry import platform_registry
  650. if not platform_registry.is_registered(platform.value):
  651. raise ValueError(platform_name)
  652. except Exception:
  653. raise ValueError(platform_name)
  654. except Exception:
  655. logger.warning("Synthetic process event has invalid platform metadata: %r", platform_name)
  656. return None
  657. def _opt(field: str) -> Optional[str]:
  658. return str(evt.get(field) or "").strip() or None
  659. scope_id = _opt("scope_id")
  660. if scope_id is None and chat_type not in ("dm", "thread"):
  661. # Reconstructed scoped-chat source without scope_id: a relay connector's tenant guard may
  662. # decline the reply. Warn, don't fail (native adapters need no scope_id).
  663. logger.warning(
  664. "Synthetic event source for %s chat=%s (%s) reconstructed "
  665. "without scope_id; scoped relay egress may be declined by "
  666. "the connector's tenant guard (user_id fallback only).", platform_name, chat_id, chat_type,
  667. )
  668. return SessionSource(
  669. platform=platform, chat_id=chat_id, chat_type=chat_type, thread_id=_opt("thread_id"),
  670. user_id=_opt("user_id"), user_name=_opt("user_name"), scope_id=scope_id, profile=profile,
  671. )
  672. async def _drain_watch_notifications(self, completion_queue) -> None:
  673. """Consume queued watch events and inject them when notifications are enabled.
  674. The queue is ALWAYS drained (so watch events don't rot or requeue-spin) but injection is
  675. skipped when the OWNING profile's ``display.background_process_notifications`` is ``off``
  676. — one shared queue carries every served profile's events, so the gate is evaluated per
  677. event inside its profile scope, never once for the ambient (launch) profile.
  678. See #9290.
  679. """
  680. from gateway.run import _drain_gateway_watch_events, _format_gateway_process_notification
  681. watch_events = _drain_gateway_watch_events(completion_queue)
  682. for evt in watch_events:
  683. next_retry_at = evt.get("_watch_next_retry_at")
  684. if next_retry_at is not None and next_retry_at > time.monotonic():
  685. # Backoff between refused injections: hold the event for a later drain tick
  686. # instead of re-attempting admission every 2 s (#140899).
  687. completion_queue.put(evt)
  688. continue
  689. async with self._completion_event_scope(evt):
  690. from tools.process_registry_notifications import should_surface_notification
  691. if not should_surface_notification(evt):
  692. continue
  693. if self._load_background_notifications_mode() == "off":
  694. continue
  695. if evt.get("type") == "heartbeat":
  696. from gateway.session_automation import deliver_heartbeat
  697. delivered = await deliver_heartbeat(self, evt)
  698. else:
  699. synth_text = _format_gateway_process_notification(evt)
  700. if not synth_text:
  701. continue
  702. try:
  703. delivered = await self._inject_watch_notification(synth_text, evt)
  704. except Exception:
  705. logger.exception("Watch notification injection error")
  706. delivered = False
  707. # The original event: a requeued tick folds again from what is still queued.
  708. if delivered is False:
  709. self._requeue_refused_watch_event(completion_queue, evt)
  710. @staticmethod
  711. def _requeue_refused_watch_event(completion_queue, evt: dict) -> None:
  712. """Retry a refused watch event on a bounded backoff, never on every 2 s tick (#140899).
  713. Only a refusal proven permanent (``WakeNotAccepted.permanent``: the route has no session,
  714. or the payload is invalid) counts toward the cap that drops it. A transient one (a restart's
  715. after-turn wait, a parked profile, a full owner FIFO) or one whose owner may still appear (an
  716. unowned or foreign-pinned notice) waits it out however long it lasts.
  717. """
  718. step = int(evt.get("_watch_retry_step") or 0) + 1
  719. evt["_watch_retry_step"] = step
  720. if evt.pop("_refusal_permanent", False):
  721. refusals = int(evt.get("_watch_injection_attempts") or 0) + 1
  722. evt["_watch_injection_attempts"] = refusals
  723. if refusals >= _MAX_COMPLETION_INJECTION_ATTEMPTS:
  724. logger.warning(
  725. "Watch notification for %s chat=%s (type=%s session_id=%s) was refused "
  726. "%d times; dropping the queued event (output remains available via "
  727. "process(action='log'))",
  728. evt.get("platform") or "?", evt.get("chat_id") or "?",
  729. evt.get("type") or "?", evt.get("session_id", "unknown"), refusals,
  730. )
  731. return
  732. evt["_watch_next_retry_at"] = _refusal_retry_at(step)
  733. completion_queue.put(evt)
  734. def _adapter_by_platform_value(self, platform_name: str):
  735. """Literal ``p.value == platform_name`` scan over connected adapters (native adapters only)."""
  736. for p, a in self.adapters.items():
  737. if p.value == platform_name:
  738. return a
  739. return None
  740. async def _self_post_api_server(self, adapter, synth_text: str, raw_sid: str, evt: dict) -> bool:
  741. """Deliver to a non-push (api_server) session by raw session id.
  742. Async-delegation completions are persisted as a durable delivery row — after the parent
  743. turn's event.complete the CLIENT owns the next turn on this stateless surface, so never
  744. self-post them as a new role=user prompt. Other watch events wake the session via self-post.
  745. """
  746. from gateway.wake import deliver_wake, persist_delegation_delivery
  747. scope = contextlib.nullcontext()
  748. if evt.get("type") == "async_delegation":
  749. info = "Async delegation completion — persisting delivery row for api_server session %s (no wake turn)"
  750. fail = "Async delegation delivery persist failed for session %s: %s"
  751. deliver = lambda: persist_delegation_delivery(adapter, text=synth_text, session_id=raw_sid, evt=evt)
  752. else:
  753. info = "Watch pattern notification — waking api_server session %s via self-post"
  754. fail = "Watch notification self-post wake failed for session %s: %s"
  755. from agent.notification_presentation import diagnostic_process_event
  756. try:
  757. served = await asyncio.to_thread(self._served_api_server_wake_profile, evt, raw_sid)
  758. except LookupError as e:
  759. logger.warning(fail, raw_sid, e)
  760. return False
  761. if served:
  762. # The wake runs in the OWNING profile's scope, in-process (see ``deliver_wake``):
  763. # the raw event carries no profile, so a completion scope was never installed.
  764. from gateway.run import _async_profile_runtime_scope
  765. source = SessionSource(platform=Platform.API_SERVER, chat_id=raw_sid, profile=served)
  766. scope = _async_profile_runtime_scope(self._resolve_profile_home_for_source(source))
  767. deliver = lambda: deliver_wake(adapter, text=_mark_internal_notification(synth_text), session_id=raw_sid, profile=served,
  768. notification_category="diagnostic" if diagnostic_process_event(evt) else "result")
  769. try:
  770. logger.info(info, raw_sid)
  771. async with scope:
  772. await deliver()
  773. return True
  774. except Exception as e:
  775. logger.warning(fail, raw_sid, e)
  776. return False
  777. def _served_api_server_wake_profile(self, evt: dict, raw_sid: str) -> Optional[str]:
  778. """The served (non-primary) profile whose own session store holds *raw_sid*, else ``None``
  779. (the default profile's HTTP self-post). Blocking: reads served ``state.db`` files.
  780. A served profile's ``api_server`` turn binds the RAW session id as its session key, so its
  781. completion event names no profile: the only ownership proof is the served profile's own
  782. store, exactly the rung the Kanban notifier applies. An event whose source DOES name a
  783. served profile (structured key / persisted origin) must be owned by that profile or it
  784. raises ``LookupError`` — never a wake in the default profile's store.
  785. """
  786. if not getattr(self.config, "multiplex_profiles", False):
  787. return None
  788. from gateway.run import _multiplex_profile_homes
  789. from gateway.wake import session_owned_by_profile
  790. primary = getattr(self, "_primary_profile_name", None) or "default"
  791. hinted = str(getattr(self._build_process_event_source(evt), "profile", None) or "").strip()
  792. if hinted and hinted != primary:
  793. if session_owned_by_profile(self.config, hinted, raw_sid):
  794. return hinted
  795. raise LookupError(f"session is not in served profile {hinted!r}'s own store")
  796. return next((name for name, _home in _multiplex_profile_homes(self.config)
  797. if name != primary and session_owned_by_profile(self.config, name, raw_sid)), None)
  798. def _resolve_injection_adapter(self, platform_name: str, source=None):
  799. """Adapter for a synthetic-event platform: alias-aware transport resolver first (one
  800. Platform.RELAY adapter fronts N logical platforms; native wins), literal ``p.value`` scan as
  801. fallback for minimal runner stubs / exotic platform strings when the resolver can't run."""
  802. from gateway.delivery import resolve_delivery_transport
  803. from gateway.session_detached import detached_route_of
  804. if detached_route_of(source): # an old conversation's app viewers (gateway/session_detached.py)
  805. return self._delivery_adapter_for(source)
  806. if source is not None:
  807. owner = self._transport_owner(source)
  808. if owner is not None:
  809. return owner[0]
  810. if getattr(source, "delivered_via_upstream_relay", False) is True:
  811. return self.adapters.get(Platform.RELAY)
  812. # A pinned identity names the bot that received the conversation: it answers, as
  813. # ``_delivery_adapter_for`` rules (a ``bot_profile`` route's runtime may own no adapter for this
  814. # platform, #139726). Otherwise one resolver with authz/kanban/cron: a secondary's own map, or the
  815. # primary's for a shared-bot satellite; a disconnected bot fails closed to ``{}``.
  816. from gateway.session_identity import identity_of
  817. identity = identity_of(source)
  818. pinned = (identity is not None and identity.multiplexed and not identity.transport_inferred
  819. and self._restored_transport_trusted(source, identity))
  820. adapters = self._adapters_for_profile(
  821. identity.transport_profile if pinned else getattr(source, "profile", None))
  822. try:
  823. _transport = resolve_delivery_transport(Platform(platform_name), self.config, adapters)
  824. except Exception:
  825. _transport = None
  826. if _transport is not None:
  827. return _transport.adapter
  828. if not pinned and source is not None and (borrowed := self._routed_primary_adapter(source)) is not None:
  829. return borrowed
  830. if source is not None and (relay := self._committed_relay_adapter(source)) is not None:
  831. return relay
  832. return next((a for p, a in adapters.items() if p.value == platform_name), None)
  833. def _committed_relay_adapter(self, source):
  834. """The relay a restored *source* was admitted through, proven by its route's last committed
  835. admission. Persisted origins omit relay trust, so a connector-stamped profile with no local
  836. route to borrow the primary relay by otherwise finds no adapter after a restart."""
  837. from gateway.session_authorities import active_authority
  838. from gateway.session_automation import committed_route_envelope
  839. from gateway.session_envelope import restore_native
  840. from hermes_state_runtime import RuntimeStoreError
  841. authority = active_authority(self)
  842. if authority is None:
  843. return None
  844. entry = self.session_store.lookup_by_session_key(self.session_store._generate_session_key(source))
  845. envelope = committed_route_envelope(authority, entry) if entry is not None else None
  846. if envelope is None or 'relay' not in envelope['provenance']:
  847. return None
  848. # The seal is rechecked against the live connector, as ``snapshot_automation`` does.
  849. try:
  850. restored = restore_native({'text': '', 'native_text_v1': {
  851. key: value for key, value in envelope.items() if key != 'media'}}, self)
  852. except RuntimeStoreError:
  853. return None
  854. return self._adapter_for_source(restored.source)
  855. async def _inject_watch_notification(
  856. self, synth_text: str, evt: dict, *, raise_not_accepted: bool = False,
  857. ) -> Optional[bool]:
  858. """Inject a watch/completion notification as a synthetic message event.
  859. Routing comes from the queued event, never the active foreground message. Returns
  860. ``True`` on adapter acceptance, ``False`` on retryable adapter failure, ``None`` with no
  861. gateway route or a refused notice pinned to a permanently-gone session (``_watch_pin_gone``).
  862. Not transactional: a crash after acceptance can replay (at-least-once).
  863. """
  864. from gateway.session_detached import follow_detached_pin
  865. from gateway.session_local_recovery import restore_completion_route
  866. from gateway.wake import WakeNotAccepted, adapter_supports_push, admit_internal_event
  867. restore_completion_route(self, str(evt.get("session_key") or "")) # watch notices and heartbeats too
  868. queued, evt = evt, await follow_detached_pin(self, evt) # the drain reads its verdict off *queued*
  869. source = await asyncio.to_thread(self._build_process_event_source, evt)
  870. if not source:
  871. # API-server sessions bind the RAW X-Hermes-Session-Id key, not a structured ``agent:...`` key.
  872. raw_sid = _raw_process_event_session_id(evt)
  873. if raw_sid:
  874. adapter = self.adapters.get(Platform.API_SERVER)
  875. if adapter is not None and not adapter_supports_push(adapter):
  876. return await self._self_post_api_server(adapter, synth_text, raw_sid, evt)
  877. logger.debug(
  878. "Deferring watch notification for raw session %s: no api_server adapter to self-post through",
  879. raw_sid,
  880. )
  881. return False
  882. logger.warning(
  883. "Dropping watch notification with no routing metadata for process %s",
  884. evt.get("session_id", "unknown"),
  885. )
  886. return None
  887. platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform)
  888. adapter = self._resolve_injection_adapter(platform_name, source)
  889. if not adapter and platform_name == Platform.API_SERVER.value and getattr(source, "profile", None):
  890. # A route-only served profile owns no adapter map; the shared listener wakes exactly the
  891. # session that profile's store owns (proven in ``_self_post_api_server``), fail-closed.
  892. adapter = self.adapters.get(Platform.API_SERVER)
  893. if not adapter:
  894. return False
  895. if not adapter_supports_push(adapter):
  896. # Non-push adapter (api_server): its chat_id IS the raw session id, so handle_message would
  897. # key the wake under a build_session_key() that never matches — self-post instead.
  898. raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "")
  899. return await self._self_post_api_server(adapter, synth_text, raw_sid, evt)
  900. try:
  901. synth_text = _mark_internal_notification(synth_text)
  902. metadata = _watch_notification_metadata(evt, source)
  903. # The queued event's ``message_id`` is the message that STARTED the process, and the
  904. # persisted origin carries the same stale id. A synthetic completion is not a reply to
  905. # it: by delivery time the user has often continued elsewhere, and an event anchored
  906. # there makes the finished job's reply quote that old message on every reply-anchoring
  907. # platform (#52694: a background completion visibly replying to a stale Discord DM
  908. # message). Routing is unaffected — topic lanes carry thread_id and the anchor-less
  909. # synthetic-send branches are covered (#87051); the original id rides metadata for
  910. # debugging only.
  911. trigger_message_id = str(evt.get("message_id") or "").strip() or None
  912. if trigger_message_id:
  913. metadata["original_trigger_message_id"] = trigger_message_id
  914. if getattr(source, "message_id", None):
  915. from gateway.session_identity import replace_source
  916. source = replace_source(source, message_id=None)
  917. synth_event = MessageEvent(
  918. text=synth_text, message_type=MessageType.TEXT, source=source, internal=True,
  919. metadata=metadata,
  920. )
  921. logger.info(
  922. "Watch pattern notification — injecting for %s chat=%s thread=%s",
  923. platform_name, source.chat_id, source.thread_id,
  924. )
  925. # Relay egress priming: post-restart routing caches are cold (they warm only on inbound), so
  926. # replies would egress without tenant discriminators and be declined by the connector.
  927. _prime = getattr(adapter, "prime_routing_cache", None)
  928. if callable(_prime):
  929. _prime(synth_event)
  930. from gateway.session_authorities import active_authority
  931. authority = active_authority(self)
  932. if authority is not None:
  933. from gateway.session_automation import admit_process_notice
  934. from hermes_state_runtime import RuntimeStoreError
  935. try:
  936. await admit_process_notice(self, authority, adapter, synth_event, evt)
  937. except RuntimeStoreError as exc:
  938. raise WakeNotAccepted(str(exc), reason=exc.reason) from exc
  939. else:
  940. await admit_internal_event(adapter, synth_event)
  941. return True
  942. except WakeNotAccepted as exc:
  943. # Permanent only on proof: a pin the authority's state shows gone (/new, #141562) or a
  944. # final code. An unowned or foreign-pinned refusal may still find its owner (d6635ffe03b).
  945. from gateway.wake import PERMANENT_REFUSAL_REASONS
  946. pin_gone = await self._watch_pin_gone(evt, str(exc), log=not raise_not_accepted)
  947. exc.permanent = pin_gone or exc.reason in PERMANENT_REFUSAL_REASONS
  948. if pin_gone and not raise_not_accepted:
  949. return None
  950. if not raise_not_accepted:
  951. queued["_refusal_permanent"] = exc.permanent # the drain counts only permanent refusals
  952. logger.warning("Watch notification not admitted for %s chat=%s: %s",
  953. platform_name, source.chat_id, exc)
  954. # Durable callers refund the claim; ordinary watch callers just requeue.
  955. if raise_not_accepted:
  956. raise
  957. return False
  958. except Exception as e:
  959. logger.error("Watch notification injection error: %s", e)
  960. return False
  961. @staticmethod
  962. def _completion_delivery_identity(evt: dict) -> Optional[tuple[str, str, object]]:
  963. """Return a producer-stable identity when one is available.
  964. Delegation UUIDs identify one producer completion. Process session IDs include the
  965. persisted spawn epoch so a reused ID is a distinct incarnation; legacy events without
  966. ``started_at`` are delivered undeduplicated rather than risk suppressing a real completion.
  967. """
  968. evt_type = str(evt.get("type") or "")
  969. if evt_type == "async_delegation":
  970. producer_id = str(evt.get("delegation_id") or "")
  971. if not producer_id:
  972. return None
  973. if evt.get("task_failure_notice"):
  974. # An interim per-task notice is its own producer event: it must not mark the
  975. # batch's final result as already delivered, nor a sibling's notice.
  976. task_idx = ((evt.get("results") or [{}])[0] or {}).get("task_index", "")
  977. return (evt_type, producer_id, f"task_failure:{task_idx}")
  978. return (evt_type, producer_id, "")
  979. # A liveness tick is one watch event like a match: without an identity the session authority
  980. # refuses it and the drain requeues it forever.
  981. if evt_type in {"watch_match", "watch_disabled", "heartbeat"} and evt.get("event_id"):
  982. return (evt_type, str(evt["session_id"]), str(evt["event_id"]))
  983. if evt_type == "completion":
  984. producer_id = str(evt.get("session_id") or "")
  985. started_at = evt.get("started_at")
  986. if producer_id and started_at is not None:
  987. return (evt_type, producer_id, started_at)
  988. return None
  989. def _mark_completions_delivered_locked(self, identities) -> None:
  990. """Move identities inflight -> delivered and trim retention. Caller holds ``_completion_delivery_lock``."""
  991. for identity in identities:
  992. self._completion_deliveries_inflight.discard(identity)
  993. self._completion_deliveries_delivered[identity] = None
  994. while len(self._completion_deliveries_delivered) > self._completion_delivery_retention:
  995. self._completion_deliveries_delivered.popitem(last=False)
  996. def _completion_identity_seen(self, identity, *, claim: bool = False) -> bool:
  997. """True when ``identity`` is inflight or already delivered this gateway lifecycle.
  998. With ``claim`` an unseen identity is atomically marked inflight (same lock hold).
  999. """
  1000. with self._completion_delivery_lock:
  1001. seen = (
  1002. identity in self._completion_deliveries_inflight
  1003. or identity in self._completion_deliveries_delivered
  1004. )
  1005. if claim and not seen:
  1006. self._completion_deliveries_inflight.add(identity)
  1007. return seen
  1008. async def _classify_completion_target(self, parent_session_id: str, session_key: str = "") -> str:
  1009. """Classify an async-completion target before adapter acceptance: ``"deliver"`` (spawning
  1010. session live or compression-rotated with a live continuation; the resolver still retargets),
  1011. ``"terminal"`` (parent gone for good — unknown / user boundary like /new; drop the durable row
  1012. rather than falsely ack), ``"retry"`` (DB unavailable / rotation mid-flight; defer the claim)."""
  1013. return (await self._classify_completion_owner(parent_session_id, session_key))[0]
  1014. async def _classify_completion_owner(self, parent_session_id: str, session_key: str = "") -> tuple[str, str, str]:
  1015. """``_classify_completion_target``'s verdict and, on ``"deliver"``, the owning session it proved
  1016. (a delegate child's pin resolved through ``model_config._delegate_from``) and the session delivery
  1017. reaches (the owner, or its compression tip)."""
  1018. from gateway.run import _USER_BOUNDARY_END_REASONS
  1019. session_db = getattr(self, "_session_db", None)
  1020. if session_db is None:
  1021. return "retry", "", ""
  1022. entry = None
  1023. if session_key:
  1024. try:
  1025. session_store: SessionStore = cast(Any, self).session_store
  1026. await asyncio.to_thread(session_store._ensure_loaded)
  1027. entry = session_store._entries.get(session_key)
  1028. except Exception:
  1029. logger.debug("Completion route lookup failed for %s", session_key, exc_info=True)
  1030. return "retry", "", ""
  1031. verdict, parent_session_id, parent = await self._lookup_completion_owner(
  1032. parent_session_id, entry.session_id if entry else "",
  1033. )
  1034. target_session_id = parent_session_id
  1035. # A local conversation a viewer closed keeps its own work as a note on its closed tip (#134889 LC-7).
  1036. closed_tip = await self._closed_local_target(session_db, entry, parent_session_id, parent)
  1037. if closed_tip:
  1038. verdict, target_session_id = "deliver", closed_tip
  1039. elif verdict == "deliver" and parent is not None and parent.get("ended_at"):
  1040. end_reason = str(parent.get("end_reason") or "")
  1041. if end_reason != "compression":
  1042. # Only a USER-closed session (/new, user_exit, session_switch) is unreachable; idle/timeout
  1043. # ends stay routable and the resolver retargets. Boundary set shared with the resolver.
  1044. verdict = "terminal" if end_reason in _USER_BOUNDARY_END_REASONS else "deliver"
  1045. else:
  1046. verdict, target_session_id = await self._resolve_compression_lineage_target(
  1047. session_db, entry, parent_session_id,
  1048. )
  1049. if verdict != "deliver":
  1050. return verdict, "", ""
  1051. return verdict, parent_session_id, target_session_id or parent_session_id
  1052. @staticmethod
  1053. def _settle_durable_claim(kind: str, delegation_id: str, claim_id: str) -> None:
  1054. """Best-effort ``drop``/``release`` of a durable completion claim."""
  1055. fn_name, fail_msg = _DURABLE_CLAIM_OPS[kind]
  1056. try:
  1057. import tools.async_delegation as _ad
  1058. getattr(_ad, fn_name)(delegation_id, claim_id)
  1059. except Exception:
  1060. logger.log(logging.WARNING if kind == "complete" else logging.DEBUG, fail_msg, exc_info=True)
  1061. async def _completion_delivery_ready(self, evt: dict) -> bool:
  1062. """Unavailable owners/transports must not spend a durable delivery attempt."""
  1063. from gateway.wake import adapter_supports_push
  1064. parent_session_id = str(evt.get("parent_session_id") or "").strip()
  1065. if parent_session_id:
  1066. verdict = await self._classify_completion_target(parent_session_id, str(evt.get("session_key") or ""))
  1067. if verdict != "deliver":
  1068. # Definitively closed targets still need the normal terminal disposition.
  1069. return verdict == "terminal"
  1070. source = await asyncio.to_thread(self._build_process_event_source, evt)
  1071. if source is not None:
  1072. platform = source.platform.value if hasattr(source.platform, "value") else str(source.platform)
  1073. adapter = self._resolve_injection_adapter(platform, source)
  1074. else:
  1075. raw_sid = _raw_process_event_session_id(evt)
  1076. adapter = self.adapters.get(Platform.API_SERVER) if raw_sid else None
  1077. if adapter is not None and adapter_supports_push(adapter):
  1078. return False
  1079. warned = self.__dict__.setdefault("_completion_no_transport_warned", set())
  1080. key = (getattr(source, "profile", None), platform) if source is not None else None
  1081. if adapter is None:
  1082. # The watcher requeues silently every tick; say once per outage which lane has no transport.
  1083. if key is not None and key not in warned:
  1084. warned.add(key)
  1085. logger.warning("Async completion held: %s", _held_reason(
  1086. source, platform, getattr(getattr(self, "config", None), "multiplex_profiles", False)))
  1087. return False
  1088. warned.discard(key)
  1089. if not adapter_supports_push(adapter):
  1090. ensure = getattr(adapter, "_ensure_session_db", None)
  1091. try:
  1092. if not callable(ensure) or await asyncio.to_thread(ensure) is None:
  1093. return False
  1094. except Exception:
  1095. logger.debug("Async-completion delivery DB unavailable", exc_info=True)
  1096. return False
  1097. return True
  1098. async def _preflight_completion_delivery(self, evt: dict) -> _CompletionClaim:
  1099. """Claim the durable row (async delegations) and verify the target before adapter acceptance.
  1100. Adapter acceptance is not proof of delivery: the inner resolver can still fail closed inside
  1101. the pipeline after acceptance, falsely acking the durable row. Verifying first gives drops an
  1102. honest durable disposition.
  1103. """
  1104. claim = self._CompletionClaim()
  1105. evt_type = evt.get("type")
  1106. if evt_type == "async_delegation" and not await self._completion_delivery_ready(evt):
  1107. claim.proceed, claim.early_result = False, False
  1108. return claim
  1109. # An interim per-task notice shares the batch's delegation_id but is not the durable
  1110. # completion; claiming that row here would acknowledge the FINAL result before it exists.
  1111. if evt_type == "async_delegation" and not evt.get("task_failure_notice"):
  1112. claim.delegation_id = str(evt.get("delegation_id") or "")
  1113. if claim.delegation_id:
  1114. try:
  1115. from tools.async_delegation import claim_completion_delivery
  1116. claim.claim_id = f"gateway:{id(self)}:{__import__('uuid').uuid4().hex}"
  1117. if not claim_completion_delivery(claim.delegation_id, claim.claim_id):
  1118. claim.proceed = False
  1119. return claim
  1120. except Exception as exc:
  1121. logger.warning("Could not claim durable async completion %s: %s", claim.delegation_id, exc)
  1122. claim.proceed, claim.early_result = False, False
  1123. return claim
  1124. elif evt_type != "completion":
  1125. return claim
  1126. # Background completions carry only session_key, so after /new the OLD session's notification
  1127. # would land in the NEW one. Stamped events get the async-delegation pre-flight; unstamped deliver.
  1128. parent_session_id = str(evt.get("parent_session_id") or "").strip()
  1129. if not parent_session_id:
  1130. return claim
  1131. # Pre-flight (#65838-class): adapter acceptance is NOT proof of delivery — the inner #55578 resolver
  1132. # can still fail closed inside the message pipeline AFTER the adapter accepted, which would falsely
  1133. # acknowledge the durable row as delivered. Verify the target here, before acceptance, and give
  1134. # drops an honest durable disposition.
  1135. verdict, claim.owner_session_id, claim.proven_session_id = await self._classify_completion_owner(
  1136. parent_session_id, str(evt.get("session_key") or ""))
  1137. if verdict == "terminal":
  1138. if evt_type == "async_delegation":
  1139. logger.warning(
  1140. "Async delegation %s targets permanently-gone session %s; "
  1141. "terminally dropping delivery (result remains in the delegation records).",
  1142. claim.delegation_id or "<legacy>", parent_session_id,
  1143. )
  1144. if claim.claim_id:
  1145. self._settle_durable_claim("drop", claim.delegation_id, claim.claim_id)
  1146. else:
  1147. logger.warning(
  1148. "Background process %s completion targets "
  1149. "permanently-gone session %s (user boundary such as "
  1150. "/new); dropping notification (output remains available via process(action='log')).",
  1151. evt.get("session_id") or "<unknown>", parent_session_id,
  1152. )
  1153. claim.proceed = False
  1154. elif verdict == "retry":
  1155. # Transient uncertainty before admission: re-poll without spending a delivery attempt.
  1156. claim.proceed, claim.early_result, claim.defer = False, False, True
  1157. return claim
  1158. def _completion_event_scope(self, evt: dict):
  1159. """Profile runtime scope of the session a completion event targets (a no-op context when the
  1160. event is the default profile's or the scope is already installed).
  1161. The pre-flight (``_classify_completion_target`` → ``_session_db``) and every durable-ledger op
  1162. (``tools.async_delegation`` → ``get_hermes_home()/state.db``) resolve from the ambient scope.
  1163. The supervised ``_async_delegation_watcher`` and startup-recovered process watchers run under
  1164. the ROOT scope, so a secondary profile's completion was looked up in the DEFAULT profile's
  1165. state.db — classified ``terminal`` and dropped, its ledger row stranded ``pending`` forever."""
  1166. from gateway.run import _async_profile_runtime_scope
  1167. from hermes_constants import get_hermes_home_override
  1168. source = self._build_process_event_source(evt)
  1169. if source is None or not getattr(source, "profile", None):
  1170. raw_sid = _raw_process_event_session_id(evt)
  1171. if raw_sid and getattr(getattr(self, "config", None), "multiplex_profiles", False):
  1172. return self._served_api_server_event_scope(evt, raw_sid)
  1173. # No routed profile: the launch profile's own completion. Bind ITS scope once the
  1174. # process multiplexes — unscoped, a fail-closed ledger read raises on a legitimate
  1175. # launch-profile event (no-op while single-profile).
  1176. from tui_gateway.launch_profile_policy import async_launch_profile_scope_if_multiplexed
  1177. return async_launch_profile_scope_if_multiplexed()
  1178. profile_home = self._resolve_profile_home_for_source(source)
  1179. if get_hermes_home_override() == str(profile_home):
  1180. return contextlib.nullcontext() # already inside this profile's scope
  1181. return _async_profile_runtime_scope(profile_home)
  1182. @contextlib.asynccontextmanager
  1183. async def _served_api_server_event_scope(self, evt: dict, raw_sid: str):
  1184. """Scope of the served profile whose own store holds a raw api_server session (its events name no
  1185. profile), else the launch profile's, as for any unrouted event."""
  1186. from gateway.run import _async_profile_runtime_scope
  1187. from tui_gateway.launch_profile_policy import async_launch_profile_scope_if_multiplexed
  1188. served = await asyncio.to_thread(self._served_api_server_wake_profile, evt, raw_sid)
  1189. if served:
  1190. source = SessionSource(platform=Platform.API_SERVER, chat_id=raw_sid, profile=served)
  1191. scope = _async_profile_runtime_scope(self._resolve_profile_home_for_source(source))
  1192. else:
  1193. scope = async_launch_profile_scope_if_multiplexed()
  1194. async with scope:
  1195. yield
  1196. async def _deliver_completion_notification(
  1197. self, synth_text: str, evt: dict, *, sibling_claims=(),
  1198. ) -> Optional[bool]:
  1199. """Acknowledge one admitted batch, refund refusals, or release failed deliveries.
  1200. True means adapter admission, not model execution; None means deduplicated or
  1201. terminal. False remains retryable. Claims are settled together for every sibling.
  1202. """
  1203. async with self._completion_event_scope(evt):
  1204. return await self._deliver_completion_notification_scoped(
  1205. synth_text, evt, sibling_claims=sibling_claims)
  1206. async def _deliver_completion_notification_scoped(
  1207. self, synth_text: str, evt: dict, *, sibling_claims=(),
  1208. ) -> Optional[bool]:
  1209. from gateway.session_local_recovery import restore_completion_route
  1210. from gateway.wake import WakeNotAccepted
  1211. from tools.process_registry_notifications import should_surface_notification
  1212. if not should_surface_notification(evt):
  1213. return None # Suppressed noise is terminal, not a consumed process result.
  1214. restore_completion_route(self, str(evt.get("session_key") or ""))
  1215. from gateway.session_detached import follow_detached_pin
  1216. evt = await follow_detached_pin(self, evt)
  1217. identity = self._completion_delivery_identity(evt)
  1218. claim = self._CompletionClaim()
  1219. accepted = identity_claimed = refused = False
  1220. try:
  1221. claim = await self._preflight_completion_delivery(evt)
  1222. if not claim.proceed:
  1223. return claim.early_result
  1224. if getattr(self, 'session_authority', None) is not None:
  1225. from gateway.session_automation import completion_admission
  1226. if completion_admission(self, evt) is not None:
  1227. accepted = True # Lost ACK: finish durable claims, never infer again.
  1228. return True
  1229. if identity is not None:
  1230. if self._completion_identity_seen(identity, claim=True):
  1231. return None
  1232. identity_claimed = True
  1233. if claim.owner_session_id:
  1234. # Admission settles the durable claim, so the resolver starts from the owner proven here.
  1235. evt = {**evt, "parent_session_id": claim.owner_session_id,
  1236. "proven_session_id": claim.proven_session_id}
  1237. # The rows this admission settles: a turn that cannot confirm the owner offers them again.
  1238. completion_ids = [claim.delegation_id] if claim.claim_id else []
  1239. completion_ids += [sibling["delegation_id"] for sibling, claim_id in sibling_claims if claim_id]
  1240. evt = {**evt, "completion_ids": completion_ids}
  1241. injection_result = await self._inject_watch_notification(synth_text, evt, raise_not_accepted=True)
  1242. if injection_result is not True:
  1243. return injection_result
  1244. accepted = True
  1245. if identity is not None:
  1246. with self._completion_delivery_lock:
  1247. self._mark_completions_delivered_locked((identity,))
  1248. return True
  1249. except WakeNotAccepted as exc:
  1250. # A proven-permanent refusal spends its durable attempt, so the row converges to
  1251. # ``dropped`` across restarts too (#141153); any other (restart wait, full FIFO, an owner
  1252. # not yet proven) is refunded and never drops a completion the owner will take later.
  1253. refused = not exc.permanent
  1254. return False
  1255. finally:
  1256. if identity_claimed and not accepted:
  1257. with self._completion_delivery_lock:
  1258. self._completion_deliveries_inflight.discard(identity)
  1259. operation = "complete" if accepted else "defer" if refused or claim.defer else "release"
  1260. if claim.claim_id:
  1261. self._settle_durable_claim(operation, claim.delegation_id, claim.claim_id)
  1262. for sibling, claim_id in sibling_claims:
  1263. if claim_id:
  1264. self._settle_durable_claim(operation, sibling["delegation_id"], claim_id)
  1265. if accepted and sibling_claims:
  1266. self._record_coalesced_completion_siblings([event for event, _claim_id in sibling_claims])
  1267. @staticmethod
  1268. def _event_route_key(evt: dict, fields: tuple[str, ...]) -> tuple[str, ...]:
  1269. return tuple(str(evt.get(field) or "") for field in fields)
  1270. @staticmethod
  1271. def _format_coalesced_process_completions(entries: list[tuple[str, dict, asyncio.Future]]) -> str:
  1272. """Build one bounded synthetic event from several redacted completions."""
  1273. from gateway.run import _redact_gateway_user_facing_secrets
  1274. lines = [
  1275. f"[IMPORTANT: {len(entries)} background processes completed for this session.",
  1276. "Treat these results as one completion batch and send at most one "
  1277. "consolidated user-facing response.",
  1278. ]
  1279. shown = entries[:10]
  1280. for _text, evt, _future in shown:
  1281. session_id = str(evt.get("session_id") or "unknown")
  1282. exit_code = evt.get("exit_code")
  1283. reason = str(evt.get("completion_reason") or "exited")
  1284. # Unconditional gateway redaction floor (the producer-seam redactor is configurable). Redact
  1285. # BEFORE slicing: truncating first can leave a credential fragment the patterns miss.
  1286. output = _redact_gateway_user_facing_secrets(str(evt.get("output") or "")).strip()
  1287. if len(output) > 800:
  1288. output = f"[… truncated …]\n{output[-800:]}"
  1289. lines.append(f"\n- {session_id}: exit_code={exit_code}, reason={reason}")
  1290. if output:
  1291. lines.append(output)
  1292. omitted = len(entries) - len(shown)
  1293. if omitted:
  1294. lines.append(
  1295. f"\n- … and {omitted} more completion(s); inspect them with "
  1296. "the process tool if they affect the conclusion."
  1297. )
  1298. lines.append("If a result does not change the current conclusion, absorb it silently.]")
  1299. return "\n".join(lines)
  1300. def _record_coalesced_completion_siblings(self, events: list[dict]) -> None:
  1301. """Extend a successful primary delivery claim to its batched siblings."""
  1302. identities = [i for i in map(self._completion_delivery_identity, events) if i is not None]
  1303. with self._completion_delivery_lock:
  1304. self._mark_completions_delivered_locked(identities)
  1305. async def _flush_process_completion_batch(self, key: tuple[str, ...]) -> None:
  1306. """Deliver one short-window completion batch and resolve its waiters."""
  1307. current_task = asyncio.current_task()
  1308. entries: list[tuple[str, dict, asyncio.Future]] = []
  1309. delivered: Optional[bool] = False
  1310. try:
  1311. await asyncio.sleep(self._completion_notification_batch_window)
  1312. entries = self._completion_notification_batches.pop(key, [])
  1313. # Detach before delivery so a completion arriving mid-flight can schedule the next flush.
  1314. if self._completion_notification_batch_tasks.get(key) is current_task:
  1315. self._completion_notification_batch_tasks.pop(key, None)
  1316. if not entries:
  1317. return
  1318. if getattr(self, 'session_authority', None) is not None:
  1319. from gateway.session_automation import completion_admission, producer_identity
  1320. entries_to_deliver = [item for item in entries if completion_admission(self, item[1]) is None]
  1321. if not entries_to_deliver:
  1322. delivered = True
  1323. return
  1324. else:
  1325. entries_to_deliver = entries
  1326. synth_text = (entries_to_deliver[0][0] if len(entries_to_deliver) == 1
  1327. else self._format_coalesced_process_completions(entries_to_deliver))
  1328. from tools.process_registry_notifications import process_completion_display_text
  1329. display_text = process_completion_display_text([evt for _text, evt, _future in entries_to_deliver])
  1330. # A duplicate primary returns None from the dedupe seam; try the next identity so a fresh
  1331. # sibling is never discarded with it.
  1332. delivered = None
  1333. for _text, candidate_evt, _future in entries_to_deliver:
  1334. candidate_evt = dict(candidate_evt, _display_text=display_text)
  1335. if getattr(self, 'session_authority', None) is not None:
  1336. candidate_evt = dict(candidate_evt, _automation_identities=[
  1337. producer_identity(self, evt) for _text, evt, _future in entries_to_deliver])
  1338. delivered = await self._deliver_completion_notification(synth_text, candidate_evt)
  1339. if delivered is not None:
  1340. break
  1341. if delivered is True and len(entries) > 1:
  1342. self._record_coalesced_completion_siblings([evt for _text, evt, _future in entries])
  1343. except asyncio.CancelledError:
  1344. # Shutdown cancellation: recover undetached entries and resolve every waiter as retryable.
  1345. delivered = False
  1346. if not entries:
  1347. entries = self._completion_notification_batches.pop(key, [])
  1348. raise
  1349. except Exception:
  1350. logger.exception("Coalesced process completion delivery failed")
  1351. delivered = False
  1352. finally:
  1353. # Never strand watcher futures: False = watcher retry path; None = ordinary dedupe result.
  1354. self._settle_batch_waiters(entries, delivered)
  1355. # Do not remove a newer flush task that reused the same route key.
  1356. if self._completion_notification_batch_tasks.get(key) is current_task:
  1357. self._completion_notification_batch_tasks.pop(key, None)
  1358. @staticmethod
  1359. def _settle_batch_waiters(entries, result) -> None:
  1360. for _text, _evt, future in entries:
  1361. if not future.done():
  1362. future.set_result(result)
  1363. async def _cancel_process_completion_batch_tasks(self) -> None:
  1364. """Settle pending completion batches before adapter teardown."""
  1365. self._completion_notification_batches_stopping = True
  1366. tasks = {
  1367. task
  1368. for task in getattr(self, "_completion_notification_batch_flush_tasks", set())
  1369. if not task.done()
  1370. }
  1371. for task in tasks:
  1372. task.cancel()
  1373. if tasks:
  1374. await asyncio.gather(*tasks, return_exceptions=True)
  1375. # Defensive cleanup for an orphaned queue with no live flush task.
  1376. batches = getattr(self, "_completion_notification_batches", {})
  1377. for entries in batches.values():
  1378. self._settle_batch_waiters(entries, False)
  1379. batches.clear()
  1380. getattr(self, "_completion_notification_batch_tasks", {}).clear()
  1381. getattr(self, "_completion_notification_batch_flush_tasks", set()).clear()
  1382. async def _enqueue_process_completion_notification(self, synth_text: str, evt: dict) -> Optional[bool]:
  1383. """Fan in policy-admitted process completions with compatible pinned owners."""
  1384. # Filter each event before it can contaminate (or suppress) a foreground batch.
  1385. # The shared queue may be drained outside the owning profile's runtime scope.
  1386. async with self._completion_event_scope(evt):
  1387. from tools.process_registry_notifications import should_surface_notification
  1388. if not should_surface_notification(evt):
  1389. return None
  1390. # Lazy defaults: lifecycle tests build GatewayRunner via object.__new__.
  1391. for attr, default in (
  1392. ("_completion_notification_batches", dict), ("_completion_notification_batch_tasks", dict),
  1393. ("_completion_notification_batch_flush_tasks", set),
  1394. ("_completion_notification_batch_window", lambda: 0.1),
  1395. ("_completion_notification_batches_stopping", lambda: False), ("_background_tasks", set),
  1396. ):
  1397. if not hasattr(self, attr):
  1398. setattr(self, attr, default())
  1399. if self._completion_notification_batches_stopping:
  1400. return False
  1401. # A route key alone is not ownership: /new can leave old and new pins
  1402. # in the same queue. Never settle one pin on another pin's acceptance.
  1403. key = self._event_route_key(
  1404. evt, (*self._COMPLETION_BATCH_KEY_FIELDS, "parent_session_id", "origin_session_id"),
  1405. )
  1406. future = asyncio.get_running_loop().create_future()
  1407. self._completion_notification_batches.setdefault(key, []).append((synth_text, evt, future))
  1408. if key not in self._completion_notification_batch_tasks:
  1409. task = asyncio.create_task(self._flush_process_completion_batch(key))
  1410. self._completion_notification_batch_tasks[key] = task
  1411. # Keep the flush alive under the gateway's normal lifecycle accounting.
  1412. self._retain_background_task(task)
  1413. self._track_task_in(self._completion_notification_batch_flush_tasks, task)
  1414. return await future
  1415. def _enrich_async_delegation_routing(self, evt: dict) -> None:
  1416. """Fill platform/chat_id/thread_id/chat_type on an async-delegation event.
  1417. Such events only carry ``session_key`` (the daemon worker lacks per-message routing
  1418. metadata). Best-effort: a CLI-origin event (empty session_key) is left as-is and won't route.
  1419. """
  1420. from gateway.run import _parse_session_key
  1421. if evt.get("platform"):
  1422. return # already enriched
  1423. parsed = _parse_session_key(evt.get("session_key", "") or "")
  1424. if not parsed:
  1425. return
  1426. evt["platform"] = parsed.get("platform", "")
  1427. evt["chat_type"] = parsed.get("chat_type", "")
  1428. evt["chat_id"] = parsed.get("chat_id", "")
  1429. if parsed.get("thread_id"):
  1430. evt["thread_id"] = parsed["thread_id"]
  1431. async def _deliver_async_delegation_group(self, group: list[dict]) -> Optional[bool]:
  1432. """Deliver a same-session batch of async completions as ONE turn: the primary carries the
  1433. consolidated text of every sibling THIS runner claimed (siblings owned elsewhere are excluded;
  1434. their claims are acked only after adapter acceptance). True after acceptance, False to requeue
  1435. the group, None when nothing is deliverable here (retry siblings requeued)."""
  1436. # The group shares one session_key, hence one profile: scope the pre-checks and sibling claims too.
  1437. async with self._completion_event_scope(group[0]):
  1438. return await self._deliver_async_delegation_group_scoped(group)
  1439. async def _deliver_async_delegation_group_scoped(self, group: list[dict]) -> Optional[bool]:
  1440. from gateway.run import _format_gateway_process_notification
  1441. from gateway.session_local_recovery import restore_completion_route
  1442. from tools.process_registry import process_registry as _pr
  1443. restore_completion_route(self, str(group[0].get("session_key") or ""))
  1444. # API delivery does not start a model turn, so there is nothing to coalesce.
  1445. # Keep each unit's stable identity with its row across partial delivery/retry.
  1446. if group and group[0].get("origin_session_id"):
  1447. outcomes = []
  1448. for evt in group:
  1449. text = _format_gateway_process_notification(evt)
  1450. if text:
  1451. outcomes.append(await self._deliver_completion_notification(text, evt))
  1452. return False if False in outcomes else True
  1453. deliverable: list[tuple[dict, str]] = []
  1454. acknowledged = False
  1455. for evt in group:
  1456. synth_text = _format_gateway_process_notification(evt)
  1457. if not synth_text:
  1458. continue
  1459. if getattr(self, 'session_authority', None) is not None:
  1460. from gateway.session_automation import completion_admission
  1461. if completion_admission(self, evt) is not None:
  1462. acknowledged = await self._deliver_completion_notification(synth_text, evt) is True or acknowledged
  1463. continue
  1464. parent = str(evt.get("parent_session_id") or "")
  1465. if parent and await self._classify_completion_target(parent, str(evt.get("session_key") or "")) == "terminal":
  1466. await self._deliver_completion_notification(synth_text, evt)
  1467. continue
  1468. identity = self._completion_delivery_identity(evt)
  1469. if identity is not None and self._completion_identity_seen(identity):
  1470. continue
  1471. deliverable.append((evt, synth_text))
  1472. if not deliverable:
  1473. return True if acknowledged else None
  1474. if len(deliverable) == 1:
  1475. evt, synth_text = deliverable[0]
  1476. return await self._deliver_completion_notification(synth_text, evt)
  1477. # Check the entire group before claiming ANY row: an unavailable sibling must
  1478. # not exhaust its budget just because the primary has a usable route.
  1479. for evt, _text in deliverable:
  1480. if not await self._completion_delivery_ready(evt):
  1481. return False
  1482. from tools.async_delegation import claim_event_delivery
  1483. primary_evt, primary_text = deliverable[0]
  1484. blocks = [primary_text]
  1485. siblings: list[tuple[dict, str]] = []
  1486. for evt, synth_text in deliverable[1:]:
  1487. claim_id = claim_event_delivery(evt, f"gateway-batch:{id(self)}")
  1488. if claim_id is None:
  1489. # Another consumer owns this row: keep it out of our text so it is never double-injected.
  1490. continue
  1491. siblings.append((evt, claim_id))
  1492. blocks.append(synth_text)
  1493. if not siblings:
  1494. return await self._deliver_completion_notification(primary_text, primary_evt)
  1495. header = (
  1496. f"[IMPORTANT: {len(blocks)} background subagent delegations "
  1497. "completed for this session. Treat these results as one "
  1498. "completion batch and send at most one consolidated user-facing "
  1499. "response. If a result does not change the current conclusion, absorb it silently.]"
  1500. )
  1501. consolidated = "\n\n".join([header, *blocks])
  1502. if getattr(self, 'session_authority', None) is not None:
  1503. from gateway.session_automation import producer_identity
  1504. primary_evt = dict(primary_evt, _automation_identities=[
  1505. producer_identity(self, event) for event in [primary_evt, *(e for e, _ in siblings)]])
  1506. delivered = await self._deliver_completion_notification(
  1507. consolidated, primary_evt, sibling_claims=siblings,
  1508. )
  1509. if delivered is None:
  1510. # Primary dropped/owned elsewhere: retry the unadmitted siblings.
  1511. for evt, _claim_id in siblings:
  1512. _pr.completion_queue.put(evt)
  1513. return delivered
  1514. def _restore_secondary_completion_ledgers(self, profile_homes) -> None:
  1515. """Re-queue undelivered async completions from every SECONDARY profile's ledger. The process
  1516. registry's ``restore_completions()`` covers only the launch profile's ``state.db``; a
  1517. secondary's rows would otherwise never be replayed after a restart."""
  1518. from tools.async_delegation import restore_undelivered_completions
  1519. from tools.process_registry import process_registry as _pr
  1520. self._each_secondary_ledger(profile_homes, lambda: restore_undelivered_completions(_pr.completion_queue),
  1521. "Restored")
  1522. def _sweep_orphaned_completion_ledgers(self) -> None:
  1523. """Offer completions whose owner process died while this gateway runs (#97202): the launch
  1524. ledger in the launch scope, each served secondary under its own. Startup replay only covers
  1525. owners that were already gone when the gateway started."""
  1526. from tools.async_delegation import sweep_orphaned_completions
  1527. from tools.process_registry import process_registry as _pr
  1528. sweep = lambda: sweep_orphaned_completions(_pr.completion_queue)
  1529. with _log_suppressed(logging.DEBUG, "Orphaned async completion sweep failed: %s"):
  1530. if count := sweep():
  1531. logger.info("Re-offered %d orphaned async completion(s)", count)
  1532. self._each_secondary_ledger((getattr(self, "_served_profile_homes", None) or {}).items(), sweep,
  1533. "Re-offered orphaned")
  1534. def _each_secondary_ledger(self, profile_homes, fn, verb: str) -> None:
  1535. """Run ``fn`` (returns a completion count) once per SECONDARY profile, bound to that profile."""
  1536. from gateway.run import _profile_runtime_scope
  1537. primary = getattr(self, "_primary_profile_name", None)
  1538. for profile_name, profile_home in profile_homes:
  1539. if profile_name == primary:
  1540. continue
  1541. try:
  1542. with _profile_runtime_scope(Path(profile_home), {}):
  1543. count = fn()
  1544. except Exception:
  1545. logger.warning("Could not replay async completions for profile %r", profile_name, exc_info=True)
  1546. continue
  1547. if count:
  1548. logger.info("%s %d undelivered async completion(s) for profile %r", verb, count, profile_name)
  1549. async def _async_delegation_watcher(self, interval: float = 2.0) -> None:
  1550. """Drain async completions and pattern notifications even while sessions are idle.
  1551. Background subagents and process pattern events have no per-process notification
  1552. consumer; both must progress without a later foreground turn.
  1553. """
  1554. await asyncio.sleep(3) # let platforms finish connecting
  1555. from tools.async_delegation import ORPHAN_SWEEP_INTERVAL_S
  1556. from tools.process_registry import process_registry as _pr
  1557. last_orphan_sweep = None
  1558. while self._running:
  1559. with _log_suppressed(logging.DEBUG, "Async delegation watcher error: %s"):
  1560. # Completions whose owner process died while this gateway runs (#97202).
  1561. if last_orphan_sweep is None or time.monotonic() - last_orphan_sweep >= ORPHAN_SWEEP_INTERVAL_S:
  1562. last_orphan_sweep = time.monotonic()
  1563. await asyncio.to_thread(self._sweep_orphaned_completion_ledgers)
  1564. # Pattern events also need an idle consumer; foreground turns are optional.
  1565. await self._drain_watch_notifications(_pr.completion_queue)
  1566. # Process completions remain owned by their per-process watchers.
  1567. requeue = []
  1568. async_events = []
  1569. while not _pr.completion_queue.empty():
  1570. try:
  1571. evt = _pr.completion_queue.get_nowait()
  1572. except Exception:
  1573. break
  1574. (async_events if evt.get("type") == "async_delegation" else requeue).append(evt)
  1575. for evt in requeue:
  1576. _pr.completion_queue.put(evt)
  1577. # A fan-out finishing together yields N completions for one session; group by full route +
  1578. # parent session so each group becomes ONE consolidated turn.
  1579. # A same-tick drain often carries several completions for the SAME originating session (a
  1580. # fan-out of background subagents finishing together). Events for different sessions never
  1581. # coalesce. See #70300.
  1582. groups: dict[tuple[str, ...], list[dict]] = {}
  1583. for evt in async_events:
  1584. self._enrich_async_delegation_routing(evt)
  1585. groups.setdefault(self._event_route_key(evt, self._ASYNC_GROUP_KEY_FIELDS), []).append(evt)
  1586. for group in groups.values():
  1587. if any(evt.get("_watch_next_retry_at", 0) > time.monotonic() for evt in group):
  1588. for evt in group:
  1589. _pr.completion_queue.put(evt)
  1590. continue
  1591. try:
  1592. delivered = await self._deliver_async_delegation_group(group)
  1593. if delivered is False:
  1594. self._back_off_refused_completions(group)
  1595. for evt in group:
  1596. _pr.completion_queue.put(evt)
  1597. except Exception as e:
  1598. for evt in group:
  1599. _pr.completion_queue.put(evt)
  1600. logger.error("Async delegation injection error: %s", e)
  1601. await asyncio.sleep(interval)
  1602. @staticmethod
  1603. def _back_off_refused_completions(group: list[dict]) -> None:
  1604. """Space out an undelivered async group's retries instead of re-offering it every 2 s pass.
  1605. The backoff only paces; the durable ledger decides when a row gives up (a permanent refusal
  1606. spends a delivery attempt there, so the count survives restarts) (#141153)."""
  1607. for evt in group:
  1608. step = int(evt.get("_watch_retry_step") or 0) + 1
  1609. evt["_watch_retry_step"] = step
  1610. evt["_watch_next_retry_at"] = _refusal_retry_at(step)
  1611. @staticmethod
  1612. def _redacted_output_tail(session, limit: int) -> str:
  1613. """Last ``limit`` chars of process output through the secret redactors (unconditional floor)."""
  1614. from gateway.run import _redact_gateway_user_facing_secrets
  1615. from tools.ansi_strip import strip_ansi
  1616. from tools.process_registry import transform_process_output
  1617. new_output = strip_ansi(session.output_buffer[-limit:]) if session.output_buffer else ""
  1618. if new_output:
  1619. from agent.redact import redact_terminal_output
  1620. _command = getattr(session, "command", "") or ""
  1621. new_output = transform_process_output(new_output, command=_command, returncode=session.exit_code,
  1622. task_id=getattr(session, "task_id", "") or "")
  1623. new_output = redact_terminal_output(new_output, _command)
  1624. # redact_terminal_output() is unforced (raw when security.redact_secrets is off); this goes
  1625. # straight to the adapter, so apply the same unconditional floor as agent-notify.
  1626. new_output = _redact_gateway_user_facing_secrets(new_output)
  1627. return new_output
  1628. async def _launching_turn_active(self, platform_name: str, watcher: dict) -> bool:
  1629. """Whether the session that launched *watcher*'s process is still inside a turn on its
  1630. adapter (``_active_sessions`` is the base adapter's busy guard)."""
  1631. session_key = str(watcher.get("session_key") or "").strip()
  1632. if not session_key:
  1633. return False
  1634. source = await asyncio.to_thread(self._build_process_event_source, watcher)
  1635. adapter = self._resolve_injection_adapter(platform_name, source)
  1636. return session_key in (getattr(adapter, "_active_sessions", None) or {})
  1637. async def _send_watcher_message(self, platform_name: str, chat_id, thread_id, message_text: str, watcher: dict) -> None:
  1638. from gateway.run import _non_conversational_metadata
  1639. source = await asyncio.to_thread(self._build_process_event_source, watcher)
  1640. adapter = self._resolve_injection_adapter(platform_name, source)
  1641. if adapter and chat_id:
  1642. with _log_suppressed(logging.ERROR, "Watcher delivery error: %s"):
  1643. send_meta = {"thread_id": thread_id} if thread_id else None
  1644. await adapter.send(
  1645. chat_id, message_text, metadata=_non_conversational_metadata(send_meta, platform=platform_name),
  1646. )
  1647. @staticmethod
  1648. def _build_process_completion_event(watcher: dict, session, session_id: str) -> dict:
  1649. """Build the synthetic ``completion`` event for an agent-notify watcher."""
  1650. from gateway.run import _redact_gateway_user_facing_secrets
  1651. from agent.redact import redact_terminal_output
  1652. from tools.ansi_strip import strip_ansi
  1653. from tools.process_registry import transform_process_output
  1654. _command = getattr(session, "command", "") or ""
  1655. _raw = strip_ansi(session.output_buffer) if session.output_buffer else ""
  1656. _raw = transform_process_output(_raw, command=_command, returncode=session.exit_code,
  1657. task_id=getattr(session, "task_id", "") or "") if _raw else _raw
  1658. _raw = redact_terminal_output(_raw, _command)
  1659. # Keep the last ~2000 chars snapped to a line boundary, with a marker when cut.
  1660. _LIMIT = 2000
  1661. # Truncate at line boundaries so notifications never start mid-line (fixes #23284). Keep the last
  1662. # ~2000 chars but snap to the nearest preceding newline, then prepend a truncation marker when
  1663. # output was cut.
  1664. if len(_raw) > _LIMIT:
  1665. _tail = _raw[-_LIMIT:]
  1666. _nl = _tail.find("\n")
  1667. _tail = _tail[_nl + 1:] if _nl != -1 else _tail
  1668. _out = f"[… output truncated — showing last {len(_tail)} chars]\n{_tail}"
  1669. else:
  1670. _out = _raw
  1671. return {
  1672. "type": "completion",
  1673. "session_id": session_id,
  1674. **{k: watcher.get(k, "") for k in _WATCHER_ROUTE_FIELDS},
  1675. "owner_task_id": getattr(session, "owner_task_id", "") or "",
  1676. "task_id": getattr(session, "task_id", "") or "",
  1677. "message_id": str(watcher.get("message_id") or "").strip() or None,
  1678. "started_at": getattr(session, "started_at", None),
  1679. "command": _redact_gateway_user_facing_secrets(_command),
  1680. "exit_code": session.exit_code,
  1681. "completion_reason": getattr(session, "completion_reason", "exited"),
  1682. "termination_source": getattr(session, "termination_source", ""),
  1683. "output": _redact_gateway_user_facing_secrets(_out),
  1684. # Spawning session-db id: lets pre-flight drop this completion if the user /new'd first.
  1685. "parent_session_id": (
  1686. watcher.get("parent_session_id") or getattr(session, "parent_session_id", "") or ""
  1687. ),
  1688. }
  1689. def _format_process_final_message(self, session_id: str, session, notify_mode: str) -> str:
  1690. """Human-facing completion message. Every mode shares the one-line status header; the
  1691. raw-output modes (all/result/error) append the bounded output tail under it instead of the
  1692. old bracketed ``[Background process proc_… finished~ …]`` debug wrapper (#54266)."""
  1693. from gateway.run import _format_concise_process_notification, _redact_gateway_user_facing_secrets
  1694. new_output = self._redacted_output_tail(session, 1000)
  1695. _started = getattr(session, "started_at", None)
  1696. _dur = max(0.0, time.time() - _started) if isinstance(_started, (int, float)) else None
  1697. command = _redact_gateway_user_facing_secrets(getattr(session, "command", "") or "")
  1698. if notify_mode == "concise":
  1699. return _format_concise_process_notification(session_id, command, session.exit_code, new_output,
  1700. duration_seconds=_dur)
  1701. header = _format_concise_process_notification(session_id, command, session.exit_code, "", duration_seconds=_dur)
  1702. return t("gateway.background.final_output", header=header, output=new_output.strip()) if new_output.strip() else header
  1703. def _format_process_running_message(self, session) -> str:
  1704. from gateway.run import _redact_gateway_user_facing_secrets, _shorten_command_for_display
  1705. new_output = self._redacted_output_tail(session, 500)
  1706. short_cmd = _shorten_command_for_display(_redact_gateway_user_facing_secrets(getattr(session, "command", "") or ""))
  1707. header = t("gateway.background.still_running") + (f" — `{short_cmd}`" if short_cmd else "")
  1708. return t("gateway.background.recent_output", header=header, output=new_output.strip()) if new_output.strip() else header
  1709. def arm_process_watcher(self, watcher: dict) -> bool:
  1710. """Start ``_run_process_watcher`` for a watcher registered mid-turn, from the agent's
  1711. tool thread. Waiting for the post-turn drain leaves a process that finishes while its
  1712. launching turn is still running with no watcher at all (#112033). False = the gateway
  1713. is not serving (startup, shutdown): the caller keeps the descriptor in
  1714. ``pending_watchers`` for the startup / post-turn drain.
  1715. The watcher starts in an empty context, as a recovered watcher does: scheduling copies the
  1716. CALLING thread's context, and the turns its completion admits inherit the watcher's. Armed
  1717. from a delegate child, a process handed to the parent completed into a parent turn that ran
  1718. as that child (its own notify watchers downgraded, sudo refused, the board fenced). Delivery
  1719. binds the owning profile's scope itself (``_completion_event_scope``)."""
  1720. loop = getattr(self, "_gateway_loop", None)
  1721. if not getattr(self, "_running", False) or loop is None or not loop.is_running():
  1722. return False
  1723. from agent.async_utils import safe_schedule_threadsafe
  1724. future = contextvars.Context().run(
  1725. safe_schedule_threadsafe, self._run_process_watcher(watcher), loop, logger=logger,
  1726. log_message="Live process watcher arming failed",
  1727. )
  1728. return future is not None
  1729. async def _run_process_watcher(self, watcher: dict) -> None:
  1730. """Run :meth:`_watch_process` inside the owning profile's scope, start to finish. A watcher starts
  1731. in an empty context (live-armed from a tool thread, or recovered at startup), and the output it
  1732. prepares runs that profile's own ``transform_terminal_output`` hooks before any delivery."""
  1733. async with self._completion_event_scope(watcher):
  1734. await self._watch_process(watcher)
  1735. async def _watch_process(self, watcher: dict) -> None:
  1736. """Poll a background process and push updates until it exits. Mode
  1737. (``display.background_process_notifications``): concise (default one-liner; failures append
  1738. the output tail) / all (running updates + final raw) / result (final raw) / error (final raw
  1739. if exit != 0) / off."""
  1740. from tools.process_registry import process_registry
  1741. from tools.process_registry_notifications import format_process_notification, should_surface_notification
  1742. session_id = watcher["session_id"]
  1743. interval = watcher["check_interval"]
  1744. platform_name = watcher.get("platform", "")
  1745. chat_id = watcher.get("chat_id", "")
  1746. thread_id = watcher.get("thread_id", "")
  1747. agent_notify = watcher.get("notify_on_complete", False)
  1748. notify_mode = self._load_background_notifications_mode() # the owning profile's (scope bound above)
  1749. logger.debug("Process watcher started: %s (every %ss, notify=%s, agent_notify=%s)",
  1750. session_id, interval, notify_mode, agent_notify)
  1751. silent = notify_mode == "off" and not agent_notify
  1752. last_output_len = 0
  1753. completion_attempts = 0
  1754. retry_interval = interval
  1755. while True:
  1756. await asyncio.sleep(retry_interval)
  1757. session = process_registry.get(session_id)
  1758. if session is None:
  1759. break
  1760. surface = should_surface_notification({
  1761. "owner_task_id": getattr(session, "owner_task_id", ""),
  1762. "task_id": getattr(session, "task_id", ""),
  1763. })
  1764. if not surface:
  1765. # Recheck live ownership next tick: handoff retains notification intent.
  1766. if session.exited:
  1767. break
  1768. continue
  1769. if silent:
  1770. # Still wait for the process to exit so we can log it, but don't push any messages.
  1771. if session.exited:
  1772. break
  1773. continue
  1774. current_output_len = len(session.output_buffer)
  1775. has_new_output = current_output_len > last_output_len
  1776. last_output_len = current_output_len
  1777. if session.exited:
  1778. # Agent-notify: inject a synthetic message unless the agent already consumed the result via
  1779. # wait/log (poll() is read-only and deliberately does NOT mark consumed).
  1780. if agent_notify and not process_registry.is_completion_consumed(session_id):
  1781. completion_evt = self._build_process_completion_event(watcher, session, session_id)
  1782. synth_text = format_process_notification(completion_evt)
  1783. if not synth_text:
  1784. break
  1785. # Captured before injection: afterwards the key is busy either way (the injected
  1786. # turn itself installs the guard).
  1787. turn_busy = await self._launching_turn_active(platform_name, watcher)
  1788. delivered = await self._enqueue_process_completion_notification(synth_text, completion_evt)
  1789. if delivered is not False:
  1790. # The agent normally reports the result itself, so the chat gets no separate
  1791. # receipt. While the launching turn is still running the injection only queues a
  1792. # follow-up, and the chat would stay mute for as long as that turn lasts
  1793. # (#112033): send the concise receipt now.
  1794. if turn_busy and (notify_mode in {"concise", "all", "result"} or (
  1795. notify_mode == "error" and session.exit_code not in {0, None}
  1796. )):
  1797. message_text = self._format_process_final_message(session_id, session, "concise")
  1798. await self._send_watcher_message(platform_name, chat_id, thread_id, message_text, watcher)
  1799. break
  1800. # A refusal may clear (adapter reconnecting, store draining), so retry with backoff
  1801. # for a few minutes, then fall through to the plain notice instead of dropping it.
  1802. completion_attempts += 1
  1803. if completion_attempts < _MAX_COMPLETION_INJECTION_ATTEMPTS:
  1804. retry_interval = jittered_backoff(
  1805. completion_attempts, base_delay=max(interval, 1) * 2,
  1806. max_delay=_COMPLETION_INJECTION_RETRY_MAX_DELAY, jitter_ratio=0.0)
  1807. continue
  1808. if (getattr(self, 'session_authority', None) is not None
  1809. and self._completion_delivery_identity(completion_evt) is not None):
  1810. from gateway.session_automation import completion_admission
  1811. try:
  1812. # The owning profile's authority, as on the delivery path.
  1813. async with self._completion_event_scope(completion_evt):
  1814. committed = completion_admission(self, completion_evt) is not None
  1815. except Exception:
  1816. # Failing open sends the notice; a dead watcher would send nothing.
  1817. logger.debug("Process watcher: admission recheck failed for %s", session_id,
  1818. exc_info=True)
  1819. committed = False
  1820. if committed:
  1821. break # The last refusal was a lost ACK: the turn is committed.
  1822. logger.warning(
  1823. "Process watcher: completion for %s was refused %d times; falling back to the "
  1824. "notify_mode=%s notice", session_id, completion_attempts, notify_mode,
  1825. )
  1826. # Text-only notification; skip when already consumed via wait/log (the agent_notify branch
  1827. # FALLS THROUGH here, hence the re-check).
  1828. if process_registry.is_completion_consumed(session_id):
  1829. logger.debug(
  1830. "Process watcher: completion for %s already consumed "
  1831. "via wait/log — skipping raw notification (#65379)", session_id,
  1832. )
  1833. break
  1834. if notify_mode in {"concise", "all", "result"} or (
  1835. notify_mode == "error" and session.exit_code not in {0, None}
  1836. ):
  1837. message_text = self._format_process_final_message(session_id, session, notify_mode)
  1838. from gateway.warning_notifications import present_notification
  1839. # Non-zero exit is the automatic diagnostic; a clean completion is the requested result.
  1840. await present_notification(
  1841. lambda: self._send_watcher_message(platform_name, chat_id, thread_id, message_text, watcher),
  1842. platform=platform_name, diagnostic=session.exit_code not in {0, None})
  1843. break
  1844. elif has_new_output and notify_mode == "all" and not agent_notify:
  1845. # New output — deliver a status update (only in "all" mode; agent_notify watchers
  1846. # only care about completion).
  1847. await self._send_watcher_message(
  1848. platform_name, chat_id, thread_id, self._format_process_running_message(session), watcher,
  1849. )
  1850. logger.debug("Process watcher ended%s: %s", " (silent)" if silent else "", session_id)