webhook_ingress.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322
  1. """Receipt-only adapter ingress: the authority, not an HTTP/WS task, owns execution."""
  2. import asyncio
  3. from contextlib import asynccontextmanager
  4. from hermes_state_runtime import RuntimeStoreError
  5. @asynccontextmanager
  6. async def producer_scope(adapter, event):
  7. from gateway.run import _async_profile_runtime_scope
  8. from gateway.session_ingress_context import native_callback
  9. from gateway.session_authorities import authority_for_home
  10. runner = getattr(adapter._message_handler, '__self__', None)
  11. if getattr(runner, 'session_authority', None) is None:
  12. raise RuntimeStoreError('not_found')
  13. registered, profile = runner._owning_profile(adapter, adapter.platform)
  14. home = getattr(runner, '_native_transport_homes', {}).get(profile)
  15. if not registered or home is None:
  16. raise RuntimeStoreError('profile_mismatch')
  17. source = event.source
  18. # Identity is pinned through the runner's single canonicalization seam (a secondary's own
  19. # bot, or the primary's routed profile); a rejected primary route is refused downstream.
  20. if profile:
  21. runner._canonicalize(source, transport_profile=profile)
  22. runtime_home = runner._resolve_profile_home_for_source(source) if source.profile else home
  23. else:
  24. runtime_home = runner._admit_primary_source(source, home)
  25. if runtime_home is None:
  26. _refuse_parked(runner, getattr(source, '_rejected_route_profile', None))
  27. source.profile_route_rejected = True
  28. runtime_home = home
  29. source._authorization_profile_home = home
  30. # The routed profile's own ledger admits the delivery; an unserved route is refused. A park
  31. # can keep the profile's retiring authority (a turn outlived its Stop), which admits nothing.
  32. authority = authority_for_home(runner, runtime_home)
  33. if authority is None or getattr(authority, 'retiring', False):
  34. _refuse_parked(runner, getattr(source, 'profile', None) or profile)
  35. if authority is None:
  36. raise RuntimeStoreError('profile_mismatch')
  37. with native_callback(runner, event, home, profile):
  38. async with _async_profile_runtime_scope(runtime_home):
  39. yield authority
  40. def _refuse_parked(runner, profile):
  41. """A delivery routed to a parked profile is retryable, never refused for good: a rescan serves the
  42. profile again, so it is ``runtime_draining`` (a webhook answers 503; relay acknowledges and tells
  43. the sender, ``parked_route_refusal``), not the permanent ``profile_mismatch`` of a route to a
  44. profile this gateway does not serve."""
  45. from gateway.run_runtime import profile_parked
  46. if profile_parked(runner, profile):
  47. from gateway.parked_route_refusal import ParkedProfileRefusal
  48. raise ParkedProfileRefusal(profile)
  49. def parked_profile_refusal(runner, profile, route_config, admission_source):
  50. """A 503 for a webhook whose profile is not served right now, else None: a ``/p/<profile>/``
  51. prefix that is parked or has no serving authority here, or a coalesced route that
  52. ``profile_routes`` sends to a profile admission would refuse (a coalesced delivery is acknowledged
  53. before admission applies that route). A parked target is retryable (``Retry-After``); any other
  54. rejected target gets the answer the immediate route gives when admission refuses it.
  55. ``admission_source`` returns the ``SessionSource`` admission builds for this delivery
  56. (``WebhookAdapter._admission_source``), so ``profile_routes`` match here exactly as there. Answered
  57. before the delivery id is recorded: coalesced, ``deliver_only`` and ``cron_job`` routes act before
  58. (or without) an admission, so the in-memory id is their only replay guard, and a recorded id would
  59. answer the provider's retry as a duplicate once the profile is served again."""
  60. from aiohttp import web
  61. if _prefix_not_served(runner, profile):
  62. error = f'Profile {profile} is not being served; retry this delivery'
  63. else:
  64. refusal = _coalesced_route_refusal(runner, profile, route_config, admission_source)
  65. if refusal is None:
  66. return None
  67. if refusal == 'refused':
  68. # ``WebhookAdapter._dispatch_agent_run``'s answer when admission refuses the delivery.
  69. return web.json_response({'error': 'Admission unavailable; retry this delivery'}, status=503)
  70. error = 'The routed profile is not being served right now; retry this delivery'
  71. from gateway.parked_route_refusal import ParkedRouteRefused
  72. return web.json_response({'error': error}, status=ParkedRouteRefused.status, headers=ParkedRouteRefused.headers)
  73. def _prefix_not_served(runner, profile):
  74. """Parked, or on disk with no serving authority here (its own gateway still runs, or it is not
  75. picked up yet): the URL prefix resolves against the profiles on disk, the registry says what is
  76. served."""
  77. from gateway.run_runtime import profile_parked
  78. if profile_parked(runner, profile):
  79. return True
  80. if not profile or getattr(runner, 'session_authorities', None) is None:
  81. return False
  82. from gateway.session_authorities import authority_for_home
  83. from hermes_cli.profiles import get_profile_dir
  84. authority = authority_for_home(runner, get_profile_dir(profile))
  85. return authority is None or getattr(authority, 'retiring', False) is True
  86. def _coalesced_route_refusal(runner, profile, route_config, admission_source):
  87. """How admission would refuse a coalesced bare-route delivery that ``profile_routes`` sends away:
  88. ``'parked'`` (retryable), ``'refused'`` (a profile neither served here nor parked: standalone,
  89. unknown, or without a live authority), else None. Reads the profile the admission's own source
  90. resolved (``build_source`` asks ``GatewayRunner._profile_name_for_source`` on the same chat_id,
  91. user_id and names), so this edge and admission give one answer: an explicit ``/p/<profile>/``
  92. wins over ``profile_routes`` there, and routes apply only under multiplexing."""
  93. if profile or not isinstance(route_config.get('coalesce'), dict):
  94. return None
  95. from gateway.run_runtime import profile_parked
  96. source = admission_source()
  97. if getattr(source, 'profile_route_rejected', False):
  98. # A parked target leaves the served set, so the resolver rejects it; admission retries
  99. # that one (``_rejected_route_profile``) and refuses any other rejection for good.
  100. rejected = getattr(source, '_rejected_route_profile', None)
  101. return 'parked' if rejected and profile_parked(runner, rejected) else 'refused'
  102. routed = source.profile
  103. if not routed:
  104. return None
  105. if profile_parked(runner, routed):
  106. return 'parked'
  107. return 'refused' if _prefix_not_served(runner, routed) else None
  108. async def admit_producer(adapter, event):
  109. """Commit a producer delivery; ``None`` when a plugin skipped or consumed it.
  110. ``pre_gateway_dispatch`` and ``post_gateway_admission`` (#129958) fire once per admission, like
  111. ``GatewayInboundMixin._handle_message``: before the commit (a skipped or consumed delivery is
  112. never admitted, a rewrite is what gets committed; a consumer's reply rides
  113. ``event._consumer_reply`` for the caller's own reply leg), and never for a replay of an
  114. already-committed delivery (provider retry, relay re-send) nor at execution (PLG-5)."""
  115. async with producer_scope(adapter, event) as authority:
  116. from gateway.config import Platform
  117. from gateway.run_inbound_consumer import POST_ADMISSION_HOOK
  118. from hermes_cli.lifecycle import has_hook
  119. webhook = adapter.platform == Platform.WEBHOOK
  120. hooks = [name for name in ('pre_gateway_dispatch', POST_ADMISSION_HOOK) if has_hook(name)]
  121. if not (webhook or hooks):
  122. return await _admit_native(authority, event)
  123. from gateway.session_authority import native_write_started
  124. payload, principal = await _prepared(authority, event)
  125. writing = native_write_started.set(False)
  126. try:
  127. # An identical delivery (provider retry, relay re-send) overlapping this one waits here, then
  128. # rechecks the ledger: the retry check, the hooks and the commit run once per delivery.
  129. async with _delivery_turn((authority.profile_id, principal, event.message_id)):
  130. committed = payload, _committed(authority, principal, event.message_id)
  131. return await _admit_once(authority, event, webhook, hooks, committed)
  132. except (Exception, asyncio.CancelledError) as exc:
  133. # Refused after preparing (a command-shaped rewrite, a drain during the hooks, a conflict),
  134. # or cancelled (relay disconnect, shutdown) before the admission write began. A cancelled
  135. # write still commits (it is shielded): the bytes it commits are never released here.
  136. if not (isinstance(exc, asyncio.CancelledError) and native_write_started.get()):
  137. await asyncio.shield(_release_preparation(authority, payload))
  138. raise
  139. finally:
  140. native_write_started.reset(writing)
  141. async def _admit_once(authority, event, webhook, hooks, committed):
  142. from gateway.run_inbound_consumer import POST_ADMISSION_HOOK
  143. if webhook:
  144. prior = await _webhook_retry(authority, event, *committed)
  145. if prior is not None:
  146. return prior
  147. admitted = event
  148. if hooks and (webhook or not committed[1]):
  149. runner = authority.runner
  150. if 'pre_gateway_dispatch' in hooks:
  151. admitted = await runner._hm_pre_gateway_dispatch_hook(event, event.source)
  152. if admitted is None:
  153. await _release_preparation(authority, committed[0])
  154. return None
  155. if admitted is not event:
  156. # The rewrite is committed with the fingerprint of the delivery as received:
  157. # a provider retry or relay re-send resends that, and resolves to this row.
  158. from gateway.session_ingress_media import ingress_fingerprint
  159. admitted._ingress_sha256 = ingress_fingerprint(committed[0], event.raw_message)
  160. if POST_ADMISSION_HOOK in hooks:
  161. from gateway.run_inbound_consumer import run_post_admission_hook
  162. consumed, event._consumer_reply = await run_post_admission_hook(
  163. runner, admitted, admitted.source, runner._session_key_for_source(admitted.source))
  164. if consumed:
  165. await _release_preparation(authority, committed[0])
  166. return None
  167. return await _admit_native(authority, admitted)
  168. async def _admit_native(authority, event):
  169. # No adapter task brackets a producer's turn: the executor runs its adapter lifecycle (typing,
  170. # processing hooks) once the row starts executing (``session_producer_lifecycle``).
  171. event._producer_delivery = True
  172. return await authority.admit_native(event)
  173. # Producer deliveries in flight by (profile, principal, provider id) -> [lock, holder + waiters]. This
  174. # gateway is the only owner of producer ingress, so in-process coordination covers every request.
  175. _deliveries = {}
  176. @asynccontextmanager
  177. async def _delivery_turn(key):
  178. if not key[2]: # no provider id: admission mints one, nothing can overlap
  179. yield
  180. return
  181. from gateway.session_authority import keyed_turn
  182. async with keyed_turn(_deliveries, key):
  183. yield
  184. async def _release_preparation(authority, payload):
  185. """A delivery a plugin skipped or consumed, or one refused or cancelled after preparing, is never
  186. admitted: release the media its preparation (``_prepared``) copied into the retained store. A path
  187. another admission or an in-flight capture holds is kept, as for a refused admission."""
  188. from gateway.session_ingress_media import release_unheld_media
  189. if media := payload['native_text_v1'].get('media'):
  190. await asyncio.to_thread(release_unheld_media, authority.db, media, retain_history=False)
  191. async def _prepared(authority, event):
  192. """``(validated payload, principal)`` of this delivery. Preparing the payload runs the same
  193. authorization as admission, so a refused sender raises here."""
  194. import json
  195. from gateway.session_envelope import prepare_native, restore_native
  196. from gateway.session_ingress_media import abandon_capture, capture_lease
  197. with capture_lease(authority) as scope:
  198. try:
  199. payload = await prepare_native(authority.runner, event)
  200. except BaseException:
  201. await abandon_capture(scope)
  202. raise
  203. source = restore_native(payload).source
  204. identity = json.dumps([source.profile, source.platform.value, source.chat_id,
  205. source.thread_id, source.user_id], separators=(',', ':'))
  206. return payload, 'messaging:' + identity
  207. def _committed(authority, principal, request_id):
  208. """The rows of this delivery's identity in the FIFO."""
  209. return authority.db._read_all('SELECT admission_id FROM session_admissions '
  210. 'WHERE principal_id=? AND request_id=?', (principal, request_id))
  211. async def _webhook_retry(authority, event, payload, rows):
  212. # One-shot finalization ends the route. Resolve an existing provider receipt
  213. # before SessionStore would rotate it into a new physical session.
  214. from gateway.session_ingress_media import ingress_fingerprint
  215. from hermes_state_runtime import get_session_admission
  216. if not rows:
  217. return None
  218. row = get_session_admission(authority.db, admission_id=rows[0]['admission_id'])
  219. retained = row['payload']['native_text_v1']
  220. if 'ingress_sha256' in retained:
  221. # Committed rewritten: the provider resends the delivery as received.
  222. same = retained['ingress_sha256'] == ingress_fingerprint(payload, event.raw_message)
  223. else:
  224. if 'webhook_delivery' not in retained and 'webhook_route' not in retained:
  225. # A pre-destination receipt is still a receipt, not permission to infer
  226. # again. Do not retrofit today's destination into its queued execution.
  227. payload['native_text_v1'].pop('webhook_delivery')
  228. payload['native_text_v1'].pop('webhook_route')
  229. if 'provenance' not in retained:
  230. payload['native_text_v1'].pop('provenance', None)
  231. same = row['payload'] == payload
  232. if len(rows) != 1 or not same:
  233. raise RuntimeStoreError('admission_conflict')
  234. event._webhook_duplicate = True
  235. receipt = authority._receipt(row)
  236. finalize_webhook(authority, receipt)
  237. return receipt
  238. def finalize_webhook(authority, receipt):
  239. """End only the settled admitted generation, atomically with the epoch fence."""
  240. import time
  241. if receipt.status != 'terminal' or receipt.execution_generation is None:
  242. return False
  243. sid = receipt.ref.session_id
  244. def write(conn):
  245. return authority.db._end_and_bump(conn, """
  246. UPDATE sessions SET ended_at=?, end_reason=?
  247. WHERE id=? AND ended_at IS NULL AND runtime_generation=?
  248. AND EXISTS (SELECT 1 FROM runtime_epoch WHERE singleton=1 AND epoch=?)
  249. AND EXISTS (SELECT 1 FROM session_admissions WHERE admission_id=?
  250. AND target_session_id=? AND status='terminal' AND generation=? AND owner_epoch=?)
  251. AND NOT EXISTS (SELECT 1 FROM session_admissions WHERE target_session_id=? AND status!='terminal')
  252. """, (time.time(), 'webhook_complete', sid, receipt.execution_generation,
  253. authority.epoch, receipt.admission_id, sid, receipt.execution_generation,
  254. receipt.authority_epoch, sid), sid, 'webhook_complete')
  255. return bool(authority.db._execute_write(write))
  256. async def recover_webhook_finalizations(authority):
  257. """Close settled original webhooks only after current route authorization succeeds."""
  258. from gateway.session_envelope import check_native_route
  259. from gateway.session_contract import SessionRef
  260. from hermes_state_runtime import RuntimeStoreError, get_session_admission
  261. rows = authority.db._read_all("""
  262. SELECT a.admission_id FROM sessions AS s
  263. JOIN session_admissions AS a ON a.target_session_id=s.id
  264. WHERE s.source='webhook' AND s.ended_at IS NULL
  265. AND a.status='terminal' AND a.generation IS NOT NULL
  266. AND json_extract(a.payload_json, '$.native_text_v1.source.platform')='webhook'
  267. AND json_extract(a.payload_json, '$.native_text_v1.automation') IS NULL
  268. """)
  269. results = {}
  270. for candidate in rows:
  271. row = get_session_admission(authority.db, admission_id=candidate['admission_id'])
  272. if row is None:
  273. continue
  274. sid = row['target_session_id']
  275. try:
  276. envelope = row['payload']['native_text_v1']
  277. entry = authority.runner.session_store.lookup_by_session_key(envelope['route'])
  278. if entry is None or authority.logical_owner(entry.session_id) != sid:
  279. raise RuntimeStoreError('admission_conflict')
  280. adapter = authority.runner._adapter_for_source(entry.origin) if entry.origin is not None else None
  281. target = authority.physical_target(SessionRef(authority.profile_id, sid))
  282. await check_native_route(authority.runner, row['payload'], target, entry.origin, adapter)
  283. results[sid] = 'finalized' if finalize_webhook(
  284. authority, authority._receipt(row)) else 'unchanged'
  285. except (KeyError, RuntimeStoreError):
  286. results[sid] = 'unavailable'
  287. return results