session_automation.py 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864
  1. """Trusted producer admission; not a client-selectable internal input flag.
  2. Reuse the private native route envelope and FIFO, retaining nonhuman turn semantics.
  3. A producer ACK means committed input, not successful inference or outbound delivery.
  4. """
  5. from copy import deepcopy
  6. from datetime import UTC, datetime
  7. import json
  8. import logging
  9. import re
  10. from gateway.config import Platform
  11. from gateway.platforms.event import MessageEvent, MessageType
  12. from gateway.session_contract import SessionRef
  13. from hermes_state_runtime import RuntimeStoreError, admit_session_input, list_session_admissions
  14. logger = logging.getLogger(__name__)
  15. def producer_identity(runner, event):
  16. identity = runner._completion_delivery_identity(event)
  17. if identity is None:
  18. raise RuntimeStoreError('invalid_params')
  19. return json.dumps(identity, separators=(',', ':'))
  20. def completion_admission(runner, event):
  21. from gateway.session_authorities import active_authority
  22. authority = active_authority(runner)
  23. if authority is None:
  24. return None
  25. from gateway.session_detached import route_entry
  26. entry = route_entry(authority, str(event.get('session_key') or ''))
  27. if entry is None:
  28. return None
  29. from hermes_state_runtime import list_session_admissions
  30. identity = producer_identity(runner, event)
  31. for row in list_session_admissions(authority.db, session_id=_route_owner(authority, entry), pending_only=False):
  32. descriptor = row['payload'].get('local_automation_v1') or row['payload'].get('native_text_v1', {}).get('automation', {})
  33. if identity in descriptor.get('identities', [descriptor.get('identity')]):
  34. return row
  35. return None
  36. async def deliver_heartbeat(runner, event):
  37. """Prepare, format and admit one liveness tick as one critical section with a completion's
  38. admit-and-withdraw (``admit_process_notice``): a completion that lands while this tick is on its
  39. way in withdraws it once it is in, and a tick checked after it sees it. True when admitted, None
  40. to drop it, False to requeue the original event."""
  41. from gateway.run import _format_gateway_process_notification
  42. from gateway.session_detached import follow_detached_pin
  43. async with _heartbeat_lock(runner):
  44. for _attempt in range(2):
  45. try:
  46. prepared = await prepare_heartbeat(runner, await follow_detached_pin(runner, event))
  47. except Exception:
  48. # One tick's failure must not drop the rest of the drained events.
  49. logger.exception("Heartbeat preparation error; the tick is requeued")
  50. return False
  51. text = prepared and _format_gateway_process_notification(prepared)
  52. if not text:
  53. return None
  54. try:
  55. delivered = await runner._inject_watch_notification(text, prepared)
  56. except Exception:
  57. logger.exception("Watch notification injection error")
  58. return False
  59. if delivered is False and prepared is not event:
  60. event['_refusal_permanent'] = prepared.get('_refusal_permanent') # the drain requeues the original
  61. if delivered is not False or not _replaced_tick_claimed(runner, prepared):
  62. return delivered
  63. # The tick it replaced started first and runs with its own output: admit this one alone.
  64. return False
  65. def _replaced_tick_claimed(runner, prepared):
  66. from gateway.session_authorities import active_authority
  67. from hermes_state_runtime import get_session_admission
  68. replaced = prepared.get('_replaces')
  69. authority = active_authority(runner) if replaced else None
  70. row = authority and get_session_admission(authority.db, admission_id=replaced)
  71. return bool(row) and row['status'] != 'queued'
  72. async def admit_process_notice(runner, authority, adapter, event, evt):
  73. """Admit a process notice. A completion and the withdrawal of its process's waiting ticks hold
  74. the heartbeat lock together; a tick already holds it (``deliver_heartbeat``)."""
  75. identity = producer_identity(runner, evt)
  76. if evt.get('type') == 'heartbeat':
  77. # The tick's raw output rides its stored admission: a later tick folds from the ledger.
  78. event.metadata['heartbeat_tick'] = {'output': str(evt.get('output') or ''),
  79. 'omitted': int(evt.get('omitted') or 0)}
  80. if evt.get('_replaces'):
  81. return await authority.admit_automation(adapter, event, identity, replaces=evt['_replaces'])
  82. if evt.get('type') != 'completion':
  83. return await authority.admit_automation(adapter, event, identity)
  84. async with _heartbeat_lock(runner):
  85. receipt = await _admit_folded_completion(authority, adapter, event, evt, identity)
  86. await withdraw_process_ticks(runner, evt)
  87. return receipt
  88. async def _admit_folded_completion(authority, adapter, event, evt, identity):
  89. """A completion landing behind another completion still waiting at the FIFO tail joins it, so a
  90. busy session runs ONE turn for every result that piled up instead of one turn each. The waiting
  91. row is cancelled in this admission's own transaction (``replaces``); when it was claimed first,
  92. this completion queues alone."""
  93. import asyncio
  94. waiting = await asyncio.to_thread(_waiting_completion, authority, evt, event)
  95. if waiting is not None:
  96. unfolded = event.text, dict(event.metadata)
  97. _fold_completion(event, waiting, identity)
  98. try:
  99. return await authority.admit_automation(adapter, event, identity, replaces=waiting['admission_id'],
  100. fold=waiting['_skip'])
  101. except RuntimeStoreError as exc:
  102. if exc.reason != 'stale_generation':
  103. raise
  104. event.text, event.metadata = unfolded
  105. return await authority.admit_automation(adapter, event, identity)
  106. def _waiting_completion(authority, evt, event):
  107. """The route's own completion row that is the LAST queued input of the session, or None. Only the
  108. tail folds: a completion never jumps ahead of, or behind, a prompt queued after it. The finishing
  109. process's own queued liveness ticks do not count as the tail: this completion withdraws them
  110. (``withdraw_process_ticks``); their admission ids ride the row as ``_skip`` for the write guard."""
  111. from gateway.session_detached import route_entry
  112. entry = route_entry(authority, str(evt.get('session_key') or ''))
  113. if entry is None:
  114. return None
  115. queued = [row for row in list_session_admissions(authority.db, session_id=_route_owner(authority, entry))
  116. if row['status'] == 'queued']
  117. process = str(evt.get('session_id') or '')
  118. skip = []
  119. while queued and _queued_tick(queued[-1], process, entry.session_key):
  120. skip.append(queued.pop()['admission_id'])
  121. tail = queued[-1] if queued else None
  122. if tail is None or tail['principal_id'] != 'automation:' + entry.session_key:
  123. return None
  124. if [identity[0] for identity in _identities([tail['request_id']])] != ['completion']:
  125. return None
  126. descriptor = _descriptor(tail['payload'])
  127. if (descriptor.get('notification_category', 'result') != event.metadata.get('notification_category', 'result')
  128. or descriptor.get('display_kind') != event.metadata.get('display_kind')):
  129. return None
  130. return {**tail, '_skip': skip}
  131. _FOLD_HEADER = ('[IMPORTANT: {count} background process results arrived while this session was busy. '
  132. 'Treat them as one batch and give one consolidated response; preserve failures and '
  133. 'actionable results.]')
  134. _FOLD_HEADER_RE = re.compile(r'\[IMPORTANT: \d+ background process results arrived while this session was busy\.'
  135. r'[^\]]*\]\n\n')
  136. def _fold_completion(event, waiting, identity):
  137. from gateway.run_notifications import INTERNAL_NOTIFICATION_FOOTER, _mark_internal_notification
  138. from tools.process_registry_notifications import process_completion_display_text
  139. descriptor = _descriptor(waiting['payload'])
  140. identities = sorted({*(descriptor.get('identities') or [waiting['request_id']]),
  141. *(event.metadata.get('automation_identities') or [identity])})
  142. def body(text):
  143. text = text.rstrip().removesuffix(INTERNAL_NOTIFICATION_FOOTER).rstrip()
  144. folded = _FOLD_HEADER_RE.match(text)
  145. return text[folded.end():] if folded else text
  146. event.text = _mark_internal_notification('\n\n'.join((
  147. _FOLD_HEADER.format(count=len(identities)), body(waiting['payload']['text']), body(event.text))))
  148. event.metadata['automation_identities'] = identities
  149. if 'display_text' in event.metadata:
  150. event.metadata['display_text'] = process_completion_display_text([{}] * len(identities))
  151. async def prepare_heartbeat(runner, event):
  152. """The liveness tick to admit for ``event``, or None to drop it because its process's completion
  153. is already in the session: a later tick would be a stale "still running" turn. The caller holds
  154. the heartbeat lock.
  155. Otherwise a tick of the same process still queued is replaced: this one carries its stored
  156. output, and ``_replaces`` names it, so its cancellation commits in this tick's own admission
  157. (``admit_automation``) and a refused replacement leaves it queued. A tick's output is never sent
  158. again, so a busy session gets one waiting turn holding every tick's output instead of one turn
  159. per tick. A queued tick that started or settled first ran with its own output, and this one
  160. queues behind it. A retry of a tick already admitted is dropped."""
  161. import asyncio
  162. from gateway.session_authorities import active_authority
  163. from gateway.session_detached import route_entry
  164. authority = active_authority(runner)
  165. entry = route_entry(authority, str(event.get('session_key') or '')) if authority else None
  166. if entry is None:
  167. return event
  168. owner, process = _route_owner(authority, entry), str(event.get('session_id') or '')
  169. if await asyncio.to_thread(_process_completed, runner, authority, owner, event):
  170. logger.debug("Dropped heartbeat %s of process %s: its completion is already admitted",
  171. event.get('event_id'), process)
  172. return None
  173. from hermes_state_runtime import find_session_admission
  174. if await asyncio.to_thread(find_session_admission, authority.db, principal_id='automation:' + entry.session_key,
  175. session_id=owner, request_id=producer_identity(runner, event)):
  176. return None # a retry of a tick already admitted (its acknowledgement lost)
  177. pending = await asyncio.to_thread(list_session_admissions, authority.db, session_id=owner)
  178. waiting = next((row for row in pending if _queued_tick(row, process, entry.session_key)), None)
  179. earlier = waiting and _descriptor(waiting['payload']).get('tick')
  180. if not isinstance(earlier, dict):
  181. return event
  182. from tools.process_registry_notifications import join_heartbeat_outputs
  183. return {**event, **join_heartbeat_outputs(earlier, event), '_replaces': waiting['admission_id']}
  184. def _descriptor(payload):
  185. return payload.get('local_automation_v1') or (payload.get('native_text_v1') or {}).get('automation') or {}
  186. def _tick_metadata(metadata):
  187. """A liveness tick's stored output (``heartbeat_tick``): what a later tick folds in."""
  188. tick = metadata.get('heartbeat_tick')
  189. if tick is None:
  190. return {}
  191. if (not isinstance(tick, dict) or set(tick) != {'output', 'omitted'} or not isinstance(tick['output'], str)
  192. or type(tick['omitted']) is not int or tick['omitted'] < 0):
  193. raise RuntimeStoreError('invalid_params')
  194. return {'tick': dict(tick)}
  195. def _replace_queued(admission_id, session_id, replaced):
  196. """Write guard: cancel the replaced queued tick in the replacement's own transaction; refuse the
  197. replacement when that tick was claimed or settled first (its output already ran)."""
  198. def guard(conn):
  199. changed = conn.execute("UPDATE session_admissions SET status='terminal',outcome='cancelled' "
  200. "WHERE admission_id=? AND target_session_id=? AND status='queued'",
  201. (admission_id, session_id)).rowcount
  202. if changed != 1:
  203. raise RuntimeStoreError('stale_generation')
  204. replaced[:] = [admission_id]
  205. return guard
  206. _FOLDED_TEXT = '[folded into a later completion batch]'
  207. def _fold_queued(admission_id, session_id, replaced, skip=()):
  208. """``_replace_queued`` for a completion fold, which must still be the FIFO tail when it commits
  209. (a prompt admitted since the tail was read keeps the earlier result ahead of it; ``skip`` names
  210. the finishing process's ticks, withdrawn right after). The replaced row keeps only its own
  211. identity: its text and earlier identities ride the batch, so no fold stores them twice."""
  212. replace = _replace_queued(admission_id, session_id, replaced)
  213. def guard(conn):
  214. row = conn.execute("SELECT seq, payload_json FROM session_admissions WHERE admission_id=? "
  215. "AND target_session_id=? AND status='queued'", (admission_id, session_id)).fetchone()
  216. later = row and conn.execute("SELECT admission_id FROM session_admissions WHERE target_session_id=? "
  217. "AND status='queued' AND seq>?", (session_id, row[0])).fetchall()
  218. if row is None or any(other[0] not in skip for other in later):
  219. raise RuntimeStoreError('stale_generation')
  220. replace(conn)
  221. payload = json.loads(row[1])
  222. payload['text'] = _FOLDED_TEXT
  223. _descriptor(payload).pop('identities', None)
  224. conn.execute('UPDATE session_admissions SET payload_json=? WHERE admission_id=?',
  225. (json.dumps(payload, separators=(',', ':'), ensure_ascii=False), admission_id))
  226. return guard
  227. async def withdraw_process_ticks(runner, event):
  228. """A process's completion is in the session: its ticks still queued there would run after it as
  229. stale "still running" turns, and the completion carries the final output, so they are cancelled.
  230. A tick that already started is left alone. Never raises: the completion is already admitted, and
  231. a retry of it takes the lost-acknowledgement path, not this one. The caller holds the heartbeat
  232. lock (``admit_process_notice``)."""
  233. import asyncio
  234. from gateway.session_authorities import active_authority
  235. try:
  236. from gateway.session_detached import route_entry
  237. authority = active_authority(runner)
  238. entry = route_entry(authority, str(event.get('session_key') or '')) if authority else None
  239. if entry is None:
  240. return
  241. finished = {identity[1] for identity in _identities(
  242. [producer_identity(runner, event), *(event.get('_automation_identities') or ())])
  243. if identity[0] == 'completion' and len(identity) > 1}
  244. pending = await asyncio.to_thread(
  245. list_session_admissions, authority.db, session_id=_route_owner(authority, entry))
  246. for row in pending:
  247. if any(_queued_tick(row, process, entry.session_key) for process in finished):
  248. await _withdraw(authority, row)
  249. except Exception:
  250. logger.warning("Could not withdraw the waiting heartbeats of finished process %s",
  251. event.get('session_id'), exc_info=True)
  252. def _heartbeat_lock(runner):
  253. # A tick's check-and-admit and a completion's admit-and-withdraw: interleaved, a tick checked
  254. # before the completion landed is admitted after its withdrawal and runs as a stale turn.
  255. import asyncio
  256. lock = runner.__dict__.get('_heartbeat_lock')
  257. if lock is None:
  258. lock = runner.__dict__['_heartbeat_lock'] = asyncio.Lock()
  259. return lock
  260. def _process_completed(runner, authority, owner, event):
  261. """Whether the completion of tick ``event``'s process is admitted into the session ``owner``: by
  262. its exact identity (``started_at`` names the process incarnation), or as a sibling in a batch,
  263. pending or settled: only the batch's primary is its ``request_id``."""
  264. completion = runner._completion_delivery_identity(
  265. {'type': 'completion', 'session_id': event.get('session_id'), 'started_at': event.get('started_at')})
  266. if completion is None:
  267. return False
  268. identity = json.dumps(completion, separators=(',', ':'))
  269. with authority.db._read_ctx() as conn:
  270. if conn.execute('SELECT 1 FROM session_admissions WHERE target_session_id=? AND request_id=?',
  271. (owner, identity)).fetchone():
  272. return True
  273. for row in list_session_admissions(authority.db, session_id=owner, pending_only=False):
  274. descriptor = row['payload'].get('local_automation_v1') or row['payload'].get('native_text_v1', {}).get('automation', {})
  275. if identity in (descriptor.get('identities') or ()):
  276. return True
  277. return False
  278. def _identities(raws):
  279. parsed = []
  280. for raw in raws:
  281. try:
  282. identity = json.loads(raw)
  283. except (TypeError, ValueError):
  284. continue
  285. if isinstance(identity, list) and identity:
  286. parsed.append(identity)
  287. return parsed
  288. def _queued_tick(row, process, route):
  289. """Whether *row* is the gateway's own queued liveness tick of *process* on *route*: the route's
  290. producer admitted ``["heartbeat", process, event_id]``. A client input whose id merely looks like
  291. one, and the ``/heartbeat`` wake (four parts), never match."""
  292. if row['status'] != 'queued' or row['principal_id'] != 'automation:' + route:
  293. return False
  294. return any(len(identity) == 3 and identity[:2] == ['heartbeat', process]
  295. for identity in _identities([row['request_id']]))
  296. async def _withdraw(authority, row):
  297. """Cancel the gateway's own queued automation *row*; False when it was claimed or settled first."""
  298. from gateway.session_contract import SessionActor
  299. # The row's own principal, with the operator access a LOCAL route requires of anyone but its viewer.
  300. actor = SessionActor(row['principal_id'], authority.profile_id,
  301. frozenset({'session:submit', 'session:control', 'session:operator'}), 'producer-withdraw')
  302. try:
  303. receipt = await authority.cancel_queued(
  304. actor, SessionRef(authority.profile_id, row['target_session_id']), row['admission_id'])
  305. except RuntimeStoreError:
  306. return False
  307. return receipt.outcome == 'cancelled'
  308. def _route_owner(authority, entry):
  309. # Compression moves the route's transcript to a child; admissions stay on the logical root.
  310. return (entry.origin.chat_id if entry.origin is not None and entry.origin.platform == Platform.LOCAL
  311. else authority.logical_owner(entry.session_id))
  312. def _owned_by(db, session_id, current):
  313. """Whether ``session_id``, or a (nested) delegate_task conversation it runs under, is the route's
  314. live ``current`` session or compressed into it. The walk stops at the first such hop: a resumed
  315. delegate run is the chat's own conversation, never the parent it was spawned from. As in the
  316. completion pre-flight (``_lookup_completion_owner``), only ``model_config._delegate_from`` leads
  317. out of an execution transcript: ``parent_session_id`` also records compression and branches.
  318. Likewise a session closed at a user boundary (/new, a resume away) leads nowhere."""
  319. from gateway.run import _USER_BOUNDARY_END_REASONS
  320. lineage = [session_id]
  321. while lineage[-1] != current and db.get_compression_tip(lineage[-1]) != current:
  322. row = db.get_session(lineage[-1]) or {}
  323. parent = db.get_session_model_config_value(lineage[-1], '_delegate_from')
  324. if (row.get('ended_at') and row.get('end_reason') in _USER_BOUNDARY_END_REASONS) or parent in (None, *lineage):
  325. return False
  326. lineage.append(parent)
  327. return True
  328. def _owner(runner, event, authority):
  329. from gateway.session_detached import route_entry
  330. route = event.metadata.get('gateway_session_key') or runner.session_store._generate_session_key(event.source)
  331. entry = route_entry(authority, route)
  332. if entry is None or entry.suspended:
  333. raise RuntimeStoreError('not_found')
  334. if runner.session_store._generate_session_key(event.source) != route:
  335. raise RuntimeStoreError('admission_conflict')
  336. expected = event.metadata.get('gateway_session_id')
  337. # A completed child may follow compression, and a process a subagent started its parent
  338. # conversation, but never /new or an unrelated resume.
  339. if not expected or not _owned_by(authority.db, expected, entry.session_id):
  340. raise RuntimeStoreError('admission_conflict')
  341. return entry
  342. def automation_notification_metadata(metadata):
  343. """Validate trusted producer category; omit the default from old fingerprints."""
  344. category = metadata.get('notification_category', 'result')
  345. if category not in ('result', 'diagnostic'):
  346. raise RuntimeStoreError('invalid_params')
  347. return {'notification_category': category} if category == 'diagnostic' else {}
  348. _DISPLAY_TEXT_LIMIT = 2000
  349. def automation_display_metadata(metadata):
  350. """Validate a producer's card (an async result, or a liveness tick's one-line receipt): both keys
  351. or neither, kind from the closed producer set."""
  352. if 'display_kind' not in metadata and 'display_text' not in metadata:
  353. return {}
  354. from gateway.response_filters import PRODUCER_CARD_KINDS
  355. kind, text = metadata.get('display_kind'), metadata.get('display_text')
  356. if (kind not in PRODUCER_CARD_KINDS or not isinstance(text, str) or not text.strip()
  357. or len(text) > _DISPLAY_TEXT_LIMIT):
  358. raise RuntimeStoreError('invalid_params')
  359. return {'display_kind': kind, 'display_text': text}
  360. # notification_origin / original_trigger_message_id are producer debug context (#52694) and are not
  361. # persisted; notification_category="diagnostic" rides the snapshot so the replayed wake still mutes,
  362. # and display_kind/display_text so the persisted row is the card the producer named; heartbeat_tick is
  363. # a liveness tick's own output, persisted so a later tick of its process folds from the ledger.
  364. # gateway_proven_session_id / gateway_completion_ids are main's delivery-time ownership proof for the
  365. # classic resolver (802fd5782f3, fd4e277c849). They are accepted and not persisted (a sibling set in the
  366. # fingerprint would make the same completion's retry a new identity); execute_admission re-stamps the
  367. # proof at execution from the target the pre-claim check validated, and the turn's resolver reads it.
  368. _AUTOMATION_METADATA = frozenset({'gateway_session_key', 'gateway_session_id', 'automation_identities',
  369. 'turn_author', 'notification_origin', 'original_trigger_message_id', 'notification_category',
  370. 'display_kind', 'display_text', 'gateway_proven_session_id', 'gateway_completion_ids', 'heartbeat_tick'})
  371. def admission_display(payload):
  372. """An automation admission's async-result card (plus its model text) for the claim's
  373. ``message.start``: a viewer that did not submit the turn (the classic CLI, idle while a
  374. background process it started finishes in the gateway) paints the same header the submitting
  375. surface would. ``{}`` for every other input."""
  376. descriptor = payload.get('local_automation_v1') or (payload.get('native_text_v1') or {}).get('automation') or {}
  377. kind, text = descriptor.get('display_kind'), descriptor.get('display_text')
  378. if not kind or not isinstance(text, str) or not text.strip():
  379. return {}
  380. card = {'display_kind': kind, 'display_text': text, 'text': str(payload.get('text') or '')}
  381. if descriptor.get('notification_category') == 'diagnostic':
  382. card['notification_category'] = 'diagnostic'
  383. return card
  384. def committed_route_envelope(authority, entry):
  385. """The route's last committed native envelope with connector provenance (what a producer turn
  386. borrows), or ``None`` while no human message was admitted there."""
  387. from hermes_state_runtime import list_session_admissions
  388. # Admissions belong to the logical root; ``entry.session_id`` is only the transcript tip.
  389. prior = list_session_admissions(authority.db, session_id=authority.logical_owner(entry.session_id),
  390. pending_only=False)
  391. envelope = next((r['payload']['native_text_v1'] for r in reversed(prior)
  392. if 'native_text_v1' in r['payload']), None)
  393. return envelope if envelope is not None and 'provenance' in envelope else None
  394. def snapshot_automation(authority, adapter, event, identity):
  395. runner = authority.runner
  396. if (not event.internal or event.message_type != MessageType.TEXT or event.is_command()
  397. or not isinstance(event.text, str) or not identity
  398. or event.media_urls or event.prompt_response or event.source.platform == Platform.API_SERVER
  399. or set(event.metadata) - _AUTOMATION_METADATA):
  400. raise RuntimeStoreError('invalid_params')
  401. notification = (automation_notification_metadata(event.metadata) | automation_display_metadata(event.metadata)
  402. | _tick_metadata(event.metadata))
  403. entry = _owner(runner, event, authority)
  404. from gateway.session_detached import detached_route_of
  405. if event.source.platform == Platform.LOCAL or detached_route_of(event.source):
  406. return snapshot_local_automation(authority, adapter, event, identity, entry)
  407. from gateway.session_envelope import restore_native
  408. owner = authority.logical_owner(entry.session_id)
  409. envelope = committed_route_envelope(authority, entry)
  410. if envelope is None:
  411. raise RuntimeStoreError('not_found')
  412. # Persisted origins deliberately omit relay trust. Borrow only an exact
  413. # committed source's private proof, revalidated against the live connector.
  414. # Recheck only the source/provenance: a completed input's retained media is already released.
  415. restored = restore_native({'text': event.text, 'native_text_v1': {
  416. key: value for key, value in envelope.items() if key != 'media'}}, runner)
  417. # The wake runs as the committed human source; the producer's source, rebuilt from the route's
  418. # stored origin, only names the route. Its labels, first speaker and stored profile drift from
  419. # the latest committed turn, so compare the route both resolve to now (``_owner`` checked it).
  420. if (runner.session_store._generate_session_key(restored.source) != entry.session_key
  421. or runner._adapter_for_source(restored.source) is not adapter):
  422. raise RuntimeStoreError('admission_conflict')
  423. provenance = deepcopy(envelope['provenance'])
  424. source = deepcopy(envelope['source'])
  425. # Never anchor the wake to the stale human message; ``to_dict`` already omits a null anchor.
  426. source.pop('message_id', None)
  427. if 'relay' in provenance:
  428. # The relay seal binds the source; restore_native verified it over the anchored source.
  429. from gateway.session_ingress_context import reseal_relay
  430. provenance = reseal_relay(adapter, source, provenance)
  431. # Producer timestamps and platform reply IDs change on retry. Neither belongs
  432. # in the identity/fingerprint of the same immutable completion.
  433. envelope = {'source': source, 'route': entry.session_key,
  434. 'timestamp': datetime.fromtimestamp(0, UTC).isoformat(),
  435. 'event': {'message_id': identity}, 'provenance': provenance,
  436. 'automation': {'identity': identity, 'owner': owner}}
  437. envelope['automation'].update(notification)
  438. if getattr(event, '_heartbeat_session_id', None):
  439. envelope['automation']['heartbeat'] = event._heartbeat_session_id
  440. identities = event.metadata.get('automation_identities')
  441. if identities:
  442. envelope['automation']['identities'] = sorted(set(identities))
  443. return {'text': event.text, 'native_text_v1': envelope}, entry
  444. def check_automation_route(runner, payload, session_id, available_source, adapter):
  445. from gateway.session_envelope import restore_native
  446. event = restore_native(payload, runner)
  447. envelope = payload['native_text_v1']
  448. from gateway.session_authorities import active_authority
  449. authority = active_authority(runner)
  450. if authority is None: # a late preflight for a profile this runtime no longer serves
  451. raise RuntimeStoreError('not_found')
  452. entry = _owner(runner, event, authority)
  453. if (entry.session_id != session_id
  454. or envelope['automation']['owner'] != authority.logical_owner(session_id)
  455. or runner.session_store._generate_session_key(available_source) != entry.session_key
  456. or adapter is None or runner._adapter_for_source(event.source) is not adapter):
  457. raise RuntimeStoreError('admission_conflict')
  458. return event.source, entry.session_key
  459. def _local_automation_owner(authority, adapter, event, entry):
  460. """``(ref, live)`` a local automation lands on. A conversation its chat moved past takes its
  461. results as a Desktop session does (gateway/session_detached.py): its app viewers get them."""
  462. from gateway.session_detached import detached_live, detached_route_of
  463. if detached_route_of(event.source):
  464. found = detached_live(authority, entry.session_key)
  465. if (found is None or found[1].source is not event.source or not adapter.authorize_source(event.source)
  466. or authority.runner._adapter_for_source(found[1].source) is not adapter):
  467. raise RuntimeStoreError('permission_denied')
  468. return found
  469. from gateway.session_local_recovery import restore_local_session
  470. ref = restore_local_session(authority, event.source.chat_id)
  471. live = authority.sessions[ref.session_id]
  472. if (live.source is not event.source or not adapter.authorize_source(event.source)
  473. or authority.runner._adapter_for_source(live.source) is not adapter):
  474. raise RuntimeStoreError('permission_denied')
  475. return ref, live
  476. def snapshot_local_automation(authority, adapter, event, identity, entry):
  477. ref, _live = _local_automation_owner(authority, adapter, event, entry)
  478. descriptor = {'identity': identity, 'owner': ref.session_id,
  479. 'route': entry.session_key, 'target': entry.session_id}
  480. descriptor.update(automation_notification_metadata(event.metadata))
  481. descriptor.update(automation_display_metadata(event.metadata))
  482. descriptor.update(_tick_metadata(event.metadata))
  483. if event.metadata.get('turn_author') is not None:
  484. from agent.turn_author import parse_turn_author
  485. descriptor['turn_author'] = parse_turn_author(event.metadata['turn_author'])
  486. if descriptor['turn_author'] is None:
  487. raise RuntimeStoreError('invalid_params')
  488. if event.metadata.get('automation_identities'):
  489. descriptor['identities'] = sorted(set(event.metadata['automation_identities']))
  490. if getattr(event, '_heartbeat_session_id', None):
  491. descriptor['heartbeat'] = event._heartbeat_session_id
  492. return {'text': event.text, 'local_automation_v1': descriptor}, entry
  493. def check_local_automation(authority, ref, row, *, route=None):
  494. live = authority.sessions.get(ref.session_id)
  495. route = route or (live.route if live is not None else None)
  496. payload = row['payload']
  497. descriptor = payload['local_automation_v1']
  498. from gateway.session_detached import own_detached_key, route_entry
  499. # Admitted while its chat was elsewhere: the conversation's own key, whichever route it is on now.
  500. admitted = descriptor.get('route') if own_detached_key(descriptor.get('route'), ref.session_id) else route
  501. if (not route or set(payload) != {'text', 'local_automation_v1'}
  502. or row['principal_id'] != 'automation:' + admitted
  503. or descriptor['owner'] != ref.session_id or descriptor['route'] != admitted
  504. or descriptor['identity'] != row['request_id']):
  505. raise RuntimeStoreError('permission_denied')
  506. entry = route_entry(authority, route)
  507. if (entry is None or entry.suspended or (entry.session_id != descriptor['target']
  508. and authority.db.get_compression_tip(descriptor['target']) != entry.session_id)):
  509. raise RuntimeStoreError('admission_conflict')
  510. def restore_local_automation(authority, ref, row):
  511. check_local_automation(authority, ref, row)
  512. live = authority.sessions[ref.session_id]
  513. descriptor = row['payload']['local_automation_v1']
  514. from gateway.session_detached import route_entry
  515. entry = route_entry(authority, live.route)
  516. event = MessageEvent(text=row['payload']['text'], source=live.source, internal=True,
  517. message_id=descriptor['identity'], metadata={'gateway_session_key': live.route,
  518. 'gateway_session_id': entry.session_id})
  519. event.metadata.update(automation_notification_metadata(descriptor))
  520. event.metadata.update(automation_display_metadata(descriptor))
  521. if descriptor.get('turn_author') is not None:
  522. event.metadata['turn_author'] = deepcopy(descriptor['turn_author'])
  523. if descriptor.get('heartbeat'):
  524. event._heartbeat_session_id = descriptor['heartbeat']
  525. return event
  526. async def admit_automation(authority, adapter, event, identity, replaces=None, fold=None):
  527. """``replaces``: a queued automation row of the same session this input supersedes (a liveness
  528. tick carrying the output of the one still waiting). Its cancellation commits in this admission's
  529. own transaction, so neither lands without the other. ``fold``: ``replaces`` is a completion this
  530. batch absorbs (``_fold_queued``), given the tick admission ids it may sit behind."""
  531. authority._require_admission_open()
  532. payload, entry = snapshot_automation(authority, adapter, event, identity)
  533. from gateway.session_authority import LiveSession
  534. sid = (payload.get('local_automation_v1') or payload['native_text_v1']['automation'])['owner']
  535. ref = SessionRef(authority.profile_id, sid)
  536. source, envelope = event.source, payload.get('native_text_v1')
  537. if envelope is not None and 'relay' in envelope['provenance']:
  538. # A restored origin carries no relay trust: bind the source the snapshot verified, or the
  539. # drain resolves no adapter for this session after a restart.
  540. from gateway.session_envelope import restore_native_source
  541. source = restore_native_source(payload, authority.runner)
  542. authority.sessions.setdefault(ref.session_id, LiveSession(source, entry.session_key))
  543. from functools import partial
  544. from gateway.session_runtime_workers import tracked_write
  545. admitted, replaced = partial(authority._admitted, ref, event), []
  546. if replaces is not None:
  547. def admitted(row, admitted=admitted):
  548. if replaced: # the guard ran in the committed write; an exact replay runs no guard
  549. authority._settle_cancelled(ref, replaces)
  550. return admitted(row)
  551. return await tracked_write(authority, partial(
  552. admit_session_input, authority.db, epoch=authority.epoch, principal_id='automation:' + entry.session_key,
  553. session_id=ref.session_id, request_id=identity, payload=deepcopy(payload),
  554. _authorize_write=authority._admission_gate(replaces and (
  555. _replace_queued(replaces, ref.session_id, replaced) if fold is None
  556. else _fold_queued(replaces, ref.session_id, replaced, fold)))),
  557. then=admitted, ordered=True, after=partial(authority._schedule_admitted, ref))
  558. _ROUTE_WAKE_PRODUCERS = frozenset({'kanban-wake', 'loop-tick'})
  559. def route_owner_lock(runner, session_id):
  560. """*session_id*'s live owner's ``mutation_lock`` (a no-op without one). Held across a route
  561. rotation and ``move_route_wakes``: its drain refuses a wake on the rotated route, so it must not
  562. claim one the move has not taken yet."""
  563. import contextlib
  564. from gateway.session_authorities import active_authority
  565. authority = active_authority(runner)
  566. live = authority.sessions.get(authority.logical_owner(session_id)) if authority and session_id else None
  567. return live.mutation_lock if live is not None else contextlib.nullcontext()
  568. async def move_route_wakes(runner, session_key, old_session_id):
  569. """After the route rotated (/new, /reset, compression-exhaustion reset), carry the old owner's
  570. still-queued route wakes to the fresh session, as main's #114456 kept a parked internal wake
  571. across /new. Left behind, the old drain refuses them (``admission_conflict``) and they stay
  572. queued forever. Completions pinned to the closed conversation are not moved (they fail closed,
  573. #55578), nor anything behind an ``unknown`` head (the operator's to resume)."""
  574. from gateway.session_authorities import active_authority
  575. authority = active_authority(runner)
  576. entry = runner.session_store.lookup_by_session_key(session_key) if authority is not None else None
  577. if entry is None or not old_session_id:
  578. return
  579. import asyncio
  580. from functools import partial
  581. from gateway.session_authority import LiveSession
  582. from gateway.session_contract import SessionActor
  583. from gateway.session_detached import lane_source
  584. from gateway.session_runtime_workers import tracked_write
  585. from hermes_state_runtime import list_session_admissions
  586. old = SessionRef(authority.profile_id, authority.logical_owner(old_session_id))
  587. new = SessionRef(authority.profile_id, authority.logical_owner(entry.session_id))
  588. if old.session_id not in authority.sessions or old == new:
  589. return
  590. rows = await asyncio.to_thread(list_session_admissions, authority.db, session_id=old.session_id)
  591. if any(row['status'] == 'unknown' for row in rows):
  592. return
  593. actor = SessionActor('messaging-control', authority.profile_id, frozenset({'session:submit', 'session:control'}), 'reset')
  594. for row in rows:
  595. automation = row['payload'].get('native_text_v1', {}).get('automation', {})
  596. identity = json.loads(row['request_id']) if automation and row['request_id'].startswith('[') else None
  597. if (row['status'] != 'queued' or row['principal_id'] != 'automation:' + session_key
  598. or not isinstance(identity, list) or identity[:1] not in ([p] for p in _ROUTE_WAKE_PRODUCERS)):
  599. continue
  600. try:
  601. await authority.cancel_queued(actor, old, row['admission_id'])
  602. except RuntimeStoreError:
  603. continue # claimed meanwhile: it runs where it is
  604. payload = deepcopy(row['payload'])
  605. payload['native_text_v1']['automation']['owner'] = new.session_id
  606. # The old conversation left the route at its rotation (gateway/session_detached.py).
  607. authority.sessions.setdefault(new.session_id, LiveSession(lane_source(authority.sessions[old.session_id].source),
  608. session_key))
  609. await tracked_write(authority, partial(
  610. admit_session_input, authority.db, epoch=authority.epoch, principal_id=row['principal_id'],
  611. session_id=new.session_id, request_id=row['request_id'], payload=payload,
  612. _authorize_write=authority._admission_gate()), then=partial(authority._admitted, new, None),
  613. ordered=True, after=partial(authority._schedule_admitted, new))
  614. def _local_owner(authority, session_key):
  615. """The LOCAL FIFO serving ``session_key`` (its owner is the receipt id, which a reset keeps)."""
  616. for sid, live in list(authority.sessions.items()):
  617. if live.route == session_key and getattr(live.source, 'platform', None) == Platform.LOCAL:
  618. return SessionRef(authority.profile_id, sid)
  619. return None
  620. async def _reset_and_retarget_local(authority, runner, session_key, old_session_id, owner):
  621. """A LOCAL route (the Bot Chat ``hermes peer dm`` lands in) keeps its FIFO owner across the
  622. reset, so nothing moves between FIFOs. Its queued automation rows (a DM that arrived while the
  623. exhausted turn ran) still name the pre-reset transcript and are retargeted at the fresh one,
  624. in the same ordered worker step as the rotation; left stale, the drain refuses them
  625. (``admission_conflict``) and the Bot Chat's FIFO pauses for good, across restarts."""
  626. from gateway.session_runtime_workers import tracked_write
  627. from hermes_state_runtime import retarget_local_automation
  628. live = authority.sessions[owner.session_id]
  629. def rotate_and_retarget():
  630. old_targets = authority.db.get_compression_lineage(old_session_id) or [old_session_id]
  631. entry = runner.session_store.reset_session(session_key)
  632. if entry is None or entry.session_id in old_targets:
  633. return entry, []
  634. return entry, retarget_local_automation(
  635. authority.db, epoch=authority.epoch, session_id=owner.session_id,
  636. principal_id='automation:' + session_key, old_targets=old_targets, new_target=entry.session_id)
  637. async with live.mutation_lock:
  638. entry, moved = await tracked_write(authority, rotate_and_retarget, ordered=True)
  639. if moved:
  640. authority._publish_pending(owner)
  641. return entry
  642. async def reset_and_rehome_messages(runner, session_key, old_session_id):
  643. """The compression-exhaustion auto-reset: rotate the route to a fresh session and move the old
  644. owner's still-queued native ``messaging:*`` rows onto it, same order and request ids. Left
  645. behind they pause the old FIFO (``admission_conflict``) and never run, while a later message
  646. runs ahead of them. /new and /stop cancel these rows instead (the user asked for a clean break);
  647. the exhausted turn itself is started, so it is never moved.
  648. The rotation (sessions.json) and the move (one SQLite transaction) run in one worker step
  649. ordered with native admissions, not atomically across the two stores: a redelivery that
  650. resolves the route to the fresh session commits after the move and dedupes onto the moved row,
  651. and one that reserved the old route before the rotation is retargeted at commit
  652. (``_admit_native_write``). Returns ``(entry, release)``. Until the caller runs ``release()``
  653. (once, after its conversation cleanup and topic binding sync, also on failure) the fresh
  654. session's ``claim_gate`` is held, so no scheduler (a redelivery, a moved route wake, this
  655. move) starts a turn there against stale state; ``release()`` then schedules its FIFO."""
  656. from gateway.session_authorities import active_authority
  657. authority = active_authority(runner)
  658. old = SessionRef(authority.profile_id, authority.logical_owner(old_session_id)) if authority else None
  659. live = authority.sessions.get(old.session_id) if old and old_session_id else None
  660. local = _local_owner(authority, session_key) if authority else None
  661. if local is not None:
  662. return await _reset_and_retarget_local(authority, runner, session_key, old_session_id, local), lambda: None
  663. if live is None:
  664. return await runner.async_session_store.reset_session(session_key), lambda: None
  665. import asyncio
  666. from gateway.session_authority import LiveSession
  667. from gateway.session_runtime_workers import tracked_write
  668. from hermes_state_runtime import rehome_queued_inputs
  669. from gateway.session_detached import lane_source
  670. chat_source = lane_source(live.source) # the rotation detaches the old conversation off the route
  671. def rotate_and_move():
  672. entry = runner.session_store.reset_session(session_key)
  673. if entry is None:
  674. return None, None, []
  675. new = SessionRef(authority.profile_id, authority.logical_owner(entry.session_id))
  676. if new == old:
  677. return entry, None, []
  678. if authority.db.get_session(new.session_id) is None:
  679. authority.db.create_session(new.session_id, source=chat_source.platform.value)
  680. # An admission that reserved the old route before this rotation commits after it.
  681. authority.rotated_owners[old.session_id] = session_key
  682. return entry, new, rehome_queued_inputs(
  683. authority.db, epoch=authority.epoch, old_session_id=old.session_id,
  684. new_session_id=new.session_id, principal_prefix='messaging:')
  685. async def fence(result):
  686. # In the write's own task step, before any later ordered admission schedules the fresh FIFO.
  687. new = result[1]
  688. if new is None:
  689. return result, None
  690. fresh = authority.sessions.setdefault(new.session_id, LiveSession(chat_source, session_key))
  691. await fresh.claim_gate.acquire()
  692. return result, fresh
  693. async with live.mutation_lock:
  694. write = asyncio.ensure_future(tracked_write(authority, rotate_and_move, fence, ordered=True))
  695. try:
  696. (entry, new, moved), fresh = await asyncio.shield(write)
  697. except asyncio.CancelledError:
  698. # The rotation still commits and takes the fence: lift it, or the fresh FIFO never runs.
  699. def lift(done):
  700. gate = None if done.cancelled() or done.exception() else done.result()[1]
  701. if gate is not None:
  702. gate.claim_gate.release()
  703. authority._schedule(done.result()[0][1])
  704. write.add_done_callback(lift)
  705. raise
  706. if fresh is None:
  707. return entry, lambda: None
  708. if moved:
  709. authority._publish_pending(old)
  710. authority._publish_pending(new)
  711. released = []
  712. def release(): # once: a second call must not lift a claim the drain took meanwhile
  713. if not released:
  714. released.append(True)
  715. fresh.claim_gate.release()
  716. if authority.sessions.get(new.session_id) is fresh:
  717. authority._schedule(new)
  718. return entry, release
  719. async def admit_route_wake(runner, adapter, source, text, producer, metadata=None):
  720. """Commit a producer wake (Kanban push wake, /loop tick) for ``source``'s route into the
  721. authority FIFO, like heartbeats and process completions, instead of a classic turn that
  722. interrupts or races the admitted one. ``False`` when no authority serves the scope or the
  723. route holds no committed human turn to borrow provenance from: the caller keeps the adapter
  724. path (nothing is queued there either). Raises ``RuntimeStoreError`` on a refusal."""
  725. from gateway.session_authorities import active_authority
  726. from gateway.session_local_recovery import restore_completion_route
  727. from gateway.session_detached import detached_live, detached_route_of, route_entry
  728. authority = active_authority(runner)
  729. key = runner._session_key_for_source(source)
  730. restore_completion_route(runner, key) # a closed local session boot left unrestored has no store entry yet
  731. entry = route_entry(authority, key) if authority is not None else None
  732. if authority is None or entry is None:
  733. return False
  734. import uuid
  735. identity = json.dumps([producer, key, uuid.uuid4().hex], separators=(',', ':'))
  736. if source.platform == Platform.LOCAL or detached_route_of(source):
  737. # A local route (or a conversation its chat moved past, answered by its app viewers) has no
  738. # messaging provenance to borrow: its wake rides local_automation_v1 on the live session's own
  739. # source object (``snapshot_local_automation`` checks identity).
  740. found = detached_live(authority, key)
  741. live = found[1] if found is not None else authority.sessions.get(source.chat_id)
  742. if live is None or live.route != key:
  743. return False
  744. event = MessageEvent(text=text, message_type=MessageType.TEXT, source=live.source, internal=True,
  745. message_id=identity, metadata={'gateway_session_key': key,
  746. 'gateway_session_id': entry.session_id,
  747. **(metadata or {})})
  748. await authority.admit_automation(adapter, event, identity)
  749. return True
  750. from gateway.session_envelope import restore_native
  751. envelope = committed_route_envelope(authority, entry)
  752. if envelope is None:
  753. return False
  754. # The route's committed source, which ``snapshot_automation`` re-derives and compares.
  755. restored = restore_native({'text': text, 'native_text_v1': {
  756. k: v for k, v in envelope.items() if k != 'media'}}, runner)
  757. event = MessageEvent(text=text, message_type=MessageType.TEXT, source=restored.source, internal=True,
  758. message_id=identity, metadata={'gateway_session_key': key,
  759. 'gateway_session_id': entry.session_id,
  760. **(metadata or {})})
  761. await authority.admit_automation(adapter, event, identity)
  762. return True