session_bot.py 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011
  1. """Bot Chat ingress into the existing authority FIFO; viewers never own delivery.
  2. The mailbox is a delivery receipt, not an execution queue. A committed canonical
  3. admission is the only consumer; unknown execution is never retried as inference.
  4. """
  5. import asyncio
  6. from functools import partial
  7. import logging
  8. from pathlib import Path
  9. import time
  10. from agent.turn_author import parse_turn_author
  11. from gateway.session_contract import SessionActor, SessionRef
  12. from gateway.config import Platform
  13. from gateway.platforms.event import MessageEvent
  14. from hermes_state_runtime import RuntimeStoreError, get_session_admission
  15. from tools.bot_failure_reasons import HELD_NOT_SENT, UNKNOWN
  16. from tools.bot_live_delivery import _TERMINAL, _delivery_id, _read
  17. logger = logging.getLogger(__name__)
  18. from gateway.session_bot_mailbox import (mailbox_lock, read_receipt, write_receipt,
  19. scan_receipts, remember_receipt, track_receipt_task, withdraw_pin, refuse_pin, requeue_pin)
  20. _CANONICAL_RECEIPT_STATUSES = frozenset({
  21. 'canonical', 'queued', 'claimed', 'ambiguous', 'settled', 'failed', 'cancelled',
  22. })
  23. _BOT_DM_MAX_CHARS = 16200
  24. # Admission refusals about the owner's state, not the delivery: with nothing committed the pin is
  25. # withdrawn and a same-id resend admits once it clears (a full FIFO, a drain for restart/update or
  26. # retirement, a fenced epoch or generation, storage, a live CLI holding the session, or a target
  27. # that moved before the write: the resend resolves it again). Every other refusal (permission_denied,
  28. # invalid_params, admission_conflict, profile_mismatch, ...) is final for that delivery: ``refuse_pin``.
  29. _OWNER_STATE_REFUSALS = frozenset({
  30. 'queue_full', 'runtime_draining', 'runtime_retired', 'stale_epoch', 'stale_generation',
  31. 'storage_unavailable', 'runtime_coordination_required', 'session_busy', 'not_found',
  32. })
  33. _PRODUCER_MAX_CHARS = 1024 * 1024
  34. def _canonical_record_error(path, record, required_strings=('admission_id', 'profile_home', 'session_id', 'principal_id')):
  35. """Return why an admission-backed receipt is unsafe for bulk recovery."""
  36. delivery_id = record.get('delivery_id')
  37. if not isinstance(delivery_id, str) or delivery_id != path.stem:
  38. return 'delivery id does not match filename'
  39. try:
  40. _delivery_id(delivery_id)
  41. except ValueError:
  42. return 'delivery id is invalid'
  43. missing = [key for key in required_strings
  44. if not isinstance(record.get(key), str) or not record[key]]
  45. if missing:
  46. return 'missing canonical fields: ' + ', '.join(missing)
  47. if not isinstance(record.get('message'), str):
  48. return 'message is not a string'
  49. status = record.get('status')
  50. if not isinstance(status, str) or status not in _CANONICAL_RECEIPT_STATUSES:
  51. return f'unknown canonical status {status!r}'
  52. return None
  53. def _record_shape_error(path, record):
  54. """Validate the known canonical, refused or legacy receipt shape for directory scans."""
  55. if record.get('refused') is True and 'admission_id' not in record:
  56. # ``refuse_pin``: the canonical pin fields, terminal ``failed``, never admitted.
  57. if record.get('status') != 'failed':
  58. return 'refused receipt is not failed'
  59. return _canonical_record_error(path, record, ('profile_home', 'session_id', 'principal_id'))
  60. if 'admission_id' in record or record.get('status') == 'canonical' or 'principal_id' in record:
  61. return _canonical_record_error(path, record)
  62. if 'owner' in record:
  63. from tools.bot_live_delivery import _ticket_shape_error
  64. return _ticket_shape_error(path, record)
  65. return 'unknown receipt schema'
  66. def _is_pre_admission_pin(path, record):
  67. """The ``ambiguous`` pin ``_admit`` writes before it admits: no admission id, never refused."""
  68. return (record.get('status') == 'ambiguous' and 'admission_id' not in record and 'refused' not in record
  69. and _canonical_record_error(path, record, ('profile_home', 'session_id', 'principal_id')) is None)
  70. def _warn_malformed(path, problem):
  71. logging.getLogger(__name__).warning("Skipping malformed Bot mailbox receipt %s (%s)", path, problem)
  72. def _scan_records(root, pins=None):
  73. """Bulk mailbox scan: isolate unreadable/malformed records; exact-id reads still fail closed.
  74. An owner scan passes ``pins``: pre-admission pins land there with their mtime, unwarned, since
  75. only the owner can tell whether their admission committed (``_scan_owner_receipts``)."""
  76. for path in root.glob('*.json'):
  77. try:
  78. record = _read(path)
  79. problem = None if record is None else _record_shape_error(path, record)
  80. if problem is not None and pins is not None and _is_pre_admission_pin(path, record):
  81. pins.append((path, record, path.stat().st_mtime))
  82. continue
  83. except (OSError, ValueError) as exc:
  84. record, problem = None, str(exc)
  85. if problem is not None:
  86. _warn_malformed(path, problem)
  87. continue
  88. if record is not None:
  89. yield path, record
  90. # An ``_admit`` pins, commits and withdraws under the mailbox lock in well under a second: a
  91. # pre-admission pin older than this is no in-flight admission, and the owner scan may withdraw it.
  92. _PIN_GRACE_SECONDS = 300
  93. async def _scan_owner_receipts(authority, home):
  94. """The owner's bulk scan, under the mailbox lock ``_admit`` holds. A pre-admission pin past the
  95. grace window whose ``bot:<key>`` never committed (refused by an older build, a death before the
  96. commit, a transient refusal on the legacy migration path) is withdrawn as ``_admit`` withdraws
  97. one: the scan stops warning and a same-id resend admits once. A pin laid over a legacy ticket
  98. (``owner``) is that never-run message's only copy: it is requeued instead, for ``_migrate`` to
  99. admit once under the same id. A pin whose admission committed stays the explicit unknown record
  100. and keeps its warning.
  101. Also returns the FIFO barrier: the lowest ``_legacy_order`` of a legacy pin of this home that is
  102. still unresolved (inside the grace window, or its requeue lost to a concurrent change). No
  103. legacy ticket ordered after it may be admitted by this pass. It holds for one scan only: past
  104. the grace window the pin is requeued and released, and a failed requeue is re-read by the next
  105. scan (gone, admitted or a ticket again), so the barrier never outlives the pin."""
  106. pins, barrier = [], None
  107. root, records = await scan_receipts(home, pins)
  108. for path, record, mtime in pins:
  109. key, age, legacy = record['delivery_id'], time.time() - mtime, 'owner' in record
  110. if (record['profile_home'] != str(home)
  111. or _pin_committed(authority, record, 'bot:' + key)):
  112. _warn_malformed(path, _record_shape_error(path, record))
  113. continue
  114. cleared = False
  115. if age >= _PIN_GRACE_SECONDS and legacy:
  116. ticket = await requeue_pin(home, key)
  117. cleared = ticket is not None and _record_shape_error(path, ticket) is None
  118. if cleared:
  119. records.append((path, ticket))
  120. elif age >= _PIN_GRACE_SECONDS:
  121. cleared = await withdraw_pin(home, key)
  122. if not cleared:
  123. if legacy:
  124. order = _legacy_order(path, record)
  125. barrier = order if barrier is None else min(barrier, order)
  126. _warn_malformed(path, _record_shape_error(path, record))
  127. continue
  128. logging.getLogger(__name__).info('%s Bot delivery %s pre-admission pin (age %ds): nothing was admitted',
  129. 'Requeued' if legacy else 'Withdrew', key, age)
  130. return root, records, barrier
  131. def _pin_committed(authority, record, request_id):
  132. """Whether the pin's admission may have committed: a live row, or a deleted target (its
  133. admissions moved to retired receipts keyed by the route principal the pin never recorded)."""
  134. if _admission_at(authority, record['session_id'], request_id) is not None:
  135. return True
  136. from hermes_state_mutation_retirement import RETIRED_PREFIX
  137. with authority.db._read_ctx() as conn:
  138. return conn.execute('SELECT 1 FROM state_meta WHERE key=?',
  139. (RETIRED_PREFIX + record['session_id'],)).fetchone() is not None
  140. def _legacy_order(path, record):
  141. """``_migrate``'s FIFO key of a legacy ticket, or of the pin laid over one (it keeps the ticket's
  142. fields). A pin without an integer order holds every legacy ticket until it resolves."""
  143. order = record.get('sequence', record.get('created_at'))
  144. return (order, path.name) if isinstance(order, int) and not isinstance(order, bool) else (float('-inf'), '')
  145. def _home(authority, actor, profile):
  146. from gateway.session_authorities import served_profile_name
  147. from tools.bot_mode_probe import _handle, _hermes_root, _roster
  148. home = Path(authority.db.db_path).parent.resolve()
  149. name = served_profile_name(home)
  150. handle = _handle(name, [n for n, _d in _roster(_hermes_root(home))]) if name == 'default' else name
  151. if actor.profile_id != authority.profile_id or profile not in (name, handle):
  152. raise RuntimeStoreError('profile_mismatch')
  153. if 'session:submit' not in actor.capabilities:
  154. raise RuntimeStoreError('permission_denied')
  155. return home
  156. async def _target(authority, actor):
  157. ref = lane = None
  158. row = authority.db.get_session_by_title('Bot Chat')
  159. if row is None or (displaced := displaceable_bot_chat(authority, row, actor.subject)) is not None:
  160. # None: a concurrent delivery created it first. A title held by an API conversation no FIFO
  161. # can own is moved aside first, under the same lock (MSG-21, #140769).
  162. ref = await _create_bot_chat(authority, actor, displace=None if row is None else displaced)
  163. if ref is None and (row := authority.db.get_session_by_title('Bot Chat')) is None:
  164. # Moved aside since it was read: a lane's /new frees a handed-off Bot Chat's title off the
  165. # titled-create lock (``release_superseded_bot_chat``). None again: a concurrent delivery minted.
  166. if (ref := await _create_bot_chat(authority, actor)) is None:
  167. row = authority.db.get_session_by_title('Bot Chat')
  168. if ref is None:
  169. tip = authority.db.get_compression_tip(row['id'])
  170. target = authority.db.get_session(tip)
  171. if target is None:
  172. raise RuntimeStoreError('not_found')
  173. from gateway.session_local_migration import resolve_local_target
  174. from gateway.session_route_recovery import handed_off_ref
  175. # A Bot Chat handed off to a messaging lane stays the bot's chat there: delivery follows it.
  176. ref = lane = handed_off_ref(authority, target['id'])
  177. if ref is None:
  178. ref = resolve_local_target(authority, actor, target['id'])
  179. authority.authorize(actor, ref, 'session:submit')
  180. live = authority.sessions[ref.session_id]
  181. entry = authority.runner.session_store.lookup_by_session_key(live.route)
  182. if (live.source.platform != Platform.LOCAL and lane is None) or entry is None:
  183. raise RuntimeStoreError('admission_conflict')
  184. return ref, live, entry
  185. def displaceable_bot_chat(authority, row, subject=None):
  186. """The id holding the canonical title when the Bot Chat is a conversation this delivery can
  187. never adopt, rename (the title is its identity) or rename over the API, so it would refuse
  188. every delivery for good: an ``api_`` row an older gateway or a plain ``POST /api/sessions``
  189. minted, a messaging row, or a lineage whose compression tip (the row a delivery adopts) is
  190. unbound local history ``_kept_local_history`` does not keep for the delivering *subject*.
  191. Else None: a bound local chat and kept history are never moved."""
  192. from hermes_state_local import POLICY_PREFIX
  193. from hermes_state_local_migration import LEGACY_PREFIX
  194. if str(row.get('chat_id') or '').startswith('local-'):
  195. return None # every delivery asks: a bound local chat answers from its own row, with no read
  196. db = authority.db
  197. tip = db.get_session(db.get_compression_tip(row['id']) or row['id'])
  198. from gateway.session_route_recovery import handed_off_ref, superseded_hand_off
  199. if tip is not None and (lane := handed_off_ref(authority, tip['id'])) is not None:
  200. # Handed off (``/handoff``): still the bot's chat while its lane holds it; history once that
  201. # lane moved on (``/new``), so a fresh Bot Chat takes the title (#143286).
  202. return row['id'] if superseded_hand_off(authority, lane) else None
  203. for candidate in (row, tip):
  204. if candidate is None or str(candidate.get('chat_id') or '').startswith('local-'):
  205. return None
  206. chat_id = str(candidate.get('chat_id') or '')
  207. with db._read_ctx() as conn:
  208. if conn.execute('SELECT 1 FROM state_meta WHERE key IN (?,?)',
  209. (POLICY_PREFIX + (chat_id or candidate['id']),
  210. LEGACY_PREFIX + (chat_id or candidate['id']))).fetchone():
  211. return None
  212. # Only the tip is adopted (``_target``), so only the tip decides: a root the owner could adopt
  213. # above a tip nobody can would otherwise hold the title and refuse every delivery.
  214. return None if _kept_local_history(db, tip, subject) else row['id']
  215. def release_superseded_bot_chat(db, store, route, session_id):
  216. """The lane on *route* moved off *session_id* (its ``/new``, a reset, another ``/resume`` or
  217. ``/handoff``). A Bot Chat handed off there (``/handoff``) is that lane's history now: free the
  218. canonical title at once (``_displace_bot_chat``), so the Bots roster and its registry lookup both
  219. stop naming it and the next click or delivery mints a fresh Bot Chat (#143286). Every route
  220. transition calls this, and so does boot (an older build's transition)."""
  221. from gateway.session_route_recovery import _messaging_platform
  222. from hermes_state_local_migration import LEGACY_LOCAL_SOURCES
  223. root = (db.get_compression_lineage(session_id) or [session_id])[0]
  224. row = db.get_session(root)
  225. if (row is None or row.get('title') != 'Bot Chat' or row.get('created_source') not in LEGACY_LOCAL_SOURCES
  226. or _messaging_platform(row.get('source')) is None):
  227. return
  228. with store._lock:
  229. entry = store._entry_locked(route)
  230. if entry is None or (db.get_compression_lineage(entry.session_id) or [entry.session_id])[0] == root:
  231. return # the lane still holds it
  232. if _displace_bot_chat(db, root):
  233. logger.info("Bot Chat %s was handed off to %s and that lane moved on: moved it aside", root, route)
  234. def release_superseded_bot_chat_at_boot(db, store):
  235. row = db.get_session_by_title('Bot Chat')
  236. if row is not None:
  237. tip = db.get_session(db.get_compression_tip(row['id']) or row['id']) or row
  238. release_superseded_bot_chat(db, store, tip.get('session_key'), row['id'])
  239. def _kept_local_history(db, row, subject=None):
  240. """Whether an unbound row is local history the Bot Chat door must leave in place: a CLI, TUI
  241. or Desktop row the delivering *subject* or the host owner adopts by the native open's own rule
  242. (``validate_legacy_row``: unstamped, or stamped with one of that adopter's identities), or a
  243. Desktop row a foreign login stamped with its label (#140183: kept even with a route column or
  244. another profile's name). Any other local-source row (a route column left by a crash or
  245. reconnect, another profile's stamp, a stamp neither adopter owns, or outside Desktop a foreign
  246. label) is not: kept, it held the title and refused every delivery for good (#141243, #143715).
  247. *subject* None: the row's own ``auth:v1`` or owner-label stamp is the adopter, as a peer DM's
  248. turn enters the chat."""
  249. from gateway.owner_identity import host_owner_subject, same_owner
  250. from gateway.session_identity import parse_authenticated_subject
  251. from hermes_state_local_migration import LEGACY_LOCAL_SOURCES, legacy_account_label, validate_legacy_row
  252. if row.get('source') not in LEGACY_LOCAL_SOURCES:
  253. return False
  254. user_id = row.get('user_id')
  255. owner = host_owner_subject()
  256. if (user_id and not legacy_account_label(row) and not same_owner(user_id, owner)
  257. and not (subject and same_owner(user_id, subject))
  258. and parse_authenticated_subject(user_id) is None):
  259. # A label neither adopter owns: Desktop keeps it (#140183); elsewhere no open adopts it.
  260. return row.get('source') == 'desktop'
  261. # A forwarded login (control socket) adopts its own stamp; the owner's history is never moved
  262. # aside for a delivery that cannot adopt it.
  263. for adopter in dict.fromkeys((user_id if subject is None else subject, owner)):
  264. try:
  265. validate_legacy_row(db, row, subject=adopter)
  266. except RuntimeStoreError:
  267. continue
  268. return True
  269. return False
  270. def _moved_aside_by_this_door(authority, holder, pinned):
  271. """Whether *pinned*, the Bot Chat the producer discovered, is the title holder this delivery's
  272. ``_target`` just moved aside. The producer pins the id it read before asking, so the first
  273. delivery after a move would otherwise be refused ``admission_conflict`` for the chat the door
  274. itself replaced, and land only on the next beat."""
  275. if holder is None:
  276. return False
  277. db = authority.db
  278. if pinned not in (holder['id'], db.get_compression_tip(holder['id'])):
  279. return False
  280. return str((db.get_session(holder['id']) or {}).get('title') or '').startswith('Bot Chat (')
  281. def _displace_bot_chat(db, sid):
  282. """Free the canonical title from *sid*: it keeps its id and history under a visible name the
  283. user can find, rename or delete: ``Bot Chat (API <id>)`` for an API conversation, else named by
  284. its platform (``Bot Chat (telegram <id>)``). False when the title moved meanwhile."""
  285. source = str((db.get_session(sid) or {}).get('source') or '')
  286. label = 'API' if source in ('', 'api_server', 'bot_peer_dm') else source
  287. name = f'Bot Chat ({label} {sid})'
  288. if len(name) > db.MAX_TITLE_LENGTH:
  289. name = name[:db.MAX_TITLE_LENGTH - 2] + '…)'
  290. def write(conn):
  291. return conn.execute("UPDATE sessions SET title=?, title_source=?, hidden=0 WHERE id=? AND title='Bot Chat'",
  292. (name, db.TITLE_SOURCE_USER, sid)).rowcount
  293. return bool(db._execute_write(write))
  294. async def _create_bot_chat(authority, actor, *, displace=None):
  295. """No Bot Chat yet (or *displace*, an API conversation holding its title, is moved aside first): mint it the way ``hermes chat --in ~ -c "Bot Chat" --create-if-missing``
  296. did on the CLI lane this door replaced, so a fresh profile's first delivery has somewhere
  297. to land instead of being recorded as failed. Same owner step as ``session.create`` with a
  298. title: lookup, creation and title run under the authority's titled-create lock, with the
  299. commits off the owner loop (N90). Creation is the actor's own capability; a submit-only
  300. principal (owner recovery) still gets ``not_found``."""
  301. if 'session:create' not in actor.capabilities:
  302. raise RuntimeStoreError('not_found')
  303. from gateway.session_local import stage_local_session, staged_creation_follows
  304. from gateway.session_local_recovery import restore_local_session
  305. from gateway.session_local_title import title_new_session
  306. from gateway.session_runtime_workers import tracked_write
  307. from hermes_state_local import commit_local_session
  308. lock = getattr(authority, '_titled_create_lock', None)
  309. if lock is None:
  310. lock = authority._titled_create_lock = asyncio.Lock()
  311. async with lock:
  312. held = authority.db.get_session_by_title('Bot Chat')
  313. if held is not None:
  314. if displace is None or held['id'] != displace or displaceable_bot_chat(authority, held, actor.subject) != displace:
  315. return None
  316. logger.warning("Bot Chat title held by %s, a conversation no FIFO can own: moving it aside", displace)
  317. # False: the title moved since the check. A lane's ``/new`` frees a handed-off Bot Chat's
  318. # title from the store's worker thread, outside this lock (``release_superseded_bot_chat``):
  319. # nobody holds it then, so this delivery mints the replacement.
  320. if (not await tracked_write(authority, partial(_displace_bot_chat, authority.db, displace))
  321. and authority.db.get_session_by_title('Bot Chat') is not None):
  322. return None
  323. # The Bot Chat follows its profile whoever mints it (a Bot or cron delivery, a peer DM),
  324. # the same contract Desktop's create asks for. A later profile change reaches it.
  325. request_id = _bot_chat_creation_id(authority, actor)
  326. params = {'request_id': request_id, 'source': 'cli', 'cwd': str(Path.home()),
  327. 'follow_profile_config': True}
  328. # A creation staged by an older build, before this flag, resumes as it was staged.
  329. if not staged_creation_follows(authority, actor, request_id):
  330. params.pop('follow_profile_config')
  331. ref = stage_local_session(authority, actor, params)
  332. if isinstance(ref, dict):
  333. await tracked_write(authority, partial(commit_local_session, authority.db, epoch=authority.epoch, receipt=ref))
  334. ref = restore_local_session(authority, ref['session_id'])
  335. await tracked_write(authority, partial(title_new_session, authority, ref, 'Bot Chat'))
  336. return ref
  337. def _bot_chat_creation_id(authority, actor):
  338. """The first creation identity whose session is not spent. Generation 0 keeps the historical
  339. ``bot-chat:<profile>`` id; a generation is spent once its session was deleted (retired: a retry
  340. must never resurrect it), renamed (titled, but not "Bot Chat" — or the lookup had found it) or
  341. handed off to a messaging lane, and a replacement takes the next one. Derived from durable rows
  342. only, so a restart and two concurrent deliveries (serialized by the titled-create lock) pick the
  343. same identity. An existing UNtitled local session is the same creation interrupted before its
  344. title: it is reused."""
  345. from gateway.session_local_recovery import local_identity
  346. from gateway.session_route_recovery import _messaging_platform
  347. from hermes_state_mutation_retirement import first_generation
  348. base = 'bot-chat:' + authority.profile_id
  349. request_id = lambda generation: base if generation == 0 else f'{base}:{generation}'
  350. def spent(conn, sid):
  351. row = conn.execute('SELECT title, source FROM sessions WHERE id=?', (sid,)).fetchone()
  352. # Handed off (``/handoff``), it is a messaging lane's conversation: its receipt went with it,
  353. # and a reset's head carried its name there (``hermes_state_local.hand_off_conversations``).
  354. return row is not None and (bool(row[0]) or _messaging_platform(row[1]) is not None)
  355. generation, _sid = first_generation(
  356. authority.db, lambda n: local_identity(authority.profile_id, actor.subject, request_id(n)), spent=spent)
  357. return request_id(generation)
  358. def _admission_outcome(authority, admission_id, fallback=None):
  359. """``(status, reply)`` of one admission, read from its committed result, never transcript recency."""
  360. row = get_session_admission(authority.db, admission_id=admission_id)
  361. if row is None:
  362. return _expired_outcome(authority, fallback)
  363. from gateway.session_results import admission_result
  364. saved = admission_result(authority.db, admission_id)
  365. if saved is not None and row['status'] != 'terminal':
  366. # The settlement committed (from its worker thread) between these two reads. Its result and
  367. # terminal outcome commit together, so re-read the row: the stale ``started`` row has no
  368. # outcome, and a completed turn would project as a terminal ``failed`` receipt.
  369. row = get_session_admission(authority.db, admission_id=admission_id)
  370. # A terminal row without a result blob is still definitive when the outcome says the
  371. # input never ran (cancelled) or was refused/failed; ambiguous is reserved for the
  372. # durable unknown state and for missing evidence (completed/interrupted without result).
  373. status = {'queued': 'queued', 'started': 'claimed', 'unknown': 'ambiguous',
  374. 'terminal': {'cancelled': 'cancelled', 'rejected': 'failed', 'failed': 'failed'}.get(row['outcome'], 'ambiguous')}[row['status']]
  375. reply = (fallback or {}).get('reply', '')
  376. if saved is not None:
  377. reply = saved['result'].get('final_response', '')
  378. # A turn stopped mid-run (the sender's message_agent cancel, or a Stop on the target) did
  379. # not fail: the sender hears ``cancelled``, never a failure to retry or report as broken.
  380. status = {'completed': 'settled', 'interrupted': 'cancelled'}.get(row['outcome'], 'failed')
  381. from gateway.session_bot_outcome import owner_stopped
  382. if status == 'cancelled' and owner_stopped(saved['result']):
  383. status = 'failed' # its gateway stopped mid-turn: never re-run, and nobody cancelled it
  384. # A successful bare silence marker is a delivery decision (same rule as the gateway and
  385. # the live Bot Chat completion): the turn stays in the target's transcript, the sender
  386. # never sees the marker as prose.
  387. from gateway.response_filters import is_intentional_silence_response, strip_trailing_silence_marker
  388. if status == 'settled':
  389. reply = '' if is_intentional_silence_response(reply) else strip_trailing_silence_marker(reply)
  390. elif fallback is not None and fallback.get('status') in {'settled', 'failed'}:
  391. status = fallback['status']
  392. return status, reply
  393. def _expired_outcome(authority, record):
  394. """A receipt outlives its admission once a deleted target's tombstone passes the retry horizon
  395. (``prune_retired_tombstones``). A final receipt keeps the answer it already holds; a pending one
  396. whose session was deleted (retired, or its row gone) can never learn its outcome and becomes
  397. ``ambiguous``, never resent. A live session missing its admission is still a storage fault."""
  398. from hermes_state_mutation_retirement import retired_session
  399. record = record or {}
  400. if record.get('status') in _TERMINAL:
  401. return record['status'], record.get('reply', '')
  402. session_id = record.get('session_id')
  403. if session_id and retired_session(authority.db, session_id):
  404. return 'ambiguous', record.get('reply', '')
  405. if session_id and authority.db.get_session(session_id) is None:
  406. # Never resent either way, but a row gone without its fence is not a plain delete: store damage,
  407. # or a marker an older release reclaimed. Logged once: the receipt is final from here on.
  408. logging.getLogger(__name__).warning(
  409. 'Bot delivery %s: session %s is gone with no deletion marker; settled as ambiguous',
  410. record.get('delivery_id'), session_id)
  411. return 'ambiguous', record.get('reply', '')
  412. raise RuntimeStoreError('storage_unavailable')
  413. def _retry_admission(authority, record):
  414. """The delivery's one retry admission id, read from the FIFO by its derived identity, so every
  415. reader (an in-memory record, the receipt file, an owner restart) follows the same retry."""
  416. return _admission_at(authority, record['session_id'], _retry_identity(record['delivery_id']))
  417. def _admission_at(authority, session_id, request_id):
  418. with authority.db._read_ctx() as conn:
  419. row = conn.execute('SELECT admission_id FROM session_admissions WHERE target_session_id=? AND request_id=?',
  420. (session_id, request_id)).fetchone()
  421. return row[0] if row else None
  422. def _result(authority, record):
  423. """The delivery's receipt. Once the one transient-failure retry was admitted, the sender's
  424. outcome is the retry's: ``status``/``reply`` follow it (``retry_admission_id``).
  425. A ``failed`` receipt carries ``error`` + a typed ``reason`` (``tools.bot_failure_reasons``):
  426. the relay's Desktop forwards ``res.reason`` to the sender, which cannot re-derive it from a
  427. reply that lacks the provider text."""
  428. retry_id = _retry_admission(authority, record)
  429. current = retry_id or record['admission_id']
  430. if retry_id:
  431. status, reply = _admission_outcome(authority, retry_id)
  432. else:
  433. status, reply = _admission_outcome(authority, record['admission_id'], record)
  434. if status == 'failed' and (record.get('retry') or {}).get('refused') == 'cancelled':
  435. status = 'cancelled' # its sender withdrew it (``session_bot_cancel``); its retry never ran
  436. if status == 'settled' and not reply:
  437. from gateway.session_bot_outcome import forgotten_reply
  438. reply = forgotten_reply(authority, current, record) or ''
  439. error = reason = None
  440. if status == 'failed':
  441. try:
  442. error, reason = _failure(authority, current)
  443. except RuntimeStoreError as exc:
  444. if exc.reason != 'not_found':
  445. raise
  446. # Its admission aged out after the chat was deleted: the receipt's own answer stands.
  447. from tools.bot_failure_reasons import UNKNOWN
  448. error, reason = record.get('error') or 'Bot Chat delivery failed', record.get('reason') or UNKNOWN
  449. return {k: v for k, v in dict(status=status, delivery_id=record['delivery_id'],
  450. profile_home=record['profile_home'], session_id=record['session_id'],
  451. admission_id=record['admission_id'], message=record['message'], reply=reply,
  452. retry_admission_id=retry_id, error=error, reason=reason).items() if v is not None}
  453. def _failure(authority, admission_id):
  454. """``(error, reason)`` of a failed admission from its committed result: the raw provider error
  455. and the turn loop's typed verdict, classified like every other Bot lane. A refusal that never
  456. ran (no result) has no provider text: ``unknown``."""
  457. from gateway.session_results import admission_result
  458. from tools.bot_failure_reasons import classify_agent_error, turn_failure_text
  459. saved = (admission_result(authority.db, admission_id) or {}).get('result') or {}
  460. from gateway.session_bot_outcome import OWNER_STOPPED_ERROR, owner_stopped
  461. if owner_stopped(saved):
  462. return OWNER_STOPPED_ERROR, UNKNOWN # never a retryable class: its steps may already have run
  463. error = saved.get('error') or saved.get('final_response') or 'Bot Chat delivery failed'
  464. return error, classify_agent_error(turn_failure_text(saved.get('error'), saved.get('failure_reason')) or error)
  465. def _delivery_result(authority, record):
  466. result = _result(authority, record)
  467. if result['status'] == 'failed' and _retry_eligible(authority, record):
  468. result['status'] = 'claimed' # owner receipt writer still owes the one allowed retry
  469. return result
  470. def peer_wait_admission(authority, record):
  471. """``(admission_id, interim)`` a live waiter must follow; ``(None, False)`` once the delivery
  472. settled for good. The retry, once admitted, is the only admission whose future fires again.
  473. An original that failed while its one retry is still eligible is ``interim``: pending, never
  474. the answer, while the owner's receipt task admits the retry (or records why it would not;
  475. the receipt file is re-read for that, the caller's in-memory record is stale)."""
  476. from tools.bot_live_delivery import read_delivery_result
  477. retry_id = _retry_admission(authority, record)
  478. if retry_id:
  479. return (retry_id, False) if _admission_outcome(authority, retry_id)[0] in {'queued', 'claimed'} else (None, False)
  480. status = _admission_outcome(authority, record['admission_id'], record)[0]
  481. if status in {'queued', 'claimed'}:
  482. return record['admission_id'], False
  483. if status == 'failed':
  484. saved = read_delivery_result(record['profile_home'], record['delivery_id']) or record
  485. if _retry_eligible(authority, {**record, 'retry': saved.get('retry')}):
  486. return record['admission_id'], True
  487. return None, False
  488. def _retry_identity(key):
  489. return 'bot:' + key + ':retry'
  490. def _retry_eligible(authority, record):
  491. """Exactly the main-lane gate (``tools.bot_failure_reasons.result_retry_action``): the original
  492. admission SETTLED ``failed`` with a committed result whose error classifies transient (429/5xx/
  493. context overflow). queued/started/unknown admissions never qualify — unknown execution is never
  494. replayed — and the retry itself is never retried."""
  495. if (record.get('retry') or {}).get('refused') or _retry_admission(authority, record):
  496. return False
  497. row = get_session_admission(authority.db, admission_id=record['admission_id'])
  498. if row is None or row['status'] != 'terminal' or row['outcome'] != 'failed':
  499. return False
  500. from gateway.session_results import admission_result
  501. saved = admission_result(authority.db, record['admission_id'])
  502. if saved is None:
  503. return False
  504. from tools.bot_failure_reasons import RETRY_NONE, result_retry_action
  505. return result_retry_action(saved['result']) != RETRY_NONE
  506. async def _maybe_retry(authority, home, path, record):
  507. """Admit the delivery's one retry under a DERIVED identity recorded on the same receipt, so a
  508. repeated deliver(), an owner restart or a second settle re-reads that admission instead of
  509. minting another. The identity is written before the admission: a death in that two-store window
  510. re-admits the SAME identity, which the FIFO answers with the existing row (never a second turn).
  511. No user-row adoption: a transient owner-side failure persists the DM row and then CLOSES it with the
  512. durable failed-turn boundary (``agent.conversation_loop._close_durable_failed_turn`` /
  513. ``_hmwa_close_failed_turn``, #107070), so ``agent.session_persistence.adopt_unanswered_turn`` would
  514. decline by contract (a plain assistant row follows the DM) and the retry is a fresh turn after that
  515. boundary. A failure that left the DM as an OPEN tail (the context-pressure classes skip the closer)
  516. is not retried: the owner execution has no adoption seam, and the retry's DM would merge into it."""
  517. if not _retry_eligible(authority, record):
  518. return
  519. identity = _retry_identity(record['delivery_id'])
  520. # The retry keeps the access its delivery was admitted on: ``operator`` is recorded only when the
  521. # transport granted ``session:operator`` (a local delivery into api-peer's Bot Chat), and the
  522. # session check below still pins the retry to the admitted target.
  523. operator = {'session:operator'} if record.get('operator') is True else set()
  524. actor = SessionActor(record['principal_id'], authority.profile_id,
  525. frozenset({'session:submit', 'session:read'} | operator), 'bot-delivery-retry')
  526. try:
  527. authority._require_admission_open()
  528. ref, live, entry = await _target(authority, actor)
  529. if ref.session_id != record['session_id']:
  530. raise RuntimeStoreError('admission_conflict')
  531. if authority.db.latest_conversation_role(entry.session_id) == 'user':
  532. raise RuntimeStoreError('open_user_tail')
  533. except RuntimeStoreError as exc:
  534. if exc.reason == 'runtime_draining':
  535. # No retry was attempted. Keep the local projection pending until startup recovery
  536. # can perform the same derived retry, rather than exposing an interim failure.
  537. record.update(status='claimed', reply='')
  538. return
  539. record['retry'] = {'identity': identity, 'refused': exc.reason}
  540. await write_receipt(home, record)
  541. return
  542. event = MessageEvent(text=record['message'], source=live.source, internal=True,
  543. message_id=identity, metadata={'gateway_session_key': live.route, 'gateway_session_id': entry.session_id})
  544. from gateway.session_automation import automation_notification_metadata
  545. event.metadata.update(automation_notification_metadata(record))
  546. if record.get('author') is not None:
  547. event.metadata['turn_author'] = dict(record['author'])
  548. record['retry'] = {'identity': identity}
  549. record.update(status='queued', reply='')
  550. await write_receipt(home, record)
  551. try:
  552. receipt = await authority.admit_automation(authority.runner._adapter_for_source(live.source), event, identity)
  553. except RuntimeStoreError as exc:
  554. if exc.reason != 'queue_full':
  555. raise
  556. # Nothing was admitted: keep the delivery pending (as for a drain) so the next wake or
  557. # delivery performs the same derived retry once the FIFO has room.
  558. record.update(status='claimed', reply='')
  559. return
  560. record.update(_result(authority, record))
  561. if record['status'] in {'queued', 'claimed'}:
  562. _watch_reply(authority, home, record['delivery_id'], receipt.admission_id)
  563. await write_receipt(home, record)
  564. remember_receipt(authority, home, record)
  565. async def _record_reply(authority, home, key, future):
  566. paused = False
  567. try:
  568. await asyncio.shield(future)
  569. except RuntimeStoreError:
  570. # A paused FIFO is still queued/unknown. Project that durable state;
  571. # never turn a wakeup refusal into a failed delivery or a fresh retry.
  572. paused = True
  573. async with mailbox_lock(authority):
  574. root, record = await read_receipt(home, key)
  575. if record is None:
  576. return
  577. path = root / f'{key}.json'
  578. record.update(_result(authority, record))
  579. if not paused:
  580. await _maybe_retry(authority, home, path, record)
  581. record.update(_delivery_result(authority, record))
  582. await write_receipt(home, record)
  583. remember_receipt(authority, home, record)
  584. async def refresh_receipt(authority, home, key):
  585. async with mailbox_lock(authority):
  586. root, record = await read_receipt(home, key)
  587. if record is None:
  588. return
  589. record.update(_result(authority, record))
  590. if record['status'] in {'queued', 'claimed'}:
  591. _watch_reply(authority, home, key, record.get('retry_admission_id') or record['admission_id'])
  592. await _maybe_retry(authority, home, root / f'{key}.json', record)
  593. await write_receipt(home, record)
  594. remember_receipt(authority, home, record)
  595. def relay_operation(connection, operation, params):
  596. authority, actor = connection.authority, connection.actor
  597. from gateway.session_authorities import require_serving, served_profile_name
  598. require_serving(authority) # the outbox/roster/replies belong to the home's next owner
  599. home = Path(authority.db.db_path).parent.resolve()
  600. name = served_profile_name(home)
  601. _home(authority, actor, name)
  602. if 'session:control' not in actor.capabilities:
  603. raise RuntimeStoreError('permission_denied')
  604. fields = {'roster': {'agents'}, 'outbox': set(), 'reply': {'id', 'reply', 'error', 'reason'}}
  605. if set(params) - fields[operation]:
  606. raise RuntimeStoreError('invalid_params')
  607. from tools.bot_relay import write_remote_roster, claim_pending_envelopes, write_reply
  608. if operation == 'roster':
  609. return {'count': write_remote_roster(home, params.get('agents'))}
  610. if operation == 'outbox':
  611. return {'envelopes': claim_pending_envelopes(home)}
  612. try:
  613. write_reply(home, params.get('id'), reply=params.get('reply', ''),
  614. error=params.get('error', ''), reason=params.get('reason', ''))
  615. except ValueError as exc:
  616. raise RuntimeStoreError('admission_conflict') from exc
  617. return {'ok': True}
  618. async def _migrate(authority, actor, home, root):
  619. # FIFO: a legacy ticket queued after an unresolved legacy pin waits for that pin's requeue. An
  620. # owner-state refusal below raises out of the pass, so nothing after the refused ticket runs.
  621. root, records, barrier = await _scan_owner_receipts(authority, home)
  622. legacy = [(path, record) for path, record in records
  623. if record and 'owner' in record and not record.get('admission_id')
  624. and record['status'] in {'queued', 'claimed'}
  625. and (barrier is None or _legacy_order(path, record) < barrier)]
  626. if not legacy:
  627. return
  628. ref, live, entry = await _target(authority, actor)
  629. for path, record in sorted(legacy, key=lambda item: _legacy_order(*item)):
  630. owner = record['owner']
  631. if owner['profile_home'] != str(home):
  632. continue
  633. if authority.db.get_compression_tip(owner['session_id']) != entry.session_id:
  634. continue
  635. from hermes_cli.active_sessions import active_session_liveness_guard
  636. with active_session_liveness_guard(owner['session_id'], registry_home=home) as active:
  637. if active:
  638. raise RuntimeStoreError('runtime_coordination_required')
  639. if record['status'] == 'claimed':
  640. record.update(status='ambiguous', reason='unknown_execution')
  641. await write_receipt(home, record, expect='claimed')
  642. continue
  643. await _admit(authority, actor, home, root, _delivery_id(record['delivery_id']),
  644. record['message'], ref, live, entry, author=parse_turn_author(record.get('author')), legacy=record,
  645. notification_category=record.get('notification_category', 'result'))
  646. async def recover_bot_deliveries(authority):
  647. """Rebuild derivative replies and queued legacy admissions at owner startup."""
  648. home = Path(authority.db.db_path).parent.resolve()
  649. async with mailbox_lock(authority):
  650. root, records, _ = await _scan_owner_receipts(authority, home)
  651. for path, record in records:
  652. if not record or record.get('profile_home') != str(home) or not record.get('admission_id'):
  653. continue
  654. try:
  655. record.update(_result(authority, record))
  656. if record['status'] in {'queued', 'claimed'}:
  657. _watch_reply(authority, home, record['delivery_id'],
  658. record.get('retry_admission_id') or record['admission_id'])
  659. if record['status'] not in {'queued', 'claimed'}:
  660. await _maybe_retry(authority, home, path, record)
  661. except RuntimeStoreError as exc:
  662. # One receipt the store cannot answer must not stop the owner (and, multiplexed, every
  663. # profile with it): log it and recover the rest. The next start reads it again (a
  664. # retry already pinned on it is then admitted once).
  665. logging.getLogger(__name__).error('Bot delivery %s not recovered: %s (%s)',
  666. record.get('delivery_id'), exc.reason, path, exc_info=True)
  667. continue
  668. await write_receipt(home, record)
  669. remember_receipt(authority, home, record)
  670. row = authority.db.get_session_by_title('Bot Chat')
  671. if row is None:
  672. return
  673. target = authority.db.get_session(authority.db.get_compression_tip(row['id']))
  674. if target is None:
  675. return
  676. from hermes_state_local import POLICY_PREFIX
  677. with authority.db._read_ctx() as conn:
  678. bound = conn.execute('SELECT 1 FROM state_meta WHERE key=?',
  679. (POLICY_PREFIX + (target.get('chat_id') or target['id']),)).fetchone()
  680. if not bound:
  681. return # Unowned history requires a native ticket, never the first remote sender.
  682. actor = SessionActor(target['user_id'], authority.profile_id,
  683. frozenset({'session:submit', 'session:read'}), 'bot-owner-recovery')
  684. await _migrate(authority, actor, home, root)
  685. def _watch_reply(authority, home, key, admission_id):
  686. watchers = getattr(authority, '_bot_reply_watchers', None)
  687. if watchers is None:
  688. watchers = authority._bot_reply_watchers = {}
  689. previous = watchers.get(key)
  690. if previous is not None and previous[0] == admission_id and not previous[2].done():
  691. return
  692. future = authority.waiters.setdefault(admission_id, asyncio.get_running_loop().create_future())
  693. task = asyncio.create_task(_record_reply(authority, home, key, future))
  694. watchers[key] = (admission_id, task, future)
  695. def done(finished):
  696. if watchers.get(key) == (admission_id, finished, future):
  697. watchers.pop(key, None)
  698. task.add_done_callback(done)
  699. track_receipt_task(authority, task)
  700. async def deliver(connection, params):
  701. from gateway.session_runtime_workers import track_mutation
  702. from gateway.session_bot_state import bot_chat_state
  703. connection.authority._require_admission_open()
  704. answer = await asyncio.shield(track_mutation(connection.authority, _deliver(connection, params)))
  705. # Not part of the receipt (it is stale a second later): the sender's ack and its runner's replays.
  706. return {**answer, 'target_state': bot_chat_state(connection.authority, answer['session_id'])}
  707. def _require_same_delivery(record, actor, message, author, category):
  708. """A delivery id names one delivery: the same message, principal, author and category (the
  709. mailbox writer's identity), whether its receipt is admission-backed or refused."""
  710. from gateway.owner_identity import same_owner
  711. if (record['message'] != message or not same_owner(record['principal_id'], actor.subject)
  712. or record.get('author') != author
  713. or record.get('notification_category', 'result') != category):
  714. raise RuntimeStoreError('admission_conflict')
  715. def _delivery_params(authority, actor, params):
  716. """``(home, key, message, author, category)`` of one ``deliver``/``settle`` request: the
  717. delivery's full identity, validated the same way for both verbs."""
  718. home = _home(authority, actor, params.get('profile'))
  719. if set(params) - {'id', 'profile', 'message', 'session_id', 'author', 'notification_category'}:
  720. raise RuntimeStoreError('invalid_params')
  721. from gateway.session_automation import automation_notification_metadata
  722. notification = automation_notification_metadata(params)
  723. category = notification.get('notification_category', 'result')
  724. try:
  725. key = _delivery_id(params.get('id'))
  726. except ValueError as exc:
  727. raise RuntimeStoreError('invalid_params') from exc
  728. message = params.get('message')
  729. # Main capped only a relayed bot DM (MESSAGE_MAX_CHARS + attribution headroom), at the relay
  730. # edge; a trusted producer's report (cron bot-chat, its deferred drain) carries no author and
  731. # was admitted whole, so it gets a resource bound only.
  732. limit = _BOT_DM_MAX_CHARS if params.get('author') is not None else _PRODUCER_MAX_CHARS
  733. if not isinstance(message, str) or not message.strip() or len(message) > limit:
  734. raise RuntimeStoreError('invalid_params')
  735. author = parse_turn_author(params.get('author'))
  736. if params.get('author') is not None and (not isinstance(params['author'], dict) or author is None):
  737. raise RuntimeStoreError('invalid_params')
  738. return home, key, message, author, category
  739. def _refused_answer(record):
  740. return {k: record[k] for k in ('status', 'refused', 'delivery_id', 'profile_home', 'session_id',
  741. 'message', 'reply', 'error', 'reason')}
  742. async def _receipt_answer(authority, actor, home, record, identity, retry_path=None):
  743. """The answer an id's existing receipt gives, or None when it has none yet: an admitted id's
  744. published result (after its one retry when ``retry_path`` is given), or a terminal refusal
  745. (``refuse_pin``, ``settle``), which is the id's answer for good. Both check the identity."""
  746. if record is None or not (record.get('admission_id') or record.get('refused') is True):
  747. return None
  748. _require_same_delivery(record, actor, *identity)
  749. if not record.get('admission_id'):
  750. return _refused_answer(record)
  751. authority.authorize(actor, SessionRef(authority.profile_id, record['session_id']), 'session:submit')
  752. if retry_path is not None:
  753. await _maybe_retry(authority, home, retry_path, record)
  754. return await _publish_result(authority, home, record)
  755. async def _deliver(connection, params):
  756. authority, actor = connection.authority, connection.actor
  757. home, key, message, author, category = _delivery_params(authority, actor, params)
  758. identity = (message, author, category)
  759. authority._require_admission_open()
  760. async with mailbox_lock(authority):
  761. root, record = await read_receipt(home, key)
  762. path = root / f'{key}.json'
  763. answer = await _receipt_answer(authority, actor, home, record, identity, retry_path=path)
  764. if answer is not None:
  765. return answer
  766. try:
  767. await _migrate(authority, actor, home, root)
  768. _, record = await read_receipt(home, key)
  769. answer = await _receipt_answer(authority, actor, home, record, identity, retry_path=path)
  770. if answer is not None:
  771. return answer
  772. if record is not None:
  773. raise RuntimeStoreError('unknown_execution')
  774. holder = authority.db.get_session_by_title('Bot Chat')
  775. ref, live, entry = await _target(authority, actor)
  776. pinned = params.get('session_id', entry.session_id)
  777. if pinned != entry.session_id and not _moved_aside_by_this_door(authority, holder, pinned):
  778. raise RuntimeStoreError('admission_conflict')
  779. return await _admit(authority, actor, home, root, key, message, ref, live, entry, author=author,
  780. notification_category=category)
  781. except RuntimeStoreError as exc:
  782. raise await _held_by_another_writer(home, key, exc) or exc
  783. async def settle(connection, params):
  784. from gateway.session_authorities import require_serving
  785. from gateway.session_runtime_workers import track_mutation
  786. # Retirement joins tracked work but cannot fence a later write: receipts carry no epoch and
  787. # mailbox_lock is this authority's own, so a stale connection is refused before it schedules.
  788. require_serving(connection.authority)
  789. connection.authority._require_admission_open()
  790. return await asyncio.shield(track_mutation(connection.authority, _settle(connection, params)))
  791. async def _settle(connection, params):
  792. """Finalize a delivery the sender was told may not have been admitted: under the mailbox lock
  793. every ``_admit`` holds, either settle the id as never sent or answer with what it already is.
  794. A ``deliver`` refusal marked ``admitted=False`` proves absence only while that request held
  795. the lock; a same-id attempt may admit right after. The sender calls this before it says "NOT
  796. sent": with no receipt and no admission row for ``bot:<id>``, a terminal ``refused`` receipt
  797. bound to the full identity is written, and every later ``deliver`` of the id answers with it
  798. instead of admitting. An admitted id gets its receipt; an in-flight pin stays unknown. Only the
  799. Bot DM tool calls this, so cron's same-id retries of a refusal are unchanged.
  800. The writer is not re-checked: it may release between the refusal and this call, and "nothing
  801. was admitted" is all that is serialized here. So the receipt claims only that, and any
  802. ``session:submit`` principal of the profile can settle an id it never delivered: the trust
  803. ``deliver`` already grants it over that id."""
  804. authority, actor = connection.authority, connection.actor
  805. home, key, message, author, category = _delivery_params(authority, actor, params)
  806. async with mailbox_lock(authority):
  807. _, record = await read_receipt(home, key)
  808. if record is None:
  809. record = _not_sent_record(authority, actor, home, key, message, author, category)
  810. await write_receipt(home, record)
  811. logging.getLogger(__name__).info('Bot Chat delivery %s settled as not sent (%s)', key, record['reason'])
  812. _require_same_delivery(record, actor, message, author, category)
  813. answer = await _receipt_answer(authority, actor, home, record, (message, author, category))
  814. if answer is None:
  815. raise RuntimeStoreError('unknown_execution')
  816. return answer
  817. def _not_sent_record(authority, actor, home, key, message, author, category):
  818. """The never-admitted id's terminal receipt, in ``refuse_pin``'s shape. Refuses (unknown) when
  819. the FIFO holds a ``bot:<id>`` row anyway or there is no Bot Chat to name."""
  820. with authority.db._read_ctx() as conn:
  821. admitted = conn.execute('SELECT 1 FROM session_admissions WHERE request_id=?', ('bot:' + key,)).fetchone()
  822. row = authority.db.get_session_by_title('Bot Chat')
  823. if admitted is not None or row is None:
  824. raise RuntimeStoreError('unknown_execution')
  825. from gateway.session_automation import automation_notification_metadata
  826. record = dict(delivery_id=key, profile_home=str(home), session_id=authority.db.get_compression_tip(row['id']),
  827. principal_id=actor.subject, message=message, status='failed', refused=True, reply='',
  828. error='Settled as not sent after a session-writer refusal', reason=HELD_NOT_SENT)
  829. record.update(automation_notification_metadata({'notification_category': category}))
  830. if author is not None:
  831. record['author'] = dict(author)
  832. return record
  833. async def _held_by_another_writer(home, key, exc):
  834. """The settleable form of a ``runtime_coordination_required`` refusal, or None.
  835. Another session writer holds the chat (a pre-authority Desktop backend's lease on a legacy Bot
  836. Chat, a live CLI). Nothing was admitted under this id when it still has no receipt: ``_admit``
  837. pins one before it admits and withdraws it only when nothing committed. That holds only while
  838. this request has the lock, so ``admitted=False`` is a hint, not the verdict: the sender asks
  839. ``settle`` before it reports a no-send. Any other refusal keeps its unknown outcome."""
  840. if exc.reason != HELD_NOT_SENT or (await read_receipt(home, key))[1] is not None:
  841. return None
  842. held = RuntimeStoreError(exc.reason, field='session_writer',
  843. detail='another session writer holds the Bot Chat; nothing was admitted yet')
  844. held.admitted = False
  845. return held
  846. async def _admit(authority, actor, home, root, key, message, ref, live, entry, author=None, legacy=None,
  847. notification_category='result'):
  848. from gateway.session_automation import automation_notification_metadata
  849. notification = automation_notification_metadata({'notification_category': notification_category})
  850. path = root / f'{key}.json'
  851. event = MessageEvent(text=message, source=live.source, internal=True,
  852. message_id='bot:' + key, metadata={'gateway_session_key': live.route,
  853. 'gateway_session_id': entry.session_id})
  854. event.metadata.update(notification)
  855. if author is not None:
  856. event.metadata['turn_author'] = dict(author)
  857. # Pin the physical target before committing. A process death in this
  858. # two-store window leaves an explicit unknown record, never a new target.
  859. record = dict(legacy or {}, delivery_id=key, profile_home=str(home), session_id=ref.session_id,
  860. principal_id=actor.subject, message=message, status='ambiguous')
  861. record.update(notification)
  862. if 'session:operator' in actor.capabilities:
  863. record['operator'] = True # admitted on operator access: its one retry keeps it (``_maybe_retry``)
  864. if author is not None:
  865. record['author'] = dict(author)
  866. # Preserve a foreign cancellation or claim that won after our legacy scan.
  867. if not await write_receipt(home, record, expect=(legacy or {}).get('status')):
  868. if legacy is not None:
  869. return None
  870. raise RuntimeStoreError('admission_conflict')
  871. try:
  872. receipt = await authority.admit_automation(authority.runner._adapter_for_source(live.source), event, 'bot:' + key)
  873. except RuntimeStoreError as exc:
  874. if _admission_at(authority, ref.session_id, 'bot:' + key) is not None:
  875. raise # committed after all: the pin stays the explicit unknown record
  876. if exc.reason in _OWNER_STATE_REFUSALS:
  877. if legacy is None:
  878. # Refused before any INSERT, not for the delivery itself: the same-id resend admits.
  879. await withdraw_pin(home, key)
  880. else:
  881. # Refused for good with nothing committed: a terminal receipt, not an unknown pin.
  882. logging.getLogger(__name__).info('Bot Chat delivery %s refused (%s)', key, exc.reason)
  883. await refuse_pin(home, key)
  884. raise
  885. record.update(status='canonical', admission_id=receipt.admission_id)
  886. record.update(_result(authority, record))
  887. if record['status'] in {'queued', 'claimed'}:
  888. _watch_reply(authority, home, key, receipt.admission_id)
  889. if record['status'] not in {'queued', 'claimed'}:
  890. await _maybe_retry(authority, home, path, record)
  891. return await _publish_result(authority, home, record)
  892. async def _publish_result(authority, home, record):
  893. """Publish the delivery's projection and answer with exactly that projection. The write is
  894. off-loop, so the admission may settle while it runs; re-deriving the answer afterwards would
  895. hand the sender a reply the receipt file does not hold yet (the reply watcher publishes it
  896. once this delivery releases the mailbox lock)."""
  897. result = _delivery_result(authority, record)
  898. record.update(result)
  899. await write_receipt(home, record)
  900. remember_receipt(authority, home, record)
  901. return result