session_unattended_unknown.py 4.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485
  1. """An ``unknown`` turn nobody is waiting on settles as interrupted (#134889 GW-10, #139532).
  2. A turn lost across an owner restart is fenced ``unknown`` and pauses its session's FIFO, because
  3. only the person who sent it knows whether to resend it. An automation delivery has no such person:
  4. a cron run (``CRON_PRINCIPAL_ID``), a process completion or heartbeat (``automation:<route>``,
  5. ``local_automation_v1``, a native envelope's ``automation`` block) or a webhook delivery
  6. (``webhook_delivery``) or a Kanban card attempt (``kanban_card_v1``, set by the dispatcher). Pausing on it stalls every message queued behind it until an operator
  7. notices (a Bot Chat sat ten hours behind one).
  8. Such a row settles through the same fenced resolution a person's Discard uses
  9. (``resolve_unknown_session_input``): terminal ``interrupted``, never requeued or replayed, with a
  10. Hermes transcript notice committed in the same transaction. No second recovery store.
  11. """
  12. from __future__ import annotations
  13. import logging
  14. from hermes_state_keys import CRON_PRINCIPAL_ID
  15. logger = logging.getLogger(__name__)
  16. def is_unattended(row) -> bool:
  17. """True for an admission no person submitted (so nobody can be asked to resend it)."""
  18. principal = row.get('principal_id') or ''
  19. if principal == CRON_PRINCIPAL_ID or principal.startswith('automation:'):
  20. return True
  21. payload = row.get('payload') or {}
  22. native = payload.get('native_text_v1') or {}
  23. # A webhook delivery: the provider already got its 202 and nobody can be asked to resend it.
  24. # A Kanban card attempt (#134889 KB-8): the dispatcher's marker, committed only by
  25. # ``SessionAuthority.submit(_kanban_card=...)``; a card row main minted without it still pauses.
  26. return ('local_automation_v1' in payload or 'kanban_card_v1' in payload
  27. or 'automation' in native or 'webhook_delivery' in native)
  28. def _unattended_unknown(rows):
  29. return [row for row in rows if row['status'] == 'unknown' and is_unattended(row)]
  30. def settle_recovered_unattended(db, epoch) -> int:
  31. """Boot recovery, right after ``recover_session_inputs`` fenced lost turns ``unknown`` and before
  32. any FIFO is scheduled: settle every unattended one. Synchronous; runs off the owner loop."""
  33. from agent.turn_failure_copy import AUTOMATION_INTERRUPTED_NOTICE
  34. from gateway.session_ingress_media import release_admission_media
  35. from gateway.session_results import close_discarded_turn
  36. from hermes_state_runtime import RuntimeStoreError, _row, resolve_unknown_session_input
  37. with db._read_ctx() as conn:
  38. rows = [_row(r) for r in conn.execute(
  39. "SELECT * FROM session_admissions WHERE status='unknown' ORDER BY seq").fetchall()]
  40. settled = 0
  41. for row in _unattended_unknown(rows):
  42. try:
  43. resolve_unknown_session_input(
  44. db, epoch=epoch, admission_id=row['admission_id'], generation=row['generation'],
  45. _terminal_write=lambda conn, lost: close_discarded_turn(db, conn, lost, AUTOMATION_INTERRUPTED_NOTICE))
  46. except RuntimeStoreError as exc:
  47. logger.warning('Lost automation admission %s left unknown: %s', row['admission_id'], exc.reason)
  48. continue
  49. release_admission_media(db, row['admission_id'])
  50. settled += 1
  51. logger.info('Lost automation admission %s on session %s settled interrupted (not replayed)',
  52. row['admission_id'], row['target_session_id'])
  53. return settled
  54. async def settle_unattended_unknown(authority, ref) -> bool:
  55. """The drain found an ``unknown`` head (fenced in this owner, e.g. a lost managed worker):
  56. settle the unattended ones through the owner's resolution. True when any settled, so the
  57. drain re-reads its FIFO instead of pausing; a person's ``unknown`` row still pauses it."""
  58. import asyncio
  59. from agent.turn_failure_copy import AUTOMATION_INTERRUPTED_NOTICE
  60. from hermes_state_runtime import RuntimeStoreError, list_session_admissions
  61. rows = await asyncio.to_thread(list_session_admissions, authority.db, session_id=ref.session_id)
  62. settled = False
  63. for row in _unattended_unknown(rows):
  64. try:
  65. await authority.resolve_unknown_as_owner(ref, row['admission_id'], row['generation'],
  66. notice=AUTOMATION_INTERRUPTED_NOTICE)
  67. except RuntimeStoreError as exc:
  68. logger.warning('Lost automation admission %s left unknown: %s', row['admission_id'], exc.reason)
  69. continue
  70. settled = True
  71. return settled