session_bot_mailbox.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206
  1. """Authority-local async serialization; file locks cover only short synchronous mailbox I/O."""
  2. import asyncio
  3. from contextlib import asynccontextmanager
  4. from tools.bot_live_delivery import _locked, _read, _write
  5. @asynccontextmanager
  6. async def mailbox_lock(authority):
  7. lock = getattr(authority, '_bot_mailbox_lock', None)
  8. if lock is None:
  9. lock = authority._bot_mailbox_lock = asyncio.Lock()
  10. async with lock:
  11. active = getattr(authority, '_bot_mailbox_operations', None)
  12. if active is None:
  13. active = authority._bot_mailbox_operations = set()
  14. task = asyncio.current_task()
  15. active.add(task)
  16. try:
  17. yield
  18. finally:
  19. active.discard(task)
  20. from gateway.session_runtime_workers import persist_work_count
  21. persist_work_count(authority)
  22. async def _mailbox_io(operation, *args):
  23. """Keep the caller's mailbox ownership until its physical I/O ends, even after cancellation."""
  24. pending = asyncio.create_task(asyncio.to_thread(operation, *args))
  25. cancelled = None
  26. while True:
  27. try:
  28. result = await asyncio.shield(pending)
  29. except asyncio.CancelledError as exc:
  30. if pending.cancelled():
  31. raise
  32. cancelled = exc
  33. except Exception:
  34. if cancelled is not None:
  35. raise cancelled
  36. raise
  37. else:
  38. if cancelled is not None:
  39. raise cancelled
  40. return result
  41. def _read_receipt(home, key):
  42. with _locked(home) as root:
  43. return root, _read(root / f'{key}.json')
  44. async def read_receipt(home, key):
  45. return await _mailbox_io(_read_receipt, home, key)
  46. _UNCHECKED = object()
  47. def _write_receipt(home, record, expect=_UNCHECKED):
  48. with _locked(home) as root:
  49. path = root / f"{record['delivery_id']}.json"
  50. current = _read(path)
  51. if expect is not _UNCHECKED and (current or {}).get('status') != expect:
  52. return False
  53. if current is not None and current.get('principal_id'):
  54. from hermes_state_runtime import RuntimeStoreError
  55. fields = ('profile_home', 'session_id', 'principal_id', 'message', 'author')
  56. if (any(current.get(key) != record.get(key) for key in fields)
  57. or current.get('notification_category', 'result') != record.get('notification_category', 'result')
  58. or (current.get('admission_id') and current['admission_id'] != record.get('admission_id'))):
  59. raise RuntimeStoreError('admission_conflict')
  60. merged = {**(current or {}), **record}
  61. if merged != current:
  62. _write(path, merged)
  63. return True
  64. async def write_receipt(home, record, *, expect=_UNCHECKED):
  65. return await _mailbox_io(_write_receipt, home, record, expect)
  66. def _scan_receipts(home, pins=None):
  67. from gateway.session_bot import _scan_records
  68. with _locked(home) as root:
  69. return root, list(_scan_records(root, pins))
  70. def _withdraw_pin(home, key):
  71. with _locked(home) as root:
  72. path = root / f'{key}.json'
  73. current = _read(path)
  74. if current is not None and current.get('status') == 'ambiguous' and not current.get('admission_id'):
  75. path.unlink()
  76. return True
  77. return False
  78. async def withdraw_pin(home, key):
  79. """Drop a pre-admission pin the FIFO definitely refused (nothing committed), only while it is
  80. still that pin, so the same-id retry admits instead of answering ``unknown_execution``."""
  81. return await _mailbox_io(_withdraw_pin, home, key)
  82. # What ``_admit`` lays over a legacy ticket when it pins it (``legacy=``); the ticket never had them.
  83. _PIN_FIELDS = ('profile_home', 'session_id', 'principal_id', 'operator')
  84. def _requeue_pin(home, key):
  85. with _locked(home) as root:
  86. path = root / f'{key}.json'
  87. current = _read(path)
  88. if (current is None or current.get('status') != 'ambiguous' or current.get('admission_id')
  89. or 'owner' not in current):
  90. return None
  91. ticket = {k: v for k, v in current.items() if k not in _PIN_FIELDS}
  92. ticket['status'] = 'queued'
  93. _write(path, ticket)
  94. return ticket
  95. async def requeue_pin(home, key):
  96. """Restore the queued legacy ticket a pre-admission pin was laid over (nothing committed), only
  97. while it is still that pin: its id, message, sequence and created_at are kept, so ``_migrate``
  98. admits it once in its FIFO place. Returns the ticket, or None when the pin is gone or changed."""
  99. return await _mailbox_io(_requeue_pin, home, key)
  100. def _refuse_pin(home, key):
  101. from tools.bot_failure_reasons import AGENT_BLOCKED
  102. with _locked(home) as root:
  103. path = root / f'{key}.json'
  104. current = _read(path)
  105. if current is not None and current.get('status') == 'ambiguous' and not current.get('admission_id'):
  106. current.update(status='failed', refused=True, reply='', error='Bot Chat delivery was refused',
  107. reason=AGENT_BLOCKED)
  108. _write(path, current)
  109. async def refuse_pin(home, key):
  110. """Settle a pre-admission pin the FIFO definitely refused for good (nothing committed) as a
  111. terminal ``failed`` receipt, only while it is still that pin: never re-sent, nothing unknown."""
  112. return await _mailbox_io(_refuse_pin, home, key)
  113. async def scan_receipts(home, pins=None):
  114. """Bulk scan; an owner scan passes ``pins`` to collect pre-admission pins instead of warning."""
  115. return await _mailbox_io(_scan_receipts, home, pins)
  116. def remember_receipt(authority, home, record):
  117. index = getattr(authority, '_bot_receipt_index', None)
  118. if index is None:
  119. index = authority._bot_receipt_index = {}
  120. if record['status'] in {'settled', 'failed', 'cancelled'}:
  121. index.pop(record['delivery_id'], None)
  122. else:
  123. index[record['delivery_id']] = (home, record['session_id'])
  124. def track_receipt_task(authority, task):
  125. tasks = getattr(authority, '_bot_receipt_tasks', None)
  126. if tasks is None:
  127. tasks = authority._bot_receipt_tasks = set()
  128. tasks.add(task)
  129. def done(finished):
  130. tasks.discard(finished)
  131. if not finished.cancelled():
  132. error = finished.exception()
  133. if error is not None:
  134. import logging
  135. logging.getLogger(__name__).warning('Bot receipt projection failed: %s', type(error).__name__)
  136. task.add_done_callback(done)
  137. def wake_bot_receipts(authority, session_id):
  138. """Re-arm paused receipts from owner publication, without making external pollers dial RPC."""
  139. pending = getattr(authority, '_bot_receipt_refreshes', None)
  140. if pending is None:
  141. pending = authority._bot_receipt_refreshes = {}
  142. authority._bot_receipt_dirty = set()
  143. for key, (home, owner) in list(getattr(authority, '_bot_receipt_index', {}).items()):
  144. if owner != session_id:
  145. continue
  146. if key in pending and not pending[key].done():
  147. authority._bot_receipt_dirty.add(key)
  148. else:
  149. _start_refresh(authority, home, key)
  150. def _start_refresh(authority, home, key):
  151. from gateway.session_bot import refresh_receipt
  152. async def refresh_until_current():
  153. while True:
  154. authority._bot_receipt_dirty.discard(key)
  155. await refresh_receipt(authority, home, key)
  156. if key not in authority._bot_receipt_dirty or key not in authority._bot_receipt_index:
  157. return
  158. task = asyncio.create_task(refresh_until_current())
  159. authority._bot_receipt_refreshes[key] = task
  160. def done(finished):
  161. if authority._bot_receipt_refreshes.get(key) is finished:
  162. authority._bot_receipt_refreshes.pop(key, None)
  163. authority._bot_receipt_dirty.discard(key)
  164. task.add_done_callback(done)
  165. track_receipt_task(authority, task)