session_local_adoption.py 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687
  1. """A local receipt's compression catch-up (#140060), committed off the owner loop.
  2. ``adopt_untracked_compression`` takes the writer lock, up to ``_WRITE_PATIENCE_S`` when contended. On
  3. the owner loop that stalls every profile it serves (streams, controls, admissions), including during
  4. a hot-profile activation. So the async paths that restore a receipt (boot and hot-profile recovery, a
  5. socket's first read) commit the catch-up in a worker first, and the restore that follows only reads.
  6. A restore reached synchronously on the loop hands it to a tracked worker (``schedule_adoption``).
  7. """
  8. import asyncio
  9. import logging
  10. from functools import partial
  11. from hermes_state_runtime import RuntimeStoreError
  12. logger = logging.getLogger(__name__)
  13. def on_owner_loop():
  14. try:
  15. asyncio.get_running_loop()
  16. except RuntimeError:
  17. return False
  18. return True
  19. def _due(authority, sid):
  20. """*sid*'s stored receipt when a catch-up would commit for it on this owner, else None. Read-only."""
  21. from gateway.session_local_recovery import owned_receipt
  22. from hermes_state_local_lineage import adoptable_compression
  23. try:
  24. receipt = owned_receipt(authority, sid)[0]
  25. with authority.db._read_ctx() as conn:
  26. return receipt if adoptable_compression(conn, receipt) is not None else None
  27. except (KeyError, TypeError, ValueError, AttributeError):
  28. return None # the restore refuses this receipt and logs the check that failed
  29. def _adopt(authority, sid, receipt):
  30. from hermes_state_local_lineage import adopt_untracked_compression
  31. adopt_untracked_compression(authority.db, epoch=authority.epoch, session_id=sid, expected=receipt)
  32. async def adopt_before_restore(authority, sid, *, tracked=True):
  33. """Commit *sid*'s due catch-up in a worker, so the restore that follows only reads.
  34. ``tracked=False``: an authority still being built, not yet published for retirement to join."""
  35. receipt = _due(authority, sid)
  36. if receipt is None:
  37. return
  38. write = partial(_adopt, authority, sid, receipt)
  39. try:
  40. if tracked:
  41. from gateway.session_runtime_workers import tracked_write
  42. await tracked_write(authority, write)
  43. else:
  44. await asyncio.to_thread(write)
  45. except RuntimeStoreError as exc: # the restore that follows refuses the receipt and logs it
  46. logger.warning('Local session %s compression catch-up failed: %s', sid, exc.reason)
  47. async def adopt_local_compressions(authority):
  48. """Boot and hot-profile recovery: every receipt's due catch-up, before ``recover_local_sessions``."""
  49. from gateway.session_local_recovery import local_receipt_ids
  50. for sid in local_receipt_ids(authority):
  51. await adopt_before_restore(authority, sid, tracked=False)
  52. def schedule_adoption(authority, sid, receipt):
  53. """A sync restore on the owner loop found *receipt*'s catch-up due: commit it in a tracked worker,
  54. then publish the session on the loop. One worker per session at a time."""
  55. from gateway.session_runtime_workers import track_mutation
  56. pending = getattr(authority, '_local_adoptions', None)
  57. if pending is None:
  58. pending = authority._local_adoptions = {}
  59. if sid in pending and not pending[sid].done():
  60. return
  61. logger.info('Local session %s: committing its compression catch-up off the owner loop', sid)
  62. async def adopt_then_publish():
  63. try:
  64. await asyncio.to_thread(_adopt, authority, sid, receipt)
  65. if not getattr(authority, 'retiring', False):
  66. from gateway.session_local_recovery import restore_local_session
  67. restore_local_session(authority, sid)
  68. except RuntimeStoreError as exc:
  69. logger.warning('Local session %s paused after its compression catch-up: %s', sid, exc.reason)
  70. pending[sid] = track_mutation(authority, adopt_then_publish())