turn_lease.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179
  1. """Per-session turn lease — serializes the [load history -> run -> flush] region.
  2. Busy guards are keyed by ROUTING KEY but the transcript is owned by SESSION_ID, and
  3. ``switch_session()`` makes key->id many-to-one (/resume from a second chat, CLI-continuity,
  4. delegation pinning, topic tip-walks), so two keys could interleave flushes on one transcript
  5. (``user;user`` wedge). The lease serializes per RESOLVED session_id: acquired right before the
  6. transcript load, released in the dispatch layer's ``finally``; identity-checked release; a
  7. timed-out waiter fails CLOSED (:class:`TurnLeaseTimeoutError`); only idle entries evict. Limits:
  8. CLI-continuity processes are outside this lock; mid-turn rotation alias is closed by ``rebind``.
  9. """
  10. import asyncio
  11. import logging
  12. import time
  13. from typing import Dict, Optional
  14. logger = logging.getLogger(__name__)
  15. # Cap on tracked leases. Idle entries evict oldest-first; live leases never do, so a burst of
  16. # distinct sessions may transiently exceed the cap rather than break serialization.
  17. DEFAULT_MAX_LEASES = 512
  18. # Fallback wait (seconds) when the caller passes no positive timeout (bridged via
  19. # HERMES_TURN_LEASE_TIMEOUT — lease contention is not agent inactivity). Fail-closed but short:
  20. # never pin a sequential platform updater for minutes.
  21. DEFAULT_LEASE_WAIT = 5.0
  22. def _holder_desc(holder: "Optional[TurnLeaseToken]") -> tuple:
  23. return (holder.owner_key, holder.generation) if holder else ("?", "?")
  24. class TurnLeaseTimeoutError(TimeoutError):
  25. """Lease held for the full wait budget; fail-closed: caller must not enter the turn region."""
  26. def __init__(self, session_id: str, *, owner_key: str, generation: int, wait_seconds: float):
  27. self.session_id, self.owner_key = session_id, owner_key
  28. self.generation, self.wait_seconds = generation, wait_seconds
  29. super().__init__(f"turn lease wait timed out after {wait_seconds:.0f}s on session "
  30. f"{session_id} for routing key {owner_key} (gen {generation})")
  31. class TurnLeaseToken:
  32. """Held-lease handle from :meth:`SessionTurnLeaseRegistry.acquire`; ``released`` makes
  33. release idempotent."""
  34. __slots__ = ("generation", "lease", "owner_key", "released", "session_id")
  35. def __init__(self, session_id: str, owner_key: str, generation: int, lease: "_SessionLease") -> None:
  36. self.session_id, self.owner_key, self.generation = session_id, owner_key, generation
  37. self.released = False
  38. # The concrete lease, so release resolves by identity even after a rotation re-aliases
  39. # ``session_id`` (both ids map to this same lease).
  40. self.lease = lease
  41. def __repr__(self) -> str: # pragma: no cover - debug aid
  42. return (f"TurnLeaseToken(session_id={self.session_id!r}, owner_key={self.owner_key!r}, "
  43. f"generation={self.generation}, released={self.released})")
  44. class _SessionLease:
  45. __slots__ = ("acquired_at", "holder", "last_used", "lock", "pending_acquires")
  46. def __init__(self) -> None:
  47. self.lock = asyncio.Lock()
  48. self.holder: Optional[TurnLeaseToken] = None
  49. self.acquired_at, self.last_used, self.pending_acquires = 0.0, time.time(), 0
  50. @property
  51. def idle(self) -> bool:
  52. """True when evictable: nobody holds or awaits it."""
  53. return self.holder is None and not self.lock.locked() and self.pending_acquires == 0
  54. class SessionTurnLeaseRegistry:
  55. """Asyncio lease per resolved session_id. Process-local, single-event-loop by design (same
  56. visibility scope as the routing-key guards it extends); call only from the gateway loop."""
  57. def __init__(self, max_entries: int = DEFAULT_MAX_LEASES) -> None:
  58. self._leases: dict[str, _SessionLease] = {}
  59. self._max_entries = max(1, int(max_entries))
  60. def _get_or_create(self, session_id: str) -> _SessionLease:
  61. if (lease := self._leases.get(session_id)) is None:
  62. self._evict_idle()
  63. lease = self._leases[session_id] = _SessionLease()
  64. lease.last_used = time.time()
  65. return lease
  66. def _evict_idle(self) -> None:
  67. """Drop oldest idle entries to fit a new lease under the cap; never a held/contended one."""
  68. if (overflow := len(self._leases) - self._max_entries + 1) <= 0:
  69. return
  70. idle = sorted((sid for sid, l in self._leases.items() if l.idle),
  71. key=lambda sid: self._leases[sid].last_used)
  72. for sid in idle[:overflow]:
  73. self._leases.pop(sid, None)
  74. async def acquire(
  75. self, session_id: str, *, owner_key: str, generation: int, timeout: Optional[float] = None
  76. ) -> Optional[TurnLeaseToken]:
  77. """Acquire the lease for ``session_id``, waiting if held. Raises
  78. :class:`TurnLeaseTimeoutError` when the wait budget expires; None for a falsy id."""
  79. if not session_id:
  80. return None
  81. wait = float(timeout) if timeout and timeout > 0 else DEFAULT_LEASE_WAIT
  82. lease = self._get_or_create(session_id)
  83. token = TurnLeaseToken(session_id, owner_key, int(generation), lease=lease)
  84. if lease.lock.locked():
  85. logger.warning(
  86. "turn lease contention on session %s: routing key %s (gen %s) waiting behind "
  87. "in-flight turn held by routing key %s (gen %s, held %.0fs) — two routing keys "
  88. "are mapped to one session_id (#64934); serializing this turn behind the previous "
  89. "turn's flush",
  90. session_id, owner_key, generation, *_holder_desc(lease.holder),
  91. time.time() - lease.acquired_at if lease.acquired_at else -1.0)
  92. # Lock.release() wakes a waiter while leaving the lock momentarily unlocked. Count every
  93. # in-progress acquire across that handoff (even apparently-uncontended ones — wait_for()
  94. # may schedule them before the lock coroutine runs) so eviction cannot orphan the old
  95. # lock and create a second lock for the same session.
  96. lease.pending_acquires += 1
  97. try:
  98. await asyncio.wait_for(lease.lock.acquire(), timeout=wait)
  99. except TimeoutError:
  100. logger.error(
  101. "turn lease wait timed out after %.0fs on session %s (waiter: routing key %s gen "
  102. "%s; holder: routing key %s gen %s) — failing closed: refusing to run this turn "
  103. "UNSERIALIZED against the still-held lease",
  104. wait, session_id, owner_key, generation, *_holder_desc(lease.holder))
  105. raise TurnLeaseTimeoutError(
  106. session_id, owner_key=owner_key, generation=generation, wait_seconds=wait) from None
  107. finally:
  108. lease.pending_acquires -= 1
  109. # Lock held and no await before holder publication, so the lease cannot become
  110. # evictable after the pending count is cleared.
  111. lease.holder = token
  112. lease.acquired_at = lease.last_used = time.time()
  113. return token
  114. def rebind(self, token: Optional[TurnLeaseToken], new_session_id: str) -> bool:
  115. """Alias a HELD lease onto ``new_session_id`` after mid-turn rotation (compression) so the
  116. flush target stays serialized: the SAME ``_SessionLease`` is registered under the new id
  117. (old mapping idle-evicts later), only the holder may rebind, the token follows. A live
  118. lease on the new id: log loudly, keep the old id (fail-open)."""
  119. if (token is None or token.released or not new_session_id
  120. or new_session_id == token.session_id):
  121. return False
  122. lease = token.lease
  123. if lease.holder is not token:
  124. return False
  125. existing = self._leases.get(new_session_id)
  126. if existing is not None and existing is not lease and not existing.idle:
  127. logger.warning(
  128. "turn lease rebind blocked: session %s rotated to %s mid-turn (holder: routing key "
  129. "%s gen %s) but the target session's lease is already live (holder: routing key %s "
  130. "gen %s) — keeping the lease on the old id; transcript writes on %s may "
  131. "interleave (#64934 rotation-alias edge)",
  132. token.session_id, new_session_id, token.owner_key, token.generation,
  133. *_holder_desc(existing.holder), new_session_id)
  134. return False
  135. self._leases[new_session_id] = lease
  136. lease.last_used = time.time()
  137. token.session_id = new_session_id
  138. return True
  139. def release(self, token: Optional[TurnLeaseToken]) -> bool:
  140. """Release ``token``'s lease. Idempotent; True only when this exact token was the current
  141. holder (a re-release or a stale token whose slot went to a newer turn is a safe no-op)."""
  142. if token is None or token.released:
  143. return False
  144. token.released = True
  145. lease = token.lease
  146. if lease.holder is not token:
  147. logger.debug("turn lease release skipped on session %s: token (key %s gen %s) is not "
  148. "the current holder", token.session_id, token.owner_key, token.generation)
  149. return False
  150. lease.holder, lease.acquired_at, lease.last_used = None, 0.0, time.time()
  151. if lease.lock.locked():
  152. lease.lock.release()
  153. return True