session_db_recovery.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177
  1. """Recoverable per-path SessionDB handle caches for the gateway."""
  2. from __future__ import annotations
  3. import contextlib
  4. import threading
  5. import time
  6. import weakref
  7. from dataclasses import dataclass
  8. from pathlib import Path
  9. from typing import Any, Callable
  10. _INITIAL_RETRY_DELAY_SECONDS = 1.0
  11. _MAX_RETRY_DELAY_SECONDS = 60.0
  12. @dataclass
  13. class _Unavailable:
  14. failures: int = 0
  15. next_retry_at: float = 0.0
  16. in_flight: bool = False
  17. error: str = ""
  18. class _HealthSource:
  19. """Weak-keyable identity for one cache's health entries."""
  20. _health_lock = threading.Lock()
  21. _health_states: weakref.WeakKeyDictionary[_HealthSource, dict[Path, str]]
  22. _health_states = weakref.WeakKeyDictionary()
  23. def _publish_health(source: _HealthSource, path: Path, state: str) -> None:
  24. """Publish one privacy-safe aggregate (no paths, no errors) across all live caches."""
  25. with _health_lock:
  26. _health_states.setdefault(source, {})[path] = state
  27. all_states = {value for item in _health_states.values() for value in item.values()}
  28. aggregate = next((s for s in ("retrying", "unavailable") if s in all_states), "ok")
  29. try:
  30. from gateway.status import publish_runtime_status
  31. publish_runtime_status(session_store={"status": aggregate})
  32. except Exception:
  33. pass # Runtime health is diagnostic only; persistence must not depend on it.
  34. def _registry_owned(handle: Any) -> bool:
  35. """True while the process-wide registry owns *handle*; an ``AsyncSessionDB`` (the only handle
  36. type with a ``_db``) is judged by the SessionDB it wraps. ``close_all`` / ``close_all_under``
  37. clear the flag when they tear the generation down."""
  38. inner = getattr(handle, "_db", handle)
  39. return getattr(inner, "_shared_registry_owned", None) is True
  40. class RecoverableHandleCache:
  41. """Cache handles by path while allowing failed opens to heal in-process.
  42. Opens run OUTSIDE ``lock`` (single-flight per path via ``in_flight``); ``close_all`` bumps
  43. ``_generation`` so a later-completing open is rejected instead of resurrecting the cache.
  44. """
  45. def __init__(
  46. self, *, handles: dict[Path, Any] | None = None, lock: threading.Lock | None = None,
  47. clock: Callable[[], float] = time.monotonic,
  48. initial_retry_delay: float = _INITIAL_RETRY_DELAY_SECONDS,
  49. max_retry_delay: float = _MAX_RETRY_DELAY_SECONDS,
  50. ) -> None:
  51. self.handles = handles if handles is not None else {}
  52. self.lock = lock if lock is not None else threading.Lock()
  53. self._clock = clock
  54. self._initial_retry_delay = max(0.0, float(initial_retry_delay))
  55. self._max_retry_delay = max(self._initial_retry_delay, float(max_retry_delay))
  56. self._unavailable: dict[Path, _Unavailable] = {}
  57. # Paths whose cached handle the registry owned when it was cached: once the registry tears
  58. # that generation down (profile unserve/delete), the entry is dead and must be reopened.
  59. self._registry_backed: set[Path] = set()
  60. self._health_source = _HealthSource()
  61. self._generation = 0
  62. self._close_rejected: Callable[[Any], None] | None = None
  63. def _is_stale(self, path: Path, unavailable: _Unavailable, generation: int) -> bool:
  64. """Caller holds ``lock``: True when close_all ran or the slot was replaced mid-open."""
  65. return generation != self._generation or self._unavailable.get(path) is not unavailable
  66. def get(
  67. self, path: Path, opener: Callable[[], Any], *, raise_on_error: bool = False,
  68. on_recovered: Callable[[], None] | None = None,
  69. non_cacheable: Callable[[Exception], bool] | None = None,
  70. ) -> Any:
  71. """Return a cached handle or make one bounded, single-flight open attempt; None while
  72. a retry is in flight or backing off. ``non_cacheable`` exceptions (e.g. a live-system
  73. guard) are re-raised without recording a failure so the next call retries at once."""
  74. path = Path(path)
  75. with self.lock:
  76. if path in self.handles:
  77. handle = self.handles[path]
  78. if path not in self._registry_backed or _registry_owned(handle):
  79. return handle
  80. # The registry force-closed this generation. Serving it would let its self-heal reopen
  81. # a writer the registry cannot see (a second writer beside the next ``acquire``), or keep
  82. # raising StateDbReplacedError after a delete + recreate: reopen through the registry.
  83. del self.handles[path]
  84. self._registry_backed.discard(path)
  85. unavailable = self._unavailable.setdefault(path, _Unavailable())
  86. if unavailable.in_flight or self._clock() < unavailable.next_retry_at:
  87. return None
  88. unavailable.in_flight = True
  89. was_unavailable = unavailable.failures > 0
  90. generation = self._generation
  91. if was_unavailable:
  92. _publish_health(self._health_source, path, "retrying")
  93. try:
  94. handle = opener()
  95. except Exception as exc:
  96. uncacheable = non_cacheable is not None and non_cacheable(exc)
  97. with self.lock:
  98. stale = self._is_stale(path, unavailable, generation)
  99. if uncacheable:
  100. if not stale:
  101. self._unavailable.pop(path, None)
  102. raise
  103. if not stale:
  104. unavailable.failures += 1
  105. unavailable.error = f"{type(exc).__name__}: {exc}"
  106. backoff = self._initial_retry_delay * (2 ** min(unavailable.failures - 1, 30))
  107. unavailable.next_retry_at = self._clock() + min(backoff, self._max_retry_delay)
  108. unavailable.in_flight = False
  109. if not stale:
  110. _publish_health(self._health_source, path, "unavailable")
  111. if raise_on_error:
  112. raise
  113. return None
  114. with self.lock:
  115. stale = self._is_stale(path, unavailable, generation)
  116. if not stale:
  117. self.handles[path] = handle
  118. if _registry_owned(handle):
  119. self._registry_backed.add(path)
  120. else:
  121. self._registry_backed.discard(path)
  122. self._unavailable.pop(path, None)
  123. close_rejected = self._close_rejected if stale else None
  124. if stale:
  125. if close_rejected is not None:
  126. with contextlib.suppress(Exception):
  127. close_rejected(handle)
  128. return None
  129. _publish_health(self._health_source, path, "ok")
  130. if was_unavailable and on_recovered is not None:
  131. on_recovered()
  132. return handle
  133. def last_error(self, path: Path) -> str:
  134. """The exception the latest failed open of *path* raised (``''`` when none is recorded)."""
  135. target = Path(path).resolve()
  136. with self.lock:
  137. return next((u.error for key, u in self._unavailable.items() if key.resolve() == target), "")
  138. def close_all(self, close: Callable[[Any], None]) -> None:
  139. """Drain cached handles under the lock and close them outside it."""
  140. with self.lock:
  141. self._generation += 1
  142. self._close_rejected = close
  143. handles = list(self.handles.values())
  144. paths = set(self.handles) | set(self._unavailable)
  145. self.handles.clear()
  146. self._registry_backed.clear()
  147. self._unavailable.clear()
  148. for handle in handles:
  149. with contextlib.suppress(Exception):
  150. close(handle)
  151. with _health_lock:
  152. states = _health_states.get(self._health_source, {})
  153. for path in paths:
  154. states.pop(path, None)