hermes_state_readpool.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259
  1. """Read-connection budgeting for SessionDB's WAL read path: per-file and
  2. process-wide permit ceilings plus a descriptor-headroom gate, so N handles on
  3. one state.db cannot walk the process into EMFILE. Idle pooled connections keep
  4. their permit and are reclaimable across peers on the same path."""
  5. import errno
  6. import logging
  7. import os
  8. import threading
  9. import time
  10. import weakref
  11. from pathlib import Path
  12. from typing import TYPE_CHECKING, Iterator, Optional
  13. if TYPE_CHECKING: # pragma: no cover
  14. from hermes_state import SessionDB
  15. # caplog tests pin the "hermes_state" logger name.
  16. logger = logging.getLogger("hermes_state")
  17. # Ceiling on read-only connections ALIVE at once against one database FILE (idle
  18. # pooled + checked out, over every SessionDB on that file). One constant for both
  19. # the pool maxsize and the permit count: a LifoQueue only caps how many are
  20. # *returned*, and EMFILE is a peak-instant condition, so a connection holds a
  21. # permit for its whole lifetime; once permits are gone reads degrade to the
  22. # locked writer connection — slower, but not a wedge the supervisor can't see.
  23. # Transient SQLITE_IOERR retry budget for READ-ONLY opens (#100436). A WAL database being actively written
  24. # (checkpoint, WAL reset/truncate, frame flush) can surface "disk I/O error" to a concurrent ``mode=ro``
  25. # reader in a millisecond-wide transition window: the read-only connection cannot perform the WAL recovery a
  26. # read through a stale or mid-update -shm file needs, because recovery requires writing the -shm index,
  27. # which mode=ro refuses. The window closes on its own (the writer finishes the transition), so a bounded
  28. # number of short retries makes the open succeed instead of 500-ing the whole /api/sessions poll (or any
  29. # other read-only opener). Deliberately NOT attempted on writable opens: a writer owns the transition, so an
  30. # IOERR there means a real storage/fd problem.
  31. _READ_POOL_MAX = 8
  32. # Ceiling ALIVE in this PROCESS across every state.db (a multiplexed gateway
  33. # opens one per profile); three profiles' worth, then readers degrade likewise.
  34. # _READ_POOL_MAX bounds one file. A multiplexed gateway serves N profiles from one process and each profile
  35. # has its OWN state.db, so a per-file ceiling still lets the descriptor cost grow with the profile count —
  36. # the same shape as the per-instance bug, one level out (#98573). Past it, readers on the (N+1)th file
  37. # degrade to their writer connection instead of opening descriptors, which is the same trade _READ_POOL_MAX
  38. # makes and for the same reason: a slow read path is recoverable, a process-wide EMFILE is not.
  39. _READ_POOL_PROCESS_MAX = 24
  40. # Warn past this many SessionDB handles on one file in one process (diagnostic:
  41. # writer connections cannot be rationed the way read connections can).
  42. _HANDLES_PER_PATH_WARN = 4
  43. # Descriptors kept in reserve for everything that is NOT this module (httpx
  44. # sockets, terminal pipes, log files): the EMFILE SQLite pushes over surfaces elsewhere.
  45. # The ceilings above bound Hermes's SQLite descriptors, which is only ever part of the fd table. The #98573
  46. # report is exactly that case: ~20 state.db descriptors were not the whole 256, they were the share that
  47. # pushed httpx and terminal pipes over, and the EMFILE surfaced in tools/terminal_tool.py rather than here.
  48. # So the read pool also yields when the PROCESS is close to its limit, whatever is consuming it.
  49. _FD_HEADROOM_RESERVE = 64
  50. # The fd count is a directory listing; cache it briefly so a read burst isn't a
  51. # syscall per query (staleness lets through at most the ceiling's worth of opens).
  52. _FD_USAGE_CACHE_SECONDS = 0.25
  53. _process_read_permits = threading.BoundedSemaphore(_READ_POOL_PROCESS_MAX)
  54. # Read opens refused for low descriptor headroom — the only visible signal the
  55. # guard fires. Guarded by _read_budgets_lock.
  56. _read_open_denied_fd_headroom = 0
  57. _fd_usage_lock = threading.Lock()
  58. _fd_usage_cache: tuple[float, Optional[int]] = (0.0, None)
  59. def _proc_fd_targets(pid: int) -> Iterator[tuple[str, str]]:
  60. """Yield ``(readlink target, fd path)`` for every entry in /proc/<pid>/fd (unreadable
  61. links skipped). Raises OSError when the fd directory itself cannot be listed."""
  62. fd_dir = f"/proc/{pid}/fd"
  63. for fd in os.listdir(fd_dir):
  64. fd_path = f"{fd_dir}/{fd}"
  65. try:
  66. yield os.readlink(fd_path), fd_path
  67. except OSError:
  68. continue
  69. def _open_fd_count() -> Optional[int]:
  70. """Open descriptors in THIS process; None when unmeasurable (Windows: no fd
  71. dir, correctly inert); -1 when the probe itself hit EMFILE/ENFILE (no headroom)."""
  72. for fd_dir in ("/proc/self/fd", "/dev/fd"):
  73. try:
  74. return len(os.listdir(fd_dir))
  75. except OSError as exc:
  76. if exc.errno in (errno.EMFILE, errno.ENFILE):
  77. return -1
  78. return None
  79. def _fd_soft_limit() -> Optional[int]:
  80. """The process's soft RLIMIT_NOFILE, or None when there is no usable one."""
  81. try:
  82. import resource
  83. except ImportError:
  84. return None
  85. try:
  86. soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
  87. except (AttributeError, OSError, ValueError):
  88. return None
  89. if soft in (resource.RLIM_INFINITY, -1):
  90. return None
  91. return int(soft)
  92. def _fd_headroom_ok() -> bool:
  93. """Can the process spare a descriptor for a new read connection? Fails OPEN
  94. when unmeasurable (refusing every read would be a self-inflicted convoy);
  95. fails CLOSED only on evidence (measured shortfall or a starved probe)."""
  96. soft = _fd_soft_limit()
  97. if soft is None:
  98. return True
  99. global _fd_usage_cache
  100. now = time.monotonic()
  101. with _fd_usage_lock:
  102. stamp, cached = _fd_usage_cache
  103. fresh = cached is not None and (now - stamp) < _FD_USAGE_CACHE_SECONDS
  104. if not fresh:
  105. cached = _open_fd_count()
  106. with _fd_usage_lock:
  107. _fd_usage_cache = (now, cached)
  108. if cached is None:
  109. return True
  110. return cached >= 0 and (soft - cached) > _FD_HEADROOM_RESERVE
  111. def _reclaim_idle_read_conn_anywhere() -> bool:
  112. """Close one idle read connection on ANY path: the process ceiling is shared
  113. across files, so a quiet profile must not hold descriptors a busy one needs."""
  114. with _read_budgets_lock:
  115. budgets = list(_read_budgets.values())
  116. return any(budget.reclaim_idle() for budget in budgets)
  117. class _PathReadBudget:
  118. """Read-connection permits for ONE database file, shared process-wide
  119. (per-instance semaphores let N SessionDBs peak at N x (1 + MAX)). An idle
  120. pooled connection keeps its permit, so a permit miss first reclaims an IDLE
  121. connection from a peer on the same path.
  122. ``_READ_POOL_MAX`` used to be enforced by a ``BoundedSemaphore`` owned by each SessionDB, which bounded
  123. the wrong noun: the descriptors are spent on a *file*, so N SessionDB objects on one state.db each got
  124. their own allowance and peak scaled as ``N x (1 + _READ_POOL_MAX)``. A long-lived gateway holds at least
  125. two (``SessionStore`` and ``GatewayRunner`` open independent handles per profile path) and the count
  126. grows with the profile count, which is how a healthy process walked into EMFILE — #98573.
  127. """
  128. def __init__(self) -> None:
  129. self.permits = threading.BoundedSemaphore(_READ_POOL_MAX)
  130. self._lock = threading.Lock()
  131. # Weak: a SessionDB dropped without close() must not pin peers' budget.
  132. self._members: weakref.WeakSet[SessionDB] = weakref.WeakSet()
  133. self._duplicate_handles_warned = False
  134. def register(self, db: "SessionDB") -> None:
  135. with self._lock:
  136. self._members.add(db)
  137. # Only writable handles carry the cost the warning names (writer connection, write
  138. # lock, close-time checkpoint). Read-only attaches (dashboard routers, status/lookup
  139. # one-shots) open per request by design and must not trip it.
  140. handles = sum(
  141. 1 for member in self._members
  142. if not member.read_only and member._conn is not None
  143. )
  144. warn = (handles > _HANDLES_PER_PATH_WARN and not self._duplicate_handles_warned)
  145. if warn:
  146. self._duplicate_handles_warned = True
  147. # Same population as `handles`: the read-only attaches the count skips
  148. # must not reappear in the list, or the list no longer explains the number.
  149. creation_sites = ", ".join(sorted(
  150. member._creation_site for member in self._members if not member.read_only
  151. ))
  152. if warn:
  153. # Writer connections cannot be capped; the only bound is not opening
  154. # redundant handles, so make the duplicate visible before it's an incident.
  155. logger.warning(
  156. # The only real bound on writers is not opening redundant handles in the first place (which
  157. # is what GatewayRunner borrowing SessionStore's handle does, #98573), so the next duplicate
  158. # should be visible before it becomes an incident rather than inferred from an lsof after
  159. # one.
  160. "%d live SessionDB handles on %s in this process; each holds "
  161. "its own writer connection (read connections are capped at %d "
  162. "for the file). A long-lived process should share one handle per path. "
  163. "Created at: %s",
  164. handles, db.db_path, _READ_POOL_MAX, creation_sites,
  165. )
  166. def unregister(self, db: "SessionDB") -> None:
  167. """Remove a closed writer from duplicate-handle diagnostics immediately."""
  168. with self._lock:
  169. self._members.discard(db)
  170. def acquire(self, requester: "SessionDB") -> bool:
  171. """Take a permit for a new read connection, or refuse (caller degrades to the
  172. locked writer connection). Gates, broadest first: fd headroom, process
  173. ceiling, this file's ceiling."""
  174. if not _fd_headroom_ok():
  175. global _read_open_denied_fd_headroom
  176. with _read_budgets_lock:
  177. _read_open_denied_fd_headroom += 1
  178. return False
  179. if not self._acquire_process_permit():
  180. return False
  181. if self._acquire_path_permit(requester):
  182. return True
  183. _process_read_permits.release()
  184. return False
  185. def release(self) -> None:
  186. """Return one connection's permits. Pairs with a successful acquire()."""
  187. self.permits.release()
  188. _process_read_permits.release()
  189. def _acquire_process_permit(self) -> bool:
  190. # Another thread may take a freed permit first: legitimate loss, no looping.
  191. return _process_read_permits.acquire(blocking=False) or (
  192. _reclaim_idle_read_conn_anywhere() and _process_read_permits.acquire(blocking=False)
  193. )
  194. def _acquire_path_permit(self, requester: "SessionDB") -> bool:
  195. return self.permits.acquire(blocking=False) or (
  196. self.reclaim_idle(exclude=requester) and self.permits.acquire(blocking=False)
  197. )
  198. def reclaim_idle(self, exclude: "Optional[SessionDB]" = None) -> bool:
  199. """Close one idle pooled connection held by a member; True if one went.
  200. Its release() returns both permits, so both ceilings reclaim through here."""
  201. with self._lock:
  202. members = [db for db in self._members if db is not exclude]
  203. return any(member._evict_one_idle_read_conn() for member in members)
  204. # canonical db path -> permits for that file. Weak values: the budget lives only
  205. # while some SessionDB on the path holds it, so tmp_path churn can't grow this.
  206. _read_budgets: weakref.WeakValueDictionary[str, _PathReadBudget] = (weakref.WeakValueDictionary())
  207. _read_budgets_lock = threading.Lock()
  208. def _read_budget_key(db_path) -> str:
  209. """Canonicalise a db path so two spellings share one budget."""
  210. try:
  211. return str(Path(db_path).resolve())
  212. except OSError:
  213. return str(db_path)
  214. def _read_budget_for(db_path) -> _PathReadBudget:
  215. key = _read_budget_key(db_path)
  216. with _read_budgets_lock:
  217. budget = _read_budgets.get(key)
  218. if budget is None:
  219. budget = _PathReadBudget()
  220. _read_budgets[key] = budget
  221. return budget