active_sessions.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846
  1. """Cross-process active chat session leases.
  2. The session database records persisted conversations; this module records
  3. currently open chat surfaces, including idle CLI/TUI sessions that have not
  4. written a transcript row yet.
  5. """
  6. from __future__ import annotations
  7. import json
  8. import logging
  9. import collections
  10. import errno
  11. import math
  12. import os
  13. import threading
  14. import time
  15. import uuid
  16. from contextlib import contextmanager, suppress
  17. from dataclasses import dataclass
  18. from pathlib import Path
  19. from typing import Any, Iterator, Optional
  20. from hermes_constants import get_default_hermes_root, get_hermes_home, named_profile_is_live
  21. from utils import atomic_json_write
  22. logger = logging.getLogger(__name__)
  23. class ActiveSessionRegistryError(RuntimeError):
  24. """The liveness registry could not prove a safe ownership decision."""
  25. def coerce_max_concurrent_sessions(value: Any, key: str = "max_concurrent_sessions") -> Optional[int]:
  26. """Return a positive integer cap, or None when disabled/invalid."""
  27. if value is None:
  28. return None
  29. try:
  30. if isinstance(value, bool) or (isinstance(value, float) and not value.is_integer()):
  31. raise ValueError(value)
  32. parsed = int(value.strip(), 10) if isinstance(value, str) else int(value)
  33. except (TypeError, ValueError):
  34. logger.warning(
  35. "Ignoring invalid %s=%r (expected a positive integer; 0/null disables)", key, value
  36. )
  37. return None
  38. return parsed if parsed > 0 else None
  39. def resolve_max_concurrent_sessions(config: Any) -> Optional[int]:
  40. """Resolve top-level max_concurrent_sessions with gateway.* fallback."""
  41. raw: Any = None
  42. key = "max_concurrent_sessions"
  43. if isinstance(config, dict):
  44. if "max_concurrent_sessions" in config:
  45. raw = config.get("max_concurrent_sessions")
  46. else:
  47. gateway_cfg = config.get("gateway")
  48. if isinstance(gateway_cfg, dict):
  49. raw = gateway_cfg.get("max_concurrent_sessions")
  50. key = "gateway.max_concurrent_sessions"
  51. else:
  52. raw = getattr(config, "max_concurrent_sessions", None)
  53. return coerce_max_concurrent_sessions(raw, key=key)
  54. def format_age(seconds: float) -> str:
  55. minutes = max(0, int(seconds // 60))
  56. if minutes < 60:
  57. return f"{minutes}m"
  58. hours, minutes = divmod(minutes, 60)
  59. return f"{hours}h" if not minutes else f"{hours}h{minutes}m"
  60. def summarize_holders(entries: list[dict[str, Any]]) -> str:
  61. """Compact "who is holding the slots" phrase, e.g. ``desktop x4, cli``."""
  62. if not entries:
  63. return ""
  64. counts = collections.Counter(str(e.get("surface") or "unknown") for e in entries)
  65. held = ", ".join(
  66. f"{surface} x{n}" if n > 1 else surface
  67. for surface, n in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
  68. )
  69. started = [t for t in (_optional_float(e.get("started_at")) for e in entries) if t]
  70. if started:
  71. held += f", oldest {format_age(time.time() - min(started))} ago"
  72. return held
  73. def active_session_limit_message(
  74. active_count: int, max_sessions: int, entries: Optional[list[dict[str, Any]]] = None
  75. ) -> str:
  76. # Name the holders: slots are shared across CLI, desktop/TUI and gateway,
  77. # so the rejected surface is usually NOT the one squatting on them.
  78. held = summarize_holders(entries or [])
  79. detail = f" Held by: {held}." if held else ""
  80. return (
  81. f"Hermes is at the active session limit ({active_count}/{max_sessions})."
  82. f"{detail} Try again when another session finishes."
  83. )
  84. # Machine-readable refusal reasons (the reason is the contract, the message is for
  85. # people). Capacity = "busy, come back later"; ownership = "a live owner exists and
  86. # writing would interleave with theirs".
  87. SESSION_NOT_OWNED = "SESSION_NOT_OWNED"
  88. MAX_CONCURRENT_SESSIONS = "MAX_CONCURRENT_SESSIONS"
  89. # Ownership could not be PROVEN (registry unreadable/corrupt). Deliberately distinct
  90. # from SESSION_NOT_OWNED: treating "can't tell" as a go-ahead is the fail-open hole
  91. # that let two writers share one session.
  92. # Distinct from SESSION_NOT_OWNED on purpose -- "someone else owns this" and "I cannot tell who owns this"
  93. # call for different operator action, and collapsing the second into a silent go-ahead is exactly the
  94. # fail-open hole that let two writers share one session (#94595 review, blocker 2).
  95. SESSION_COORDINATION_UNAVAILABLE = "SESSION_COORDINATION_UNAVAILABLE"
  96. # Advertised through the gateway. A module constant, not a config flag: it holds
  97. # because try_acquire_active_session checks atomically, so it cannot drift from the
  98. # enforcement without this file changing.
  99. PER_SESSION_EXCLUSIVE_SUBMIT = True
  100. class ActiveSessionRefusal(str):
  101. """Refusal message (a ``str``, so callers are untouched) with a machine-readable ``reason``."""
  102. reason: str
  103. def __new__(cls, message: str, reason: str) -> ActiveSessionRefusal:
  104. obj = super().__new__(cls, message)
  105. obj.reason = reason
  106. return obj
  107. def __getnewargs_ex__(self) -> tuple[tuple[str, str], dict[str, Any]]:
  108. # copy/pickle rebuild via __new__: without the reason, publishing a refused turn's
  109. # message.complete (deepcopy of the payload) raises and the viewer hangs.
  110. return (str(self), self.reason), {}
  111. def format_refusal_stderr(message: str) -> str:
  112. """Keep the refusal contract across the one-shot CLI subprocess boundary."""
  113. reason = getattr(message, "reason", "")
  114. return f"hermes-refusal-reason: {reason}\n{message}" if reason else str(message)
  115. def _is_same_writer(entry: dict[str, Any], metadata: Optional[dict[str, Any]]) -> bool:
  116. """True when an existing lease belongs to the very caller re-acquiring it.
  117. Identity is (pid, live_session_id): pid alone lets two live sessions in one process
  118. steal each other's lease; the live id alone lets another process with an equal id."""
  119. try:
  120. if int(entry.get("pid") or -1) != os.getpid():
  121. return False
  122. except (TypeError, ValueError):
  123. return False
  124. existing_live = str((entry.get("metadata") or {}).get("live_session_id") or "")
  125. incoming_live = str((metadata or {}).get("live_session_id") or "")
  126. return bool(existing_live and incoming_live) and existing_live == incoming_live
  127. def session_owner_details(session_id: str, entry: dict[str, Any]) -> str:
  128. """The ``Details:`` line shared by every owner-refusal message."""
  129. surface = str(entry.get("surface") or "another surface")
  130. started = _optional_float(entry.get("started_at"))
  131. age = f" {format_age(time.time() - started)} ago" if started else ""
  132. return f"Details: session {session_id} opened by {surface}{age}."
  133. def session_already_owned_message(session_id: str, entry: dict[str, Any]) -> str:
  134. """Refusal text for a session another live process holds.
  135. Contract shared with the TUI/Desktop surfaces: the FIRST line is the plain user sentence
  136. (no lease/pid/owner jargon); the second line is ``Details: ...`` for logs and bug reports.
  137. """
  138. return (
  139. "This chat is open in another Hermes window/terminal. Use it there, or start a new chat here.\n"
  140. + session_owner_details(session_id, entry)
  141. )
  142. def _registry_home(registry_home: str | Path | None = None) -> Path:
  143. return Path(registry_home) if registry_home is not None else Path(get_hermes_home())
  144. def _state_path(registry_home: str | Path | None = None) -> Path:
  145. return _registry_home(registry_home) / "runtime" / "active_sessions.json"
  146. def _lock_path(registry_home: str | Path | None = None) -> Path:
  147. return _registry_home(registry_home) / "runtime" / "active_sessions.lock"
  148. def _lease_paths(
  149. lease: Optional[ActiveSessionLease] = None, registry_home: str | Path | None = None
  150. ) -> tuple[Path, Path]:
  151. if lease is not None and lease.state_path is not None and lease.lock_path is not None:
  152. return lease.state_path, lease.lock_path
  153. home = _registry_home(registry_home)
  154. return home / "runtime" / "active_sessions.json", home / "runtime" / "active_sessions.lock"
  155. def _flock(fh, *, lock: bool, timeout: Optional[float] = None) -> None:
  156. """Exclusive whole-file lock/unlock on ``fh`` (fcntl on POSIX, msvcrt on Windows). With ``timeout``
  157. the lock is retried only while another holder has it, then BlockingIOError is raised, so a holder
  158. stopped mid-write (SIGSTOP, a debugger) cannot stall the caller; other errors raise at once."""
  159. if lock and timeout is not None:
  160. from pm.filesystem import lock_fd
  161. if not lock_fd(fh.fileno(), wait=True, timeout=timeout):
  162. raise BlockingIOError(errno.EWOULDBLOCK, f"file lock still held after {timeout}s")
  163. return
  164. if os.name == "nt":
  165. import msvcrt
  166. fh.seek(0)
  167. msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK if lock else msvcrt.LK_UNLCK, 1)
  168. else:
  169. import fcntl
  170. fcntl.flock(fh.fileno(), fcntl.LOCK_EX if lock else fcntl.LOCK_UN)
  171. class _FileLock:
  172. def __init__(self, path: Path, *, timeout: Optional[float] = None):
  173. self.path = path
  174. self.timeout = timeout
  175. self._fh = None
  176. def __enter__(self):
  177. self.path.parent.mkdir(parents=True, exist_ok=True)
  178. self._fh = open(self.path, "a+b")
  179. try:
  180. _flock(self._fh, lock=True, timeout=self.timeout)
  181. except Exception as exc:
  182. self._fh.close()
  183. self._fh = None
  184. raise RuntimeError(f"file lock unavailable: {self.path}") from exc
  185. return self
  186. def __exit__(self, exc_type, exc, tb):
  187. fh, self._fh = self._fh, None
  188. if fh is not None:
  189. with suppress(Exception):
  190. _flock(fh, lock=False)
  191. fh.close()
  192. def _read_entries(path: Path, *, strict: bool = False) -> list[dict[str, Any]]:
  193. def invalid(what: str) -> ActiveSessionRegistryError:
  194. return ActiveSessionRegistryError(f"active session registry {what}: {path}")
  195. try:
  196. with open(path, "r", encoding="utf-8-sig") as fh:
  197. data = json.load(fh)
  198. except FileNotFoundError:
  199. return []
  200. except Exception as exc:
  201. if strict:
  202. raise invalid("unreadable") from exc
  203. logger.warning("Ignoring corrupt active session registry at %s", path)
  204. return []
  205. entries = data.get("entries") if isinstance(data, dict) else data
  206. if not isinstance(entries, list):
  207. if strict:
  208. raise invalid("has invalid shape")
  209. return []
  210. valid = [entry for entry in entries if isinstance(entry, dict)]
  211. if not strict:
  212. return _without_queued_releases(path, valid)
  213. if len(valid) != len(entries):
  214. raise invalid("contains invalid entries")
  215. seen_leases: set[str] = set()
  216. for entry in valid:
  217. lease_id = entry.get("lease_id")
  218. # (problem-if-True predicate, message fragment) — checked lazily, in
  219. # this order, so an unhashable lease id is reported before the dup check.
  220. for bad, what in (
  221. (lambda: not _nonblank_str(lease_id), "an invalid lease id"),
  222. (lambda: lease_id in seen_leases, "a duplicate lease id"),
  223. (lambda: not _nonblank_str(entry.get("session_id")), "an invalid session id"),
  224. (lambda: _registry_pid(entry.get("pid")) <= 0, "an invalid pid"),
  225. (lambda: not _optional_isinstance(entry.get("surface"), str), "an invalid surface"),
  226. (lambda: not _optional_isinstance(entry.get("track_liveness"), bool), "an invalid liveness marker"),
  227. (lambda: not _optional_isinstance(entry.get("metadata"), dict), "invalid metadata"),
  228. (lambda: not _valid_process_start(entry.get("process_start_time")), "an invalid process start time"),
  229. ):
  230. if bad():
  231. raise invalid(f"contains {what}")
  232. seen_leases.add(lease_id)
  233. return _without_queued_releases(path, valid)
  234. # Leases this process has queued for release (gateway/active_lease_io.py) whose durable drop has not
  235. # landed yet, keyed by registry file. Every in-process read hides them, so a claim, count or
  236. # ownership check made right after a turn end sees the registry as if the release had already run
  237. # (a slot freed, the session unowned); the next locked write persists the drop. Other processes
  238. # still see the entry until the release worker rewrites the file.
  239. _queued_releases: dict[str, set[str]] = {}
  240. _queued_releases_lock = threading.Lock()
  241. def _registry_key(path: Path) -> str:
  242. return os.path.normcase(os.path.realpath(path))
  243. def mark_release_queued(lease: ActiveSessionLease) -> None:
  244. """Retire ``lease`` from this process's view of the registry now; its file write comes later."""
  245. if not isinstance(lease, ActiveSessionLease) or not lease.enabled or lease.released:
  246. return
  247. state_path, _ = _lease_paths(lease)
  248. with _queued_releases_lock:
  249. _queued_releases.setdefault(_registry_key(state_path), set()).add(lease.lease_id)
  250. def _unmark_release_queued(state_path: Path, lease_id: str) -> None:
  251. """Make ``lease_id`` visible again to the caller holding the registry lock, which drops it."""
  252. key = _registry_key(state_path)
  253. with _queued_releases_lock:
  254. queued = _queued_releases.get(key)
  255. if queued is not None:
  256. queued.discard(lease_id)
  257. if not queued:
  258. del _queued_releases[key]
  259. def _without_queued_releases(path: Path, entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
  260. if not _queued_releases:
  261. return entries
  262. with _queued_releases_lock:
  263. queued = set(_queued_releases.get(_registry_key(path), ()))
  264. if not queued:
  265. return entries
  266. return [e for e in entries if str(e.get("lease_id") or "") not in queued]
  267. def _nonblank_str(v: Any) -> bool:
  268. return isinstance(v, str) and bool(v.strip())
  269. def _optional_isinstance(v: Any, typ) -> bool:
  270. return v is None or isinstance(v, typ)
  271. def _registry_pid(pid: Any) -> int:
  272. """Registry pid as int; 0 for bools, non-int/str, or unparseable values."""
  273. if isinstance(pid, bool) or not isinstance(pid, (int, str)):
  274. return 0
  275. try:
  276. return int(pid)
  277. except (TypeError, ValueError):
  278. return 0
  279. def _valid_process_start(v: Any) -> bool:
  280. if v in (None, ""):
  281. return True
  282. parsed = _optional_float(v)
  283. return parsed is not None and math.isfinite(parsed)
  284. def _write_entries(path: Path, entries: list[dict[str, Any]]) -> None:
  285. atomic_json_write(path, {"entries": entries}, indent=None, sort_keys=True)
  286. def _process_start_time(pid: int) -> Optional[float]:
  287. # Pair pid with create_time when psutil can read it, so a recycled pid does not
  288. # keep a stale lease alive indefinitely.
  289. try:
  290. import psutil # type: ignore
  291. return float(psutil.Process(pid).create_time())
  292. except Exception:
  293. return None
  294. _OWN_START: tuple[int, float] | None = None # (pid, create_time); published atomically, re-read after fork
  295. def _own_start_time() -> Optional[float]:
  296. """This process's create_time, read from psutil once instead of per lease probe."""
  297. global _OWN_START
  298. pid = os.getpid()
  299. if _OWN_START is None or _OWN_START[0] != pid:
  300. start = _process_start_time(pid)
  301. if start is None:
  302. return None
  303. _OWN_START = (pid, start)
  304. return _OWN_START[1]
  305. def _optional_float(value: Any) -> Optional[float]:
  306. if value is None or value == "":
  307. return None
  308. try:
  309. return float(value)
  310. except (TypeError, ValueError):
  311. return None
  312. def _pid_liveness(pid: Any, process_start_time: Any = None, *, lenient: bool = False) -> Optional[bool]:
  313. """True/False for live/dead, or None when unknowable. ``lenient`` never returns None:
  314. an unparseable pid or failed existence probe counts as dead, an unreadable start as alive."""
  315. unknown_dead = False if lenient else None
  316. try:
  317. pid_int = int(pid)
  318. except (TypeError, ValueError):
  319. pid_int = 0
  320. if pid_int <= 0:
  321. return unknown_dead
  322. is_self = pid_int == os.getpid() # trivially exists; the (pid, start) identity check still applies
  323. if not is_self:
  324. try:
  325. from gateway.status import _pid_exists
  326. if not _pid_exists(pid_int):
  327. return False
  328. except Exception:
  329. return unknown_dead
  330. expected_start = _optional_float(process_start_time)
  331. if expected_start is None:
  332. return True
  333. current_start = _own_start_time() if is_self else _process_start_time(pid_int)
  334. if current_start is None:
  335. return True if lenient else None
  336. return abs(current_start - expected_start) < 0.001
  337. def _prune_dead(
  338. entries: list[dict[str, Any]], *, strict: bool = False,
  339. target_session_id: str | None = None, target_pid: int | None = None,
  340. ) -> list[dict[str, Any]]:
  341. """Keep entries whose owner is alive; tracked/strict entries must be provably so.
  342. With ``target_session_id`` only THAT session's owner has to be provable: an
  343. unrelated sibling whose liveness is unknowable (pid present, start time
  344. unreadable — an LXC ``/proc`` after a backend restart) stays in the live set,
  345. so it still fences its own session and still counts toward capacity, but no
  346. longer refuses every claim/release for a different session id. See #113683.
  347. ``target_pid`` scopes the same way by owner pid (the orphan sweep only ever
  348. reclaims this process's own leases).
  349. """
  350. targeted = target_session_id is not None or target_pid is not None
  351. live: list[dict[str, Any]] = []
  352. for entry in entries:
  353. tracked = strict or bool(entry.get("track_liveness"))
  354. state = _pid_liveness(
  355. entry.get("pid"), entry.get("process_start_time"), lenient=not tracked
  356. )
  357. if state is None:
  358. if (
  359. not targeted
  360. or (target_session_id is not None
  361. and str(entry.get("session_id") or "") == str(target_session_id))
  362. or (target_pid is not None and entry.get("pid") == target_pid)
  363. ):
  364. raise ActiveSessionRegistryError("active session owner liveness is unknown")
  365. state = True
  366. if state:
  367. live.append(entry)
  368. return live
  369. @dataclass
  370. class ActiveSessionLease:
  371. lease_id: str
  372. session_id: str
  373. surface: str
  374. enabled: bool = True
  375. released: bool = False
  376. # Pinned at acquisition: a lease taken under the root HERMES_HOME must release
  377. # against the same registry even inside a profile-home override, or phantom
  378. # leases fill the session cap.
  379. # See #85431.
  380. state_path: Optional[Path] = None
  381. lock_path: Optional[Path] = None
  382. track_liveness: bool = False
  383. def release(self) -> None:
  384. if self.released or not self.enabled:
  385. return
  386. release_active_session(self)
  387. def _clean_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
  388. return {str(k): v for k, v in metadata.items() if isinstance(k, str)}
  389. def _drop_lease(
  390. state_path: Path, entries: list[dict[str, Any]], lease_id: str
  391. ) -> list[dict[str, Any]]:
  392. """Remove ``lease_id`` from ``entries``, writing the registry only if it was present."""
  393. kept = [e for e in entries if str(e.get("lease_id") or "") != lease_id]
  394. if len(kept) != len(entries):
  395. _write_entries(state_path, kept)
  396. return kept
  397. def _holds_session(entries: list[dict[str, Any]], session_id: str) -> bool:
  398. target = str(session_id or "")
  399. return bool(target) and any(str(e.get("session_id") or "") == target for e in entries)
  400. def _read_live_entries(
  401. state_path: Path, *, track_liveness: bool, warn: str,
  402. target_session_id: str | None = None, target_pid: int | None = None,
  403. ) -> Optional[tuple[list[dict[str, Any]], list[dict[str, Any]]]]:
  404. """``(raw, pruned)`` from the registry, or None when it is unreadable.
  405. Liveness-tracked callers re-raise instead (they must not proceed on an unprovable
  406. registry); untracked callers get ``warn`` logged and decide how to degrade.
  407. ``target_session_id`` is the session the caller is about to claim/release (see
  408. ``_prune_dead``).
  409. """
  410. try:
  411. raw_entries = _read_entries(state_path, strict=True)
  412. return raw_entries, _prune_dead(
  413. raw_entries, strict=track_liveness,
  414. target_session_id=target_session_id, target_pid=target_pid,
  415. )
  416. except ActiveSessionRegistryError:
  417. if track_liveness:
  418. raise
  419. logger.warning(warn)
  420. return None
  421. def _lease_entry(
  422. *, lease_id: str, session_id: str, surface: str,
  423. metadata: Optional[dict[str, Any]] = None, track_liveness: bool = False,
  424. ) -> dict[str, Any]:
  425. now = time.time()
  426. entry: dict[str, Any] = {
  427. "lease_id": lease_id,
  428. "session_id": str(session_id),
  429. "surface": str(surface),
  430. "pid": os.getpid(),
  431. "process_start_time": _own_start_time(),
  432. "started_at": now,
  433. "updated_at": now,
  434. }
  435. if track_liveness:
  436. entry["track_liveness"] = True
  437. if metadata:
  438. entry["metadata"] = _clean_metadata(metadata)
  439. return entry
  440. def try_acquire_active_session(
  441. *, session_id: str, surface: str, config: Any, metadata: Optional[dict[str, Any]] = None,
  442. registry_home: str | Path | None = None, track_liveness: bool = False,
  443. ) -> tuple[Optional[ActiveSessionLease], Optional[str]]:
  444. """Acquire an active-session slot: ``(lease, None)`` or ``(None, ActiveSessionRefusal)``.
  445. Per-session exclusivity is CORRECTNESS, enforced unconditionally (at most one live
  446. owner per stored session); ``max_concurrent_sessions`` is resource POLICY, applied
  447. only when configured. ``registry_home`` lets profile-scoped backends share the owning
  448. profile's registry. Ownership uncertainty fails CLOSED (SESSION_COORDINATION_UNAVAILABLE).
  449. Liveness tracking keeps richer desktop lifecycle semantics; ``registry_home`` lets profile-scoped
  450. backends share the owning profile's registry even when launched from another home. See #94595.
  451. """
  452. max_sessions = resolve_max_concurrent_sessions(config)
  453. lease_id = uuid.uuid4().hex
  454. key = str(session_id or "")
  455. # No stored id yet => nothing to fence or record (and the strict schema
  456. # refuses empty session ids): hand back a no-op lease.
  457. if not key and not track_liveness:
  458. return ActiveSessionLease(
  459. lease_id=lease_id, session_id=key, surface=str(surface), enabled=False
  460. ), None
  461. entry = _lease_entry(
  462. lease_id=lease_id, session_id=key, surface=str(surface), metadata=metadata,
  463. track_liveness=track_liveness,
  464. )
  465. state_path, lock_path = _lease_paths(registry_home=registry_home)
  466. lease = ActiveSessionLease(
  467. lease_id=lease_id, session_id=key, surface=str(surface), state_path=state_path,
  468. lock_path=lock_path, track_liveness=track_liveness,
  469. )
  470. with _FileLock(lock_path):
  471. # A capacity cap could degrade open; exclusivity cannot: "could not
  472. # prove ownership" must never become "no owner exists".
  473. loaded = _read_live_entries(
  474. state_path, track_liveness=track_liveness,
  475. warn="Active-session registry is unavailable; refusing the session "
  476. "rather than risking a concurrent writer",
  477. target_session_id=key,
  478. )
  479. if loaded is None:
  480. return None, ActiveSessionRefusal(
  481. "Hermes could not read the active-session registry at "
  482. f"{state_path}, so it cannot prove this session has no other "
  483. "live owner. Fix or remove that file and try again.",
  484. SESSION_COORDINATION_UNAVAILABLE,
  485. )
  486. raw_entries, entries = loaded
  487. pruned = len(raw_entries) - len(entries)
  488. if pruned:
  489. logger.info("Pruned %d stale active session lease(s)", pruned)
  490. def refuse(message: str, reason: str, log: str, *args) -> tuple[None, ActiveSessionRefusal]:
  491. _write_entries(state_path, entries) # persist the prune even when refusing
  492. logger.info(log, *args)
  493. return None, ActiveSessionRefusal(message, reason)
  494. # Correctness first, under the same lock that just pruned dead owners.
  495. # An empty key is exempt: treating "" as an identity would make every
  496. # unsaved draft exclude every other one.
  497. if key:
  498. for index, existing in enumerate(entries):
  499. if str(existing.get("session_id") or "") != key:
  500. continue
  501. # The same writer is not a second writer: a live session that
  502. # leaked its lease reference would otherwise be fenced out of
  503. # its own session permanently (pruning only removes entries
  504. # whose PROCESS is dead). Re-entrancy, not concurrency.
  505. if _is_same_writer(existing, metadata):
  506. entries[index] = entry
  507. _write_entries(state_path, entries)
  508. return lease, None
  509. return refuse(
  510. session_already_owned_message(key, existing), SESSION_NOT_OWNED,
  511. "Refused active session %s: already held by pid=%s surface=%s",
  512. key, existing.get("pid"), existing.get("surface"),
  513. )
  514. # Capacity second, and only when an operator asked for one.
  515. if max_sessions is not None and len(entries) >= max_sessions:
  516. return refuse(
  517. active_session_limit_message(len(entries), max_sessions, entries),
  518. MAX_CONCURRENT_SESSIONS,
  519. "Active session limit reached: active=%d max=%d surface=%s",
  520. len(entries), max_sessions, surface,
  521. )
  522. entries.append(entry)
  523. _write_entries(state_path, entries)
  524. return lease, None
  525. def release_active_session(lease: ActiveSessionLease) -> None:
  526. # Prefer the registry the lease was acquired against: the caller may be
  527. # running under a profile HERMES_HOME override.
  528. # See #85431.
  529. state_path, lock_path = _lease_paths(lease)
  530. with _FileLock(lock_path):
  531. _unmark_release_queued(state_path, lease.lease_id)
  532. if lease.released:
  533. return
  534. loaded = _read_live_entries(
  535. state_path, track_liveness=lease.track_liveness,
  536. warn="Active-session registry is unavailable; preserving it while "
  537. "releasing an untracked lease",
  538. target_session_id=lease.session_id,
  539. )
  540. if loaded is not None:
  541. _drop_lease(state_path, loaded[1], lease.lease_id)
  542. lease.released = True
  543. def transfer_active_session(
  544. lease: ActiveSessionLease, *, session_id: str, metadata: Optional[dict[str, Any]] = None
  545. ) -> bool:
  546. """Move an existing lease to a new session id without dropping the slot."""
  547. new_session_id = str(session_id or "")
  548. if not new_session_id or lease.released:
  549. return False
  550. if not lease.enabled:
  551. lease.session_id = new_session_id
  552. return True
  553. state_path, lock_path = _lease_paths(lease)
  554. with _FileLock(lock_path):
  555. # release() may have won after the optimistic precheck but before this
  556. # thread acquired the file lock. Never resurrect a durably removed lease.
  557. if lease.released:
  558. return False
  559. loaded = _read_live_entries(
  560. state_path, track_liveness=lease.track_liveness,
  561. warn="Active-session registry is unavailable; refusing to overwrite "
  562. "it during lease transfer",
  563. target_session_id=lease.session_id,
  564. )
  565. if loaded is None:
  566. return False
  567. entries = loaded[1]
  568. own = next((e for e in entries if str(e.get("lease_id") or "") == lease.lease_id), None)
  569. if own is not None:
  570. own["session_id"] = new_session_id
  571. own["updated_at"] = time.time()
  572. if metadata:
  573. own["metadata"] = _clean_metadata(metadata)
  574. elif lease.track_liveness:
  575. entries.append(_lease_entry(
  576. lease_id=lease.lease_id, session_id=new_session_id, surface=lease.surface,
  577. metadata=metadata, track_liveness=True,
  578. ))
  579. else:
  580. return False
  581. _write_entries(state_path, entries)
  582. lease.session_id = new_session_id
  583. return True
  584. # A lease this process wrote in the last few seconds may not be in the caller's
  585. # ``own_live_lease_ids`` yet: ``try_acquire_active_session`` writes the registry entry under
  586. # the file lock and the server attaches the lease to its session record only after that
  587. # returns. A concurrent finalize that snapshotted its live ids in between would otherwise
  588. # read the brand-new lease as an orphan and drop it. Real orphans are minutes old.
  589. # See #101415.
  590. _SELF_ORPHAN_GRACE_SECONDS = 30.0
  591. def _drop_self_orphans(
  592. entries: list[dict[str, Any]], own_live_lease_ids: set[str] | None
  593. ) -> list[dict[str, Any]]:
  594. """Drop this process's leases only when its caller can vouch for owners."""
  595. if own_live_lease_ids is None:
  596. return entries
  597. pid = os.getpid()
  598. cutoff = time.time() - _SELF_ORPHAN_GRACE_SECONDS
  599. return [
  600. entry for entry in entries
  601. if entry.get("pid") != pid
  602. or str(entry.get("lease_id") or "") in own_live_lease_ids
  603. or (_optional_float(entry.get("started_at")) or 0.0) > cutoff
  604. ]
  605. def _release_orphaned_leases_in_home(registry_home: Path, live_lease_ids: set[str]) -> int:
  606. state_path = _state_path(registry_home)
  607. # No registry file yet means no leases have ever been written under this
  608. # home — don't take a lock (or create its file) on the idle-reaper tick.
  609. if not state_path.exists():
  610. return 0
  611. with _FileLock(_lock_path(registry_home)):
  612. loaded = _read_live_entries(
  613. state_path, track_liveness=False,
  614. warn="Active-session registry is unavailable; skipping orphaned-lease sweep",
  615. target_pid=os.getpid(),
  616. )
  617. if loaded is None:
  618. return 0
  619. entries = loaded[1]
  620. kept = _drop_self_orphans(entries, live_lease_ids)
  621. dropped = len(entries) - len(kept)
  622. if dropped:
  623. _write_entries(state_path, kept)
  624. return dropped
  625. def release_orphaned_leases(live_lease_ids: set[str]) -> int:
  626. """Drop this process's registry entries that no live session owns.
  627. ``_prune_dead`` only reclaims leases of dead processes, so on a days-long server a
  628. lease whose session skipped teardown is held until restart. The owning process is the
  629. only authority on its own leases — exact, no heartbeat on the turn path, no threshold.
  630. Sweeps the root home and every profile home (a multiplexed server leases across them).
  631. """
  632. root = get_default_hermes_root()
  633. homes = [root]
  634. try:
  635. homes.extend(p for p in (root / "profiles").iterdir() if named_profile_is_live(p))
  636. except OSError:
  637. pass
  638. dropped = 0
  639. for home in homes:
  640. try:
  641. dropped += _release_orphaned_leases_in_home(home, live_lease_ids)
  642. except OSError as exc:
  643. logger.debug("orphaned-lease sweep failed for %s: %s", home, exc)
  644. return dropped
  645. def active_session_registry_snapshot(
  646. registry_home: str | Path | None = None, *, strict: bool = False,
  647. ) -> list[dict[str, Any]]:
  648. """Return live leases; attachment callers require provable liveness.
  649. The per-entry liveness probes in ``_prune_dead`` run AFTER the file lock
  650. is released: holding an exclusive, unfair lock across process-introspection
  651. syscalls starves concurrent pollers once a handful of leases exist
  652. (#115578). The prune write-back re-locks and drops only the lease ids
  653. already proven dead, so a lease created between the snapshot and the
  654. write-back is never lost.
  655. """
  656. state_path, lock_path = _lease_paths(registry_home=registry_home)
  657. with _FileLock(lock_path):
  658. raw_entries = _read_entries(state_path, strict=True)
  659. entries = _prune_dead(raw_entries, strict=strict)
  660. if entries != raw_entries:
  661. live_lease_ids = {str(entry.get("lease_id") or "") for entry in entries}
  662. dead_lease_ids = {
  663. str(entry.get("lease_id") or "")
  664. for entry in raw_entries
  665. if str(entry.get("lease_id") or "") not in live_lease_ids
  666. }
  667. with _FileLock(lock_path):
  668. current_entries = _read_entries(state_path, strict=True)
  669. kept_entries = [
  670. entry for entry in current_entries
  671. if str(entry.get("lease_id") or "") not in dead_lease_ids
  672. ]
  673. if len(kept_entries) != len(current_entries):
  674. _write_entries(state_path, kept_entries)
  675. return entries
  676. @contextmanager
  677. def active_session_liveness_guard(
  678. session_id: str, *, registry_home: str | Path | None = None,
  679. own_live_lease_ids: set[str] | None = None,
  680. ) -> Iterator[bool]:
  681. """Hold the registry lock while reporting whether ``session_id`` is leased, so no
  682. new backend can acquire a lease between the check and the caller's ``end_session``."""
  683. state_path, lock_path = _lease_paths(registry_home=registry_home)
  684. with _FileLock(lock_path):
  685. entries = _prune_dead(
  686. _read_entries(state_path, strict=True), strict=True, target_session_id=session_id,
  687. )
  688. entries = _drop_self_orphans(entries, own_live_lease_ids)
  689. _write_entries(state_path, entries)
  690. yield _holds_session(entries, session_id)
  691. @contextmanager
  692. def release_active_session_liveness_guard(
  693. lease: ActiveSessionLease, session_id: str, *, own_live_lease_ids: set[str] | None = None,
  694. ) -> Iterator[bool]:
  695. """Remove ``lease`` and hold its registry lock through a lifecycle write, making
  696. cleanup one atomic decision (release, check siblings, end the durable row)."""
  697. if not lease.enabled or lease.released:
  698. home = lease.state_path.parent.parent if lease.state_path is not None else None
  699. with active_session_liveness_guard(
  700. session_id, registry_home=home, own_live_lease_ids=own_live_lease_ids,
  701. ) as active:
  702. yield active
  703. return
  704. state_path, lock_path = _lease_paths(lease)
  705. with _FileLock(lock_path):
  706. _unmark_release_queued(state_path, lease.lease_id)
  707. entries = _prune_dead(
  708. _read_entries(state_path, strict=True), strict=True, target_session_id=session_id,
  709. )
  710. kept = [e for e in entries if str(e.get("lease_id") or "") != lease.lease_id]
  711. kept = _drop_self_orphans(kept, own_live_lease_ids)
  712. if len(kept) != len(entries):
  713. _write_entries(state_path, kept)
  714. lease.released = True
  715. yield _holds_session(kept, session_id)