deadline.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506
  1. """Unified deadline layer — one bounded-execution primitive, one timeout resolver (#85125).
  2. * :func:`resolve_timeout` — ``timeouts:`` in config.yaml > legacy env var > default.
  3. * :func:`clamp_timeout` — huge timeouts overflow ``time_t`` in ``Lock.acquire`` /
  4. ``Thread.join`` on macOS (#83220), so every timeout is capped.
  5. * :func:`run_bounded_async` / :func:`run_bounded_sync` — wall-clock deadlines driven by
  6. a daemon ``threading.Timer`` / worker thread, so a blocked event loop cannot disable them.
  7. * :func:`kill_process_tree` — portable whole-tree termination.
  8. * :func:`poll_until` — the one "check until it holds or the deadline passes" loop; use it
  9. instead of hand-writing ``deadline = time.monotonic() + t; while ...: time.sleep(...)``.
  10. Invariants: operation exceptions propagate unchanged (only the *timeout* outcome is reified
  11. as :class:`BoundedResult`); a timeout here is OUR deadline, not the provider's (classify
  12. :class:`DeadlineExpired` distinctly from transport timeouts); ``None`` / non-positive means unbounded.
  13. """
  14. from __future__ import annotations
  15. import asyncio
  16. import contextvars
  17. from contextlib import contextmanager
  18. import faulthandler
  19. import logging
  20. import os
  21. import subprocess
  22. import sys
  23. import threading
  24. import time
  25. from dataclasses import dataclass
  26. from typing import Any, Awaitable, Callable, Optional, Protocol, TypeVar
  27. logger = logging.getLogger(__name__)
  28. __all__ = [
  29. "MAX_SAFE_TIMEOUT_S",
  30. "BoundedResult",
  31. "DeadlineExpired",
  32. "clamp_timeout",
  33. "kill_process_tree",
  34. "poll_until",
  35. "resolve_timeout",
  36. "run_bounded_async",
  37. "run_bounded_sync",
  38. ]
  39. _T = TypeVar("_T")
  40. # Upper bound for any timeout handed to platform wait primitives.
  41. #
  42. # CPython converts ``threading.Lock.acquire(timeout=...)`` /
  43. # ``Thread.join(timeout=...)`` deadlines to an absolute timestamp; very large
  44. # relative timeouts overflow ``time_t`` on macOS and raise
  45. # ``OverflowError: timestamp out of range for platform time_t`` (#83220).
  46. # On Windows the binding constraint is narrower: the underlying
  47. # ``WaitForSingleObject`` takes a DWORD *milliseconds* argument, so any
  48. # wait above ~49.7 days (0xFFFFFFFF ms) raises OverflowError rather than
  49. # waiting (verified live: 4294967.0s ok, 4294967.3s overflows). One year
  50. # is the semantic "unbounded" on platforms that can express it; Windows
  51. # takes the DWORD-ms cap, which is still semantically "unbounded" for
  52. # every wait in this codebase.
  53. _WINDOWS_MAX_WAIT_S = (0xFFFFFFFF - 1000) / 1000.0 # DWORD ms, minus a 1s rounding guard
  54. MAX_SAFE_TIMEOUT_S = (
  55. _WINDOWS_MAX_WAIT_S if sys.platform == "win32" else 31_536_000.0
  56. ) # 365 days off-Windows
  57. # Grace after a deadline fires before concluding the loop thread is blocked and dumping stacks.
  58. _LOOP_BLOCKED_DUMP_GRACE_S = 5.0
  59. # ``Event.wait`` is a C-level block: KeyboardInterrupt / SetAsyncExc only land when the
  60. # thread returns to Python, so the sync wait is sliced to observe /stop or SIGINT promptly.
  61. # Slice the wait so a /stop or SIGINT during a bounded sync call is observed within this window rather than
  62. # at the full deadline (#94285, tools/test_local_interrupt_cleanup).
  63. _BOUNDED_SYNC_WAIT_SLICE_S = 0.2
  64. class DeadlineExpired(TimeoutError):
  65. """A deadline enforced by this layer expired (Hermes's own bound, not the provider's)."""
  66. def __init__(self, label: str, timeout_s: float):
  67. super().__init__(f"deadline expired after {timeout_s:.1f}s: {label}")
  68. self.label = label
  69. self.timeout_s = timeout_s
  70. class SuspectableBackend(Protocol):
  71. """A stateful backend (MCP connection, browser session, LSP client) ``run_bounded_*`` flags via
  72. ``mark_suspect`` on timeout so the owner can health-check/recycle it before reuse. ``mark_suspect``
  73. MUST be cheap, non-blocking, and must not acquire locks the guarded operation may hold — it runs
  74. inline on the event loop / caller's thread while the wedged worker is still alive."""
  75. def mark_suspect(self, reason: str) -> None: ...
  76. def ensure_healthy(self) -> bool: ...
  77. def _mark_backend_suspect(backend: object | None, label: str, timeout_s: float) -> None:
  78. """Best-effort ``mark_suspect``; never raises, non-adopting backends tolerated."""
  79. if backend is None:
  80. return
  81. try:
  82. mark = getattr(backend, "mark_suspect", None)
  83. if callable(mark):
  84. mark(f"{label} timed out after {timeout_s:.1f}s")
  85. except Exception:
  86. logger.debug("deadline mark_suspect failed", exc_info=True)
  87. @dataclass(frozen=True, kw_only=True)
  88. class BoundedResult:
  89. """Outcome of a bounded operation; operation exceptions are never captured here."""
  90. timed_out: bool
  91. value: Any
  92. elapsed_s: float
  93. timeout_s: Optional[float]
  94. label: str
  95. def _result(start: float, timeout_s: Optional[float], label: str, *, value: Any = None, timed_out: bool = False) -> BoundedResult:
  96. return BoundedResult(
  97. timed_out=timed_out, value=value, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label
  98. )
  99. def clamp_timeout(timeout: Optional[float]) -> Optional[float]:
  100. """Normalize a timeout: None/non-positive/non-numeric/NaN -> None (unbounded), else capped."""
  101. if timeout is None:
  102. return None
  103. try:
  104. value = float(timeout)
  105. except (TypeError, ValueError):
  106. logger.warning("clamp_timeout: non-numeric timeout %r; treating as unbounded", timeout)
  107. return None
  108. if value != value: # NaN
  109. logger.warning("clamp_timeout: NaN timeout; treating as unbounded")
  110. return None
  111. return None if value <= 0 else min(value, MAX_SAFE_TIMEOUT_S)
  112. def poll_until(predicate: Callable[[], _T], timeout: float, interval: float) -> _T:
  113. """Call ``predicate`` every ``interval`` s until it returns something truthy or ``timeout`` s
  114. pass; return its last result (falsy on timeout).
  115. The first check runs at once and the last sleep is cut to the time left, so a condition that
  116. holds when the deadline arrives still counts. ``timeout <= 0`` means exactly one check.
  117. Only the sleeps are bounded: time spent inside ``predicate`` is not, so a predicate that blocks
  118. (socket probes, subprocess calls) can carry the final check past the deadline.
  119. """
  120. deadline = time.monotonic() + timeout
  121. while True:
  122. result = predicate()
  123. if result:
  124. return result
  125. remaining = deadline - time.monotonic()
  126. if remaining <= 0:
  127. return result
  128. time.sleep(min(interval, remaining))
  129. # --- Timeout resolution: config ``timeouts:`` > legacy env var > default ------
  130. def _timeouts_section() -> dict:
  131. """Read the ``timeouts:`` root section from config.yaml (read-only, fail-open)."""
  132. try:
  133. from hermes_cli.config import load_config_readonly
  134. section = load_config_readonly().get("timeouts")
  135. return section if isinstance(section, dict) else {}
  136. except Exception:
  137. logger.debug("timeouts: config read failed; using defaults", exc_info=True)
  138. return {}
  139. def _lookup_dotted(section: dict, key: str) -> Any:
  140. """Walk ``a.b.c`` through nested dicts; return None when absent."""
  141. node: Any = section
  142. for part in key.split("."):
  143. if not isinstance(node, dict) or part not in node:
  144. return None
  145. node = node[part]
  146. return node
  147. def resolve_timeout(key: str, *, default: Optional[float], env_var: Optional[str] = None) -> Optional[float]:
  148. """Resolve a timeout (seconds): dotted ``timeouts.<key>`` > ``env_var`` > ``default``; the winner
  149. goes through :func:`clamp_timeout`, invalid config/env values fall through with a warning."""
  150. raw = _lookup_dotted(_timeouts_section(), key)
  151. if raw is not None:
  152. # Explicit float() so invalid config values FALL THROUGH to env/default instead of
  153. # resolving as unbounded. bool rejected (YAML `true` would become a 1s deadline);
  154. # NaN rejected for the same fall-through reason.
  155. if not isinstance(raw, bool):
  156. try:
  157. value = float(raw)
  158. if value == value: # not NaN
  159. return clamp_timeout(value)
  160. except (TypeError, ValueError):
  161. pass
  162. logger.warning("timeouts.%s: invalid value %r in config.yaml; ignoring", key, raw)
  163. if env_var:
  164. env_raw = os.getenv(env_var, "").strip()
  165. if env_raw:
  166. try:
  167. return clamp_timeout(float(env_raw))
  168. except ValueError:
  169. logger.warning("invalid %s=%r; ignoring", env_var, env_raw)
  170. return clamp_timeout(default)
  171. # --- Bounded execution — async flavor ------------------------------------------
  172. # The deadline is a daemon threading.Timer so a blocked event loop cannot disable it; a
  173. # second timer dumps all thread stacks when the loop provably failed to process the expiry.
  174. # --------------------------------------------------------------------------- Bounded execution — async
  175. # flavor. Generalizes plugins/platforms/telegram/adapter.py:_await_with_thread_deadline (the #63309 fix):
  176. # the deadline is driven by a daemon threading.Timer so a blocked event loop cannot disable it, and a second
  177. # timer dumps all thread stacks when the loop provably failed to process the expiry — the one piece of
  178. # information loop-blocked hangs otherwise never surface.
  179. # ---------------------------------------------------------------------------
  180. def _consume_abandoned(task: asyncio.Future[Any]) -> None:
  181. """Observe an abandoned task's outcome so it never logs 'never retrieved'."""
  182. try:
  183. if not task.cancelled():
  184. task.exception()
  185. except Exception:
  186. pass
  187. def _abandon(task: asyncio.Future[Any]) -> None:
  188. """Cancel ``task`` and never await it; its outcome is consumed so it stays unobserved-safe."""
  189. task.cancel()
  190. task.add_done_callback(_consume_abandoned)
  191. async def _run_abandon_cleanup(on_abandon: Callable[[], Awaitable[Any]]) -> None:
  192. """Run abandonment cleanup fire-and-forget (its failures swallowed)."""
  193. try:
  194. await on_abandon()
  195. except Exception:
  196. logger.debug("deadline abandon-cleanup failed", exc_info=True)
  197. def _dump_blocked_loop_diagnostics(label: str, timeout_s: float) -> None:
  198. logger.warning(
  199. "[deadline] %r deadline (%.0fs) expired but the event loop has not processed the expiry "
  200. "after a further %.0fs — the loop thread appears BLOCKED in a synchronous call, which is "
  201. "why no asyncio timeout can fire. Dumping all thread stacks to stderr to identify the "
  202. "blocking frame.",
  203. label, timeout_s, _LOOP_BLOCKED_DUMP_GRACE_S,
  204. )
  205. try:
  206. faulthandler.dump_traceback(all_threads=True)
  207. except Exception:
  208. logger.debug("faulthandler traceback dump failed", exc_info=True)
  209. async def run_bounded_async(
  210. awaitable: Awaitable[Any],
  211. timeout: Optional[float],
  212. *,
  213. label: str = "operation",
  214. on_abandon: Optional[Callable[[], Awaitable[Any]]] = None,
  215. dump_on_blocked_loop: bool = True,
  216. backend: object | None = None,
  217. ) -> BoundedResult:
  218. """Await ``awaitable`` under a wall-clock deadline independent of loop timers.
  219. Operation exceptions (incl. ``CancelledError`` from a caller cancelling *us*) propagate
  220. unchanged. On timeout the task is cancelled and **abandoned** (never awaited —
  221. cancellation-shielded scopes are exactly the paths that wedge); ``on_abandon`` runs detached."""
  222. timeout_s = clamp_timeout(timeout)
  223. start = time.monotonic()
  224. if timeout_s is None:
  225. return _result(start, None, label, value=await awaitable)
  226. task = asyncio.ensure_future(awaitable)
  227. loop = asyncio.get_running_loop()
  228. deadline: asyncio.Future[None] = loop.create_future()
  229. loop_processed_expiry = threading.Event()
  230. def _mark_expired() -> None:
  231. loop_processed_expiry.set()
  232. if not deadline.done():
  233. deadline.set_result(None)
  234. def _watchdog_check() -> None:
  235. if not loop_processed_expiry.is_set():
  236. _dump_blocked_loop_diagnostics(label, timeout_s)
  237. timers = [threading.Timer(timeout_s, lambda: loop.call_soon_threadsafe(_mark_expired))]
  238. if dump_on_blocked_loop:
  239. timers.append(threading.Timer(timeout_s + _LOOP_BLOCKED_DUMP_GRACE_S, _watchdog_check))
  240. for t in timers:
  241. t.daemon = True
  242. t.start()
  243. try:
  244. try:
  245. done, _ = await asyncio.wait({task, deadline}, return_when=asyncio.FIRST_COMPLETED)
  246. except asyncio.CancelledError:
  247. _abandon(task) # the CALLER cancelled us; `task` must not run unobserved
  248. raise
  249. if task in done:
  250. if not deadline.done():
  251. deadline.cancel()
  252. return _result(start, timeout_s, label, value=await task)
  253. _abandon(task)
  254. if on_abandon is not None:
  255. asyncio.ensure_future(_run_abandon_cleanup(on_abandon)).add_done_callback(_consume_abandoned)
  256. # Deliberately INLINE on the loop: the mark must happen-before this result returns
  257. # AND before the ensure_future'd on_abandon cleanup starts (next tick).
  258. _mark_backend_suspect(backend, label, timeout_s)
  259. logger.warning("[deadline] %r timed out after %.1fs; task abandoned", label, timeout_s)
  260. return _result(start, timeout_s, label, timed_out=True)
  261. finally:
  262. for t in timers:
  263. t.cancel()
  264. # cancel() cannot stop a Timer whose callback is already running; setting the
  265. # event closes that race so a completed await is never misreported as blocked.
  266. loop_processed_expiry.set()
  267. # --- Bounded execution — sync flavor -------------------------------------------
  268. def run_bounded_sync(
  269. fn: Callable[[], Any],
  270. timeout: Optional[float],
  271. *,
  272. label: str = "operation",
  273. on_timeout: Optional[Callable[[], None]] = None,
  274. backend: object | None = None,
  275. ) -> BoundedResult:
  276. """Run ``fn`` in a daemon worker thread under a wall-clock deadline; exceptions re-raise in
  277. the caller. On expiry the worker is **abandoned** (every timeout leaks one daemon thread, so
  278. do NOT use per-item in hot loops) and ``on_timeout`` runs best-effort in the caller's thread.
  279. The worker runs under ``contextvars.copy_context()`` so secret scope / session id survive.
  280. See #94285.
  281. """
  282. timeout_s = clamp_timeout(timeout)
  283. start = time.monotonic()
  284. if timeout_s is None:
  285. return _result(start, None, label, value=fn())
  286. box: dict[str, Any] = {}
  287. done = threading.Event()
  288. ctx = contextvars.copy_context()
  289. def _worker() -> None:
  290. try:
  291. box["value"] = ctx.run(fn)
  292. except BaseException as exc: # re-raised in caller; must not vanish
  293. box["exc"] = exc
  294. finally:
  295. done.set()
  296. threading.Thread(target=_worker, name=f"deadline-{label}", daemon=True).start()
  297. deadline = start + timeout_s
  298. while not done.is_set():
  299. remaining = deadline - time.monotonic()
  300. if remaining <= 0:
  301. break
  302. done.wait(min(_BOUNDED_SYNC_WAIT_SLICE_S, remaining))
  303. if not done.is_set():
  304. logger.warning("[deadline] %r timed out after %.1fs; worker abandoned", label, timeout_s)
  305. # Mark suspect BEFORE owner cleanup so a recycle in on_timeout never
  306. # inherits a stale flag on the healed replacement.
  307. _mark_backend_suspect(backend, label, timeout_s)
  308. if on_timeout is not None:
  309. try:
  310. on_timeout()
  311. except Exception:
  312. logger.debug("deadline on_timeout callback failed", exc_info=True)
  313. return _result(start, timeout_s, label, timed_out=True)
  314. if "exc" in box:
  315. raise box["exc"]
  316. return _result(start, timeout_s, label, value=box.get("value"))
  317. # --- Whole-tree process termination --------------------------------------------
  318. @contextmanager
  319. def _process_tree_snapshot(pid: int, *, hard_kill: bool):
  320. """Stop each hard-kill target before discovering its children: a running
  321. parent can fork after psutil builds its PID map and escape the final signal.
  322. Resume anything we stopped if signalling fails. Graceful signals never stop
  323. their recipients, since their handlers must remain able to run.
  324. """
  325. descendants = []
  326. stopped = []
  327. try:
  328. try:
  329. import psutil
  330. root = psutil.Process(pid)
  331. descendants = root.children(recursive=True)
  332. if hard_kill:
  333. pending = [root]
  334. seen = {os.getpid()}
  335. known = {process.pid: process for process in descendants}
  336. stop_deadline = time.monotonic() + 1.0
  337. while pending:
  338. for process in pending:
  339. if process.pid in seen:
  340. continue
  341. seen.add(process.pid)
  342. if time.monotonic() >= stop_deadline:
  343. raise TimeoutError("process tree did not stop before snapshot deadline")
  344. try:
  345. status = process.status()
  346. if status in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD):
  347. continue
  348. if status != psutil.STATUS_STOPPED:
  349. process.suspend()
  350. stopped.append(process)
  351. while process.status() != psutil.STATUS_STOPPED:
  352. if time.monotonic() >= stop_deadline:
  353. raise TimeoutError("process did not stop before snapshot deadline")
  354. time.sleep(0.001)
  355. except psutil.NoSuchProcess:
  356. continue
  357. # Rescan after stopping the discovered generation. A child
  358. # may have forked while that generation was being stopped.
  359. for process in root.children(recursive=True):
  360. known.setdefault(process.pid, process)
  361. descendants = list(known.values())
  362. pending = [process for process in descendants if process.pid not in seen]
  363. except Exception:
  364. # Preserve the existing best-effort group fallback when discovery or
  365. # stopping is unavailable; never strand a successfully stopped target.
  366. logger.debug("kill_process_tree: snapshot incomplete for pid %s", pid, exc_info=True)
  367. yield descendants
  368. finally:
  369. for process in stopped:
  370. try:
  371. process.resume()
  372. except Exception:
  373. logger.debug("kill_process_tree: target already gone or resume refused", exc_info=True)
  374. def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool:
  375. """Terminate ``pid`` and all its descendants, portably; True when anything was signalled.
  376. Windows: ``taskkill /F /T`` (``sig`` ignored). POSIX: snapshot descendants via
  377. psutil; for SIGKILL, stop and rescan the live tree so a concurrent fork cannot
  378. escape a stale snapshot. Signal identity-checked descendants before their
  379. parent, then its group when ``pid`` leads one. Stopping is best-effort with a
  380. bounded wait; unavailable psutil still leaves process-group cleanup. Other
  381. signals do not suspend recipients. ``sig`` defaults to ``SIGKILL``."""
  382. if sys.platform == "win32":
  383. try:
  384. from hermes_cli._subprocess_compat import windows_hide_flags
  385. creationflags = windows_hide_flags()
  386. except Exception:
  387. creationflags = 0
  388. try:
  389. proc = subprocess.run(
  390. ["taskkill", "/F", "/T", "/PID", str(pid)],
  391. stdin=subprocess.DEVNULL, capture_output=True, timeout=15, check=False, creationflags=creationflags,
  392. )
  393. # taskkill exits non-zero for not-found / access-denied (False = nothing terminated).
  394. return proc.returncode == 0
  395. except Exception:
  396. logger.debug("kill_process_tree: taskkill failed for pid %s", pid, exc_info=True)
  397. return False
  398. import signal as _signal
  399. if sig is None:
  400. sig = _signal.SIGKILL
  401. with _process_tree_snapshot(int(pid), hard_kill=sig == _signal.SIGKILL) as descendants:
  402. signalled = False
  403. # Signal descendants while their ownership ancestry is still observable.
  404. # Frozen hard-kill targets cannot fork during this bottom-up teardown.
  405. for child in reversed(descendants):
  406. try:
  407. if child.is_running():
  408. child.send_signal(sig)
  409. signalled = True
  410. except Exception:
  411. continue
  412. try:
  413. # getpgid→killpg has an inherent TOCTOU shared by every killpg site; the psutil
  414. # sweep below is identity-aware (PID + create time) and does not.
  415. pgid = os.getpgid(pid)
  416. except (ProcessLookupError, PermissionError, OSError):
  417. pgid = None
  418. try:
  419. if pgid is not None and pgid == pid:
  420. # pid leads its own group (the == check avoids signalling the caller's group).
  421. os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only branch (win32 returns above)
  422. else:
  423. os.kill(pid, sig)
  424. signalled = True
  425. except ProcessLookupError:
  426. pass
  427. except (PermissionError, OSError):
  428. logger.debug("kill_process_tree: signal failed for pid %s", pid, exc_info=True)
  429. return signalled