review_idle_queue.py 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218
  1. """Idle deferral for background reviews on the managed local runtime.
  2. On the managed llama-server the post-turn review fork monopolizes the GPU the next prompt
  3. needs and the next live turn cancels it (decode cost paid, learning lost). Reviews bound for
  4. the managed endpoint are therefore queued and dispatched when the machine is quiet
  5. (``auxiliary.background_review.defer``: ``auto`` = exactly that case, ``never`` = old behavior;
  6. explicit /refine never defers). One slot per session, newest snapshot wins (a review replays
  7. the whole conversation, so coalescing is dedup, not loss); aged-out items (defer_max_age_s,
  8. default 30 min) dispatch regardless of idleness; in-memory best-effort like the immediate
  9. fork. Idle truth is the supervisor's /slots held for a settle window.
  10. """
  11. from __future__ import annotations
  12. import contextvars
  13. import json
  14. import logging
  15. import threading
  16. import time
  17. import urllib.request
  18. from dataclasses import dataclass
  19. from typing import Any, Callable, Dict, Optional
  20. logger = logging.getLogger(__name__)
  21. _IDLE_SETTLE_S = 15.0 # quiet window: two back-to-back prompts must not look idle, a coffee break must
  22. _POLL_INTERVAL_S = 5.0 # poll cadence while non-empty; the thread parks when empty
  23. _MAX_AGE_DEFAULT_S = 30.0 * 60.0 # dispatch regardless of idleness past this age
  24. def defer_mode(task_cfg: Optional[dict[str, Any]]) -> str:
  25. """'auto' (default) or 'never' from auxiliary.background_review.defer."""
  26. raw = str((task_cfg or {}).get("defer", "auto")).strip().lower()
  27. return raw if raw in ("auto", "never") else "auto"
  28. def defer_max_age_s(task_cfg: Optional[dict[str, Any]]) -> float:
  29. try:
  30. value = float((task_cfg or {}).get("defer_max_age_s", _MAX_AGE_DEFAULT_S))
  31. except (TypeError, ValueError):
  32. return _MAX_AGE_DEFAULT_S
  33. return value if value > 0 else _MAX_AGE_DEFAULT_S
  34. def review_targets_managed_local(agent: Any, task_cfg: Optional[dict[str, Any]]) -> bool:
  35. """Would this review fork decode on the llama-server WE manage? Exact netloc match against the
  36. supervisor state file; any failure reads False (immediate spawn is the safe default). The cheap
  37. TTL-cached netloc probe runs FIRST so cloud-only installs skip runtime resolution on the turn's tail."""
  38. try:
  39. from agent.auxiliary_managed_local import _is_managed_local_endpoint, _managed_local_netloc
  40. if not _managed_local_netloc():
  41. return False
  42. from agent.background_review import _resolve_review_runtime
  43. runtime = _resolve_review_runtime(agent, task_cfg)
  44. return _is_managed_local_endpoint(runtime.get("base_url"))
  45. except Exception:
  46. return False
  47. @dataclass(slots=True)
  48. class _PendingReview:
  49. agent: Any
  50. session_key: str
  51. kwargs: dict[str, Any]
  52. enqueued_at: float
  53. context: contextvars.Context
  54. class ReviewIdleQueue:
  55. """Session-coalescing queue + idle-gated dispatcher thread."""
  56. def __init__(self) -> None:
  57. self._lock = threading.Lock()
  58. self._pending: dict[str, _PendingReview] = {}
  59. self._wake = threading.Event()
  60. self._thread: Optional[threading.Thread] = None
  61. self._live_turns = 0
  62. self._quiet_since: Optional[float] = None
  63. # Test seams — replaced by unit tests, never in production.
  64. self._now: Callable[[], float] = time.monotonic
  65. self._server_idle: Callable[[], bool] = _managed_server_idle
  66. def note_turn_started(self) -> None:
  67. with self._lock:
  68. self._live_turns += 1
  69. self._quiet_since = None
  70. def note_turn_finished(self) -> None:
  71. with self._lock:
  72. self._live_turns = max(0, self._live_turns - 1)
  73. if self._live_turns == 0:
  74. self._quiet_since = self._now()
  75. self._wake.set()
  76. def enqueue(self, agent: Any, session_key: str, kwargs: dict[str, Any]) -> None:
  77. """Add (or replace — newest snapshot wins) a session's pending review, keeping the ORIGINAL
  78. enqueue time on coalesce so a busy session cannot push its age-out forever."""
  79. with self._lock:
  80. existing = self._pending.get(session_key)
  81. enqueued_at = existing.enqueued_at if existing is not None else self._now()
  82. self._pending[session_key] = _PendingReview(
  83. agent, session_key, kwargs, enqueued_at, contextvars.copy_context())
  84. self._ensure_thread()
  85. self._wake.set()
  86. logger.info("Background review deferred (session=%s, queued=%d)", session_key[-12:], len(self._pending))
  87. def pending_count(self) -> int:
  88. with self._lock:
  89. return len(self._pending)
  90. def _ensure_thread(self) -> None:
  91. with self._lock:
  92. if self._thread is None or not self._thread.is_alive():
  93. self._thread = threading.Thread(target=self._run, daemon=True, name="bg-review-idle-queue")
  94. self._thread.start()
  95. def quiet_period(self) -> tuple[float, Optional[float]]:
  96. """Process-wide quiet clock: (seconds since the last live turn ended, the stamp that quiet period began
  97. at). ``(0.0, None)`` while any turn runs or before the first turn ends."""
  98. with self._lock:
  99. if self._live_turns > 0 or self._quiet_since is None:
  100. return 0.0, None
  101. return self._now() - self._quiet_since, self._quiet_since
  102. def _quiet_for(self) -> float:
  103. """Seconds this process has been turn-free (0 while a turn runs)."""
  104. return self.quiet_period()[0]
  105. def _pop_dispatchable(self) -> Optional[_PendingReview]:
  106. """Oldest aged-out item, else the oldest item once quiet+idle hold."""
  107. with self._lock:
  108. if not self._pending:
  109. return None
  110. now = self._now()
  111. aged = [p for p in self._pending.values()
  112. if now - p.enqueued_at >= defer_max_age_s(p.kwargs.get("task_cfg"))]
  113. candidate = min(aged, key=lambda p: p.enqueued_at) if aged else None
  114. if candidate is None and (self._quiet_for() < _IDLE_SETTLE_S or not self._server_idle()):
  115. return None
  116. with self._lock:
  117. if candidate is None:
  118. if not self._pending:
  119. return None
  120. candidate = min(self._pending.values(), key=lambda p: p.enqueued_at)
  121. return self._pending.pop(candidate.session_key, None)
  122. def _run(self) -> None:
  123. while True:
  124. self._wake.wait()
  125. with self._lock:
  126. if not self._pending:
  127. self._wake.clear()
  128. continue
  129. item = None
  130. try:
  131. item = self._pop_dispatchable()
  132. if item is not None:
  133. # The shared dispatcher has no caller profile. Enter the item's context
  134. # before reading config AND spawning the context-propagating review worker.
  135. item.context.run(self._dispatch, item)
  136. except Exception:
  137. logger.warning("Deferred review dispatch failed", exc_info=True)
  138. if item is None:
  139. time.sleep(_POLL_INTERVAL_S)
  140. def _dispatch(self, item: _PendingReview) -> None:
  141. if not self._still_enabled(item):
  142. logger.info(
  143. "Deferred background review dropped: reviews were disabled while it was queued (session=%s)",
  144. item.session_key[-12:])
  145. return
  146. logger.info(
  147. "Dispatching deferred background review (session=%s, waited=%.0fs, queued=%d)",
  148. item.session_key[-12:], self._now() - item.enqueued_at, self.pending_count())
  149. item.agent._spawn_background_review_now(**item.kwargs)
  150. @staticmethod
  151. def _still_enabled(item: _PendingReview) -> bool:
  152. """Re-check the enabled gate at DISPATCH time (disabling reviews while queued must stick). Fail-open."""
  153. try:
  154. from agent.background_review import load_background_review_settings
  155. return load_background_review_settings()[0]
  156. except Exception:
  157. return True
  158. def _managed_server_idle() -> bool:
  159. """No processing slot on any loaded model of the managed router; unreachable/no state file reads idle."""
  160. try:
  161. from hermes_cli.local_runtime.supervisor import state_path
  162. from urllib.parse import quote
  163. state = json.loads(state_path().read_text(encoding="utf-8-sig"))
  164. base = str(state.get("base_url", "")).rsplit("/v1", 1)[0]
  165. headers = {"Authorization": f"Bearer {state.get('api_key', '')}"}
  166. if not base:
  167. return True
  168. def _get(path: str) -> Any:
  169. with urllib.request.urlopen(urllib.request.Request(f"{base}{path}", headers=headers), timeout=3) as r:
  170. return json.loads(r.read())
  171. loaded = [m["id"] for m in _get("/models").get("data", [])
  172. if (m.get("status") or {}).get("value") in ("loaded", "ready")]
  173. return not any(
  174. # autoload=false: a model unloaded since the /models read must not load again for a status read.
  175. s.get("is_processing") for mid in loaded
  176. for s in _get(f"/slots?model={quote(mid)}&autoload=false") if isinstance(s, dict)
  177. )
  178. except Exception:
  179. return True
  180. # Module singleton — one queue per process, like the load-progress watcher.
  181. QUEUE = ReviewIdleQueue()