async_delegation.py 80 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449
  1. #!/usr/bin/env python3
  2. """Async (background) delegation registry behind ``delegate_task(background=true)``.
  3. The parent dispatches a subagent on a module-level daemon executor and returns a handle
  4. immediately. On completion a ``type="async_delegation"`` event (self-contained task-source
  5. block) is pushed onto the SHARED ``process_registry.completion_queue`` the CLI/gateway drain
  6. while idle, so results surface as a NEW turn (never mid-turn) and inherit its de-dup and
  7. crash-recovery wiring. Only the async lifecycle lives here; the child run is an injected ``runner``."""
  8. from __future__ import annotations
  9. import collections
  10. import contextvars
  11. import json
  12. import logging
  13. import os
  14. import sqlite3
  15. import threading
  16. import time
  17. import uuid
  18. from concurrent.futures import ThreadPoolExecutor
  19. from typing import Any, Callable, Dict, List, Optional
  20. from hermes_constants import get_hermes_home, hermes_home_key
  21. from tools.daemon_pool import DaemonThreadPoolExecutor
  22. from tools.thread_context import propagate_context_to_thread
  23. from tools.async_delegation_worker import worker_ledger, require_ledger_owner
  24. from tools.delegation_redaction import redact_child_result, redact_completion_event
  25. logger = logging.getLogger(__name__)
  26. # ── Module-level state ──────────────────────────────────────────────────────
  27. # Persistent daemon executor (never a `with ThreadPoolExecutor()` block, which
  28. # would join on exit and defeat async); daemon workers can't hang a hard exit.
  29. _executor: Optional[ThreadPoolExecutor] = None
  30. _executor_lock = threading.Lock()
  31. _executor_max_workers: int = 0
  32. _records_lock = threading.Lock()
  33. # delegation_id -> record dict; kept for the run plus a short completed tail.
  34. _records: dict[str, dict[str, Any]] = {}
  35. # A closed session's token -> the (field, value) selectors a new dispatch is refused for (``refuse_for_session``),
  36. # and each selector -> the tokens refusing it, so a dispatch checks its own selectors, not every fence.
  37. _refused: dict[Any, frozenset] = {}
  38. _refused_by: dict[tuple, set] = {}
  39. _FENCED_FIELDS = ("session_key", "parent_session_id")
  40. _DEFAULT_MAX_ASYNC_CHILDREN = 3
  41. # Completed records retained (in memory and in the ledger) for status queries.
  42. _MAX_RETAINED_COMPLETED = 50
  43. _DURABLE_RETENTION_SECONDS = 7 * 24 * 60 * 60
  44. _MAX_DURABLE_PENDING = 1000
  45. # Cap retried deliveries so an unroutable row converges to terminal 'dropped'.
  46. _MAX_DELIVERY_ATTEMPTS = 8
  47. # Pending completions older than this are dropped on restart replay instead of
  48. # re-run as a full-context turn; 48h keeps weekend results deliverable.
  49. _MAX_COMPLETION_REPLAY_AGE_S = 48 * 3600.0
  50. # A delivery claim older than this is abandoned and may be re-claimed.
  51. _CLAIM_LEASE_S = 300.0
  52. _DB_LOCK = threading.Lock()
  53. # ── Orphaned-completion sweep ────────────────────────────────────────────────
  54. # Startup replay runs once per process, so a completion whose owner died while THIS process was
  55. # already running (a desktop reload) would wait for the next restart (#97202). Delivery loops (gateway
  56. # watcher, TUI poller) sweep each home they serve at most once per interval.
  57. ORPHAN_SWEEP_INTERVAL_S = 30.0
  58. # Idle time before a dead owner's pending row is re-offered; keeps the sweep off a row just touched.
  59. _ORPHAN_STALE_S = 60.0
  60. _orphan_lock = threading.Lock()
  61. # (home key, delegation_id) put on this process's queue by replay or sweep and not re-offered while
  62. # that copy is alive. A consumer that discards its copy with the row still pending hands it back
  63. # (``return_completion_offer``); the delivery claim stays the only thing that settles the row.
  64. _offered: set = set()
  65. # Delegation ids this process first offered from its startup replay. Their copies carry
  66. # ``startup_replay`` on the sweep's later re-offers too, so a refused one stays gated.
  67. _startup_replayed: set = set()
  68. _last_orphan_sweep: dict[str, float] = {}
  69. # (home key, delegation_id) -> when the sweep last re-offered a lost copy. A consumer that keeps
  70. # handing the copy back would otherwise be re-offered every tick: re-offering touches no ledger
  71. # column, so each re-offer gets its own ``_CLAIM_LEASE_S`` here, and only the first one warns.
  72. _reoffered_at: dict[tuple[str, str], float] = {}
  73. # delegation_id -> number of copies a consumer holds off-queue (``mark_copy_held``). The sweep counts
  74. # them as live alongside the queued ones, so it never adds a duplicate beside a held copy.
  75. _held_copies: collections.Counter = collections.Counter()
  76. _held_lock = threading.Lock() # taken after a queue's mutex, never before
  77. # ── Stale-delegation detection (progress-based, on by default) ──────────────
  78. # A runner wedged before returning never reaches its finalizer, so it would show
  79. # "dispatched" forever. No wall-clock timeout (heavy work must never be killed for
  80. # taking long): one monitor thread samples per-dispatch PROGRESS via an injected
  81. # ``progress_fn``; a frozen child is interrupted, given a grace window to unwind via
  82. # the normal finalize path, and only force-finalized (terminal ``stalled`` event) if
  83. # it never returns. Thresholds mirror delegate_tool's sync heartbeat monitor.
  84. _STALE_CHECK_INTERVAL = 30.0
  85. _STALE_IDLE_SECONDS = 450.0
  86. _STALE_IN_TOOL_SECONDS = 1200.0
  87. _STALL_GRACE_SECONDS = 120.0
  88. _monitor_lock = threading.Lock()
  89. _monitor_thread: Optional[threading.Thread] = None
  90. _monitor_stop = threading.Event()
  91. _LIVE_STATES = {"running", "stalling", "finalizing"}
  92. _ACTIVE_STATES = ("running", "stalling")
  93. # Routing origin persisted at dispatch so a restart-recovered completion can
  94. # reconstruct a full SessionSource (scope_id drives relay tenant egress).
  95. _ROUTING_KEYS = ("scope_id", "user_id", "user_name")
  96. # Structured stall metadata — additive, present only on stall finalizations.
  97. _STALL_META_KEYS = ("stalled_after_quiet_seconds", "stall_threshold_seconds", "stall_phase", "stall_grace_seconds")
  98. # Private stall bookkeeping on the record -> public field in list_async_delegations().
  99. _STALL_FIELD_MAP = (("_stall_quiet_seconds", "stalled_after_quiet_seconds"),
  100. ("_stall_threshold_seconds", "stall_threshold_seconds"), ("_stall_in_tool", "stall_in_tool"))
  101. # ── Durable ledger (state.db / async_delegations) ───────────────────────────
  102. def _db_path():
  103. return get_hermes_home() / "state.db"
  104. def _connect() -> sqlite3.Connection:
  105. from agent.runtime_session_store import WorkerPersistenceError, is_worker_process
  106. if is_worker_process():
  107. raise WorkerPersistenceError('worker_delegation_ledger_unavailable')
  108. from hermes_cli.sqlite_util import open_db
  109. # Same state.db as hermes_state.SessionDB -- reuse its owner-only (0600)
  110. # hardening so this writer doesn't create/leave the file (and its WAL
  111. # sidecars) at the process umask. See hermes_state._secure_state_db_files.
  112. from hermes_constants import mkdir_under_hermes_home
  113. from hermes_state import _secure_state_db_files
  114. path = _db_path()
  115. # A late replay or writer must not resurrect a removed named profile (#123265).
  116. mkdir_under_hermes_home(path.parent)
  117. _secure_state_db_files(path, create_main=True)
  118. # wal=False: SessionDB owns state.db's journal mode (_initialize_schema applies the barriers).
  119. conn = open_db(path, db_label="state.db (async_delegation)", busy_timeout_ms=10_000,
  120. wal=False, row_factory=None, initialize=_initialize_schema)
  121. _secure_state_db_files(path)
  122. return conn
  123. def _initialize_schema(conn: sqlite3.Connection) -> None:
  124. from hermes_state_repair import apply_durability_barriers
  125. from hermes_state_schema import reconcile_state_schema
  126. # Preserve the journal mode SessionDB configured on state.db: forcing WAL from
  127. # every short-lived connection collides with live transcript/FTS writers.
  128. apply_durability_barriers(conn)
  129. # Single durable-shape authority: the canonical SCHEMA_SQL drives both
  130. # table creation and column backfill (reconcile_state_schema replays the
  131. # canonical DDL and reuses SessionDB's declarative reconciliation). This
  132. # module previously carried its own CREATE TABLE + ALTER column list,
  133. # which drifted from SCHEMA_SQL — same-name columns with different
  134. # nullability/defaults depending on which authority touched the database
  135. # first (#94691).
  136. reconcile_state_schema(conn)
  137. def _transaction():
  138. from hermes_cli.sqlite_util import transaction
  139. return transaction(_connect())
  140. def _capture_routing_origin() -> dict[str, Any]:
  141. """Snapshot scope_id/user_id/user_name on the PARENT thread (the daemon worker
  142. has no contextvars) so a restart-replayed completion can rebuild a SessionSource.
  143. Best-effort: empty values are omitted."""
  144. try:
  145. from gateway.session_context import get_session_env
  146. return {k: v for k in _ROUTING_KEYS if (v := get_session_env(f"HERMES_SESSION_{k.upper()}", ""))}
  147. except Exception:
  148. return {}
  149. def _persist_dispatch(record: dict[str, Any]) -> None:
  150. if remote := worker_ledger():
  151. return remote.dispatch(record)
  152. now = time.time()
  153. try:
  154. from gateway.status import get_process_start_time
  155. owner_started_at = get_process_start_time(os.getpid())
  156. except Exception:
  157. owner_started_at = None
  158. task_payload = {
  159. key: record.get(key)
  160. for key in ("goal", "goals", "context", "toolsets", "role", "model", "is_batch", "task_indexes", "task_transcripts", *_ROUTING_KEYS)
  161. if key in record}
  162. try: # where the children's terminals started; lets recovery add a git-state hint
  163. task_payload["owner_cwd"] = os.getcwd()
  164. except OSError:
  165. pass
  166. with _DB_LOCK, _transaction() as conn:
  167. conn.execute("""INSERT OR REPLACE INTO async_delegations
  168. (delegation_id, origin_session, origin_ui_session_id,
  169. parent_session_id, state, dispatched_at, updated_at,
  170. delivery_state, delivery_attempts, owner_pid,
  171. owner_started_at, task_json, origin_session_id)
  172. VALUES (?, ?, ?, ?, 'running', ?, ?, 'pending', 0, ?, ?, ?, ?)""",
  173. (record["delegation_id"], record.get("session_key", ""), record.get("origin_ui_session_id", ""),
  174. record.get("parent_session_id"), record["dispatched_at"], now, os.getpid(), owner_started_at,
  175. json.dumps(task_payload), record.get("origin_session_id", "")))
  176. _prune_durable_records()
  177. def _prune_durable_records() -> None:
  178. """Bound terminal history, preferring delivered records for deletion."""
  179. require_ledger_owner()
  180. cutoff = time.time() - _DURABLE_RETENTION_SECONDS
  181. with _DB_LOCK, _transaction() as conn:
  182. conn.execute(
  183. "DELETE FROM async_delegations WHERE delivery_state='delivered' AND updated_at < ?", (cutoff,))
  184. terminal_count = conn.execute(
  185. "SELECT COUNT(*) FROM async_delegations WHERE state NOT IN ('running','finalizing')").fetchone()[0]
  186. if terminal_count > _MAX_RETAINED_COMPLETED:
  187. conn.execute("""DELETE FROM async_delegations WHERE delegation_id IN (
  188. SELECT delegation_id FROM async_delegations
  189. WHERE state NOT IN ('running','finalizing')
  190. ORDER BY CASE delivery_state WHEN 'delivered' THEN 0 ELSE 1 END,
  191. updated_at ASC LIMIT ?
  192. )""", (terminal_count - _MAX_RETAINED_COMPLETED,))
  193. pending_count = conn.execute("""SELECT COUNT(*) FROM async_delegations
  194. WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'""").fetchone()[0]
  195. if pending_count > _MAX_DURABLE_PENDING:
  196. conn.execute("""DELETE FROM async_delegations WHERE delegation_id IN (
  197. SELECT delegation_id FROM async_delegations
  198. WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'
  199. ORDER BY updated_at ASC LIMIT ?
  200. )""", (pending_count - _MAX_DURABLE_PENDING,))
  201. def _persist_completion(event: dict[str, Any], result: dict[str, Any]) -> None:
  202. event, result = redact_completion_event(event), redact_child_result(result)
  203. if remote := worker_ledger():
  204. return remote.complete(event, result)
  205. now = time.time()
  206. with _DB_LOCK, _transaction() as conn:
  207. conn.execute("""UPDATE async_delegations SET state=?, completed_at=?, updated_at=?,
  208. event_json=?, result_json=?, delivery_state='pending'
  209. WHERE delegation_id=?""",
  210. (event.get("status", "completed"), event.get("completed_at", now), now,
  211. json.dumps(event), json.dumps(result), event["delegation_id"]))
  212. def record_unit_child(delegation_id: str, entry: dict[str, Any]) -> None:
  213. """Durably record ONE finished child of a still-running multi-child unit on the unit's own row, so a crash before
  214. the unit joins loses only the children that had not finished. Stored in ``result_json`` (overwritten by the real
  215. result at finalize); ``recover_abandoned_delegations`` replays it. Best-effort: a failed write costs recovery
  216. fidelity, never the live result."""
  217. entry = redact_child_result(entry)
  218. if remote := worker_ledger():
  219. return remote.child(delegation_id, entry)
  220. try:
  221. with _DB_LOCK, _transaction() as conn:
  222. row = conn.execute("SELECT result_json FROM async_delegations WHERE delegation_id=? AND state='running'",
  223. (delegation_id,)).fetchone()
  224. if row is None:
  225. return
  226. partial = redact_child_result(json.loads(row[0] or "{}") or {})
  227. results = [r for r in partial.get("results") or [] if r.get("task_index") != entry.get("task_index")]
  228. results.append(entry)
  229. conn.execute("UPDATE async_delegations SET result_json=?, updated_at=? WHERE delegation_id=? AND state='running'",
  230. (json.dumps({"results": results, "partial": True}), time.time(), delegation_id))
  231. except Exception:
  232. logger.warning("Async delegation %s: could not record finished child %s", delegation_id, entry.get("task_index"), exc_info=True)
  233. def _recovered_results(task: dict[str, Any], result_json: Optional[str], error: str) -> Optional[list[dict[str, Any]]]:
  234. """Per-task results for an abandoned unit: recorded children as they finished, the rest ``unknown``."""
  235. partial = json.loads(result_json or "{}") or {}
  236. if not (task.get("is_batch") and partial.get("partial") and partial.get("results")):
  237. return None
  238. recorded = {r["task_index"]: r for r in partial["results"] if isinstance(r.get("task_index"), int)}
  239. indexes = task.get("task_indexes") or list(range(len(task.get("goals") or [])))
  240. return [recorded.get(i) or {"task_index": i, "status": "unknown", "summary": None, "error": error} for i in indexes]
  241. def _owner_liveness() -> Optional[Callable[[Any, Any], bool]]:
  242. """``alive(owner_pid, owner_started_at)`` over the shared drift-tolerant start-time comparator,
  243. or None when the liveness probes cannot be imported."""
  244. try:
  245. from gateway.status import _pid_exists, get_process_start_time, start_time_fingerprints_match
  246. except Exception:
  247. return None
  248. def alive(pid, started) -> bool:
  249. return bool(pid) and _pid_exists(int(pid)) and (
  250. started is None or start_time_fingerprints_match(started, get_process_start_time(int(pid)) or 0))
  251. return alive
  252. def recover_abandoned_delegations() -> int:
  253. """Classify records whose owning process disappeared as outcome unknown; children a multi-child unit had already
  254. recorded (``record_unit_child``) are replayed with their real results."""
  255. require_ledger_owner()
  256. alive = _owner_liveness()
  257. if alive is None:
  258. return 0
  259. now, recovered = time.time(), 0
  260. with _DB_LOCK, _transaction() as conn:
  261. rows = conn.execute("""SELECT delegation_id, origin_session, origin_ui_session_id,
  262. parent_session_id, dispatched_at, owner_pid,
  263. owner_started_at, task_json, origin_session_id, result_json, state
  264. FROM async_delegations WHERE state IN ('running','finalizing')""").fetchall()
  265. for row in rows:
  266. delegation_id, session_key, origin_ui, parent_id, dispatched_at, pid, started, task_json, origin_sid, result_json, last_state = row
  267. if alive(pid, started):
  268. continue
  269. task = json.loads(task_json or "{}")
  270. error = "Delegation owner exited before recording a terminal result; outcome unknown."
  271. recovered_results = _recovered_results(task, result_json, error)
  272. if recovered_results:
  273. done = sum(1 for r in recovered_results if r.get("status") != "unknown")
  274. error = (f"Delegation owner exited before the unit finished; {done}/{len(recovered_results)} child "
  275. "results were recorded and are included below, the rest are unknown.")
  276. diagnostics = {"last_known_status": last_state, "task_transcripts": task.get("task_transcripts") or {}}
  277. # Verbatim transcript tails + a git snapshot of the owner's cwd, so the parent can
  278. # continue or re-dispatch from the event alone instead of opening files (#116000).
  279. from tools.async_delegation_recovery_hints import git_state_hint, transcript_tails
  280. if tails := transcript_tails(diagnostics["task_transcripts"]):
  281. diagnostics["transcript_tails"] = tails
  282. if hint := git_state_hint(task.get("owner_cwd")):
  283. diagnostics["git_state_hint"] = hint
  284. event = {
  285. "type": "async_delegation", "delegation_id": delegation_id, "session_key": session_key,
  286. "origin_ui_session_id": origin_ui, "origin_session_id": origin_sid or "",
  287. "parent_session_id": parent_id, "goal": task.get("goal", ""), "goals": task.get("goals"),
  288. "context": task.get("context"), "toolsets": task.get("toolsets"), "role": task.get("role"),
  289. "model": task.get("model"), "is_batch": bool(task.get("is_batch")),
  290. "status": "unknown", "summary": None, "error": error, **diagnostics,
  291. **({"results": recovered_results} if recovered_results else {}),
  292. "dispatched_at": dispatched_at, "completed_at": now,
  293. **{k: task[k] for k in _ROUTING_KEYS if task.get(k)}}
  294. result = {"status": "unknown", "summary": None, "error": event["error"], **diagnostics,
  295. **({"results": recovered_results} if recovered_results else {})}
  296. event, result = redact_completion_event(event), redact_child_result(result)
  297. conn.execute("""UPDATE async_delegations SET state='unknown', completed_at=?,
  298. updated_at=?, event_json=?, result_json=?, delivery_state='pending'
  299. WHERE delegation_id=?""", (now, now, json.dumps(event), json.dumps(result), delegation_id))
  300. recovered += 1
  301. return recovered
  302. def restore_undelivered_completions(target_queue) -> int:
  303. """Enqueue durable pending completions as fresh turns after process start.
  304. Restored events are stamped ``restored=True`` in memory only: they came from a PREVIOUS
  305. process, so drains without an ownership filter must leave them for a consumer that can
  306. prove ownership. Rows older than ``_MAX_COMPLETION_REPLAY_AGE_S`` are terminally dropped
  307. instead of replaying a turn nobody is waiting on.
  308. Every restored event is stamped ``restored=True`` (in-memory only — the stamp is added after the durable
  309. payload is deserialized and is never persisted). Restored events originate from a *previous* process, so
  310. no consumer in THIS process implicitly owns them: drain paths that run without an ownership filter (the
  311. legacy single-session behavior) must leave them queued for a consumer that can positively prove
  312. ownership, otherwise a brand-new session adopts a dead session's delegation results seconds after boot
  313. (#64484).
  314. """
  315. from agent.runtime_session_store import is_worker_process
  316. if is_worker_process():
  317. # The ordinary owner restores delivery-only results; a compute import
  318. # must neither reap that owner's ledger nor steal its delivery queue.
  319. return 0
  320. if not _db_path().exists():
  321. return 0 # nothing to replay; a replay must not create (or migrate) the ledger (#123265)
  322. recover_abandoned_delegations()
  323. now = time.time()
  324. with _DB_LOCK, _transaction() as conn:
  325. rows = conn.execute("""SELECT delegation_id, event_json, completed_at, dispatched_at
  326. FROM async_delegations
  327. WHERE state != 'running' AND delivery_state='pending' AND event_json IS NOT NULL
  328. ORDER BY completed_at, delegation_id""").fetchall()
  329. return _replay_pending(conn, rows, target_queue, now, startup=True)
  330. def _replay_pending(conn, rows, target_queue, now: float, *, startup: bool = False) -> int:
  331. """Put each pending ``(delegation_id, event_json, completed_at, dispatched_at)`` row on ``target_queue``
  332. stamped ``restored``, or terminally drop it past ``_MAX_COMPLETION_REPLAY_AGE_S``. Records the offer so
  333. the orphan sweep skips the row until the copy is handed back (``return_completion_offer``)."""
  334. home, restored = hermes_home_key(get_hermes_home()), 0
  335. for delegation_id, payload, completed_at, dispatched_at in rows:
  336. age_basis = completed_at or dispatched_at
  337. if age_basis and (now - age_basis) > _MAX_COMPLETION_REPLAY_AGE_S:
  338. conn.execute("""UPDATE async_delegations SET delivery_state='dropped',
  339. delivery_claim=NULL, delivery_claimed_at=NULL,
  340. updated_at=?
  341. WHERE delegation_id=? AND delivery_state='pending'""", (now, delegation_id))
  342. logger.warning("Async delegation %s: pending completion is %.1fh old "
  343. "(cap %.1fh); terminally dropping the replay (result remains queryable).",
  344. delegation_id, (now - age_basis) / 3600.0, _MAX_COMPLETION_REPLAY_AGE_S / 3600.0)
  345. continue
  346. with _orphan_lock:
  347. if (home, delegation_id) in _offered:
  348. # The owner's explicit startup replay and the first-consume replay
  349. # (process_registry.restore_completions) may both run: offer once.
  350. continue
  351. evt = json.loads(payload)
  352. if isinstance(evt, dict):
  353. evt = redact_completion_event(evt)
  354. with _orphan_lock:
  355. if startup:
  356. _startup_replayed.add(delegation_id)
  357. if isinstance(evt, dict):
  358. evt["restored"] = True
  359. if delegation_id in _startup_replayed:
  360. evt["startup_replay"] = True
  361. target_queue.put(evt)
  362. with _orphan_lock:
  363. _offered.add((home, delegation_id))
  364. restored += 1
  365. return restored
  366. def completion_copy_id(evt: Any) -> str:
  367. """The delegation id ``evt`` is the final-result copy of, or "" (other events, interim notices)."""
  368. if not isinstance(evt, dict) or evt.get("type") != "async_delegation" or is_interim_delegation_event(evt):
  369. return ""
  370. return str(evt.get("delegation_id") or "")
  371. def mark_copy_held(delegation_id: str) -> None:
  372. """Record that a consumer holds a copy of ``delegation_id`` off every queue (a poller between dequeue and
  373. requeue, a parked copy). The orphan sweep treats a held copy like a queued one. A holder that moves a copy
  374. onto a queue marks before it dequeues, or under the queue's mutex, and unmarks only after any requeue, so
  375. the sweep never sees the copy as neither queued nor held. Pair every call with ``unmark_copy_held``."""
  376. if delegation_id:
  377. with _held_lock:
  378. _held_copies[delegation_id] += 1
  379. def unmark_copy_held(delegation_id: str) -> None:
  380. """End one ``mark_copy_held`` hold on ``delegation_id``."""
  381. if delegation_id:
  382. with _held_lock:
  383. _held_copies[delegation_id] -= 1
  384. if _held_copies[delegation_id] <= 0:
  385. del _held_copies[delegation_id]
  386. def _queued_completion_ids(*queues) -> Optional[set]:
  387. """Delegation ids with a final-result copy sitting in any of ``queues`` or held off-queue by a consumer
  388. (``mark_copy_held``), or None when a queue cannot be inspected. Each queue is read under its own mutex
  389. (the lock ``put``/``get`` and ``take_ready`` take) with the held ids read inside it, so a copy moving
  390. between a queue and a hold is always seen in one of them. Interim per-task notices share their batch's
  391. id but are not the result, so they never count as its copy."""
  392. ids: set = set()
  393. for q in {id(q): q for q in queues if q is not None}.values():
  394. mutex, items = getattr(q, "mutex", None), getattr(q, "queue", None)
  395. if mutex is None or items is None:
  396. return None
  397. with mutex:
  398. snapshot = list(items)
  399. with _held_lock:
  400. ids.update(_held_copies)
  401. ids.update(completion_copy_id(evt) for evt in snapshot)
  402. ids.discard("")
  403. return ids
  404. def _copy_may_be_live(delegation_id: str, pid, updated_at: float, live_ids: set, home: str,
  405. offered: set, now: float) -> bool:
  406. """For a row whose copy this process made (it owns the row or offered it): whether that copy may still be
  407. delivered, so the sweep must not add another. Live = queued or held (``live_ids``), or the row or this
  408. process's last re-offer was touched within a claim lease, the span after which a silent claim counts as
  409. abandoned. A row owned by ANOTHER live process is left to it: its queue is out of sight."""
  410. if not (delegation_id in offered or int(pid) == os.getpid()):
  411. return True
  412. if delegation_id in live_ids or updated_at >= now - _CLAIM_LEASE_S:
  413. return True
  414. with _orphan_lock:
  415. return _reoffered_at.get((home, delegation_id), float("-inf")) >= now - _CLAIM_LEASE_S
  416. def _note_reoffers(home: str, lost: list, now: float) -> None:
  417. """Start each lost copy's re-offer lease and log it: WARNING the first time a row is re-offered, DEBUG
  418. after that (a consumer that keeps handing the copy back would otherwise warn once per lease)."""
  419. with _orphan_lock:
  420. first = [d for d in lost if (home, d) not in _reoffered_at]
  421. for key in [k for k, at in _reoffered_at.items() if now - at > _MAX_COMPLETION_REPLAY_AGE_S]:
  422. del _reoffered_at[key] # its row is past the replay cap, so it was dropped
  423. _reoffered_at.update({(home, d): now for d in lost})
  424. _offered.difference_update({(home, d) for d in lost})
  425. log = logger.warning if first else logger.debug
  426. log("Re-offering %d pending async completion(s) whose in-memory copy was lost while this process ran: %s",
  427. len(lost), ", ".join(lost))
  428. def sweep_orphaned_completions(target_queue, *, now: Optional[float] = None) -> int:
  429. """Re-offer this home's pending completions that no live in-memory copy will deliver (#97202, #140718).
  430. Startup replay (``restore_undelivered_completions``) covers owners that died before the process
  431. started; this covers the rest while it runs. Abandoned in-flight rows are first classified by
  432. ``recover_abandoned_delegations``. A terminal row qualifies when it is pending with an event, idle
  433. past ``_ORPHAN_STALE_S`` and not under a live delivery claim, and then:
  434. - its owner is dead and this process has not offered it yet; or
  435. - its owner is THIS process, or this process already offered it, the row has been idle for a whole
  436. ``_CLAIM_LEASE_S`` and no copy of it sits in ``target_queue`` or the producers'
  437. ``completion_queue``: the one in-memory copy was lost (a consumer discarded it, or it went onto
  438. a queue no delivery loop drains), and without this the row would stay pending with zero
  439. attempts until a restart (#144563).
  440. The sweep never adds a copy beside a live one: queued, held off-queue by a consumer
  441. (``mark_copy_held``), under a claim lease (filtered above), or touched within a lease (a claim, defer
  442. or release refreshes ``updated_at``). Each re-offer starts its own lease, so a consumer that keeps
  443. handing the copy back sees it at most once per lease, and only the first re-offer warns. A row owned
  444. by ANOTHER live process is left to that process, whose queue this one cannot see. The consumer's
  445. ``claim_completion_delivery`` stays the atomic cross-process gate, so a duplicate copy can never
  446. deliver twice. Rows past the delivery budget or the replay age converge to ``dropped``. Reads the
  447. current profile's ledger: callers bind the owning profile first."""
  448. alive = _owner_liveness()
  449. if alive is None or not _db_path().exists():
  450. return 0 # never create a ledger just to sweep it
  451. recover_abandoned_delegations()
  452. now = time.time() if now is None else now
  453. home = hermes_home_key(get_hermes_home())
  454. from tools.process_registry import process_registry
  455. live = _queued_completion_ids(target_queue, process_registry.completion_queue)
  456. with _orphan_lock:
  457. offered = {delegation_id for key, delegation_id in _offered if key == home}
  458. with _DB_LOCK, _transaction() as conn:
  459. rows = conn.execute("""SELECT delegation_id, event_json, completed_at, dispatched_at,
  460. owner_pid, owner_started_at, delivery_attempts, updated_at
  461. FROM async_delegations
  462. WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'
  463. AND event_json IS NOT NULL AND updated_at < ?
  464. AND (delivery_claim IS NULL OR delivery_claimed_at < ?)
  465. ORDER BY completed_at, delegation_id""", (now - _ORPHAN_STALE_S, now - _CLAIM_LEASE_S)).fetchall()
  466. orphans, lost = [], []
  467. for delegation_id, payload, completed_at, dispatched_at, pid, started, attempts, updated_at in rows:
  468. copy_lost = alive(pid, started) or delegation_id in offered
  469. if copy_lost and (live is None or _copy_may_be_live(delegation_id, pid, updated_at, live, home,
  470. offered, now)):
  471. continue
  472. if (attempts or 0) >= _MAX_DELIVERY_ATTEMPTS:
  473. # Its last claimant died holding the final attempt; converge like release_completion_delivery.
  474. conn.execute("""UPDATE async_delegations SET delivery_state='dropped',
  475. delivery_claim=NULL, delivery_claimed_at=NULL, updated_at=?
  476. WHERE delegation_id=? AND delivery_state='pending'""", (now, delegation_id))
  477. logger.warning("Async delegation %s exhausted its %d delivery attempts; "
  478. "marking terminally dropped (result remains queryable).",
  479. delegation_id, _MAX_DELIVERY_ATTEMPTS)
  480. continue
  481. orphans.append((delegation_id, payload, completed_at, dispatched_at))
  482. if copy_lost:
  483. lost.append(delegation_id)
  484. if lost:
  485. _note_reoffers(home, lost, now)
  486. return _replay_pending(conn, orphans, target_queue, now)
  487. def maybe_sweep_orphaned_completions(target_queue, *, now: Optional[float] = None) -> int:
  488. """``sweep_orphaned_completions`` at most once per ``ORPHAN_SWEEP_INTERVAL_S`` per home (``now`` is
  489. monotonic), for delivery loops that tick far more often. Never raises into the loop."""
  490. home = hermes_home_key(get_hermes_home())
  491. now = time.monotonic() if now is None else now
  492. with _orphan_lock:
  493. last = _last_orphan_sweep.get(home)
  494. if last is not None and now - last < ORPHAN_SWEEP_INTERVAL_S:
  495. return 0
  496. _last_orphan_sweep[home] = now
  497. try:
  498. return sweep_orphaned_completions(target_queue)
  499. except Exception:
  500. logger.debug("Orphaned async delegation sweep failed", exc_info=True)
  501. return 0
  502. def _update_delivery(sql: str, params: tuple) -> bool:
  503. """Run one UPDATE on the ledger; True iff exactly one row changed."""
  504. with _DB_LOCK, _transaction() as conn:
  505. return conn.execute(sql, params).rowcount == 1
  506. def mark_completion_delivered(delegation_id: str) -> bool:
  507. """Atomically acknowledge successful injection of a durable completion."""
  508. require_ledger_owner()
  509. now = time.time()
  510. return _update_delivery(
  511. """UPDATE async_delegations SET delivery_state='delivered', delivered_at=?, updated_at=?
  512. WHERE delegation_id=? AND delivery_state!='delivered'""", (now, now, delegation_id))
  513. def claim_completion_delivery(delegation_id: str, claim_id: str) -> bool:
  514. """Claim one pending completion across competing consumers/processes."""
  515. if remote := worker_ledger():
  516. return remote.claim(delegation_id, claim_id)
  517. now = time.time()
  518. with _DB_LOCK, _transaction() as conn:
  519. row = conn.execute(
  520. "SELECT delivery_state FROM async_delegations WHERE delegation_id=?", (delegation_id,)).fetchone()
  521. if row is None:
  522. return True # legacy event created before durable dispatch
  523. cur = conn.execute("""UPDATE async_delegations SET delivery_claim=?, delivery_claimed_at=?,
  524. delivery_attempts=delivery_attempts+1, updated_at=?
  525. WHERE delegation_id=? AND delivery_state='pending'
  526. AND (delivery_claim IS NULL OR delivery_claimed_at < ?)""",
  527. (claim_id, now, now, delegation_id, now - _CLAIM_LEASE_S))
  528. return cur.rowcount == 1
  529. def is_interim_delegation_event(evt: dict[str, Any]) -> bool:
  530. """An early per-task notice for a batch that is still running. It shares the batch's
  531. ``delegation_id`` but is NOT the durable completion: it must never claim, acknowledge or
  532. dedup against the final result's row (independent review reproduced exactly that loss)."""
  533. return evt.get("type") == "async_delegation" and bool(evt.get("task_failure_notice"))
  534. def claim_event_delivery(evt: dict[str, Any], consumer: str) -> Optional[str]:
  535. """Claim a durable delegation event; non-durable events (and interim notices) need no token."""
  536. if is_interim_delegation_event(evt):
  537. return ""
  538. delegation_id = str(evt.get("delegation_id") or "") if evt.get("type") == "async_delegation" else ""
  539. if not delegation_id:
  540. return ""
  541. claim_id = f"{consumer}:{os.getpid()}:{uuid.uuid4().hex}"
  542. return claim_id if claim_completion_delivery(delegation_id, claim_id) else None
  543. def release_completion_delivery(delegation_id: str, claim_id: str) -> bool:
  544. """Release a failed delivery claim so another consumer may retry. Attempts are
  545. counted at claim time; once the budget is exhausted the row converges to
  546. terminal ``dropped`` (only pending rows replay on restart)."""
  547. if remote := worker_ledger():
  548. return remote.release(delegation_id, claim_id)
  549. now = time.time()
  550. with _DB_LOCK, _transaction() as conn:
  551. capped = conn.execute("""UPDATE async_delegations SET delivery_state='dropped',
  552. delivery_claim=NULL, delivery_claimed_at=NULL, updated_at=?
  553. WHERE delegation_id=? AND delivery_state='pending'
  554. AND delivery_claim=? AND delivery_attempts>=?""",
  555. (now, delegation_id, claim_id, _MAX_DELIVERY_ATTEMPTS))
  556. if capped.rowcount == 1:
  557. logger.warning("Async delegation %s exhausted its %d delivery attempts; "
  558. "marking terminally dropped (result remains queryable).",
  559. delegation_id, _MAX_DELIVERY_ATTEMPTS)
  560. return True
  561. cur = conn.execute("""UPDATE async_delegations SET delivery_claim=NULL,
  562. delivery_claimed_at=NULL, updated_at=?
  563. WHERE delegation_id=? AND delivery_state='pending'
  564. AND delivery_claim=?""", (now, delegation_id, claim_id))
  565. return cur.rowcount == 1
  566. def defer_completion_delivery(delegation_id: str, claim_id: str) -> bool:
  567. """Return an unadmitted completion to pending without spending a delivery attempt."""
  568. if remote := worker_ledger():
  569. return remote.defer(delegation_id, claim_id)
  570. return _update_delivery("""UPDATE async_delegations SET delivery_claim=NULL,
  571. delivery_claimed_at=NULL, delivery_attempts=MAX(0, delivery_attempts-1),
  572. updated_at=?
  573. WHERE delegation_id=? AND delivery_state='pending' AND delivery_claim=?""",
  574. (time.time(), delegation_id, claim_id))
  575. def drop_completion_delivery(delegation_id: str, claim_id: str) -> bool:
  576. """Terminally drop a claimed completion whose target is permanently gone (the
  577. spawning session ended at an explicit user boundary such as /new or reset).
  578. ``dropped`` — not ``delivered`` — keeps the ack honest; not ``pending`` keeps
  579. restart recovery from replaying it into a fail-closed drop forever."""
  580. if remote := worker_ledger():
  581. return remote.drop(delegation_id, claim_id)
  582. return _update_delivery("""UPDATE async_delegations SET delivery_state='dropped',
  583. updated_at=?, delivery_claim=NULL,
  584. delivery_claimed_at=NULL
  585. WHERE delegation_id=? AND delivery_state='pending'
  586. AND delivery_claim=?""", (time.time(), delegation_id, claim_id))
  587. def complete_completion_delivery(delegation_id: str, claim_id: str) -> bool:
  588. """Acknowledge acceptance for the consumer holding this claim."""
  589. if remote := worker_ledger():
  590. return remote.ack(delegation_id, claim_id)
  591. now = time.time()
  592. return _update_delivery("""UPDATE async_delegations SET delivery_state='delivered',
  593. delivered_at=?, updated_at=?, delivery_claim=NULL,
  594. delivery_claimed_at=NULL
  595. WHERE delegation_id=? AND delivery_state='pending'
  596. AND delivery_claim=?""", (now, now, delegation_id, claim_id))
  597. def requeue_admitted_completion(delegation_id: str, target_queue) -> bool:
  598. """Offer an acknowledged completion again when its receiving turn could not confirm the owner.
  599. The row returns to pending and its event goes back on ``target_queue``, so the next delivery
  600. re-runs the pre-flight. A row past the delivery budget converges to ``dropped`` instead, which
  601. bounds the loop; a row without a stored event is left as it is."""
  602. now = time.time()
  603. with _DB_LOCK, _transaction() as conn:
  604. exhausted = conn.execute("""UPDATE async_delegations SET delivery_state='dropped', updated_at=?
  605. WHERE delegation_id=? AND delivery_state='delivered' AND delivery_attempts>=?""",
  606. (now, delegation_id, _MAX_DELIVERY_ATTEMPTS))
  607. if exhausted.rowcount == 1:
  608. logger.warning("Async delegation %s exhausted its %d delivery attempts; "
  609. "marking terminally dropped (result remains queryable).",
  610. delegation_id, _MAX_DELIVERY_ATTEMPTS)
  611. return False
  612. reopened = conn.execute("""UPDATE async_delegations SET delivery_state='pending',
  613. delivered_at=NULL, updated_at=?
  614. WHERE delegation_id=? AND delivery_state='delivered' AND event_json IS NOT NULL""",
  615. (now, delegation_id))
  616. if reopened.rowcount != 1:
  617. return False
  618. rows = conn.execute("""SELECT delegation_id, event_json, completed_at, dispatched_at
  619. FROM async_delegations WHERE delegation_id=?""", (delegation_id,)).fetchall()
  620. with _orphan_lock: # an admitted copy already had its consumer: requeue keeps live routing
  621. _startup_replayed.discard(delegation_id)
  622. return _replay_pending(conn, rows, target_queue, now) == 1
  623. def complete_event_delivery(evt: dict[str, Any], claim_id: str) -> None:
  624. _event_delivery(complete_completion_delivery, evt, claim_id)
  625. def release_event_delivery(evt: dict[str, Any], claim_id: str) -> None:
  626. """Release a failed claim for a consumer that discards its copy (the TUI poller): the row is pending
  627. again, so it must stay eligible for the orphan sweep."""
  628. _event_delivery(release_completion_delivery, evt, claim_id)
  629. return_completion_offer(evt)
  630. def return_completion_offer(evt: dict[str, Any]) -> None:
  631. """Hand an offered completion back to the orphan sweep after its in-memory copy was discarded while
  632. the durable row stays pending, e.g. a TUI session that cannot prove it owns the event drops it (every
  633. session poller drains one process-wide queue). The next sweep may offer the row again. Delegation ids
  634. are unique across profiles, so this clears the offer in every home."""
  635. delegation_id = str(evt.get("delegation_id") or "") if evt.get("type") == "async_delegation" else ""
  636. if not delegation_id or is_interim_delegation_event(evt):
  637. return
  638. with _orphan_lock:
  639. _offered.difference_update({key for key in _offered if key[1] == delegation_id})
  640. def _event_delivery(fn, evt: dict[str, Any], claim_id: str) -> None:
  641. if claim_id and evt.get("type") == "async_delegation":
  642. fn(str(evt.get("delegation_id") or ""), claim_id)
  643. def get_durable_delegation(delegation_id: str) -> Optional[dict[str, Any]]:
  644. if remote := worker_ledger():
  645. saved = remote.read(delegation_id)
  646. if saved is not None and saved.get("result") is not None:
  647. return {**saved, "result": redact_child_result(saved["result"])}
  648. return saved
  649. with _DB_LOCK, _transaction() as conn:
  650. row = conn.execute("""SELECT origin_session, state, dispatched_at, completed_at,
  651. result_json, delivery_state, delivery_attempts,
  652. origin_session_id
  653. FROM async_delegations WHERE delegation_id=?""", (delegation_id,)).fetchone()
  654. return None if row is None else {
  655. "delegation_id": delegation_id, "origin_session": row[0], "state": row[1], "dispatched_at": row[2],
  656. "completed_at": row[3], "result": redact_child_result(json.loads(row[4])) if row[4] else None, "delivery_state": row[5],
  657. "delivery_attempts": row[6], "origin_session_id": row[7] or ""}
  658. _FAILED_TASK_STATES = frozenset({"error", "failed", "failure", "timeout", "stalled", "unknown", "interrupted"})
  659. _FAILURE_SURFACE_WINDOW_S = 24 * 3600.0
  660. def _json_object(raw: Optional[str]) -> dict[str, Any]:
  661. try:
  662. value = json.loads(raw) if raw else {}
  663. except ValueError:
  664. return {}
  665. return value if isinstance(value, dict) else {}
  666. def failed_delegations_for_session(
  667. origin_ui_session_id: str = "", parent_session_id: str = "", *, limit: int = 20, now: Optional[float] = None,
  668. ) -> list[dict[str, Any]]:
  669. """Recently failed async delegation tasks owned by a session, newest first.
  670. The live roster forgets a child once it ends and does not survive a renderer reload, so a failed
  671. delegation had nowhere to show (#97202). This reads the durable row instead: one entry per failed
  672. task (a batch unit that "completed" can still carry failed tasks) with ``delegation_id``,
  673. ``task_index``, ``goal``, ``status``, ``error``, ``dispatched_at`` and ``completed_at``. Either selector claims a row:
  674. the UI session id at dispatch, or the spawner's durable session id (survives a reload re-mint)."""
  675. selectors = [(col, val) for col, val in (
  676. ("origin_ui_session_id", origin_ui_session_id), ("parent_session_id", parent_session_id)) if val]
  677. if not selectors:
  678. return []
  679. cutoff = (now if now is not None else time.time()) - _FAILURE_SURFACE_WINDOW_S
  680. owner_sql = " OR ".join(f"{col}=?" for col, _ in selectors)
  681. with _DB_LOCK, _transaction() as conn:
  682. rows = conn.execute(
  683. f"""SELECT delegation_id, state, dispatched_at, completed_at, task_json, result_json FROM async_delegations
  684. WHERE ({owner_sql}) AND state NOT IN ('running','finalizing') AND completed_at >= ?
  685. ORDER BY completed_at DESC LIMIT ?""",
  686. (*(val for _, val in selectors), cutoff, limit)).fetchall()
  687. failed: list[dict[str, Any]] = []
  688. for delegation_id, state, dispatched_at, completed_at, task_json, result_json in rows:
  689. task, result = _json_object(task_json), redact_child_result(_json_object(result_json))
  690. goals = task.get("goals") if isinstance(task.get("goals"), list) and task["goals"] else [task.get("goal") or ""]
  691. goal_for = dict(zip(task.get("task_indexes") or range(len(goals)), goals))
  692. tasks = result["results"] if isinstance(result.get("results"), list) else [] if task.get("is_batch") else [result]
  693. if not tasks and str(state).lower() in _FAILED_TASK_STATES:
  694. tasks = [{"task_index": 0, "error": result.get("error")}]
  695. for entry in tasks:
  696. status = str(entry.get("status") or state or "").lower()
  697. if status not in _FAILED_TASK_STATES:
  698. continue
  699. index = entry.get("task_index") if isinstance(entry.get("task_index"), int) else 0
  700. error = entry.get("error") or result.get("error")
  701. failed.append({
  702. "delegation_id": delegation_id, "task_index": index, "status": status,
  703. "goal": str(goal_for.get(index, goals[0]) or ""), "error": str(error) if error else None,
  704. "dispatched_at": dispatched_at, "completed_at": completed_at})
  705. return failed[:limit]
  706. # ── In-memory registry queries ──────────────────────────────────────────────
  707. def _get_executor(max_workers: int) -> ThreadPoolExecutor:
  708. """Lazily create (or grow in place, never shrink) the shared daemon executor. Raising
  709. ``_max_workers`` is enough: the next ``submit`` spawns threads up to the new cap."""
  710. global _executor, _executor_max_workers
  711. with _executor_lock:
  712. if _executor is None:
  713. _executor = DaemonThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="async-delegate")
  714. _executor_max_workers = max_workers
  715. elif max_workers > _executor_max_workers:
  716. _executor._max_workers = max_workers
  717. _executor_max_workers = max_workers
  718. return _executor
  719. def active_count() -> int:
  720. """Number of live async delegation UNITS (one per completion message: a task group or an ungrouped task)."""
  721. with _records_lock:
  722. return sum(1 for r in _records.values() if r.get("status") in _LIVE_STATES)
  723. def active_task_count() -> int:
  724. """Number of running child subagents (a batch of N contributes N; a batch with
  725. no goal list counts 1) — the truthful observability figure, unlike slots."""
  726. with _records_lock:
  727. return sum(
  728. len(r.get("task_indexes") or r["goals"])
  729. if r.get("is_batch") and isinstance(r.get("goals"), (list, tuple)) and r["goals"] else 1
  730. for r in _records.values() if r.get("status") in {"running", "finalizing"})
  731. def _session_records(statuses, session_key: str, origin_ui_session_id: str, parent_session_id: str) -> list:
  732. """Records in ``statuses`` owned by a session: any non-empty selector claims the
  733. record — ``origin_ui_session_id`` (TUI tab), ``session_key`` (routing key at
  734. dispatch), or ``parent_session_id`` (spawner's durable id — the right one for
  735. gateway chats, whose session_key survives ``/new`` while the session id rotates)."""
  736. selectors = [(field, wanted) for field, wanted in (
  737. ("origin_ui_session_id", origin_ui_session_id), ("session_key", session_key),
  738. ("parent_session_id", parent_session_id)) if wanted]
  739. if not selectors:
  740. return []
  741. with _records_lock:
  742. return [r for r in _records.values() if r.get("status") in statuses
  743. and any(str(r.get(field) or "") == wanted for field, wanted in selectors)]
  744. def has_live_for_session(session_key: str = "", origin_ui_session_id: str = "", parent_session_id: str = "") -> bool:
  745. """Whether a session still owns any live (running/stalling/finalizing) delegation."""
  746. return bool(_session_records(_LIVE_STATES, session_key, origin_ui_session_id, parent_session_id))
  747. def _new_delegation_id() -> str:
  748. return f"deleg_{uuid.uuid4().hex[:8]}"
  749. def _prune_completed_locked() -> None:
  750. """Drop the oldest completed records beyond the cap. Caller holds ``_records_lock``.
  751. ``stalling``/``finalizing`` are still live: evicting one makes the late runner return hit
  752. ``_finalize``'s missing-record path and silently drop a real result."""
  753. completed = [(rid, r) for rid, r in _records.items() if r.get("status") not in _LIVE_STATES]
  754. completed.sort(key=lambda kv: kv[1].get("completed_at") or kv[1].get("dispatched_at") or 0)
  755. for rid, _ in completed[: max(0, len(completed) - _MAX_RETAINED_COMPLETED)]:
  756. _records.pop(rid, None)
  757. def _current_origin_session_id() -> str:
  758. """Raw session id of the ORIGINATING api_server request, or ``""``. ``HERMES_SESSION_ID``
  759. is unsafe here: building the child agent calls ``set_current_session_id(child.session_id)``
  760. just before dispatch, so the wake would self-post into the subagent's own session. The
  761. request-scoped ``HERMES_SESSION_CHAT_ID`` (raw X-Hermes-Session-Id on api_server) survives
  762. child construction; on push platforms chat_id is a chat, not a session => ``""``."""
  763. try:
  764. from gateway.session_context import get_session_env
  765. is_api = get_session_env("HERMES_SESSION_PLATFORM", "") == "api_server"
  766. return (get_session_env("HERMES_SESSION_CHAT_ID", "") or "") if is_api else ""
  767. except Exception:
  768. return ""
  769. # ── Dispatch ────────────────────────────────────────────────────────────────
  770. def _single_crash(error: str, duration: float) -> dict[str, Any]:
  771. return {"status": "error", "summary": None, "error": error, "api_calls": 0, "duration_seconds": duration}
  772. def _batch_crash(error: str, duration: float) -> dict[str, Any]:
  773. return {"results": [], "error": error, "total_duration_seconds": duration}
  774. def _batch_status(combined: dict[str, Any]) -> str:
  775. """Batch status: completed unless every child errored/was interrupted."""
  776. child_results = combined.get("results") or []
  777. ok = ("completed", "success")
  778. return "error" if child_results and all(r.get("status") not in ok for r in child_results) else "completed"
  779. def _dispatch(**kwargs) -> dict[str, Any]:
  780. from hermes_cli.backend_retirement import retirement
  781. with retirement.work() as admitted:
  782. if not admitted:
  783. return {"status": "rejected", "error": "backend is retiring; reconnect to continue"}
  784. return _dispatch_admitted(**kwargs)
  785. def _dispatch_admitted(
  786. *, delegation_id: str, goal: str, goals: Optional[list[str]], context: Optional[str],
  787. toolsets: Optional[list[str]], role: str, model: Optional[str], session_key: str,
  788. parent_session_id: Optional[str], runner: Callable[[], dict[str, Any]], origin_ui_session_id: str,
  789. origin_session_id: str, interrupt_fn: Optional[Callable[[], None]], max_async_children: int,
  790. progress_fn: Optional[Callable[[], tuple]], capacity_error: str, slot_key: Optional[str] = None,
  791. task_indexes: Optional[list[int]] = None,
  792. task_transcripts: Optional[dict[str, str]] = None,
  793. ) -> dict[str, Any]:
  794. """Shared dispatch core for single (``goals is None``) and batch units. Capacity check +
  795. record insert happen under ONE lock hold so concurrent dispatches can't both pass the check
  796. and exceed the cap. At capacity the dispatch is REJECTED (never queued) so a runaway model
  797. can't pile up unbounded background work. ``slot_key`` names the pool slot the unit occupies
  798. (default: its own id); the units of one delegate_task call share the first unit's id so
  799. splitting a call into per-group completions never consumes more capacity than the call did."""
  800. worker_ledger() # refuse unbound workers before publishing a live record
  801. is_batch = goals is not None
  802. label = " batch" if is_batch else ""
  803. classify = _batch_status if is_batch else (lambda r: r.get("status") or "completed")
  804. crash_result = _batch_crash if is_batch else _single_crash
  805. dispatched_at = time.time()
  806. record: dict[str, Any] = {
  807. "delegation_id": delegation_id, "goal": goal, **({"goals": list(goals)} if is_batch else {}),
  808. "context": context, "toolsets": list(toolsets) if toolsets else None, "role": role, "model": model,
  809. "session_key": session_key, "origin_ui_session_id": origin_ui_session_id,
  810. "origin_session_id": origin_session_id, "parent_session_id": parent_session_id,
  811. **_capture_routing_origin(),
  812. "status": "running", "dispatched_at": dispatched_at, "completed_at": None,
  813. "interrupt_fn": interrupt_fn, **({"is_batch": True} if is_batch else {}), "progress_fn": progress_fn,
  814. "slot_key": slot_key or delegation_id,
  815. **({"task_transcripts": dict(task_transcripts)} if task_transcripts else {}),
  816. # Which of the call's ``goals`` this unit runs (None = all of them).
  817. **({"task_indexes": list(task_indexes)} if task_indexes is not None else {}),
  818. # The one stale-monitor thread serves every profile and starts with an empty Context;
  819. # a forced finalization runs under the dispatcher's so it settles the same state.db.
  820. "_context": contextvars.copy_context(),
  821. # Stale-monitor bookkeeping (see _stale_monitor_loop).
  822. "_progress_token": None, "_progress_ts": dispatched_at, "_interrupted_at": None}
  823. with _records_lock:
  824. if any((field, record[field]) in _refused_by for field in _FENCED_FIELDS):
  825. return {"status": "rejected", "error": "This session was closed (exit or /new): no new background work."}
  826. active_slots = {r.get("slot_key") or r["delegation_id"] for r in _records.values() if r.get("status") in _ACTIVE_STATES}
  827. if record["slot_key"] not in active_slots and len(active_slots) >= max_async_children:
  828. return {"status": "rejected", "error": capacity_error}
  829. _records[delegation_id] = record
  830. live_units = sum(1 for r in _records.values() if r.get("status") in _LIVE_STATES)
  831. _persist_dispatch(record)
  832. # Units of one call share a slot, so live units can exceed slots: size the pool by units or a
  833. # unit queues behind a full pool and the stale monitor kills it before its child ever starts.
  834. executor = _get_executor(max(max_async_children, live_units))
  835. def _worker() -> None:
  836. result: dict[str, Any] = {}
  837. status = "error"
  838. with _records_lock:
  839. rec = _records.get(delegation_id)
  840. if rec is not None:
  841. # The stall clock starts when the runner starts; a unit queued behind a full pool is not stalled.
  842. rec.update(_started=True, _progress_ts=time.time())
  843. try:
  844. result = runner() or {}
  845. status = classify(result)
  846. except Exception as exc:
  847. logger.exception(f"Async delegation{label} %s crashed", delegation_id)
  848. result = crash_result(f"{type(exc).__name__}: {exc}", round(time.time() - dispatched_at, 2))
  849. finally:
  850. _finalize(delegation_id, result, status)
  851. from hermes_cli.backend_retirement import retirement
  852. # The outer dispatch reservation prevents a freeze during this handoff. Retain a worker
  853. # reservation too: the stall monitor may finalize its registry record before it really exits.
  854. retirement.acquire()
  855. try:
  856. future = executor.submit(propagate_context_to_thread(_worker))
  857. future.add_done_callback(lambda _: retirement.release())
  858. except Exception as exc: # pragma: no cover — pool submit failure is rare
  859. retirement.release()
  860. with _records_lock:
  861. _records.pop(delegation_id, None)
  862. if remote := worker_ledger():
  863. remote.unscheduled(delegation_id)
  864. else:
  865. with _DB_LOCK, _transaction() as conn:
  866. conn.execute("DELETE FROM async_delegations WHERE delegation_id=?", (delegation_id,))
  867. return {"status": "rejected", "error": f"Failed to schedule async delegation{label}: {exc}"}
  868. if progress_fn is not None:
  869. _ensure_stale_monitor()
  870. return {"status": "dispatched", "delegation_id": delegation_id}
  871. def dispatch_async_delegation(
  872. *, goal: str, context: Optional[str], toolsets: Optional[list[str]], role: str, model: Optional[str],
  873. session_key: str, parent_session_id: Optional[str] = None, runner: Callable[[], dict[str, Any]],
  874. origin_ui_session_id: str = "", origin_session_id: str = "", interrupt_fn: Optional[Callable[[], None]] = None,
  875. max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, progress_fn: Optional[Callable[[], tuple]] = None,
  876. ) -> dict[str, Any]:
  877. """Spawn ``runner`` on the daemon executor and return a handle immediately.
  878. ``session_key``/``parent_session_id`` are captured on the parent thread (the worker carries
  879. no contextvars) and route the completion back to the spawning session.
  880. ``progress_fn() -> (token, in_tool)`` enables stale monitoring; omitted = unmonitored.
  881. Returns ``{"status": "dispatched", "delegation_id"}`` or ``{"status": "rejected", "error"}``."""
  882. delegation_id = _new_delegation_id()
  883. handle = _dispatch(
  884. delegation_id=delegation_id, goal=goal, goals=None, context=context,
  885. toolsets=toolsets, role=role, model=model, session_key=session_key,
  886. parent_session_id=parent_session_id, runner=runner,
  887. origin_ui_session_id=origin_ui_session_id, origin_session_id=origin_session_id,
  888. interrupt_fn=interrupt_fn, max_async_children=max_async_children, progress_fn=progress_fn,
  889. capacity_error=(
  890. f"Async delegation capacity reached ({max_async_children} running). Wait for one to finish "
  891. "(its result will re-enter the chat), or run this task synchronously (background=false). "
  892. "Raise delegation.max_concurrent_children in config.yaml to allow more concurrent background subagents."))
  893. if handle["status"] == "dispatched":
  894. logger.info("Dispatched async delegation %s (session_key=%s): %s",
  895. delegation_id, session_key or "<cli>", (goal or "")[:80])
  896. return handle
  897. def dispatch_async_delegation_batch(
  898. *, goals: list[str], context: Optional[str], toolsets: Optional[list[str]], role: str, model: Optional[str],
  899. session_key: str, parent_session_id: Optional[str] = None, runner: Callable[[], dict[str, Any]],
  900. origin_ui_session_id: str = "", origin_session_id: str = "", interrupt_fn: Optional[Callable[[], None]] = None,
  901. max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, delegation_id: Optional[str] = None,
  902. progress_fn: Optional[Callable[[], tuple]] = None, slot_key: Optional[str] = None,
  903. task_indexes: Optional[list[int]] = None,
  904. task_transcripts: Optional[dict[str, str]] = None,
  905. ) -> dict[str, Any]:
  906. """Dispatch a fan-out unit (a whole batch, or one ``group`` of a delegate_task call) as ONE
  907. background unit: ``runner`` runs its tasks and returns the combined ``{"results": [...],
  908. "total_duration_seconds": N}`` dict. The unit occupies ONE async slot — or joins the slot named
  909. by ``slot_key`` (in-unit parallelism is bounded separately) — and produces a SINGLE completion
  910. event carrying per-task ``results``."""
  911. delegation_id = delegation_id or _new_delegation_id()
  912. # ``goals`` is the whole call (result task_index indexes it); the unit's own goals label the record.
  913. unit_goals = [goals[i] for i in task_indexes] if task_indexes is not None else list(goals)
  914. n = len(unit_goals)
  915. combined_goal = unit_goals[0] if n == 1 else f"{n} parallel subagents: " + "; ".join(g[:40] for g in unit_goals)
  916. handle = _dispatch(
  917. delegation_id=delegation_id, goal=combined_goal, goals=goals, context=context,
  918. toolsets=toolsets, role=role, model=model, session_key=session_key,
  919. parent_session_id=parent_session_id, runner=runner,
  920. origin_ui_session_id=origin_ui_session_id, origin_session_id=origin_session_id,
  921. interrupt_fn=interrupt_fn, max_async_children=max_async_children, progress_fn=progress_fn, slot_key=slot_key,
  922. task_indexes=task_indexes, task_transcripts=task_transcripts,
  923. capacity_error=(
  924. f"Async delegation capacity reached ({max_async_children} running). Wait for one to finish "
  925. "(its result will re-enter the chat), or raise delegation.max_concurrent_children in "
  926. "config.yaml to allow more concurrent background units."))
  927. if handle["status"] == "dispatched":
  928. logger.info("Dispatched async delegation batch %s (%d task(s), session_key=%s)",
  929. delegation_id, n, session_key or "<cli>")
  930. return handle
  931. # ── Finalization + completion events ────────────────────────────────────────
  932. def _finalize(delegation_id: str, result: Any, status: str) -> None:
  933. """Atomically claim terminal delivery, push the completion event, then mark ``status``.
  934. ``result`` is a dict or a callable receiving the record snapshot (stall path). The record
  935. stays active ("finalizing") until durable persistence and queue publication finish; otherwise
  936. process shutdown can kill this daemon worker after status flips but before SQLite commits.
  937. A second call for the same id (late runner return after a forced stall) is a no-op."""
  938. with _records_lock:
  939. record = _records.get(delegation_id)
  940. if record is None or record.get("status") not in _ACTIVE_STATES:
  941. return
  942. record["status"] = "finalizing"
  943. record["completed_at"] = time.time()
  944. record["interrupt_fn"] = None # drop the closure; child is done
  945. record["progress_fn"] = None # stop stale-monitor sampling
  946. snapshot = dict(record)
  947. _push_completion_event(snapshot, result(snapshot) if callable(result) else result, status)
  948. with _records_lock:
  949. if delegation_id in _records:
  950. _records[delegation_id]["status"] = status
  951. # Only an active record is ever force-finalized under its dispatcher's Context; a
  952. # retained completed one would otherwise pin that turn's agent and history.
  953. _records[delegation_id].pop("_context", None)
  954. _prune_completed_locked()
  955. def _push_completion_event(record: dict[str, Any], result: dict[str, Any], status: str) -> None:
  956. """Push a type='async_delegation' event onto the shared completion queue. Batch records
  957. (``is_batch``) carry the per-task ``results`` list (plus live transcript paths, the
  958. full-fidelity record of each child's run) instead of a single summary. Best-effort: failure
  959. must not crash the worker, but it WOULD mean a silently-lost result, so we log loudly.
  960. The queued event is the parent's in-process copy and honours ``security.redact_secrets: false``;
  961. ``_persist_completion`` re-scrubs with force for the durable row (and the worker IPC)."""
  962. result = redact_child_result(result, force=False)
  963. is_batch = bool(record.get("is_batch"))
  964. label = " batch" if is_batch else ""
  965. try:
  966. from tools.process_registry import process_registry
  967. except Exception as exc: # pragma: no cover
  968. logger.error(f"Async delegation{label} %s finished but process_registry import failed; "
  969. "result lost: %s", record.get("delegation_id"), exc)
  970. return
  971. dispatched_at = record.get("dispatched_at") or time.time()
  972. completed_at = record.get("completed_at") or time.time()
  973. if is_batch:
  974. payload = {
  975. "is_batch": True, "results": result.get("results") or [],
  976. "live_transcripts": result.get("live_transcripts"), "error": result.get("error"),
  977. "total_duration_seconds": result.get("total_duration_seconds"),
  978. **({"group": result["group"]} if result.get("group") is not None else {})}
  979. else:
  980. # A runner with no measured count (a cron run without a session) omits api_calls: never a fake 0.
  981. payload = {
  982. "summary": result.get("summary"), "error": result.get("error"),
  983. **({"api_calls": result["api_calls"]} if result.get("api_calls") is not None else {}),
  984. "duration_seconds": result.get("duration_seconds", round(completed_at - dispatched_at, 2))}
  985. evt = {
  986. "type": "async_delegation", "delegation_id": record.get("delegation_id"),
  987. # session_key routes back to the originating gateway session; "" => CLI.
  988. "session_key": record.get("session_key", ""),
  989. "origin_ui_session_id": record.get("origin_ui_session_id", ""),
  990. "origin_session_id": record.get("origin_session_id", ""),
  991. "parent_session_id": record.get("parent_session_id"),
  992. "goal": record.get("goal", ""), **({"goals": record.get("goals")} if is_batch else {}),
  993. "context": record.get("context"), "toolsets": record.get("toolsets"), "role": record.get("role"),
  994. "model": record.get("model") if is_batch else (result.get("model") or record.get("model")),
  995. "status": status, **payload, "dispatched_at": dispatched_at, "completed_at": completed_at,
  996. **({} if is_batch else {"exit_reason": result.get("exit_reason")}),
  997. **{k: record[k] for k in _ROUTING_KEYS if record.get(k)},
  998. **{k: result[k] for k in _STALL_META_KEYS if k in result}}
  999. try:
  1000. _persist_completion(evt, result)
  1001. except Exception as exc:
  1002. logger.error(f"Async delegation{label} %s: durable completion write failed; delivering in-memory "
  1003. "only (a restart may report this unit as unknown): %s", record.get("delegation_id"), exc)
  1004. try:
  1005. process_registry.completion_queue.put(evt)
  1006. except Exception as exc: # pragma: no cover
  1007. logger.error(f"Async delegation{label} %s: failed to enqueue completion event; "
  1008. "result lost: %s", record.get("delegation_id"), exc)
  1009. def push_task_failure_notice(delegation_id: str, entry: dict[str, Any], *, n_tasks: int) -> None:
  1010. """Surface ONE failed child of a still-running detached batch to the parent now, instead of
  1011. when the slowest sibling finishes. In a 1,393-agent run every wave-1 child died in a 401 storm
  1012. at 08:29 and the parent learned of it at 09:36, when the batch's "unknown outcome" block finally
  1013. arrived: 66 minutes of a dead wave with nothing running. The notice rides the same
  1014. ``type="async_delegation"`` event shape as the batch result (so every drain/route/format path
  1015. treats it identically) with ``task_failure_notice=True`` and a single-entry ``results`` list; the
  1016. batch record is NOT finalized and its consolidated result still arrives as before. In-process
  1017. queue only (never persisted), so the parent's copy honours ``security.redact_secrets: false``."""
  1018. entry = redact_child_result(entry, force=False)
  1019. with _records_lock:
  1020. record = _records.get(delegation_id)
  1021. if record is None or record.get("status") not in _ACTIVE_STATES:
  1022. return
  1023. snapshot = dict(record)
  1024. try:
  1025. from tools.process_registry import process_registry
  1026. except Exception as exc: # pragma: no cover
  1027. logger.error("Async delegation batch %s: task failure notice dropped (process_registry import): %s", delegation_id, exc)
  1028. return
  1029. evt = {
  1030. "type": "async_delegation", "task_failure_notice": True, "is_batch": True, "n_tasks": n_tasks,
  1031. "delegation_id": delegation_id, "results": [entry],
  1032. "session_key": snapshot.get("session_key", ""),
  1033. "origin_ui_session_id": snapshot.get("origin_ui_session_id", ""),
  1034. "origin_session_id": snapshot.get("origin_session_id", ""),
  1035. "parent_session_id": snapshot.get("parent_session_id"),
  1036. "goal": snapshot.get("goal", ""), "goals": snapshot.get("goals"), "context": snapshot.get("context"),
  1037. "toolsets": snapshot.get("toolsets"), "role": snapshot.get("role"), "model": snapshot.get("model"),
  1038. "status": "running", "dispatched_at": snapshot.get("dispatched_at") or time.time(), "completed_at": time.time(),
  1039. **{k: snapshot[k] for k in _ROUTING_KEYS if snapshot.get(k)}}
  1040. try:
  1041. process_registry.completion_queue.put(evt)
  1042. except Exception as exc: # pragma: no cover
  1043. logger.error("Async delegation batch %s: failed to enqueue task failure notice: %s", delegation_id, exc)
  1044. # ── Stale monitor ───────────────────────────────────────────────────────────
  1045. def _ensure_stale_monitor() -> None:
  1046. """Start (once) the stale-delegation monitor thread. One daemon thread serves
  1047. every dispatch; it exits when no monitorable records remain and is restarted
  1048. by the next dispatch with a ``progress_fn``."""
  1049. global _monitor_thread
  1050. with _monitor_lock:
  1051. if _monitor_thread is not None and _monitor_thread.is_alive():
  1052. return
  1053. _monitor_stop.clear()
  1054. _monitor_thread = threading.Thread(
  1055. target=_stale_monitor_loop, name="async-delegate-stale-monitor", daemon=True)
  1056. _monitor_thread.start()
  1057. def _sweep_stale_locked(now: float):
  1058. """One monitor pass over ``_records``; caller holds ``_records_lock``. Returns
  1059. ``(stalled, expired, any_monitorable)``: newly-stalling ``(delegation_id, quiet_for, in_tool)``
  1060. tuples, stalling ids past the grace window, and whether anything is left to monitor."""
  1061. stalled, expired, any_monitorable = [], [], False # (delegation_id, quiet_for, in_tool) / ids past grace
  1062. for record in _records.values():
  1063. status = record.get("status")
  1064. if status == "stalling":
  1065. any_monitorable = True
  1066. if now - (record.get("_interrupted_at") or now) >= _STALL_GRACE_SECONDS:
  1067. expired.append(record["delegation_id"])
  1068. continue
  1069. progress_fn = record.get("progress_fn")
  1070. if status != "running" or progress_fn is None:
  1071. continue
  1072. any_monitorable = True
  1073. if not record.get("_started"):
  1074. continue # queued behind a full pool: not stalled, but keep the monitor alive for when it starts
  1075. try:
  1076. token, in_tool = progress_fn()
  1077. except Exception:
  1078. # An unreadable child must not look permanently healthy —
  1079. # keep the last timestamp running instead of refreshing it.
  1080. token, in_tool = record.get("_progress_token"), False
  1081. if token != record.get("_progress_token"):
  1082. record.update(_progress_token=token, _progress_ts=now)
  1083. continue
  1084. quiet_for = now - (record.get("_progress_ts") or now)
  1085. limit = _STALE_IN_TOOL_SECONDS if in_tool else _STALE_IDLE_SECONDS
  1086. if quiet_for >= limit:
  1087. # Stall context feeds the terminal event and status listings.
  1088. record.update(
  1089. status="stalling", _interrupted_at=now, _stall_quiet_seconds=round(quiet_for, 2),
  1090. _stall_threshold_seconds=limit, _stall_in_tool=bool(in_tool))
  1091. stalled.append((record["delegation_id"], quiet_for, in_tool))
  1092. return stalled, expired, any_monitorable
  1093. def _call_interrupt(fn, msg: str, *args) -> bool:
  1094. """Invoke an ``interrupt_fn``; True on success, else debug-log ``msg`` (+ exc)."""
  1095. if not callable(fn):
  1096. return False
  1097. try:
  1098. fn()
  1099. return True
  1100. except Exception as exc:
  1101. logger.debug(msg, *args, exc)
  1102. return False
  1103. def _stale_monitor_loop() -> None:
  1104. """Sweep running delegations for stalled progress. A changed progress token refreshes the
  1105. record's timestamp; a frozen token past the idle/in-tool threshold marks the record
  1106. ``stalling`` and calls ``interrupt_fn``; a ``stalling`` record still unreturned after the
  1107. grace window is force-finalized with a terminal ``stalled`` event."""
  1108. while not _monitor_stop.wait(_STALE_CHECK_INTERVAL):
  1109. now = time.time()
  1110. with _records_lock:
  1111. stalled, expired, any_monitorable = _sweep_stale_locked(now)
  1112. for delegation_id, quiet_for, in_tool in stalled:
  1113. logger.warning("Async delegation %s made no progress for %.0fs "
  1114. "(in_tool=%s) — interrupting; grace window %.0fs",
  1115. delegation_id, quiet_for, in_tool, _STALL_GRACE_SECONDS)
  1116. with _records_lock:
  1117. fn = (_records.get(delegation_id) or {}).get("interrupt_fn")
  1118. _call_interrupt(fn, "Async delegation %s stall interrupt failed: %s", delegation_id)
  1119. for delegation_id in expired:
  1120. with _records_lock:
  1121. ctx = (_records.get(delegation_id) or {}).get("_context") or contextvars.copy_context()
  1122. ctx.run(_finalize, delegation_id, lambda rec, d=delegation_id: _stalled_result(d, rec), "stalled")
  1123. if not any_monitorable:
  1124. return
  1125. def _stalled_error_text(event_record: dict[str, Any]) -> str:
  1126. """Human wording for a force-finalized stall. This string reaches the user (CLI timeline, Desktop
  1127. async-result card), so it names the task, how long it was silent, and what to do — no issue
  1128. numbers or worker internals (those stay in the log line and the stall_* metadata)."""
  1129. goal = " ".join(str(event_record.get("goal") or "").split())
  1130. label = f'Background task "{goal[:120]}"' if goal else "The background task"
  1131. quiet = float(event_record.get("_stall_quiet_seconds") or 0)
  1132. silence = f" after {round(quiet / 60)} min of no progress" if quiet >= 60 else ""
  1133. return (f"{label} stopped responding{silence} and was cancelled. Nothing else was affected; "
  1134. "ask me to run it again if you still need it.")
  1135. def _stalled_result(delegation_id: str, event_record: dict[str, Any]) -> dict[str, Any]:
  1136. """Synthetic terminal result for a stalling delegation whose runner never returned."""
  1137. completed_at = event_record.get("completed_at") or time.time()
  1138. duration = round(completed_at - (event_record.get("dispatched_at") or completed_at), 2)
  1139. error = _stalled_error_text(event_record)
  1140. logger.error("Async delegation %s force-finalized as stalled after %.0fs", delegation_id, duration)
  1141. # Structured stall metadata lets parents/UIs distinguish a stall-monitor
  1142. # kill from other failures without parsing the error string.
  1143. stall_in_tool = event_record.get("_stall_in_tool")
  1144. stall_meta = {
  1145. "stalled_after_quiet_seconds": event_record.get("_stall_quiet_seconds"),
  1146. "stall_threshold_seconds": event_record.get("_stall_threshold_seconds"),
  1147. "stall_phase": "in_tool" if stall_in_tool else "idle" if stall_in_tool is not None else None,
  1148. "stall_grace_seconds": _STALL_GRACE_SECONDS}
  1149. if event_record.get("is_batch"):
  1150. return {**_batch_crash(error, duration), **stall_meta}
  1151. return {**_single_crash(error, duration), "status": "stalled", "exit_reason": "stalled", **stall_meta}
  1152. # ── Observability + control ─────────────────────────────────────────────────
  1153. def _children_activity_from_token(token: Any, now: float) -> Optional[list]:
  1154. """Parse a progress token into per-child activity dicts (best-effort): delegate_tool
  1155. emits one ``(api_call_count, current_tool, last_activity_ts)`` tuple per child;
  1156. foreign token shapes degrade to ``None`` entries."""
  1157. try:
  1158. parts = list(token)
  1159. except TypeError:
  1160. return None
  1161. out: list[Optional[dict[str, Any]]] = []
  1162. for part in parts:
  1163. if not (isinstance(part, (list, tuple)) and len(part) >= 2):
  1164. out.append(None)
  1165. continue
  1166. entry: dict[str, Any] = {"api_calls": part[0], "current_tool": part[1]}
  1167. if len(part) >= 3 and isinstance(part[2], (int, float)):
  1168. entry["seconds_since_activity"] = round(max(0.0, now - float(part[2])), 1)
  1169. out.append(entry)
  1170. return out
  1171. def list_async_delegations() -> list[dict[str, Any]]:
  1172. """Snapshot of async delegations (running + recently completed) without callables or private
  1173. monitor bookkeeping; adds computed live fields for UIs (``seconds_since_progress``,
  1174. ``children_activity``/``in_tool`` sampled from ``progress_fn``) and stall context once tripped.
  1175. Safe to call from any thread. See #51690.
  1176. """
  1177. now = time.time()
  1178. samplers: dict[str, Callable] = {}
  1179. with _records_lock:
  1180. items = []
  1181. for r in _records.values():
  1182. item = {k: v for k, v in r.items() if k not in {"interrupt_fn", "progress_fn"} and not k.startswith("_")}
  1183. status = r.get("status")
  1184. if status in _ACTIVE_STATES:
  1185. if r.get("_progress_ts"):
  1186. item["seconds_since_progress"] = round(now - r["_progress_ts"], 1)
  1187. if callable(r.get("progress_fn")):
  1188. samplers[r["delegation_id"]] = r["progress_fn"]
  1189. if status in ("stalling", "stalled"):
  1190. for src, dst in _STALL_FIELD_MAP:
  1191. if r.get(src) is not None:
  1192. item[dst] = r.get(src)
  1193. items.append(item)
  1194. # Sample OUTSIDE the lock — progress_fn reads child-agent attributes and a
  1195. # slow/broken sampler must not block every dispatch/finalize.
  1196. for item in items:
  1197. fn = samplers.get(item.get("delegation_id"))
  1198. if fn is None:
  1199. continue
  1200. try:
  1201. token, in_tool = fn()
  1202. except Exception:
  1203. continue
  1204. activity = _children_activity_from_token(token, now)
  1205. if activity is not None:
  1206. item["children_activity"] = activity
  1207. item["in_tool"] = bool(in_tool)
  1208. return items
  1209. def _interrupt_records(targets: list[dict[str, Any]], caller: str, reason: str, msg: str) -> int:
  1210. """Call ``interrupt_fn`` on each record; log ``msg`` once; returns how many succeeded."""
  1211. count = sum(
  1212. _call_interrupt(r.get("interrupt_fn"), "%s: %s interrupt failed: %s", caller, r.get("delegation_id"))
  1213. for r in targets)
  1214. if count:
  1215. logger.info(msg, count, reason)
  1216. return count
  1217. def interrupt_all(reason: str = "shutdown") -> int:
  1218. """Signal every running async delegation to stop (``/stop``, shutdown). Returns how
  1219. many. The child still emits a completion event (status='interrupted') via the
  1220. normal finalize path."""
  1221. with _records_lock:
  1222. targets = [r for r in _records.values() if r.get("status") in _ACTIVE_STATES]
  1223. return _interrupt_records(targets, "interrupt_all", reason, "Interrupted %d async delegation(s) (%s)")
  1224. def interrupt_for_session(
  1225. session_key: str = "", origin_ui_session_id: str = "", parent_session_id: str = "", reason: str = "session_end",
  1226. ) -> int:
  1227. """Signal running async delegations owned by ONE ending session to stop (any
  1228. selector matches, see ``_session_records``). Returns how many."""
  1229. targets = _session_records(_ACTIVE_STATES, session_key, origin_ui_session_id, parent_session_id)
  1230. return _interrupt_records(
  1231. targets, "interrupt_for_session", reason, "Interrupted %d async delegation(s) for ending session (%s)")
  1232. def refuse_for_session(token: Any, *, session_key: str = "", parent_session_ids=()) -> None:
  1233. """Refuse new dispatches for ONE closing session (any selector matches) until ``allow_for_session``.
  1234. Set before ``interrupt_for_session``: a dispatch registered first is interrupted, a later one refused."""
  1235. fence = {("session_key", session_key)} if session_key else set()
  1236. fence.update(("parent_session_id", parent) for parent in parent_session_ids if parent)
  1237. with _records_lock:
  1238. _unfence(token)
  1239. _refused[token] = frozenset(fence)
  1240. for selector in fence:
  1241. _refused_by.setdefault(selector, set()).add(token)
  1242. def allow_for_session(token: Any) -> None:
  1243. """Lift ONE session's refusal: a viewer attached again, or the owner retired its live state."""
  1244. with _records_lock:
  1245. _unfence(token)
  1246. def _unfence(token: Any) -> None:
  1247. """Drop *token*'s refusal; caller holds ``_records_lock``."""
  1248. for selector in _refused.pop(token, ()):
  1249. tokens = _refused_by[selector]
  1250. tokens.discard(token)
  1251. if not tokens:
  1252. del _refused_by[selector]
  1253. def _reset_for_tests() -> None:
  1254. """Test-only: clear all state and tear down the executor + monitor."""
  1255. global _executor, _executor_max_workers, _monitor_thread
  1256. with _executor_lock:
  1257. if _executor is not None:
  1258. _executor.shutdown(wait=False)
  1259. _executor = None
  1260. _executor_max_workers = 0
  1261. _monitor_stop.set()
  1262. with _monitor_lock:
  1263. thread, _monitor_thread = _monitor_thread, None
  1264. if thread is not None and thread.is_alive():
  1265. thread.join(timeout=2)
  1266. with _records_lock:
  1267. _records.clear()
  1268. _refused.clear()
  1269. _refused_by.clear()
  1270. with _orphan_lock:
  1271. _offered.clear()
  1272. _startup_replayed.clear()
  1273. _last_orphan_sweep.clear()
  1274. _reoffered_at.clear()
  1275. with _held_lock:
  1276. _held_copies.clear()