| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449 |
- #!/usr/bin/env python3
- """Async (background) delegation registry behind ``delegate_task(background=true)``.
- The parent dispatches a subagent on a module-level daemon executor and returns a handle
- immediately. On completion a ``type="async_delegation"`` event (self-contained task-source
- block) is pushed onto the SHARED ``process_registry.completion_queue`` the CLI/gateway drain
- while idle, so results surface as a NEW turn (never mid-turn) and inherit its de-dup and
- crash-recovery wiring. Only the async lifecycle lives here; the child run is an injected ``runner``."""
- from __future__ import annotations
- import collections
- import contextvars
- import json
- import logging
- import os
- import sqlite3
- import threading
- import time
- import uuid
- from concurrent.futures import ThreadPoolExecutor
- from typing import Any, Callable, Dict, List, Optional
- from hermes_constants import get_hermes_home, hermes_home_key
- from tools.daemon_pool import DaemonThreadPoolExecutor
- from tools.thread_context import propagate_context_to_thread
- from tools.async_delegation_worker import worker_ledger, require_ledger_owner
- from tools.delegation_redaction import redact_child_result, redact_completion_event
- logger = logging.getLogger(__name__)
- # ── Module-level state ──────────────────────────────────────────────────────
- # Persistent daemon executor (never a `with ThreadPoolExecutor()` block, which
- # would join on exit and defeat async); daemon workers can't hang a hard exit.
- _executor: Optional[ThreadPoolExecutor] = None
- _executor_lock = threading.Lock()
- _executor_max_workers: int = 0
- _records_lock = threading.Lock()
- # delegation_id -> record dict; kept for the run plus a short completed tail.
- _records: dict[str, dict[str, Any]] = {}
- # A closed session's token -> the (field, value) selectors a new dispatch is refused for (``refuse_for_session``),
- # and each selector -> the tokens refusing it, so a dispatch checks its own selectors, not every fence.
- _refused: dict[Any, frozenset] = {}
- _refused_by: dict[tuple, set] = {}
- _FENCED_FIELDS = ("session_key", "parent_session_id")
- _DEFAULT_MAX_ASYNC_CHILDREN = 3
- # Completed records retained (in memory and in the ledger) for status queries.
- _MAX_RETAINED_COMPLETED = 50
- _DURABLE_RETENTION_SECONDS = 7 * 24 * 60 * 60
- _MAX_DURABLE_PENDING = 1000
- # Cap retried deliveries so an unroutable row converges to terminal 'dropped'.
- _MAX_DELIVERY_ATTEMPTS = 8
- # Pending completions older than this are dropped on restart replay instead of
- # re-run as a full-context turn; 48h keeps weekend results deliverable.
- _MAX_COMPLETION_REPLAY_AGE_S = 48 * 3600.0
- # A delivery claim older than this is abandoned and may be re-claimed.
- _CLAIM_LEASE_S = 300.0
- _DB_LOCK = threading.Lock()
- # ── Orphaned-completion sweep ────────────────────────────────────────────────
- # Startup replay runs once per process, so a completion whose owner died while THIS process was
- # already running (a desktop reload) would wait for the next restart (#97202). Delivery loops (gateway
- # watcher, TUI poller) sweep each home they serve at most once per interval.
- ORPHAN_SWEEP_INTERVAL_S = 30.0
- # Idle time before a dead owner's pending row is re-offered; keeps the sweep off a row just touched.
- _ORPHAN_STALE_S = 60.0
- _orphan_lock = threading.Lock()
- # (home key, delegation_id) put on this process's queue by replay or sweep and not re-offered while
- # that copy is alive. A consumer that discards its copy with the row still pending hands it back
- # (``return_completion_offer``); the delivery claim stays the only thing that settles the row.
- _offered: set = set()
- # Delegation ids this process first offered from its startup replay. Their copies carry
- # ``startup_replay`` on the sweep's later re-offers too, so a refused one stays gated.
- _startup_replayed: set = set()
- _last_orphan_sweep: dict[str, float] = {}
- # (home key, delegation_id) -> when the sweep last re-offered a lost copy. A consumer that keeps
- # handing the copy back would otherwise be re-offered every tick: re-offering touches no ledger
- # column, so each re-offer gets its own ``_CLAIM_LEASE_S`` here, and only the first one warns.
- _reoffered_at: dict[tuple[str, str], float] = {}
- # delegation_id -> number of copies a consumer holds off-queue (``mark_copy_held``). The sweep counts
- # them as live alongside the queued ones, so it never adds a duplicate beside a held copy.
- _held_copies: collections.Counter = collections.Counter()
- _held_lock = threading.Lock() # taken after a queue's mutex, never before
- # ── Stale-delegation detection (progress-based, on by default) ──────────────
- # A runner wedged before returning never reaches its finalizer, so it would show
- # "dispatched" forever. No wall-clock timeout (heavy work must never be killed for
- # taking long): one monitor thread samples per-dispatch PROGRESS via an injected
- # ``progress_fn``; a frozen child is interrupted, given a grace window to unwind via
- # the normal finalize path, and only force-finalized (terminal ``stalled`` event) if
- # it never returns. Thresholds mirror delegate_tool's sync heartbeat monitor.
- _STALE_CHECK_INTERVAL = 30.0
- _STALE_IDLE_SECONDS = 450.0
- _STALE_IN_TOOL_SECONDS = 1200.0
- _STALL_GRACE_SECONDS = 120.0
- _monitor_lock = threading.Lock()
- _monitor_thread: Optional[threading.Thread] = None
- _monitor_stop = threading.Event()
- _LIVE_STATES = {"running", "stalling", "finalizing"}
- _ACTIVE_STATES = ("running", "stalling")
- # Routing origin persisted at dispatch so a restart-recovered completion can
- # reconstruct a full SessionSource (scope_id drives relay tenant egress).
- _ROUTING_KEYS = ("scope_id", "user_id", "user_name")
- # Structured stall metadata — additive, present only on stall finalizations.
- _STALL_META_KEYS = ("stalled_after_quiet_seconds", "stall_threshold_seconds", "stall_phase", "stall_grace_seconds")
- # Private stall bookkeeping on the record -> public field in list_async_delegations().
- _STALL_FIELD_MAP = (("_stall_quiet_seconds", "stalled_after_quiet_seconds"),
- ("_stall_threshold_seconds", "stall_threshold_seconds"), ("_stall_in_tool", "stall_in_tool"))
- # ── Durable ledger (state.db / async_delegations) ───────────────────────────
- def _db_path():
- return get_hermes_home() / "state.db"
- def _connect() -> sqlite3.Connection:
- from agent.runtime_session_store import WorkerPersistenceError, is_worker_process
- if is_worker_process():
- raise WorkerPersistenceError('worker_delegation_ledger_unavailable')
- from hermes_cli.sqlite_util import open_db
- # Same state.db as hermes_state.SessionDB -- reuse its owner-only (0600)
- # hardening so this writer doesn't create/leave the file (and its WAL
- # sidecars) at the process umask. See hermes_state._secure_state_db_files.
- from hermes_constants import mkdir_under_hermes_home
- from hermes_state import _secure_state_db_files
- path = _db_path()
- # A late replay or writer must not resurrect a removed named profile (#123265).
- mkdir_under_hermes_home(path.parent)
- _secure_state_db_files(path, create_main=True)
- # wal=False: SessionDB owns state.db's journal mode (_initialize_schema applies the barriers).
- conn = open_db(path, db_label="state.db (async_delegation)", busy_timeout_ms=10_000,
- wal=False, row_factory=None, initialize=_initialize_schema)
- _secure_state_db_files(path)
- return conn
- def _initialize_schema(conn: sqlite3.Connection) -> None:
- from hermes_state_repair import apply_durability_barriers
- from hermes_state_schema import reconcile_state_schema
- # Preserve the journal mode SessionDB configured on state.db: forcing WAL from
- # every short-lived connection collides with live transcript/FTS writers.
- apply_durability_barriers(conn)
- # Single durable-shape authority: the canonical SCHEMA_SQL drives both
- # table creation and column backfill (reconcile_state_schema replays the
- # canonical DDL and reuses SessionDB's declarative reconciliation). This
- # module previously carried its own CREATE TABLE + ALTER column list,
- # which drifted from SCHEMA_SQL — same-name columns with different
- # nullability/defaults depending on which authority touched the database
- # first (#94691).
- reconcile_state_schema(conn)
- def _transaction():
- from hermes_cli.sqlite_util import transaction
- return transaction(_connect())
- def _capture_routing_origin() -> dict[str, Any]:
- """Snapshot scope_id/user_id/user_name on the PARENT thread (the daemon worker
- has no contextvars) so a restart-replayed completion can rebuild a SessionSource.
- Best-effort: empty values are omitted."""
- try:
- from gateway.session_context import get_session_env
- return {k: v for k in _ROUTING_KEYS if (v := get_session_env(f"HERMES_SESSION_{k.upper()}", ""))}
- except Exception:
- return {}
- def _persist_dispatch(record: dict[str, Any]) -> None:
- if remote := worker_ledger():
- return remote.dispatch(record)
- now = time.time()
- try:
- from gateway.status import get_process_start_time
- owner_started_at = get_process_start_time(os.getpid())
- except Exception:
- owner_started_at = None
- task_payload = {
- key: record.get(key)
- for key in ("goal", "goals", "context", "toolsets", "role", "model", "is_batch", "task_indexes", "task_transcripts", *_ROUTING_KEYS)
- if key in record}
- try: # where the children's terminals started; lets recovery add a git-state hint
- task_payload["owner_cwd"] = os.getcwd()
- except OSError:
- pass
- with _DB_LOCK, _transaction() as conn:
- conn.execute("""INSERT OR REPLACE INTO async_delegations
- (delegation_id, origin_session, origin_ui_session_id,
- parent_session_id, state, dispatched_at, updated_at,
- delivery_state, delivery_attempts, owner_pid,
- owner_started_at, task_json, origin_session_id)
- VALUES (?, ?, ?, ?, 'running', ?, ?, 'pending', 0, ?, ?, ?, ?)""",
- (record["delegation_id"], record.get("session_key", ""), record.get("origin_ui_session_id", ""),
- record.get("parent_session_id"), record["dispatched_at"], now, os.getpid(), owner_started_at,
- json.dumps(task_payload), record.get("origin_session_id", "")))
- _prune_durable_records()
- def _prune_durable_records() -> None:
- """Bound terminal history, preferring delivered records for deletion."""
- require_ledger_owner()
- cutoff = time.time() - _DURABLE_RETENTION_SECONDS
- with _DB_LOCK, _transaction() as conn:
- conn.execute(
- "DELETE FROM async_delegations WHERE delivery_state='delivered' AND updated_at < ?", (cutoff,))
- terminal_count = conn.execute(
- "SELECT COUNT(*) FROM async_delegations WHERE state NOT IN ('running','finalizing')").fetchone()[0]
- if terminal_count > _MAX_RETAINED_COMPLETED:
- conn.execute("""DELETE FROM async_delegations WHERE delegation_id IN (
- SELECT delegation_id FROM async_delegations
- WHERE state NOT IN ('running','finalizing')
- ORDER BY CASE delivery_state WHEN 'delivered' THEN 0 ELSE 1 END,
- updated_at ASC LIMIT ?
- )""", (terminal_count - _MAX_RETAINED_COMPLETED,))
- pending_count = conn.execute("""SELECT COUNT(*) FROM async_delegations
- WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'""").fetchone()[0]
- if pending_count > _MAX_DURABLE_PENDING:
- conn.execute("""DELETE FROM async_delegations WHERE delegation_id IN (
- SELECT delegation_id FROM async_delegations
- WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'
- ORDER BY updated_at ASC LIMIT ?
- )""", (pending_count - _MAX_DURABLE_PENDING,))
- def _persist_completion(event: dict[str, Any], result: dict[str, Any]) -> None:
- event, result = redact_completion_event(event), redact_child_result(result)
- if remote := worker_ledger():
- return remote.complete(event, result)
- now = time.time()
- with _DB_LOCK, _transaction() as conn:
- conn.execute("""UPDATE async_delegations SET state=?, completed_at=?, updated_at=?,
- event_json=?, result_json=?, delivery_state='pending'
- WHERE delegation_id=?""",
- (event.get("status", "completed"), event.get("completed_at", now), now,
- json.dumps(event), json.dumps(result), event["delegation_id"]))
- def record_unit_child(delegation_id: str, entry: dict[str, Any]) -> None:
- """Durably record ONE finished child of a still-running multi-child unit on the unit's own row, so a crash before
- the unit joins loses only the children that had not finished. Stored in ``result_json`` (overwritten by the real
- result at finalize); ``recover_abandoned_delegations`` replays it. Best-effort: a failed write costs recovery
- fidelity, never the live result."""
- entry = redact_child_result(entry)
- if remote := worker_ledger():
- return remote.child(delegation_id, entry)
- try:
- with _DB_LOCK, _transaction() as conn:
- row = conn.execute("SELECT result_json FROM async_delegations WHERE delegation_id=? AND state='running'",
- (delegation_id,)).fetchone()
- if row is None:
- return
- partial = redact_child_result(json.loads(row[0] or "{}") or {})
- results = [r for r in partial.get("results") or [] if r.get("task_index") != entry.get("task_index")]
- results.append(entry)
- conn.execute("UPDATE async_delegations SET result_json=?, updated_at=? WHERE delegation_id=? AND state='running'",
- (json.dumps({"results": results, "partial": True}), time.time(), delegation_id))
- except Exception:
- logger.warning("Async delegation %s: could not record finished child %s", delegation_id, entry.get("task_index"), exc_info=True)
- def _recovered_results(task: dict[str, Any], result_json: Optional[str], error: str) -> Optional[list[dict[str, Any]]]:
- """Per-task results for an abandoned unit: recorded children as they finished, the rest ``unknown``."""
- partial = json.loads(result_json or "{}") or {}
- if not (task.get("is_batch") and partial.get("partial") and partial.get("results")):
- return None
- recorded = {r["task_index"]: r for r in partial["results"] if isinstance(r.get("task_index"), int)}
- indexes = task.get("task_indexes") or list(range(len(task.get("goals") or [])))
- return [recorded.get(i) or {"task_index": i, "status": "unknown", "summary": None, "error": error} for i in indexes]
- def _owner_liveness() -> Optional[Callable[[Any, Any], bool]]:
- """``alive(owner_pid, owner_started_at)`` over the shared drift-tolerant start-time comparator,
- or None when the liveness probes cannot be imported."""
- try:
- from gateway.status import _pid_exists, get_process_start_time, start_time_fingerprints_match
- except Exception:
- return None
- def alive(pid, started) -> bool:
- return bool(pid) and _pid_exists(int(pid)) and (
- started is None or start_time_fingerprints_match(started, get_process_start_time(int(pid)) or 0))
- return alive
- def recover_abandoned_delegations() -> int:
- """Classify records whose owning process disappeared as outcome unknown; children a multi-child unit had already
- recorded (``record_unit_child``) are replayed with their real results."""
- require_ledger_owner()
- alive = _owner_liveness()
- if alive is None:
- return 0
- now, recovered = time.time(), 0
- with _DB_LOCK, _transaction() as conn:
- rows = conn.execute("""SELECT delegation_id, origin_session, origin_ui_session_id,
- parent_session_id, dispatched_at, owner_pid,
- owner_started_at, task_json, origin_session_id, result_json, state
- FROM async_delegations WHERE state IN ('running','finalizing')""").fetchall()
- for row in rows:
- delegation_id, session_key, origin_ui, parent_id, dispatched_at, pid, started, task_json, origin_sid, result_json, last_state = row
- if alive(pid, started):
- continue
- task = json.loads(task_json or "{}")
- error = "Delegation owner exited before recording a terminal result; outcome unknown."
- recovered_results = _recovered_results(task, result_json, error)
- if recovered_results:
- done = sum(1 for r in recovered_results if r.get("status") != "unknown")
- error = (f"Delegation owner exited before the unit finished; {done}/{len(recovered_results)} child "
- "results were recorded and are included below, the rest are unknown.")
- diagnostics = {"last_known_status": last_state, "task_transcripts": task.get("task_transcripts") or {}}
- # Verbatim transcript tails + a git snapshot of the owner's cwd, so the parent can
- # continue or re-dispatch from the event alone instead of opening files (#116000).
- from tools.async_delegation_recovery_hints import git_state_hint, transcript_tails
- if tails := transcript_tails(diagnostics["task_transcripts"]):
- diagnostics["transcript_tails"] = tails
- if hint := git_state_hint(task.get("owner_cwd")):
- diagnostics["git_state_hint"] = hint
- event = {
- "type": "async_delegation", "delegation_id": delegation_id, "session_key": session_key,
- "origin_ui_session_id": origin_ui, "origin_session_id": origin_sid or "",
- "parent_session_id": parent_id, "goal": task.get("goal", ""), "goals": task.get("goals"),
- "context": task.get("context"), "toolsets": task.get("toolsets"), "role": task.get("role"),
- "model": task.get("model"), "is_batch": bool(task.get("is_batch")),
- "status": "unknown", "summary": None, "error": error, **diagnostics,
- **({"results": recovered_results} if recovered_results else {}),
- "dispatched_at": dispatched_at, "completed_at": now,
- **{k: task[k] for k in _ROUTING_KEYS if task.get(k)}}
- result = {"status": "unknown", "summary": None, "error": event["error"], **diagnostics,
- **({"results": recovered_results} if recovered_results else {})}
- event, result = redact_completion_event(event), redact_child_result(result)
- conn.execute("""UPDATE async_delegations SET state='unknown', completed_at=?,
- updated_at=?, event_json=?, result_json=?, delivery_state='pending'
- WHERE delegation_id=?""", (now, now, json.dumps(event), json.dumps(result), delegation_id))
- recovered += 1
- return recovered
- def restore_undelivered_completions(target_queue) -> int:
- """Enqueue durable pending completions as fresh turns after process start.
- Restored events are stamped ``restored=True`` in memory only: they came from a PREVIOUS
- process, so drains without an ownership filter must leave them for a consumer that can
- prove ownership. Rows older than ``_MAX_COMPLETION_REPLAY_AGE_S`` are terminally dropped
- instead of replaying a turn nobody is waiting on.
- Every restored event is stamped ``restored=True`` (in-memory only — the stamp is added after the durable
- payload is deserialized and is never persisted). Restored events originate from a *previous* process, so
- no consumer in THIS process implicitly owns them: drain paths that run without an ownership filter (the
- legacy single-session behavior) must leave them queued for a consumer that can positively prove
- ownership, otherwise a brand-new session adopts a dead session's delegation results seconds after boot
- (#64484).
- """
- from agent.runtime_session_store import is_worker_process
- if is_worker_process():
- # The ordinary owner restores delivery-only results; a compute import
- # must neither reap that owner's ledger nor steal its delivery queue.
- return 0
- if not _db_path().exists():
- return 0 # nothing to replay; a replay must not create (or migrate) the ledger (#123265)
- recover_abandoned_delegations()
- now = time.time()
- with _DB_LOCK, _transaction() as conn:
- rows = conn.execute("""SELECT delegation_id, event_json, completed_at, dispatched_at
- FROM async_delegations
- WHERE state != 'running' AND delivery_state='pending' AND event_json IS NOT NULL
- ORDER BY completed_at, delegation_id""").fetchall()
- return _replay_pending(conn, rows, target_queue, now, startup=True)
- def _replay_pending(conn, rows, target_queue, now: float, *, startup: bool = False) -> int:
- """Put each pending ``(delegation_id, event_json, completed_at, dispatched_at)`` row on ``target_queue``
- stamped ``restored``, or terminally drop it past ``_MAX_COMPLETION_REPLAY_AGE_S``. Records the offer so
- the orphan sweep skips the row until the copy is handed back (``return_completion_offer``)."""
- home, restored = hermes_home_key(get_hermes_home()), 0
- for delegation_id, payload, completed_at, dispatched_at in rows:
- age_basis = completed_at or dispatched_at
- if age_basis and (now - age_basis) > _MAX_COMPLETION_REPLAY_AGE_S:
- conn.execute("""UPDATE async_delegations SET delivery_state='dropped',
- delivery_claim=NULL, delivery_claimed_at=NULL,
- updated_at=?
- WHERE delegation_id=? AND delivery_state='pending'""", (now, delegation_id))
- logger.warning("Async delegation %s: pending completion is %.1fh old "
- "(cap %.1fh); terminally dropping the replay (result remains queryable).",
- delegation_id, (now - age_basis) / 3600.0, _MAX_COMPLETION_REPLAY_AGE_S / 3600.0)
- continue
- with _orphan_lock:
- if (home, delegation_id) in _offered:
- # The owner's explicit startup replay and the first-consume replay
- # (process_registry.restore_completions) may both run: offer once.
- continue
- evt = json.loads(payload)
- if isinstance(evt, dict):
- evt = redact_completion_event(evt)
- with _orphan_lock:
- if startup:
- _startup_replayed.add(delegation_id)
- if isinstance(evt, dict):
- evt["restored"] = True
- if delegation_id in _startup_replayed:
- evt["startup_replay"] = True
- target_queue.put(evt)
- with _orphan_lock:
- _offered.add((home, delegation_id))
- restored += 1
- return restored
- def completion_copy_id(evt: Any) -> str:
- """The delegation id ``evt`` is the final-result copy of, or "" (other events, interim notices)."""
- if not isinstance(evt, dict) or evt.get("type") != "async_delegation" or is_interim_delegation_event(evt):
- return ""
- return str(evt.get("delegation_id") or "")
- def mark_copy_held(delegation_id: str) -> None:
- """Record that a consumer holds a copy of ``delegation_id`` off every queue (a poller between dequeue and
- requeue, a parked copy). The orphan sweep treats a held copy like a queued one. A holder that moves a copy
- onto a queue marks before it dequeues, or under the queue's mutex, and unmarks only after any requeue, so
- the sweep never sees the copy as neither queued nor held. Pair every call with ``unmark_copy_held``."""
- if delegation_id:
- with _held_lock:
- _held_copies[delegation_id] += 1
- def unmark_copy_held(delegation_id: str) -> None:
- """End one ``mark_copy_held`` hold on ``delegation_id``."""
- if delegation_id:
- with _held_lock:
- _held_copies[delegation_id] -= 1
- if _held_copies[delegation_id] <= 0:
- del _held_copies[delegation_id]
- def _queued_completion_ids(*queues) -> Optional[set]:
- """Delegation ids with a final-result copy sitting in any of ``queues`` or held off-queue by a consumer
- (``mark_copy_held``), or None when a queue cannot be inspected. Each queue is read under its own mutex
- (the lock ``put``/``get`` and ``take_ready`` take) with the held ids read inside it, so a copy moving
- between a queue and a hold is always seen in one of them. Interim per-task notices share their batch's
- id but are not the result, so they never count as its copy."""
- ids: set = set()
- for q in {id(q): q for q in queues if q is not None}.values():
- mutex, items = getattr(q, "mutex", None), getattr(q, "queue", None)
- if mutex is None or items is None:
- return None
- with mutex:
- snapshot = list(items)
- with _held_lock:
- ids.update(_held_copies)
- ids.update(completion_copy_id(evt) for evt in snapshot)
- ids.discard("")
- return ids
- def _copy_may_be_live(delegation_id: str, pid, updated_at: float, live_ids: set, home: str,
- offered: set, now: float) -> bool:
- """For a row whose copy this process made (it owns the row or offered it): whether that copy may still be
- delivered, so the sweep must not add another. Live = queued or held (``live_ids``), or the row or this
- process's last re-offer was touched within a claim lease, the span after which a silent claim counts as
- abandoned. A row owned by ANOTHER live process is left to it: its queue is out of sight."""
- if not (delegation_id in offered or int(pid) == os.getpid()):
- return True
- if delegation_id in live_ids or updated_at >= now - _CLAIM_LEASE_S:
- return True
- with _orphan_lock:
- return _reoffered_at.get((home, delegation_id), float("-inf")) >= now - _CLAIM_LEASE_S
- def _note_reoffers(home: str, lost: list, now: float) -> None:
- """Start each lost copy's re-offer lease and log it: WARNING the first time a row is re-offered, DEBUG
- after that (a consumer that keeps handing the copy back would otherwise warn once per lease)."""
- with _orphan_lock:
- first = [d for d in lost if (home, d) not in _reoffered_at]
- for key in [k for k, at in _reoffered_at.items() if now - at > _MAX_COMPLETION_REPLAY_AGE_S]:
- del _reoffered_at[key] # its row is past the replay cap, so it was dropped
- _reoffered_at.update({(home, d): now for d in lost})
- _offered.difference_update({(home, d) for d in lost})
- log = logger.warning if first else logger.debug
- log("Re-offering %d pending async completion(s) whose in-memory copy was lost while this process ran: %s",
- len(lost), ", ".join(lost))
- def sweep_orphaned_completions(target_queue, *, now: Optional[float] = None) -> int:
- """Re-offer this home's pending completions that no live in-memory copy will deliver (#97202, #140718).
- Startup replay (``restore_undelivered_completions``) covers owners that died before the process
- started; this covers the rest while it runs. Abandoned in-flight rows are first classified by
- ``recover_abandoned_delegations``. A terminal row qualifies when it is pending with an event, idle
- past ``_ORPHAN_STALE_S`` and not under a live delivery claim, and then:
- - its owner is dead and this process has not offered it yet; or
- - its owner is THIS process, or this process already offered it, the row has been idle for a whole
- ``_CLAIM_LEASE_S`` and no copy of it sits in ``target_queue`` or the producers'
- ``completion_queue``: the one in-memory copy was lost (a consumer discarded it, or it went onto
- a queue no delivery loop drains), and without this the row would stay pending with zero
- attempts until a restart (#144563).
- The sweep never adds a copy beside a live one: queued, held off-queue by a consumer
- (``mark_copy_held``), under a claim lease (filtered above), or touched within a lease (a claim, defer
- or release refreshes ``updated_at``). Each re-offer starts its own lease, so a consumer that keeps
- handing the copy back sees it at most once per lease, and only the first re-offer warns. A row owned
- by ANOTHER live process is left to that process, whose queue this one cannot see. The consumer's
- ``claim_completion_delivery`` stays the atomic cross-process gate, so a duplicate copy can never
- deliver twice. Rows past the delivery budget or the replay age converge to ``dropped``. Reads the
- current profile's ledger: callers bind the owning profile first."""
- alive = _owner_liveness()
- if alive is None or not _db_path().exists():
- return 0 # never create a ledger just to sweep it
- recover_abandoned_delegations()
- now = time.time() if now is None else now
- home = hermes_home_key(get_hermes_home())
- from tools.process_registry import process_registry
- live = _queued_completion_ids(target_queue, process_registry.completion_queue)
- with _orphan_lock:
- offered = {delegation_id for key, delegation_id in _offered if key == home}
- with _DB_LOCK, _transaction() as conn:
- rows = conn.execute("""SELECT delegation_id, event_json, completed_at, dispatched_at,
- owner_pid, owner_started_at, delivery_attempts, updated_at
- FROM async_delegations
- WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'
- AND event_json IS NOT NULL AND updated_at < ?
- AND (delivery_claim IS NULL OR delivery_claimed_at < ?)
- ORDER BY completed_at, delegation_id""", (now - _ORPHAN_STALE_S, now - _CLAIM_LEASE_S)).fetchall()
- orphans, lost = [], []
- for delegation_id, payload, completed_at, dispatched_at, pid, started, attempts, updated_at in rows:
- copy_lost = alive(pid, started) or delegation_id in offered
- if copy_lost and (live is None or _copy_may_be_live(delegation_id, pid, updated_at, live, home,
- offered, now)):
- continue
- if (attempts or 0) >= _MAX_DELIVERY_ATTEMPTS:
- # Its last claimant died holding the final attempt; converge like release_completion_delivery.
- conn.execute("""UPDATE async_delegations SET delivery_state='dropped',
- delivery_claim=NULL, delivery_claimed_at=NULL, updated_at=?
- WHERE delegation_id=? AND delivery_state='pending'""", (now, delegation_id))
- logger.warning("Async delegation %s exhausted its %d delivery attempts; "
- "marking terminally dropped (result remains queryable).",
- delegation_id, _MAX_DELIVERY_ATTEMPTS)
- continue
- orphans.append((delegation_id, payload, completed_at, dispatched_at))
- if copy_lost:
- lost.append(delegation_id)
- if lost:
- _note_reoffers(home, lost, now)
- return _replay_pending(conn, orphans, target_queue, now)
- def maybe_sweep_orphaned_completions(target_queue, *, now: Optional[float] = None) -> int:
- """``sweep_orphaned_completions`` at most once per ``ORPHAN_SWEEP_INTERVAL_S`` per home (``now`` is
- monotonic), for delivery loops that tick far more often. Never raises into the loop."""
- home = hermes_home_key(get_hermes_home())
- now = time.monotonic() if now is None else now
- with _orphan_lock:
- last = _last_orphan_sweep.get(home)
- if last is not None and now - last < ORPHAN_SWEEP_INTERVAL_S:
- return 0
- _last_orphan_sweep[home] = now
- try:
- return sweep_orphaned_completions(target_queue)
- except Exception:
- logger.debug("Orphaned async delegation sweep failed", exc_info=True)
- return 0
- def _update_delivery(sql: str, params: tuple) -> bool:
- """Run one UPDATE on the ledger; True iff exactly one row changed."""
- with _DB_LOCK, _transaction() as conn:
- return conn.execute(sql, params).rowcount == 1
- def mark_completion_delivered(delegation_id: str) -> bool:
- """Atomically acknowledge successful injection of a durable completion."""
- require_ledger_owner()
- now = time.time()
- return _update_delivery(
- """UPDATE async_delegations SET delivery_state='delivered', delivered_at=?, updated_at=?
- WHERE delegation_id=? AND delivery_state!='delivered'""", (now, now, delegation_id))
- def claim_completion_delivery(delegation_id: str, claim_id: str) -> bool:
- """Claim one pending completion across competing consumers/processes."""
- if remote := worker_ledger():
- return remote.claim(delegation_id, claim_id)
- now = time.time()
- with _DB_LOCK, _transaction() as conn:
- row = conn.execute(
- "SELECT delivery_state FROM async_delegations WHERE delegation_id=?", (delegation_id,)).fetchone()
- if row is None:
- return True # legacy event created before durable dispatch
- cur = conn.execute("""UPDATE async_delegations SET delivery_claim=?, delivery_claimed_at=?,
- delivery_attempts=delivery_attempts+1, updated_at=?
- WHERE delegation_id=? AND delivery_state='pending'
- AND (delivery_claim IS NULL OR delivery_claimed_at < ?)""",
- (claim_id, now, now, delegation_id, now - _CLAIM_LEASE_S))
- return cur.rowcount == 1
- def is_interim_delegation_event(evt: dict[str, Any]) -> bool:
- """An early per-task notice for a batch that is still running. It shares the batch's
- ``delegation_id`` but is NOT the durable completion: it must never claim, acknowledge or
- dedup against the final result's row (independent review reproduced exactly that loss)."""
- return evt.get("type") == "async_delegation" and bool(evt.get("task_failure_notice"))
- def claim_event_delivery(evt: dict[str, Any], consumer: str) -> Optional[str]:
- """Claim a durable delegation event; non-durable events (and interim notices) need no token."""
- if is_interim_delegation_event(evt):
- return ""
- delegation_id = str(evt.get("delegation_id") or "") if evt.get("type") == "async_delegation" else ""
- if not delegation_id:
- return ""
- claim_id = f"{consumer}:{os.getpid()}:{uuid.uuid4().hex}"
- return claim_id if claim_completion_delivery(delegation_id, claim_id) else None
- def release_completion_delivery(delegation_id: str, claim_id: str) -> bool:
- """Release a failed delivery claim so another consumer may retry. Attempts are
- counted at claim time; once the budget is exhausted the row converges to
- terminal ``dropped`` (only pending rows replay on restart)."""
- if remote := worker_ledger():
- return remote.release(delegation_id, claim_id)
- now = time.time()
- with _DB_LOCK, _transaction() as conn:
- capped = conn.execute("""UPDATE async_delegations SET delivery_state='dropped',
- delivery_claim=NULL, delivery_claimed_at=NULL, updated_at=?
- WHERE delegation_id=? AND delivery_state='pending'
- AND delivery_claim=? AND delivery_attempts>=?""",
- (now, delegation_id, claim_id, _MAX_DELIVERY_ATTEMPTS))
- if capped.rowcount == 1:
- logger.warning("Async delegation %s exhausted its %d delivery attempts; "
- "marking terminally dropped (result remains queryable).",
- delegation_id, _MAX_DELIVERY_ATTEMPTS)
- return True
- cur = conn.execute("""UPDATE async_delegations SET delivery_claim=NULL,
- delivery_claimed_at=NULL, updated_at=?
- WHERE delegation_id=? AND delivery_state='pending'
- AND delivery_claim=?""", (now, delegation_id, claim_id))
- return cur.rowcount == 1
- def defer_completion_delivery(delegation_id: str, claim_id: str) -> bool:
- """Return an unadmitted completion to pending without spending a delivery attempt."""
- if remote := worker_ledger():
- return remote.defer(delegation_id, claim_id)
- return _update_delivery("""UPDATE async_delegations SET delivery_claim=NULL,
- delivery_claimed_at=NULL, delivery_attempts=MAX(0, delivery_attempts-1),
- updated_at=?
- WHERE delegation_id=? AND delivery_state='pending' AND delivery_claim=?""",
- (time.time(), delegation_id, claim_id))
- def drop_completion_delivery(delegation_id: str, claim_id: str) -> bool:
- """Terminally drop a claimed completion whose target is permanently gone (the
- spawning session ended at an explicit user boundary such as /new or reset).
- ``dropped`` — not ``delivered`` — keeps the ack honest; not ``pending`` keeps
- restart recovery from replaying it into a fail-closed drop forever."""
- if remote := worker_ledger():
- return remote.drop(delegation_id, claim_id)
- return _update_delivery("""UPDATE async_delegations SET delivery_state='dropped',
- updated_at=?, delivery_claim=NULL,
- delivery_claimed_at=NULL
- WHERE delegation_id=? AND delivery_state='pending'
- AND delivery_claim=?""", (time.time(), delegation_id, claim_id))
- def complete_completion_delivery(delegation_id: str, claim_id: str) -> bool:
- """Acknowledge acceptance for the consumer holding this claim."""
- if remote := worker_ledger():
- return remote.ack(delegation_id, claim_id)
- now = time.time()
- return _update_delivery("""UPDATE async_delegations SET delivery_state='delivered',
- delivered_at=?, updated_at=?, delivery_claim=NULL,
- delivery_claimed_at=NULL
- WHERE delegation_id=? AND delivery_state='pending'
- AND delivery_claim=?""", (now, now, delegation_id, claim_id))
- def requeue_admitted_completion(delegation_id: str, target_queue) -> bool:
- """Offer an acknowledged completion again when its receiving turn could not confirm the owner.
- The row returns to pending and its event goes back on ``target_queue``, so the next delivery
- re-runs the pre-flight. A row past the delivery budget converges to ``dropped`` instead, which
- bounds the loop; a row without a stored event is left as it is."""
- now = time.time()
- with _DB_LOCK, _transaction() as conn:
- exhausted = conn.execute("""UPDATE async_delegations SET delivery_state='dropped', updated_at=?
- WHERE delegation_id=? AND delivery_state='delivered' AND delivery_attempts>=?""",
- (now, delegation_id, _MAX_DELIVERY_ATTEMPTS))
- if exhausted.rowcount == 1:
- logger.warning("Async delegation %s exhausted its %d delivery attempts; "
- "marking terminally dropped (result remains queryable).",
- delegation_id, _MAX_DELIVERY_ATTEMPTS)
- return False
- reopened = conn.execute("""UPDATE async_delegations SET delivery_state='pending',
- delivered_at=NULL, updated_at=?
- WHERE delegation_id=? AND delivery_state='delivered' AND event_json IS NOT NULL""",
- (now, delegation_id))
- if reopened.rowcount != 1:
- return False
- rows = conn.execute("""SELECT delegation_id, event_json, completed_at, dispatched_at
- FROM async_delegations WHERE delegation_id=?""", (delegation_id,)).fetchall()
- with _orphan_lock: # an admitted copy already had its consumer: requeue keeps live routing
- _startup_replayed.discard(delegation_id)
- return _replay_pending(conn, rows, target_queue, now) == 1
- def complete_event_delivery(evt: dict[str, Any], claim_id: str) -> None:
- _event_delivery(complete_completion_delivery, evt, claim_id)
- def release_event_delivery(evt: dict[str, Any], claim_id: str) -> None:
- """Release a failed claim for a consumer that discards its copy (the TUI poller): the row is pending
- again, so it must stay eligible for the orphan sweep."""
- _event_delivery(release_completion_delivery, evt, claim_id)
- return_completion_offer(evt)
- def return_completion_offer(evt: dict[str, Any]) -> None:
- """Hand an offered completion back to the orphan sweep after its in-memory copy was discarded while
- the durable row stays pending, e.g. a TUI session that cannot prove it owns the event drops it (every
- session poller drains one process-wide queue). The next sweep may offer the row again. Delegation ids
- are unique across profiles, so this clears the offer in every home."""
- delegation_id = str(evt.get("delegation_id") or "") if evt.get("type") == "async_delegation" else ""
- if not delegation_id or is_interim_delegation_event(evt):
- return
- with _orphan_lock:
- _offered.difference_update({key for key in _offered if key[1] == delegation_id})
- def _event_delivery(fn, evt: dict[str, Any], claim_id: str) -> None:
- if claim_id and evt.get("type") == "async_delegation":
- fn(str(evt.get("delegation_id") or ""), claim_id)
- def get_durable_delegation(delegation_id: str) -> Optional[dict[str, Any]]:
- if remote := worker_ledger():
- saved = remote.read(delegation_id)
- if saved is not None and saved.get("result") is not None:
- return {**saved, "result": redact_child_result(saved["result"])}
- return saved
- with _DB_LOCK, _transaction() as conn:
- row = conn.execute("""SELECT origin_session, state, dispatched_at, completed_at,
- result_json, delivery_state, delivery_attempts,
- origin_session_id
- FROM async_delegations WHERE delegation_id=?""", (delegation_id,)).fetchone()
- return None if row is None else {
- "delegation_id": delegation_id, "origin_session": row[0], "state": row[1], "dispatched_at": row[2],
- "completed_at": row[3], "result": redact_child_result(json.loads(row[4])) if row[4] else None, "delivery_state": row[5],
- "delivery_attempts": row[6], "origin_session_id": row[7] or ""}
- _FAILED_TASK_STATES = frozenset({"error", "failed", "failure", "timeout", "stalled", "unknown", "interrupted"})
- _FAILURE_SURFACE_WINDOW_S = 24 * 3600.0
- def _json_object(raw: Optional[str]) -> dict[str, Any]:
- try:
- value = json.loads(raw) if raw else {}
- except ValueError:
- return {}
- return value if isinstance(value, dict) else {}
- def failed_delegations_for_session(
- origin_ui_session_id: str = "", parent_session_id: str = "", *, limit: int = 20, now: Optional[float] = None,
- ) -> list[dict[str, Any]]:
- """Recently failed async delegation tasks owned by a session, newest first.
- The live roster forgets a child once it ends and does not survive a renderer reload, so a failed
- delegation had nowhere to show (#97202). This reads the durable row instead: one entry per failed
- task (a batch unit that "completed" can still carry failed tasks) with ``delegation_id``,
- ``task_index``, ``goal``, ``status``, ``error``, ``dispatched_at`` and ``completed_at``. Either selector claims a row:
- the UI session id at dispatch, or the spawner's durable session id (survives a reload re-mint)."""
- selectors = [(col, val) for col, val in (
- ("origin_ui_session_id", origin_ui_session_id), ("parent_session_id", parent_session_id)) if val]
- if not selectors:
- return []
- cutoff = (now if now is not None else time.time()) - _FAILURE_SURFACE_WINDOW_S
- owner_sql = " OR ".join(f"{col}=?" for col, _ in selectors)
- with _DB_LOCK, _transaction() as conn:
- rows = conn.execute(
- f"""SELECT delegation_id, state, dispatched_at, completed_at, task_json, result_json FROM async_delegations
- WHERE ({owner_sql}) AND state NOT IN ('running','finalizing') AND completed_at >= ?
- ORDER BY completed_at DESC LIMIT ?""",
- (*(val for _, val in selectors), cutoff, limit)).fetchall()
- failed: list[dict[str, Any]] = []
- for delegation_id, state, dispatched_at, completed_at, task_json, result_json in rows:
- task, result = _json_object(task_json), redact_child_result(_json_object(result_json))
- goals = task.get("goals") if isinstance(task.get("goals"), list) and task["goals"] else [task.get("goal") or ""]
- goal_for = dict(zip(task.get("task_indexes") or range(len(goals)), goals))
- tasks = result["results"] if isinstance(result.get("results"), list) else [] if task.get("is_batch") else [result]
- if not tasks and str(state).lower() in _FAILED_TASK_STATES:
- tasks = [{"task_index": 0, "error": result.get("error")}]
- for entry in tasks:
- status = str(entry.get("status") or state or "").lower()
- if status not in _FAILED_TASK_STATES:
- continue
- index = entry.get("task_index") if isinstance(entry.get("task_index"), int) else 0
- error = entry.get("error") or result.get("error")
- failed.append({
- "delegation_id": delegation_id, "task_index": index, "status": status,
- "goal": str(goal_for.get(index, goals[0]) or ""), "error": str(error) if error else None,
- "dispatched_at": dispatched_at, "completed_at": completed_at})
- return failed[:limit]
- # ── In-memory registry queries ──────────────────────────────────────────────
- def _get_executor(max_workers: int) -> ThreadPoolExecutor:
- """Lazily create (or grow in place, never shrink) the shared daemon executor. Raising
- ``_max_workers`` is enough: the next ``submit`` spawns threads up to the new cap."""
- global _executor, _executor_max_workers
- with _executor_lock:
- if _executor is None:
- _executor = DaemonThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="async-delegate")
- _executor_max_workers = max_workers
- elif max_workers > _executor_max_workers:
- _executor._max_workers = max_workers
- _executor_max_workers = max_workers
- return _executor
- def active_count() -> int:
- """Number of live async delegation UNITS (one per completion message: a task group or an ungrouped task)."""
- with _records_lock:
- return sum(1 for r in _records.values() if r.get("status") in _LIVE_STATES)
- def active_task_count() -> int:
- """Number of running child subagents (a batch of N contributes N; a batch with
- no goal list counts 1) — the truthful observability figure, unlike slots."""
- with _records_lock:
- return sum(
- len(r.get("task_indexes") or r["goals"])
- if r.get("is_batch") and isinstance(r.get("goals"), (list, tuple)) and r["goals"] else 1
- for r in _records.values() if r.get("status") in {"running", "finalizing"})
- def _session_records(statuses, session_key: str, origin_ui_session_id: str, parent_session_id: str) -> list:
- """Records in ``statuses`` owned by a session: any non-empty selector claims the
- record — ``origin_ui_session_id`` (TUI tab), ``session_key`` (routing key at
- dispatch), or ``parent_session_id`` (spawner's durable id — the right one for
- gateway chats, whose session_key survives ``/new`` while the session id rotates)."""
- selectors = [(field, wanted) for field, wanted in (
- ("origin_ui_session_id", origin_ui_session_id), ("session_key", session_key),
- ("parent_session_id", parent_session_id)) if wanted]
- if not selectors:
- return []
- with _records_lock:
- return [r for r in _records.values() if r.get("status") in statuses
- and any(str(r.get(field) or "") == wanted for field, wanted in selectors)]
- def has_live_for_session(session_key: str = "", origin_ui_session_id: str = "", parent_session_id: str = "") -> bool:
- """Whether a session still owns any live (running/stalling/finalizing) delegation."""
- return bool(_session_records(_LIVE_STATES, session_key, origin_ui_session_id, parent_session_id))
- def _new_delegation_id() -> str:
- return f"deleg_{uuid.uuid4().hex[:8]}"
- def _prune_completed_locked() -> None:
- """Drop the oldest completed records beyond the cap. Caller holds ``_records_lock``.
- ``stalling``/``finalizing`` are still live: evicting one makes the late runner return hit
- ``_finalize``'s missing-record path and silently drop a real result."""
- completed = [(rid, r) for rid, r in _records.items() if r.get("status") not in _LIVE_STATES]
- completed.sort(key=lambda kv: kv[1].get("completed_at") or kv[1].get("dispatched_at") or 0)
- for rid, _ in completed[: max(0, len(completed) - _MAX_RETAINED_COMPLETED)]:
- _records.pop(rid, None)
- def _current_origin_session_id() -> str:
- """Raw session id of the ORIGINATING api_server request, or ``""``. ``HERMES_SESSION_ID``
- is unsafe here: building the child agent calls ``set_current_session_id(child.session_id)``
- just before dispatch, so the wake would self-post into the subagent's own session. The
- request-scoped ``HERMES_SESSION_CHAT_ID`` (raw X-Hermes-Session-Id on api_server) survives
- child construction; on push platforms chat_id is a chat, not a session => ``""``."""
- try:
- from gateway.session_context import get_session_env
- is_api = get_session_env("HERMES_SESSION_PLATFORM", "") == "api_server"
- return (get_session_env("HERMES_SESSION_CHAT_ID", "") or "") if is_api else ""
- except Exception:
- return ""
- # ── Dispatch ────────────────────────────────────────────────────────────────
- def _single_crash(error: str, duration: float) -> dict[str, Any]:
- return {"status": "error", "summary": None, "error": error, "api_calls": 0, "duration_seconds": duration}
- def _batch_crash(error: str, duration: float) -> dict[str, Any]:
- return {"results": [], "error": error, "total_duration_seconds": duration}
- def _batch_status(combined: dict[str, Any]) -> str:
- """Batch status: completed unless every child errored/was interrupted."""
- child_results = combined.get("results") or []
- ok = ("completed", "success")
- return "error" if child_results and all(r.get("status") not in ok for r in child_results) else "completed"
- def _dispatch(**kwargs) -> dict[str, Any]:
- from hermes_cli.backend_retirement import retirement
- with retirement.work() as admitted:
- if not admitted:
- return {"status": "rejected", "error": "backend is retiring; reconnect to continue"}
- return _dispatch_admitted(**kwargs)
- def _dispatch_admitted(
- *, delegation_id: str, goal: str, goals: Optional[list[str]], context: Optional[str],
- toolsets: Optional[list[str]], role: str, model: Optional[str], session_key: str,
- parent_session_id: Optional[str], runner: Callable[[], dict[str, Any]], origin_ui_session_id: str,
- origin_session_id: str, interrupt_fn: Optional[Callable[[], None]], max_async_children: int,
- progress_fn: Optional[Callable[[], tuple]], capacity_error: str, slot_key: Optional[str] = None,
- task_indexes: Optional[list[int]] = None,
- task_transcripts: Optional[dict[str, str]] = None,
- ) -> dict[str, Any]:
- """Shared dispatch core for single (``goals is None``) and batch units. Capacity check +
- record insert happen under ONE lock hold so concurrent dispatches can't both pass the check
- and exceed the cap. At capacity the dispatch is REJECTED (never queued) so a runaway model
- can't pile up unbounded background work. ``slot_key`` names the pool slot the unit occupies
- (default: its own id); the units of one delegate_task call share the first unit's id so
- splitting a call into per-group completions never consumes more capacity than the call did."""
- worker_ledger() # refuse unbound workers before publishing a live record
- is_batch = goals is not None
- label = " batch" if is_batch else ""
- classify = _batch_status if is_batch else (lambda r: r.get("status") or "completed")
- crash_result = _batch_crash if is_batch else _single_crash
- dispatched_at = time.time()
- record: dict[str, Any] = {
- "delegation_id": delegation_id, "goal": goal, **({"goals": list(goals)} if is_batch else {}),
- "context": context, "toolsets": list(toolsets) if toolsets else None, "role": role, "model": model,
- "session_key": session_key, "origin_ui_session_id": origin_ui_session_id,
- "origin_session_id": origin_session_id, "parent_session_id": parent_session_id,
- **_capture_routing_origin(),
- "status": "running", "dispatched_at": dispatched_at, "completed_at": None,
- "interrupt_fn": interrupt_fn, **({"is_batch": True} if is_batch else {}), "progress_fn": progress_fn,
- "slot_key": slot_key or delegation_id,
- **({"task_transcripts": dict(task_transcripts)} if task_transcripts else {}),
- # Which of the call's ``goals`` this unit runs (None = all of them).
- **({"task_indexes": list(task_indexes)} if task_indexes is not None else {}),
- # The one stale-monitor thread serves every profile and starts with an empty Context;
- # a forced finalization runs under the dispatcher's so it settles the same state.db.
- "_context": contextvars.copy_context(),
- # Stale-monitor bookkeeping (see _stale_monitor_loop).
- "_progress_token": None, "_progress_ts": dispatched_at, "_interrupted_at": None}
- with _records_lock:
- if any((field, record[field]) in _refused_by for field in _FENCED_FIELDS):
- return {"status": "rejected", "error": "This session was closed (exit or /new): no new background work."}
- active_slots = {r.get("slot_key") or r["delegation_id"] for r in _records.values() if r.get("status") in _ACTIVE_STATES}
- if record["slot_key"] not in active_slots and len(active_slots) >= max_async_children:
- return {"status": "rejected", "error": capacity_error}
- _records[delegation_id] = record
- live_units = sum(1 for r in _records.values() if r.get("status") in _LIVE_STATES)
- _persist_dispatch(record)
- # Units of one call share a slot, so live units can exceed slots: size the pool by units or a
- # unit queues behind a full pool and the stale monitor kills it before its child ever starts.
- executor = _get_executor(max(max_async_children, live_units))
- def _worker() -> None:
- result: dict[str, Any] = {}
- status = "error"
- with _records_lock:
- rec = _records.get(delegation_id)
- if rec is not None:
- # The stall clock starts when the runner starts; a unit queued behind a full pool is not stalled.
- rec.update(_started=True, _progress_ts=time.time())
- try:
- result = runner() or {}
- status = classify(result)
- except Exception as exc:
- logger.exception(f"Async delegation{label} %s crashed", delegation_id)
- result = crash_result(f"{type(exc).__name__}: {exc}", round(time.time() - dispatched_at, 2))
- finally:
- _finalize(delegation_id, result, status)
- from hermes_cli.backend_retirement import retirement
- # The outer dispatch reservation prevents a freeze during this handoff. Retain a worker
- # reservation too: the stall monitor may finalize its registry record before it really exits.
- retirement.acquire()
- try:
- future = executor.submit(propagate_context_to_thread(_worker))
- future.add_done_callback(lambda _: retirement.release())
- except Exception as exc: # pragma: no cover — pool submit failure is rare
- retirement.release()
- with _records_lock:
- _records.pop(delegation_id, None)
- if remote := worker_ledger():
- remote.unscheduled(delegation_id)
- else:
- with _DB_LOCK, _transaction() as conn:
- conn.execute("DELETE FROM async_delegations WHERE delegation_id=?", (delegation_id,))
- return {"status": "rejected", "error": f"Failed to schedule async delegation{label}: {exc}"}
- if progress_fn is not None:
- _ensure_stale_monitor()
- return {"status": "dispatched", "delegation_id": delegation_id}
- def dispatch_async_delegation(
- *, goal: str, context: Optional[str], toolsets: Optional[list[str]], role: str, model: Optional[str],
- session_key: str, parent_session_id: Optional[str] = None, runner: Callable[[], dict[str, Any]],
- origin_ui_session_id: str = "", origin_session_id: str = "", interrupt_fn: Optional[Callable[[], None]] = None,
- max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, progress_fn: Optional[Callable[[], tuple]] = None,
- ) -> dict[str, Any]:
- """Spawn ``runner`` on the daemon executor and return a handle immediately.
- ``session_key``/``parent_session_id`` are captured on the parent thread (the worker carries
- no contextvars) and route the completion back to the spawning session.
- ``progress_fn() -> (token, in_tool)`` enables stale monitoring; omitted = unmonitored.
- Returns ``{"status": "dispatched", "delegation_id"}`` or ``{"status": "rejected", "error"}``."""
- delegation_id = _new_delegation_id()
- handle = _dispatch(
- delegation_id=delegation_id, goal=goal, goals=None, context=context,
- toolsets=toolsets, role=role, model=model, session_key=session_key,
- parent_session_id=parent_session_id, runner=runner,
- origin_ui_session_id=origin_ui_session_id, origin_session_id=origin_session_id,
- interrupt_fn=interrupt_fn, max_async_children=max_async_children, progress_fn=progress_fn,
- capacity_error=(
- f"Async delegation capacity reached ({max_async_children} running). Wait for one to finish "
- "(its result will re-enter the chat), or run this task synchronously (background=false). "
- "Raise delegation.max_concurrent_children in config.yaml to allow more concurrent background subagents."))
- if handle["status"] == "dispatched":
- logger.info("Dispatched async delegation %s (session_key=%s): %s",
- delegation_id, session_key or "<cli>", (goal or "")[:80])
- return handle
- def dispatch_async_delegation_batch(
- *, goals: list[str], context: Optional[str], toolsets: Optional[list[str]], role: str, model: Optional[str],
- session_key: str, parent_session_id: Optional[str] = None, runner: Callable[[], dict[str, Any]],
- origin_ui_session_id: str = "", origin_session_id: str = "", interrupt_fn: Optional[Callable[[], None]] = None,
- max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, delegation_id: Optional[str] = None,
- progress_fn: Optional[Callable[[], tuple]] = None, slot_key: Optional[str] = None,
- task_indexes: Optional[list[int]] = None,
- task_transcripts: Optional[dict[str, str]] = None,
- ) -> dict[str, Any]:
- """Dispatch a fan-out unit (a whole batch, or one ``group`` of a delegate_task call) as ONE
- background unit: ``runner`` runs its tasks and returns the combined ``{"results": [...],
- "total_duration_seconds": N}`` dict. The unit occupies ONE async slot — or joins the slot named
- by ``slot_key`` (in-unit parallelism is bounded separately) — and produces a SINGLE completion
- event carrying per-task ``results``."""
- delegation_id = delegation_id or _new_delegation_id()
- # ``goals`` is the whole call (result task_index indexes it); the unit's own goals label the record.
- unit_goals = [goals[i] for i in task_indexes] if task_indexes is not None else list(goals)
- n = len(unit_goals)
- combined_goal = unit_goals[0] if n == 1 else f"{n} parallel subagents: " + "; ".join(g[:40] for g in unit_goals)
- handle = _dispatch(
- delegation_id=delegation_id, goal=combined_goal, goals=goals, context=context,
- toolsets=toolsets, role=role, model=model, session_key=session_key,
- parent_session_id=parent_session_id, runner=runner,
- origin_ui_session_id=origin_ui_session_id, origin_session_id=origin_session_id,
- interrupt_fn=interrupt_fn, max_async_children=max_async_children, progress_fn=progress_fn, slot_key=slot_key,
- task_indexes=task_indexes, task_transcripts=task_transcripts,
- capacity_error=(
- f"Async delegation capacity reached ({max_async_children} running). Wait for one to finish "
- "(its result will re-enter the chat), or raise delegation.max_concurrent_children in "
- "config.yaml to allow more concurrent background units."))
- if handle["status"] == "dispatched":
- logger.info("Dispatched async delegation batch %s (%d task(s), session_key=%s)",
- delegation_id, n, session_key or "<cli>")
- return handle
- # ── Finalization + completion events ────────────────────────────────────────
- def _finalize(delegation_id: str, result: Any, status: str) -> None:
- """Atomically claim terminal delivery, push the completion event, then mark ``status``.
- ``result`` is a dict or a callable receiving the record snapshot (stall path). The record
- stays active ("finalizing") until durable persistence and queue publication finish; otherwise
- process shutdown can kill this daemon worker after status flips but before SQLite commits.
- A second call for the same id (late runner return after a forced stall) is a no-op."""
- with _records_lock:
- record = _records.get(delegation_id)
- if record is None or record.get("status") not in _ACTIVE_STATES:
- return
- record["status"] = "finalizing"
- record["completed_at"] = time.time()
- record["interrupt_fn"] = None # drop the closure; child is done
- record["progress_fn"] = None # stop stale-monitor sampling
- snapshot = dict(record)
- _push_completion_event(snapshot, result(snapshot) if callable(result) else result, status)
- with _records_lock:
- if delegation_id in _records:
- _records[delegation_id]["status"] = status
- # Only an active record is ever force-finalized under its dispatcher's Context; a
- # retained completed one would otherwise pin that turn's agent and history.
- _records[delegation_id].pop("_context", None)
- _prune_completed_locked()
- def _push_completion_event(record: dict[str, Any], result: dict[str, Any], status: str) -> None:
- """Push a type='async_delegation' event onto the shared completion queue. Batch records
- (``is_batch``) carry the per-task ``results`` list (plus live transcript paths, the
- full-fidelity record of each child's run) instead of a single summary. Best-effort: failure
- must not crash the worker, but it WOULD mean a silently-lost result, so we log loudly.
- The queued event is the parent's in-process copy and honours ``security.redact_secrets: false``;
- ``_persist_completion`` re-scrubs with force for the durable row (and the worker IPC)."""
- result = redact_child_result(result, force=False)
- is_batch = bool(record.get("is_batch"))
- label = " batch" if is_batch else ""
- try:
- from tools.process_registry import process_registry
- except Exception as exc: # pragma: no cover
- logger.error(f"Async delegation{label} %s finished but process_registry import failed; "
- "result lost: %s", record.get("delegation_id"), exc)
- return
- dispatched_at = record.get("dispatched_at") or time.time()
- completed_at = record.get("completed_at") or time.time()
- if is_batch:
- payload = {
- "is_batch": True, "results": result.get("results") or [],
- "live_transcripts": result.get("live_transcripts"), "error": result.get("error"),
- "total_duration_seconds": result.get("total_duration_seconds"),
- **({"group": result["group"]} if result.get("group") is not None else {})}
- else:
- # A runner with no measured count (a cron run without a session) omits api_calls: never a fake 0.
- payload = {
- "summary": result.get("summary"), "error": result.get("error"),
- **({"api_calls": result["api_calls"]} if result.get("api_calls") is not None else {}),
- "duration_seconds": result.get("duration_seconds", round(completed_at - dispatched_at, 2))}
- evt = {
- "type": "async_delegation", "delegation_id": record.get("delegation_id"),
- # session_key routes back to the originating gateway session; "" => CLI.
- "session_key": record.get("session_key", ""),
- "origin_ui_session_id": record.get("origin_ui_session_id", ""),
- "origin_session_id": record.get("origin_session_id", ""),
- "parent_session_id": record.get("parent_session_id"),
- "goal": record.get("goal", ""), **({"goals": record.get("goals")} if is_batch else {}),
- "context": record.get("context"), "toolsets": record.get("toolsets"), "role": record.get("role"),
- "model": record.get("model") if is_batch else (result.get("model") or record.get("model")),
- "status": status, **payload, "dispatched_at": dispatched_at, "completed_at": completed_at,
- **({} if is_batch else {"exit_reason": result.get("exit_reason")}),
- **{k: record[k] for k in _ROUTING_KEYS if record.get(k)},
- **{k: result[k] for k in _STALL_META_KEYS if k in result}}
- try:
- _persist_completion(evt, result)
- except Exception as exc:
- logger.error(f"Async delegation{label} %s: durable completion write failed; delivering in-memory "
- "only (a restart may report this unit as unknown): %s", record.get("delegation_id"), exc)
- try:
- process_registry.completion_queue.put(evt)
- except Exception as exc: # pragma: no cover
- logger.error(f"Async delegation{label} %s: failed to enqueue completion event; "
- "result lost: %s", record.get("delegation_id"), exc)
- def push_task_failure_notice(delegation_id: str, entry: dict[str, Any], *, n_tasks: int) -> None:
- """Surface ONE failed child of a still-running detached batch to the parent now, instead of
- when the slowest sibling finishes. In a 1,393-agent run every wave-1 child died in a 401 storm
- at 08:29 and the parent learned of it at 09:36, when the batch's "unknown outcome" block finally
- arrived: 66 minutes of a dead wave with nothing running. The notice rides the same
- ``type="async_delegation"`` event shape as the batch result (so every drain/route/format path
- treats it identically) with ``task_failure_notice=True`` and a single-entry ``results`` list; the
- batch record is NOT finalized and its consolidated result still arrives as before. In-process
- queue only (never persisted), so the parent's copy honours ``security.redact_secrets: false``."""
- entry = redact_child_result(entry, force=False)
- with _records_lock:
- record = _records.get(delegation_id)
- if record is None or record.get("status") not in _ACTIVE_STATES:
- return
- snapshot = dict(record)
- try:
- from tools.process_registry import process_registry
- except Exception as exc: # pragma: no cover
- logger.error("Async delegation batch %s: task failure notice dropped (process_registry import): %s", delegation_id, exc)
- return
- evt = {
- "type": "async_delegation", "task_failure_notice": True, "is_batch": True, "n_tasks": n_tasks,
- "delegation_id": delegation_id, "results": [entry],
- "session_key": snapshot.get("session_key", ""),
- "origin_ui_session_id": snapshot.get("origin_ui_session_id", ""),
- "origin_session_id": snapshot.get("origin_session_id", ""),
- "parent_session_id": snapshot.get("parent_session_id"),
- "goal": snapshot.get("goal", ""), "goals": snapshot.get("goals"), "context": snapshot.get("context"),
- "toolsets": snapshot.get("toolsets"), "role": snapshot.get("role"), "model": snapshot.get("model"),
- "status": "running", "dispatched_at": snapshot.get("dispatched_at") or time.time(), "completed_at": time.time(),
- **{k: snapshot[k] for k in _ROUTING_KEYS if snapshot.get(k)}}
- try:
- process_registry.completion_queue.put(evt)
- except Exception as exc: # pragma: no cover
- logger.error("Async delegation batch %s: failed to enqueue task failure notice: %s", delegation_id, exc)
- # ── Stale monitor ───────────────────────────────────────────────────────────
- def _ensure_stale_monitor() -> None:
- """Start (once) the stale-delegation monitor thread. One daemon thread serves
- every dispatch; it exits when no monitorable records remain and is restarted
- by the next dispatch with a ``progress_fn``."""
- global _monitor_thread
- with _monitor_lock:
- if _monitor_thread is not None and _monitor_thread.is_alive():
- return
- _monitor_stop.clear()
- _monitor_thread = threading.Thread(
- target=_stale_monitor_loop, name="async-delegate-stale-monitor", daemon=True)
- _monitor_thread.start()
- def _sweep_stale_locked(now: float):
- """One monitor pass over ``_records``; caller holds ``_records_lock``. Returns
- ``(stalled, expired, any_monitorable)``: newly-stalling ``(delegation_id, quiet_for, in_tool)``
- tuples, stalling ids past the grace window, and whether anything is left to monitor."""
- stalled, expired, any_monitorable = [], [], False # (delegation_id, quiet_for, in_tool) / ids past grace
- for record in _records.values():
- status = record.get("status")
- if status == "stalling":
- any_monitorable = True
- if now - (record.get("_interrupted_at") or now) >= _STALL_GRACE_SECONDS:
- expired.append(record["delegation_id"])
- continue
- progress_fn = record.get("progress_fn")
- if status != "running" or progress_fn is None:
- continue
- any_monitorable = True
- if not record.get("_started"):
- continue # queued behind a full pool: not stalled, but keep the monitor alive for when it starts
- try:
- token, in_tool = progress_fn()
- except Exception:
- # An unreadable child must not look permanently healthy —
- # keep the last timestamp running instead of refreshing it.
- token, in_tool = record.get("_progress_token"), False
- if token != record.get("_progress_token"):
- record.update(_progress_token=token, _progress_ts=now)
- continue
- quiet_for = now - (record.get("_progress_ts") or now)
- limit = _STALE_IN_TOOL_SECONDS if in_tool else _STALE_IDLE_SECONDS
- if quiet_for >= limit:
- # Stall context feeds the terminal event and status listings.
- record.update(
- status="stalling", _interrupted_at=now, _stall_quiet_seconds=round(quiet_for, 2),
- _stall_threshold_seconds=limit, _stall_in_tool=bool(in_tool))
- stalled.append((record["delegation_id"], quiet_for, in_tool))
- return stalled, expired, any_monitorable
- def _call_interrupt(fn, msg: str, *args) -> bool:
- """Invoke an ``interrupt_fn``; True on success, else debug-log ``msg`` (+ exc)."""
- if not callable(fn):
- return False
- try:
- fn()
- return True
- except Exception as exc:
- logger.debug(msg, *args, exc)
- return False
- def _stale_monitor_loop() -> None:
- """Sweep running delegations for stalled progress. A changed progress token refreshes the
- record's timestamp; a frozen token past the idle/in-tool threshold marks the record
- ``stalling`` and calls ``interrupt_fn``; a ``stalling`` record still unreturned after the
- grace window is force-finalized with a terminal ``stalled`` event."""
- while not _monitor_stop.wait(_STALE_CHECK_INTERVAL):
- now = time.time()
- with _records_lock:
- stalled, expired, any_monitorable = _sweep_stale_locked(now)
- for delegation_id, quiet_for, in_tool in stalled:
- logger.warning("Async delegation %s made no progress for %.0fs "
- "(in_tool=%s) — interrupting; grace window %.0fs",
- delegation_id, quiet_for, in_tool, _STALL_GRACE_SECONDS)
- with _records_lock:
- fn = (_records.get(delegation_id) or {}).get("interrupt_fn")
- _call_interrupt(fn, "Async delegation %s stall interrupt failed: %s", delegation_id)
- for delegation_id in expired:
- with _records_lock:
- ctx = (_records.get(delegation_id) or {}).get("_context") or contextvars.copy_context()
- ctx.run(_finalize, delegation_id, lambda rec, d=delegation_id: _stalled_result(d, rec), "stalled")
- if not any_monitorable:
- return
- def _stalled_error_text(event_record: dict[str, Any]) -> str:
- """Human wording for a force-finalized stall. This string reaches the user (CLI timeline, Desktop
- async-result card), so it names the task, how long it was silent, and what to do — no issue
- numbers or worker internals (those stay in the log line and the stall_* metadata)."""
- goal = " ".join(str(event_record.get("goal") or "").split())
- label = f'Background task "{goal[:120]}"' if goal else "The background task"
- quiet = float(event_record.get("_stall_quiet_seconds") or 0)
- silence = f" after {round(quiet / 60)} min of no progress" if quiet >= 60 else ""
- return (f"{label} stopped responding{silence} and was cancelled. Nothing else was affected; "
- "ask me to run it again if you still need it.")
- def _stalled_result(delegation_id: str, event_record: dict[str, Any]) -> dict[str, Any]:
- """Synthetic terminal result for a stalling delegation whose runner never returned."""
- completed_at = event_record.get("completed_at") or time.time()
- duration = round(completed_at - (event_record.get("dispatched_at") or completed_at), 2)
- error = _stalled_error_text(event_record)
- logger.error("Async delegation %s force-finalized as stalled after %.0fs", delegation_id, duration)
- # Structured stall metadata lets parents/UIs distinguish a stall-monitor
- # kill from other failures without parsing the error string.
- stall_in_tool = event_record.get("_stall_in_tool")
- stall_meta = {
- "stalled_after_quiet_seconds": event_record.get("_stall_quiet_seconds"),
- "stall_threshold_seconds": event_record.get("_stall_threshold_seconds"),
- "stall_phase": "in_tool" if stall_in_tool else "idle" if stall_in_tool is not None else None,
- "stall_grace_seconds": _STALL_GRACE_SECONDS}
- if event_record.get("is_batch"):
- return {**_batch_crash(error, duration), **stall_meta}
- return {**_single_crash(error, duration), "status": "stalled", "exit_reason": "stalled", **stall_meta}
- # ── Observability + control ─────────────────────────────────────────────────
- def _children_activity_from_token(token: Any, now: float) -> Optional[list]:
- """Parse a progress token into per-child activity dicts (best-effort): delegate_tool
- emits one ``(api_call_count, current_tool, last_activity_ts)`` tuple per child;
- foreign token shapes degrade to ``None`` entries."""
- try:
- parts = list(token)
- except TypeError:
- return None
- out: list[Optional[dict[str, Any]]] = []
- for part in parts:
- if not (isinstance(part, (list, tuple)) and len(part) >= 2):
- out.append(None)
- continue
- entry: dict[str, Any] = {"api_calls": part[0], "current_tool": part[1]}
- if len(part) >= 3 and isinstance(part[2], (int, float)):
- entry["seconds_since_activity"] = round(max(0.0, now - float(part[2])), 1)
- out.append(entry)
- return out
- def list_async_delegations() -> list[dict[str, Any]]:
- """Snapshot of async delegations (running + recently completed) without callables or private
- monitor bookkeeping; adds computed live fields for UIs (``seconds_since_progress``,
- ``children_activity``/``in_tool`` sampled from ``progress_fn``) and stall context once tripped.
- Safe to call from any thread. See #51690.
- """
- now = time.time()
- samplers: dict[str, Callable] = {}
- with _records_lock:
- items = []
- for r in _records.values():
- item = {k: v for k, v in r.items() if k not in {"interrupt_fn", "progress_fn"} and not k.startswith("_")}
- status = r.get("status")
- if status in _ACTIVE_STATES:
- if r.get("_progress_ts"):
- item["seconds_since_progress"] = round(now - r["_progress_ts"], 1)
- if callable(r.get("progress_fn")):
- samplers[r["delegation_id"]] = r["progress_fn"]
- if status in ("stalling", "stalled"):
- for src, dst in _STALL_FIELD_MAP:
- if r.get(src) is not None:
- item[dst] = r.get(src)
- items.append(item)
- # Sample OUTSIDE the lock — progress_fn reads child-agent attributes and a
- # slow/broken sampler must not block every dispatch/finalize.
- for item in items:
- fn = samplers.get(item.get("delegation_id"))
- if fn is None:
- continue
- try:
- token, in_tool = fn()
- except Exception:
- continue
- activity = _children_activity_from_token(token, now)
- if activity is not None:
- item["children_activity"] = activity
- item["in_tool"] = bool(in_tool)
- return items
- def _interrupt_records(targets: list[dict[str, Any]], caller: str, reason: str, msg: str) -> int:
- """Call ``interrupt_fn`` on each record; log ``msg`` once; returns how many succeeded."""
- count = sum(
- _call_interrupt(r.get("interrupt_fn"), "%s: %s interrupt failed: %s", caller, r.get("delegation_id"))
- for r in targets)
- if count:
- logger.info(msg, count, reason)
- return count
- def interrupt_all(reason: str = "shutdown") -> int:
- """Signal every running async delegation to stop (``/stop``, shutdown). Returns how
- many. The child still emits a completion event (status='interrupted') via the
- normal finalize path."""
- with _records_lock:
- targets = [r for r in _records.values() if r.get("status") in _ACTIVE_STATES]
- return _interrupt_records(targets, "interrupt_all", reason, "Interrupted %d async delegation(s) (%s)")
- def interrupt_for_session(
- session_key: str = "", origin_ui_session_id: str = "", parent_session_id: str = "", reason: str = "session_end",
- ) -> int:
- """Signal running async delegations owned by ONE ending session to stop (any
- selector matches, see ``_session_records``). Returns how many."""
- targets = _session_records(_ACTIVE_STATES, session_key, origin_ui_session_id, parent_session_id)
- return _interrupt_records(
- targets, "interrupt_for_session", reason, "Interrupted %d async delegation(s) for ending session (%s)")
- def refuse_for_session(token: Any, *, session_key: str = "", parent_session_ids=()) -> None:
- """Refuse new dispatches for ONE closing session (any selector matches) until ``allow_for_session``.
- Set before ``interrupt_for_session``: a dispatch registered first is interrupted, a later one refused."""
- fence = {("session_key", session_key)} if session_key else set()
- fence.update(("parent_session_id", parent) for parent in parent_session_ids if parent)
- with _records_lock:
- _unfence(token)
- _refused[token] = frozenset(fence)
- for selector in fence:
- _refused_by.setdefault(selector, set()).add(token)
- def allow_for_session(token: Any) -> None:
- """Lift ONE session's refusal: a viewer attached again, or the owner retired its live state."""
- with _records_lock:
- _unfence(token)
- def _unfence(token: Any) -> None:
- """Drop *token*'s refusal; caller holds ``_records_lock``."""
- for selector in _refused.pop(token, ()):
- tokens = _refused_by[selector]
- tokens.discard(token)
- if not tokens:
- del _refused_by[selector]
- def _reset_for_tests() -> None:
- """Test-only: clear all state and tear down the executor + monitor."""
- global _executor, _executor_max_workers, _monitor_thread
- with _executor_lock:
- if _executor is not None:
- _executor.shutdown(wait=False)
- _executor = None
- _executor_max_workers = 0
- _monitor_stop.set()
- with _monitor_lock:
- thread, _monitor_thread = _monitor_thread, None
- if thread is not None and thread.is_alive():
- thread.join(timeout=2)
- with _records_lock:
- _records.clear()
- _refused.clear()
- _refused_by.clear()
- with _orphan_lock:
- _offered.clear()
- _startup_replayed.clear()
- _last_orphan_sweep.clear()
- _reoffered_at.clear()
- with _held_lock:
- _held_copies.clear()
|