kanban_turn_recovery.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. """In-place recovery for kanban worker turns that die on a retryable failure.
  2. A worker spawned by the kanban dispatcher used to end its run silently when a
  3. provider call failed: the one-shot CLI returned, the process exited ``rc=0``
  4. with no terminal ``kanban_complete`` / ``kanban_block`` call, the dispatcher
  5. booked a *protocol violation* and respawned the task from scratch — discarding
  6. the whole session context the worker had accumulated. Under a provider stream
  7. storm that turns one slow morning into several cold restarts per card.
  8. This module retries such a turn IN PLACE: same session, same conversation
  9. history, bounded budget, explicit backoff, and a continuation nudge that tells
  10. the model to finish the task with a real terminal call. When the budget is
  11. exhausted (or the retry is not authorised) the caller exits NON-ZERO so the run
  12. is booked honestly instead of masquerading as a clean exit.
  13. Retry authority is POSITIVE and TYPED — never inferred from an outcome shape:
  14. * only a ``failed`` turn whose classifier marked the failure ``retryable`` is
  15. eligible, and quota/billing walls are left to the dispatcher's
  16. cooldown/breaker accounting (``KANBAN_RATE_LIMIT_EXIT_CODE``, see
  17. ``hermes_cli/cli_single_query.py::_single_query_exit_code``);
  18. * ``interrupted=True`` is never retried (the interrupt was persisted and
  19. cleared by ``agent/turn_recovery.py::abort_turn_on_interrupt`` — re-entering
  20. the model would resurrect explicitly cancelled work);
  21. * a turn that ended through the bounded terminal settlement is never retried
  22. (``agent/turn_finalizer.py::_record_kanban_budget_exhausted`` already recorded
  23. ``outcome="timed_out"`` and released the claim — #87096);
  24. * before EVERY attempt the worker proves it still owns the exact live run and
  25. unexpired claim lease (``worker_claim_is_live``). The proof is the carrier the
  26. dispatcher pinned at spawn — ``HERMES_KANBAN_DB`` + ``HERMES_KANBAN_RUN_ID`` +
  27. ``HERMES_KANBAN_CLAIM_LOCK`` are REQUIRED and compared exactly, and the
  28. task/run ``claim_expires`` lease must be unexpired; any missing coordinate or
  29. expired lease means no proof, no retry (fail closed).
  30. Incomplete-but-not-failed results (``partial`` / ``completed=False``) are
  31. deliberately NOT retry authority: truncation and compression repair are owned
  32. inside the conversation loop (#89289) and by the dispatcher. They still fail the
  33. process exit code (``hermes_cli/cli_single_query.py::_single_query_exit_code``) so such a run is booked
  34. honestly instead of ending as a silent ``rc=0``.
  35. """
  36. from __future__ import annotations
  37. import logging
  38. import os
  39. import sqlite3
  40. import time
  41. from pathlib import Path
  42. from typing import Any, Callable, Optional
  43. from agent.error_classifier import FailoverReason
  44. logger = logging.getLogger(__name__)
  45. #: Recovery attempts when ``HERMES_KANBAN_TURN_RECOVERY`` is unset.
  46. DEFAULT_MAX_RECOVERY_ATTEMPTS = 3
  47. #: Backoff before recovery attempt N (1-based). The last entry repeats.
  48. RECOVERY_DELAYS_SECONDS: tuple[float, ...] = (15.0, 45.0, 90.0)
  49. #: Failure reasons that must NOT be retried in place: quota walls and billing
  50. #: blocks need the dispatcher's cooldown/breaker accounting, not another call.
  51. #: Aligned with the exit side of main's ``cli._TRANSIENT_PROVIDER_REASONS``
  52. #: (which also releases ``timeout`` / ``overloaded`` / ``server_error`` runs as
  53. #: neutral 75s AFTER in-place recovery has been exhausted). The two sets answer
  54. #: different questions: this one says who may re-enter the model here; main's
  55. #: says how the process exit is booked.
  56. _NON_RECOVERABLE_REASONS = frozenset({
  57. FailoverReason.rate_limit.value,
  58. FailoverReason.billing.value,
  59. FailoverReason.upstream_rate_limit.value,
  60. })
  61. #: ``turn_exit_reason`` prefixes that mean the turn already went through a
  62. #: durable terminal settlement (the run may be closed and its claim released):
  63. #: ``agent/turn_finalizer.py`` writes ``max_iterations_reached(N/M)`` right after
  64. #: recording ``outcome="timed_out"`` for an out-of-budget kanban worker (#87096).
  65. _TERMINAL_TURN_EXIT_PREFIXES = ("max_iterations_reached",)
  66. _OFF_VALUES = frozenset({"0", "false", "no", "off"})
  67. def kanban_task_id() -> Optional[str]:
  68. """The dispatcher-set kanban task id, or ``None`` when this is not a worker run.
  69. Single source of truth for every kanban-worker predicate. The env value is
  70. stripped exactly once, here, so a whitespace-only value can never read as
  71. "worker" to one caller and "not a worker" to another (the exit-code guard and
  72. the recovery gate must agree).
  73. """
  74. task_id = (os.environ.get("HERMES_KANBAN_TASK") or "").strip()
  75. return task_id or None
  76. def _resolve_task_id(task_id: Optional[str]) -> Optional[str]:
  77. """An explicit carrier id (managed worker frame) wins; ``None`` reads the env pin."""
  78. if task_id is None:
  79. return kanban_task_id()
  80. return task_id.strip() or None
  81. def kanban_turn_recovery_enabled(task_id: Optional[str] = None) -> bool:
  82. """On when a worker task id is carried and the attempt budget is non-zero."""
  83. if _resolve_task_id(task_id) is None:
  84. return False
  85. return max_recovery_attempts() > 0
  86. def max_recovery_attempts() -> int:
  87. """``HERMES_KANBAN_TURN_RECOVERY`` parsed as an attempt count (0 disables).
  88. Unset/blank -> :data:`DEFAULT_MAX_RECOVERY_ATTEMPTS`; explicit 0/false/no/off
  89. -> 0; anything unparseable -> the default. Clamped to [0, 10] so a bad value
  90. can never create an unbounded loop.
  91. """
  92. raw = (os.environ.get("HERMES_KANBAN_TURN_RECOVERY") or "").strip()
  93. if not raw:
  94. return DEFAULT_MAX_RECOVERY_ATTEMPTS
  95. if raw.lower() in _OFF_VALUES:
  96. return 0
  97. try:
  98. value = int(raw)
  99. except (TypeError, ValueError):
  100. return DEFAULT_MAX_RECOVERY_ATTEMPTS
  101. return max(0, min(value, 10))
  102. def recovery_delay_seconds(attempt: int) -> float:
  103. """Backoff before recovery attempt ``attempt`` (1-based); last entry repeats."""
  104. attempt = max(attempt, 1)
  105. index = min(attempt - 1, len(RECOVERY_DELAYS_SECONDS) - 1)
  106. return RECOVERY_DELAYS_SECONDS[index]
  107. def _turn_exit_is_terminal(result: dict) -> bool:
  108. reason = str(result.get("turn_exit_reason") or "")
  109. return reason.startswith(_TERMINAL_TURN_EXIT_PREFIXES)
  110. def should_recover_turn(result: Any, *, attempt: int, task_id: Optional[str] = None) -> bool:
  111. """True when a settled worker turn grants in-place retry authority.
  112. ``attempt`` is the number of recovery attempts ALREADY made. Authority is
  113. positive and typed: an enabled recovery within budget, a FAILED turn whose
  114. classifier marked the failure ``retryable``, no quota/billing wall, no
  115. explicit cancellation, and no durable terminal settlement. Everything else —
  116. including ``partial`` / ``completed=False`` — is not authority to re-enter
  117. the model. See the module docstring for the full rule set.
  118. """
  119. if not kanban_turn_recovery_enabled(task_id):
  120. return False
  121. if attempt >= max_recovery_attempts():
  122. return False
  123. if not isinstance(result, dict):
  124. return False
  125. if result.get("interrupted") is True:
  126. return False
  127. if _turn_exit_is_terminal(result):
  128. return False
  129. if result.get("failed") is not True:
  130. return False
  131. if result.get("failure_retryable") is not True:
  132. return False
  133. if str(result.get("failure_reason") or "") in _NON_RECOVERABLE_REASONS:
  134. return False
  135. return True
  136. def _kanban_db_path() -> Optional[str]:
  137. """The dispatcher-pinned board DB — no ambient resolution, ever.
  138. The dispatcher pins ``HERMES_KANBAN_DB`` (plus run id + claim lock) at spawn
  139. (``hermes_cli/kanban_db_dispatch.py``); that pin IS the exact authority
  140. carrier the retry proof must re-check. If the pin is absent there is no
  141. exact carrier to re-prove, so this returns ``None`` and the caller fails
  142. closed: resolving whatever board is ambient *now* would silently move the
  143. proof onto a different board — precisely the weaker proof this fail-closed
  144. contract exists to prevent.
  145. """
  146. pinned = (os.environ.get("HERMES_KANBAN_DB") or "").strip()
  147. return pinned or None
  148. def _now() -> int:
  149. """Wall-clock seconds — one seam so tests can advance the clock across a backoff."""
  150. return int(time.time())
  151. def worker_claim_is_live(
  152. *,
  153. task_id: Optional[str] = None,
  154. db_path: Optional[str] = None,
  155. run_id: Optional[str] = None,
  156. claim_lock: Optional[str] = None,
  157. verify_pid: bool = True,
  158. ) -> bool:
  159. """True when THIS process still owns a live, unexpired run/claim lease.
  160. Verified read-only against the dispatcher-pinned board before EVERY retry.
  161. All three pinned coordinates are REQUIRED — ``HERMES_KANBAN_DB``,
  162. ``HERMES_KANBAN_RUN_ID``, ``HERMES_KANBAN_CLAIM_LOCK`` — and compared with
  163. exact equality; a missing pin means there is no exact authority carrier to
  164. re-prove, never a fallback to ambient board state.
  165. The lease itself is part of the liveness contract: the canonical stale-claim
  166. selector (``hermes_cli/kanban_db.py::release_stale_claims``) treats
  167. ``status='running' AND claim_expires < now`` as stale and reclaims the task,
  168. and the worker-liveness path treats an expired claim as non-live. So both
  169. the task row AND the open run row must carry an UNEXPIRED ``claim_expires``
  170. (``heartbeat_claim`` extends the task expiry and mirrors it onto the active
  171. run row). An expired lease means mutation authority has lapsed even before
  172. the reconciler processes the row; a NULL expiry is not a provable live
  173. lease either and fails closed.
  174. FAIL-CLOSED: any missing coordinate, missing/unreadable DB, mismatched
  175. run/lock/pid, closed run, or expired lease returns False and the caller
  176. does not re-enter the model — an honest non-zero exit is always available,
  177. whereas mutating state under a released or expired claim is not recoverable.
  178. An owner-side (managed) worker passes its carrier explicitly from the bound
  179. worker frame instead of the env pins, and ``verify_pid=False`` because there
  180. the run's ``worker_pid`` names the dispatcher-spawned submitter, not this
  181. interpreter — that caller proves interpreter identity from its own
  182. ``worker_bound`` event.
  183. """
  184. task_id = _resolve_task_id(task_id)
  185. if task_id is None:
  186. return False
  187. db_path = _kanban_db_path() if db_path is None else (db_path.strip() or None)
  188. if not db_path or not Path(db_path).exists():
  189. logger.warning("kanban claim check: pinned board db %r not found — not retrying in place", db_path)
  190. return False
  191. run_id_env = str(run_id if run_id is not None else os.environ.get("HERMES_KANBAN_RUN_ID") or "").strip()
  192. lock_env = str(claim_lock if claim_lock is not None else os.environ.get("HERMES_KANBAN_CLAIM_LOCK") or "").strip()
  193. if not run_id_env or not lock_env:
  194. logger.warning(
  195. "kanban claim check: missing dispatcher-pinned run-id/claim-lock carrier "
  196. "— no exact authority to re-prove, not retrying in place"
  197. )
  198. return False
  199. now = _now()
  200. try:
  201. conn = sqlite3.connect(f"{Path(db_path).absolute().as_uri()}?mode=ro", uri=True)
  202. try:
  203. conn.row_factory = sqlite3.Row
  204. row = conn.execute(
  205. "SELECT status, worker_pid, current_run_id, claim_lock, claim_expires "
  206. "FROM tasks WHERE id = ?",
  207. (task_id,),
  208. ).fetchone()
  209. if row is None:
  210. return False
  211. if (row["status"] or "") != "running":
  212. return False
  213. worker_pid = row["worker_pid"]
  214. if verify_pid and worker_pid is not None and int(worker_pid) != os.getpid():
  215. return False
  216. # A running task with no run pointer is not proof of a live run: the
  217. # dispatcher writes it on claim and clears it on release/end.
  218. current_run_id = row["current_run_id"]
  219. if current_run_id is None:
  220. return False
  221. if str(current_run_id) != run_id_env:
  222. return False
  223. if (row["claim_lock"] or "") != lock_env:
  224. return False
  225. if row["claim_expires"] is None or int(row["claim_expires"]) < now:
  226. return False
  227. run = conn.execute(
  228. "SELECT ended_at, worker_pid, claim_expires FROM task_runs WHERE id = ?",
  229. (int(current_run_id),),
  230. ).fetchone()
  231. if run is None:
  232. return False
  233. if run["ended_at"] is not None:
  234. return False
  235. run_pid = run["worker_pid"]
  236. if verify_pid and run_pid is not None and int(run_pid) != os.getpid():
  237. return False
  238. if run["claim_expires"] is None or int(run["claim_expires"]) < now:
  239. return False
  240. return True
  241. finally:
  242. conn.close()
  243. except Exception:
  244. logger.warning("kanban claim check failed — not retrying in place", exc_info=True)
  245. return False
  246. def _truncate(text: str, limit: int = 300) -> str:
  247. text = (text or "").strip().replace("\n", " ")
  248. return text[: limit - 1] + "…" if len(text) > limit else text
  249. def build_recovery_nudge(result: Any, *, attempt: int, max_attempts: int,
  250. task_id: Optional[str] = None) -> str:
  251. """The synthetic user turn sent after a failed worker turn.
  252. Mirrors the stop-nudge contract (agent/kanban_stop.py): plain text is not a
  253. terminal state; the worker must finish with ``kanban_complete`` /
  254. ``kanban_block``. Emphasises "same session, do not start over" so the model
  255. reuses everything it already read/wrote instead of re-doing the work.
  256. """
  257. error = ""
  258. if isinstance(result, dict):
  259. error = _truncate(str(result.get("error") or result.get("final_response") or ""))
  260. task_id = _resolve_task_id(task_id) or "this task"
  261. return (
  262. "[System: the previous turn ended UNFINISHED — the API call failed after "
  263. f"all retries and the turn was interrupted (recovery attempt {attempt}/{max_attempts}). "
  264. f"Error: {error or 'provider stream failure'}.\n\n"
  265. f"Task `{task_id}` is still `running`. This is the SAME session, with your full "
  266. "context — nothing you already read, wrote, or computed is lost. Do NOT start over.\n\n"
  267. "Do this immediately:\n"
  268. "1. If the interrupted turn was mid tool-call, re-check the on-disk state of "
  269. "whatever it was writing before continuing (the action may not have executed).\n"
  270. "2. Continue the task from where it stopped and finish any remaining deliverable.\n"
  271. "3. End with a terminal `kanban_complete(summary=..., artifacts=[...])` if the work "
  272. "is done, or `kanban_block(reason=...)` if you are blocked.]"
  273. )
  274. def _emit_recovery_skipped(emit: Optional[Callable[[str], None]], task_id: Optional[str] = None) -> None:
  275. """The one "no proof, no retry" line — fail-closed is worth saying out loud."""
  276. message = (
  277. f"[kanban] in-place recovery skipped for {_resolve_task_id(task_id) or 'task'}: this worker no "
  278. "longer holds a live run/claim (already settled, reclaimed, or unverifiable)"
  279. )
  280. logger.warning("%s", message)
  281. if emit is not None:
  282. try:
  283. emit(message)
  284. except Exception:
  285. logger.debug("kanban turn-recovery emit failed", exc_info=True)
  286. def recover_failed_kanban_turns(
  287. turn_fn: Callable[[str], Any],
  288. get_result: Callable[[], Any],
  289. *,
  290. sleep_fn: Callable[[float], None] = time.sleep,
  291. emit: Optional[Callable[[str], None]] = None,
  292. claim_check: Optional[Callable[[], bool]] = None,
  293. task_id: Optional[str] = None,
  294. ) -> int:
  295. """Retry an authorised kanban worker turn in place; returns attempts made.
  296. ``turn_fn(nudge)`` runs one more turn with the given synthetic user message.
  297. ``get_result()`` returns the LATEST settled turn result (``None`` stops the
  298. loop). Before EVERY attempt the worker must still own its live run/claim
  299. (``claim_check``, default :func:`worker_claim_is_live`); the loop is bounded
  300. by the attempt budget even when ``get_result`` never changes. The loop
  301. deliberately does NOT extend its own claim lease during the backoff —
  302. self-holding a claim is claim-lifetime policy (#95318), not this module's;
  303. if the lease expires while sleeping, the re-proof fails and the attempt is
  304. not made. ``task_id`` carries the worker identity explicitly (owner-side
  305. workers pass their bound frame's task); such callers must also pass a
  306. ``claim_check`` built from the same explicit carrier.
  307. """
  308. attempts = 0
  309. check = claim_check or worker_claim_is_live
  310. while True:
  311. result = get_result()
  312. if not should_recover_turn(result, attempt=attempts, task_id=task_id):
  313. return attempts
  314. if not check():
  315. _emit_recovery_skipped(emit, task_id)
  316. return attempts
  317. attempts += 1
  318. delay = recovery_delay_seconds(attempts)
  319. message = (
  320. f"[kanban] provider failure on {_resolve_task_id(task_id) or 'task'} — retrying the turn in place "
  321. f"(attempt {attempts}/{max_recovery_attempts()}) after {int(delay)}s; session context preserved"
  322. )
  323. logger.warning("%s", message)
  324. if emit is not None:
  325. try:
  326. emit(message)
  327. except Exception:
  328. logger.debug("kanban turn-recovery emit failed", exc_info=True)
  329. sleep_fn(delay)
  330. # Re-prove ownership AFTER the backoff and immediately before model re-entry: the
  331. # dispatcher can reclaim or settle this run while we sleep, and the pre-sleep proof
  332. # alone would let that one attempt run without live ownership (round-5 finding F1 —
  333. # TOCTOU window). No proof, no retry; the caller's honest exit still applies.
  334. if not check():
  335. _emit_recovery_skipped(emit, task_id)
  336. return attempts
  337. turn_fn(build_recovery_nudge(result, attempt=attempts, max_attempts=max_recovery_attempts(),
  338. task_id=task_id))
  339. __all__ = [
  340. "DEFAULT_MAX_RECOVERY_ATTEMPTS",
  341. "RECOVERY_DELAYS_SECONDS",
  342. "build_recovery_nudge",
  343. "kanban_task_id",
  344. "kanban_turn_recovery_enabled",
  345. "max_recovery_attempts",
  346. "recover_failed_kanban_turns",
  347. "recovery_delay_seconds",
  348. "should_recover_turn",
  349. "worker_claim_is_live",
  350. ]