kanban_db_dispatch.py 134 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964
  1. """Dispatcher: crash/stale/orphan detection, failure accounting and the respawn circuit breaker, memory-aware concurrency caps, the one-shot ``dispatch_once`` pass, worker spawning (``_default_spawn``), worker-log rotation and the long-lived ``run_daemon`` loop.
  2. Split out of ``hermes_cli.kanban_db``; origin-resident helpers are reached
  3. late-bound via ``_kb`` (import-cycle breaking) so monkeypatching
  4. ``kanban_db.<name>`` keeps working.
  5. """
  6. from __future__ import annotations
  7. import contextlib
  8. import os
  9. import re
  10. import signal
  11. import sqlite3
  12. import subprocess
  13. import sys
  14. import time
  15. from dataclasses import dataclass
  16. from dataclasses import field
  17. from pathlib import Path
  18. from typing import Any
  19. from typing import Callable
  20. from typing import Iterable
  21. from typing import Iterator
  22. from typing import Mapping
  23. from typing import Optional
  24. from typing import TYPE_CHECKING
  25. from hermes_cli.quiet_single_query import KANBAN_WORKER_EXIT_TRAILER
  26. if TYPE_CHECKING:
  27. from hermes_cli.kanban_db import Task
  28. # After this many consecutive non-success attempts on a task/profile the
  29. # dispatcher parks the task in ``blocked`` with a reason — prevents retry storms.
  30. DEFAULT_FAILURE_LIMIT = 2
  31. # Keep a little wall-clock budget for the worker to observe a terminal timeout
  32. # and make a terminal board call (kanban_block/kanban_complete/kanban_request_review)
  33. # before max_runtime_seconds kills it.
  34. KANBAN_TERMINAL_TIMEOUT_GRACE_SECONDS = 30
  35. # A healthy worker is still alive for a while after kanban_complete /
  36. # kanban_request_review returns (final assistant turn, session persistence), so
  37. # a run's retained worker is only reaped once ended_at is at least this old
  38. # (two default dispatch ticks).
  39. TERMINAL_WORKER_REAP_GRACE_SECONDS = 120
  40. # ---------------------------------------------------------------------------
  41. # Respawn guard constants
  42. # ---------------------------------------------------------------------------
  43. # Patterns in last_failure_error that indicate a quota / auth blocker.
  44. # These errors won't resolve by retrying immediately — auto-block instead.
  45. # The auth family is a curated list, not an open `auth\w*` stem: that stem
  46. # also matched ordinary English words like "author"/"authored"/"authoring"/
  47. # "authoritative" in worker progress prose, parking a healthy card forever
  48. # (#117009).
  49. _RESPAWN_BLOCKER_RE = re.compile(
  50. r"\b(quota|rate[\s_\-]?limit|429|403|"
  51. r"auth|authenticat(?:e|es|ed|ing|ion)|authoriz(?:e|es|ed|ing|ation)|"
  52. r"authoris(?:e|es|ed|ing|ation)|authz|"
  53. r"unauthorized|forbidden|billing|subscription|"
  54. r"access[\s_]denied|permission[\s_]denied|"
  55. r"invalid[\s_]api[\s_]key)\b",
  56. re.IGNORECASE,
  57. )
  58. # Within this window a completed run counts as "recent proof"; don't re-spawn.
  59. _RESPAWN_GUARD_SUCCESS_WINDOW = 3600 # 1 hour
  60. # Cooldown after a rate-limited (quota-wall) requeue before re-spawning. Without
  61. # it the task would re-spawn on the very next tick and bounce off the same quota
  62. # wall, burning a worker slot every tick for hours. Overridable via
  63. # ``HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS``.
  64. DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS = 300 # 5 minutes
  65. # Within this window a GitHub PR URL in a comment blocks re-spawn.
  66. _RESPAWN_GUARD_PR_WINDOW = 86400 # 24 hours
  67. _RESPAWN_GUARD_PR_URL_RE = re.compile(
  68. r"https?://github\.com/[^/\s]+/[^/\s]+/pull/\d+",
  69. re.IGNORECASE,
  70. )
  71. @dataclass
  72. class DispatchResult:
  73. """Outcome of a single ``dispatch`` pass.
  74. ``kanban.default_assignee`` applied this tick before spawning (#27145). Surfaces the auto-assignment to
  75. telemetry / CLI / dashboard so the operator can see when the dispatcher is acting on the fallback rule
  76. ``kanban.max_in_progress_per_profile`` (#21582). Each entry is ``(task_id, assignee,
  77. current_running_count)``. NOT an operator-actionable failure — the task will be picked up on a
  78. subsequent tick when the assignee has capacity. Separate bucket so telemetry / dashboards can show "this
  79. profile is busy" vs
  80. the board's dispatch lock (issue #35240). A losing dispatcher does no DB writes this tick — the lock
  81. holder is making progress on the same board. This is the steady-state signal that a single-writer guard
  82. is
  83. """
  84. reclaimed: int = 0
  85. promoted: int = 0
  86. reconciled_orphans: list[str] = field(default_factory=list)
  87. """``running`` cards requeued by :func:`reconcile_orphaned_running` (broken
  88. claim bookkeeping, dead/gone worker)."""
  89. reaped_terminal_workers: list[str] = field(default_factory=list)
  90. """Task ids whose worker outlived its closed run and was terminated by
  91. :func:`reap_terminal_workers`."""
  92. spawned: list[tuple[str, str, str]] = field(default_factory=list)
  93. """``(task_id, assignee, workspace_path)`` triples."""
  94. skipped_unassigned: list[str] = field(default_factory=list)
  95. """Ready task ids with no assignee at all — operator-actionable (usually a
  96. misfiled task waiting for routing)."""
  97. auto_assigned_default: list[str] = field(default_factory=list)
  98. """Unassigned task ids that had ``kanban.default_assignee`` applied this
  99. tick before spawning, so telemetry/CLI/dashboard can show the dispatcher
  100. acting on the fallback rule rather than explicit assignments."""
  101. skipped_nonspawnable: list[str] = field(default_factory=list)
  102. """Ready task ids whose assignee names a control-plane lane (e.g. a Claude
  103. Code terminal like ``orion-cc``), not a Hermes profile. Expected steady-state
  104. on multi-lane setups, NOT operator-actionable; tracked apart so health
  105. telemetry can tell "stuck" from "correctly idle"."""
  106. skipped_per_profile_capped: list[tuple[str, str, int]] = field(default_factory=list)
  107. """``(task_id, assignee, current_running_count)`` deferred because the
  108. assignee is at ``kanban.max_in_progress_per_profile``. Picked up on a later
  109. tick; separate bucket so dashboards show "profile busy" vs "stuck"."""
  110. crashed: list[str] = field(default_factory=list)
  111. """Task ids reclaimed because their worker PID disappeared."""
  112. auto_blocked: list[str] = field(default_factory=list)
  113. """Task ids auto-blocked by the spawn-failure circuit breaker."""
  114. timed_out: list[str] = field(default_factory=list)
  115. """Task ids whose workers exceeded ``max_runtime_seconds``."""
  116. stale: list[str] = field(default_factory=list)
  117. """Task ids reclaimed for no heartbeat within ``dispatch_stale_timeout_seconds``."""
  118. respawn_guarded: list[tuple[str, str]] = field(default_factory=list)
  119. """``(task_id, reason)`` skipped by the respawn guard: ``"blocker_auth"``
  120. (quota/auth error — also auto-blocked), ``"recent_success"`` (completed run
  121. within guard window), ``"active_pr"`` (GitHub PR URL in a recent comment)."""
  122. rate_limited: list[str] = field(default_factory=list)
  123. """Task ids whose workers bailed on a provider rate-limit / quota wall
  124. (EX_TEMPFAIL sentinel exit) and were released to ``ready`` WITHOUT counting
  125. a failure — a long quota window must never trip the circuit breaker."""
  126. skipped_locked: bool = False
  127. """True when another process held the board's dispatch lock: this tick did
  128. no DB writes; the lock holder is making progress on the same board."""
  129. memory_pressure: Optional[str] = None
  130. """Memory pressure that restricted this tick: ``"critical"`` (no new
  131. workers), ``"elevated"`` (at most one), ``None`` (no restriction).
  132. Reclaim/promotion bookkeeping still ran; deferred tasks stay queued."""
  133. def describe_suppression(results: Iterable[Optional[DispatchResult]]) -> str:
  134. """One line naming why the tick(s) held ready work back, or ``""``.
  135. ``active_pr=1, recent_success=2, rate_limited=1, skipped_locked=1,
  136. memory_pressure=critical`` — the respawn-guard reasons counted per task
  137. plus the tick-level holds. Feeds the "dispatcher stuck" warnings of the
  138. CLI daemon and the embedded gateway dispatcher, which otherwise report a
  139. bare zero-spawn count while ``hermes kanban tail`` is the only place the
  140. guard reason is written (#111910).
  141. """
  142. counts: dict[str, int] = {}
  143. pressure: Optional[str] = None
  144. for res in results:
  145. if res is None:
  146. continue
  147. for _task_id, reason in res.respawn_guarded:
  148. counts[reason] = counts.get(reason, 0) + 1
  149. if res.rate_limited:
  150. counts["rate_limited"] = counts.get("rate_limited", 0) + len(res.rate_limited)
  151. if res.skipped_locked:
  152. counts["skipped_locked"] = counts.get("skipped_locked", 0) + 1
  153. if res.memory_pressure:
  154. pressure = res.memory_pressure
  155. parts = [f"{k}={v}" for k, v in sorted(counts.items())]
  156. if pressure:
  157. parts.append(f"memory_pressure={pressure}")
  158. return ", ".join(parts)
  159. # Bounded registry of recently-reaped worker exits, filled by the reap loop in
  160. # ``dispatch_once`` and read by ``detect_crashed_workers`` to classify a dead-pid
  161. # task. Entry: ``pid -> (raw_wait_status, reaped_at_epoch)``; raw status kept so
  162. # both WIFEXITED/WEXITSTATUS and WIFSIGNALED can be consulted. Trimmed by age
  163. # plus a total size cap. Process-local by nature (``waitpid`` only reaps our own
  164. # children): a per-tick ``hermes kanban dispatch`` process finds it empty, so
  165. # ``_classify_dead_worker_exit`` falls back to the exit trailer the worker
  166. # leaves in its own log (``KANBAN_WORKER_EXIT_TRAILER``).
  167. _RECENT_WORKER_EXIT_TTL_SECONDS = 600
  168. _RECENT_WORKER_EXITS_MAX = 4096
  169. _recent_worker_exits: dict[int, tuple[int, float]] = {}
  170. # Windows has no ``waitpid(-1)``: a child's exit code is only recoverable
  171. # through a live handle, so ``_default_spawn`` parks each worker's ``Popen``
  172. # here (Windows only) and ``reap_worker_zombies`` polls it. Entry: ``pid -> Popen``.
  173. _live_worker_procs: dict[int, subprocess.Popen] = {}
  174. def _wait_status_from_returncode(returncode: int) -> int:
  175. """Encode a ``Popen.returncode`` in the wait-status layout the registry stores."""
  176. return (int(returncode) & 0xFF) << 8
  177. def _record_worker_exit(pid: int, raw_status: int) -> None:
  178. """Record a reaped child's exit status; duplicate pids overwrite (latest wins)."""
  179. if not pid or pid <= 0:
  180. return
  181. now = time.time()
  182. _recent_worker_exits[int(pid)] = (int(raw_status), now)
  183. if len(_recent_worker_exits) > _RECENT_WORKER_EXITS_MAX // 2:
  184. cutoff = now - _RECENT_WORKER_EXIT_TTL_SECONDS
  185. for _pid in [p for p, (_s, t) in _recent_worker_exits.items() if t < cutoff]:
  186. _recent_worker_exits.pop(_pid, None)
  187. if len(_recent_worker_exits) > _RECENT_WORKER_EXITS_MAX:
  188. # Drop oldest half.
  189. ordered = sorted(_recent_worker_exits.items(), key=lambda kv: kv[1][1])
  190. for _pid, _ in ordered[: len(ordered) // 2]:
  191. _recent_worker_exits.pop(_pid, None)
  192. def _classify_worker_exit(pid: int) -> tuple[str, Optional[int]]:
  193. """``(kind, code)`` for a reaped worker PID: ``clean_exit`` (rc 0 while
  194. still ``running`` = protocol violation), ``rate_limited``
  195. (``KANBAN_RATE_LIMIT_EXIT_CODE``, never counts as a failure),
  196. ``nonzero_exit``, ``signaled`` (``code`` is the signal), ``unknown`` (pid
  197. not in the reap registry; ``code`` None)."""
  198. entry = _recent_worker_exits.get(int(pid))
  199. if entry is None:
  200. return ("unknown", None)
  201. raw, _ = entry
  202. # Bit-level POSIX wait-status decode instead of os.WIFEXITED/WEXITSTATUS/
  203. # WIFSIGNALED/WTERMSIG: those helpers do not exist on Windows, where the
  204. # registry is fed by reap_worker_zombies' Popen poll. Low 7 bits = signal
  205. # (0 = normal exit, 0x7F = stopped), bits 8-15 = exit code.
  206. raw = int(raw)
  207. signal_number = raw & 0x7F
  208. if signal_number == 0:
  209. return _exit_code_kind((raw >> 8) & 0xFF)
  210. if signal_number != 0x7F:
  211. return ("signaled", signal_number)
  212. return ("unknown", None)
  213. def _exit_code_kind(code: int) -> tuple[str, int]:
  214. """``(kind, code)`` for a worker's exit code, however it was observed."""
  215. if code == 0:
  216. return ("clean_exit", 0)
  217. if code == _kb.KANBAN_RATE_LIMIT_EXIT_CODE:
  218. return ("rate_limited", code)
  219. if code == _kb.KANBAN_TERMINAL_PROVIDER_EXIT_CODE:
  220. return ("terminal_provider", code)
  221. return ("nonzero_exit", code)
  222. _EXIT_TRAILER_RE = re.compile(
  223. r"^" + re.escape(KANBAN_WORKER_EXIT_TRAILER) + r"(\d+)\s*$", re.MULTILINE,
  224. )
  225. def _worker_log_exit_code(task_id: str, board: Optional[str] = None) -> Optional[int]:
  226. """Exit code from the trailer the worker CLI wrote to its own log; None when absent.
  227. The durable twin of ``_recent_worker_exits``: written by the worker itself
  228. (``hermes_cli.quiet_single_query.exit_single_query``), so it is there whether
  229. or not the process running this sweep ever reaped the worker. Last trailer
  230. wins — the log is append-mode across re-runs.
  231. """
  232. try:
  233. raw = _kb.read_worker_log(task_id, tail_bytes=4000, board=board)
  234. except Exception:
  235. return None
  236. matches = _EXIT_TRAILER_RE.findall(raw or "")
  237. return int(matches[-1]) if matches else None
  238. def reap_worker_zombies() -> list[int]:
  239. """Reap exited workers without blocking; returns reaped PIDs. POSIX reaps
  240. every child via ``waitpid(-1)``; Windows polls the ``Popen`` handles
  241. parked by ``_default_spawn`` (the only way to learn a child's exit code
  242. there), so the rate-limit sentinel exit is classified on both hosts."""
  243. reaped: list[int] = []
  244. if _kb._IS_WINDOWS:
  245. for pid, proc in list(_live_worker_procs.items()):
  246. returncode = proc.poll()
  247. if returncode is None:
  248. continue
  249. _record_worker_exit(pid, _wait_status_from_returncode(returncode))
  250. _live_worker_procs.pop(pid, None)
  251. reaped.append(pid)
  252. return reaped
  253. try:
  254. while True:
  255. try:
  256. pid, status = os.waitpid(-1, os.WNOHANG)
  257. except ChildProcessError:
  258. break
  259. if pid == 0:
  260. break
  261. _record_worker_exit(pid, status)
  262. reaped.append(pid)
  263. except Exception:
  264. pass
  265. return reaped
  266. def _pid_alive(pid: Optional[int]) -> bool:
  267. """Return True if ``pid`` is still running on this host.
  268. Uses ``gateway.status._pid_exists`` (OpenProcess on Windows, ``os.kill(pid, 0)``
  269. on POSIX). **DO NOT** call ``os.kill(pid, 0)`` directly on Windows — there
  270. ``sig=0`` is ``CTRL_C_EVENT`` broadcast to the console group, potentially
  271. killing unrelated processes.
  272. Zombies (exited, not yet reaped) still pass the existence check, so a
  273. worker would look "alive" forever between exit and reap. Linux: peek at
  274. ``/proc/<pid>/status`` and treat ``State: Z`` as dead; macOS: ask ``ps``
  275. for the BSD ``stat`` field and treat ``Z`` as dead.
  276. """
  277. if not pid or pid <= 0:
  278. return False
  279. from gateway.status import _pid_exists
  280. if not _pid_exists(int(pid)):
  281. return False
  282. if sys.platform == "linux":
  283. try:
  284. with open(f"/proc/{int(pid)}/status", "r", encoding="utf-8") as f:
  285. for line in f:
  286. if line.startswith("State:"):
  287. # "State:\tZ (zombie)" → dead
  288. if "Z" in line.split(":", 1)[1]:
  289. return False
  290. break
  291. except (FileNotFoundError, PermissionError, OSError):
  292. # proc entry gone → already reaped; treat as dead.
  293. pass
  294. elif sys.platform == "darwin":
  295. try:
  296. proc = subprocess.run(
  297. ["ps", "-o", "stat=", "-p", str(int(pid))],
  298. stdout=subprocess.PIPE,
  299. stderr=subprocess.DEVNULL,
  300. text=True, encoding='utf-8', errors='replace',
  301. timeout=1,
  302. check=False,
  303. )
  304. if proc.returncode != 0:
  305. return False
  306. if "Z" in (proc.stdout or "").strip():
  307. return False
  308. except (OSError, subprocess.SubprocessError, TimeoutError):
  309. # If the secondary probe fails, keep the kill(0) answer.
  310. pass
  311. return True
  312. # ``worker_started_at`` value for a spawn whose fingerprint could not be captured. Distinct from the
  313. # NULL legacy row (pre-fingerprint spawn): such a worker is held (its claim is never released beside
  314. # the live PID) but NEVER signalled — missing process identity is refusal, not permission (#99558).
  315. UNVERIFIED_WORKER_FINGERPRINT = "unverified"
  316. def _process_fingerprint(pid: int) -> Optional[str]:
  317. """Restart-stable identity of a live process: ``"<instantiation epoch>|<start time>"``. The start
  318. time alone (``/proc/<pid>/stat`` field 22 on Linux) is clock ticks since THIS boot, so a row that
  319. survives a reboot could match an unrelated process with the same PID and the same tick value;
  320. ``gateway.drain_control.current_instantiation_epoch`` (``boot_id`` + PID-1 start) changes on every
  321. reboot / container recreate, so the composed value never survives one. ``None`` when unreadable."""
  322. from gateway.drain_control import current_instantiation_epoch
  323. from gateway.status import get_process_start_time
  324. start = get_process_start_time(int(pid))
  325. if start is None:
  326. return None
  327. return f"{current_instantiation_epoch()}|{start}"
  328. def _worker_alive(pid: Optional[int], started_at) -> bool:
  329. """True when ``pid`` is live AND is still the worker we spawned. ``started_at`` is the fingerprint
  330. recorded by ``_set_worker_pid``; after a reboot (or any PID recycle) an unrelated process can own
  331. the number, so bare existence is never enough to extend a claim or to signal. A legacy row without
  332. a fingerprint keeps the existence answer: killing it is the pre-fingerprint behaviour and the row is
  333. rewritten with a fingerprint on its next spawn. An UNVERIFIED spawn also keeps the existence answer
  334. (a claim is never released beside a possibly-live worker) but ``_terminate_reclaimed_worker``
  335. refuses to signal it."""
  336. if not _kb._pid_alive(pid):
  337. return False
  338. if started_at == UNVERIFIED_WORKER_FINGERPRINT:
  339. return True
  340. return not _pid_recycled(pid, started_at)
  341. def _pid_recycled(pid: Optional[int], started_at) -> bool:
  342. """True when a live ``pid`` is NOT the process fingerprinted at spawn (or the fingerprint can no
  343. longer be read). Signalling it would hit a stranger. ``None`` fingerprint = legacy row, never
  344. recycled; the UNVERIFIED marker is always foreign. An integer fingerprint (rows written before the
  345. boot witness was added) compares the start time only."""
  346. if started_at is None or not pid:
  347. return False
  348. if started_at == UNVERIFIED_WORKER_FINGERPRINT:
  349. return True
  350. if isinstance(started_at, str) and "|" in started_at:
  351. return _process_fingerprint(int(pid)) != started_at
  352. from gateway.status import _start_times_agree, get_process_start_time
  353. current = get_process_start_time(int(pid))
  354. if current is None:
  355. return True
  356. try:
  357. return not _start_times_agree(current, started_at)
  358. except (TypeError, ValueError):
  359. return True
  360. def _kill_fn(signal_fn) -> Optional[Callable[[int, int], None]]:
  361. """``signal_fn`` test hook, else ``os.kill`` when the platform has one."""
  362. if signal_fn is not None:
  363. return signal_fn
  364. return os.kill if hasattr(os, "kill") else None
  365. def _poll_worker_exit(pid: int, started_at: Optional[int] = None) -> bool:
  366. """Poll ~5 s (10 x 0.5 s) for ``pid`` to die; True once it is gone."""
  367. for _ in range(10):
  368. if not _worker_alive(pid, started_at):
  369. return True
  370. time.sleep(0.5)
  371. return False
  372. def _sigkill(kill, pid: int) -> bool:
  373. """Best-effort SIGKILL; True when the signal was delivered."""
  374. try:
  375. # signal.SIGKILL doesn't exist on Windows; SIGTERM maps to TerminateProcess.
  376. kill(int(pid), getattr(signal, "SIGKILL", signal.SIGTERM))
  377. return True
  378. except (ProcessLookupError, OSError):
  379. return False
  380. def _terminate_reclaimed_worker(
  381. pid: Optional[int],
  382. claim_lock: Optional[str],
  383. *,
  384. signal_fn=None,
  385. started_at=None,
  386. ) -> dict[str, Any]:
  387. """Best-effort host-local worker termination for reclaim paths. ``started_at`` is the spawn-time
  388. fingerprint: when the live process no longer matches it, the PID was recycled and nothing is
  389. signalled — the worker is gone, which is what the reclaim wanted (``terminated`` = True). An
  390. UNVERIFIED spawn (fingerprint capture failed) that is still live is never signalled either, but
  391. it is reported as surviving (``signal_refused``) so the reclaim holds the claim instead of
  392. spawning a duplicate beside it."""
  393. info: dict[str, Any] = {
  394. "prev_pid": int(pid) if pid else None,
  395. "host_local": False,
  396. "termination_attempted": False,
  397. "terminated": False,
  398. "sigkill": False,
  399. }
  400. if not pid or pid <= 0 or not claim_lock:
  401. return info
  402. if not str(claim_lock).startswith(_kb._host_prefix()):
  403. return info
  404. info["host_local"] = True
  405. kill = _kill_fn(signal_fn)
  406. if kill is None:
  407. return info
  408. if started_at == UNVERIFIED_WORKER_FINGERPRINT:
  409. # Never signal by bare number: a dead PID is "gone" (reclaim proceeds), a live one is held.
  410. info["signal_refused"] = True
  411. info["terminated"] = not _kb._pid_alive(pid)
  412. return info
  413. if _kb._pid_alive(pid) and _pid_recycled(pid, started_at):
  414. info["terminated"] = True
  415. info["pid_recycled"] = True
  416. return info
  417. info["termination_attempted"] = True
  418. try:
  419. kill(int(pid), signal.SIGTERM)
  420. except OSError:
  421. # Already gone = successful termination (Windows reports an absent pid as a bare OSError, not
  422. # ProcessLookupError). Leaving terminated=False would make the reclaim defer forever.
  423. info["terminated"] = not _worker_alive(pid, started_at)
  424. return info
  425. if _poll_worker_exit(pid, started_at):
  426. info["terminated"] = True
  427. return info
  428. if _worker_alive(pid, started_at):
  429. if not _sigkill(kill, pid):
  430. return info
  431. info["sigkill"] = True
  432. info["terminated"] = not _worker_alive(pid, started_at)
  433. return info
  434. def reap_terminal_workers(conn: sqlite3.Connection, *, signal_fn=None) -> list[str]:
  435. """End host-local workers that outlived their run (issue #111791) — a worker
  436. that called ``kanban_complete`` and then hung keeps its ``state.db`` sidecar
  437. fds open and no ``running``-only sweep can see it once ``tasks.worker_pid`` is
  438. cleared. Keys on the closed ``task_runs`` row's retained pid + spawn
  439. fingerprint: a legacy row (NULL fingerprint) or a recycled PID is never
  440. signalled; a pid that is simply gone just has its evidence cleared. A run
  441. that ended less than ``TERMINAL_WORKER_REAP_GRACE_SECONDS`` ago is left
  442. alone so a worker still finalising after its own transition is not killed.
  443. One row's failure (signal, /proc probe) is logged and skips only that row.
  444. Returns the task ids whose worker was terminated."""
  445. rows = conn.execute(
  446. "SELECT id, task_id, worker_pid, worker_started_at, claim_lock FROM task_runs "
  447. "WHERE ended_at IS NOT NULL AND ended_at <= ? "
  448. "AND worker_pid IS NOT NULL AND worker_started_at IS NOT NULL",
  449. (int(time.time()) - TERMINAL_WORKER_REAP_GRACE_SECONDS,),
  450. ).fetchall()
  451. host_prefix = _kb._host_prefix()
  452. reaped: list[str] = []
  453. for row in rows:
  454. try:
  455. _reap_terminal_worker_row(conn, row, host_prefix, signal_fn, reaped)
  456. except Exception:
  457. _kb._log.debug(
  458. "kanban dispatch: terminal worker reap failed for run %s (task %s)",
  459. row["id"], row["task_id"], exc_info=True,
  460. )
  461. return reaped
  462. def _reap_terminal_worker_row(conn, row, host_prefix: str, signal_fn, reaped: list[str]) -> None:
  463. pid, fingerprint = int(row["worker_pid"]), row["worker_started_at"]
  464. if pid == os.getpid() or not str(row["claim_lock"] or "").startswith(host_prefix):
  465. return
  466. if fingerprint == UNVERIFIED_WORKER_FINGERPRINT and _kb._pid_alive(pid):
  467. return # unproven identity: never signalled; its evidence is cleared once the pid is gone
  468. alive = _worker_alive(pid, fingerprint)
  469. termination = None
  470. if alive:
  471. termination = _terminate_reclaimed_worker(
  472. pid, row["claim_lock"], signal_fn=signal_fn, started_at=fingerprint)
  473. if not termination["terminated"]:
  474. return # still alive: try again next tick
  475. with _kb.write_txn(conn):
  476. conn.execute(
  477. "UPDATE task_runs SET worker_pid = NULL, worker_started_at = NULL "
  478. "WHERE id = ? AND worker_pid = ? AND worker_started_at = ?",
  479. (row["id"], pid, fingerprint),
  480. )
  481. if alive:
  482. _kb._append_event(
  483. conn, row["task_id"], "terminal_worker_reaped",
  484. {"pid": pid, "worker_started_at": fingerprint, **termination}, run_id=row["id"],
  485. )
  486. if alive:
  487. reaped.append(row["task_id"])
  488. def _worker_survived_termination(termination: dict) -> bool:
  489. """True when we tried to kill our own host-local worker and it is still alive.
  490. Reclaiming then would release the claim and spawn a second worker while the
  491. first still runs — the duplication loop. Only host-local workers we actually
  492. signalled count; a non-local lock or no-op attempt (no ``os.kill``) must fall
  493. through to the normal release path since we cannot manage that worker anyway.
  494. """
  495. return bool(
  496. termination.get("host_local")
  497. and (termination.get("termination_attempted") or termination.get("signal_refused"))
  498. and not termination.get("terminated")
  499. )
  500. def _defer_reclaim_for_live_worker(
  501. conn: sqlite3.Connection,
  502. task_id: str,
  503. claim_lock: Optional[str],
  504. now: int,
  505. termination: dict,
  506. *,
  507. reason: str,
  508. ) -> None:
  509. """Hold a claim whose worker survived termination instead of releasing it.
  510. Extends ``claim_expires`` by ``RECLAIM_DEFER_GRACE_SECONDS`` so the task
  511. stays ``running`` (no duplicate spawn) and records ``reclaim_deferred``.
  512. The next tick retries the kill; not spawning a duplicate is what lets the
  513. throttled worker finally die.
  514. """
  515. grace = now + _kb.RECLAIM_DEFER_GRACE_SECONDS
  516. with _kb.write_txn(conn):
  517. cur = conn.execute(
  518. "UPDATE tasks SET claim_expires = ? "
  519. "WHERE id = ? AND status = 'running' AND claim_lock IS ?",
  520. (grace, task_id, claim_lock),
  521. )
  522. if cur.rowcount != 1:
  523. return
  524. run_id = _kb._current_run_id(conn, task_id)
  525. if run_id is not None:
  526. conn.execute("UPDATE task_runs SET claim_expires = ? WHERE id = ?", (grace, run_id))
  527. payload = {"reason": reason, "claim_lock": claim_lock, "claim_expires_now": grace}
  528. payload.update(termination)
  529. _kb._append_event(conn, task_id, "reclaim_deferred", payload, run_id=run_id)
  530. def heartbeat_worker(
  531. conn: sqlite3.Connection,
  532. task_id: str,
  533. *,
  534. note: Optional[str] = None,
  535. expected_run_id: Optional[int] = None,
  536. ) -> bool:
  537. """Record a ``heartbeat`` event + touch ``last_heartbeat_at``.
  538. Liveness signal orthogonal to the PID check: a worker whose forked child
  539. (train loop, crawl) is stuck can still have a live Python process.
  540. Returns False if the task is not running or its claim expired.
  541. """
  542. now = int(time.time())
  543. with _kb.write_txn(conn):
  544. sql = "UPDATE tasks SET last_heartbeat_at = ? WHERE id = ? AND status = 'running'"
  545. params: tuple = (now, task_id)
  546. if expected_run_id is not None:
  547. sql += " AND current_run_id = ?"
  548. params += (int(expected_run_id),)
  549. cur = conn.execute(sql, params)
  550. if cur.rowcount != 1:
  551. return False
  552. run_id = (
  553. int(expected_run_id)
  554. if expected_run_id is not None
  555. else _kb._current_run_id(conn, task_id)
  556. )
  557. if run_id is not None:
  558. conn.execute("UPDATE task_runs SET last_heartbeat_at = ? WHERE id = ?", (now, run_id))
  559. _kb._append_event(
  560. conn, task_id, "heartbeat",
  561. {"note": note} if note else None,
  562. run_id=run_id,
  563. )
  564. return True
  565. def enforce_max_runtime(conn: sqlite3.Connection, *, signal_fn=None) -> list[str]:
  566. """Terminate workers whose per-task ``max_runtime_seconds`` has elapsed.
  567. SIGTERM, short grace, then SIGKILL. Emits ``timed_out`` and restores the
  568. task's source phase so the next tick re-spawns the same kind of worker —
  569. unless the circuit breaker already gave up, leaving it blocked. Host-local
  570. only (same reasoning as ``detect_crashed_workers``). ``signal_fn`` is a test hook.
  571. """
  572. timed_out: list[str] = []
  573. now = int(time.time())
  574. host_prefix = _kb._host_prefix()
  575. rows = conn.execute(
  576. "SELECT t.id, t.worker_pid, t.worker_started_at, "
  577. " COALESCE(r.started_at, t.started_at) AS active_started_at, "
  578. " t.max_runtime_seconds, t.claim_lock "
  579. "FROM tasks t "
  580. "LEFT JOIN task_runs r ON r.id = t.current_run_id "
  581. "WHERE t.status = 'running' AND t.max_runtime_seconds IS NOT NULL "
  582. " AND COALESCE(r.started_at, t.started_at) IS NOT NULL "
  583. " AND t.worker_pid IS NOT NULL"
  584. ).fetchall()
  585. from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked, owner_reclaim_blocked_after_kill
  586. for row in rows:
  587. if owner_reclaim_blocked(conn, row["id"]):
  588. continue
  589. lock = row["claim_lock"] or ""
  590. if not lock.startswith(host_prefix):
  591. continue
  592. # Runtime is per attempt: ``tasks.started_at`` records the FIRST start,
  593. # so retries must be measured from the active task_runs row.
  594. elapsed = now - (_kb._to_epoch(row["active_started_at"]) or now) # unreadable clock: never kill
  595. limit = int(row["max_runtime_seconds"])
  596. if elapsed < limit:
  597. continue
  598. pid = int(row["worker_pid"])
  599. tid = row["id"]
  600. started_at = _kb._row_get(row, "worker_started_at")
  601. if started_at == UNVERIFIED_WORKER_FINGERPRINT and _kb._pid_alive(pid):
  602. # Fingerprint capture failed at spawn: we cannot prove this live PID is our worker, so
  603. # it is neither signalled nor released beside (duplicate). It is reclaimed once it exits.
  604. _kb._log.warning("kanban: task %s worker pid %s exceeded max runtime but has no verified "
  605. "identity; not signalled", tid, pid)
  606. continue
  607. # SIGTERM then SIGKILL after 5 s grace; workers wanting a cleaner
  608. # shutdown install their own SIGTERM handler. A recycled PID (fingerprint
  609. # mismatch) is never signalled: the worker is already gone.
  610. killed = False
  611. kill = _kill_fn(signal_fn)
  612. if kill is not None and not (_kb._pid_alive(pid) and _pid_recycled(pid, started_at)):
  613. with contextlib.suppress(ProcessLookupError, OSError):
  614. kill(pid, signal.SIGTERM)
  615. # Short polling wait — no time.sleep on the write txn.
  616. _poll_worker_exit(pid, started_at)
  617. if _worker_alive(pid, started_at):
  618. killed = _sigkill(kill, pid)
  619. error = f"elapsed {int(elapsed)}s > limit {limit}s"
  620. with _kb.write_txn(conn):
  621. # Again under the write lock, after the kill wait: an interpreter that bound
  622. # meanwhile is invisible to the pid/claim CAS below. Host-local by the filter above.
  623. if owner_reclaim_blocked_after_kill(conn, tid, signalled=True):
  624. continue
  625. retry_status = _kb._retry_status_for_run(conn, tid)
  626. cur = conn.execute(
  627. "UPDATE tasks SET status = ?, claim_lock = NULL, "
  628. "claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "
  629. "last_heartbeat_at = NULL "
  630. "WHERE id = ? AND status = 'running' "
  631. " AND worker_pid = ? AND claim_lock IS ?",
  632. (retry_status, tid, pid, row["claim_lock"]),
  633. )
  634. if cur.rowcount == 1:
  635. payload = {
  636. "pid": pid,
  637. "elapsed_seconds": int(elapsed),
  638. "limit_seconds": limit,
  639. "sigkill": killed,
  640. "retry_status": retry_status,
  641. }
  642. run_id = _kb._end_run(
  643. conn, tid, outcome="timed_out", status="timed_out",
  644. error=error, metadata=payload,
  645. )
  646. _kb._append_event(conn, tid, "timed_out", payload, run_id=run_id)
  647. timed_out.append(tid)
  648. # Outside the write_txn above because ``_record_task_failure`` opens its
  649. # own. If the breaker trips this flips the task to ``blocked`` and emits
  650. # ``gave_up`` on top of the ``timed_out`` already emitted.
  651. if cur.rowcount == 1:
  652. _record_task_failure(
  653. conn, tid,
  654. error=error,
  655. outcome="timed_out",
  656. release_claim=False,
  657. end_run=False,
  658. event_payload_extra={"pid": pid, "sigkill": killed, "retry_status": retry_status},
  659. )
  660. return timed_out
  661. # A running task with no heartbeat for this long is inactive regardless of
  662. # ``dispatch_stale_timeout_seconds`` (spec: ">4h started + no commits in 1h").
  663. _STALE_HEARTBEAT_GAP_SECONDS = 3600
  664. def detect_stale_running(
  665. conn: sqlite3.Connection,
  666. *,
  667. stale_timeout_seconds: int = 0,
  668. signal_fn=None,
  669. ) -> list[str]:
  670. """Reclaim ``running`` tasks with no heartbeat progress; returns their ids.
  671. Stale = running longer than ``stale_timeout_seconds`` (active run's
  672. ``started_at``, else ``tasks.started_at``) AND ``last_heartbeat_at`` NULL or
  673. older than ``_STALE_HEARTBEAT_GAP_SECONDS``. Task returns to its source
  674. phase, run closes ``outcome='stale'``, a live host-local worker is killed.
  675. ``0`` disables the check; ``signal_fn`` is a test hook. Deliberately NOT
  676. counted via ``_record_task_failure``: an absent heartbeat is not a worker
  677. failure, and counting it would let long-running tasks trip the breaker.
  678. """
  679. if stale_timeout_seconds <= 0:
  680. return []
  681. now = int(time.time())
  682. reclaimed: list[str] = []
  683. rows = conn.execute(
  684. "SELECT t.id, t.worker_pid, t.worker_started_at, t.last_heartbeat_at, t.claim_lock, "
  685. " COALESCE(r.started_at, t.started_at) AS active_started_at "
  686. "FROM tasks t "
  687. "LEFT JOIN task_runs r ON r.id = t.current_run_id "
  688. "WHERE t.status = 'running'"
  689. ).fetchall()
  690. from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked, owner_reclaim_blocked_after_kill
  691. for row in rows:
  692. if owner_reclaim_blocked(conn, row["id"]):
  693. continue
  694. if row["active_started_at"] is None:
  695. continue
  696. elapsed = now - (_kb._to_epoch(row["active_started_at"]) or now) # unreadable clock: never reclaim
  697. if elapsed < stale_timeout_seconds:
  698. continue
  699. last_hb = _kb._to_epoch(row["last_heartbeat_at"])
  700. hb_age = (now - last_hb) if last_hb is not None else None
  701. if hb_age is not None and hb_age < _STALE_HEARTBEAT_GAP_SECONDS:
  702. continue
  703. pid = row["worker_pid"]
  704. tid = row["id"]
  705. lock = row["claim_lock"] or ""
  706. termination = _kb._terminate_reclaimed_worker(
  707. pid, lock, signal_fn=signal_fn, started_at=_kb._row_get(row, "worker_started_at"))
  708. # Never release a claim while our own worker is still alive: that would
  709. # spawn a duplicate beside it. Hold the claim and retry next tick.
  710. if _worker_survived_termination(termination):
  711. _defer_reclaim_for_live_worker(
  712. conn, tid, lock, now, termination,
  713. reason="heartbeat_stale_worker_alive",
  714. )
  715. continue
  716. with _kb.write_txn(conn):
  717. # Again under the write lock, after the termination wait (see enforce_max_runtime).
  718. if owner_reclaim_blocked_after_kill(conn, tid, signalled=bool(termination.get("termination_attempted"))):
  719. continue
  720. retry_status = _kb._retry_status_for_run(conn, tid)
  721. cur = conn.execute(
  722. "UPDATE tasks SET status = ?, claim_lock = NULL, "
  723. "claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "
  724. "last_heartbeat_at = NULL "
  725. "WHERE id = ? AND status = 'running' "
  726. " AND claim_lock IS ?",
  727. (retry_status, tid, row["claim_lock"]),
  728. )
  729. if cur.rowcount != 1:
  730. continue
  731. payload = {
  732. "elapsed_seconds": int(elapsed),
  733. "last_heartbeat_at": _kb._opt_int(last_hb),
  734. "heartbeat_age_seconds": _kb._opt_int(hb_age),
  735. "timeout_seconds": stale_timeout_seconds,
  736. "pid": int(pid) if pid else None,
  737. "retry_status": retry_status,
  738. }
  739. payload.update(termination)
  740. run_id = _kb._end_run(
  741. conn, tid,
  742. outcome="stale", status="stale",
  743. error=(
  744. f"no heartbeat for {int(hb_age)}s "
  745. if hb_age is not None
  746. else "no heartbeat ever"
  747. ) + f" after {int(elapsed)}s running",
  748. metadata=payload,
  749. )
  750. _kb._append_event(conn, tid, "stale", payload, run_id=run_id)
  751. reclaimed.append(tid)
  752. return reclaimed
  753. def reconcile_orphaned_running(conn: sqlite3.Connection) -> list[str]:
  754. """Requeue ``running`` cards with broken claim bookkeeping; returns their ids.
  755. A task ``running`` with NULL ``claim_lock``/``claim_expires`` (crash
  756. mid-claim, manual SQL, DB restore) is a zombie forever: ``release_stale_claims``
  757. needs ``claim_expires``, ``detect_crashed_workers`` needs a host-local lock +
  758. pid, ``detect_stale_running`` is off by default. Orphans go back to ``ready``
  759. with a comment, leaked run closed, ``reconciled`` event; a row with a live
  760. host-local PID is deferred so no duplicate spawns beside it.
  761. """
  762. now = int(time.time())
  763. reconciled: list[str] = []
  764. rows = conn.execute(
  765. "SELECT id, claim_lock, claim_expires, worker_pid, worker_started_at FROM tasks "
  766. "WHERE status = 'running' "
  767. " AND (claim_lock IS NULL OR claim_expires IS NULL)"
  768. ).fetchall()
  769. from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked
  770. for row in rows:
  771. if owner_reclaim_blocked(conn, row["id"]):
  772. continue
  773. tid = row["id"]
  774. pid = row["worker_pid"]
  775. if pid and _worker_alive(pid, _kb._row_get(row, "worker_started_at")):
  776. # Never requeue beside a live process. Retry next tick.
  777. _kb._log.debug(
  778. "kanban reconcile: task %s has broken claim bookkeeping but "
  779. "pid %s is alive on this host — deferring", tid, pid,
  780. )
  781. continue
  782. with _kb.write_txn(conn):
  783. if owner_reclaim_blocked(conn, tid):
  784. continue
  785. cur = conn.execute(
  786. "UPDATE tasks SET status = 'ready', claim_lock = NULL, "
  787. "claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "
  788. "last_heartbeat_at = NULL "
  789. "WHERE id = ? AND status = 'running' "
  790. " AND claim_lock IS ? AND claim_expires IS ?",
  791. (tid, row["claim_lock"], row["claim_expires"]),
  792. )
  793. if cur.rowcount != 1:
  794. continue
  795. payload = {
  796. "reason": "orphaned_running",
  797. "claim_lock": row["claim_lock"],
  798. "claim_expires": _kb._to_epoch(row["claim_expires"]),
  799. "worker_pid": int(pid) if pid else None,
  800. "now": now,
  801. }
  802. run_id = _kb._end_run(
  803. conn, tid,
  804. outcome="reclaimed", status="reclaimed",
  805. error="orphaned running card (broken claim bookkeeping)",
  806. metadata=payload,
  807. )
  808. _kb._insert_comment(
  809. conn, tid, "dispatcher",
  810. "reconciliation: card was 'running' with no valid claim "
  811. "(dead/gone worker) — requeued to ready",
  812. now,
  813. )
  814. _kb._append_event(conn, tid, "reconciled", payload, run_id=run_id)
  815. reconciled.append(tid)
  816. _kb._log.info(
  817. "kanban reconcile: requeued orphaned running task %s "
  818. "(claim_lock=%r, worker_pid=%r)", tid, row["claim_lock"], pid,
  819. )
  820. return reconciled
  821. def _error_fingerprint(error_text: str) -> str:
  822. """Normalize an error message (strip PIDs, timestamps) so same-root-cause errors group."""
  823. fp = re.sub(r'\bpid \d+\b', 'pid N', error_text[:80])
  824. fp = re.sub(r'\b\d{10,}\b', '<TS>', fp)
  825. return fp.lower().strip()
  826. # ~96% of "clean exit without a terminal tool call" tasks complete on a later
  827. # run, so a protocol violation gets a bounded retry before the breaker trips.
  828. # The budget is a violation-only STREAK (``_protocol_violation_streak``),
  829. # independent of ``consecutive_failures``: other failure kinds neither consume
  830. # nor extend it. Per-task ``max_retries`` overrides it.
  831. _PROTOCOL_VIOLATION_FAILURE_LIMIT = 3
  832. # Closed runs to walk when counting the streak; it trips at a handful anyway.
  833. _PROTOCOL_VIOLATION_SCAN_LIMIT = 50
  834. def _protocol_violation_streak(conn: sqlite3.Connection, task_id: str) -> int:
  835. """Count the task's trailing run of clean-exit protocol violations.
  836. Walks closed runs newest-first (including the one ``detect_crashed_workers``
  837. just closed). ``rate_limited`` runs are neutral and skipped (a quota wall
  838. says nothing about the task); any other closed run breaks the streak, so
  839. the budget counts ONLY protocol violations. Violations are recognized by the
  840. ``protocol_violation`` run-metadata marker, with the error text as fallback
  841. for runs recorded before the marker existed.
  842. """
  843. streak = 0
  844. rows = conn.execute(
  845. "SELECT outcome, error, metadata FROM task_runs "
  846. "WHERE task_id = ? AND ended_at IS NOT NULL "
  847. "ORDER BY id DESC LIMIT ?",
  848. (task_id, _PROTOCOL_VIOLATION_SCAN_LIMIT),
  849. ).fetchall()
  850. for row in rows:
  851. outcome = row["outcome"] or ""
  852. if outcome == "rate_limited":
  853. continue
  854. head = (row["error"] or "").partition(WORKER_OUTPUT_SEP)[0] # never the worker's own words
  855. if outcome == "crashed" and (
  856. _kb._json_dict(row["metadata"]).get("protocol_violation") or "protocol violation" in head):
  857. streak += 1
  858. continue
  859. break
  860. return streak
  861. WORKER_OUTPUT_SEP = " Worker's last output: " # dead-worker error text: our words, then the worker's
  862. _PROTOCOL_VIOLATION_ERROR = (
  863. # Worker subprocess returned 0 but its task is still ``running`` in the DB — it exited without calling
  864. # ``kanban_complete`` / ``kanban_block`` / ``kanban_request_review``. Overwhelmingly the work itself succeeded and only the
  865. # paperwork was skipped, so a retry usually completes; the corrective sentence below is surfaced to the
  866. # retry worker via the prior-attempt error in ``build_worker_context`` (guidance approach from #61817).
  867. # Keep this short: ``_record_task_failure`` caps the stored error at 500 chars and the worker's own
  868. # last output (``_worker_final_output``, up to 400 chars) is appended after it — a longer preamble
  869. # truncates away the worker's explanation, which is the part the board and the retry worker need.
  870. "worker exited cleanly (rc=0) without kanban_complete, kanban_block "
  871. "or kanban_request_review — protocol violation. "
  872. "If the prior run already did the work, verify it and "
  873. "report it via kanban_complete (or kanban_request_review); "
  874. "a run without a terminal kanban call counts as failed no "
  875. "matter what it did."
  876. )
  877. # Rich panel/rule chrome around the rendered response, and the CLI's own preamble lines.
  878. _LOG_CHROME = re.compile(r"[─━═╭╮╰╯│┃┌┐└┘]+|☤\s*Hermes")
  879. def _exit_summary_marker() -> str:
  880. """The CLI exit-summary header (``cli_session_mixin.show_exit_summary``), in the active language."""
  881. from agent.i18n import t
  882. return t("cli.session.exit_resume_hint")
  883. def _log_noise_prefixes() -> tuple[str, ...]:
  884. from agent.i18n import t
  885. return ("session_id:", "Query:", t("cli.chat.initializing_agent"))
  886. def _worker_final_output(task_id: str, board: Optional[str] = None) -> str:
  887. """Best-effort read of a dead worker's last printed text, for the board diagnostic.
  888. A ``chat -q`` worker's stdout/stderr are redirected to its per-task log
  889. (``_default_spawn``), so when it exits without a terminal board call the
  890. reason is usually sitting there: the model's own explanation of why it could
  891. not comply (#88603), or the rendered provider error (#46593). The reap used to
  892. discard it in favour of a canned message on every retry. Trims the CLI exit
  893. summary, rule lines and the ``session_id:`` trailer; returns "" (never raises)
  894. on a missing/empty log.
  895. ``board`` must come from the dispatching tick: ambient current-board resolution
  896. is wrong for every board but the one the dispatcher thread happens to call
  897. "current", so the log would silently not be found.
  898. """
  899. try:
  900. raw = _kb.read_worker_log(task_id, tail_bytes=4000, board=board)
  901. except Exception:
  902. return ""
  903. if not raw:
  904. return ""
  905. raw = _EXIT_TRAILER_RE.sub("", raw)
  906. cut = raw.rfind(_exit_summary_marker())
  907. if cut != -1:
  908. raw = raw[:cut]
  909. lines = []
  910. for ln in raw.splitlines():
  911. ln = _LOG_CHROME.sub("", ln).strip()
  912. if ln and not ln.startswith(_log_noise_prefixes()):
  913. lines.append(ln)
  914. return " ".join(lines)[-400:]
  915. @dataclass
  916. class _DeadWorker:
  917. """How ``detect_crashed_workers`` should book one dead worker."""
  918. kind: str
  919. code: Optional[int]
  920. error_text: str
  921. event_kind: str
  922. event_payload: dict
  923. protocol_violation: bool = False
  924. rate_limited: bool = False
  925. terminal_provider: bool = False
  926. """``KANBAN_TERMINAL_PROVIDER_EXIT_CODE``: the provider rejected the worker's
  927. credential/model — trips the breaker on this first occurrence."""
  928. @property
  929. def run_outcome(self) -> str:
  930. # A rate-limited requeue is recorded as ``rate_limited`` so board history
  931. # doesn't show a phantom crash for a quota wall.
  932. return "rate_limited" if self.rate_limited else "crashed"
  933. def _classify_dead_worker(
  934. pid: int, claimer: Optional[str], exit_code=None, *, task_id: Optional[str] = None,
  935. board: Optional[str] = None,
  936. ) -> _DeadWorker:
  937. """Map a dead worker's reaped exit status to its reclaim bookkeeping.
  938. A clean exit or a crash carries the worker's own last output (``worker_output``
  939. in the event payload, appended to the error text) so the board and the retry
  940. worker see WHY instead of a bare label; a rate-limited requeue does not need it.
  941. """
  942. dead = _classify_dead_worker_exit(pid, claimer, exit_code, task_id=task_id, board=board)
  943. if task_id and not dead.rate_limited:
  944. worker_output = _worker_final_output(task_id, board=board)
  945. if worker_output:
  946. dead.error_text += f"{WORKER_OUTPUT_SEP}{worker_output!r}"
  947. dead.event_payload["worker_output"] = worker_output
  948. return dead
  949. def _classify_dead_worker_exit(
  950. pid: int,
  951. claimer: Optional[str],
  952. exit_code=None,
  953. *,
  954. task_id: Optional[str] = None,
  955. board: Optional[str] = None,
  956. ) -> _DeadWorker:
  957. """Exit status -> reclaim bookkeeping, before the worker's own words are folded in.
  958. ``exit_code`` is the authority's exact-run result for a managed worker (a child of the
  959. profile owner, never of this dispatcher, so it cannot be reaped here). Otherwise the reap
  960. registry only knows children of THIS process; a per-tick dispatcher reads the exit trailer
  961. the worker left in its log instead, so the same death gets the same booking (protocol
  962. violation / rate-limit requeue / crash) as under the gateway-embedded dispatcher. A worker
  963. that never reached its exit epilogue (killed, OOM) leaves no trailer and stays a plain crash.
  964. """
  965. kind, code = _classify_worker_exit(pid)
  966. if exit_code is not None:
  967. kind, code = _exit_code_kind(int(exit_code))
  968. elif kind == "unknown" and task_id:
  969. logged = _worker_log_exit_code(task_id, board=board)
  970. if logged is not None:
  971. kind, code = _exit_code_kind(logged)
  972. if kind == "clean_exit":
  973. # rc=0 while still ``running``: usually the work succeeded and only the
  974. # paperwork was skipped; the corrective sentence reaches the retry
  975. # worker via ``build_worker_context``.
  976. return _DeadWorker(
  977. kind, code, _PROTOCOL_VIOLATION_ERROR, "protocol_violation",
  978. # ``protocol_violation`` is the durable marker for
  979. # _protocol_violation_streak: _end_run copies this payload into the
  980. # run metadata.
  981. {"pid": pid, "claimer": claimer, "exit_code": code, "protocol_violation": True},
  982. protocol_violation=True,
  983. )
  984. if kind == "rate_limited":
  985. # Quota wall — NOT a task failure. Release to the source phase and do
  986. # NOT count a failure so a long quota window can't trip the breaker.
  987. return _DeadWorker(
  988. kind, code,
  989. f"pid {pid} exited rate-limited (quota wall) — requeued without counting a failure",
  990. "rate_limited",
  991. {"pid": pid, "claimer": claimer, "exit_code": code},
  992. rate_limited=True,
  993. )
  994. if kind == "terminal_provider":
  995. # The worker classified its own provider failure as unhealable (credential
  996. # revoked, model gone): every further spawn would hit the same wall, so
  997. # ``_account_crashes`` trips the breaker now instead of after ``failure_limit``.
  998. return _DeadWorker(
  999. kind, code,
  1000. f"pid {pid} exited on a terminal provider error (exit {code}): the provider rejected "
  1001. "this profile's credential or model — fix the configuration, then unblock.",
  1002. "crashed",
  1003. {"pid": pid, "claimer": claimer, "exit_kind": kind, "exit_code": code, "terminal_provider": True},
  1004. terminal_provider=True,
  1005. )
  1006. if kind == "nonzero_exit":
  1007. error_text = f"pid {pid} exited with code {code}"
  1008. elif kind == "signaled":
  1009. error_text = f"pid {pid} killed by signal {code}"
  1010. else:
  1011. error_text = f"pid {pid} not alive"
  1012. event_payload = {"pid": pid, "claimer": claimer}
  1013. if code is not None and kind != "unknown":
  1014. event_payload["exit_kind"] = kind
  1015. event_payload["exit_code"] = code
  1016. return _DeadWorker(kind, code, error_text, "crashed", event_payload)
  1017. @dataclass
  1018. class _CrashSweep:
  1019. """Everything ``detect_crashed_workers`` collects inside its reclaim txn."""
  1020. crashed: list[str] = field(default_factory=list)
  1021. rate_limited: list[str] = field(default_factory=list)
  1022. # ``(task_id, pid, claimer, dead_worker)``: accounted after the txn via
  1023. # ``_record_task_failure`` (needs its own write_txn).
  1024. crash_details: list[tuple[str, int, str, _DeadWorker]] = field(default_factory=list)
  1025. # Worker-exit observer payloads, fired only after every reclaim/accounting
  1026. # txn has committed.
  1027. exited_hook_payloads: list[dict] = field(default_factory=list)
  1028. def _reclaim_dead_workers(conn: sqlite3.Connection, board: Optional[str] = None) -> _CrashSweep:
  1029. """Release every host-local ``running`` task whose worker PID is dead."""
  1030. sweep = _CrashSweep()
  1031. with _kb.write_txn(conn):
  1032. rows = conn.execute(
  1033. "SELECT id, worker_pid, worker_started_at, claim_lock, started_at, assignee "
  1034. "FROM tasks "
  1035. "WHERE status = 'running' AND worker_pid IS NOT NULL"
  1036. ).fetchall()
  1037. host_prefix = _kb._host_prefix()
  1038. from hermes_cli.kanban_owner_recovery import apply_attempt_note, managed_run_exit_code, owner_reclaim_blocked
  1039. for row in rows:
  1040. lock = row["claim_lock"] or ""
  1041. if not lock.startswith(host_prefix):
  1042. continue
  1043. # Launch-window grace so a freshly-spawned worker isn't reclaimed
  1044. # before its PID is visible on /proc.
  1045. started_at = _kb._to_epoch(_kb._row_get(row, "started_at"))
  1046. if started_at is not None and time.time() - started_at < _kb._resolve_crash_grace_seconds():
  1047. continue
  1048. if _worker_alive(row["worker_pid"], _kb._row_get(row, "worker_started_at")):
  1049. continue
  1050. # An ambiguous receipt holds the card while the attempt may still run; once nothing
  1051. # can be executing it (a dead bound interpreter, or a run that never bound one): crash.
  1052. if owner_reclaim_blocked(conn, row["id"], crash_sweep=True):
  1053. continue
  1054. pid = int(row["worker_pid"])
  1055. # A managed run's authority-recorded exit (its bound interpreter's, under this claim).
  1056. exit_code = managed_run_exit_code(conn, row["id"], pid, row["claim_lock"])
  1057. dead = _classify_dead_worker(pid, row["claim_lock"], exit_code, task_id=row["id"], board=board)
  1058. apply_attempt_note(conn, row["id"], pid, dead)
  1059. retry_status = _kb._retry_status_for_run(conn, row["id"])
  1060. dead.event_payload["retry_status"] = retry_status
  1061. cur = conn.execute(
  1062. "UPDATE tasks SET status = ?, claim_lock = NULL, "
  1063. "claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL "
  1064. "WHERE id = ? AND status = 'running' "
  1065. " AND worker_pid = ? AND claim_lock IS ?",
  1066. (retry_status, row["id"], pid, row["claim_lock"]),
  1067. )
  1068. if cur.rowcount != 1:
  1069. continue
  1070. run_id = _kb._end_run(
  1071. conn, row["id"],
  1072. outcome=dead.run_outcome, status=dead.run_outcome,
  1073. error=dead.error_text,
  1074. metadata=dict(dead.event_payload),
  1075. )
  1076. _kb._append_event(conn, row["id"], dead.event_kind, dead.event_payload, run_id=run_id)
  1077. sweep.exited_hook_payloads.append({
  1078. "task_id": row["id"],
  1079. "assignee": row["assignee"],
  1080. "run_id": run_id,
  1081. "worker_pid": pid,
  1082. "exit_kind": dead.kind,
  1083. "exit_code": dead.code,
  1084. "outcome": dead.run_outcome,
  1085. "retry_status": retry_status,
  1086. })
  1087. if dead.rate_limited or dead.protocol_violation:
  1088. # Stamp last_failure_error WITHOUT touching ``consecutive_failures``:
  1089. # a rate-limited requeue must show ``check_respawn_guard`` a quota
  1090. # blocker; a below-budget protocol violation never reaches
  1091. # ``_record_task_failure`` (which stamps this column), yet the
  1092. # board UI and retry worker need the corrective message.
  1093. conn.execute(
  1094. "UPDATE tasks SET last_failure_error = ? WHERE id = ?",
  1095. (dead.error_text[:500], row["id"]),
  1096. )
  1097. if dead.rate_limited:
  1098. sweep.rate_limited.append(row["id"])
  1099. else:
  1100. sweep.crashed.append(row["id"])
  1101. sweep.crash_details.append((row["id"], pid, row["claim_lock"], dead))
  1102. return sweep
  1103. def _account_crashes(conn: sqlite3.Connection, crash_details: list) -> list[str]:
  1104. """Count each crash against the breaker; returns the task ids it tripped.
  1105. Protocol violations get a BOUNDED violation-only budget independent of
  1106. ``consecutive_failures`` (per-task ``max_retries`` takes precedence);
  1107. systemic same-error crashes (>= 3 identical fingerprints this tick) and
  1108. terminal provider errors (credential revoked, model gone — a retry cannot
  1109. heal them) trip immediately.
  1110. """
  1111. auto_blocked: list[str] = []
  1112. fp_counts: dict[str, int] = {}
  1113. for _, _, _, dead in crash_details:
  1114. fp = _error_fingerprint(dead.error_text)
  1115. fp_counts[fp] = fp_counts.get(fp, 0) + 1
  1116. for tid, pid, claimer, dead in crash_details:
  1117. error_text = dead.error_text
  1118. if dead.protocol_violation:
  1119. streak = _protocol_violation_streak(conn, tid)
  1120. trow = conn.execute("SELECT max_retries FROM tasks WHERE id = ?", (tid,)).fetchone()
  1121. if trow is None:
  1122. continue # task deleted mid-loop
  1123. task_override = _kb._row_get(trow, "max_retries")
  1124. violation_limit = (
  1125. int(task_override) if task_override is not None else _PROTOCOL_VIOLATION_FAILURE_LIMIT
  1126. )
  1127. if streak < violation_limit:
  1128. # Below budget: already back at ``ready`` with the error stamped.
  1129. # No ``_record_task_failure`` — must not consume the unified budget.
  1130. continue
  1131. # ``force_trip``: the decision (incl. per-task ``max_retries``) was
  1132. # already made against the violation streak above.
  1133. tripped = _record_task_failure(
  1134. conn, tid,
  1135. error=error_text,
  1136. outcome="crashed",
  1137. failure_limit=violation_limit,
  1138. force_trip=True,
  1139. release_claim=False,
  1140. end_run=False,
  1141. event_payload_extra={
  1142. "pid": pid,
  1143. "claimer": claimer,
  1144. "protocol_violations": streak,
  1145. "protocol_violation_limit": violation_limit,
  1146. },
  1147. )
  1148. elif dead.terminal_provider:
  1149. # A retry cannot heal a revoked credential or a missing model, so
  1150. # the whole ``failure_limit`` budget would be spent on identical
  1151. # failures. ``force_trip`` blocks now, sticky: ``recompute_ready``
  1152. # must not auto-resume it before the operator fixes the provider.
  1153. tripped = _record_task_failure(
  1154. conn, tid,
  1155. error=error_text,
  1156. outcome="crashed",
  1157. force_trip=True,
  1158. release_claim=False,
  1159. end_run=False,
  1160. event_payload_extra={"pid": pid, "claimer": claimer, "terminal_provider": True},
  1161. )
  1162. else:
  1163. is_systemic = fp_counts.get(_error_fingerprint(error_text), 0) >= 3
  1164. extra = {"pid": pid, "claimer": claimer}
  1165. if is_systemic:
  1166. # Trips at 1, below any ``failure_limit``: hold it for an operator.
  1167. extra["sticky"] = True
  1168. tripped = _record_task_failure(
  1169. conn, tid,
  1170. error=error_text,
  1171. outcome="crashed",
  1172. failure_limit=1 if is_systemic else None,
  1173. release_claim=False,
  1174. end_run=False,
  1175. event_payload_extra=extra,
  1176. )
  1177. if tripped:
  1178. auto_blocked.append(tid)
  1179. return auto_blocked
  1180. def detect_crashed_workers(conn: sqlite3.Connection, board: Optional[str] = None) -> list[str]:
  1181. """Reclaim ``running`` tasks whose worker PID is no longer alive.
  1182. Restores the source phase immediately (no waiting for the claim TTL), for
  1183. tasks claimed by *this host* only — other hosts' PIDs are meaningless.
  1184. Clean exit while ``running`` is a protocol violation with a bounded
  1185. violation-only retry budget; ``KANBAN_RATE_LIMIT_EXIT_CODE`` is a quota
  1186. wall, released WITHOUT counting a failure and surfaced via the
  1187. ``_last_rate_limited`` attribute (the return stays crashed-only).
  1188. """
  1189. sweep = _reclaim_dead_workers(conn, board=board)
  1190. # Outside the main txn: account each crash and maybe trip the breaker.
  1191. auto_blocked = _account_crashes(conn, sweep.crash_details) if sweep.crash_details else []
  1192. # Side-channel attributes keep the public ``list[str]`` return stable;
  1193. # ``dispatch_once`` reads them to populate ``DispatchResult``. Rate-limited
  1194. # requeues did NOT count a failure and are NOT crashes.
  1195. detect_crashed_workers._last_auto_blocked = auto_blocked # type: ignore[attr-defined]
  1196. detect_crashed_workers._last_rate_limited = sweep.rate_limited # type: ignore[attr-defined]
  1197. # Fired only now, after the reclaim txn AND breaker accounting have
  1198. # committed, so subscribers always observe fully durable board state.
  1199. if sweep.exited_hook_payloads and _kb._kanban_observer_consumed("on_kanban_worker_exited"):
  1200. _board = _kb.get_current_board()
  1201. for hook_fields in sweep.exited_hook_payloads:
  1202. hook_fields = dict(hook_fields)
  1203. _kb._fire_kanban_lifecycle_hook(
  1204. # Kanban worker-lifecycle, task-mutation, and dispatcher-tick observers (RFC #58548,
  1205. # accepted as the design basis in the #64231 batch disposition; on_kanban_dispatch_tick is
  1206. # the re-port of PR #56066). All five are observers only: return values are ignored, and
  1207. # every fire site is fully best-effort, so a broken callback can never break dispatch or a
  1208. # task mutation. Cost rule: every call site short-circuits on has_hook(), so when nothing
  1209. # subscribes no payload is built and the hot paths (each dispatcher tick, each task write)
  1210. # pay one dict probe. WHICH PROCESS: worker spawn/exit/stale-claim and the dispatch tick
  1211. # fire in the DISPATCHER process (gateway-embedded dispatcher or ``hermes kanban
  1212. # dispatch``); on_kanban_task_updated fires in whichever process committed the mutation
  1213. # (CLI, worker, or the gateway-embedded dashboard API). Common kwargs (task-scoped hooks):
  1214. # task_id: str, profile_name: str, board: str | None, assignee: str | None, run_id: int |
  1215. # None. on_kanban_worker_spawned fires after ``spawn_fn`` returns AND the worker PID (when
  1216. # one was reported) is durably persisted, per the RFC timing contract; like
  1217. # kanban_task_claimed it runs inside the board's dispatch lock, so callbacks must stay fast.
  1218. # Adds: worker_pid: int | None, workspace_path: str. Privacy: workspace_path is a filesystem
  1219. # path and may reveal project layout or usernames.
  1220. "on_kanban_worker_exited",
  1221. hook_fields.pop("task_id"),
  1222. board=_board,
  1223. **hook_fields,
  1224. )
  1225. return sweep.crashed
  1226. def _record_task_failure(
  1227. conn: sqlite3.Connection,
  1228. task_id: str,
  1229. error: str,
  1230. *,
  1231. outcome: str,
  1232. failure_limit: int | None = None,
  1233. force_trip: bool = False,
  1234. release_claim: bool = False,
  1235. end_run: bool = False,
  1236. event_payload_extra: Optional[dict] = None,
  1237. infrastructure: bool = False,
  1238. ) -> bool:
  1239. """Record a non-success outcome and maybe trip the circuit breaker; every
  1240. non-success path funnels through here so ``consecutive_failures`` stays
  1241. consistent. Returns True when the task was auto-blocked.
  1242. ``release_claim=True, end_run=True``: spawn-failure path (task still
  1243. running with an open run — restore source phase or ``blocked``, release
  1244. claim, close run). Both False: timeout/crash path (caller already restored
  1245. the phase and closed the run; only the counter moves, a trip flips to
  1246. ``blocked`` + ``gave_up``). Threshold: per-task ``max_retries`` >
  1247. ``failure_limit`` > ``DEFAULT_FAILURE_LIMIT``. ``force_trip`` trips
  1248. unconditionally (caller applied its own bounded-retry policy).
  1249. ``infrastructure=True``: the host refused the spawn (no restart-safe scope,
  1250. #114720) — nothing about the card ran, so the run and event are recorded
  1251. with ``infrastructure: true`` but ``consecutive_failures`` is left alone and
  1252. the breaker never trips; the card stays retryable and
  1253. :func:`check_respawn_guard` spaces the retries.
  1254. """
  1255. if failure_limit is None:
  1256. failure_limit = DEFAULT_FAILURE_LIMIT
  1257. error = error[:500]
  1258. with _kb.write_txn(conn):
  1259. row = conn.execute(
  1260. "SELECT consecutive_failures, status, max_retries, current_run_id "
  1261. "FROM tasks WHERE id = ?", (task_id,),
  1262. ).fetchone()
  1263. if row is None:
  1264. return False
  1265. retry_status = (
  1266. _kb._retry_status_for_run(conn, task_id, row["current_run_id"])
  1267. if release_claim
  1268. else ("review" if row["status"] == "review" else "ready")
  1269. )
  1270. failures = int(row["consecutive_failures"]) + (0 if infrastructure else 1)
  1271. # Per-task override wins over caller-supplied and default thresholds.
  1272. task_override = _kb._row_get(row, "max_retries")
  1273. if task_override is not None:
  1274. effective_limit, limit_source = int(task_override), "task"
  1275. else:
  1276. effective_limit, limit_source = int(failure_limit), "dispatcher"
  1277. if infrastructure or not (force_trip or failures >= effective_limit):
  1278. if release_claim:
  1279. # Spawn path: restore the claimed source phase + clear claim.
  1280. conn.execute(
  1281. "UPDATE tasks SET status = ?, claim_lock = NULL, "
  1282. "claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "
  1283. "consecutive_failures = ?, last_failure_error = ? "
  1284. "WHERE id = ? AND status = 'running'",
  1285. (retry_status, failures, error, task_id),
  1286. )
  1287. else:
  1288. conn.execute(
  1289. "UPDATE tasks SET consecutive_failures = ?, "
  1290. "last_failure_error = ? WHERE id = ?",
  1291. (failures, error, task_id),
  1292. )
  1293. # Timeout/crash path's caller already emitted its own event.
  1294. if end_run:
  1295. detail = {"failures": failures, "retry_status": retry_status}
  1296. if infrastructure:
  1297. detail["infrastructure"] = True
  1298. run_id = _kb._end_run(
  1299. conn, task_id, outcome=outcome, status=outcome, error=error, metadata=detail,
  1300. )
  1301. _kb._append_event(conn, task_id, outcome, {"error": error, **detail}, run_id=run_id)
  1302. return False
  1303. # Spawn path (release_claim) is still running and also clears claim
  1304. # state; the timeout/crash path already did.
  1305. conn.execute(
  1306. "UPDATE tasks SET status = 'blocked', "
  1307. + ("claim_lock = NULL, claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "
  1308. if release_claim else "")
  1309. + "consecutive_failures = ?, last_failure_error = ? "
  1310. "WHERE id = ? AND status IN ('running', 'ready', 'review')",
  1311. (failures, error, task_id),
  1312. )
  1313. payload = {
  1314. "failures": failures,
  1315. "effective_limit": effective_limit,
  1316. "limit_source": limit_source,
  1317. "error": error,
  1318. "trigger_outcome": outcome,
  1319. "retry_status": retry_status,
  1320. }
  1321. run_id = None
  1322. if end_run:
  1323. # Only the spawn path has an open run to close.
  1324. run_id = _kb._end_run(
  1325. conn, task_id, outcome="gave_up", status="gave_up", error=error,
  1326. metadata={
  1327. "failures": failures,
  1328. "trigger_outcome": outcome,
  1329. "effective_limit": effective_limit,
  1330. "limit_source": limit_source,
  1331. "retry_status": retry_status,
  1332. },
  1333. )
  1334. if force_trip:
  1335. # The caller applied its own bounded policy, so the counter cannot
  1336. # judge this block: ``recompute_ready`` holds it for an operator.
  1337. payload["sticky"] = True
  1338. if event_payload_extra:
  1339. payload.update(event_payload_extra)
  1340. _kb._append_event(conn, task_id, "gave_up", payload, run_id=run_id)
  1341. return True
  1342. def _set_worker_pid(conn: sqlite3.Connection, task_id: str, pid: int, *, run_id=None, claim_lock=None) -> None:
  1343. """Publish the launcher only while this claim has no executing worker yet, together with its
  1344. restart-stable fingerprint (``_process_fingerprint``); the ``spawned`` event carries both. The
  1345. fingerprint is what lets every later liveness/kill decision tell OUR worker from a process that
  1346. recycled the PID after a reboot. A failed capture is persisted as ``UNVERIFIED_WORKER_FINGERPRINT``,
  1347. never NULL: NULL is the legacy pre-fingerprint row whose bare-PID kill authority a new spawn must
  1348. not inherit."""
  1349. started_at = _process_fingerprint(int(pid)) or UNVERIFIED_WORKER_FINGERPRINT
  1350. with _kb.write_txn(conn):
  1351. if run_id is None:
  1352. run_id = _kb._current_run_id(conn, task_id)
  1353. cur = conn.execute(
  1354. "UPDATE tasks SET worker_pid = ?, worker_started_at = ? WHERE id = ? AND status = 'running' "
  1355. "AND current_run_id IS ? AND worker_pid IS NULL "
  1356. "AND (? IS NULL OR claim_lock = ?)",
  1357. (int(pid), started_at, task_id, run_id, claim_lock, claim_lock))
  1358. if cur.rowcount:
  1359. conn.execute("UPDATE task_runs SET worker_pid = ?, worker_started_at = ? WHERE id = ?",
  1360. (int(pid), started_at, run_id))
  1361. _kb._append_event(conn, task_id, "spawned", {"pid": int(pid), "started_at": started_at}, run_id=run_id)
  1362. def adopt_worker_pid(conn: sqlite3.Connection, task_id: str, run_id: int, pid: int) -> bool:
  1363. """Worker-side half of ``_set_worker_pid``, run by the worker before its first model call.
  1364. A dispatcher killed between spawning the worker and ``_set_worker_pid`` leaves the run with no
  1365. pid: no liveness check can see the worker, so a TTL expiry reclaims the card and spawns a second
  1366. worker beside it. The worker fills the missing pid itself (``worker_registered``). False when
  1367. ``run_id`` is no longer the card's live run: the card was reclaimed before this worker got here,
  1368. and it must exit without working it."""
  1369. started_at = _process_fingerprint(int(pid)) or UNVERIFIED_WORKER_FINGERPRINT
  1370. with _kb.write_txn(conn):
  1371. row = conn.execute("SELECT status, current_run_id, worker_pid, claim_lock FROM tasks WHERE id = ?",
  1372. (task_id,)).fetchone()
  1373. if row is None or row["status"] != "running" or row["current_run_id"] != int(run_id):
  1374. return False
  1375. # Liveness checks are host-local: a pid from another host (or pid namespace) proves nothing here.
  1376. if row["worker_pid"] is None and (row["claim_lock"] or "").startswith(_kb._host_prefix()):
  1377. conn.execute("UPDATE tasks SET worker_pid = ?, worker_started_at = ? WHERE id = ?",
  1378. (int(pid), started_at, task_id))
  1379. conn.execute("UPDATE task_runs SET worker_pid = ?, worker_started_at = ? WHERE id = ?",
  1380. (int(pid), started_at, int(run_id)))
  1381. _kb._append_event(conn, task_id, "worker_registered", {"pid": int(pid), "started_at": started_at},
  1382. run_id=int(run_id))
  1383. return True
  1384. def _clear_failure_counter(conn: sqlite3.Connection, task_id: str) -> None:
  1385. """Reset the unified consecutive-failures counter.
  1386. Called from ``complete_task`` on success. NOT called on spawn success: a
  1387. spawn proves the worker could start, not that the run will succeed, so
  1388. timeouts and crashes must accumulate across spawn boundaries.
  1389. """
  1390. with _kb.write_txn(conn):
  1391. conn.execute(
  1392. "UPDATE tasks SET consecutive_failures = 0, "
  1393. "last_failure_error = NULL WHERE id = ?",
  1394. (task_id,),
  1395. )
  1396. def check_respawn_guard(
  1397. conn: sqlite3.Connection, task_id: str, *, lane: str = "ready",
  1398. ) -> Optional[str]:
  1399. """Return a guard reason if ``task_id`` should NOT be re-spawned, else None.
  1400. Called per ready/review row before any claim attempt. Priority order:
  1401. ``"infrastructure_cooldown"`` (latest run is a ``spawn_failed`` the host
  1402. refused — no restart-safe scope — within the cooldown; never counted),
  1403. ``"rate_limit_cooldown"`` (latest run ``rate_limited`` within the cooldown;
  1404. checked BEFORE ``blocker_auth`` because the requeue stamps a quota-flavored
  1405. ``last_failure_error`` that would otherwise park the task forever — that
  1406. path never increments ``consecutive_failures``), ``"blocker_auth"``
  1407. (quota/auth pattern; the breaker still trips eventually), then for the
  1408. ready lane only ``"recent_success"`` (completed run within the window, unless
  1409. a re-queue event arrived after it — a deliberate re-run) and ``"active_pr"``
  1410. (PR URL in a recent comment; re-spawning risks a duplicate PR — unless a
  1411. handoff event followed the comment: the named profile must work on that
  1412. PR). The review lane skips the last two: they are the *inputs* to a review
  1413. handoff. Stale / dead claim locks are NOT a guard reason — the reclaim
  1414. passes own those.
  1415. """
  1416. row = conn.execute(
  1417. "SELECT last_failure_error FROM tasks WHERE id = ?",
  1418. (task_id,),
  1419. ).fetchone()
  1420. if row is None:
  1421. return None
  1422. now = int(time.time())
  1423. # 1. Rate-limit cooldown — see docstring for why this precedes blocker_auth.
  1424. # LATEST run only: a newer crash/completion supersedes the rate-limit run.
  1425. # An infrastructure spawn refusal (#114720) shares the cooldown: the host
  1426. # condition is not the card's, so it retries forever, spaced, and never
  1427. # reaches the breaker.
  1428. rl_cooldown = _kb._resolve_rate_limit_cooldown_seconds()
  1429. latest_run = conn.execute(
  1430. "SELECT outcome, ended_at, metadata FROM task_runs "
  1431. "WHERE task_id = ? AND ended_at IS NOT NULL "
  1432. "ORDER BY ended_at DESC LIMIT 1",
  1433. (task_id,),
  1434. ).fetchone()
  1435. if latest_run is not None and latest_run["outcome"] == "spawn_failed":
  1436. if rl_cooldown > 0 and _kb._json_dict(latest_run["metadata"]).get("infrastructure"):
  1437. ended_at = _kb._to_epoch(latest_run["ended_at"])
  1438. if ended_at is not None and (now - ended_at) < rl_cooldown:
  1439. return "infrastructure_cooldown"
  1440. if latest_run is not None and latest_run["outcome"] == "rate_limited":
  1441. if rl_cooldown <= 0:
  1442. # Cooldown disabled — respawn immediately, skipping blocker_auth so
  1443. # the stamped rate-limit text doesn't re-trap the task.
  1444. return None
  1445. ended_at = _kb._to_epoch(latest_run["ended_at"])
  1446. if ended_at is not None and (now - ended_at) < rl_cooldown:
  1447. return "rate_limit_cooldown"
  1448. # Cooldown elapsed — return early so blocker_auth doesn't catch the
  1449. # stamped rate-limit text; this path intentionally retries forever
  1450. # (spaced by the cooldown) until quota returns or a real run supersedes it.
  1451. return None
  1452. # 2. Quota / auth blocker: retrying immediately will not help. A plain
  1453. # crash is different: its persisted error includes the worker's last
  1454. # captured output, which is context rather than a diagnosis and may contain
  1455. # benign commands such as ``claude auth status`` (#117097).
  1456. err = _kb._lossy_text(row["last_failure_error"])
  1457. latest_outcome = latest_run["outcome"] if latest_run is not None else None
  1458. if err and latest_outcome != "crashed" and _RESPAWN_BLOCKER_RE.search(err):
  1459. return "blocker_auth"
  1460. # Review-lane spawns stop here: a recent completed run and a fresh PR URL
  1461. # are the canonical *inputs* to a review handoff, not duplicate-work signals.
  1462. if lane == "review":
  1463. return None
  1464. # 3. Completed run within guard window. Exception: an explicit re-queue
  1465. # AFTER that success (done→ready drag, re-promotion, unblock, reclaim) is
  1466. # a deliberate "run it again" — otherwise a manual done→ready would sit
  1467. # silently held until the window elapses.
  1468. success = max((t for t, _ in _rows_since(
  1469. conn, "task_runs", task_id, now - _RESPAWN_GUARD_SUCCESS_WINDOW, "AND outcome = 'completed'")), default=None)
  1470. if success is not None and not any(_rows_since(
  1471. conn, "task_events", task_id, success, "AND kind IN ('status', 'promoted', 'unblocked', 'reclaimed')")):
  1472. return "recent_success"
  1473. # 4. GitHub PR URL in a recent comment — prior worker already opened a PR.
  1474. # Exception: a handoff AFTER the newest PR comment (operator reassign,
  1475. # reviewer changes_requested, review reopen) names the profile that must
  1476. # now work on THAT PR — a closer or the implementer finishing it, not a
  1477. # duplicate implementation (#111910). A crash/reclaim is not a handoff,
  1478. # so the worker that opened the PR is still not re-spawned against it.
  1479. pr_times = [t for t, c in _rows_since(conn, "task_comments", task_id, now - _RESPAWN_GUARD_PR_WINDOW,
  1480. cols=", body", unreadable=now) # duplicate-PR risk: fail closed
  1481. if _RESPAWN_GUARD_PR_URL_RE.search(_kb._lossy_text(c["body"]) or "")]
  1482. if pr_times:
  1483. # Strictly after: a same-second tie stays guarded (fail closed).
  1484. if any(_is_handoff_event(e["kind"], e["payload"]) for _, e in _rows_since(
  1485. conn, "task_events", task_id, max(pr_times),
  1486. "AND kind IN ('assigned', 'changes_requested', 'review_reopened')", cols=", kind, payload", strict=True)):
  1487. return None
  1488. return "active_pr"
  1489. return None
  1490. _GUARD_TIME_COLS = {"task_runs": "ended_at", "task_events": "created_at", "task_comments": "created_at"}
  1491. def _rows_since(conn: sqlite3.Connection, table: str, task_id: str, since: int, where: str = "", *,
  1492. cols: str = "", strict: bool = False, unreadable: Optional[int] = None) -> Iterator[tuple[int, Any]]:
  1493. """``(epoch, row)`` for ``task_id``'s ``table`` rows timed at/after ``since`` (after, if ``strict``).
  1494. SQLite sorts TEXT above every INTEGER, so the indexed range also returns every text stamp (written
  1495. outside the mutators, #97455); those are decided on their parsed epoch. An unreadable time is
  1496. dropped (never guards or lifts) unless the caller substitutes ``unreadable``."""
  1497. col = _GUARD_TIME_COLS[table]
  1498. sql = f"SELECT {col}{cols} FROM {table} WHERE task_id = ? {where} AND {col} {'>' if strict else '>='} ?"
  1499. for row in conn.execute(sql, (task_id, since)):
  1500. t = _kb._to_epoch(row[0])
  1501. t = unreadable if t is None else t
  1502. if t is not None and (t > since if strict else t >= since):
  1503. yield t, row
  1504. def _is_handoff_event(kind: str, payload: Optional[str]) -> bool:
  1505. """Only an ``assigned`` event that moves the card to a DIFFERENT profile is
  1506. a handoff. A no-op re-assign (dev→dev via CLI/dashboard/``reassign
  1507. --reclaim``), an unassign, or the dispatcher's own
  1508. ``kanban.default_assignee`` write would otherwise lift ``active_pr`` for
  1509. the very implementer that opened the PR. Events without ``from`` (written
  1510. before it was recorded) are not trusted as handoffs — fail closed."""
  1511. if kind != "assigned":
  1512. return True
  1513. data = _kb._json_or(payload, {})
  1514. if not isinstance(data, dict) or data.get("source") == "kanban.default_assignee":
  1515. return False
  1516. to = data.get("assignee")
  1517. return bool(to) and "from" in data and data["from"] != to
  1518. def _profile_exists_fn() -> Optional[Callable[[str], bool]]:
  1519. """``hermes_cli.profiles.profile_exists``, or ``None`` when it cannot be
  1520. imported (local import avoids a cycle; callers fall back to trusting the
  1521. assignee).
  1522. When ``kanban.dispatch_profiles`` is set (#110995) the returned predicate
  1523. additionally requires the assignee to be listed, fail-closed — so a card
  1524. assigned to ``default`` is only claimable by homes that opted into it.
  1525. Foreign assignees land in the existing ``skipped_nonspawnable`` bucket.
  1526. """
  1527. try:
  1528. from hermes_cli.profiles import normalize_profile_name, profile_exists
  1529. except Exception:
  1530. return None
  1531. allowlist = _dispatch_profile_allowlist(normalize_profile_name)
  1532. if allowlist is None:
  1533. return profile_exists
  1534. def _gated(name: str) -> bool:
  1535. try:
  1536. canon = normalize_profile_name(name)
  1537. except ValueError:
  1538. return False
  1539. return canon in allowlist and bool(profile_exists(name))
  1540. return _gated
  1541. def _dispatch_profile_allowlist(normalize_profile_name) -> Optional[frozenset]:
  1542. """Per-home claim allowlist ``kanban.dispatch_profiles`` (#110995).
  1543. On a shared board (one ``kanban.db`` mounted across several Hermes homes),
  1544. every home's ``profile_exists`` returns True for ``default`` — the root
  1545. profile every home has — so a card assigned to ``default`` is claimable by
  1546. every home's dispatcher. A home opts out of foreign claims by declaring
  1547. which assignees it may claim::
  1548. kanban:
  1549. dispatch_profiles: ["sage", "researcher"] # or "sage,researcher"
  1550. Returns ``None`` only when the key is absent from the user config (upstream
  1551. behavior: any existing profile is claimable). A present value is
  1552. fail-closed: an empty list, ``null`` or a bare ``dispatch_profiles:`` claims
  1553. nothing. The user layer is read without the ``DEFAULT_CONFIG`` merge (whose
  1554. ``None`` placeholder would make the key look present in every home), and a
  1555. config read that raises also claims nothing — a corrupt config on a shared
  1556. board must never widen this home's claim scope silently (#113620).
  1557. """
  1558. try:
  1559. from hermes_cli.config_effective import load_user_config_effective
  1560. kanban = (load_user_config_effective(fail_closed=True) or {}).get("kanban", {})
  1561. except Exception as exc:
  1562. _kb._log.warning(
  1563. "kanban: could not read kanban.dispatch_profiles (%s: %s) — "
  1564. "this home claims no cards until the config is readable",
  1565. type(exc).__name__, exc,
  1566. )
  1567. return frozenset()
  1568. if not isinstance(kanban, Mapping) or "dispatch_profiles" not in kanban:
  1569. return None
  1570. raw = kanban["dispatch_profiles"]
  1571. if raw is None or (isinstance(raw, str) and not raw.strip()):
  1572. _kb._log.warning(
  1573. "kanban: kanban.dispatch_profiles is present but empty — this home "
  1574. "claims no cards; omit the key to allow any existing profile"
  1575. )
  1576. return frozenset()
  1577. names = [str(n) for n in raw] if isinstance(raw, (list, tuple)) else str(raw).split(",")
  1578. allowed = set()
  1579. for n in names:
  1580. try:
  1581. allowed.add(normalize_profile_name(n))
  1582. except ValueError:
  1583. continue
  1584. return frozenset(allowed)
  1585. def dispatch_profile_allowlist_summary() -> str:
  1586. """Human-readable resolution of ``kanban.dispatch_profiles`` for this home.
  1587. Surfaced by ``hermes kanban diagnostics`` so an operator on a shared board
  1588. can see what a home believes it may claim (#113620): ``any`` (key absent),
  1589. the sorted allowed names, or ``none (fail-closed: ...)``.
  1590. """
  1591. try:
  1592. from hermes_cli.profiles import normalize_profile_name
  1593. except Exception as exc:
  1594. return f"none (fail-closed: profiles unavailable: {exc})"
  1595. allowlist = _dispatch_profile_allowlist(normalize_profile_name)
  1596. if allowlist is None:
  1597. return "any"
  1598. if allowlist:
  1599. return ", ".join(sorted(allowlist))
  1600. return ("none (fail-closed: kanban.dispatch_profiles is present but names no valid "
  1601. "profile, or the config could not be read — omit the key to allow any)")
  1602. def _has_spawnable(conn: sqlite3.Connection, status: str) -> bool:
  1603. rows = conn.execute(
  1604. "SELECT DISTINCT assignee FROM tasks "
  1605. "WHERE status = ? AND assignee IS NOT NULL AND claim_lock IS NULL",
  1606. (status,),
  1607. ).fetchall()
  1608. if not rows:
  1609. return False
  1610. profile_exists = _profile_exists_fn()
  1611. if profile_exists is None:
  1612. # Can't introspect — assume spawnable, preserve legacy behavior.
  1613. return True
  1614. return any(profile_exists(row["assignee"]) for row in rows)
  1615. def has_spawnable_ready(conn: sqlite3.Connection) -> bool:
  1616. """True iff a ready+assigned+unclaimed task maps to a real Hermes profile.
  1617. Lets health telemetry tell "stuck" (``0 spawned`` with spawnable work) from
  1618. "correctly idle" (only control-plane lanes waiting on ``claim_task``). Falls
  1619. back to "any assigned" when ``profile_exists`` is unimportable.
  1620. """
  1621. return _has_spawnable(conn, "ready")
  1622. def has_spawnable_review(conn: sqlite3.Connection) -> bool:
  1623. """:func:`has_spawnable_ready` for the review column."""
  1624. return _has_spawnable(conn, "review")
  1625. def review_dispatch_enabled() -> bool:
  1626. """Whether review tasks dispatch automatically. Default true (Hermes ships
  1627. ``sdlc-review``); operators disable it for human-only review boards.
  1628. """
  1629. try:
  1630. from hermes_cli.config import load_config
  1631. return bool((load_config() or {}).get("kanban", {}).get("review_dispatch", True))
  1632. except Exception:
  1633. return True
  1634. # Memory-aware dispatch guard: an uncapped board once OOM'd a 1 GiB host. Two
  1635. # safeguards — a memory-DERIVED default cap when none is configured
  1636. # (``resolve_max_in_progress``) and a live memory-PRESSURE guard inside the
  1637. # tick (``_memory_pressure_level``) because a static cap can't see other
  1638. # tenants. Both fail open: non-Linux / read error → no cap / "unknown".
  1639. # Assumed per-worker footprint for the derived cap; deliberately conservative
  1640. # so the cap errs toward fewer workers on small VMs.
  1641. MEMORY_GUARD_MB_PER_WORKER = 512
  1642. # Derived default bounds: never below 2 (smallest VM must still progress),
  1643. # never above 8 (more fan-out must be explicit in config).
  1644. DERIVED_MAX_IN_PROGRESS_FLOOR = 2
  1645. DERIVED_MAX_IN_PROGRESS_CEILING = 8
  1646. def _system_memory_sample() -> dict:
  1647. """Best-effort system memory snapshot (KiB values), ``{}`` when unknown.
  1648. Local import keeps ``kanban_db`` importable without the gateway package.
  1649. Module-level indirection is also the test seam — conftest patches this to
  1650. ``{}`` so results don't depend on the CI runner's live memory.
  1651. """
  1652. try:
  1653. from gateway.lifecycle_ledger import sample_memory
  1654. return sample_memory() or {}
  1655. except Exception:
  1656. return {}
  1657. def derive_default_max_in_progress(sample: Optional[Mapping[str, Any]] = None) -> Optional[int]:
  1658. """Memory-derived default for ``kanban.max_in_progress`` when unset:
  1659. ``clamp(MemTotal / MEMORY_GUARD_MB_PER_WORKER, FLOOR, CEILING)``. Returns
  1660. ``None`` (no cap) when total memory is unknown, so macOS/Windows dev
  1661. machines are unaffected.
  1662. """
  1663. if sample is None:
  1664. sample = _system_memory_sample()
  1665. total_kib = sample.get("mem_total_kib")
  1666. if isinstance(total_kib, bool) or not isinstance(total_kib, int) or total_kib <= 0:
  1667. return None
  1668. workers = (total_kib // 1024) // MEMORY_GUARD_MB_PER_WORKER
  1669. return max(DERIVED_MAX_IN_PROGRESS_FLOOR, min(workers, DERIVED_MAX_IN_PROGRESS_CEILING))
  1670. def resolve_max_in_progress(configured: Optional[int]) -> Optional[int]:
  1671. """Effective global concurrency cap: explicit config wins, else the
  1672. memory-derived default. All config-parsing callers route through this so
  1673. both paths agree.
  1674. """
  1675. if configured is not None:
  1676. return configured
  1677. return derive_default_max_in_progress()
  1678. def configured_max_in_progress() -> Optional[int]:
  1679. """Read ``kanban.max_in_progress`` from config, or None when unset/invalid.
  1680. Shared so every dispatch entry point agrees on "explicitly configured": a
  1681. positive integer wins, anything else falls through to the derived default.
  1682. """
  1683. try:
  1684. from hermes_cli.config import load_config_readonly
  1685. raw = (load_config_readonly() or {}).get("kanban", {}).get("max_in_progress")
  1686. except Exception:
  1687. return None
  1688. if raw is None:
  1689. return None
  1690. try:
  1691. ival = int(raw)
  1692. except (TypeError, ValueError):
  1693. return None
  1694. return ival if ival >= 1 else None
  1695. def count_running_tasks(conn: sqlite3.Connection) -> int:
  1696. """Number of tasks in ``status='running'``.
  1697. Used by the multi-board sweep to count OTHER boards' workers against the
  1698. host-level budget — the memory-derived cap bounds the machine, not the
  1699. board. Fails open to 0 so a broken board doesn't brick dispatch on healthy ones.
  1700. """
  1701. try:
  1702. return int(
  1703. conn.execute(
  1704. "SELECT COUNT(*) FROM tasks WHERE status = 'running'"
  1705. ).fetchone()[0]
  1706. )
  1707. except Exception:
  1708. return 0
  1709. def count_running_tasks_other_boards(board: Optional[str] = None) -> int:
  1710. """Total ``running`` tasks across every board EXCEPT ``board``.
  1711. Caps bound the HOST, but each board's tick only sees its own DB; without
  1712. this a derived cap of N gets multiplied by the number of active boards.
  1713. Boards are matched by resolved DB path, so ``HERMES_KANBAN_DB`` (pins every
  1714. board to one file) yields 0. Fails open per board.
  1715. """
  1716. try:
  1717. current_path = str(_kb.kanban_db_path(board=board).expanduser().resolve())
  1718. except Exception:
  1719. current_path = None
  1720. try:
  1721. boards = _kb.list_boards(include_archived=False)
  1722. except Exception:
  1723. return 0
  1724. total = 0
  1725. for meta in boards:
  1726. slug = meta.get("slug") or _kb.DEFAULT_BOARD
  1727. try:
  1728. path = _kb.kanban_db_path(board=slug).expanduser()
  1729. resolved = str(path.resolve())
  1730. if current_path is not None and resolved == current_path:
  1731. continue
  1732. if not path.exists():
  1733. continue
  1734. other = _kbc.connect(board=slug)
  1735. try:
  1736. total += count_running_tasks(other)
  1737. finally:
  1738. with contextlib.suppress(Exception):
  1739. other.close()
  1740. except Exception:
  1741. continue
  1742. return total
  1743. def _memory_pressure_level(sample: Optional[Mapping[str, Any]] = None) -> str:
  1744. """Classify system memory pressure: ok/elevated/critical/unknown.
  1745. Reuses :func:`gateway.memory_status.classify_pressure` so "critical" matches
  1746. the dashboard banner and lifecycle-ledger OOM heuristics. ``unknown``
  1747. (non-Linux, read failure) imposes no restriction — never brick dispatch
  1748. where /proc is unavailable.
  1749. """
  1750. if sample is None:
  1751. sample = _system_memory_sample()
  1752. if not sample:
  1753. return "unknown"
  1754. try:
  1755. from gateway.memory_status import classify_pressure
  1756. return classify_pressure(sample.get("mem_available_kib"), sample.get("mem_total_kib"))
  1757. except Exception:
  1758. return "unknown"
  1759. def dispatch_once(
  1760. conn: sqlite3.Connection,
  1761. *,
  1762. spawn_fn=None,
  1763. ttl_seconds: Optional[int] = None,
  1764. dry_run: bool = False,
  1765. max_spawn: Optional[int] = None,
  1766. max_in_progress: Optional[int] = None,
  1767. failure_limit: int = DEFAULT_FAILURE_LIMIT,
  1768. stale_timeout_seconds: int = 0,
  1769. board: Optional[str] = None,
  1770. default_assignee: Optional[str] = None,
  1771. max_in_progress_per_profile: Optional[int] = None,
  1772. reconcile_orphans: bool = True,
  1773. ) -> DispatchResult:
  1774. """Run one dispatcher tick under the board's single-writer lock.
  1775. Wraps :func:`_dispatch_once_locked` in the non-blocking :func:`_dispatch_tick_lock`
  1776. so two dispatchers on one ``kanban.db`` never race a write tick on WAL
  1777. frames. The loser returns an empty ``DispatchResult`` with
  1778. ``skipped_locked=True`` and writes nothing; the lock is keyed on the
  1779. resolved DB path so unrelated boards tick in parallel.
  1780. """
  1781. def _locked_tick() -> DispatchResult:
  1782. return _dispatch_once_locked(
  1783. conn,
  1784. spawn_fn=spawn_fn,
  1785. ttl_seconds=ttl_seconds,
  1786. dry_run=dry_run,
  1787. max_spawn=max_spawn,
  1788. max_in_progress=max_in_progress,
  1789. failure_limit=failure_limit,
  1790. stale_timeout_seconds=stale_timeout_seconds,
  1791. board=board,
  1792. default_assignee=default_assignee,
  1793. max_in_progress_per_profile=max_in_progress_per_profile,
  1794. reconcile_orphans=reconcile_orphans,
  1795. )
  1796. try:
  1797. db_path = _kb.kanban_db_path(board=board)
  1798. except Exception:
  1799. # Must not lose the tick — fall through to an unguarded dispatch.
  1800. result = _locked_tick()
  1801. _kb._fire_dispatch_tick_hook(result, board=board, dry_run=dry_run)
  1802. return result
  1803. with _kbc._dispatch_tick_lock(db_path) as held:
  1804. if not held:
  1805. result = DispatchResult(skipped_locked=True)
  1806. else:
  1807. result = _locked_tick()
  1808. # Still under the dispatch lock: periodic PASSIVE WAL checkpoint.
  1809. _kbc._maybe_checkpoint_wal(conn, db_path)
  1810. # Lock released. Fire the tick observer strictly OUTSIDE the critical
  1811. # section: a slow subscriber must never stall a sibling dispatcher's tick.
  1812. _kb._fire_dispatch_tick_hook(result, board=board, dry_run=dry_run)
  1813. return result
  1814. def _call_spawn_fn(spawn_fn, task: Task, workspace: str, board: Optional[str]) -> Optional[int]:
  1815. """Back-compat: older spawn_fn signatures (and test stubs) accept only
  1816. ``(task, workspace)``; pass ``board`` only when the callable supports it."""
  1817. import inspect
  1818. try:
  1819. sig = inspect.signature(spawn_fn)
  1820. if "board" in sig.parameters:
  1821. return spawn_fn(task, workspace, board=board)
  1822. return spawn_fn(task, workspace)
  1823. except (TypeError, ValueError):
  1824. return spawn_fn(task, workspace)
  1825. def _dispatch_lane_task(
  1826. conn: sqlite3.Connection,
  1827. row: sqlite3.Row,
  1828. assignee: str,
  1829. result: DispatchResult,
  1830. *,
  1831. lane: str,
  1832. dry_run: bool,
  1833. ttl_seconds: Optional[int],
  1834. board: Optional[str],
  1835. failure_limit: int,
  1836. spawn_fn,
  1837. per_profile_cap: Optional[int],
  1838. per_profile_running: dict[str, int],
  1839. ) -> bool:
  1840. """Guard, claim, resolve the workspace and spawn one ready/review row.
  1841. Returns True when a spawn slot was consumed (real or ``dry_run``); every
  1842. skip is recorded on ``result``.
  1843. """
  1844. task_id = row["id"]
  1845. # Non-profile assignees (control-plane lanes that pull via ``claim_task``)
  1846. # would fail ``hermes -p <assignee>`` at startup and loop ready→crash→ready
  1847. # forever. Bucketed apart from skipped_unassigned: the operator cannot fix
  1848. # it by assigning a profile, and health telemetry suppresses "stuck" for it.
  1849. profile_exists = _profile_exists_fn()
  1850. if profile_exists is not None and not profile_exists(assignee):
  1851. result.skipped_nonspawnable.append(task_id)
  1852. # Per-task diagnostic so ``show``/``tail`` name the missing profile instead of leaving
  1853. # the card in ``ready`` with zero board evidence (#122422). Unlike a respawn guard the
  1854. # condition never expires on its own, so write it once: a repeat only when something
  1855. # else happened on the card since (reassign, comment) — not one row per tick forever,
  1856. # and not one row per foreign home per tick on a shared board (#101015).
  1857. if not dry_run:
  1858. with _kb.write_txn(conn):
  1859. last = conn.execute(
  1860. "SELECT kind, payload FROM task_events WHERE task_id = ? "
  1861. "ORDER BY created_at DESC, id DESC LIMIT 1", (task_id,)).fetchone()
  1862. if (last is None or last["kind"] != "skipped_nonspawnable"
  1863. or last["payload"] != _kb._json_or_null({"assignee": assignee})):
  1864. _kb._append_event(conn, task_id, "skipped_nonspawnable", {"assignee": assignee})
  1865. return False
  1866. # Per-profile cap: one profile's local model / API quota / browser pool
  1867. # must not be overwhelmed by a fan-out even with global headroom.
  1868. if per_profile_cap is not None:
  1869. current = per_profile_running.get(assignee, 0)
  1870. if current >= per_profile_cap:
  1871. result.skipped_per_profile_capped.append((task_id, assignee, current))
  1872. return False
  1873. guard_reason = check_respawn_guard(conn, task_id, lane=lane)
  1874. if guard_reason is not None:
  1875. result.respawn_guarded.append((task_id, guard_reason))
  1876. # Event so ``hermes kanban tail`` shows why the task looks stuck.
  1877. # Honour kanban.default_assignee: when the dispatcher hits an unassigned ready task and an
  1878. # operator-configured fallback exists, persist the assignment and proceed. This removes the
  1879. # dashboard footgun where a task created without an assignee parks in 'ready' forever even though
  1880. # the operator's intent ("default") was perfectly clear (#27145). Mutating the row (not just the
  1881. # in-memory view) keeps diagnostics and the board state consistent: the task is now legitimately
  1882. # owned by ``kanban.default_assignee``, not "unassigned but secretly routed".
  1883. if not dry_run:
  1884. with _kb.write_txn(conn):
  1885. _kb._append_event(conn, task_id, "respawn_guarded", {"reason": guard_reason})
  1886. return False
  1887. def _count_spawn(name: str) -> None:
  1888. # Later rows in this tick respect the per-profile cap; subsequent
  1889. # ticks re-query from the DB.
  1890. if per_profile_cap is not None and name:
  1891. per_profile_running[name] = per_profile_running.get(name, 0) + 1
  1892. if dry_run:
  1893. result.spawned.append((task_id, assignee, ""))
  1894. _count_spawn(assignee)
  1895. return True
  1896. claim = _kb.claim_review_task if lane == "review" else _kb.claim_task
  1897. claimed = claim(conn, task_id, ttl_seconds=ttl_seconds)
  1898. if claimed is None:
  1899. return False
  1900. try:
  1901. resolved_branch_name = None
  1902. if claimed.workspace_kind == "worktree":
  1903. workspace, resolved_branch_name = _kbw._resolve_worktree_workspace(claimed, board=board)
  1904. else:
  1905. workspace = _kbw.resolve_workspace(claimed, board=board)
  1906. except Exception as exc:
  1907. if _record_task_failure(
  1908. conn, claimed.id, f"workspace: {exc}",
  1909. outcome="spawn_failed", failure_limit=failure_limit, release_claim=True, end_run=True,
  1910. ):
  1911. result.auto_blocked.append(claimed.id)
  1912. return False
  1913. _kbw.set_workspace_path(conn, claimed.id, str(workspace))
  1914. if claimed.workspace_kind == "worktree":
  1915. _kbw.set_branch_name(conn, claimed.id, resolved_branch_name or (claimed.branch_name or "").strip() or f"wt/{claimed.id}")
  1916. _kbw._maybe_emit_scratch_tip(conn, claimed.id, claimed.workspace_kind)
  1917. if lane == "review":
  1918. # Force-load sdlc-review; the kanban lifecycle is already in every
  1919. # worker's system prompt via KANBAN_GUIDANCE.
  1920. claimed.skills = list(dict.fromkeys([*(claimed.skills or []), "sdlc-review"]))
  1921. try:
  1922. pid = _call_spawn_fn(spawn_fn if spawn_fn is not None else _default_spawn, claimed, str(workspace), board)
  1923. if pid:
  1924. _set_worker_pid(conn, claimed.id, int(pid), run_id=claimed.current_run_id, claim_lock=claimed.claim_lock)
  1925. # Fires AFTER the PID (when reported) is durably persisted. Best-effort.
  1926. _kb._fire_worker_spawned_hook(conn, claimed, str(workspace), pid, board=board)
  1927. # consecutive_failures is deliberately NOT reset here: resetting on
  1928. # spawn would let a task that keeps timing out loop forever. Cleared
  1929. # only on successful completion (complete_task).
  1930. result.spawned.append((claimed.id, claimed.assignee or "", str(workspace)))
  1931. _count_spawn(claimed.assignee)
  1932. return True
  1933. except Exception as exc:
  1934. from tools.process_registry import RestartSafeScopeUnavailable
  1935. # The host refused the spawn (no restart-safe scope): nothing about the
  1936. # card ran, so it must not spend the card's retry budget (#114720).
  1937. infrastructure = isinstance(exc, RestartSafeScopeUnavailable)
  1938. if infrastructure:
  1939. _kb._log.warning("kanban dispatcher: spawn of %s deferred, host cannot place the worker: %s", claimed.id, exc)
  1940. if _record_task_failure(
  1941. conn, claimed.id, str(exc),
  1942. outcome="spawn_failed", failure_limit=failure_limit, release_claim=True, end_run=True,
  1943. infrastructure=infrastructure,
  1944. ):
  1945. result.auto_blocked.append(claimed.id)
  1946. return False
  1947. def _apply_default_assignee(
  1948. conn: sqlite3.Connection, task_id: str, assignee: str, *, dry_run: bool,
  1949. ) -> bool:
  1950. """Persist ``kanban.default_assignee`` on an unassigned ready row.
  1951. Mutating the row keeps board state honest: the task is legitimately owned
  1952. by the default, not "unassigned but secretly routed". ``dry_run`` reports
  1953. without writing. Returns False when the write failed.
  1954. """
  1955. if dry_run:
  1956. return True
  1957. try:
  1958. with _kb.write_txn(conn):
  1959. conn.execute(
  1960. "UPDATE tasks SET assignee = ? WHERE id = ? "
  1961. "AND (assignee IS NULL OR assignee = '')",
  1962. (assignee, task_id),
  1963. )
  1964. _kb._append_event(
  1965. conn, task_id, "assigned",
  1966. {"assignee": assignee, "source": "kanban.default_assignee"},
  1967. )
  1968. except Exception:
  1969. _kb._log.debug(
  1970. "kanban dispatch: failed to apply default_assignee=%r to task %s",
  1971. assignee, task_id, exc_info=True,
  1972. )
  1973. return False
  1974. return True
  1975. def _run_reclaim_phase(
  1976. conn: sqlite3.Connection,
  1977. result: DispatchResult,
  1978. *,
  1979. stale_timeout_seconds: int,
  1980. failure_limit: int,
  1981. reconcile_orphans: bool,
  1982. board: Optional[str] = None,
  1983. ) -> None:
  1984. """Reclaim stale/orphaned/crashed/timed-out running tasks, then promote."""
  1985. reap_worker_zombies()
  1986. result.reaped_terminal_workers = reap_terminal_workers(conn)
  1987. result.reclaimed = _kb.release_stale_claims(conn, failure_limit=failure_limit)
  1988. if reconcile_orphans:
  1989. result.reconciled_orphans = reconcile_orphaned_running(conn)
  1990. result.stale = detect_stale_running(conn, stale_timeout_seconds=stale_timeout_seconds)
  1991. result.crashed = detect_crashed_workers(conn, board=board)
  1992. # Side-channel attributes (see detect_crashed_workers); rate-limited tasks
  1993. # went back to ``ready`` and the respawn guard defers them until quota clears.
  1994. result.auto_blocked.extend(getattr(detect_crashed_workers, "_last_auto_blocked", []))
  1995. result.rate_limited.extend(getattr(detect_crashed_workers, "_last_rate_limited", []))
  1996. result.timed_out = enforce_max_runtime(conn)
  1997. result.promoted = _kb.recompute_ready(conn, failure_limit=failure_limit)
  1998. def _tick_spawn_budget(
  1999. conn: sqlite3.Connection,
  2000. result: DispatchResult,
  2001. *,
  2002. max_spawn: Optional[int],
  2003. max_in_progress: Optional[int],
  2004. board: Optional[str],
  2005. ) -> tuple[bool, Optional[int]]:
  2006. """``(may_spawn, spawn_budget)`` for this tick; ``budget None`` = uncapped.
  2007. ``max_spawn`` is a live per-board concurrency cap (running + this tick's
  2008. spawns), not a per-tick budget — a per-tick reading would grow concurrency
  2009. by N every tick. ``max_in_progress`` is a HOST-level cap: running workers on
  2010. every other board count against the same budget, else N boards multiply the
  2011. cap by N — exactly the fan-out the memory-derived default exists to prevent.
  2012. """
  2013. # Count already-running tasks so max_spawn enforces concurrency, not a
  2014. # per-tick budget: "running" tasks stay running until the worker makes a terminal
  2015. # board call (kanban_complete/kanban_block/kanban_request_review) or the TTL reclaims them.
  2016. running_count = 0
  2017. spawn_budget: Optional[int] = None
  2018. if max_spawn is not None or max_in_progress is not None:
  2019. running_count = count_running_tasks(conn)
  2020. # Both ready and review loops consume from the same budget.
  2021. if max_spawn is not None:
  2022. if running_count >= max_spawn:
  2023. return False, None
  2024. spawn_budget = max_spawn - running_count
  2025. if max_in_progress is not None:
  2026. total_running = running_count + count_running_tasks_other_boards(board)
  2027. if total_running >= max_in_progress:
  2028. return False, None
  2029. remaining = max_in_progress - total_running
  2030. if spawn_budget is None or spawn_budget > remaining:
  2031. spawn_budget = remaining
  2032. # Memory-pressure guard: a static cap can't see the host's actual state.
  2033. # critical -> spawn nothing this tick; elevated -> at most one new worker.
  2034. # Reclaim/promotion already ran, so bookkeeping stays live; deferred tasks
  2035. # wait for a later tick. "unknown" imposes no restriction.
  2036. pressure = _memory_pressure_level()
  2037. if pressure == "critical":
  2038. result.memory_pressure = pressure
  2039. _kb._log.warning(
  2040. "kanban dispatch: system memory pressure is critical; "
  2041. "spawning no new workers this tick (deferred, not dropped)"
  2042. )
  2043. return False, None
  2044. if pressure == "elevated":
  2045. result.memory_pressure = pressure
  2046. if spawn_budget is None or spawn_budget > 1:
  2047. _kb._log.warning(
  2048. "kanban dispatch: system memory pressure is elevated; "
  2049. "limiting to at most 1 new worker this tick"
  2050. )
  2051. spawn_budget = 1
  2052. return True, spawn_budget
  2053. def _lane_rows(conn: sqlite3.Connection, status: str) -> list[sqlite3.Row]:
  2054. """Unclaimed rows of one lane in dispatch order."""
  2055. return conn.execute(
  2056. "SELECT id, assignee FROM tasks "
  2057. f"WHERE status = '{status}' AND claim_lock IS NULL "
  2058. "ORDER BY priority DESC, created_at ASC"
  2059. ).fetchall()
  2060. def _any_spawnable_review(
  2061. conn: sqlite3.Connection,
  2062. review_rows: list[sqlite3.Row],
  2063. *,
  2064. per_profile_cap: Optional[int] = None,
  2065. per_profile_running: Optional[dict[str, int]] = None,
  2066. ) -> bool:
  2067. """Mirror review dispatch gates before reserving ready-lane capacity.
  2068. Unavailable profile metadata retains the historic fail-open behavior. A
  2069. review row that :func:`_dispatch_lane_task` would refuse this tick — its
  2070. assignee already at the per-profile cap, or respawn-guarded — cannot
  2071. consume the reservation, so it must not withhold capacity from an
  2072. otherwise ready task (one such row would pin ``ready_budget`` to 0).
  2073. """
  2074. if not review_rows:
  2075. return False
  2076. profile_exists = _profile_exists_fn()
  2077. running = per_profile_running or {}
  2078. for row in review_rows:
  2079. assignee = row["assignee"]
  2080. if not assignee:
  2081. continue
  2082. if profile_exists is not None and not profile_exists(assignee):
  2083. continue
  2084. if per_profile_cap is not None and running.get(assignee, 0) >= per_profile_cap:
  2085. continue
  2086. if check_respawn_guard(conn, row["id"], lane="review") is None:
  2087. return True
  2088. return False
  2089. def _resolve_default_assignee(default_assignee: Optional[str]) -> Optional[str]:
  2090. """``kanban.default_assignee`` when it names a real profile this home may
  2091. claim (``kanban.dispatch_profiles`` gated, same predicate as the spawn
  2092. gate). Otherwise ``None`` so an unassigned shared-board card is never
  2093. written to. When the profiles module isn't importable trust the
  2094. operator's config: the downstream check still buckets a missing profile
  2095. as nonspawnable."""
  2096. name = (default_assignee or "").strip() or None
  2097. if name:
  2098. profile_exists = _profile_exists_fn()
  2099. if profile_exists is not None and not profile_exists(name):
  2100. return None
  2101. return name
  2102. # The dispatch lock has been released here. Fire the tick observer strictly OUTSIDE the single-writer
  2103. # critical section (#56066 sweeper finding / #64231 disposition): a slow subscriber must never extend the
  2104. # lock hold and stall a sibling dispatcher's tick.
  2105. def _dispatch_once_locked(
  2106. conn: sqlite3.Connection,
  2107. *,
  2108. spawn_fn=None,
  2109. ttl_seconds: Optional[int] = None,
  2110. dry_run: bool = False,
  2111. max_spawn: Optional[int] = None,
  2112. max_in_progress: Optional[int] = None,
  2113. failure_limit: int = DEFAULT_FAILURE_LIMIT,
  2114. stale_timeout_seconds: int = 0,
  2115. board: Optional[str] = None,
  2116. default_assignee: Optional[str] = None,
  2117. max_in_progress_per_profile: Optional[int] = None,
  2118. reconcile_orphans: bool = True,
  2119. ) -> DispatchResult:
  2120. """One dispatcher tick: reclaim stale/crashed running tasks, promote
  2121. todo -> ready, then atomically claim each spawnable ready/review row and
  2122. call ``spawn_fn(task, workspace_path, board) -> Optional[int]``, recording
  2123. the PID so later ticks catch crashes before the TTL. Cap semantics:
  2124. :func:`_tick_spawn_budget`."""
  2125. result = DispatchResult()
  2126. _run_reclaim_phase(
  2127. conn, result, stale_timeout_seconds=stale_timeout_seconds,
  2128. failure_limit=failure_limit, reconcile_orphans=reconcile_orphans, board=board,
  2129. )
  2130. may_spawn, spawn_budget = _tick_spawn_budget(
  2131. conn, result, max_spawn=max_spawn, max_in_progress=max_in_progress, board=board,
  2132. )
  2133. if not may_spawn:
  2134. return result
  2135. ready_rows = _lane_rows(conn, "ready")
  2136. # Review rows are enumerated up front so the budget split can see whether
  2137. # review work exists at all.
  2138. review_rows = _lane_rows(conn, "review") if review_dispatch_enabled() else []
  2139. # Per-profile cap. Deferred tasks go to skipped_per_profile_capped, not
  2140. # skipped_unassigned — "busy, retry later" differs from "needs routing".
  2141. # Resolved BEFORE the review reservation so the reservation can see which
  2142. # review rows the lane loop would refuse this tick.
  2143. per_profile_cap = max_in_progress_per_profile if (
  2144. # Per-profile concurrency cap (#21582): when set, track how many workers each assignee already has
  2145. # in flight, and refuse to spawn when this would push that assignee past the cap. Prevents fan-out
  2146. # workloads from melting a single profile's local model / API quota / browser pool while leaving
  2147. # other profiles idle.
  2148. isinstance(max_in_progress_per_profile, int)
  2149. and max_in_progress_per_profile > 0
  2150. ) else None
  2151. per_profile_running: dict[str, int] = {}
  2152. if per_profile_cap is not None:
  2153. for prow in conn.execute(
  2154. "SELECT assignee, COUNT(*) AS n FROM tasks "
  2155. "WHERE status = 'running' AND assignee IS NOT NULL "
  2156. "GROUP BY assignee"
  2157. ):
  2158. per_profile_running[prow["assignee"]] = int(prow["n"])
  2159. # Review-lane reservation: the ready loop runs first and would otherwise
  2160. # consume the ENTIRE shared budget, starving reviews under a sustained ready
  2161. # backlog. When spawnable review work exists and there is any budget, hold
  2162. # one slot back.
  2163. ready_budget = spawn_budget
  2164. if spawn_budget is not None and spawn_budget > 0 and _any_spawnable_review(
  2165. conn, review_rows,
  2166. per_profile_cap=per_profile_cap, per_profile_running=per_profile_running,
  2167. ):
  2168. ready_budget = max(spawn_budget - 1, 0)
  2169. lane_kwargs: dict[str, Any] = dict(
  2170. dry_run=dry_run, ttl_seconds=ttl_seconds, board=board,
  2171. failure_limit=failure_limit, spawn_fn=spawn_fn,
  2172. per_profile_cap=per_profile_cap, per_profile_running=per_profile_running,
  2173. )
  2174. default_assignee = _resolve_default_assignee(default_assignee)
  2175. spawned = 0
  2176. for row in ready_rows:
  2177. if ready_budget is not None and spawned >= ready_budget:
  2178. break
  2179. row_assignee = row["assignee"]
  2180. if not row_assignee:
  2181. # Honour kanban.default_assignee so an unassigned task doesn't
  2182. # park in 'ready' forever.
  2183. if not default_assignee or not _apply_default_assignee(
  2184. conn, row["id"], default_assignee, dry_run=dry_run,
  2185. ):
  2186. result.skipped_unassigned.append(row["id"])
  2187. continue
  2188. row_assignee = default_assignee
  2189. result.auto_assigned_default.append(row["id"])
  2190. if _dispatch_lane_task(conn, row, row_assignee, result, lane="ready", **lane_kwargs):
  2191. spawned += 1
  2192. # A review agent (sdlc-review) approves (→ done) or requests changes
  2193. # (→ ready/todo). Review spawns share max_spawn with ready tasks. The loop
  2194. # checks the FULL shared ``spawn_budget`` — the reservation above caps the
  2195. # ready lane, it grants no extra capacity here.
  2196. for row in review_rows:
  2197. if spawn_budget is not None and spawned >= spawn_budget:
  2198. break
  2199. if not row["assignee"]:
  2200. result.skipped_unassigned.append(row["id"])
  2201. continue
  2202. if _dispatch_lane_task(conn, row, row["assignee"], result, lane="review", **lane_kwargs):
  2203. spawned += 1
  2204. return result
  2205. def _positive_int(value: Any, default: int, *, minimum: int = 1) -> int:
  2206. try:
  2207. parsed = int(value)
  2208. except (TypeError, ValueError):
  2209. return default
  2210. return parsed if parsed >= minimum else default
  2211. def _module_hermes_argv() -> list[str]:
  2212. """Interpreter-bound Hermes CLI invocation (``hermes_cli.main`` is the
  2213. console-script target — there is no top-level ``hermes`` package)."""
  2214. return [sys.executable, "-m", "hermes_cli.main"]
  2215. def _absolute_hermes_path(path: str) -> str:
  2216. """Return an absolute filesystem path for a resolved Hermes shim."""
  2217. expanded = os.path.expanduser(path)
  2218. return expanded if os.path.isabs(expanded) else os.path.abspath(expanded)
  2219. def _looks_like_path(value: str) -> bool:
  2220. """Return true when a command override is an explicit path, not a name."""
  2221. expanded = os.path.expanduser(value)
  2222. return (
  2223. expanded.startswith("~")
  2224. or os.path.isabs(expanded)
  2225. or bool(os.path.dirname(expanded))
  2226. or "\\" in expanded
  2227. or bool(re.match(r"^[A-Za-z]:", expanded))
  2228. )
  2229. def _is_windows_batch_shim(path: str) -> bool:
  2230. """Return true for Windows shell/batch shims that should not be argv[0]."""
  2231. return path.lower().endswith((".cmd", ".bat"))
  2232. def _path_search_names(command: str) -> list[str]:
  2233. """Return executable names to try for an unqualified command."""
  2234. if not _kb._IS_WINDOWS or os.path.splitext(command)[1]:
  2235. return [command]
  2236. raw = os.environ.get("PATHEXT") or ".COM;.EXE;.BAT;.CMD"
  2237. return [command + ext for ext in raw.split(";") if ext]
  2238. def _safe_which_no_cwd(command: str) -> Optional[str]:
  2239. """Resolve a bare command from PATH without implicit current-dir search.
  2240. On Windows ``shutil.which`` may search the current directory before PATH
  2241. for bare names — unsafe for a dispatcher. Only explicit PATH entries are
  2242. considered; empty / ``.`` entries are skipped.
  2243. """
  2244. for raw_dir in os.environ.get("PATH", "").split(os.pathsep):
  2245. if not raw_dir or raw_dir == ".":
  2246. continue
  2247. directory = os.path.expanduser(raw_dir)
  2248. for name in _path_search_names(command):
  2249. candidate = os.path.join(directory, name)
  2250. if os.path.isfile(candidate) and (_kb._IS_WINDOWS or os.access(candidate, os.X_OK)):
  2251. return candidate
  2252. return None
  2253. def _hermes_path_argv(path: str) -> list[str]:
  2254. """argv for a resolved Hermes executable path. Windows batch shims
  2255. (``.cmd``/``.bat``) are unsafe as argv[0] because the argument vector
  2256. includes task-derived values; prefer the module form."""
  2257. if _kb._IS_WINDOWS and _is_windows_batch_shim(path):
  2258. return _module_hermes_argv()
  2259. return [_absolute_hermes_path(path)]
  2260. def _resolve_hermes_argv() -> list[str]:
  2261. """Resolve the ``hermes`` invocation as argv for ``Popen``: ``$HERMES_BIN``
  2262. (path-like -> absolute; bare names keep PATH semantics, never a
  2263. same-directory file), then the running interpreter's ``sys.executable -m
  2264. hermes_cli.main`` (exactly this install; also covers shim-less cron,
  2265. systemd ``User=``, launchd), then ``which("hermes")`` (Windows: safe PATH
  2266. search, batch shims fall back to the module form) only when ``hermes_cli``
  2267. is not importable. The module argv must win over PATH: a PATH-first lookup
  2268. lets an attacker-planted ``hermes`` shadow the running install (#111569).
  2269. Mirrors ``gateway.run._resolve_hermes_bin``; local because ``hermes_cli``
  2270. sits below ``gateway`` in the dependency order.
  2271. """
  2272. import importlib.util
  2273. import shutil
  2274. env_bin = os.environ.get("HERMES_BIN", "").strip()
  2275. if env_bin:
  2276. if _looks_like_path(env_bin):
  2277. return _hermes_path_argv(env_bin)
  2278. resolved_env_bin = _safe_which_no_cwd(env_bin)
  2279. if resolved_env_bin:
  2280. return _hermes_path_argv(resolved_env_bin)
  2281. return _module_hermes_argv()
  2282. try:
  2283. if importlib.util.find_spec("hermes_cli") is not None:
  2284. return _module_hermes_argv()
  2285. except Exception:
  2286. pass
  2287. hermes_bin = _safe_which_no_cwd("hermes") if _kb._IS_WINDOWS else shutil.which("hermes")
  2288. if hermes_bin:
  2289. return _hermes_path_argv(hermes_bin)
  2290. return _module_hermes_argv()
  2291. def _worker_terminal_timeout_env(
  2292. max_runtime_seconds: Optional[int],
  2293. current_timeout: Optional[str],
  2294. ) -> Optional[str]:
  2295. """Return a worker-scoped TERMINAL_TIMEOUT override, if needed.
  2296. When ``max_runtime_seconds`` exceeds the terminal tool's default timeout,
  2297. raise only the child's default so a long command isn't killed by the
  2298. generic terminal default first.
  2299. """
  2300. if max_runtime_seconds is None:
  2301. return None
  2302. try:
  2303. runtime = int(max_runtime_seconds)
  2304. except (TypeError, ValueError):
  2305. return None
  2306. if runtime <= 0:
  2307. return None
  2308. desired = max(1, runtime - KANBAN_TERMINAL_TIMEOUT_GRACE_SECONDS)
  2309. try:
  2310. existing = int(str(current_timeout).strip()) if current_timeout else 0
  2311. except (TypeError, ValueError):
  2312. existing = 0
  2313. if existing >= desired:
  2314. return None
  2315. return str(desired)
  2316. @contextlib.contextmanager
  2317. def _worker_profile_scope(hermes_home: str, *, bind_home: bool = True):
  2318. """Bind an assigned profile's runtime scope (secrets + terminal policy, optionally home) for
  2319. one dispatch-side read or spawn-env build.
  2320. The dispatcher runs detached from any turn, so nothing binds a profile for it: ``load_config``,
  2321. the toolset probes' ``get_secret`` reads and ``build_subprocess_env``'s passthrough resolution
  2322. all fall back to the LAUNCH profile's ambient ``os.environ`` / ``TERMINAL_*``. Binding was
  2323. previously conditional on ``is_multiplex_active()``, so on a single-profile host a worker for
  2324. profile B was built entirely from the dispatcher's own environment.
  2325. ``bind_home=False`` for the spawn-env build: which variables may cross into a child is the
  2326. DISPATCHER's ``terminal.env_passthrough`` policy (#109494, read through the home override) —
  2327. only their VALUES come from the assignee's scope, so that branch binds the secret scope alone.
  2328. Toolset resolution binds the home and the terminal policy, as it always has.
  2329. The secret mapping is never widened: a profile that is not this process's own home gets its own
  2330. ``.env`` + external sources ONLY, while the launch home keeps its established
  2331. env-over-``.env`` precedence (``launch_secret_scope``) so systemd / ``op run`` injection still
  2332. resolves for a standalone dispatcher.
  2333. """
  2334. from agent.secret_scope import build_profile_secret_scope, reset_secret_scope, set_secret_scope
  2335. from hermes_constants import get_process_hermes_home, reset_hermes_home_override, set_hermes_home_override
  2336. from tools.terminal_scope import install_profile_terminal_scope, reset_terminal_scope
  2337. from tui_gateway.launch_profile_policy import launch_secret_scope, launch_terminal_env
  2338. home = Path(hermes_home)
  2339. is_launch_home = str(home.resolve()) == str(Path(get_process_hermes_home()).resolve())
  2340. home_token = secret_token = terminal_token = None
  2341. try:
  2342. home_token = set_hermes_home_override(str(home)) if bind_home else None
  2343. secret_token = set_secret_scope(
  2344. launch_secret_scope(home) if is_launch_home else build_profile_secret_scope(home),
  2345. profile_home=None if is_launch_home else str(home))
  2346. terminal_token = install_profile_terminal_scope(
  2347. home, env_overlay=launch_terminal_env() if is_launch_home else None) if bind_home else None
  2348. yield
  2349. finally:
  2350. if terminal_token is not None:
  2351. reset_terminal_scope(terminal_token)
  2352. if secret_token is not None:
  2353. reset_secret_scope(secret_token)
  2354. if home_token is not None:
  2355. reset_hermes_home_override(home_token)
  2356. def _resolve_worker_cli_toolsets(hermes_home: Optional[str]) -> Optional[list[str]]:
  2357. """Return the assigned profile's effective CLI toolsets for a worker.
  2358. Resolved at dispatch time and passed as an explicit ``--toolsets`` pin so
  2359. worker startup cannot fall back to a stale root/active-profile config or a
  2360. profile whose top-level ``toolsets`` is only the kanban orchestrator
  2361. surface. ``model_tools`` still appends the task-scoped kanban lifecycle
  2362. tools when ``HERMES_KANBAN_TASK`` is set.
  2363. """
  2364. if not hermes_home:
  2365. return None
  2366. try:
  2367. from hermes_cli.config import load_config
  2368. from hermes_cli.tools_config import _get_platform_tools
  2369. with _worker_profile_scope(hermes_home):
  2370. cfg = load_config()
  2371. toolsets = sorted(_get_platform_tools(cfg, "cli"))
  2372. return toolsets or None
  2373. except Exception as exc:
  2374. _kb._log.debug(
  2375. "kanban worker: could not resolve CLI toolsets for HERMES_HOME=%r (%s)",
  2376. hermes_home,
  2377. exc,
  2378. )
  2379. return None
  2380. _retagged_workspace_roots: set[str] = set()
  2381. def _retag_legacy_worker_sessions(workspaces_root_path: str) -> None:
  2382. """Reclaim pre-tag worker rows in state.db so they leave the session lists.
  2383. Best-effort: the durable gate is ``state_meta`` in
  2384. ``retag_kanban_worker_sessions``; the in-process set avoids reopening
  2385. state.db on every spawn. A tick must never fail because a session DB was
  2386. busy or missing.
  2387. """
  2388. if workspaces_root_path in _retagged_workspace_roots:
  2389. return
  2390. try:
  2391. from hermes_state_registry import acquire, release_or_close
  2392. # Inside the gateway the dispatcher shares the process's registry handle; a bare
  2393. # SessionDB() here was one more writer connection on the same state.db (#100896).
  2394. db = acquire()
  2395. try:
  2396. db.retag_kanban_worker_sessions(workspaces_root_path)
  2397. finally:
  2398. release_or_close(db)
  2399. _retagged_workspace_roots.add(workspaces_root_path)
  2400. except Exception as exc:
  2401. _kb._log.debug("kanban worker: legacy session retag skipped (%s)", exc)
  2402. def _worker_argv(task: Task, profile_arg: str, hermes_home: Optional[str]) -> list[str]:
  2403. """The profile owner executes; this subprocess only waits for its receipt."""
  2404. import sys
  2405. return [sys.executable, "-m", "hermes_cli.kanban_worker_client"]
  2406. def _open_worker_log(task: Task, board: Optional[str]):
  2407. """Append-mode per-task log (a re-run on unblock appends, never overwrites),
  2408. rotated first. Anchored at the board root (not the shared kanban root) so
  2409. `hermes kanban log` reads its own file and boards sharing task ids don't
  2410. collide."""
  2411. log_dir = _kb.worker_logs_dir(board=board)
  2412. log_dir.mkdir(parents=True, exist_ok=True)
  2413. log_path = log_dir / f"{task.id}.log"
  2414. from hermes_cli.kanban_worker_log import rotate_worker_log, worker_log_rotation_config
  2415. rotate_worker_log(log_path, *worker_log_rotation_config())
  2416. return open(log_path, "ab")
  2417. def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]:
  2418. """Wrap a systemd-hosted dispatcher's worker in the shared restart-safe scope.
  2419. Kanban workers are long-lived agentic runs that outlive the dispatcher
  2420. tick, so they never take cron's degraded mode under the managed gateway:
  2421. ``require_restart_safe_scope=True`` makes the helper raise
  2422. ``RestartSafeScopeUnavailable`` there (an infrastructure spawn failure the
  2423. dispatcher does not charge to the card). Under any other systemd unit
  2424. (``Type=oneshot`` dispatch timers, #113612) ``outlives_parent=True`` gets the
  2425. worker its own scope so the unit's cgroup teardown cannot kill it.
  2426. """
  2427. from tools.process_registry import restart_safe_gateway_child_argv
  2428. if task.current_run_id is None:
  2429. # Outside managed systemd this is harmless, but a managed dispatch must
  2430. # never mint an untraceable worker. Check topology through the shared
  2431. # helper first, using a placeholder suffix that cannot be launched.
  2432. dispatch = restart_safe_gateway_child_argv(
  2433. command,
  2434. unit_suffix=f"kanban-{task.id}-run-missing",
  2435. require_restart_safe_scope=True,
  2436. outlives_parent=True,
  2437. )
  2438. if dispatch.mode != "in_process":
  2439. raise RuntimeError(
  2440. "cannot create restart-safe systemd scope for Kanban worker: "
  2441. "the claimed task has no current run id"
  2442. )
  2443. return command
  2444. return restart_safe_gateway_child_argv(
  2445. command,
  2446. unit_suffix=f"kanban-{task.id}-run-{task.current_run_id}",
  2447. require_restart_safe_scope=True,
  2448. outlives_parent=True,
  2449. ).argv
  2450. def _default_spawn(task: Task, workspace: str, *, board: Optional[str] = None) -> Optional[int]:
  2451. """Fire-and-forget ``hermes -p <profile> chat -q ...`` subprocess.
  2452. Returns the child's PID so the dispatcher can detect crashes before the
  2453. claim TTL expires; completion is still observed via the worker's own
  2454. ``complete`` / ``block`` transitions. ``board`` pins the child's
  2455. ``HERMES_KANBAN_DB`` / ``HERMES_KANBAN_BOARD`` / workspaces_root to the
  2456. board the task was claimed from, so workers cannot see other boards.
  2457. """
  2458. if not task.assignee:
  2459. raise ValueError(f"task {task.id} has no assignee")
  2460. from hermes_cli.profiles import normalize_profile_name, resolve_profile_env
  2461. profile_arg = normalize_profile_name(task.assignee)
  2462. from agent.secret_scope import is_multiplex_active
  2463. from tools.environments.local import _is_routed_home, build_subprocess_env, strip_launch_profile_env
  2464. try:
  2465. profile_home = resolve_profile_env(profile_arg)
  2466. except FileNotFoundError:
  2467. # No profile dir (isolated test fixtures) — the CLI resolves it from
  2468. # HERMES_PROFILE (set below) instead.
  2469. profile_home = None
  2470. # Scrub for a ROUTED home, not only under multiplex: the authority test is "does this worker act
  2471. # for another profile", exactly as served_profile_child_env decides it (tools/environments/local.py).
  2472. # Gating on the gateway-wide flag left B's worker inheriting the dispatcher's own OPENAI_API_KEY and
  2473. # systemd-injected tokens on every single-profile host.
  2474. routed = bool(profile_home) and _is_routed_home(profile_home)
  2475. # build_subprocess_env's secret scrub resolves terminal.env_passthrough vars through get_secret(),
  2476. # which without a bound scope reads the LAUNCH profile's ambient environment for a worker spawned
  2477. # on B's behalf (and raises under multiplex) — so bind B's secret scope around the build.
  2478. with (_worker_profile_scope(profile_home, bind_home=False) if profile_home
  2479. else contextlib.nullcontext()):
  2480. env = build_subprocess_env(
  2481. scrub_secrets=is_multiplex_active() or routed,
  2482. inherit_profile_home=True,
  2483. )
  2484. # The dispatcher is detached from every conversation; its worker must never
  2485. # inherit routing mirrored by a previous gateway turn.
  2486. from gateway.session_context import _VAR_MAP
  2487. for key in _VAR_MAP:
  2488. env.pop(key, None)
  2489. # Inject HERMES_HOME so the worker reads the profile-scoped config.yaml:
  2490. # without it the child's get_hermes_home() falls back to the DEFAULT
  2491. # profile root because `hermes -p` applies its override before
  2492. # hermes_constants is imported.
  2493. if profile_home:
  2494. env["HERMES_HOME"] = profile_home
  2495. # A multiplexer dispatching for another profile must not hand it the launch
  2496. # profile's .env settings / TERMINAL_* policy — a standalone dispatcher never would.
  2497. strip_launch_profile_env(env, profile_home)
  2498. if task.tenant:
  2499. env["HERMES_TENANT"] = task.tenant
  2500. env["HERMES_KANBAN_TASK"] = task.id
  2501. env["HERMES_KANBAN_WORKSPACE"] = workspace
  2502. # Tag the session `kanban` so session-browsing surfaces filter it out by
  2503. # source instead of rendering one sidebar row per attempt.
  2504. env["HERMES_SESSION_SOURCE"] = "kanban"
  2505. # TERMINAL_CWD takes precedence over process cwd in file_tools and
  2506. # build_context_files_prompt; without it relative writes land in the gateway
  2507. # user's home and workers load the gateway's AGENTS.md. file_tools rejects
  2508. # relative / sentinel values, so only set a real absolute directory.
  2509. # Pin TERMINAL_CWD to the task's workspace so the worker's file tools and context-file loader anchor on
  2510. # the workspace, not whatever cwd the dispatching gateway happened to export. The worker subprocess is
  2511. # already launched with cwd=workspace, but TERMINAL_CWD takes precedence over the process cwd in both
  2512. # file_tools._resolve_base_dir (#41312 — relative write_file paths were landing in the gateway user's
  2513. # home) and build_context_files_prompt (#34619 — workers loaded the dispatching gateway's AGENTS.md
  2514. # instead of the task's). Setting it to the workspace fixes both: the workspace is where the task's work
  2515. # actually happens.
  2516. if workspace and os.path.isabs(workspace) and os.path.isdir(workspace):
  2517. env["TERMINAL_CWD"] = workspace
  2518. if task.branch_name:
  2519. env["HERMES_KANBAN_BRANCH"] = task.branch_name
  2520. if task.current_run_id is not None:
  2521. env["HERMES_KANBAN_RUN_ID"] = str(task.current_run_id)
  2522. if task.claim_lock:
  2523. env["HERMES_KANBAN_CLAIM_LOCK"] = task.claim_lock
  2524. # Goal-loop mode (Ralph-style /goal judge loop in cli.py quiet-mode path).
  2525. # Only set when enabled so non-goal tasks keep a clean env.
  2526. if task.goal_mode:
  2527. env["HERMES_KANBAN_GOAL_MODE"] = "1"
  2528. if task.goal_max_turns is not None:
  2529. env["HERMES_KANBAN_GOAL_MAX_TURNS"] = str(int(task.goal_max_turns))
  2530. for var in ("TERMINAL_TIMEOUT", "TERMINAL_MAX_FOREGROUND_TIMEOUT"):
  2531. override = _worker_terminal_timeout_env(task.max_runtime_seconds, env.get(var))
  2532. if override is not None:
  2533. env[var] = override
  2534. # Pin the board DB + workspaces root so the worker's kanban paths still
  2535. # match after `hermes -p` rewrites HERMES_HOME (symlink / Docker layouts).
  2536. env["HERMES_KANBAN_DB"] = str(_kb.kanban_db_path(board=board))
  2537. env["HERMES_KANBAN_WORKSPACES_ROOT"] = str(_kb.workspaces_root(board=board))
  2538. # Board slug — defense-in-depth pin if a path is resolved without the
  2539. # DB / workspaces env vars.
  2540. env["HERMES_KANBAN_BOARD"] = _kb._normalize_board_slug(board) or _kb.get_current_board()
  2541. # kanban_comment reads HERMES_PROFILE for its default author; `-p` alone
  2542. # doesn't set the env var.
  2543. env["HERMES_PROFILE"] = profile_arg
  2544. # This is the grant boundary: the dispatcher assigned this new worker's task.
  2545. from agent.delegation_context import DELEGATED_CHILD_ENV_MARKER
  2546. env.pop(DELEGATED_CHILD_ENV_MARKER, None)
  2547. # `--cli` is the highest-precedence TUI override; dropping HERMES_TUI covers
  2548. # older hermes builds on PATH that predate the flag's precedence.
  2549. env.pop("HERMES_TUI", None)
  2550. cmd = _worker_argv(task, profile_arg, env.get("HERMES_HOME"))
  2551. # The worker is `-m hermes_cli.kanban_worker_client`: Hermes itself, with no console-script
  2552. # bootstrap. The routed-profile scrub above strips Hermes-owned PYTHONPATH entries (user
  2553. # children must not see our tree), so a PYTHONPATH-launched dispatcher spawned a child that
  2554. # died on `No module named 'hermes_cli'` before it ever reached the owner. Same pin as cron's
  2555. # external worker (#112729): applied to the sanitized env, after every other decision stands.
  2556. from cron.scheduler_worker_env import pin_hermes_tree_on_pythonpath
  2557. env = pin_hermes_tree_on_pythonpath(env, Path(__file__).resolve().parent.parent)
  2558. # A worker spawned by a managed systemd gateway must leave the gateway's
  2559. # cgroup before startup; otherwise restarting the service kills the worker
  2560. # that is performing the handoff.
  2561. cmd = _restart_safe_worker_argv(task, cmd)
  2562. from tools.process_registry import systemd_user_bus_env
  2563. env = systemd_user_bus_env(env)
  2564. log_f = _open_worker_log(task, board)
  2565. try:
  2566. proc = subprocess.Popen(
  2567. cmd,
  2568. cwd=workspace if os.path.isdir(workspace) else None,
  2569. stdin=subprocess.DEVNULL,
  2570. stdout=log_f,
  2571. stderr=subprocess.STDOUT,
  2572. env=env,
  2573. start_new_session=True, # detached-spawn: ok — restart-safe kanban worker, must survive the dispatcher
  2574. creationflags=subprocess.CREATE_NO_WINDOW if _kb._IS_WINDOWS else 0,
  2575. )
  2576. except FileNotFoundError:
  2577. log_f.close()
  2578. raise RuntimeError(
  2579. "`hermes` executable not found on PATH. "
  2580. "Install Hermes Agent or activate its venv before running the kanban dispatcher."
  2581. )
  2582. # Intentionally NOT closing log_f: the child keeps writing after return;
  2583. # the OS-level FD stays open in the child until it exits.
  2584. if _kb._IS_WINDOWS:
  2585. _live_worker_procs[proc.pid] = proc
  2586. return proc.pid
  2587. # ---------------------------------------------------------------------------
  2588. # Long-lived dispatcher daemon
  2589. # ---------------------------------------------------------------------------
  2590. def run_daemon(
  2591. *,
  2592. interval: float = 60.0,
  2593. max_spawn: Optional[int] = None,
  2594. failure_limit: int = DEFAULT_FAILURE_LIMIT,
  2595. stop_event=None,
  2596. on_tick=None,
  2597. ) -> None:
  2598. """Run the dispatcher in a loop until interrupted.
  2599. Calls :func:`dispatch_once` every ``interval`` seconds; exits cleanly on
  2600. SIGINT / SIGTERM so it is systemd-friendly. ``stop_event`` and ``on_tick``
  2601. are test hooks. Each tick resolves ``kanban.max_in_progress`` exactly like
  2602. the gateway dispatcher and ``hermes kanban dispatch`` — the standalone
  2603. daemon must not be the one uncapped entry point.
  2604. """
  2605. import threading
  2606. if stop_event is None:
  2607. stop_event = threading.Event()
  2608. def _handle(_signum, _frame):
  2609. stop_event.set()
  2610. # Install handlers only on the main thread — tests call this inline from
  2611. # worker threads and signal() would raise there.
  2612. if threading.current_thread() is threading.main_thread():
  2613. for sig_name in ("SIGINT", "SIGTERM"):
  2614. sig = getattr(signal, sig_name, None)
  2615. if sig is not None:
  2616. with contextlib.suppress(ValueError, OSError):
  2617. signal.signal(sig, _handle)
  2618. while not stop_event.is_set():
  2619. try:
  2620. # Re-resolved every tick (config load is mtime-cached) so operator
  2621. # edits apply without a restart.
  2622. max_in_progress = resolve_max_in_progress(configured_max_in_progress())
  2623. with contextlib.closing(_kbc.connect()) as conn:
  2624. res = dispatch_once(
  2625. conn,
  2626. max_spawn=max_spawn,
  2627. max_in_progress=max_in_progress,
  2628. failure_limit=failure_limit,
  2629. )
  2630. if on_tick is not None:
  2631. with contextlib.suppress(Exception):
  2632. on_tick(res)
  2633. except Exception:
  2634. # Don't let any single tick kill the daemon.
  2635. import traceback
  2636. traceback.print_exc()
  2637. stop_event.wait(timeout=interval)
  2638. # Late-bound origin namespace (see module docstring); imported LAST so this
  2639. # module is fully populated before ``kanban_db`` imports from it.
  2640. from hermes_cli import kanban_db as _kb
  2641. from hermes_cli import kanban_db_connect as _kbc
  2642. from hermes_cli import kanban_db_workspace as _kbw