kanban_db.py 181 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231
  1. """SQLite-backed Kanban board shared across profiles (the cross-profile coordination primitive).
  2. Lives under the shared Hermes root: ``default`` board DB at ``<root>/kanban.db`` (pre-boards
  3. back-compat), other boards at ``<root>/kanban/boards/<slug>/``; a worker on one board never sees
  4. another. Board resolution: ``board=`` arg > ``HERMES_KANBAN_BOARD`` > ``HERMES_KANBAN_DB`` (pins the
  5. file path) > ``<root>/kanban/current`` > ``default`` — but only for unfenced callers; the dispatcher
  6. injects these into workers, and dispatched workers (``HERMES_KANBAN_TASK``), delegated children and
  7. board-enumerating machine flows (gateway notifier/watcher/dispatcher, ``pin_first_board_resolution``)
  8. always resolve through the pin, so workers physically cannot see other boards.
  9. Concurrency: WAL + ``BEGIN IMMEDIATE`` + compare-and-swap on ``tasks.status``/``claim_lock`` —
  10. SQLite serializes writers so one claimer wins, losers see zero rows (no retries, no distributed
  11. locks). Schema: tasks, task_links, task_comments, task_events, task_runs, attachments, notify subs.
  12. """
  13. from __future__ import annotations
  14. import contextlib
  15. import json
  16. import os
  17. import re
  18. import secrets
  19. import sqlite3
  20. import subprocess
  21. import sys
  22. import logging
  23. import time
  24. from contextvars import ContextVar, Token
  25. from dataclasses import dataclass
  26. from pathlib import Path
  27. from typing import Any, Iterable, Optional
  28. from hermes_cli.kanban_workflow import DEFAULT_STATUSES as VALID_STATUSES
  29. from toolsets import get_toolset_names
  30. from utils import unlink_files_older_than
  31. _log = logging.getLogger(__name__)
  32. # --- Shared micro-helpers (row access, JSON, env, git) ---
  33. def _lossy_text(value: Any) -> Any:
  34. """``bytes`` -> ``str`` with U+FFFD for undecodable sequences; anything else passes through.
  35. Installed as every board connection's ``text_factory`` (a TEXT cell holding
  36. invalid UTF-8 otherwise aborts the whole ``fetchall`` with "Could not decode
  37. to UTF-8") and applied to BLOB-typed cells in the ``from_row`` constructors
  38. (task, comment, event, run), so one
  39. corrupt row degrades to replacement characters instead of taking the board
  40. listing down."""
  41. if isinstance(value, bytes):
  42. return value.decode("utf-8", errors="replace")
  43. return value
  44. def _row_get(row: Any, col: str, default: Any = None) -> Any:
  45. """``row[col]`` tolerant of the column being absent from the SELECT / schema."""
  46. if row is None or col not in row.keys():
  47. return default
  48. return row[col]
  49. def _json_or(value: Any, default: Any = None) -> Any:
  50. """Decode a JSON text column; any decode failure or empty value yields ``default``."""
  51. if not value:
  52. return default
  53. try:
  54. return json.loads(value)
  55. except Exception:
  56. return default
  57. def _json_dict(value: Any) -> dict:
  58. """Decode a JSON text column that must be an object; anything else yields ``{}``."""
  59. parsed = _json_or(value, {})
  60. return parsed if isinstance(parsed, dict) else {}
  61. def _env_int(name: str, default: int, *, minimum: int = 0) -> int:
  62. """Integer env override: absent/empty/non-integer/below ``minimum`` falls back to ``default``."""
  63. raw = os.environ.get(name, "").strip()
  64. if raw:
  65. try:
  66. parsed = int(raw)
  67. except ValueError:
  68. return default
  69. if parsed >= minimum:
  70. return parsed
  71. return default
  72. def _git_out(cwd: Path, *args: str, timeout: int = 30) -> Optional[str]:
  73. """Run ``git -C cwd args`` and return stripped stdout, or ``None`` on any failure / empty output."""
  74. try:
  75. result = subprocess.run(
  76. ["git", "-C", str(cwd), *args],
  77. capture_output=True, text=True, encoding="utf-8", errors="replace",
  78. timeout=timeout, check=False,
  79. )
  80. except Exception:
  81. return None
  82. if result.returncode != 0:
  83. return None
  84. return (result.stdout or "").strip() or None
  85. # --- Constants ---
  86. VALID_INITIAL_STATUSES = {"running", "blocked"}
  87. # Typed block reasons (routing in ``_route_block``); ``None`` = legacy un-typed.
  88. VALID_BLOCK_KINDS = {"dependency", "needs_input", "capability", "transient"}
  89. # Same-reason block -> unblock -> re-block cycles before routing to ``triage``.
  90. # Counts unblock recurrences, NOT dispatcher failures (``DEFAULT_FAILURE_LIMIT``).
  91. BLOCK_RECURRENCE_LIMIT = 2
  92. VALID_WORKSPACE_KINDS = {"scratch", "worktree", "dir"}
  93. def normalize_reasoning_effort(effort: Optional[str]) -> Optional[str]:
  94. """``VALID_REASONING_EFFORTS`` or ``"none"`` (thinking off), case-insensitive;
  95. empty/None = inherit the profile's own effort (NULL). Anything else raises —
  96. a typo'd level must not quietly hand the task back to the profile default."""
  97. from hermes_constants import VALID_REASONING_EFFORTS
  98. value = str(effort or "").strip().lower()
  99. if not value:
  100. return None
  101. if value == "none" or value in VALID_REASONING_EFFORTS:
  102. return value
  103. allowed = ", ".join(("none", *VALID_REASONING_EFFORTS))
  104. raise ValueError(f"reasoning_effort must be one of {allowed}, got {effort!r}")
  105. KNOWN_TOOLSET_NAMES = frozenset(name.casefold() for name in get_toolset_names())
  106. _IS_WINDOWS = sys.platform == "win32"
  107. KANBAN_ATTACHMENT_MAX_BYTES = 25 * 1024 * 1024 # one cap for dashboard, tools and CLI
  108. def _assert_not_delegated_child_mutation(path: str | Path | None = None) -> None:
  109. """Reject Kanban mutations from ``delegate_task`` child contexts.
  110. The tool/CLI fast-fail guards are UX, not a trust boundary (a child can shell
  111. out or import this module); the invariant lives here so every ``write_txn``
  112. user and board-metadata mutator fails closed before touching durable state.
  113. *path* is the board DB / metadata root being mutated; ``None`` means the
  114. lineage's own board (``kanban_home()``).
  115. """
  116. from agent.delegation_context import kanban_path_is_fenced
  117. if kanban_path_is_fenced(kanban_home() if path is None else path):
  118. raise PermissionError("delegate_task child contexts cannot mutate Kanban tasks or boards")
  119. def _fire_kanban_lifecycle_hook(event: str, task_id: str, **fields: Any) -> None:
  120. """Best-effort lifecycle hook. Call AFTER the write txn commits (plugins never
  121. run under the SQLite write lock, always see durable state); failures are
  122. swallowed so an observer can never break a transition."""
  123. try:
  124. from hermes_cli.lifecycle import invoke_hook
  125. invoke_hook(event, task_id=task_id, profile_name=_hook_profile_name(), **fields)
  126. except Exception as exc: # pragma: no cover - defensive
  127. _log.debug("kanban lifecycle hook %s failed: %s", event, exc)
  128. def _fire_task_hook(event: str, task: Optional[Task], task_id: str, run_id: Optional[int], **fields: Any) -> None:
  129. """Lifecycle hook for a task transition; ``assignee`` from the (possibly missing) row."""
  130. _fire_kanban_lifecycle_hook(
  131. event, task_id, board=get_current_board(),
  132. assignee=task.assignee if task else None, run_id=run_id, **fields,
  133. )
  134. def _hook_profile_name() -> str:
  135. """Active profile for hook payloads; ``"default"`` when it cannot be resolved."""
  136. from hermes_cli.profiles import get_active_profile_name
  137. try:
  138. return get_active_profile_name()
  139. except Exception:
  140. return "default"
  141. def _kanban_observer_consumed(event: str) -> bool:
  142. """Hot-path short-circuit: skip payload assembly when nothing subscribes.
  143. Inspection failure counts as unconsumed (dropping an observer is always safe)."""
  144. try:
  145. from hermes_cli.lifecycle import has_hook
  146. return has_hook(event)
  147. except Exception: # pragma: no cover - defensive
  148. return False
  149. def _fire_worker_spawned_hook(
  150. conn: sqlite3.Connection, task: Task, workspace_path: str, pid: Optional[int], *,
  151. board: Optional[str] = None,
  152. ) -> None:
  153. """``on_kanban_worker_spawned`` AFTER the PID is durably persisted; best-effort."""
  154. if not _kanban_observer_consumed("on_kanban_worker_spawned"):
  155. return
  156. try:
  157. _fire_kanban_lifecycle_hook(
  158. "on_kanban_worker_spawned", task.id, board=board or get_current_board(),
  159. assignee=task.assignee, run_id=_current_run_id(conn, task.id),
  160. worker_pid=int(pid) if pid else None, workspace_path=str(workspace_path),
  161. )
  162. except Exception as exc: # pragma: no cover - defensive
  163. _log.debug("kanban worker spawned hook failed: %s", exc)
  164. def notify_task_updated(
  165. conn: sqlite3.Connection, task_id: str, changed_fields: Iterable[str], *,
  166. board: Optional[str] = None,
  167. ) -> None:
  168. """``on_kanban_task_updated`` AFTER a non-lifecycle task mutation commits
  169. (also for direct-SQL surfaces like dashboard field editors).
  170. ``changed_fields`` carries field NAMES only, never values."""
  171. if not _kanban_observer_consumed("on_kanban_task_updated"):
  172. return
  173. try:
  174. row = conn.execute(
  175. "SELECT assignee, current_run_id FROM tasks WHERE id = ?", (task_id,),
  176. ).fetchone()
  177. _fire_kanban_lifecycle_hook(
  178. "on_kanban_task_updated", task_id, board=board or get_current_board(),
  179. assignee=row["assignee"] if row else None,
  180. run_id=row["current_run_id"] if row else None, changed_fields=list(changed_fields),
  181. )
  182. except Exception as exc: # pragma: no cover - defensive
  183. _log.debug("kanban task updated hook failed: %s", exc)
  184. # DispatchResult counters whose non-zero value means the tick did something.
  185. _TICK_ACTIVITY_FIELDS = (
  186. "spawned", "reclaimed", "promoted", "reconciled_orphans", "reaped_terminal_workers", "crashed", "stale",
  187. "timed_out", "auto_blocked", "rate_limited", "auto_assigned_default",
  188. "respawn_guarded", "skipped_per_profile_capped", "skipped_unassigned",
  189. "skipped_nonspawnable",
  190. )
  191. def _fire_dispatch_tick_hook(
  192. result: DispatchResult, *, board: Optional[str] = None, dry_run: bool = False,
  193. ) -> None:
  194. """``on_kanban_dispatch_tick`` — strictly AFTER ``_dispatch_tick_lock`` is
  195. released so a slow subscriber cannot stall a sibling dispatcher.
  196. Re-port of PR #56066 per the #64231 batch disposition: renamed to the taxonomy form and called by
  197. ``dispatch_once`` strictly AFTER ``_dispatch_tick_lock`` has been released — the original fired inside
  198. the lock, so a slow subscriber could extend the single-writer critical section and stall a sibling
  199. dispatcher's tick. Observer-only and fully best-effort: any subscriber failure is swallowed.
  200. """
  201. if not _kanban_observer_consumed("on_kanban_dispatch_tick"):
  202. return
  203. try:
  204. from hermes_cli.lifecycle import invoke_hook
  205. profile_name = _hook_profile_name()
  206. if board is None:
  207. try:
  208. board = get_current_board()
  209. except Exception:
  210. board = None
  211. outcome = "ok"
  212. if result.skipped_locked:
  213. outcome = "skipped_locked"
  214. elif not any(getattr(result, f) for f in _TICK_ACTIVITY_FIELDS):
  215. outcome = "idle"
  216. invoke_hook(
  217. "on_kanban_dispatch_tick", board=board, profile_name=profile_name,
  218. dry_run=bool(dry_run), outcome=outcome, result=result,
  219. )
  220. except Exception as exc: # pragma: no cover - defensive
  221. _log.debug("kanban dispatch tick hook failed: %s", exc)
  222. # Claim window before the next tick reclaims a running task; long workers
  223. # ``heartbeat_claim`` or raise it via HERMES_KANBAN_CLAIM_TTL_SECONDS.
  224. DEFAULT_CLAIM_TTL_SECONDS = 15 * 60
  225. # A live PID with a heartbeat older than this is wedged and reclaimed anyway
  226. # (``_touch_activity`` keeps genuinely active workers fresh).
  227. # If a worker's PID is still alive but its ``last_heartbeat_at`` is older than this when
  228. # ``release_stale_claims`` runs, treat the worker as wedged and reclaim regardless of PID liveness (#29747
  229. # gap 3). This catches the logic-loop case where the process is technically running but not making
  230. # observable progress. ``_touch_activity`` bridges chunk-level liveness into ``last_heartbeat_at`` via
  231. # #31752, so any genuinely active worker keeps its heartbeat fresh as a side effect of normal API traffic.
  232. DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS = 60 * 60
  233. # Grace when a host-local worker survived termination (e.g. parked in D state
  234. # under memory.high, SIGKILL pending): releasing now would spawn a duplicate.
  235. RECLAIM_DEFER_GRACE_SECONDS = 120
  236. def _resolve_claim_ttl_seconds(ttl_seconds: Optional[int] = None) -> int:
  237. """Explicit ``ttl_seconds`` > ``HERMES_KANBAN_CLAIM_TTL_SECONDS`` > default."""
  238. if ttl_seconds is not None:
  239. return max(1, int(ttl_seconds))
  240. return _env_int("HERMES_KANBAN_CLAIM_TTL_SECONDS", DEFAULT_CLAIM_TTL_SECONDS, minimum=1)
  241. # ``detect_crashed_workers`` skips ``_pid_alive`` this long after start: the
  242. # fork -> /proc window can report a fresh worker dead.
  243. DEFAULT_CRASH_GRACE_SECONDS = 30
  244. # Worker exit "provider rate-limited": released WITHOUT counting a failure (the
  245. # breaker must never trip on a throttle). 75 == BSD EX_TEMPFAIL.
  246. KANBAN_RATE_LIMIT_EXIT_CODE = 75
  247. # Worker exit "provider rejected the configuration": credential revoked (401/403), model gone
  248. # (404), TLS chain broken — a retry cannot fix it, so the dispatcher parks the card blocked on
  249. # the FIRST occurrence instead of spending ``failure_limit`` identical spawns. 78 == BSD EX_CONFIG.
  250. KANBAN_TERMINAL_PROVIDER_EXIT_CODE = 78
  251. def _resolve_crash_grace_seconds() -> int:
  252. """``HERMES_KANBAN_CRASH_GRACE_SECONDS`` (0 = immediate, for tests) else default."""
  253. return _env_int("HERMES_KANBAN_CRASH_GRACE_SECONDS", DEFAULT_CRASH_GRACE_SECONDS)
  254. def _resolve_rate_limit_cooldown_seconds() -> int:
  255. """``HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS`` (0 = next tick, for tests) else default."""
  256. return _env_int("HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS", DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS)
  257. # --- Paths ---
  258. DEFAULT_BOARD = "default"
  259. _CURRENT_BOARD_OVERRIDE: ContextVar[str | None] = ContextVar(
  260. "hermes_kanban_current_board_override", default=None,
  261. )
  262. # Machine flows that enumerate boards (gateway notifier / watcher / dispatcher
  263. # ticks) resolve board paths env-pin-first — see pin_first_board_resolution().
  264. _PIN_FIRST_BOARD_RESOLUTION: ContextVar[bool] = ContextVar(
  265. "hermes_kanban_pin_first_board_resolution", default=False,
  266. )
  267. @contextlib.contextmanager
  268. def pin_first_board_resolution():
  269. """Resolve board paths env-pin-first for machine flows that enumerate boards.
  270. The gateway notifier, per-subscription cursor writes and the embedded
  271. dispatcher poll every board slug from ``list_boards()`` — the slug is not
  272. caller intent, it is an iteration variable. On a box whose environment pins
  273. ``HERMES_KANBAN_DB`` (the dispatcher default) every slug must map to that
  274. one pinned file: the notifier dedupes resolved DB paths, and per-slug
  275. paths read empty boards nobody writes, silently killing wake
  276. notifications. Explicit cross-board intent (CLI ``--board``, a model
  277. tool's ``board=``) is a USER property and must never run inside this
  278. context; with no pin set this context changes nothing.
  279. """
  280. token = _PIN_FIRST_BOARD_RESOLUTION.set(True)
  281. try:
  282. yield
  283. finally:
  284. _PIN_FIRST_BOARD_RESOLUTION.reset(token)
  285. @contextlib.contextmanager
  286. def scoped_current_board(slug: str):
  287. """Pin the active board for the current context only."""
  288. token: Token[str | None] = _CURRENT_BOARD_OVERRIDE.set(slug)
  289. try:
  290. yield
  291. finally:
  292. _CURRENT_BOARD_OVERRIDE.reset(token)
  293. # Slug = directory name: strict enough to stop traversal / separators, loose
  294. # enough for kebab-case. Display names (spaces, emoji) live in board.json.
  295. _BOARD_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9\-_]{0,63}$")
  296. def _normalize_board_slug(slug: Optional[str]) -> Optional[str]:
  297. """Lowercase + strip a slug; validate; return ``None`` for empty."""
  298. s = str(slug).strip().lower() if slug is not None else ""
  299. if not s:
  300. return None
  301. if not _BOARD_SLUG_RE.match(s):
  302. raise ValueError(
  303. f"invalid board slug {slug!r}: must be 1-64 chars, lowercase "
  304. f"alphanumerics / hyphens / underscores, not starting with '-' or '_'"
  305. )
  306. return s
  307. def _slug_or_default(board: Optional[str]) -> str:
  308. return _normalize_board_slug(board) or DEFAULT_BOARD
  309. def _require_slug(slug: str) -> str:
  310. normed = _normalize_board_slug(slug)
  311. if not normed:
  312. raise ValueError("board slug is required")
  313. return normed
  314. def kanban_home() -> Path:
  315. """``HERMES_KANBAN_HOME`` else ``get_default_hermes_root()``. Shared across
  316. profiles BY DESIGN: resolving through the active profile's HERMES_HOME would
  317. fork the board per profile and break the dispatcher/worker handoff."""
  318. override = os.environ.get("HERMES_KANBAN_HOME", "").strip()
  319. if override:
  320. return Path(override).expanduser()
  321. from hermes_constants import get_default_hermes_root
  322. return get_default_hermes_root()
  323. def boards_root() -> Path:
  324. """``<root>/kanban/boards`` — parent of the *additional* named boards.
  325. ``default`` is deliberately not here (its DB stays at ``<root>/kanban.db``)."""
  326. return kanban_home() / "kanban" / "boards"
  327. def current_board_path() -> Path:
  328. """``<root>/kanban/current`` — one-line slug written by ``boards switch``; absent = ``default``."""
  329. return kanban_home() / "kanban" / "current"
  330. def get_current_board() -> str:
  331. """Active slug: context override -> ``HERMES_KANBAN_BOARD`` -> ``<root>/kanban/current``
  332. (only while that board exists) -> ``DEFAULT_BOARD``. A malformed/stale slug
  333. falls through — the dispatcher must never crash on a hand-edited file."""
  334. def _existing(candidate: str) -> Optional[str]:
  335. if not candidate:
  336. return None
  337. try:
  338. normed = _normalize_board_slug(candidate)
  339. except ValueError:
  340. return None
  341. if not (normed and board_exists(normed)):
  342. return None
  343. # An archived board leaves a tombstone board.json; a stale `current`
  344. # file or HERMES_KANBAN_BOARD pointing at one falls through to
  345. # default rather than resurrecting it (#43243).
  346. if read_board_metadata(normed).get("archived"):
  347. return None
  348. return normed
  349. for candidate in (
  350. (_CURRENT_BOARD_OVERRIDE.get() or "").strip(),
  351. os.environ.get("HERMES_KANBAN_BOARD", "").strip(),
  352. ):
  353. found = _existing(candidate)
  354. if found:
  355. return found
  356. try:
  357. f = current_board_path()
  358. if f.exists():
  359. # utf-8-sig read fix (ours): tolerate BOM-persisted current-board files.
  360. val = f.read_text(encoding="utf-8-sig").strip()
  361. if val:
  362. try:
  363. normed = _normalize_board_slug(val)
  364. if normed and board_exists(normed):
  365. return normed
  366. except ValueError:
  367. pass
  368. except OSError:
  369. pass
  370. return DEFAULT_BOARD
  371. def set_current_board(slug: str) -> Path:
  372. """Persist ``slug`` as the active board; returns the file written. Does NOT
  373. check the board exists — callers do (so ``boards switch <typo>`` errors)."""
  374. _assert_not_delegated_child_mutation()
  375. normed = _require_slug(slug)
  376. path = current_board_path()
  377. path.parent.mkdir(parents=True, exist_ok=True)
  378. path.write_text(normed + "\n", encoding="utf-8")
  379. return path
  380. def clear_current_board() -> None:
  381. """Remove ``<root>/kanban/current`` so the active board reverts to ``default``."""
  382. _assert_not_delegated_child_mutation()
  383. with contextlib.suppress(FileNotFoundError):
  384. current_board_path().unlink()
  385. def board_dir(board: Optional[str] = None) -> Path:
  386. """``<root>/kanban/boards/<slug>/``. For ``default`` this holds metadata
  387. only (board.json, workspaces/, logs/) — its DB stays at ``<root>/kanban.db``
  388. for back-compat (:func:`kanban_db_path`).
  389. """
  390. return boards_root() / _slug_or_default(board)
  391. def board_exists(board: Optional[str] = None) -> bool:
  392. """Board has a ``board.json`` on disk; ``default`` always exists."""
  393. slug = _slug_or_default(board)
  394. if slug == DEFAULT_BOARD:
  395. return True
  396. return _dir_holds_board(board_dir(slug))
  397. def _explicit_board_slug(board: Optional[str]) -> Optional[str]:
  398. """Explicit caller intent: a direct ``board=`` argument, else the scoped
  399. ``--board`` context (CLI ``hermes kanban --board``, dashboard plugin_api);
  400. ``None`` when the caller expressed neither."""
  401. if board is not None:
  402. return _normalize_board_slug(board)
  403. # A caller-scoped board (CLI `hermes kanban --board B ...`, dashboard
  404. # plugin_api) is explicit intent just like a direct board= argument —
  405. # without this a worker-pinned HERMES_KANBAN_DB silently outranks --board
  406. # (os-reviewer P1 on PR#107195 / t_11c4afd8).
  407. ctx = (_CURRENT_BOARD_OVERRIDE.get() or "").strip()
  408. if ctx:
  409. try:
  410. return _normalize_board_slug(ctx)
  411. except ValueError:
  412. return None
  413. return None
  414. def _explicit_board_intent_pinned() -> bool:
  415. """Whether an explicit ``board=`` (or scoped ``--board``) must still resolve
  416. through the ``HERMES_KANBAN_DB``-style env pins instead of its own board dir.
  417. True for machine flows wrapped in :func:`pin_first_board_resolution` and
  418. for every execution the dispatcher fences: its own workers (they carry
  419. ``HERMES_KANBAN_TASK``) and delegated children / descendants (the
  420. ``HERMES_DELEGATED_CHILD_CONTEXT`` marker). The pins ARE the "workers
  421. physically cannot see other boards" isolation (5ec6baa), and
  422. ``agent.delegation_context.kanban_path_is_fenced`` checks the pinned path /
  423. fenced root — an explicit board that resolved elsewhere would also escape
  424. that fence."""
  425. if _PIN_FIRST_BOARD_RESOLUTION.get():
  426. return True
  427. from agent.delegation_context import explicit_board_intent_is_pinned
  428. return explicit_board_intent_is_pinned()
  429. def _board_path(
  430. env_var: Optional[str], board: Optional[str], default_parts: tuple[str, ...], leaf: str,
  431. ) -> Path:
  432. """Shared resolver. An explicit ``board=`` argument — or the scoped
  433. ``--board`` context (:func:`scoped_current_board`) — outranks the ``env_var``
  434. pin ONLY where no fence applies (see :func:`_explicit_board_intent_pinned`):
  435. user-facing cross-board intent (CLI ``--board``, a model tool's ``board=``)
  436. is honored, but machine flows that enumerate boards (gateway notifier /
  437. watcher / dispatcher ticks) and dispatched or delegated workers keep
  438. resolving through the pin. Without explicit intent the ``env_var`` override
  439. pins the file, else legacy ``<root>/<default_parts>`` for the ``default``
  440. board, else ``board_dir(slug)/leaf``."""
  441. pin = os.environ.get(env_var, "").strip() if env_var else ""
  442. slug = _explicit_board_slug(board)
  443. if pin and (slug is None or _explicit_board_intent_pinned()):
  444. return Path(pin).expanduser()
  445. if slug is None:
  446. slug = get_current_board()
  447. if slug == DEFAULT_BOARD:
  448. return kanban_home().joinpath(*default_parts)
  449. return board_dir(slug) / leaf
  450. def kanban_db_path(board: Optional[str] = None) -> Path:
  451. """``kanban.db`` path: ``HERMES_KANBAN_DB`` pins it (injected into workers);
  452. ``default`` -> ``<root>/kanban.db`` (back-compat), else the board dir."""
  453. return _board_path("HERMES_KANBAN_DB", board, ("kanban.db",), "kanban.db")
  454. def workspaces_root(board: Optional[str] = None) -> Path:
  455. """Per-board scratch workspace root (``HERMES_KANBAN_WORKSPACES_ROOT`` wins);
  456. ``default`` keeps the legacy ``<root>/kanban/workspaces/``."""
  457. return _board_path("HERMES_KANBAN_WORKSPACES_ROOT", board, ("kanban", "workspaces"), "workspaces")
  458. def attachments_root(board: Optional[str] = None) -> Path:
  459. """Per-board attachments root (``HERMES_KANBAN_ATTACHMENTS_ROOT`` wins). Workers
  460. read attachments by absolute path, so remote terminal backends must mount it."""
  461. return _board_path("HERMES_KANBAN_ATTACHMENTS_ROOT", board, ("kanban", "attachments"), "attachments")
  462. def task_attachments_dir(task_id: str, board: Optional[str] = None) -> Path:
  463. """Return the per-task attachment directory ``<root>/<task_id>/``."""
  464. return attachments_root(board=board) / task_id
  465. def worker_logs_dir(board: Optional[str] = None) -> Path:
  466. """Per-board worker log dir (logs follow the board so ``hermes kanban log``
  467. is unambiguous when two boards share a task id)."""
  468. return _board_path(None, board, ("kanban", "logs"), "logs")
  469. # --- Data classes ---
  470. @dataclass
  471. class Task:
  472. """In-memory view of a row from the ``tasks`` table."""
  473. id: str
  474. title: str
  475. body: Optional[str]
  476. assignee: Optional[str]
  477. status: str
  478. priority: int
  479. created_by: Optional[str]
  480. created_at: Optional[int] # None only for an unparseable stamp
  481. started_at: Optional[int]
  482. completed_at: Optional[int]
  483. workspace_kind: str
  484. workspace_path: Optional[str]
  485. claim_lock: Optional[str]
  486. claim_expires: Optional[int]
  487. tenant: Optional[str]
  488. branch_name: Optional[str] = None
  489. project_id: Optional[str] = None
  490. result: Optional[str] = None
  491. idempotency_key: Optional[str] = None
  492. # Column semantics: see SCHEMA_SQL.
  493. consecutive_failures: int = 0
  494. worker_pid: Optional[int] = None
  495. last_failure_error: Optional[str] = None
  496. max_runtime_seconds: Optional[int] = None
  497. last_heartbeat_at: Optional[int] = None
  498. current_run_id: Optional[int] = None
  499. workflow_template_id: Optional[str] = None
  500. current_step_key: Optional[str] = None
  501. skills: Optional[list] = None # None = defaults only; [] = explicitly none
  502. model_override: Optional[str] = None
  503. provider_override: Optional[str] = None # provider ``model_override`` belongs to
  504. reasoning_effort: Optional[str] = None # VALID_REASONING_EFFORTS | "none"; NULL = profile's
  505. # Breaker trip count; None -> ``kanban.failure_limit`` -> DEFAULT_FAILURE_LIMIT.
  506. max_retries: Optional[int] = None
  507. # ``/goal``-style loop: a judge re-checks each turn IN THE SAME SESSION until
  508. # done / budget exhausted (-> kanban_block); ``goal_max_turns`` None -> goals default.
  509. goal_mode: bool = False
  510. goal_max_turns: Optional[int] = None
  511. session_id: Optional[str] = None # originating HERMES_SESSION_ID; NULL from CLI/dashboard
  512. # VALID_BLOCK_KINDS or None (legacy); kept across unblock so a same-kind re-block reads as a loop.
  513. block_kind: Optional[str] = None
  514. block_recurrences: int = 0 # unblock-loop counter, see BLOCK_RECURRENCE_LIMIT
  515. completion_contract: Optional[str] = None
  516. @classmethod
  517. def from_row(cls, row: sqlite3.Row) -> Task:
  518. g = lambda col, default=None: _lossy_text(_row_get(row, col, default))
  519. parsed = _json_or(g("skills"))
  520. skills_value = [str(s) for s in parsed if s] if isinstance(parsed, list) else None
  521. return cls(
  522. **{col: _lossy_text(row[col]) for col in _TASK_REQUIRED_COLUMNS},
  523. **{col: _to_epoch(g(col)) for col in _TASK_EPOCH_COLUMNS}, # outside writers may store text: epoch or None
  524. **{col: g(col) for col in _TASK_OPTIONAL_COLUMNS},
  525. **{col: g(col) or None for col in _TASK_EMPTY_IS_NULL_COLUMNS},
  526. # Pre-migration fallbacks (spawn_failures / last_spawn_error) are only
  527. # reachable on a DB never opened since the rename migration landed.
  528. consecutive_failures=g("consecutive_failures", g("spawn_failures", 0)),
  529. last_failure_error=g("last_failure_error", g("last_spawn_error")),
  530. skills=skills_value,
  531. goal_mode=bool(g("goal_mode")),
  532. block_recurrences=int(g("block_recurrences") or 0),
  533. )
  534. # Columns every schema version has (KeyError if the SELECT omitted them).
  535. _TASK_REQUIRED_COLUMNS = (
  536. "id", "title", "body", "assignee", "status", "priority", "created_by",
  537. "workspace_kind", "workspace_path", "claim_lock",
  538. )
  539. _TASK_EPOCH_COLUMNS = ("created_at", "started_at", "completed_at", "claim_expires", "last_heartbeat_at")
  540. # Later-added columns read as NULL when absent from the row.
  541. _TASK_OPTIONAL_COLUMNS = (
  542. "branch_name", "project_id", "tenant", "result", "idempotency_key", "worker_pid",
  543. "max_runtime_seconds", "current_run_id", "workflow_template_id",
  544. "current_step_key", "max_retries", "session_id", "completion_contract",
  545. )
  546. # Text columns where "" is stored/read as "not set".
  547. _TASK_EMPTY_IS_NULL_COLUMNS = (
  548. "model_override", "provider_override", "reasoning_effort", "goal_max_turns", "block_kind",
  549. )
  550. @dataclass
  551. class Run:
  552. """One attempt at a task (``task_runs`` row): opened on claim, closed on
  553. complete/block/crash/timeout/reclaim; carries the handoff summary."""
  554. id: int
  555. task_id: str
  556. profile: Optional[str]
  557. step_key: Optional[str]
  558. status: str
  559. claim_lock: Optional[str]
  560. claim_expires: Optional[int]
  561. worker_pid: Optional[int]
  562. max_runtime_seconds: Optional[int]
  563. last_heartbeat_at: Optional[int]
  564. started_at: int
  565. ended_at: Optional[int]
  566. outcome: Optional[str]
  567. summary: Optional[str]
  568. metadata: Optional[dict]
  569. error: Optional[str]
  570. @classmethod
  571. def from_row(cls, row: sqlite3.Row) -> Run:
  572. return cls(
  573. **{
  574. col: _lossy_text(row[col]) for col in (
  575. "task_id", "profile", "step_key", "status", "claim_lock",
  576. "worker_pid", "max_runtime_seconds", "outcome", "summary", "error",
  577. )
  578. },
  579. **{col: _to_epoch(row[col]) for col in ("claim_expires", "last_heartbeat_at", "ended_at")},
  580. id=int(row["id"]), started_at=_to_epoch(row["started_at"]) or 0,
  581. metadata=_json_or(_lossy_text(row["metadata"])),
  582. )
  583. @dataclass
  584. class Comment:
  585. id: int
  586. task_id: str
  587. author: str
  588. body: str
  589. created_at: int
  590. @classmethod
  591. def from_row(cls, r: sqlite3.Row) -> Comment:
  592. return cls(
  593. id=r["id"], task_id=r["task_id"], author=_lossy_text(r["author"]),
  594. body=_lossy_text(r["body"]), created_at=_to_epoch(r["created_at"]) or 0,
  595. )
  596. @dataclass
  597. class Attachment:
  598. """In-memory view of a row from the ``task_attachments`` table."""
  599. id: int
  600. task_id: str
  601. filename: str
  602. stored_path: str
  603. content_type: Optional[str]
  604. size: int
  605. uploaded_by: Optional[str]
  606. created_at: int
  607. @classmethod
  608. def from_row(cls, r: sqlite3.Row) -> Attachment:
  609. return cls(
  610. id=r["id"], task_id=r["task_id"], filename=r["filename"],
  611. stored_path=r["stored_path"], content_type=r["content_type"],
  612. size=r["size"] or 0, uploaded_by=r["uploaded_by"], created_at=_to_epoch(r["created_at"]) or 0,
  613. )
  614. @dataclass
  615. class Event:
  616. id: int
  617. task_id: str
  618. kind: str
  619. payload: Optional[dict]
  620. created_at: int
  621. run_id: Optional[int] = None
  622. @classmethod
  623. def from_row(cls, row: sqlite3.Row) -> Event:
  624. run_id = _row_get(row, "run_id")
  625. return cls(
  626. id=row["id"], task_id=row["task_id"], kind=_lossy_text(row["kind"]),
  627. payload=_json_or(_lossy_text(row["payload"])), created_at=_to_epoch(row["created_at"]) or 0,
  628. run_id=_opt_int(run_id),
  629. )
  630. # --- Schema ---
  631. SCHEMA_SQL = """
  632. CREATE TABLE IF NOT EXISTS tasks (
  633. id TEXT PRIMARY KEY,
  634. title TEXT NOT NULL,
  635. body TEXT,
  636. assignee TEXT,
  637. status TEXT NOT NULL,
  638. priority INTEGER DEFAULT 0,
  639. created_by TEXT,
  640. created_at INTEGER NOT NULL,
  641. started_at INTEGER,
  642. completed_at INTEGER,
  643. workspace_kind TEXT NOT NULL DEFAULT 'scratch',
  644. workspace_path TEXT,
  645. branch_name TEXT,
  646. -- Optional link to a first-class Project (hermes_cli/projects_db). When set,
  647. -- the task's worktree is anchored under the project's primary repo with a
  648. -- deterministic branch name instead of a random wt/<task-id> fallback.
  649. project_id TEXT,
  650. claim_lock TEXT,
  651. claim_expires INTEGER,
  652. tenant TEXT,
  653. result TEXT,
  654. idempotency_key TEXT,
  655. -- Unified consecutive-failure counter. Incremented on spawn
  656. -- failure, timeout, or crash; reset only on successful completion.
  657. -- The circuit breaker in _record_task_failure trips when this
  658. -- exceeds DEFAULT_FAILURE_LIMIT consecutive non-successes.
  659. consecutive_failures INTEGER NOT NULL DEFAULT 0,
  660. worker_pid INTEGER,
  661. -- Restart-stable fingerprint of worker_pid ("<boot/instantiation epoch>|<start time>",
  662. -- kanban_db_dispatch._process_fingerprint) recorded at spawn: liveness and kills require pid
  663. -- AND fingerprint to agree, so a PID recycled after a reboot is never read as our worker or
  664. -- signalled. NULL = legacy row (pre-fingerprint spawn); 'unverified' = capture failed at
  665. -- spawn (held while live, never signalled). Column keeps its INTEGER affinity for the
  666. -- start-time-only integer values older rows carry.
  667. worker_started_at INTEGER,
  668. -- Short excerpt of the most recent failure's error text.
  669. last_failure_error TEXT,
  670. max_runtime_seconds INTEGER,
  671. last_heartbeat_at INTEGER,
  672. -- Pointer into task_runs for the currently-active run (NULL if no
  673. -- run is in-flight). Denormalised for cheap reads.
  674. current_run_id INTEGER,
  675. -- Forward-compat for v2 workflow routing. In v1 the kernel writes
  676. -- these when the task is opted into a template but otherwise ignores
  677. -- them; the dispatcher doesn't consult them for routing yet.
  678. workflow_template_id TEXT,
  679. current_step_key TEXT,
  680. -- Force-loaded skills for the worker on this task, stored as JSON.
  681. -- Passed to the worker via `--skills`. NULL or empty array = no extras.
  682. skills TEXT,
  683. -- Per-task model override. When set, the dispatcher passes -m <model>
  684. -- to the worker, overriding the profile's default model. NULL = use
  685. -- the profile default.
  686. model_override TEXT,
  687. -- Provider the model override belongs to. When set (alongside
  688. -- model_override), the dispatcher passes --provider <name> so the
  689. -- worker resolves the model against the right backend instead of the
  690. -- profile's configured provider. NULL = profile provider.
  691. provider_override TEXT,
  692. -- Per-task reasoning effort for the worker (minimal|low|medium|high|
  693. -- xhigh|max|ultra, or 'none' for thinking off). When set, the dispatcher
  694. -- passes --reasoning <level> so the worker runs at that depth regardless
  695. -- of the profile's agent.reasoning_effort. NULL = profile setting.
  696. reasoning_effort TEXT,
  697. -- Per-task override for the consecutive-failure circuit breaker.
  698. -- The value is the failure count at which the breaker trips — e.g.
  699. -- ``max_retries=1`` blocks on the first failure. NULL (the common
  700. -- case) falls through to the dispatcher-level ``kanban.failure_limit``
  701. -- config and then ``DEFAULT_FAILURE_LIMIT``.
  702. max_retries INTEGER,
  703. -- When 1, the dispatched worker runs in a Ralph-style goal loop: an
  704. -- auxiliary judge re-evaluates the worker's response against the
  705. -- card title/body after each turn and feeds a continuation prompt
  706. -- back into the SAME session until the judge agrees the work is done
  707. -- or ``goal_max_turns`` is exhausted. NULL/0 = classic single-shot
  708. -- worker (the default).
  709. goal_mode INTEGER NOT NULL DEFAULT 0,
  710. -- Goal-loop turn budget for ``goal_mode`` workers. NULL = use the
  711. -- goals-engine default.
  712. goal_max_turns INTEGER,
  713. -- Originating chat/agent session id when the task was created from
  714. -- inside an agent loop that propagated ``HERMES_SESSION_ID``. NULL
  715. -- for tasks created from the CLI, dashboard, or any path that doesn't
  716. -- set the env var, and for an id with no ``sessions`` row in this
  717. -- profile's state.db (kanban_create verifies before stamping). Indexed
  718. -- so per-session list queries stay cheap on larger boards.
  719. session_id TEXT,
  720. -- Typed block reason set by ``block_task`` (one of VALID_BLOCK_KINDS, or
  721. -- NULL for legacy/un-typed blocks). Drives routing: ``dependency`` never
  722. -- sits in ``blocked`` (goes to ``todo`` for parent-gating); the others go
  723. -- to ``blocked`` for a human. Preserved across unblock so a re-block for
  724. -- the SAME kind can be recognised as a loop.
  725. block_kind TEXT,
  726. -- Unblock-loop counter. Incremented each time a task is re-blocked for the
  727. -- same truly-blocked reason after having been unblocked. When it reaches
  728. -- BLOCK_RECURRENCE_LIMIT the task is routed to ``triage`` instead of
  729. -- ``blocked`` so a cron can't spin it forever. Reset to 0 only on a
  730. -- successful completion — NOT on unblock (resetting on unblock is exactly
  731. -- the amnesia that let the loop run unbounded).
  732. block_recurrences INTEGER NOT NULL DEFAULT 0
  733. );
  734. CREATE TABLE IF NOT EXISTS task_links (
  735. parent_id TEXT NOT NULL,
  736. child_id TEXT NOT NULL,
  737. PRIMARY KEY (parent_id, child_id)
  738. );
  739. CREATE TABLE IF NOT EXISTS task_comments (
  740. id INTEGER PRIMARY KEY AUTOINCREMENT,
  741. task_id TEXT NOT NULL,
  742. author TEXT NOT NULL,
  743. body TEXT NOT NULL,
  744. created_at INTEGER NOT NULL
  745. );
  746. CREATE TABLE IF NOT EXISTS task_events (
  747. id INTEGER PRIMARY KEY AUTOINCREMENT,
  748. task_id TEXT NOT NULL,
  749. run_id INTEGER,
  750. kind TEXT NOT NULL,
  751. payload TEXT,
  752. created_at INTEGER NOT NULL
  753. );
  754. -- Historical attempt record. Each time the dispatcher claims a task, a
  755. -- new row is created here; claim state, PID, heartbeat, runtime cap,
  756. -- and structured summary all live on the run, not the task. Multiple
  757. -- rows per task id when the task was retried after crash/timeout/block.
  758. -- v2 of the kanban schema will use ``step_key`` to drive per-stage
  759. -- workflow routing; in v1 the column is nullable and unused (kernel
  760. -- ignores it).
  761. CREATE TABLE IF NOT EXISTS task_runs (
  762. id INTEGER PRIMARY KEY AUTOINCREMENT,
  763. task_id TEXT NOT NULL,
  764. profile TEXT,
  765. step_key TEXT,
  766. status TEXT NOT NULL,
  767. -- status: running | done | blocked | crashed | timed_out | failed | released
  768. claim_lock TEXT,
  769. claim_expires INTEGER,
  770. worker_pid INTEGER,
  771. -- Spawn-time start fingerprint of worker_pid (see tasks.worker_started_at). Retained with
  772. -- worker_pid after the run ends so a worker that outlives its terminal transition can
  773. -- still be found and reaped; NULL = legacy row, never signalled.
  774. worker_started_at INTEGER,
  775. max_runtime_seconds INTEGER,
  776. last_heartbeat_at INTEGER,
  777. started_at INTEGER NOT NULL,
  778. ended_at INTEGER,
  779. outcome TEXT,
  780. -- outcome: completed | blocked | crashed | timed_out | spawn_failed |
  781. -- gave_up | reclaimed | (null while still running)
  782. summary TEXT,
  783. metadata TEXT,
  784. error TEXT
  785. );
  786. -- Files attached to a task (PDFs, images, source documents). The blob
  787. -- lives on disk under ``attachments_root(board)/<task_id>/<stored_name>``;
  788. -- this row carries metadata + the absolute ``stored_path`` so the
  789. -- dashboard can list/download and ``build_worker_context`` can surface
  790. -- the absolute path to the worker (which has full file-tool access). See
  791. -- #35338.
  792. CREATE TABLE IF NOT EXISTS task_attachments (
  793. id INTEGER PRIMARY KEY AUTOINCREMENT,
  794. task_id TEXT NOT NULL,
  795. filename TEXT NOT NULL,
  796. stored_path TEXT NOT NULL,
  797. content_type TEXT,
  798. size INTEGER NOT NULL DEFAULT 0,
  799. uploaded_by TEXT,
  800. created_at INTEGER NOT NULL
  801. );
  802. -- Subscription from a gateway source (platform + chat + thread) to a
  803. -- task. The gateway's kanban-notifier watcher tails task_events and
  804. -- pushes ``completed`` / ``blocked`` / ``spawn_auto_blocked`` events to
  805. -- the original requester so human-in-the-loop workflows close the loop.
  806. CREATE TABLE IF NOT EXISTS kanban_notify_subs (
  807. task_id TEXT NOT NULL,
  808. platform TEXT NOT NULL,
  809. chat_id TEXT NOT NULL,
  810. thread_id TEXT NOT NULL DEFAULT '',
  811. user_id TEXT,
  812. user_id_alt TEXT,
  813. chat_type TEXT,
  814. notifier_profile TEXT,
  815. delivery_mode TEXT NOT NULL DEFAULT 'notify',
  816. delivery_metadata TEXT,
  817. created_at INTEGER NOT NULL,
  818. last_event_id INTEGER NOT NULL DEFAULT 0,
  819. last_ping_event_id INTEGER NOT NULL DEFAULT 0,
  820. PRIMARY KEY (task_id, platform, chat_id, thread_id)
  821. );
  822. CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
  823. CREATE INDEX IF NOT EXISTS idx_links_child ON task_links(child_id);
  824. CREATE INDEX IF NOT EXISTS idx_links_parent ON task_links(parent_id);
  825. CREATE INDEX IF NOT EXISTS idx_comments_task ON task_comments(task_id, created_at);
  826. CREATE INDEX IF NOT EXISTS idx_events_task ON task_events(task_id, created_at);
  827. CREATE INDEX IF NOT EXISTS idx_runs_task ON task_runs(task_id, started_at);
  828. CREATE INDEX IF NOT EXISTS idx_runs_status ON task_runs(status);
  829. CREATE INDEX IF NOT EXISTS idx_attachments_task ON task_attachments(task_id, created_at);
  830. CREATE INDEX IF NOT EXISTS idx_notify_task ON kanban_notify_subs(task_id);
  831. """
  832. # --- ID generation ---
  833. def _new_task_id() -> str:
  834. """``t_`` + 4 hex bytes (collision ~1e-3 at 100k tasks; 2 bytes would hit 50%
  835. by 10k). Idempotency belongs to ``idempotency_key``, not id uniqueness."""
  836. return "t_" + secrets.token_hex(4)
  837. def _claimer_id() -> str:
  838. """Return a ``host:pid`` string that identifies this claimer."""
  839. import socket
  840. try:
  841. host = socket.gethostname() or "unknown"
  842. except Exception:
  843. host = "unknown"
  844. return f"{host}:{os.getpid()}"
  845. def _host_prefix() -> str:
  846. """``"<host>:"`` prefix shared by every claim lock issued from this host."""
  847. return f"{_claimer_id().split(':', 1)[0]}:"
  848. # --- Task creation / mutation ---
  849. def _validate_model_override(model: Optional[str], provider: Optional[str]) -> tuple[Optional[str], Optional[str]]:
  850. """Strip both; a provider without a model is rejected (a bare ``--provider``
  851. would re-resolve the profile's model against another backend — exactly
  852. the mismatch the override exists to kill)."""
  853. model = (model or "").strip() or None
  854. provider = (provider or "").strip() or None
  855. if provider and not model:
  856. raise ValueError("provider_override requires a model_override")
  857. return model, provider
  858. def _canonical_assignee(assignee: Optional[str]) -> Optional[str]:
  859. """Lowercase-assignee normalization for Kanban rows (dashboard/CLI parity)."""
  860. if assignee is None:
  861. return None
  862. from hermes_cli.profiles import normalize_profile_name
  863. return normalize_profile_name(assignee)
  864. def _resolve_project_link(
  865. conn: sqlite3.Connection, project_id: Optional[str], project_source_task_id: Optional[str],
  866. workspace_kind: str, workspace_path: Optional[str],
  867. ) -> tuple[Optional[str], Any, Optional[str], str]:
  868. """``(project_id, project_obj, project_repo, workspace_kind)`` for ``create_task``.
  869. A project-linked task is anchored to the project's primary repo as a
  870. worktree with a deterministic branch (slug + task id). Projects live in the
  871. creator's per-profile projects.db, but the stored repo path is absolute so
  872. the cross-profile dispatcher needs no projects.db access. ``project_repo``
  873. is set when the worktree path must still be derived from the new task id.
  874. """
  875. project_id = (str(project_id).strip() or None) if project_id is not None else None
  876. if not project_id:
  877. return None, None, None, workspace_kind
  878. from hermes_cli import projects_db as _pdb
  879. project_repo: Optional[str] = None
  880. try:
  881. with _pdb.connect_closing() as _pconn:
  882. project_obj = _pdb.get_project(_pconn, project_id)
  883. except Exception:
  884. project_obj = None
  885. if project_obj is None and project_source_task_id:
  886. project_obj, project_repo = _project_from_source_task(
  887. conn, _pdb, project_id, str(project_source_task_id),
  888. )
  889. if project_obj is not None and workspace_kind == "scratch":
  890. workspace_kind = "worktree"
  891. if project_obj is None:
  892. # Unresolvable id/slug: drop the link (never a dangling reference,
  893. # never a crash) and create an ordinary scratch task.
  894. return None, None, None, workspace_kind
  895. # Canonicalise (a slug may have been passed) and anchor the worktree
  896. # under the project's primary repo.
  897. if workspace_kind == "scratch" and project_obj.primary_path:
  898. workspace_kind = "worktree"
  899. if workspace_kind == "worktree" and workspace_path is None and project_obj.primary_path:
  900. # Concrete path is deferred to the insert loop: a fresh
  901. # ``<repo>/.worktrees/<task-id>`` keyed on the new task id.
  902. project_repo = str(project_obj.primary_path)
  903. return project_obj.id, project_obj, project_repo, workspace_kind
  904. def _project_from_source_task(
  905. conn: sqlite3.Connection, _pdb: Any, project_id: str, source_task_id: str,
  906. ) -> tuple[Any, Optional[str]]:
  907. """Recover a Project (and its repo) from a canonical project-linked
  908. worktree task on this board. Worker profiles have their own projects.db
  909. while the Kanban DB is shared, so this carries the repo + branch
  910. convention forward without opening the creator's store and without
  911. reusing the source task's literal worktree path. ``(None, None)`` when
  912. the source task is not a ``<repo>/.worktrees/<id>`` project worktree."""
  913. source_task = get_task(conn, source_task_id)
  914. if not (
  915. source_task is not None
  916. and source_task.project_id == project_id
  917. and source_task.workspace_kind == "worktree"
  918. and source_task.workspace_path
  919. ):
  920. return None, None
  921. source_path = Path(source_task.workspace_path)
  922. if not (
  923. source_path.is_absolute()
  924. and source_path.name == source_task.id
  925. and source_path.parent.name == ".worktrees"
  926. ):
  927. return None, None
  928. project_slug = None
  929. if source_task.branch_name:
  930. prefix, separator, leaf = source_task.branch_name.partition("/")
  931. if separator and (leaf == source_task.id or leaf.startswith(f"{source_task.id}-")):
  932. with contextlib.suppress(ValueError):
  933. project_slug = _pdb.normalize_slug(prefix)
  934. if project_slug is None:
  935. with contextlib.suppress(ValueError):
  936. project_slug = _pdb.normalize_slug(project_id)
  937. if not project_slug:
  938. return None, None
  939. project_repo = str(source_path.parent.parent)
  940. project_obj = _pdb.Project(
  941. id=project_id, slug=project_slug, name=project_slug, created_at=0, primary_path=project_repo,
  942. )
  943. return project_obj, project_repo
  944. def _normalize_task_skills(skills: Optional[Iterable[str]]) -> Optional[list[str]]:
  945. """Strip/dedupe a skills list. Commas are refused (a comma-joined string must
  946. not land in one argv slot); toolset names are rejected all at once because
  947. agents that confuse the two usually pass several."""
  948. if skills is None:
  949. return None
  950. cleaned: list[str] = []
  951. seen: set[str] = set()
  952. toolset_typos: list[str] = []
  953. for s in skills:
  954. if not s:
  955. continue
  956. name = str(s).strip()
  957. if not name:
  958. continue
  959. if "," in name:
  960. raise ValueError(
  961. f"skill name cannot contain comma: {name!r} "
  962. f"(pass a list of separate names instead of a comma-joined string)"
  963. )
  964. if name.casefold() in KNOWN_TOOLSET_NAMES:
  965. toolset_typos.append(name)
  966. continue
  967. if name in seen:
  968. continue
  969. seen.add(name)
  970. cleaned.append(name)
  971. if toolset_typos:
  972. quoted = ", ".join(repr(n) for n in toolset_typos)
  973. noun = "is a toolset name" if len(toolset_typos) == 1 else "are toolset names"
  974. raise ValueError(
  975. f"{quoted} {noun}, not skill name(s). "
  976. "Put toolsets in the assignee profile's `toolsets:` config "
  977. "instead of per-task skills. Skills are named skill bundles "
  978. "(e.g. `blogwatcher`, `github-code-review`); toolsets are runtime "
  979. "capabilities (e.g. `web`, `browser`, `terminal`)."
  980. )
  981. return cleaned
  982. def create_task(
  983. conn: sqlite3.Connection, *, title: str, body: Optional[str] = None,
  984. assignee: Optional[str] = None, created_by: Optional[str] = None,
  985. workspace_kind: Optional[str] = None, workspace_path: Optional[str] = None,
  986. branch_name: Optional[str] = None, tenant: Optional[str] = None, priority: int = 0,
  987. parents: Iterable[str] = (), triage: bool = False, idempotency_key: Optional[str] = None,
  988. max_runtime_seconds: Optional[int] = None, skills: Optional[Iterable[str]] = None,
  989. max_retries: Optional[int] = None, model_override: Optional[str] = None,
  990. provider_override: Optional[str] = None, reasoning_effort: Optional[str] = None,
  991. goal_mode: bool = False, goal_max_turns: Optional[int] = None, initial_status: str = "running",
  992. session_id: Optional[str] = None, board: Optional[str] = None, project_id: Optional[str] = None,
  993. project_source_task_id: Optional[str] = None,
  994. creator_task_id: Optional[str] = None,
  995. completion_contract: Optional[str] = None,
  996. ) -> str:
  997. """Create a task (optionally under ``parents``); returns its id.
  998. Status: ``ready`` unless a parent is not ``done`` (``todo``); ``triage=True``
  999. forces ``triage``; ``initial_status="blocked"`` parks it for human ops.
  1000. ``idempotency_key``: an existing non-archived task with the key is returned
  1001. instead of a duplicate. ``max_runtime_seconds``: cap before the dispatcher
  1002. SIGTERMs and re-queues. ``model_override``/``provider_override`` pin the
  1003. worker model (provider requires model); ``reasoning_effort`` is independent.
  1004. ``creator_task_id``: inherit durable session/subscriptions independently of
  1005. dependency edges; an explicit ``session_id`` still wins.
  1006. ``project_source_task_id``: cross-profile fallback when ``project_id`` is not
  1007. in the active profile's projects.db — see ``_resolve_project_link``.
  1008. ``workspace_kind=None`` (omitted) inherits a project-scoped board's project;
  1009. an explicit ``"scratch"`` or ``project_id=""`` is a request for no project.
  1010. """
  1011. from hermes_cli.kanban_db_graph import initial_task_state, inherit_creator_origin
  1012. from hermes_cli.kanban_pr_acceptance import validate_contract
  1013. completion_contract = validate_contract(completion_contract)
  1014. model_override, provider_override = _validate_model_override(model_override, provider_override)
  1015. reasoning_effort = normalize_reasoning_effort(reasoning_effort)
  1016. assignee = _canonical_assignee(assignee)
  1017. if not title or not title.strip():
  1018. raise ValueError("title is required")
  1019. if initial_status not in VALID_INITIAL_STATUSES:
  1020. raise ValueError(f"initial_status must be one of {sorted(VALID_INITIAL_STATUSES)}")
  1021. # A project-scoped board anchors every new task to its project's repo
  1022. # (deterministic worktree + branch) without each surface repeating it.
  1023. # An explicit ``scratch`` (or ``project_id=""``) is a request for no project:
  1024. # it must not be upgraded to a worktree in the board's repo (#106342).
  1025. if project_id is None and workspace_kind != "scratch":
  1026. try:
  1027. project_id = (_board_meta_for(board).get("project_id") or "").strip() or None
  1028. except Exception:
  1029. pass
  1030. if workspace_kind is None:
  1031. workspace_kind = "scratch"
  1032. if workspace_kind not in VALID_WORKSPACE_KINDS:
  1033. raise ValueError(
  1034. f"workspace_kind must be one of {sorted(VALID_WORKSPACE_KINDS)}, "
  1035. f"got {workspace_kind!r}"
  1036. )
  1037. if branch_name is not None:
  1038. branch_name = str(branch_name).strip() or None
  1039. if branch_name and workspace_kind != "worktree":
  1040. raise ValueError("branch_name is only valid for worktree workspaces")
  1041. project_id, project_obj, project_repo, workspace_kind = _resolve_project_link(
  1042. conn, project_id, project_source_task_id, workspace_kind, workspace_path
  1043. )
  1044. parents = tuple(p for p in parents if p)
  1045. skills_list = _normalize_task_skills(skills)
  1046. # Idempotency check BEFORE the write txn (no lock held); a concurrent-create
  1047. # race may insert twice, the next lookup stabilises on the newest.
  1048. if idempotency_key:
  1049. row = conn.execute(
  1050. "SELECT id FROM tasks WHERE idempotency_key = ? "
  1051. "AND status != 'archived' "
  1052. "ORDER BY created_at DESC LIMIT 1", (idempotency_key,),
  1053. ).fetchone()
  1054. if row:
  1055. return row["id"]
  1056. now = int(time.time())
  1057. # Only persistent kinds inherit the board ``default_workdir``: a scratch
  1058. # task inheriting it would point cleanup at the user's source tree.
  1059. if workspace_path is None and project_repo is None and workspace_kind in {"dir", "worktree"}:
  1060. board_default = _board_meta_for(board).get("default_workdir")
  1061. if board_default:
  1062. workspace_path = str(board_default)
  1063. # Retry once on the extremely unlikely id collision.
  1064. for attempt in range(2):
  1065. task_id = _new_task_id()
  1066. try:
  1067. # allow_nested: graph builders compose create_task under one outer
  1068. # commit so the dispatcher never sees a half-built graph.
  1069. with write_txn(conn, allow_nested=True):
  1070. task_status, tenant = initial_task_state(conn, parents, initial_status, triage, tenant)
  1071. # Project worktree: fresh dir under the repo + deterministic
  1072. # branch, instead of the random ``wt/<id>`` worker fallback.
  1073. if project_obj is not None and workspace_kind == "worktree":
  1074. if project_repo and not workspace_path:
  1075. workspace_path = os.path.join(project_repo, ".worktrees", task_id)
  1076. if not branch_name:
  1077. branch_name = _project_branch_name(project_obj, task_id, title)
  1078. conn.execute(
  1079. """
  1080. INSERT INTO tasks (
  1081. id, title, body, assignee, status, priority,
  1082. created_by, created_at, workspace_kind, workspace_path,
  1083. branch_name, project_id, tenant, idempotency_key,
  1084. max_runtime_seconds,
  1085. skills, max_retries, model_override, provider_override,
  1086. reasoning_effort,
  1087. goal_mode, goal_max_turns, session_id, completion_contract
  1088. ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  1089. """,
  1090. (
  1091. task_id, title.strip(), body, assignee, task_status, priority,
  1092. created_by, now, workspace_kind, workspace_path,
  1093. branch_name, project_id, tenant, idempotency_key,
  1094. _opt_int(max_runtime_seconds),
  1095. json.dumps(skills_list) if skills_list is not None else None,
  1096. _opt_int(max_retries), model_override, provider_override, reasoning_effort,
  1097. 1 if goal_mode else 0, _opt_int(goal_max_turns), session_id, completion_contract,
  1098. ),
  1099. )
  1100. for pid in parents:
  1101. _link(conn, pid, task_id)
  1102. _append_event(
  1103. conn,
  1104. task_id,
  1105. "created",
  1106. {
  1107. "assignee": assignee,
  1108. "status": task_status,
  1109. "parents": list(parents),
  1110. "creator_task_id": creator_task_id,
  1111. "tenant": tenant,
  1112. "workspace_kind": workspace_kind,
  1113. "workspace_path": workspace_path,
  1114. "branch_name": branch_name,
  1115. "project_id": project_id,
  1116. "skills": list(skills_list) if skills_list else None,
  1117. "goal_mode": bool(goal_mode) or None,
  1118. "model_override": model_override,
  1119. "provider_override": provider_override,
  1120. },
  1121. )
  1122. if task_status == "blocked":
  1123. _append_event(
  1124. conn,
  1125. task_id,
  1126. "blocked",
  1127. {"reason": "initial_status", "status": "blocked", "actor": created_by or "user"},
  1128. )
  1129. if task_status == "todo":
  1130. # Parked behind an open parent: record why, exactly as
  1131. # link_tasks does, so the board never shows an unexplained todo.
  1132. gating = [p for p in parents if _task_status(conn, p) not in ("done", "archived")]
  1133. if gating:
  1134. _append_event(
  1135. conn,
  1136. task_id,
  1137. "dependency_wait",
  1138. {"reason": "parent_not_done", "parent": gating[0]},
  1139. )
  1140. # ACK-edge: the originating channel hears a child BLOCK, not just the fan-in.
  1141. inherit_creator_origin(conn, task_id, creator_task_id, created_at=now)
  1142. _inherit_notify_subs(conn, task_id, parents, created_at=now)
  1143. return task_id
  1144. except sqlite3.IntegrityError:
  1145. if attempt == 1:
  1146. raise
  1147. raise RuntimeError("unreachable")
  1148. def _board_meta_for(board: Optional[str]) -> dict:
  1149. return read_board_metadata(board if board else get_current_board())
  1150. def _project_branch_name(project_obj: Any, task_id: str, title: Optional[str]) -> Optional[str]:
  1151. from hermes_cli import projects_db as _pdb
  1152. try:
  1153. return _pdb.branch_name_for(project_obj, task_id, title=title or "")
  1154. except Exception:
  1155. return None
  1156. def _link(conn: sqlite3.Connection, parent_id: str, child_id: str) -> None:
  1157. conn.execute(
  1158. "INSERT OR IGNORE INTO task_links (parent_id, child_id) VALUES (?, ?)",
  1159. (parent_id, child_id),
  1160. )
  1161. def _missing_task_ids(conn: sqlite3.Connection, ids: Iterable[str]) -> list[str]:
  1162. """Subset of ``ids`` (order kept) with no ``tasks`` row."""
  1163. ids = list(ids)
  1164. if not ids:
  1165. return []
  1166. placeholders = ",".join("?" * len(ids))
  1167. rows = conn.execute(f"SELECT id FROM tasks WHERE id IN ({placeholders})", ids).fetchall()
  1168. present = {r["id"] for r in rows}
  1169. return [p for p in ids if p not in present]
  1170. def _inherit_notify_subs(
  1171. conn: sqlite3.Connection, child_id: str, parents: Iterable[str], *,
  1172. created_at: Optional[int] = None,
  1173. ) -> None:
  1174. """Copy parents' notify subscriptions to a child, cursor caught up to the
  1175. child's current event so a late ``link_tasks`` never replays history.
  1176. Single owner of inheritance (create_task, link_tasks, decompose). It must
  1177. copy EVERY routing/delivery column: dropping ``chat_type`` made DM-originated
  1178. completions wake a fresh group session instead of the originating DM.
  1179. Omitting columns here silently degrades routing: a DM-originated child completion falls back to
  1180. chat_type='group' and wakes a fresh group-scoped session instead of the originating DM (issue #73030).
  1181. """
  1182. parent_ids = tuple(dict.fromkeys(p for p in parents if p))
  1183. if not parent_ids:
  1184. return
  1185. row = conn.execute(
  1186. "SELECT COALESCE(MAX(id), 0) AS cursor FROM task_events WHERE task_id = ?", (child_id,),
  1187. ).fetchone()
  1188. cursor = int(row["cursor"] if row is not None else 0)
  1189. placeholders = ",".join("?" * len(parent_ids))
  1190. conn.execute(
  1191. f"""
  1192. INSERT OR IGNORE INTO kanban_notify_subs
  1193. (task_id, platform, chat_id, thread_id, user_id, user_id_alt,
  1194. chat_type, notifier_profile, delivery_mode, delivery_metadata,
  1195. created_at, last_event_id)
  1196. SELECT ?, platform, chat_id, thread_id, user_id, user_id_alt,
  1197. COALESCE(chat_type, 'dm'), notifier_profile,
  1198. COALESCE(delivery_mode, 'notify'), delivery_metadata, ?, ?
  1199. FROM kanban_notify_subs
  1200. WHERE task_id IN ({placeholders})
  1201. """,
  1202. (child_id, int(created_at if created_at is not None else time.time()), cursor, *parent_ids),
  1203. )
  1204. def get_task(conn: sqlite3.Connection, task_id: str) -> Optional[Task]:
  1205. row = conn.execute("SELECT * FROM tasks WHERE id = ?", (task_id,)).fetchone()
  1206. return Task.from_row(row) if row else None
  1207. # Canonical sort-order mappings for ``hermes kanban list --sort``.
  1208. # Each value is a raw SQL fragment appended after ``ORDER BY``.
  1209. VALID_SORT_ORDERS: dict[str, str] = {
  1210. "created": "created_at ASC, id ASC",
  1211. "created-desc": "created_at DESC, id DESC",
  1212. "priority": "priority DESC, created_at ASC",
  1213. "priority-desc": "priority ASC, created_at ASC",
  1214. "status": "status ASC, created_at ASC",
  1215. "assignee": "assignee ASC, created_at ASC",
  1216. "title": "title ASC, id ASC",
  1217. "updated": "started_at DESC NULLS LAST, created_at DESC",
  1218. "completed-desc": "completed_at DESC NULLS LAST, id DESC",
  1219. }
  1220. def list_tasks(
  1221. conn: sqlite3.Connection, *, assignee: Optional[str] = None, status: Optional[str] = None,
  1222. tenant: Optional[str] = None, session_id: Optional[str] = None, include_archived: bool = False,
  1223. limit: Optional[int] = None, order_by: Optional[str] = None,
  1224. workflow_template_id: Optional[str] = None, current_step_key: Optional[str] = None,
  1225. ) -> list[Task]:
  1226. if status is not None and status not in VALID_STATUSES:
  1227. raise ValueError(f"status must be one of {sorted(VALID_STATUSES)}")
  1228. query = "SELECT * FROM tasks WHERE 1=1"
  1229. params: list[Any] = []
  1230. for col, val in (
  1231. ("assignee", _canonical_assignee(assignee)), ("status", status), ("tenant", tenant),
  1232. ("session_id", session_id), ("workflow_template_id", workflow_template_id),
  1233. ("current_step_key", current_step_key),
  1234. ):
  1235. if val is not None:
  1236. query += f" AND {col} = ?"
  1237. params.append(val)
  1238. if not include_archived and status != "archived":
  1239. query += " AND status != 'archived'"
  1240. if order_by is not None:
  1241. order_by = order_by.strip().lower()
  1242. if order_by not in VALID_SORT_ORDERS:
  1243. raise ValueError(f"order_by must be one of {sorted(VALID_SORT_ORDERS.keys())}")
  1244. query += f" ORDER BY {VALID_SORT_ORDERS[order_by]}"
  1245. else:
  1246. query += " ORDER BY priority DESC, created_at ASC"
  1247. if limit:
  1248. query += f" LIMIT {int(limit)}"
  1249. rows = conn.execute(query, params).fetchall()
  1250. return [Task.from_row(r) for r in rows]
  1251. def assign_task(conn: sqlite3.Connection, task_id: str, profile: Optional[str]) -> bool:
  1252. """Assign/reassign; raises RuntimeError while the task is running under a claim."""
  1253. profile = _canonical_assignee(profile)
  1254. with write_txn(conn):
  1255. row = conn.execute(
  1256. "SELECT status, claim_lock, assignee FROM tasks WHERE id = ?", (task_id,)
  1257. ).fetchone()
  1258. if not row:
  1259. return False
  1260. if row["claim_lock"] is not None and row["status"] == "running":
  1261. raise RuntimeError(
  1262. f"cannot reassign {task_id}: currently running (claimed). "
  1263. "Wait for completion or reclaim the stale lock first."
  1264. )
  1265. if row["assignee"] != profile:
  1266. # The failure streak is per task/profile; a new profile starts fresh.
  1267. conn.execute(
  1268. "UPDATE tasks SET assignee = ?, consecutive_failures = 0, "
  1269. "last_failure_error = NULL WHERE id = ?", (profile, task_id),
  1270. )
  1271. else:
  1272. conn.execute("UPDATE tasks SET assignee = ? WHERE id = ?", (profile, task_id))
  1273. # ``from`` lets the respawn guard tell a real handoff (dev→closer) from
  1274. # a no-op re-assign or an unassign, which must not lift ``active_pr``.
  1275. _append_event(
  1276. conn, task_id, "assigned", {"assignee": profile, "from": row["assignee"]},
  1277. )
  1278. # Observer fires AFTER commit so subscribers see durable state.
  1279. notify_task_updated(conn, task_id, ("assignee",))
  1280. return True
  1281. def set_model_override(
  1282. conn: sqlite3.Connection, task_id: str, model: Optional[str], provider: Optional[str] = None,
  1283. ) -> bool:
  1284. """Set (empty ``model`` clears BOTH) the per-task model/provider override.
  1285. Allowed while ``running``: it applies on the NEXT dispatch, which is the
  1286. rate-limit-recovery flow (set, then reclaim/retry)."""
  1287. model, provider = _validate_model_override(model, provider)
  1288. return _set_task_override(
  1289. conn, task_id,
  1290. "UPDATE tasks SET model_override = ?, provider_override = ? WHERE id = ?", (model, provider),
  1291. "model_override_set", {"model": model, "provider": provider},
  1292. ("model_override", "provider_override"), archived_msg="cannot set model override",
  1293. )
  1294. def _set_task_override(
  1295. conn: sqlite3.Connection, task_id: str, sql: str, params: tuple, event_kind: str, payload: dict,
  1296. changed_fields: tuple[str, ...], *, archived_msg: str,
  1297. ) -> bool:
  1298. """Per-task override write: refuse archived tasks, record ``event_kind``,
  1299. then fire the task-updated observer AFTER commit (RFC #58548)."""
  1300. with write_txn(conn):
  1301. status = _task_status(conn, task_id)
  1302. if status is None:
  1303. return False
  1304. if status == "archived":
  1305. raise RuntimeError(f"{archived_msg} on archived task {task_id}")
  1306. conn.execute(sql, (*params, task_id))
  1307. _append_event(conn, task_id, event_kind, payload)
  1308. notify_task_updated(conn, task_id, changed_fields)
  1309. return True
  1310. def set_reasoning_effort(conn: sqlite3.Connection, task_id: str, effort: Optional[str]) -> bool:
  1311. """Set (empty clears; ``"none"`` pins thinking OFF) the per-task reasoning
  1312. effort. Independent of the model override so clearing one never resets the
  1313. other; applies on the NEXT dispatch, so settable while running."""
  1314. effort = normalize_reasoning_effort(effort)
  1315. return _set_task_override(
  1316. conn, task_id, "UPDATE tasks SET reasoning_effort = ? WHERE id = ?", (effort,),
  1317. "reasoning_effort_set", {"reasoning_effort": effort},
  1318. ("reasoning_effort",), archived_msg="cannot set reasoning effort",
  1319. )
  1320. # --- Links ---
  1321. def link_tasks(
  1322. conn: sqlite3.Connection,
  1323. parent_id: str,
  1324. child_id: str,
  1325. *,
  1326. expected_child_run_id: Optional[int] = None,
  1327. ) -> bool:
  1328. """Link ``parent_id -> child_id``. Returns True when the link gated a
  1329. ``ready`` child back to ``todo`` (the new parent is not yet terminal), so
  1330. callers can surface the demotion instead of a silent status flip.
  1331. A running child cannot normally be gated retroactively, so reject the edge
  1332. rather than record a dependency that did not constrain the active run. The
  1333. owning worker may link its own active run for a subsequent dependency-block
  1334. handoff by supplying its trusted ``expected_child_run_id``.
  1335. """
  1336. if parent_id == child_id:
  1337. raise ValueError("a task cannot depend on itself")
  1338. gated = False
  1339. with write_txn(conn):
  1340. missing = _missing_task_ids(conn, [parent_id, child_id])
  1341. if missing:
  1342. raise ValueError(f"unknown task(s): {', '.join(missing)}")
  1343. child = conn.execute(
  1344. "SELECT status, current_run_id FROM tasks WHERE id = ?", (child_id,),
  1345. ).fetchone()
  1346. if child["status"] == "running" and (
  1347. expected_child_run_id is None
  1348. or child["current_run_id"] != expected_child_run_id
  1349. ):
  1350. raise ValueError(f"cannot link {parent_id} -> {child_id}: child is already running")
  1351. if _would_cycle(conn, parent_id, child_id):
  1352. raise ValueError(f"linking {parent_id} -> {child_id} would create a cycle")
  1353. _link(conn, parent_id, child_id)
  1354. # If child was ready but parent is not yet terminal, demote child to todo
  1355. # (archived counts as terminal, matching _parents_satisfied/recompute_ready).
  1356. if _task_status(conn, parent_id) not in ("done", "archived"):
  1357. cur = conn.execute(
  1358. "UPDATE tasks SET status = 'todo' WHERE id = ? AND status = 'ready'",
  1359. (child_id,),
  1360. )
  1361. gated = cur.rowcount == 1
  1362. if gated:
  1363. _append_event(
  1364. conn,
  1365. child_id,
  1366. "dependency_wait",
  1367. {"reason": "parent_not_done", "demoted": True, "parent": parent_id},
  1368. )
  1369. _append_event(
  1370. conn,
  1371. child_id,
  1372. "linked",
  1373. {"parent": parent_id, "child": child_id},
  1374. )
  1375. _inherit_notify_subs(conn, child_id, (parent_id,))
  1376. return gated
  1377. def _would_cycle(conn: sqlite3.Connection, parent_id: str, child_id: str) -> bool:
  1378. """True iff ``parent_id`` is already a descendant of ``child_id``."""
  1379. seen = set()
  1380. stack = [child_id]
  1381. while stack:
  1382. node = stack.pop()
  1383. if node == parent_id:
  1384. return True
  1385. if node in seen:
  1386. continue
  1387. seen.add(node)
  1388. rows = conn.execute(
  1389. "SELECT child_id FROM task_links WHERE parent_id = ?", (node,)
  1390. ).fetchall()
  1391. stack.extend(r["child_id"] for r in rows)
  1392. return False
  1393. def unlink_tasks(conn: sqlite3.Connection, parent_id: str, child_id: str) -> bool:
  1394. with write_txn(conn):
  1395. cur = conn.execute(
  1396. "DELETE FROM task_links WHERE parent_id = ? AND child_id = ?", (parent_id, child_id),
  1397. )
  1398. removed = cur.rowcount > 0
  1399. if removed:
  1400. _append_event(conn, child_id, "unlinked", {"parent": parent_id, "child": child_id})
  1401. if removed:
  1402. # Re-gate the child now (as complete_task/unblock_task do) instead of
  1403. # leaving it in todo until the next tick.
  1404. recompute_ready(conn)
  1405. return removed
  1406. def _linked_ids(conn: sqlite3.Connection, want: str, where: str, task_id: str) -> list[str]:
  1407. rows = conn.execute(
  1408. f"SELECT {want} FROM task_links WHERE {where} = ? ORDER BY {want}", (task_id,)
  1409. ).fetchall()
  1410. return [r[want] for r in rows]
  1411. # Dependency edge removed — re-evaluate promotion eligibility for the child immediately. Matches the
  1412. # contract of complete_task and unblock_task; without this the child stays stuck in todo until the next
  1413. # dispatcher tick or a manual `hermes kanban recompute` (issue #22459).
  1414. def parent_ids(conn: sqlite3.Connection, task_id: str) -> list[str]:
  1415. return _linked_ids(conn, "parent_id", "child_id", task_id)
  1416. def child_ids(conn: sqlite3.Connection, task_id: str) -> list[str]:
  1417. return _linked_ids(conn, "child_id", "parent_id", task_id)
  1418. def task_graph_contexts(conn: sqlite3.Connection, task_ids: Iterable[str]) -> dict[str, dict]:
  1419. """Bulk-load compact direct graph state for graph-aware diagnostics."""
  1420. ordered_ids = list(dict.fromkeys(str(task_id) for task_id in task_ids if task_id))
  1421. contexts = {task_id: {"parents": [], "children": []} for task_id in ordered_ids}
  1422. if not ordered_ids:
  1423. return contexts
  1424. placeholders = ",".join("?" for _ in ordered_ids)
  1425. for bucket, own, other in (("parents", "child_id", "parent_id"), ("children", "parent_id", "child_id")):
  1426. for row in conn.execute(
  1427. f"SELECT l.{own} AS owner_id, t.id, t.title, t.status "
  1428. f"FROM task_links l JOIN tasks t ON t.id = l.{other} "
  1429. f"WHERE l.{own} IN ({placeholders}) ORDER BY l.{own}, t.id", tuple(ordered_ids),
  1430. ).fetchall():
  1431. contexts[row["owner_id"]][bucket].append(
  1432. {"id": row["id"], "title": row["title"], "status": row["status"]}
  1433. )
  1434. return contexts
  1435. def task_graph_context(conn: sqlite3.Connection, task_id: str) -> dict:
  1436. """Return compact direct parent/child state for one task."""
  1437. return task_graph_contexts(conn, [task_id])[task_id]
  1438. # --- Comments & events ---
  1439. def add_comment(conn: sqlite3.Connection, task_id: str, author: str, body: str) -> int:
  1440. if not body or not body.strip():
  1441. raise ValueError("comment body is required")
  1442. if not author or not author.strip():
  1443. raise ValueError("comment author is required")
  1444. now = int(time.time())
  1445. # ``allow_nested=True``: graph builders (kanban_swarm blackboard seeding)
  1446. # compose comment writes under one outer commit.
  1447. with write_txn(conn, allow_nested=True):
  1448. _require_task(conn, task_id)
  1449. cur = conn.execute(
  1450. "INSERT INTO task_comments (task_id, author, body, created_at) "
  1451. "VALUES (?, ?, ?, ?)", (task_id, author.strip(), body.strip(), now),
  1452. )
  1453. _append_event(conn, task_id, "commented", {"author": author, "len": len(body)})
  1454. return int(cur.lastrowid or 0)
  1455. def _require_task(conn: sqlite3.Connection, task_id: str) -> None:
  1456. if not conn.execute("SELECT 1 FROM tasks WHERE id = ?", (task_id,)).fetchone():
  1457. raise ValueError(f"unknown task {task_id}")
  1458. def _task_rows(conn: sqlite3.Connection, table: str, task_id: str, order: str) -> list[sqlite3.Row]:
  1459. return conn.execute(
  1460. f"SELECT * FROM {table} WHERE task_id = ? ORDER BY {order}", (task_id,)
  1461. ).fetchall()
  1462. def list_comments(conn: sqlite3.Connection, task_id: str) -> list[Comment]:
  1463. return [Comment.from_row(r) for r in _task_rows(conn, "task_comments", task_id, "created_at ASC")]
  1464. def list_comments_after(
  1465. conn: sqlite3.Connection, task_id: str, *, after_id: int = 0
  1466. ) -> list[Comment]:
  1467. """Comments with ``id > after_id`` — keyed on rowid, not ``created_at``, so a
  1468. same-second burst is never skipped (live worker comment bridge)."""
  1469. rows = conn.execute(
  1470. "SELECT id, task_id, author, body, created_at FROM task_comments "
  1471. "WHERE task_id = ? AND id > ? ORDER BY id ASC", (task_id, int(after_id)),
  1472. ).fetchall()
  1473. return [Comment.from_row(r) for r in rows]
  1474. # --- Attachments ---
  1475. class AttachmentTooLarge(ValueError):
  1476. """Attachment over the size cap. A ``ValueError`` so generic 400 handlers
  1477. still catch it while the tool/CLI can give a 413-style message."""
  1478. def _safe_attachment_name(raw: str) -> str:
  1479. """Client filename -> safe basename: strip directories (both separators),
  1480. control chars and leading dots (no dotfiles, no traversal); ValueError when
  1481. nothing usable remains. Only ever joined under the per-task attachments dir."""
  1482. name = (raw or "").replace("\\", "/").split("/")[-1].strip()
  1483. name = "".join(ch for ch in name if ch.isprintable() and ch not in "\x00").strip()
  1484. name = name.lstrip(".").strip()
  1485. if not name:
  1486. raise ValueError("invalid attachment filename")
  1487. return name[:200]
  1488. def _collision_free_path(dest_dir: Path, safe_name: str) -> Path:
  1489. """``foo.pdf`` -> ``foo.pdf``, ``foo (1).pdf``, ... first one that doesn't exist."""
  1490. stem, dot, ext = safe_name.partition(".")
  1491. candidate = safe_name
  1492. n = 1
  1493. while (dest_dir / candidate).exists():
  1494. candidate = f"{stem} ({n}){dot}{ext}"
  1495. n += 1
  1496. return dest_dir / candidate
  1497. def store_attachment_bytes(
  1498. conn: sqlite3.Connection, task_id: str, filename: str, data: bytes, *,
  1499. content_type: Optional[str] = None, uploaded_by: Optional[str] = None,
  1500. board: Optional[str] = None, max_bytes: Optional[int] = None,
  1501. ) -> int:
  1502. """Single attachment write path (dashboard, tools, CLI): size cap, safe
  1503. basename, collision-free blob under :func:`task_attachments_dir`, then the
  1504. metadata row. Raises :class:`AttachmentTooLarge` / ``ValueError``; a blob
  1505. whose row insert fails is removed before re-raising. Returns the new id."""
  1506. if max_bytes is None:
  1507. max_bytes = KANBAN_ATTACHMENT_MAX_BYTES
  1508. if len(data) > max_bytes:
  1509. raise AttachmentTooLarge(f"attachment exceeds {max_bytes // (1024 * 1024)} MB limit")
  1510. safe_name = _safe_attachment_name(filename)
  1511. dest_dir = task_attachments_dir(task_id, board=board)
  1512. dest_dir.mkdir(parents=True, exist_ok=True)
  1513. dest_path = _collision_free_path(dest_dir, safe_name)
  1514. dest_path.write_bytes(data)
  1515. try:
  1516. return add_attachment(
  1517. conn, task_id, filename=dest_path.name, stored_path=str(dest_path.resolve()),
  1518. content_type=content_type, size=len(data), uploaded_by=uploaded_by,
  1519. )
  1520. except Exception:
  1521. # Don't leave an orphan blob if the metadata insert fails (most
  1522. # commonly: the task id doesn't exist).
  1523. with contextlib.suppress(OSError):
  1524. dest_path.unlink(missing_ok=True)
  1525. raise
  1526. def add_attachment(
  1527. conn: sqlite3.Connection, task_id: str, *, filename: str, stored_path: str,
  1528. content_type: Optional[str] = None, size: int = 0, uploaded_by: Optional[str] = None,
  1529. ) -> int:
  1530. """Record the metadata row (+ ``attached`` event) for a blob the caller already wrote."""
  1531. if not filename or not filename.strip():
  1532. raise ValueError("attachment filename is required")
  1533. if not stored_path or not stored_path.strip():
  1534. raise ValueError("attachment stored_path is required")
  1535. now = int(time.time())
  1536. with write_txn(conn):
  1537. _require_task(conn, task_id)
  1538. cur = conn.execute(
  1539. "INSERT INTO task_attachments "
  1540. "(task_id, filename, stored_path, content_type, size, uploaded_by, created_at) "
  1541. "VALUES (?, ?, ?, ?, ?, ?, ?)",
  1542. (task_id, filename.strip(), stored_path, content_type, int(size), uploaded_by, now),
  1543. )
  1544. _append_event(
  1545. conn, task_id, "attached",
  1546. {"filename": filename.strip(), "size": int(size), "by": uploaded_by},
  1547. )
  1548. return int(cur.lastrowid or 0)
  1549. def list_attachments(conn: sqlite3.Connection, task_id: str) -> list[Attachment]:
  1550. return [Attachment.from_row(r) for r in _task_rows(conn, "task_attachments", task_id, "created_at ASC, id ASC")]
  1551. def get_attachment(conn: sqlite3.Connection, attachment_id: int) -> Optional[Attachment]:
  1552. r = conn.execute("SELECT * FROM task_attachments WHERE id = ?", (attachment_id,)).fetchone()
  1553. return None if r is None else Attachment.from_row(r)
  1554. def delete_attachment(conn: sqlite3.Connection, attachment_id: int) -> Optional[Attachment]:
  1555. """Delete the row (source of truth) and best-effort its blob; None when no row matched."""
  1556. with write_txn(conn):
  1557. att = get_attachment(conn, attachment_id)
  1558. if att is None:
  1559. return None
  1560. conn.execute("DELETE FROM task_attachments WHERE id = ?", (attachment_id,))
  1561. has_remaining_blob_reference = conn.execute(
  1562. "SELECT 1 FROM task_attachments WHERE stored_path = ? LIMIT 1",
  1563. (att.stored_path,),
  1564. ).fetchone() is not None
  1565. _append_event(conn, att.task_id, "attachment_removed", {"filename": att.filename})
  1566. if not has_remaining_blob_reference:
  1567. with contextlib.suppress(OSError):
  1568. p = Path(att.stored_path)
  1569. if p.is_file():
  1570. p.unlink()
  1571. return att
  1572. def list_events(conn: sqlite3.Connection, task_id: str) -> list[Event]:
  1573. return [Event.from_row(r) for r in _task_rows(conn, "task_events", task_id, "created_at ASC, id ASC")]
  1574. def _insert_comment(
  1575. conn: sqlite3.Connection, task_id: str, author: str, body: str, created_at: int,
  1576. ) -> None:
  1577. """Raw comment INSERT for callers already inside a write txn (``add_comment``
  1578. opens its own txn and emits ``commented``)."""
  1579. conn.execute(
  1580. "INSERT INTO task_comments (task_id, author, body, created_at) "
  1581. "VALUES (?, ?, ?, ?)", (task_id, author, body, created_at),
  1582. )
  1583. def _append_event(
  1584. conn: sqlite3.Connection, task_id: str, kind: str, payload: Optional[dict] = None, *,
  1585. run_id: Optional[int] = None,
  1586. ) -> None:
  1587. """Insert an event row inside the caller's txn; ``run_id`` groups it by attempt (NULL = task-scoped)."""
  1588. conn.execute(
  1589. "INSERT INTO task_events (task_id, run_id, kind, payload, created_at) "
  1590. "VALUES (?, ?, ?, ?, ?)", (task_id, run_id, kind, _json_or_null(payload), int(time.time())),
  1591. )
  1592. def _end_run(
  1593. conn: sqlite3.Connection, task_id: str, *, outcome: str, summary: Optional[str] = None,
  1594. error: Optional[str] = None, metadata: Optional[dict] = None, status: Optional[str] = None,
  1595. ) -> Optional[int]:
  1596. """Close the active run (``status`` defaults to ``outcome``) and clear
  1597. ``current_run_id``; None when no run was active (never-claimed task).
  1598. ``worker_pid`` / ``worker_started_at`` / ``claim_lock`` stay on the closed
  1599. row: they are the only evidence left of the OS process once the task row
  1600. is wiped, and :func:`kanban_db_dispatch.reap_terminal_workers` needs them
  1601. to end a worker that survived its own terminal transition."""
  1602. now = int(time.time())
  1603. run_id = _current_run_id(conn, task_id)
  1604. if run_id is None:
  1605. return None
  1606. conn.execute(
  1607. """
  1608. UPDATE task_runs
  1609. SET status = ?,
  1610. outcome = ?,
  1611. summary = ?,
  1612. error = ?,
  1613. metadata = ?,
  1614. ended_at = ?,
  1615. claim_expires = NULL
  1616. WHERE id = ?
  1617. AND ended_at IS NULL
  1618. """,
  1619. (status or outcome, outcome, summary, error, _json_or_null(metadata), now, run_id),
  1620. )
  1621. conn.execute("UPDATE tasks SET current_run_id = NULL WHERE id = ?", (task_id,))
  1622. return run_id
  1623. def _first_line(text: Optional[str], limit: int) -> str:
  1624. """First non-blank-stripped line of ``text`` capped at ``limit`` chars; "" when empty."""
  1625. lines = (text or "").strip().splitlines()
  1626. return lines[0][:limit] if lines else ""
  1627. def _opt_int(value: Any) -> Optional[int]:
  1628. """``int(value)`` or ``None`` when ``value`` is ``None`` (NULL column passthrough)."""
  1629. return int(value) if value is not None else None
  1630. def _json_or_null(obj: Any) -> Optional[str]:
  1631. """JSON text for a payload/metadata column; falsy -> NULL."""
  1632. return json.dumps(obj, ensure_ascii=False) if obj else None
  1633. def _task_status(conn: sqlite3.Connection, task_id: str) -> Optional[str]:
  1634. """Current ``tasks.status`` for ``task_id``, or ``None`` when no such row."""
  1635. row = conn.execute("SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone()
  1636. return row["status"] if row else None
  1637. def _current_run_id(conn: sqlite3.Connection, task_id: str) -> Optional[int]:
  1638. row = conn.execute("SELECT current_run_id FROM tasks WHERE id = ?", (task_id,)).fetchone()
  1639. return int(row["current_run_id"]) if row and row["current_run_id"] else None
  1640. # Distinguishes "caller named the acting profile" (which may legitimately be
  1641. # None for an unassigned card) from "read the card's current assignee".
  1642. _UNSET: Any = object()
  1643. def _end_or_synthesize_run(
  1644. conn: sqlite3.Connection, task_id: str, *, outcome: str, status: str,
  1645. summary: Optional[str] = None, metadata: Optional[dict] = None, synthesize: bool,
  1646. profile: Any = _UNSET,
  1647. ) -> Optional[int]:
  1648. """:func:`_end_run`; when no run was active and ``synthesize`` holds, record a
  1649. zero-duration run instead so the handoff fields survive in attempt history.
  1650. ``profile`` overrides the profile read off the task row for the synthesized
  1651. run — transitions that reassign the task (e.g. review handoff) pass the
  1652. acting profile captured before the rewrite."""
  1653. run_id = _end_run(conn, task_id, outcome=outcome, status=status, summary=summary, metadata=metadata)
  1654. if run_id is None and synthesize:
  1655. run_id = _synthesize_ended_run(conn, task_id, outcome=outcome, summary=summary, metadata=metadata, profile=profile)
  1656. return run_id
  1657. def _synthesize_ended_run(
  1658. conn: sqlite3.Connection, task_id: str, *, outcome: str, summary: Optional[str] = None,
  1659. error: Optional[str] = None, metadata: Optional[dict] = None,
  1660. profile: Any = _UNSET,
  1661. ) -> int:
  1662. """Zero-duration closed run for a terminal transition on a never-claimed
  1663. task, so the handoff fields aren't silently dropped (``_end_run`` is a
  1664. no-op then). ``started_at == ended_at`` keeps elapsed stats honest. Does
  1665. NOT touch the tasks row.
  1666. ``profile`` overrides the profile read off the task row: transitions that
  1667. reassign the task (e.g. review handoff) pass the acting profile captured
  1668. before the rewrite, so the run names the actor, not the new assignee."""
  1669. now = int(time.time())
  1670. trow = conn.execute(
  1671. "SELECT assignee, current_step_key FROM tasks WHERE id = ?", (task_id,),
  1672. ).fetchone()
  1673. if profile is _UNSET:
  1674. profile = trow["assignee"] if trow else None
  1675. step_key = trow["current_step_key"] if trow else None
  1676. cur = conn.execute(
  1677. """
  1678. INSERT INTO task_runs (
  1679. task_id, profile, step_key,
  1680. status, outcome,
  1681. summary, error, metadata,
  1682. started_at, ended_at
  1683. ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  1684. """,
  1685. (
  1686. task_id, profile, step_key, outcome, outcome, summary, error, _json_or_null(metadata),
  1687. now, now,
  1688. ),
  1689. )
  1690. return int(cur.lastrowid or 0)
  1691. # --- Dependency resolution (todo -> ready) ---
  1692. def _has_sticky_block(conn: sqlite3.Connection, task_id: str) -> bool:
  1693. """True when the newest ``blocked``/``unblocked``/``gave_up`` event says the
  1694. block must wait for an operator: an explicit ``kanban_block`` (#28712), or a
  1695. breaker trip ``_record_task_failure`` stamped ``sticky`` — the clean-exit
  1696. protocol-violation budget or a systemic same-error wave. Those trip on a
  1697. policy independent of ``consecutive_failures``, so ``recompute_ready``'s
  1698. counter check cannot see them — without this the trip is promoted back to
  1699. ``ready`` in the same tick and the card respawns forever. A plain
  1700. (unified-budget) ``gave_up`` carries no marker and is judged by the counter,
  1701. so raising ``failure_limit`` or ``assign_task`` to a fresh profile still
  1702. releases it; a task with no such event at all (direct DB edit) auto-recovers.
  1703. """
  1704. row = conn.execute(
  1705. "SELECT kind FROM task_events "
  1706. "WHERE task_id = ? AND kind IN ('blocked', 'unblocked') "
  1707. "ORDER BY id DESC LIMIT 1", (task_id,),
  1708. ).fetchone()
  1709. if row and row["kind"] == "blocked":
  1710. return True
  1711. trip = conn.execute(
  1712. "SELECT payload FROM task_events "
  1713. "WHERE task_id = ? AND kind = 'gave_up' AND id > COALESCE("
  1714. " (SELECT MAX(id) FROM task_events WHERE task_id = ? AND kind = 'unblocked'), 0) "
  1715. "ORDER BY id DESC LIMIT 1", (task_id, task_id),
  1716. ).fetchone()
  1717. return bool(trip) and bool(_json_dict(trip["payload"]).get("sticky"))
  1718. def _latest_event(
  1719. conn: sqlite3.Connection, task_id: str, kind: str, run_id: Optional[int] = None,
  1720. ) -> Optional[sqlite3.Row]:
  1721. """Newest ``task_events`` row of ``kind`` (optionally scoped to one run)."""
  1722. sql = "SELECT payload FROM task_events WHERE task_id = ? AND kind = ?"
  1723. params: tuple[Any, ...] = (task_id, kind)
  1724. if run_id is not None:
  1725. sql += " AND run_id = ?"
  1726. params = (*params, int(run_id))
  1727. return conn.execute(sql + " ORDER BY id DESC LIMIT 1", params).fetchone()
  1728. def _resume_status_from_events(conn: sqlite3.Connection, task_id: str) -> str:
  1729. """``review`` when the newest lifecycle event carries a review
  1730. ``resume_status``/``retry_status``/``source_status``, else ``ready`` (legacy)."""
  1731. row = conn.execute(
  1732. "SELECT payload FROM task_events "
  1733. "WHERE task_id = ? AND kind IN ("
  1734. "'blocked', 'block_loop_detected', 'dependency_wait', 'gave_up', "
  1735. "'unblocked', 'changes_requested', 'review_reopened', 'status', 'reclaimed', "
  1736. "'stale', 'timed_out', 'crashed', 'spawn_failed', 'rate_limited'"
  1737. ") ORDER BY id DESC LIMIT 1", (task_id,),
  1738. ).fetchone()
  1739. payload = _json_dict(_row_get(row, "payload"))
  1740. for key in ("resume_status", "retry_status", "source_status"):
  1741. if payload.get(key) == "review":
  1742. return "review"
  1743. return "ready"
  1744. def recompute_ready(conn: sqlite3.Connection, failure_limit: int | None = None) -> int:
  1745. """Promote ``todo``/``blocked`` tasks whose parents are all done/archived;
  1746. returns the count. Opens its own IMMEDIATE txn — call OUTSIDE any write txn.
  1747. ``blocked`` is skipped when sticky (explicit ``kanban_block``) or when
  1748. ``consecutive_failures`` reached the limit (else the breaker could never
  1749. trip). Limit order matches ``_record_task_failure``: ``max_retries`` >
  1750. ``failure_limit`` > ``DEFAULT_FAILURE_LIMIT``.
  1751. 1. The most recent block event was a worker-initiated ``kanban_block`` — those stay blocked until an
  1752. explicit ``kanban_unblock`` (#28712).
  1753. """
  1754. if failure_limit is None:
  1755. failure_limit = DEFAULT_FAILURE_LIMIT
  1756. promoted = 0
  1757. with write_txn(conn):
  1758. todo_rows = conn.execute(
  1759. "SELECT id, status, consecutive_failures, max_retries "
  1760. "FROM tasks WHERE status IN ('todo', 'blocked')"
  1761. ).fetchall()
  1762. for row in todo_rows:
  1763. task_id = row["id"]
  1764. cur_status = row["status"]
  1765. if cur_status == "blocked" and _has_sticky_block(conn, task_id):
  1766. # Explicit human-intervention block; only ``unblock_task`` may exit it.
  1767. continue
  1768. parents = conn.execute(
  1769. "SELECT t.status FROM tasks t "
  1770. "JOIN task_links l ON l.parent_id = t.id "
  1771. "WHERE l.child_id = ?", (task_id,),
  1772. ).fetchall()
  1773. if all(p["status"] in ("done", "archived") for p in parents):
  1774. resume_status = _resume_status_from_events(conn, task_id)
  1775. if cur_status == "blocked":
  1776. # At the breaker limit, no auto-recovery (else block ->
  1777. # recover -> respawn -> exhaust -> block forever). The
  1778. # counter is preserved so it accumulates across cycles.
  1779. failures = int(row["consecutive_failures"] or 0)
  1780. task_limit = row["max_retries"]
  1781. effective_limit = (
  1782. int(task_limit) if task_limit is not None
  1783. else int(failure_limit)
  1784. )
  1785. if failures >= effective_limit:
  1786. continue
  1787. conn.execute(
  1788. "UPDATE tasks SET status = ? "
  1789. "WHERE id = ? AND status = 'blocked'", (resume_status, task_id),
  1790. )
  1791. else:
  1792. conn.execute(
  1793. "UPDATE tasks SET status = ? WHERE id = ? AND status = 'todo'",
  1794. (resume_status, task_id),
  1795. )
  1796. _append_event(
  1797. conn, task_id, "promoted",
  1798. {"status": resume_status} if resume_status != "ready" else None,
  1799. )
  1800. promoted += 1
  1801. return promoted
  1802. # --- Claim / complete / block ---
  1803. def _parents_satisfied(conn: sqlite3.Connection, task_id: str) -> bool:
  1804. """Return whether every direct parent is terminal for dependency gating."""
  1805. return conn.execute(
  1806. "SELECT 1 FROM task_links l "
  1807. "JOIN tasks p ON p.id = l.parent_id "
  1808. "WHERE l.child_id = ? "
  1809. "AND p.status NOT IN ('done', 'archived') LIMIT 1", (task_id,),
  1810. ).fetchone() is None
  1811. def unsatisfied_parents(conn: sqlite3.Connection, task_id: str) -> list[tuple[str, str]]:
  1812. """``(parent_id, status)`` for every direct parent :func:`_parents_satisfied`
  1813. still counts as open (``done`` / ``archived`` release the child), in id
  1814. order, so a refusal or a board view can name the blockers instead of the
  1815. caller guessing. Read-only."""
  1816. rows = conn.execute(
  1817. "SELECT p.id, p.status FROM task_links l "
  1818. "JOIN tasks p ON p.id = l.parent_id "
  1819. "WHERE l.child_id = ? AND p.status NOT IN ('done', 'archived') "
  1820. "ORDER BY p.id", (task_id,),
  1821. ).fetchall()
  1822. return [(row["id"], row["status"]) for row in rows]
  1823. def _claim_and_open_run(
  1824. conn: sqlite3.Connection, task_id: str, source_status: str, lock: str, expires: int, now: int,
  1825. *, event_extra: Optional[dict] = None,
  1826. ) -> Optional[int]:
  1827. """CAS ``source_status -> running``, open a run row, emit ``claimed``; None
  1828. when the CAS lost. Caller holds the txn."""
  1829. cur = conn.execute(
  1830. f"""
  1831. UPDATE tasks
  1832. SET status = 'running',
  1833. claim_lock = ?,
  1834. claim_expires = ?,
  1835. started_at = COALESCE(started_at, ?)
  1836. WHERE id = ?
  1837. AND status = '{source_status}'
  1838. AND claim_lock IS NULL
  1839. """,
  1840. (lock, expires, now, task_id),
  1841. )
  1842. if cur.rowcount != 1:
  1843. return None
  1844. trow = conn.execute(
  1845. "SELECT assignee, max_runtime_seconds, current_step_key "
  1846. "FROM tasks WHERE id = ?", (task_id,),
  1847. ).fetchone()
  1848. run_cur = conn.execute(
  1849. """
  1850. INSERT INTO task_runs (
  1851. task_id, profile, step_key, status,
  1852. claim_lock, claim_expires, max_runtime_seconds,
  1853. started_at
  1854. ) VALUES (?, ?, ?, 'running', ?, ?, ?, ?)
  1855. """,
  1856. (
  1857. task_id, trow["assignee"] if trow else None, trow["current_step_key"] if trow else None,
  1858. lock, expires, trow["max_runtime_seconds"] if trow else None, now,
  1859. ),
  1860. )
  1861. run_id = run_cur.lastrowid
  1862. conn.execute("UPDATE tasks SET current_run_id = ? WHERE id = ?", (run_id, task_id))
  1863. _append_event(
  1864. conn, task_id, "claimed",
  1865. {"lock": lock, "expires": expires, "run_id": run_id, **(event_extra or {})}, run_id=run_id,
  1866. )
  1867. return run_id
  1868. def claim_task(
  1869. conn: sqlite3.Connection, task_id: str, *, ttl_seconds: Optional[int] = None,
  1870. claimer: Optional[str] = None,
  1871. ) -> Optional[Task]:
  1872. """Atomically transition ``ready -> running``.
  1873. Returns the claimed ``Task`` on success, ``None`` if the task was
  1874. already claimed (or is not in ``ready`` status).
  1875. """
  1876. now = int(time.time())
  1877. lock = claimer or _claimer_id()
  1878. expires = now + _resolve_claim_ttl_seconds(ttl_seconds)
  1879. with write_txn(conn):
  1880. # Single enforcement point: never ready -> running with an undone
  1881. # parent, whichever writer set 'ready'. Demote to 'todo';
  1882. # recompute_ready re-promotes when the parents finish.
  1883. if not _parents_satisfied(conn, task_id):
  1884. conn.execute(
  1885. "UPDATE tasks SET status = 'todo' "
  1886. "WHERE id = ? AND status = 'ready'", (task_id,),
  1887. )
  1888. _append_event(conn, task_id, "claim_rejected", {"reason": "parents_not_done"})
  1889. return None
  1890. # Close a leaked prior run so the CAS below doesn't strand it.
  1891. _reclaim_dangling_run(
  1892. conn, task_id, statuses=("ready",), now=now, note="invariant recovery on re-claim",
  1893. )
  1894. run_id = _claim_and_open_run(conn, task_id, "ready", lock, expires, now)
  1895. if run_id is None:
  1896. return None
  1897. claimed = get_task(conn, task_id)
  1898. _fire_task_hook("kanban_task_claimed", claimed, task_id, run_id)
  1899. return claimed
  1900. def claim_review_task(
  1901. conn: sqlite3.Connection, task_id: str, *, ttl_seconds: Optional[int] = None,
  1902. claimer: Optional[str] = None,
  1903. ) -> Optional[Task]:
  1904. """Atomic ``review -> running`` (None when lost). Parents are re-checked
  1905. (one may have reopened meanwhile) and a NEW run tracks the reviewer
  1906. separately from the implementer."""
  1907. now = int(time.time())
  1908. lock = claimer or _claimer_id()
  1909. expires = now + _resolve_claim_ttl_seconds(ttl_seconds)
  1910. with write_txn(conn):
  1911. if not _parents_satisfied(conn, task_id):
  1912. demoted = conn.execute(
  1913. "UPDATE tasks SET status = 'todo' "
  1914. "WHERE id = ? AND status = 'review' AND claim_lock IS NULL", (task_id,),
  1915. )
  1916. if demoted.rowcount == 1:
  1917. _append_event(
  1918. conn, task_id, "dependency_wait",
  1919. {"reason": "parent_reopened", "source_status": "review"},
  1920. )
  1921. return None
  1922. run_id = _claim_and_open_run(
  1923. conn, task_id, "review", lock, expires, now, event_extra={"source_status": "review"},
  1924. )
  1925. if run_id is None:
  1926. return None
  1927. return get_task(conn, task_id)
  1928. def _retry_status_for_run(
  1929. conn: sqlite3.Connection, task_id: str, run_id: Optional[int] = None,
  1930. ) -> str:
  1931. """``review`` when the run's ``claimed`` event says ``source_status=review``,
  1932. else ``ready`` — one place, so crash/timeout/reclaim can't silently turn a
  1933. reviewer run into an implementation run."""
  1934. if run_id is None:
  1935. run_id = _current_run_id(conn, task_id)
  1936. if run_id is None:
  1937. return "ready"
  1938. event = _latest_event(conn, task_id, "claimed", run_id)
  1939. payload = _json_dict(_row_get(event, "payload"))
  1940. return "review" if payload.get("source_status") == "review" else "ready"
  1941. # Run outcome -> lifecycle status a goal loop should report for a handed-off run.
  1942. _RUN_OUTCOME_TERMINAL_STATUS = {
  1943. "completed": "done",
  1944. "review_requested": "review",
  1945. "changes_requested": "changes_requested",
  1946. "blocked": "blocked",
  1947. "dependency_wait": "blocked",
  1948. }
  1949. def goal_run_status(
  1950. conn: sqlite3.Connection, task_id: str, expected_run_id: Optional[int] = None,
  1951. ) -> Optional[str]:
  1952. """Lifecycle status as seen by ONE run: terminal handoffs bind to that run,
  1953. any other ownership loss is ``superseded`` — otherwise an old goal loop
  1954. would read the successor's live ``running`` and mutate it."""
  1955. task = get_task(conn, task_id)
  1956. if task is None:
  1957. return None
  1958. if expected_run_id is not None:
  1959. row = conn.execute(
  1960. "SELECT outcome FROM task_runs WHERE id = ? AND task_id = ?",
  1961. (int(expected_run_id), task_id),
  1962. ).fetchone()
  1963. outcome = str(row["outcome"]) if row and row["outcome"] is not None else None
  1964. terminal_status = _RUN_OUTCOME_TERMINAL_STATUS.get(outcome)
  1965. if terminal_status is not None:
  1966. return terminal_status
  1967. if outcome is not None or task.current_run_id != int(expected_run_id):
  1968. return "superseded"
  1969. if task.status in {"ready", "todo"}:
  1970. event = conn.execute(
  1971. "SELECT kind FROM task_events WHERE task_id = ? "
  1972. "ORDER BY id DESC LIMIT 1", (task_id,),
  1973. ).fetchone()
  1974. if event and event["kind"] == "changes_requested":
  1975. return "changes_requested"
  1976. return task.status
  1977. def heartbeat_claim(
  1978. conn: sqlite3.Connection, task_id: str, *, ttl_seconds: Optional[int] = None,
  1979. claimer: Optional[str] = None,
  1980. ) -> bool:
  1981. """Extend a running claim; True if we still own it."""
  1982. expires = int(time.time()) + _resolve_claim_ttl_seconds(ttl_seconds)
  1983. lock = claimer or _claimer_id()
  1984. with write_txn(conn):
  1985. cur = conn.execute(
  1986. "UPDATE tasks SET claim_expires = ? "
  1987. "WHERE id = ? AND status = 'running' AND claim_lock = ?", (expires, task_id, lock),
  1988. )
  1989. if cur.rowcount != 1:
  1990. return False
  1991. _extend_run_claim(conn, task_id, expires)
  1992. return True
  1993. def _extend_run_claim(conn: sqlite3.Connection, task_id: str, expires: int) -> Optional[int]:
  1994. """Mirror a task claim extension onto its active run row; returns that run id."""
  1995. run_id = _current_run_id(conn, task_id)
  1996. if run_id is not None:
  1997. conn.execute("UPDATE task_runs SET claim_expires = ? WHERE id = ?", (expires, run_id))
  1998. return run_id
  1999. def release_stale_claims(
  2000. conn: sqlite3.Connection, *, signal_fn=None, failure_limit: Optional[int] = None,
  2001. ) -> int:
  2002. """Reclaim ``running`` tasks whose claim expired; returns the count reclaimed.
  2003. Every reclaim that actually releases a claim is a non-success attempt and
  2004. is booked through ``_record_task_failure`` (#111306): a claim that expired
  2005. without a worker ever spawning otherwise loops claim -> reclaim -> claim
  2006. with ``consecutive_failures`` stuck at 0, so the breaker never trips.
  2007. ``reclaim_task`` (operator path) deliberately resets the counter instead.
  2008. A host-local worker that is still alive gets its claim *extended* instead
  2009. (a slow model can sit longer than the TTL inside one tool-free call, so no
  2010. heartbeat) — unless ``last_heartbeat_at`` is older than
  2011. ``DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS`` (wedged; ``_touch_activity``
  2012. keeps any genuinely active worker fresh). Safe to call often.
  2013. Reclaiming a live worker mid-flight produces the spawn- then-immediately-reclaim loop seen on slow
  2014. models that spend longer than ``DEFAULT_CLAIM_TTL_SECONDS`` inside a single tool-free LLM call (#23025):
  2015. no tool calls means no ``kanban_heartbeat``, even though the subprocess is healthy.
  2016. Backstop (#29747 gap 3): if the worker's PID is still alive but its ``last_heartbeat_at`` is stale by
  2017. more than ``DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS`` (1h), the worker has been making no observable
  2018. progress and we reclaim anyway — even if ``_pid_alive`` is still true. This catches the
  2019. wedged-in-a-logic-loop case where the process is technically running but accomplishing nothing.
  2020. ``_touch_activity`` (run_agent.py) bridges chunk-level liveness into ``last_heartbeat_at`` via #31752,
  2021. so any genuinely active worker keeps its heartbeat fresh as a side effect of normal API traffic.
  2022. ``enforce_max_runtime`` and ``detect_crashed_workers`` remain the upper bounds for genuinely wedged or
  2023. dead workers.
  2024. """
  2025. now = int(time.time())
  2026. reclaimed = 0
  2027. host_prefix = _host_prefix()
  2028. stale = conn.execute(
  2029. "SELECT id, claim_lock, worker_pid, worker_started_at, claim_expires, last_heartbeat_at, "
  2030. " assignee "
  2031. "FROM tasks "
  2032. "WHERE status = 'running' AND claim_expires IS NOT NULL "
  2033. " AND claim_expires < ?", (now,),
  2034. ).fetchall()
  2035. from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked, owner_reclaim_blocked_after_kill, reclaim_error
  2036. for row in stale:
  2037. if owner_reclaim_blocked(conn, row["id"]):
  2038. continue
  2039. host_local = (row["claim_lock"] or "").startswith(host_prefix)
  2040. hb = _to_epoch(row["last_heartbeat_at"])
  2041. # Backstop: a heartbeat older than the max-stale threshold means no
  2042. # observable progress — reclaim even if the PID is alive (logic loop).
  2043. heartbeat_stale = hb is not None and (now - hb) > DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS
  2044. started_at = _row_get(row, "worker_started_at")
  2045. if (host_local and row["worker_pid"] and _worker_alive(row["worker_pid"], started_at)
  2046. and not heartbeat_stale):
  2047. _extend_live_stale_claim(conn, row, now)
  2048. continue
  2049. termination = _terminate_reclaimed_worker(
  2050. row["worker_pid"], row["claim_lock"], signal_fn=signal_fn, started_at=started_at,
  2051. )
  2052. # A live worker of ours must keep its claim (else a duplicate spawns beside it).
  2053. if _worker_survived_termination(termination):
  2054. _defer_reclaim_for_live_worker(
  2055. conn, row["id"], row["claim_lock"], now, termination,
  2056. reason="ttl_expired_worker_alive",
  2057. )
  2058. continue
  2059. with write_txn(conn):
  2060. # Again under the write lock: an interpreter that bound since the check above
  2061. # changes neither claim_lock nor worker_pid, so the CAS below cannot see it.
  2062. if owner_reclaim_blocked_after_kill(conn, row["id"], signalled=bool(termination.get("termination_attempted"))):
  2063. continue
  2064. retry_status, error = _retry_status_for_run(conn, row["id"]), reclaim_error(conn, row["id"], row["claim_lock"])
  2065. cur = conn.execute(
  2066. "UPDATE tasks SET status = ?, claim_lock = NULL, "
  2067. "claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL "
  2068. "WHERE id = ? AND status = 'running' AND claim_lock IS ? "
  2069. "AND claim_expires IS NOT NULL AND claim_expires < ? "
  2070. # A worker that registered its own pid since the SELECT keeps its claim.
  2071. "AND worker_pid IS ?",
  2072. (retry_status, row["id"], row["claim_lock"], now, row["worker_pid"]),
  2073. )
  2074. if cur.rowcount != 1:
  2075. continue
  2076. run_id = _record_reclaim(
  2077. conn, row["id"], termination,
  2078. error=error,
  2079. payload={
  2080. "stale_lock": row["claim_lock"],
  2081. "worker_pid": _opt_int(row["worker_pid"]),
  2082. "claim_expires": _to_epoch(row["claim_expires"]),
  2083. "last_heartbeat_at": hb,
  2084. "now": now,
  2085. "host_local": host_local,
  2086. "heartbeat_stale": bool(heartbeat_stale),
  2087. "retry_status": retry_status,
  2088. },
  2089. )
  2090. reclaimed += 1
  2091. # Own txn, after the reclaim commit (same shape as ``enforce_max_runtime``):
  2092. # the run ended without a verdict, so it counts toward the breaker and a
  2093. # trip flips the task to ``blocked`` + ``gave_up`` on top of ``reclaimed``.
  2094. _record_task_failure(
  2095. conn, row["id"], error,
  2096. outcome="reclaimed", failure_limit=failure_limit,
  2097. release_claim=False, end_run=False,
  2098. event_payload_extra={"worker_pid": _opt_int(row["worker_pid"]), "retry_status": retry_status},
  2099. )
  2100. # Post-commit observer; every non-reclaim branch ``continue``d above.
  2101. if _kanban_observer_consumed("on_kanban_worker_stale_claim"):
  2102. _fire_kanban_lifecycle_hook(
  2103. "on_kanban_worker_stale_claim", row["id"], board=get_current_board(),
  2104. assignee=row["assignee"], run_id=run_id, worker_pid=_opt_int(row["worker_pid"]),
  2105. heartbeat_stale=bool(heartbeat_stale), retry_status=retry_status,
  2106. )
  2107. return reclaimed
  2108. def _record_reclaim(
  2109. conn: sqlite3.Connection, task_id: str, termination: dict, *, error: str, payload: dict,
  2110. ) -> Optional[int]:
  2111. """Close the active run as ``reclaimed`` and emit the ``reclaimed`` event
  2112. (payload merged with the termination report). Caller holds the txn."""
  2113. run_id = _end_run(
  2114. conn, task_id, outcome="reclaimed", status="reclaimed", error=error, metadata=termination,
  2115. )
  2116. payload.update(termination)
  2117. _append_event(conn, task_id, "reclaimed", payload, run_id=run_id)
  2118. return run_id
  2119. def _extend_live_stale_claim(conn: sqlite3.Connection, row: sqlite3.Row, now: int) -> None:
  2120. """TTL-expired claim whose host-local worker is alive: extend instead of
  2121. reclaiming (``claim_extended`` event). CAS on the same expired lock so a
  2122. concurrent reclaimer wins cleanly."""
  2123. new_expires = now + _resolve_claim_ttl_seconds()
  2124. with write_txn(conn):
  2125. cur = conn.execute(
  2126. "UPDATE tasks SET claim_expires = ? "
  2127. "WHERE id = ? AND status = 'running' "
  2128. " AND claim_lock IS ? "
  2129. " AND claim_expires IS NOT NULL "
  2130. " AND claim_expires < ?", (new_expires, row["id"], row["claim_lock"], now),
  2131. )
  2132. if cur.rowcount != 1:
  2133. return
  2134. run_id = _extend_run_claim(conn, row["id"], new_expires)
  2135. _append_event(
  2136. conn, row["id"], "claim_extended",
  2137. {
  2138. "reason": "pid_alive",
  2139. "worker_pid": int(row["worker_pid"]),
  2140. "claim_lock": row["claim_lock"],
  2141. "claim_expires_was": _to_epoch(row["claim_expires"]),
  2142. "claim_expires_now": new_expires,
  2143. "last_heartbeat_at": _to_epoch(row["last_heartbeat_at"]),
  2144. },
  2145. run_id=run_id,
  2146. )
  2147. def reclaim_task(
  2148. conn: sqlite3.Connection, task_id: str, *, reason: Optional[str] = None, signal_fn=None,
  2149. ) -> bool:
  2150. """Operator reclaim regardless of TTL: release the claim, restore the source
  2151. phase, reset the failure counter. False when not running."""
  2152. row = conn.execute(
  2153. "SELECT status, claim_lock, worker_pid, worker_started_at FROM tasks WHERE id = ?", (task_id,),
  2154. ).fetchone()
  2155. if not row:
  2156. return False
  2157. if row["status"] != "running" and row["claim_lock"] is None:
  2158. # Nothing to reclaim — already ready / blocked / done.
  2159. return False
  2160. prev_lock = row["claim_lock"]
  2161. termination = _terminate_reclaimed_worker(
  2162. row["worker_pid"], prev_lock, signal_fn=signal_fn, started_at=row["worker_started_at"])
  2163. with write_txn(conn):
  2164. retry_status = _retry_status_for_run(conn, task_id)
  2165. cur = conn.execute(
  2166. "UPDATE tasks SET status = ?, claim_lock = NULL, "
  2167. "claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL "
  2168. "WHERE id = ? AND status IN ('running', 'ready', 'blocked') "
  2169. "AND claim_lock IS ?", (retry_status, task_id, prev_lock),
  2170. )
  2171. if cur.rowcount != 1:
  2172. return False
  2173. _record_reclaim(
  2174. conn, task_id, termination,
  2175. error=f"manual_reclaim: {reason}" if reason else f"manual_reclaim lock={prev_lock}",
  2176. payload={"manual": True, "reason": reason, "prev_lock": prev_lock, "retry_status": retry_status},
  2177. )
  2178. # Operator intervention = fresh retry budget (own txn, runs after commit).
  2179. _clear_failure_counter(conn, task_id)
  2180. return True
  2181. def reassign_task(
  2182. conn: sqlite3.Connection, task_id: str, profile: Optional[str], *, reclaim_first: bool = False,
  2183. reason: Optional[str] = None,
  2184. ) -> bool:
  2185. """Reassign (None unassigns); a running task is refused unless
  2186. ``reclaim_first`` releases its claim — the "this profile's model is broken" path."""
  2187. if reclaim_first:
  2188. # Safe to call even if nothing to reclaim.
  2189. reclaim_task(conn, task_id, reason=reason or "reassign")
  2190. # assign_task handles its own txn + the still-running guard.
  2191. try:
  2192. return assign_task(conn, task_id, profile)
  2193. except RuntimeError:
  2194. # Task is still running and reclaim_first was False; caller
  2195. # needs to decide whether to retry with reclaim.
  2196. return False
  2197. def _verify_created_cards(
  2198. conn: sqlite3.Connection, completing_task_id: str, claimed_ids: Iterable[str],
  2199. ) -> tuple[list[str], list[str]]:
  2200. """Partition ``claimed_ids`` into (verified, phantom). Verified = the row
  2201. exists AND ``created_by`` is the completing task's assignee or id, OR the
  2202. card is linked as its child (created elsewhere, attached by the worker).
  2203. Never mutates."""
  2204. ordered = list(dict.fromkeys(str(x).strip() for x in (claimed_ids or []) if str(x).strip()))
  2205. if not ordered:
  2206. return [], []
  2207. row = conn.execute("SELECT assignee FROM tasks WHERE id = ?", (completing_task_id,)).fetchone()
  2208. if row is None:
  2209. # Completing task not found — nothing resolves.
  2210. return [], ordered
  2211. completing_assignee = row["assignee"]
  2212. # Batch-fetch existence + created_by in one query.
  2213. placeholders = ",".join(["?"] * len(ordered))
  2214. rows = conn.execute(
  2215. f"SELECT id, created_by FROM tasks WHERE id IN ({placeholders})", tuple(ordered),
  2216. ).fetchall()
  2217. found = {r["id"]: r["created_by"] for r in rows}
  2218. # Pull the set of cards linked as children of the completing task.
  2219. # Cheap: one query, indexed on parent_id.
  2220. linked_children: set[str] = set(child_ids(conn, completing_task_id))
  2221. verified: list[str] = []
  2222. phantom: list[str] = []
  2223. for cid in ordered:
  2224. created_by = found.get(cid)
  2225. trusted = created_by is not None and (
  2226. (completing_assignee and created_by == completing_assignee)
  2227. or created_by == completing_task_id
  2228. or cid in linked_children
  2229. )
  2230. (verified if trusted else phantom).append(cid)
  2231. return verified, phantom
  2232. # Matches ``kanban_create`` (12 hex) and ``_new_task_id`` (8 hex) ids; 8+ for forward compat.
  2233. _TASK_ID_PROSE_RE = re.compile(r"\bt_[a-f0-9]{8,}\b")
  2234. def _scan_prose_for_phantom_ids(conn: sqlite3.Connection, text: str) -> list[str]:
  2235. """``t_<hex>`` references in ``text`` that don't resolve to a task (deduped; advisory)."""
  2236. if not text:
  2237. return []
  2238. return _missing_task_ids(conn, dict.fromkeys(_TASK_ID_PROSE_RE.findall(text)))
  2239. class HallucinatedCardsError(ValueError):
  2240. """``complete_task`` refused: ``created_cards`` has ids that don't exist or
  2241. weren't created by this worker (``.phantom``). A ``ValueError`` so tool
  2242. error handlers treat it as recoverable."""
  2243. def __init__(self, phantom: list[str], completing_task_id: str):
  2244. self.phantom = list(phantom)
  2245. self.completing_task_id = completing_task_id
  2246. super().__init__(
  2247. f"completion blocked: claimed created_cards that do not exist "
  2248. f"or were not created by this worker: {', '.join(phantom)}"
  2249. )
  2250. class EmptyCompletionError(ValueError):
  2251. """``complete_task`` refused: no substantive ``result``, ``summary``, or
  2252. stored result. A ``ValueError`` so tool error handlers treat it as
  2253. recoverable. Review approvals are exempt (the human is the record)."""
  2254. def __init__(self, task_id: str):
  2255. self.task_id = task_id
  2256. super().__init__(
  2257. f"completion blocked: {task_id} has no result or summary evidence"
  2258. )
  2259. class ArtifactPreservationError(RuntimeError):
  2260. """Raised when a declared scratch deliverable cannot be preserved."""
  2261. class LiveClaimError(ValueError):
  2262. """``complete_task`` refused: the task is ``running`` under a live claim and
  2263. the caller neither owns its run (``expected_run_id``) nor passed ``force``.
  2264. Completing anyway would close the worker's run row underneath a process
  2265. that is still executing. A ``ValueError`` so tool error handlers treat it
  2266. as recoverable."""
  2267. def __init__(self, task_id: str):
  2268. super().__init__(
  2269. f"{task_id} is running under a live worker claim; pass expected_run_id "
  2270. "(worker ownership) or force=True (explicit operator override) instead "
  2271. "of closing the live run"
  2272. )
  2273. def _claim_is_live(trow) -> bool:
  2274. """True when a ``running`` task's claim still protects a run: the worker process
  2275. it spawned exists (PID + start-time fingerprint). A claim whose worker is gone,
  2276. or a library/CLI claim that never spawned one, has no run to protect. TTL expiry
  2277. is deliberately not consulted: ``reclaim_stale_tasks`` extends, not reclaims, the
  2278. claim of a live worker, so the process is the liveness authority here too."""
  2279. return bool(
  2280. trow["status"] == "running"
  2281. and trow["claim_lock"] is not None
  2282. and trow["worker_pid"]
  2283. and _worker_alive(trow["worker_pid"], trow["worker_started_at"])
  2284. )
  2285. def complete_task(
  2286. conn: sqlite3.Connection, task_id: str, *, result: Optional[str] = None,
  2287. summary: Optional[str] = None, metadata: Optional[dict] = None,
  2288. created_cards: Optional[Iterable[str]] = None, expected_run_id: Optional[int] = None,
  2289. fire_lifecycle_hook: bool = True, force: bool = False,
  2290. ) -> bool:
  2291. """``running|ready|blocked|review -> done``; records ``result``.
  2292. ``ready`` is accepted for manual CLI completion, ``review`` for human
  2293. approval. A ``running`` task under a live claim is only completed with
  2294. proof of ownership (``expected_run_id``) or ``force=True`` (explicit
  2295. operator override) — otherwise :class:`LiveClaimError`, the same fence
  2296. :func:`request_review` applies. With no active run the handoff fields survive via
  2297. :func:`_synthesize_ended_run`. ``summary`` (defaults to ``result``) and
  2298. ``metadata`` land on the closing run for :func:`build_worker_context`.
  2299. ``created_cards`` are verified first — a phantom id raises
  2300. :class:`HallucinatedCardsError` after an auditable event; afterwards the
  2301. prose is scanned for unresolvable ``t_<hex>`` refs (advisory event only).
  2302. Completions from non-review statuses need evidence: a stripped ``result``
  2303. or ``summary``, or a stripped result already stored on the card. Empty or
  2304. whitespace-only evidence raises :class:`EmptyCompletionError` after an
  2305. auditable event. Approving a card out of ``review`` stays exempt.
  2306. """
  2307. now = int(time.time())
  2308. # Cheap pre-check; re-checked inside the txn to close the parent-reopen race.
  2309. if not _parents_satisfied(conn, task_id):
  2310. return False
  2311. from hermes_cli.kanban_pr_acceptance_store import prepare_acceptance, record_acceptance
  2312. verified_cards = _gate_created_cards(conn, task_id, created_cards, summary or result)
  2313. _gate_empty_completion(conn, task_id, result=result, summary=summary)
  2314. metadata = _merge_completion_prose_artifacts(
  2315. conn, task_id, metadata, summary=summary, result=result,
  2316. )
  2317. handoff_summary = summary if summary is not None else result
  2318. acceptance = prepare_acceptance(conn, task_id, expected_run_id, metadata)
  2319. if acceptance is False:
  2320. return False
  2321. with write_txn(conn):
  2322. # Hard invariant even for human review approval: a parent may have
  2323. # reopened while this task waited.
  2324. if not _parents_satisfied(conn, task_id):
  2325. return False
  2326. if acceptance is not None and not record_acceptance(conn, task_id, acceptance):
  2327. return False
  2328. trow = conn.execute(
  2329. "SELECT status, claim_lock, worker_pid, worker_started_at FROM tasks WHERE id = ?",
  2330. (task_id,),
  2331. ).fetchone()
  2332. prior_status = trow["status"] if trow else None
  2333. # Refuse to close a LIVE worker's run without proof of ownership
  2334. # (expected_run_id) or an explicit human override (force=True); see
  2335. # _claim_is_live for what "live" means.
  2336. if expected_run_id is None and not force and trow and _claim_is_live(trow):
  2337. raise LiveClaimError(task_id)
  2338. sql = """
  2339. UPDATE tasks
  2340. SET status = 'done',
  2341. result = ?,
  2342. completed_at = ?,
  2343. claim_lock = NULL,
  2344. claim_expires= NULL,
  2345. worker_pid = NULL,
  2346. block_kind = NULL,
  2347. block_recurrences = 0
  2348. WHERE id = ?
  2349. AND status IN ('running', 'ready', 'blocked', 'review')
  2350. """
  2351. params: tuple = (result, now, task_id)
  2352. if expected_run_id is not None:
  2353. sql += " AND current_run_id = ?"
  2354. params = (*params, int(expected_run_id))
  2355. if conn.execute(sql, params).rowcount != 1:
  2356. return False
  2357. if isinstance(metadata, dict):
  2358. _stage_completion_artifacts(conn, task_id, metadata, now)
  2359. run_id = _end_run(
  2360. conn, task_id, outcome="completed", status="done", summary=handoff_summary,
  2361. metadata=metadata,
  2362. )
  2363. # Never-claimed task: synthesize a run so the handoff fields survive.
  2364. if run_id is None and (summary or metadata or result or prior_status == "review"):
  2365. synth_summary, synth_metadata = handoff_summary, metadata
  2366. if prior_status == "review" and not synth_summary and not synth_metadata:
  2367. synth_summary = _REVIEW_APPROVED_NOTE
  2368. synth_metadata = {"source_status": "review", "approval": "manual"}
  2369. run_id = _synthesize_ended_run(
  2370. conn, task_id, outcome="completed", summary=synth_summary, metadata=synth_metadata,
  2371. )
  2372. event_summary = handoff_summary
  2373. if prior_status == "review" and not event_summary:
  2374. event_summary = _REVIEW_APPROVED_NOTE
  2375. _append_event(
  2376. conn, task_id, "completed",
  2377. _completed_event_payload(result, event_summary, verified_cards, metadata),
  2378. run_id=run_id,
  2379. )
  2380. _flag_phantom_prose_refs(conn, task_id, run_id, summary, result, verified_cards)
  2381. # Success wipes the breaker counter (history stays on the event log).
  2382. _clear_failure_counter(conn, task_id)
  2383. recompute_ready(conn) # separate txn so children see ``done``
  2384. _cleanup_workspace(conn, task_id)
  2385. _done_task = get_task(conn, task_id)
  2386. if fire_lifecycle_hook:
  2387. _fire_task_hook("kanban_task_completed", _done_task, task_id, run_id, summary=handoff_summary)
  2388. return True
  2389. _REVIEW_APPROVED_NOTE = "Review approved without additional evidence."
  2390. def _gate_created_cards(
  2391. conn: sqlite3.Connection, task_id: str, created_cards: Optional[Iterable[str]], preview_text: Optional[str],
  2392. ) -> list[str]:
  2393. """Verify ``created_cards`` BEFORE the main write txn; returns the verified
  2394. ids. A phantom id is recorded in its own tiny txn (auditable) then raised
  2395. as :class:`HallucinatedCardsError` without touching task state."""
  2396. if not created_cards:
  2397. return []
  2398. verified_cards, phantom_cards = _verify_created_cards(conn, task_id, created_cards)
  2399. if phantom_cards:
  2400. with write_txn(conn):
  2401. _append_event(
  2402. conn, task_id, "completion_blocked_hallucination",
  2403. {
  2404. "phantom_cards": phantom_cards,
  2405. "verified_cards": verified_cards,
  2406. "summary_preview": _first_line(preview_text, 200) or None,
  2407. },
  2408. )
  2409. raise HallucinatedCardsError(phantom_cards, task_id)
  2410. return verified_cards
  2411. def _substantive_text(value: Optional[str]) -> bool:
  2412. return bool(value is not None and str(value).strip())
  2413. def _gate_empty_completion(
  2414. conn: sqlite3.Connection,
  2415. task_id: str,
  2416. *,
  2417. result: Optional[str],
  2418. summary: Optional[str],
  2419. ) -> None:
  2420. """Refuse a completion that would leave the card with no evidence.
  2421. Review approvals are exempt: a human vouches for the card and
  2422. ``_REVIEW_APPROVED_NOTE`` is the documented record.
  2423. """
  2424. row = conn.execute(
  2425. "SELECT status, result FROM tasks WHERE id = ?",
  2426. (task_id,),
  2427. ).fetchone()
  2428. if row is None:
  2429. return
  2430. if row["status"] == "review":
  2431. return
  2432. stored = row["result"]
  2433. if _substantive_text(result) or _substantive_text(summary) or _substantive_text(stored):
  2434. return
  2435. with write_txn(conn):
  2436. _append_event(
  2437. conn, task_id, "completion_blocked_empty_result",
  2438. {
  2439. "result_preview": _first_line(result, 200) or None,
  2440. "summary_preview": _first_line(summary, 200) or None,
  2441. },
  2442. )
  2443. raise EmptyCompletionError(task_id)
  2444. def _stage_completion_artifacts(
  2445. conn: sqlite3.Connection, task_id: str, metadata: dict, now: int, *,
  2446. uploaded_by: str = "kanban_complete",
  2447. ) -> list[Path]:
  2448. """Copy scratch artifacts to the attachments dir and record each as an
  2449. attachment row; returns the copies so the caller can discard them if its
  2450. transaction rolls back."""
  2451. _persist_scratch_completion_artifacts(conn, task_id, metadata)
  2452. staged = [Path(stored_path) for stored_path in metadata.pop("_staged_artifacts", [])]
  2453. for path in staged:
  2454. _insert_completion_attachment(
  2455. conn, task_id, filename=path.name, stored_path=str(path),
  2456. size=path.stat().st_size, created_at=now, uploaded_by=uploaded_by,
  2457. )
  2458. return staged
  2459. def _cleaned_artifact_paths(metadata: Any) -> list[str]:
  2460. """Non-blank string paths declared in ``metadata["artifacts"]``."""
  2461. if not isinstance(metadata, dict):
  2462. return []
  2463. raw = metadata.get("artifacts")
  2464. if not isinstance(raw, (list, tuple)):
  2465. return []
  2466. return [str(p).strip() for p in raw if isinstance(p, str) and str(p).strip()]
  2467. def _completed_event_payload(
  2468. result: Optional[str], event_summary: Optional[str], verified_cards: list[str], metadata: Any,
  2469. ) -> dict:
  2470. """``completed`` event payload: first summary line (400 chars) so gateway
  2471. notifiers / dashboard WS render without a second round-trip; verified
  2472. cards; and ``metadata["artifacts"]`` promoted so the notifier can upload
  2473. them as native attachments without fetching the run row."""
  2474. # Mirror CLI's _show_voice_status: include STT/TTS provider availability so the user can tell at a
  2475. # glance *why* voice mode isn't working ("STT provider: MISSING ..." is the common case). ``record_key``
  2476. # mirrors the configured ``voice.record_key`` so the TUI can both bind it (frontend
  2477. # ``isVoiceToggleKey``) and display it in /voice status — previously the TUI hardcoded Ctrl+B and
  2478. # ignored the config (#18994).
  2479. payload: dict = {
  2480. "result_len": len(result) if result else 0,
  2481. "summary": _first_line(event_summary, 400) or None,
  2482. }
  2483. if verified_cards:
  2484. payload["verified_cards"] = verified_cards
  2485. if isinstance(metadata, dict):
  2486. cleaned = _cleaned_artifact_paths(metadata)
  2487. if cleaned:
  2488. payload["artifacts"] = cleaned
  2489. return payload
  2490. def _flag_phantom_prose_refs(
  2491. conn: sqlite3.Connection, task_id: str, run_id: Optional[int],
  2492. summary: Optional[str], result: Optional[str], verified_cards: list[str],
  2493. ) -> None:
  2494. """Advisory post-commit scan of summary+result for unresolvable ``t_<hex>``
  2495. references; emits ``suspected_hallucinated_references`` in its own txn so
  2496. the completion is already durable. Never blocks."""
  2497. scan_text = " ".join(filter(None, [summary, result]))
  2498. if not scan_text:
  2499. return
  2500. phantom_refs = [p for p in _scan_prose_for_phantom_ids(conn, scan_text) if p not in set(verified_cards)]
  2501. if phantom_refs:
  2502. with write_txn(conn):
  2503. _append_event(
  2504. conn, task_id, "suspected_hallucinated_references",
  2505. {"phantom_refs": phantom_refs, "source": "completion_summary"}, run_id=run_id,
  2506. )
  2507. def _merge_completion_prose_artifacts(
  2508. conn: sqlite3.Connection, task_id: str, metadata: Optional[dict], *, summary: Optional[str],
  2509. result: Optional[str],
  2510. ) -> Optional[dict]:
  2511. """Legacy workers named deliverables only by absolute path in prose; add
  2512. those that exist under the scratch workspace to ``metadata["artifacts"]``
  2513. before cleanup can erase them."""
  2514. workspace = _scratch_workspace(conn, task_id)
  2515. if workspace is None:
  2516. return metadata
  2517. if not _is_managed_scratch_path(workspace):
  2518. return metadata
  2519. text = "\n".join(part for part in (summary, result) if part)
  2520. if not text:
  2521. return metadata
  2522. prefix = re.escape(str(workspace))
  2523. discovered: list[str] = []
  2524. for match in re.finditer(prefix + r"(?:[/\\][^\s`\"'<>]+)", text):
  2525. raw = match.group(0).rstrip(".,;:!?)]}")
  2526. candidate = Path(raw)
  2527. if candidate.is_file():
  2528. discovered.append(str(candidate))
  2529. if not discovered:
  2530. return metadata
  2531. updated = dict(metadata) if isinstance(metadata, dict) else {}
  2532. existing = updated.get("artifacts")
  2533. merged = list(existing) if isinstance(existing, (list, tuple)) else []
  2534. seen = {str(path) for path in merged}
  2535. for path in discovered:
  2536. if path not in seen:
  2537. merged.append(path)
  2538. seen.add(path)
  2539. updated["artifacts"] = merged
  2540. return updated
  2541. def _persist_scratch_completion_artifacts(
  2542. conn: sqlite3.Connection, task_id: str, metadata: dict,
  2543. ) -> None:
  2544. """Copy scratch-workspace completion artifacts before cleanup removes them."""
  2545. raw_artifacts = metadata.get("artifacts")
  2546. if not isinstance(raw_artifacts, (list, tuple)):
  2547. return
  2548. workspace = _scratch_workspace(conn, task_id)
  2549. if workspace is None:
  2550. return
  2551. is_managed, board = _managed_scratch_path_info(workspace)
  2552. if not is_managed:
  2553. return
  2554. try:
  2555. workspace_root = workspace.resolve()
  2556. except OSError:
  2557. return
  2558. attachment_dir = task_attachments_dir(task_id, board=board)
  2559. persisted: list[str] = []
  2560. used_destinations: set[Path] = set()
  2561. changed = False
  2562. def _discard_copies() -> None:
  2563. _discard_staged_copies(used_destinations, attachment_dir)
  2564. for item in raw_artifacts:
  2565. artifact = str(item).strip() if isinstance(item, str) else ""
  2566. if not artifact:
  2567. continue
  2568. src = Path(artifact).expanduser()
  2569. try:
  2570. resolved_src = src.resolve()
  2571. except OSError:
  2572. persisted.append(artifact)
  2573. continue
  2574. if not resolved_src.is_relative_to(workspace_root):
  2575. persisted.append(artifact)
  2576. continue
  2577. problem = None
  2578. if not src.is_file():
  2579. problem = f"declared scratch artifact is unavailable or not a regular file: {artifact}"
  2580. elif resolved_src.stat().st_size > KANBAN_ATTACHMENT_MAX_BYTES:
  2581. problem = (
  2582. f"declared scratch artifact exceeds the "
  2583. f"{KANBAN_ATTACHMENT_MAX_BYTES}-byte limit: {artifact}"
  2584. )
  2585. if problem:
  2586. _discard_copies()
  2587. raise ArtifactPreservationError(problem)
  2588. dest: Optional[Path] = None
  2589. try:
  2590. attachment_dir.mkdir(parents=True, exist_ok=True)
  2591. dest = _unique_attachment_path(attachment_dir, resolved_src.name, used_destinations)
  2592. _copy_capped(resolved_src, dest, artifact)
  2593. except Exception as exc:
  2594. if dest is not None:
  2595. with contextlib.suppress(OSError):
  2596. dest.unlink(missing_ok=True)
  2597. _discard_copies()
  2598. if isinstance(exc, ArtifactPreservationError):
  2599. raise
  2600. raise ArtifactPreservationError(
  2601. f"could not preserve declared scratch artifact {artifact}: {exc}"
  2602. ) from exc
  2603. used_destinations.add(dest)
  2604. persisted.append(str(dest.resolve()))
  2605. changed = True
  2606. if changed:
  2607. metadata["artifacts"] = persisted
  2608. metadata["_staged_artifacts"] = [
  2609. path for path in persisted if path.startswith(str(attachment_dir.resolve()))
  2610. ]
  2611. def _discard_staged_copies(copies: Iterable[Path], attachment_dir: Path) -> None:
  2612. """Remove staged attachment copies whose DB rows never committed; a leaked
  2613. copy would make the retry stage ``name_1.ext`` next to an orphan."""
  2614. for copied in copies:
  2615. with contextlib.suppress(OSError):
  2616. Path(copied).unlink(missing_ok=True)
  2617. with contextlib.suppress(OSError):
  2618. attachment_dir.rmdir()
  2619. def _copy_capped(src: Path, dest: Path, artifact: str) -> None:
  2620. """Chunked copy that aborts if the file grows past the attachment cap mid-copy."""
  2621. with src.open("rb") as source_file, dest.open("xb") as destination_file:
  2622. copied = 0
  2623. while chunk := source_file.read(1024 * 1024):
  2624. copied += len(chunk)
  2625. if copied > KANBAN_ATTACHMENT_MAX_BYTES:
  2626. raise ArtifactPreservationError(
  2627. f"declared scratch artifact grew beyond the size limit: {artifact}"
  2628. )
  2629. destination_file.write(chunk)
  2630. def _insert_completion_attachment(
  2631. conn: sqlite3.Connection, task_id: str, *, filename: str, stored_path: str, size: int,
  2632. created_at: int, uploaded_by: str = "kanban_complete",
  2633. ) -> None:
  2634. """Record a worker-produced artifact in the existing attachment table."""
  2635. conn.execute(
  2636. "INSERT INTO task_attachments "
  2637. "(task_id, filename, stored_path, content_type, size, uploaded_by, created_at) "
  2638. "VALUES (?, ?, ?, NULL, ?, ?, ?)",
  2639. (task_id, filename, stored_path, size, uploaded_by, created_at),
  2640. )
  2641. _append_event(conn, task_id, "attached", {"filename": filename, "size": size, "by": uploaded_by})
  2642. def _unique_attachment_path(directory: Path, filename: str, used: set[Path]) -> Path:
  2643. """Return a non-conflicting path under ``directory`` for ``filename``."""
  2644. safe_name = Path(filename).name or "artifact"
  2645. stem, suffix = Path(safe_name).stem or "artifact", Path(safe_name).suffix
  2646. candidate = directory / safe_name
  2647. idx = 1
  2648. while candidate in used or candidate.exists():
  2649. candidate = directory / f"{stem}_{idx}{suffix}"
  2650. idx += 1
  2651. return candidate
  2652. def edit_task(
  2653. conn: sqlite3.Connection, task_id: str, *, title: Optional[str] = None,
  2654. body: Optional[str] = None, priority: Optional[int] = None,
  2655. result: Optional[str] = None, summary: Optional[str] = None,
  2656. metadata: Optional[dict] = None, board: Optional[str] = None,
  2657. ) -> bool:
  2658. """Edit task fields, optionally backfilling a completed task's result."""
  2659. changed_fields = [
  2660. field for field, value in (("title", title), ("body", body), ("priority", priority))
  2661. if value is not None
  2662. ]
  2663. with write_txn(conn):
  2664. status = _task_status(conn, task_id)
  2665. if status is None or (result is not None and status != "done"):
  2666. return False
  2667. assignments = []
  2668. params = []
  2669. for field, value in (("title", title), ("body", body), ("priority", priority)):
  2670. if value is not None:
  2671. assignments.append(f"{field} = ?")
  2672. params.append(value)
  2673. if result is not None:
  2674. assignments.append("result = ?")
  2675. params.append(result)
  2676. changed_fields.append("result")
  2677. if not assignments:
  2678. return False
  2679. conn.execute(
  2680. f"UPDATE tasks SET {', '.join(assignments)} WHERE id = ?",
  2681. (*params, task_id),
  2682. )
  2683. if priority is not None:
  2684. _append_event(conn, task_id, "reprioritized", {"priority": priority})
  2685. if result is None:
  2686. non_priority_fields = [field for field in changed_fields if field != "priority"]
  2687. if non_priority_fields:
  2688. _append_event(conn, task_id, "edited", {"fields": non_priority_fields})
  2689. else:
  2690. handoff_summary = summary if summary is not None else result
  2691. changed_fields.append("summary")
  2692. if metadata is not None:
  2693. changed_fields.append("metadata")
  2694. run = conn.execute(
  2695. """
  2696. SELECT id FROM task_runs
  2697. WHERE task_id = ?
  2698. AND outcome = 'completed'
  2699. ORDER BY COALESCE(ended_at, started_at, 0) DESC, id DESC
  2700. LIMIT 1
  2701. """,
  2702. (task_id,),
  2703. ).fetchone()
  2704. if run is None:
  2705. run_id = _synthesize_ended_run(
  2706. conn, task_id, outcome="completed", summary=handoff_summary, metadata=metadata,
  2707. )
  2708. else:
  2709. run_id = int(run["id"])
  2710. conn.execute("UPDATE task_runs SET summary = ? WHERE id = ?", (handoff_summary, run_id))
  2711. if metadata is not None:
  2712. conn.execute(
  2713. "UPDATE task_runs SET metadata = ? WHERE id = ?",
  2714. (json.dumps(metadata, ensure_ascii=False), run_id),
  2715. )
  2716. _append_event(
  2717. conn, task_id, "edited",
  2718. {
  2719. "fields": ["result", "summary"] + (["metadata"] if metadata is not None else []),
  2720. "result_len": len(result) if result else 0,
  2721. "summary": _first_line(handoff_summary, 400) or None,
  2722. },
  2723. run_id=run_id,
  2724. )
  2725. notify_task_updated(conn, task_id, changed_fields, board=board)
  2726. return True
  2727. def block_task(
  2728. conn: sqlite3.Connection, task_id: str, *, reason: Optional[str] = None,
  2729. kind: Optional[str] = None, expected_run_id: Optional[int] = None,
  2730. ) -> bool:
  2731. """``running``/``ready`` -> ``blocked`` (or ``todo`` / ``triage``, see
  2732. :func:`_route_block`). ``kind='dependency'`` with no incomplete parent is
  2733. re-kinded to ``needs_input`` (sticky) so ``recompute_ready`` cannot
  2734. promote it into a context-free respawn. ``transient`` still counts
  2735. toward the loop breaker so a forever-flaky task escalates. True on any
  2736. transition.
  2737. An already-``blocked`` card that the failure breaker parked UNTYPED
  2738. (``block_kind IS NULL``, no live run) is classified in place when *kind*
  2739. is supplied: ``block_kind``/``block_recurrences`` are set and a ``blocked``
  2740. audit event is appended, while status, failure evidence and the terminal
  2741. runs stay exactly as the breaker left them. A typed block, a card with a
  2742. live run, or a kind-less call on a blocked card are still refused.
  2743. """
  2744. if kind is not None and kind not in VALID_BLOCK_KINDS:
  2745. raise ValueError(f"block kind must be one of {sorted(VALID_BLOCK_KINDS)} or None")
  2746. with write_txn(conn):
  2747. cur_row = conn.execute(
  2748. "SELECT status, block_kind, block_recurrences FROM tasks WHERE id = ?", (task_id,),
  2749. ).fetchone()
  2750. if cur_row is None:
  2751. return False
  2752. # The breaker (``_record_task_failure``) parks cards ``blocked`` with no
  2753. # ``block_kind`` and no ``blocked`` event -- the policy is the
  2754. # supervisor's, not the kernel's -- but the transition guard below only
  2755. # matches running/ready, so that policy could never be attached later
  2756. # (#117363). Classify in place; never re-type or flap status. A caller
  2757. # asserting run ownership (``expected_run_id``) cannot own a parked
  2758. # card -- its run is over -- so it is refused like any stale worker.
  2759. if cur_row["status"] == "blocked":
  2760. if kind is None or expected_run_id is not None or _row_get(cur_row, "block_kind") is not None:
  2761. return False
  2762. classified = conn.execute(
  2763. "UPDATE tasks SET block_kind = ?, block_recurrences = 1 "
  2764. "WHERE id = ? AND status = 'blocked' AND block_kind IS NULL "
  2765. "AND current_run_id IS NULL",
  2766. (kind, task_id),
  2767. ).rowcount
  2768. if classified != 1:
  2769. return False
  2770. _append_event(conn, task_id, "blocked", {
  2771. "kind": kind, "reason": reason, "classified_in_place": True,
  2772. })
  2773. return True
  2774. source_status = _retry_status_for_run(conn, task_id) if cur_row["status"] == "running" else "ready"
  2775. requested_kind = kind
  2776. rekind_reason = None
  2777. # ``dependency`` only waits on incomplete parents. A worker filing that
  2778. # kind with none open would park in ``todo`` and ``recompute_ready``
  2779. # would promote+respawn it context-free on the next tick. Re-kind to
  2780. # ``needs_input`` so it is sticky until a human unblocks.
  2781. if kind == "dependency" and _parents_satisfied(conn, task_id):
  2782. kind = "needs_input"
  2783. rekind_reason = "no_open_parent"
  2784. new_status, event_kind, set_sql, params, payload = _route_block(
  2785. kind, reason, source_status, prev_kind=_row_get(cur_row, "block_kind"),
  2786. prev_recurrences=int(_row_get(cur_row, "block_recurrences") or 0),
  2787. )
  2788. if rekind_reason:
  2789. payload["requested_kind"] = requested_kind
  2790. payload["rekind_reason"] = rekind_reason
  2791. sql = f"""
  2792. UPDATE tasks
  2793. SET status = '{new_status}',
  2794. claim_lock = NULL,
  2795. claim_expires = NULL,
  2796. worker_pid = NULL,
  2797. {set_sql}
  2798. WHERE id = ?
  2799. AND status IN ('running', 'ready')
  2800. """
  2801. params = (*params, task_id)
  2802. if expected_run_id is not None:
  2803. sql += " AND current_run_id = ?"
  2804. params = (*params, int(expected_run_id))
  2805. if conn.execute(sql, params).rowcount != 1:
  2806. return False
  2807. run_id = _end_or_synthesize_run(
  2808. conn, task_id, outcome="blocked", status="blocked", summary=reason, synthesize=bool(reason),
  2809. )
  2810. _append_event(conn, task_id, event_kind, payload, run_id=run_id)
  2811. blocked_task = get_task(conn, task_id)
  2812. _fire_task_hook("kanban_task_blocked", blocked_task, task_id, run_id, reason=reason)
  2813. return True
  2814. def _route_block(
  2815. kind: Optional[str], reason: Optional[str], source_status: str, *,
  2816. prev_kind: Optional[str], prev_recurrences: int,
  2817. ) -> tuple[str, str, str, tuple, dict]:
  2818. """``(new_status, event_kind, set_sql, params, payload)`` for :func:`block_task`.
  2819. ``dependency`` never enters the human ``blocked`` bucket: it waits in
  2820. ``todo`` for ``recompute_ready``, so a cron never sees a dependency-wait
  2821. as something to "unblock". Callers that pass ``dependency`` with no
  2822. incomplete parent are re-kinded to ``needs_input`` before this runs
  2823. (see :func:`block_task`). Every other kind counts unblock-loop
  2824. recurrences: block_task only fires from running/ready (AFTER an unblock
  2825. returned the task to the pool), so a stored ``block_kind`` equal to the
  2826. incoming one means blocked -> unblocked -> re-block for the same cause
  2827. (un-typed None compares equal to a prior un-typed block). At
  2828. ``BLOCK_RECURRENCE_LIMIT`` the task routes to ``triage`` for a human.
  2829. """
  2830. payload = {"reason": reason, "kind": kind, "source_status": source_status}
  2831. if kind == "dependency":
  2832. return "todo", "dependency_wait", "block_kind = ?", (kind,), payload
  2833. recurrences = prev_recurrences + 1 if prev_kind == kind else 1
  2834. set_sql = "block_kind = ?,\n block_recurrences = ?"
  2835. payload = {"reason": reason, "kind": kind, "recurrences": recurrences, "source_status": source_status}
  2836. if recurrences >= BLOCK_RECURRENCE_LIMIT:
  2837. payload["limit"] = BLOCK_RECURRENCE_LIMIT
  2838. return "triage", "block_loop_detected", set_sql, (kind, recurrences), payload
  2839. return "blocked", "blocked", set_sql, (kind, recurrences), payload
  2840. def redact_review_value(value: Any) -> Any:
  2841. """Redact secrets at the domain boundary for durable review handoffs."""
  2842. if isinstance(value, str):
  2843. from agent.redact import redact_sensitive_text
  2844. return redact_sensitive_text(value, force=True)
  2845. if isinstance(value, dict):
  2846. return {key: redact_review_value(item) for key, item in value.items()}
  2847. if isinstance(value, list):
  2848. return [redact_review_value(item) for item in value]
  2849. if isinstance(value, tuple):
  2850. return tuple(redact_review_value(item) for item in value)
  2851. return value
  2852. def request_review(
  2853. conn: sqlite3.Connection, task_id: str, *, summary: Optional[str] = None,
  2854. metadata: Optional[dict] = None, reviewer: Optional[str] = None,
  2855. expected_run_id: Optional[int] = None, force: bool = False, with_reason: bool = False,
  2856. ):
  2857. """``running``/``ready`` -> ``review``; never touches block recurrence accounting.
  2858. Implementer and reviewer are recorded on the event so requested changes
  2859. route back to the right profile; ``reviewer`` reassigns the task, and on
  2860. re-review defaults to the latest ``changes_requested`` provenance. A live
  2861. claim is only cleared with proof of ownership (``expected_run_id``) or
  2862. ``force=True``. Returns ``bool``, or ``(ok, reason)`` with ``with_reason``.
  2863. ``metadata["artifacts"]`` names the handoff's deliverable
  2864. files; a review handoff is the last implementer transition, and the
  2865. *reviewer's* completion is what cleans the managed scratch workspace up, so
  2866. the files are staged into the task's durable attachments dir here and the
  2867. staged paths ride the ``review_requested`` payload for the notifier to
  2868. upload. A declared artifact that cannot be preserved raises
  2869. :class:`ArtifactPreservationError`, rolling the whole transition back: the
  2870. task stays ``running`` and retryable, with no attachments and no event.
  2871. """
  2872. def _ret(ok: bool, reason: Optional[str] = None):
  2873. return (ok, reason) if with_reason else ok
  2874. summary = redact_review_value(summary)
  2875. metadata = redact_review_value(metadata)
  2876. # Declared (metadata["artifacts"]) and prose-referenced files
  2877. # must be durable BEFORE anything can clean the scratch workspace up: for a
  2878. # review-bound card the reviewer's completion is the cleanup trigger.
  2879. metadata = _merge_completion_prose_artifacts(conn, task_id, metadata, summary=summary, result=None)
  2880. now = int(time.time())
  2881. # Staged copies live outside the txn: a rollback after staging must not
  2882. # leave orphans that make the retry stage ``name_1.ext`` beside them.
  2883. staged_copies: list[Path] = []
  2884. try:
  2885. with write_txn(conn):
  2886. if not _parents_satisfied(conn, task_id):
  2887. return _ret(False, "parent dependencies are not satisfied")
  2888. trow = conn.execute(
  2889. "SELECT assignee, status, claim_lock, current_run_id, worker_pid, "
  2890. "worker_started_at FROM tasks WHERE id = ?", (task_id,),
  2891. ).fetchone()
  2892. if trow is None:
  2893. return _ret(False, "task not found")
  2894. # Refuse to clear a live worker's claim without proof of ownership
  2895. # (expected_run_id) or an explicit human override (force=True);
  2896. # the same fence as complete_task (_claim_is_live).
  2897. if expected_run_id is None and not force and _claim_is_live(trow):
  2898. return _ret(
  2899. False, "task is running under a live claim; pass expected_run_id "
  2900. "(worker ownership) or force=True (explicit operator "
  2901. "override) instead of clearing the live run's claim",
  2902. )
  2903. if reviewer is None:
  2904. reviewer = _prior_reviewer(conn, task_id)
  2905. if reviewer is False:
  2906. return _ret(
  2907. False, "re-review has no durable reviewer provenance (the "
  2908. "latest changes_requested event is missing or "
  2909. "malformed); pass reviewer= explicitly",
  2910. )
  2911. reviewer = _canonical_assignee(reviewer)
  2912. # The actor is the run that did the work. ``assignee`` is the actor
  2913. # only while a worker holds the card; on a never-claimed card it is
  2914. # whoever the operator assigned -- possibly the reviewer itself,
  2915. # which is what ``kanban create --assignee <reviewer>`` followed by
  2916. # ``request-review`` produces. Recording the reviewer as its own
  2917. # implementer is worse than recording nothing: request_changes()
  2918. # routes on this field, and it already refuses a handoff that
  2919. # carries no implementer provenance.
  2920. implementer = None
  2921. if trow["current_run_id"] is not None:
  2922. arow = conn.execute(
  2923. "SELECT profile FROM task_runs WHERE id = ?",
  2924. (trow["current_run_id"],),
  2925. ).fetchone()
  2926. implementer = arow["profile"] if arow else None
  2927. if implementer is None and trow["assignee"] != reviewer:
  2928. implementer = trow["assignee"]
  2929. assignee_sql = ", assignee = ?" if reviewer is not None else ""
  2930. run_guard = "" if expected_run_id is None else " AND current_run_id = ?"
  2931. params: tuple[Any, ...] = (
  2932. *(() if reviewer is None else (reviewer,)), task_id,
  2933. *(() if expected_run_id is None else (int(expected_run_id),)),
  2934. )
  2935. cur = conn.execute(
  2936. """
  2937. UPDATE tasks
  2938. SET status = 'review',
  2939. claim_lock = NULL,
  2940. claim_expires = NULL,
  2941. worker_pid = NULL
  2942. """ + assignee_sql + """
  2943. WHERE id = ?
  2944. AND status IN ('running', 'ready')
  2945. """ + run_guard,
  2946. params,
  2947. )
  2948. if cur.rowcount != 1:
  2949. return _ret(
  2950. False, "task is not in running/ready (or expected_run_id did not match the current run)",
  2951. )
  2952. if isinstance(metadata, dict):
  2953. staged_copies = _stage_completion_artifacts(
  2954. conn, task_id, metadata, now, uploaded_by="kanban_request_review",
  2955. )
  2956. run_id = _end_or_synthesize_run(
  2957. conn, task_id, outcome="review_requested", status="review",
  2958. summary=summary, metadata=metadata, synthesize=bool(summary or metadata),
  2959. profile=implementer,
  2960. )
  2961. payload: dict = {
  2962. "summary": _first_line(summary, 400) or None,
  2963. "implementer": implementer,
  2964. "reviewer": reviewer,
  2965. }
  2966. staged = _cleaned_artifact_paths(metadata)
  2967. if staged:
  2968. payload["artifacts"] = staged
  2969. _append_event(conn, task_id, "review_requested", payload, run_id=run_id)
  2970. except Exception:
  2971. if staged_copies:
  2972. _discard_staged_copies(staged_copies, staged_copies[0].parent)
  2973. raise
  2974. return _ret(True)
  2975. def _prior_reviewer(conn: sqlite3.Connection, task_id: str):
  2976. """Reviewer recorded by the latest ``changes_requested`` run's event.
  2977. ``None`` = first review (no such run); ``False`` = a run exists but its
  2978. provenance is missing/malformed."""
  2979. changes_run = conn.execute(
  2980. "SELECT id FROM task_runs "
  2981. "WHERE task_id = ? AND outcome = 'changes_requested' "
  2982. "ORDER BY id DESC LIMIT 1", (task_id,),
  2983. ).fetchone()
  2984. if changes_run is None:
  2985. return None
  2986. changes_event = _latest_event(conn, task_id, "changes_requested", changes_run["id"])
  2987. reviewer = _json_dict(_row_get(changes_event, "payload")).get("reviewer")
  2988. return reviewer if isinstance(reviewer, str) and reviewer.strip() else False
  2989. def _nonblank_str(value: Any) -> Optional[str]:
  2990. return value if isinstance(value, str) and value.strip() else None
  2991. def request_changes(
  2992. conn: sqlite3.Connection, task_id: str, *, reason: str, expected_run_id: Optional[int] = None,
  2993. ) -> tuple[bool, Optional[str]]:
  2994. """Close an active reviewer run (claimed from ``review``) and hand the task
  2995. back to the implementer from the latest ``review_requested`` event, parent
  2996. gating reapplied. Returns ``(ok, implementer | reason)``."""
  2997. reason = str(redact_review_value(reason or "")).strip()
  2998. if not reason:
  2999. return False, "reason is required"
  3000. with write_txn(conn):
  3001. task_row = conn.execute(
  3002. "SELECT status, assignee, current_run_id FROM tasks WHERE id = ?", (task_id,),
  3003. ).fetchone()
  3004. if task_row is None:
  3005. return False, "task not found"
  3006. current_run_id = task_row["current_run_id"]
  3007. if task_row["status"] != "running" or current_run_id is None:
  3008. return False, "task is not in an active review run"
  3009. if expected_run_id is not None and int(current_run_id) != int(expected_run_id):
  3010. return False, "run_id mismatch"
  3011. claimed_event = _latest_event(conn, task_id, "claimed", current_run_id)
  3012. claimed_payload = _json_dict(_row_get(claimed_event, "payload"))
  3013. if claimed_payload.get("source_status") != "review":
  3014. return False, "active run was not claimed from review"
  3015. requested_event = _latest_event(conn, task_id, "review_requested")
  3016. if requested_event is None:
  3017. return False, "no prior review_requested event"
  3018. implementer = _nonblank_str(_json_dict(requested_event["payload"]).get("implementer"))
  3019. if implementer is None:
  3020. return False, "review handoff has no valid implementer provenance"
  3021. reviewer = _canonical_assignee(_nonblank_str(task_row["assignee"]))
  3022. new_status = _landing_status_after_parents(conn, task_id)
  3023. # consecutive_failures deliberately PRESERVED: a review transition is
  3024. # not evidence the pathology cleared; only complete_task resets it.
  3025. cur = conn.execute(
  3026. """
  3027. UPDATE tasks
  3028. SET status = ?,
  3029. assignee = COALESCE(?, assignee),
  3030. claim_lock = NULL,
  3031. claim_expires = NULL,
  3032. worker_pid = NULL, worker_started_at = NULL
  3033. WHERE id = ? AND status = 'running' AND current_run_id = ?
  3034. """,
  3035. (new_status, implementer, task_id, int(current_run_id)),
  3036. )
  3037. if cur.rowcount != 1:
  3038. return False, "task changed during review handoff"
  3039. run_id = _end_run(
  3040. conn, task_id, outcome="changes_requested", status=new_status, summary=reason,
  3041. )
  3042. _append_event(
  3043. conn,
  3044. task_id,
  3045. "changes_requested",
  3046. {
  3047. "reason": reason,
  3048. "implementer": implementer,
  3049. "reviewer": reviewer,
  3050. "status": new_status,
  3051. },
  3052. run_id=run_id,
  3053. )
  3054. return True, implementer
  3055. def promote_task(
  3056. conn: sqlite3.Connection, task_id: str, *, actor: str, reason: Optional[str] = None,
  3057. dry_run: bool = False,
  3058. ) -> tuple[bool, Optional[str]]:
  3059. """Operator promotion ``todo``/``blocked`` -> ``ready`` with an audit event.
  3060. Refused while a parent is unfinished; ``dry_run`` only validates.
  3061. Returns ``(ok, reason)``."""
  3062. cur_status = _task_status(conn, task_id)
  3063. if cur_status is None:
  3064. return False, f"task {task_id} not found"
  3065. if cur_status not in ("todo", "blocked"):
  3066. return False, (
  3067. f"task {task_id} is {cur_status!r}; promote only applies to "
  3068. f"'todo' or 'blocked'"
  3069. )
  3070. # No override: claim_task demotes ready -> todo on an undone parent whichever
  3071. # writer set 'ready', so a forced promotion would only report a success the
  3072. # first claim silently reverts (#106195). The dependency itself is the knob.
  3073. parents = conn.execute(
  3074. "SELECT t.id, t.status FROM tasks t "
  3075. "JOIN task_links l ON l.parent_id = t.id "
  3076. "WHERE l.child_id = ?", (task_id,),
  3077. ).fetchall()
  3078. unsatisfied = [p["id"] for p in parents if p["status"] not in ("done", "archived")]
  3079. if unsatisfied:
  3080. return False, (
  3081. f"unsatisfied parent dependencies: {', '.join(unsatisfied)} "
  3082. f"(the ready -> running claim re-checks parents, so promotion cannot "
  3083. f"bypass them; complete the parents or drop the link with "
  3084. f"`hermes kanban unlink <parent_id> {task_id}`)"
  3085. )
  3086. if dry_run:
  3087. return True, None
  3088. with write_txn(conn):
  3089. upd = conn.execute(
  3090. "UPDATE tasks SET status = 'ready' "
  3091. "WHERE id = ? AND status IN ('todo', 'blocked')", (task_id,),
  3092. )
  3093. if upd.rowcount != 1:
  3094. return False, f"task {task_id} status changed during promotion"
  3095. _append_event(conn, task_id, "promoted_manual", {"actor": actor, "reason": reason})
  3096. return True, None
  3097. def _reclaim_dangling_run(
  3098. conn: sqlite3.Connection, task_id: str, *, statuses, now: int, note: str,
  3099. ) -> None:
  3100. """Close a leaked open run before a status flip so the invariant
  3101. ``current_run_id IS NULL <=> run row terminal`` holds; no-op normally."""
  3102. placeholders = ", ".join("?" for _ in statuses)
  3103. stale = conn.execute(
  3104. f"SELECT current_run_id FROM tasks WHERE id = ? AND status IN ({placeholders})",
  3105. (task_id, *statuses),
  3106. ).fetchone()
  3107. if stale and stale["current_run_id"]:
  3108. conn.execute(
  3109. """
  3110. UPDATE task_runs
  3111. SET status = 'reclaimed', outcome = 'reclaimed',
  3112. summary = COALESCE(summary, ?),
  3113. ended_at = ?,
  3114. claim_lock = NULL, claim_expires = NULL, worker_pid = NULL
  3115. WHERE id = ? AND ended_at IS NULL
  3116. """,
  3117. (note, now, int(stale["current_run_id"])),
  3118. )
  3119. def _landing_status_after_parents(conn: sqlite3.Connection, task_id: str) -> str:
  3120. """``ready`` if every parent is terminal else ``todo`` — the re-gate shared by
  3121. unblock/reopen so neither can spawn a child whose upstream is unfinished."""
  3122. return "ready" if _parents_satisfied(conn, task_id) else "todo"
  3123. def unblock_task(conn: sqlite3.Connection, task_id: str) -> bool:
  3124. """``blocked``/``scheduled`` -> its resumable phase (parent re-gated; ``review``
  3125. when that is where it left off), closing any leaked run first."""
  3126. now = int(time.time())
  3127. with write_txn(conn):
  3128. resume_status = (
  3129. _resume_status_from_events(conn, task_id)
  3130. if _task_status(conn, task_id) == "blocked"
  3131. else "ready"
  3132. )
  3133. _reclaim_dangling_run(
  3134. conn, task_id, statuses=("blocked", "scheduled"), now=now,
  3135. note="invariant recovery on unblock",
  3136. )
  3137. # Re-gate on parent completion before restoring the source phase.
  3138. landing_status = _landing_status_after_parents(conn, task_id)
  3139. new_status = (
  3140. "review"
  3141. if landing_status == "ready" and resume_status == "review"
  3142. else landing_status
  3143. )
  3144. # ``block_kind``/``block_recurrences`` deliberately survive the unblock:
  3145. # resetting them is the amnesia that let cron-unblock <-> re-block loop
  3146. # unbounded; only complete_task clears them. ``consecutive_failures``
  3147. # (the dispatcher's spawn/crash counter) IS reset — a deliberate unblock
  3148. # is a fresh start for the retry budget.
  3149. cur = conn.execute(
  3150. "UPDATE tasks SET status = ?, current_run_id = NULL, "
  3151. "consecutive_failures = 0, last_failure_error = NULL "
  3152. "WHERE id = ? AND status IN ('blocked', 'scheduled')", (new_status, task_id),
  3153. )
  3154. if cur.rowcount != 1:
  3155. return False
  3156. _append_event(
  3157. conn, task_id, "unblocked",
  3158. (
  3159. {"status": new_status, "resume_status": resume_status}
  3160. if new_status != "ready" or resume_status != "ready"
  3161. else None
  3162. ),
  3163. )
  3164. return True
  3165. def reopen_review_task(conn: sqlite3.Connection, task_id: str) -> bool:
  3166. """``review`` -> ``ready``/``todo`` so the implementer re-runs on the new
  3167. comments; restores the implementer from the ``review_requested`` event.
  3168. Preserves ``consecutive_failures`` and the block loop counter (review is
  3169. not a block; only :func:`complete_task` clears them)."""
  3170. now = int(time.time())
  3171. with write_txn(conn):
  3172. _reclaim_dangling_run(
  3173. conn, task_id, statuses=("review",), now=now,
  3174. note="invariant recovery on review reopen",
  3175. )
  3176. new_status = _landing_status_after_parents(conn, task_id)
  3177. review_event = _latest_event(conn, task_id, "review_requested")
  3178. handoff = _json_dict(_row_get(review_event, "payload"))
  3179. implementer = _nonblank_str(handoff.get("implementer"))
  3180. params: tuple[Any, ...] = (new_status, *((implementer,) if implementer else ()), task_id)
  3181. cur = conn.execute(
  3182. # consecutive_failures deliberately PRESERVED: review reopen is not
  3183. # a success signal; only complete_task resets the breaker (#35072).
  3184. "UPDATE tasks SET status = ?, current_run_id = NULL, "
  3185. "claim_lock = NULL, claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL "
  3186. + (", assignee = ?" if implementer else "")
  3187. + " WHERE id = ? AND status = 'review'",
  3188. params,
  3189. )
  3190. if cur.rowcount != 1:
  3191. return False
  3192. payload: dict[str, Any] = {"status": new_status}
  3193. if implementer:
  3194. payload["implementer"] = implementer
  3195. _append_event(
  3196. conn, task_id, "review_reopened", payload if payload != {"status": "ready"} else None,
  3197. )
  3198. return True
  3199. def invalidate_descendants_for_parent_reopen(
  3200. conn: sqlite3.Connection, task_id: str, *, author: str,
  3201. ) -> dict[str, Any]:
  3202. """THE done-reopen invalidation: every ``ready``/``review``/``running``/``done``
  3203. descendant of a reopened ancestor is demoted to ``todo`` and re-gated.
  3204. Every surface that reopens a done task (dashboard PATCH/drag) routes here.
  3205. Composes under the caller's txn (``allow_nested=True``) so the flip and the
  3206. retractions commit atomically. Each descendant gets a
  3207. ``descendant_invalidated`` event, the legacy ``status`` event the live feed
  3208. renders, and a comment naming the ancestor. Running descendants are closed
  3209. ``reclaimed`` and their workers killed strictly post-commit (audit trail
  3210. before death) — when composed, the CALLER must drain ``terminations``
  3211. after its own commit. ``consecutive_failures`` resets (deliberate operator
  3212. action), the opposite of :func:`reopen_review_task`.
  3213. Returns ``{"invalidated": [{id, prior_status, new_status, resume_status}],
  3214. "terminations": [(worker_pid, claim_lock, worker_started_at)]}``.
  3215. """
  3216. caller_owns_txn = bool(conn.in_transaction)
  3217. now = int(time.time())
  3218. invalidated: list[dict[str, Any]] = []
  3219. terminations: list[tuple[Optional[int], Optional[str], Optional[int]]] = []
  3220. with write_txn(conn, allow_nested=True):
  3221. rows = conn.execute(
  3222. """
  3223. WITH RECURSIVE descendants(id) AS (
  3224. SELECT child_id FROM task_links WHERE parent_id = ?
  3225. UNION
  3226. SELECT l.child_id
  3227. FROM task_links l
  3228. JOIN descendants d ON d.id = l.parent_id
  3229. )
  3230. SELECT t.id, t.status, t.current_run_id, t.worker_pid, t.claim_lock, t.worker_started_at
  3231. FROM descendants d
  3232. JOIN tasks t ON t.id = d.id
  3233. ORDER BY t.id
  3234. """,
  3235. (task_id,),
  3236. ).fetchall()
  3237. for row in rows:
  3238. previous_status = row["status"]
  3239. if previous_status not in {"ready", "review", "running", "done"}:
  3240. continue
  3241. resume_status = "ready"
  3242. run_id = None
  3243. if previous_status == "review":
  3244. resume_status = "review"
  3245. elif previous_status == "running":
  3246. resume_status = _retry_status_for_run(conn, row["id"], row["current_run_id"])
  3247. terminations.append((row["worker_pid"], row["claim_lock"], row["worker_started_at"]))
  3248. run_id = _end_run(
  3249. conn, row["id"], outcome="reclaimed", status="todo",
  3250. summary=f"ancestor {task_id} reopened",
  3251. )
  3252. # consecutive_failures = 0: deliberate operator reset — see
  3253. # docstring for why this diverges from reopen_review_task.
  3254. conn.execute(
  3255. "UPDATE tasks SET status = 'todo', completed_at = NULL, "
  3256. "claim_lock = NULL, claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "
  3257. "current_run_id = NULL, consecutive_failures = 0 WHERE id = ?", (row["id"],),
  3258. )
  3259. entry = {
  3260. "id": row["id"], "prior_status": previous_status,
  3261. "new_status": "todo", "resume_status": resume_status,
  3262. }
  3263. _append_event(
  3264. conn, row["id"], "descendant_invalidated",
  3265. {"ancestor": task_id, **{k: v for k, v in entry.items() if k != "id"}},
  3266. run_id=run_id,
  3267. )
  3268. # Legacy 'status' event so existing live-feed consumers still see
  3269. # the move without learning the new event kind.
  3270. _append_event(
  3271. conn, row["id"], "status",
  3272. {
  3273. "status": "todo", "reason": "ancestor_reopened", "parent": task_id,
  3274. "previous_status": previous_status, "resume_status": resume_status,
  3275. },
  3276. run_id=run_id,
  3277. )
  3278. _insert_comment(
  3279. conn, row["id"], author, f"Invalidated: ancestor {task_id} was reopened; "
  3280. f"retracted from '{previous_status}' to 'todo' "
  3281. f"(will resume via '{resume_status}').", now,
  3282. )
  3283. invalidated.append(entry)
  3284. if not caller_owns_txn:
  3285. # Standalone: committed above, audit trail durable, safe to kill now.
  3286. # Composed calls leave this to the caller post-commit.
  3287. for pid, claim_lock, started_at in terminations:
  3288. _terminate_reclaimed_worker(pid, claim_lock, started_at=started_at)
  3289. return {"invalidated": invalidated, "terminations": terminations}
  3290. def specify_triage_task(
  3291. conn: sqlite3.Connection, task_id: str, *, title: Optional[str] = None,
  3292. body: Optional[str] = None, assignee: Optional[str] = None, author: Optional[str] = None,
  3293. ) -> bool:
  3294. """Update title/body/assignee (when given) and move ``triage -> todo`` in one
  3295. txn; False when not in triage. Lands in ``todo`` (not ``ready``) so parent
  3296. gating still applies; the audit comment is written only when a field changed.
  3297. """
  3298. if title is not None and not title.strip():
  3299. raise ValueError("title cannot be blank")
  3300. assignee = _canonical_assignee(assignee)
  3301. with write_txn(conn):
  3302. existing = conn.execute(
  3303. "SELECT title, body, assignee FROM tasks WHERE id = ? AND status = 'triage'",
  3304. (task_id,),
  3305. ).fetchone()
  3306. if existing is None:
  3307. return False
  3308. sets: list[str] = ["status = 'todo'"]
  3309. params: list[Any] = []
  3310. changed_fields: list[str] = []
  3311. if title is not None and title.strip() != (existing["title"] or ""):
  3312. sets.append("title = ?")
  3313. params.append(title.strip())
  3314. changed_fields.append("title")
  3315. if body is not None and (body or "") != (existing["body"] or ""):
  3316. sets.append("body = ?")
  3317. params.append(body)
  3318. changed_fields.append("body")
  3319. if assignee is not None and assignee != (existing["assignee"] or None):
  3320. sets.append("assignee = ?")
  3321. params.append(assignee)
  3322. changed_fields.append("assignee")
  3323. params.append(task_id)
  3324. cur = conn.execute(
  3325. f"UPDATE tasks SET {', '.join(sets)} "
  3326. f"WHERE id = ? AND status = 'triage'", tuple(params),
  3327. )
  3328. if cur.rowcount != 1:
  3329. return False
  3330. if changed_fields and author and author.strip():
  3331. # Not add_comment (own txn + 'commented' event); 'specified' below records it.
  3332. _insert_comment(
  3333. conn, task_id, author.strip(),
  3334. "Specified — updated " + ", ".join(changed_fields) + " and promoted to todo.",
  3335. int(time.time()),
  3336. )
  3337. _append_event(
  3338. conn, task_id, "specified",
  3339. {"changed_fields": changed_fields} if changed_fields else None,
  3340. )
  3341. # Own IMMEDIATE txn (outside the one above): a parent-free specified task
  3342. # flips to 'ready' now instead of idling until the next tick.
  3343. recompute_ready(conn)
  3344. return True
  3345. def archive_task(conn: sqlite3.Connection, task_id: str, *, signal_fn=None) -> bool:
  3346. """Archive a task; a *running* task's host-local worker is terminated.
  3347. Clearing ``worker_pid`` in the DB alone left the OS process running past its
  3348. own archive — it kept executing (and pushing work) against a task nothing
  3349. tracked anymore (#76196). Snapshot pid+claim inside the archive txn so the
  3350. kill is contingent on THIS caller winning the archive transition (a losing
  3351. concurrent archiver must never signal the pid); the kill itself runs after
  3352. commit — ``_poll_worker_exit`` can wait ~5 s and must not hold the write
  3353. lock. Post-release kill is safe here because ``archived`` is terminal: no
  3354. dispatcher can spawn a duplicate worker off the released claim. The
  3355. termination outcome lands as its own ``archive_worker_termination`` event so
  3356. the ``archived`` event stays atomic with the status flip.
  3357. """
  3358. with write_txn(conn):
  3359. row = conn.execute(
  3360. "SELECT status, claim_lock, worker_pid, worker_started_at FROM tasks WHERE id = ?",
  3361. (task_id,),
  3362. ).fetchone()
  3363. if not row:
  3364. return False
  3365. was_running = row["status"] == "running"
  3366. prev_pid, prev_lock, prev_started = row["worker_pid"], row["claim_lock"], row["worker_started_at"]
  3367. cur = conn.execute(
  3368. "UPDATE tasks SET status = 'archived', "
  3369. " claim_lock = NULL, claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL "
  3370. "WHERE id = ? AND status != 'archived'", (task_id,),
  3371. )
  3372. if cur.rowcount != 1:
  3373. return False
  3374. # Archived mid-run (dashboard): close the run so history isn't orphaned.
  3375. run_id = _end_run(
  3376. conn, task_id, outcome="reclaimed", status="reclaimed",
  3377. summary="task archived with run still active",
  3378. )
  3379. _append_event(conn, task_id, "archived", None, run_id=run_id)
  3380. if was_running:
  3381. termination = _terminate_reclaimed_worker(prev_pid, prev_lock, signal_fn=signal_fn, started_at=prev_started)
  3382. with write_txn(conn):
  3383. _append_event(conn, task_id, "archive_worker_termination", termination, run_id=run_id)
  3384. # ``archived`` parents no longer block children; promote them now.
  3385. recompute_ready(conn)
  3386. # Reap the workspace on archive too (never-completed tasks kept it forever).
  3387. _cleanup_workspace(conn, task_id)
  3388. return True
  3389. def _delete_task_relations(conn: sqlite3.Connection, task_id: str) -> None:
  3390. """Delete every row referencing ``task_id`` (schema has no ON DELETE CASCADE)."""
  3391. conn.execute("DELETE FROM task_links WHERE parent_id = ? OR child_id = ?", (task_id, task_id))
  3392. for table in ("task_comments", "task_events", "task_runs", "kanban_notify_subs"):
  3393. conn.execute(f"DELETE FROM {table} WHERE task_id = ?", (task_id,))
  3394. def delete_archived_task(conn: sqlite3.Connection, task_id: str) -> bool:
  3395. """Hard-delete an ARCHIVED task (+ related rows); anything else must be
  3396. archived first so data loss takes two deliberate actions."""
  3397. with write_txn(conn):
  3398. if _task_status(conn, task_id) != "archived":
  3399. return False
  3400. _delete_task_relations(conn, task_id)
  3401. cur = conn.execute("DELETE FROM tasks WHERE id = ?", (task_id,))
  3402. return cur.rowcount == 1
  3403. def delete_task(conn: sqlite3.Connection, task_id: str) -> bool:
  3404. """Hard-delete a task and its related rows in one txn; False when not found."""
  3405. with write_txn(conn):
  3406. cur = conn.execute("DELETE FROM tasks WHERE id = ?", (task_id,))
  3407. if cur.rowcount != 1:
  3408. return False
  3409. _delete_task_relations(conn, task_id)
  3410. recompute_ready(conn)
  3411. return True
  3412. def schedule_task(
  3413. conn: sqlite3.Connection, task_id: str, *, reason: Optional[str] = None,
  3414. expected_run_id: Optional[int] = None,
  3415. ) -> bool:
  3416. """Park in ``scheduled`` (waiting on time, not a human; not dispatchable)
  3417. until ``unblock_task`` re-gates it."""
  3418. with write_txn(conn):
  3419. params: list[Any] = [task_id]
  3420. sql = """
  3421. UPDATE tasks
  3422. SET status = 'scheduled',
  3423. claim_lock = NULL,
  3424. claim_expires= NULL,
  3425. worker_pid = NULL
  3426. WHERE id = ?
  3427. AND status IN ('todo', 'ready', 'running', 'blocked')
  3428. """
  3429. if expected_run_id is not None:
  3430. sql += " AND current_run_id = ?"
  3431. params.append(int(expected_run_id))
  3432. if conn.execute(sql, params).rowcount != 1:
  3433. return False
  3434. run_id = _end_or_synthesize_run(
  3435. conn, task_id, outcome="scheduled", status="scheduled", summary=reason, synthesize=bool(reason),
  3436. )
  3437. _append_event(conn, task_id, "scheduled", {"reason": reason}, run_id=run_id)
  3438. return True
  3439. # --- Worker context builder (what a spawned worker sees) ---
  3440. def build_worker_context(conn: sqlite3.Connection, task_id: str) -> str:
  3441. """Everything a worker should read about its task: header, body,
  3442. attachments, prior attempts, done-parent handoffs, the assignee's recent
  3443. work, comments. Lists are tail-capped and fields char-capped
  3444. (``_CTX_MAX_*``) so the prompt stays bounded on pathological boards."""
  3445. task = get_task(conn, task_id)
  3446. if not task:
  3447. raise ValueError(f"unknown task {task_id}")
  3448. from hermes_cli.kanban_db_context import (
  3449. _ctx_attachments, _ctx_comments, _ctx_header, _ctx_parent_results, _ctx_prior_attempts,
  3450. _ctx_role_history,
  3451. )
  3452. # One clock reading so every relative age in this rendering agrees.
  3453. now = int(time.time())
  3454. lines: list[str] = []
  3455. _ctx_header(lines, task)
  3456. _ctx_attachments(lines, list_attachments(conn, task_id))
  3457. _ctx_prior_attempts(lines, conn, task_id, now)
  3458. _ctx_parent_results(lines, conn, task_id, now)
  3459. _ctx_role_history(lines, conn, task, now)
  3460. _ctx_comments(lines, list_comments(conn, task_id), now)
  3461. return "\n".join(lines).rstrip() + "\n"
  3462. # --- Stats + SLA helpers ---
  3463. def board_stats(conn: sqlite3.Connection) -> dict:
  3464. """Per-status + per-assignee counts and the oldest ``ready`` age (staleness signal)."""
  3465. by_status: dict[str, int] = {}
  3466. for row in conn.execute(
  3467. "SELECT status, COUNT(*) AS n FROM tasks "
  3468. "WHERE status != 'archived' GROUP BY status"
  3469. ):
  3470. by_status[row["status"]] = int(row["n"])
  3471. by_assignee = _counts_by_assignee(conn)
  3472. oldest_row = conn.execute(
  3473. "SELECT MIN(created_at) AS ts FROM tasks WHERE status = 'ready'"
  3474. ).fetchone()
  3475. now = int(time.time())
  3476. oldest_ready_age = (
  3477. (now - int(oldest_row["ts"]))
  3478. if oldest_row and oldest_row["ts"] is not None else None
  3479. )
  3480. return {
  3481. "by_status": by_status,
  3482. "by_assignee": by_assignee,
  3483. "oldest_ready_age_seconds": oldest_ready_age,
  3484. "now": now,
  3485. }
  3486. def _counts_by_assignee(conn: sqlite3.Connection) -> dict[str, dict[str, int]]:
  3487. """``{assignee: {status: n}}`` over non-archived tasks."""
  3488. counts: dict[str, dict[str, int]] = {}
  3489. for row in conn.execute(
  3490. "SELECT assignee, status, COUNT(*) AS n FROM tasks "
  3491. "WHERE status != 'archived' AND assignee IS NOT NULL "
  3492. "GROUP BY assignee, status"
  3493. ):
  3494. counts.setdefault(row["assignee"], {})[row["status"]] = int(row["n"])
  3495. return counts
  3496. def _to_epoch(val) -> Optional[int]:
  3497. """Epoch seconds from int/float/numeric string/ISO-8601; None for empty/invalid."""
  3498. if val is None:
  3499. return None
  3500. if isinstance(val, (int, float)):
  3501. return int(val)
  3502. s = str(val).strip()
  3503. if not s:
  3504. return None
  3505. try:
  3506. return int(s)
  3507. except ValueError:
  3508. pass
  3509. # ISO-8601 fallback (e.g. '2026-05-10T15:00:00Z')
  3510. try:
  3511. from datetime import datetime
  3512. dt = datetime.fromisoformat(s)
  3513. return int(dt.timestamp())
  3514. except (ValueError, OSError):
  3515. return None
  3516. def task_age(task: Task) -> dict:
  3517. """Return age metrics for a single task. All values are seconds or None."""
  3518. now = int(time.time())
  3519. _c, _s, _co = task.created_at, task.started_at, task.completed_at # from_row coerced
  3520. return {
  3521. "created_age_seconds": now - _c if _c is not None else None,
  3522. "started_age_seconds": now - _s if _s is not None else None,
  3523. "time_to_complete_seconds": _co - (_s or _c) if _co is not None else None,
  3524. }
  3525. # --- Retention + garbage collection ---
  3526. def _retention_seconds(older_than_seconds: int) -> int:
  3527. """Normalise a gc retention window, rejecting negatives.
  3528. Shared by both gc sweeps: a negative window puts the cutoff in the future,
  3529. so "older than cutoff" would match every row / file instead of none —
  3530. refuse before any sweep runs.
  3531. """
  3532. older_than_seconds = int(older_than_seconds)
  3533. if older_than_seconds < 0:
  3534. raise ValueError(
  3535. f"older_than_seconds must be >= 0, got {older_than_seconds!r}: "
  3536. "a negative retention selects everything."
  3537. )
  3538. return older_than_seconds
  3539. def gc_events(conn: sqlite3.Connection, *, older_than_seconds: int = 30 * 24 * 3600) -> int:
  3540. """Prune old done/archived events, retaining decomposition identity until task deletion.
  3541. ``older_than_seconds=0`` means everything older than now; the CLI maps
  3542. ``--event-retention-days 0`` to "disabled" before calling this.
  3543. """
  3544. cutoff = int(time.time()) - _retention_seconds(older_than_seconds)
  3545. with write_txn(conn):
  3546. cur = conn.execute(
  3547. "DELETE FROM task_events WHERE created_at < ? AND kind != 'decomposed' AND task_id IN "
  3548. "(SELECT id FROM tasks WHERE status IN ('done', 'archived'))", (cutoff,),
  3549. )
  3550. return int(cur.rowcount or 0)
  3551. def gc_worker_logs(*, older_than_seconds: int = 30 * 24 * 3600, board: Optional[str] = None) -> int:
  3552. """Delete worker log files older than the cutoff on one board; returns the count.
  3553. ``older_than_seconds=0`` means everything older than now; the CLI maps
  3554. ``--log-retention-days 0`` to "disabled" before calling this.
  3555. """
  3556. older_than_seconds = _retention_seconds(older_than_seconds)
  3557. log_dir = worker_logs_dir(board=board)
  3558. if not log_dir.exists():
  3559. return 0
  3560. return unlink_files_older_than(log_dir, "*", older_than_seconds)
  3561. # --- Worker log accessor ---
  3562. def worker_log_path(task_id: str, *, board: Optional[str] = None) -> Path:
  3563. """Worker log path (may not exist). The dispatcher always passes ``board``
  3564. explicitly to avoid resolution ambiguity."""
  3565. return worker_logs_dir(board=board) / f"{task_id}.log"
  3566. def read_worker_log(
  3567. task_id: str, *, tail_bytes: Optional[int] = None, board: Optional[str] = None,
  3568. ) -> Optional[str]:
  3569. """Worker log text (last ``tail_bytes`` when set); None when the file is missing."""
  3570. path = worker_log_path(task_id, board=board)
  3571. if not path.exists():
  3572. return None
  3573. try:
  3574. if tail_bytes is None:
  3575. return path.read_text(encoding="utf-8-sig", errors="replace")
  3576. size = path.stat().st_size
  3577. with open(path, "rb") as f:
  3578. if size > tail_bytes:
  3579. f.seek(size - tail_bytes)
  3580. # Skip the partial first line unless the window has no newline
  3581. # at all (readline() would eat everything).
  3582. probe = f.tell()
  3583. if not f.readline().endswith(b"\n") and f.tell() >= size:
  3584. f.seek(probe)
  3585. return f.read().decode("utf-8", errors="replace")
  3586. except OSError:
  3587. return None
  3588. # --- Assignee enumeration (known profiles + per-profile board stats) ---
  3589. def list_profiles_on_disk() -> list[str]:
  3590. """Profiles with a ``config.yaml`` plus the implicit ``default``; reads paths
  3591. directly to avoid importing ``hermes_cli.profiles`` at startup."""
  3592. try:
  3593. from hermes_constants import get_default_hermes_root
  3594. default_root = get_default_hermes_root()
  3595. profiles_dir = default_root / "profiles"
  3596. except Exception:
  3597. return []
  3598. names: set[str] = set()
  3599. if default_root.exists():
  3600. names.add("default")
  3601. if profiles_dir.is_dir():
  3602. try:
  3603. names.update(e.name for e in profiles_dir.iterdir() if e.is_dir() and (e / "config.yaml").is_file()) # config-reader: ok — profile directory discovery marker
  3604. except OSError:
  3605. pass
  3606. return sorted(names)
  3607. def known_assignees(conn: sqlite3.Connection) -> list[dict]:
  3608. """``{"name", "on_disk", "counts"}`` for every on-disk profile or task
  3609. assignee, so a fresh profile appears in pickers before it has a task."""
  3610. on_disk = set(list_profiles_on_disk())
  3611. counts = _counts_by_assignee(conn)
  3612. return [
  3613. {"name": name, "on_disk": name in on_disk, "counts": counts.get(name, {})}
  3614. for name in sorted(on_disk | set(counts))
  3615. ]
  3616. # --- Runs (attempt history on a task) ---
  3617. def list_runs(
  3618. conn: sqlite3.Connection, task_id: str, *, include_active: bool = True,
  3619. state_type: Optional[str] = None, state_name: Optional[str] = None,
  3620. ) -> list[Run]:
  3621. """Runs in start order; ``include_active=False`` = closed only; ``state_type``
  3622. (``status``/``outcome``) + ``state_name`` filter together."""
  3623. if (state_type is None) ^ (state_name is None):
  3624. raise ValueError("state_type and state_name must both be set or both omitted")
  3625. if state_type is not None and state_type not in ("status", "outcome"):
  3626. raise ValueError("state_type must be 'status' or 'outcome'")
  3627. q = "SELECT * FROM task_runs WHERE task_id = ?"
  3628. params: list[Any] = [task_id]
  3629. if not include_active:
  3630. q += " AND ended_at IS NOT NULL"
  3631. if state_type is not None:
  3632. q += f" AND {state_type} = ?"
  3633. params.append(state_name)
  3634. q += " ORDER BY started_at ASC, id ASC"
  3635. rows = conn.execute(q, params).fetchall()
  3636. return [Run.from_row(r) for r in rows]
  3637. def get_run(conn: sqlite3.Connection, run_id: int) -> Optional[Run]:
  3638. row = conn.execute("SELECT * FROM task_runs WHERE id = ?", (int(run_id),)).fetchone()
  3639. return Run.from_row(row) if row else None
  3640. def latest_run(conn: sqlite3.Connection, task_id: str) -> Optional[Run]:
  3641. """Return the most recent run regardless of outcome (active or closed)."""
  3642. row = conn.execute(
  3643. "SELECT * FROM task_runs WHERE task_id = ? "
  3644. "ORDER BY started_at DESC, id DESC LIMIT 1", (task_id,),
  3645. ).fetchone()
  3646. return Run.from_row(row) if row else None
  3647. def latest_summary(conn: sqlite3.Connection, task_id: str) -> Optional[str]:
  3648. """Newest non-empty run summary, or None. Workers hand off via ``summary`` and
  3649. leave ``tasks.result`` NULL, so views need this or a done task looks empty."""
  3650. row = conn.execute(
  3651. "SELECT summary FROM task_runs "
  3652. "WHERE task_id = ? AND summary IS NOT NULL AND summary != '' "
  3653. "ORDER BY COALESCE(ended_at, started_at) DESC, id DESC LIMIT 1", (task_id,),
  3654. ).fetchone()
  3655. return row["summary"] if row else None
  3656. def latest_summaries(conn: sqlite3.Connection, task_ids: Iterable[str]) -> dict[str, str]:
  3657. """``{task_id: newest non-empty run summary}`` in one query (window function,
  3658. SQLite >= 3.25); tasks without a summary are omitted."""
  3659. ids = list(task_ids)
  3660. if not ids:
  3661. return {}
  3662. placeholders = ",".join("?" for _ in ids)
  3663. rows = conn.execute(
  3664. f"""
  3665. SELECT task_id, summary FROM (
  3666. SELECT task_id, summary,
  3667. ROW_NUMBER() OVER (
  3668. PARTITION BY task_id
  3669. ORDER BY COALESCE(ended_at, started_at) DESC, id DESC
  3670. ) AS rn
  3671. FROM task_runs
  3672. WHERE task_id IN ({placeholders})
  3673. AND summary IS NOT NULL AND summary != ''
  3674. ) WHERE rn = 1
  3675. """,
  3676. ids,
  3677. ).fetchall()
  3678. return {r["task_id"]: r["summary"] for r in rows}
  3679. def current_run_started_ats(conn: sqlite3.Connection, task_ids: Iterable[str]) -> dict[str, int]:
  3680. """``{task_id: started_at of the run ``tasks.current_run_id`` points at}``
  3681. in one query; tasks with no active run (NULL or dangling pointer) are omitted."""
  3682. ids = list(task_ids)
  3683. if not ids:
  3684. return {}
  3685. placeholders = ",".join("?" for _ in ids)
  3686. rows = conn.execute(
  3687. "SELECT t.id AS task_id, r.started_at AS started_at FROM tasks t "
  3688. "JOIN task_runs r ON r.id = t.current_run_id "
  3689. f"WHERE t.id IN ({placeholders})",
  3690. ids,
  3691. ).fetchall()
  3692. return {r["task_id"]: r["started_at"] for r in rows}
  3693. # --- Split modules (imported at the tail: they import this module as ``_kb``) ---
  3694. from hermes_cli.kanban_db_boards import (
  3695. _default_board_display_name,
  3696. _dir_holds_board,
  3697. board_metadata_path,
  3698. create_board,
  3699. list_boards,
  3700. read_board_metadata,
  3701. remove_board,
  3702. write_board_metadata,
  3703. )
  3704. from hermes_cli.kanban_db_connect import (
  3705. _INITIALIZED_PATHS,
  3706. init_db,
  3707. write_txn,
  3708. )
  3709. from hermes_cli.kanban_db_workspace import (
  3710. _cleanup_workspace,
  3711. _is_managed_scratch_path,
  3712. _managed_scratch_path_info,
  3713. _scratch_workspace,
  3714. )
  3715. from hermes_cli.kanban_db_dispatch import (
  3716. DEFAULT_FAILURE_LIMIT,
  3717. DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS,
  3718. DispatchResult,
  3719. _clear_failure_counter,
  3720. _defer_reclaim_for_live_worker,
  3721. _pid_alive,
  3722. _record_task_failure,
  3723. _terminate_reclaimed_worker,
  3724. _worker_alive,
  3725. _worker_survived_termination,
  3726. _worker_terminal_timeout_env,
  3727. )