scheduler.py 214 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416
  1. # health: allow FILE_LINES -- security fix for #82010: distinguish an explicitly-empty toolset allowlist (fail closed, nothing allowed) from an absent one (no restriction); the added lines are minimal fail-closed branches at this existing chokepoint. Also #141468: match the `mcp-<server>` toolset spelling so a narrow job's MCP allowlist is not widened into every enabled server — a minimal branch at the same chokepoint.
  2. """Cron job scheduler: tick() runs due jobs (gateway calls it every 60s from a background thread).
  3. A file lock (~/.hermes/cron/.tick.lock) keeps overlapping processes to one tick at a time.
  4. """
  5. import atexit
  6. import concurrent.futures
  7. import contextlib
  8. import contextvars
  9. import errno
  10. import json
  11. import logging
  12. import os
  13. import re
  14. import subprocess
  15. import sys
  16. import threading
  17. import time
  18. import uuid
  19. from dataclasses import dataclass
  20. from datetime import datetime, timezone, UTC
  21. # fcntl is Unix-only; Windows uses msvcrt
  22. try:
  23. import fcntl
  24. except ImportError:
  25. fcntl = None
  26. try:
  27. import msvcrt
  28. except ImportError:
  29. msvcrt = None
  30. from pathlib import Path
  31. from typing import Any, Callable, Dict, List, Optional, Union
  32. # Must precede repo-level imports: standalone invocations (e.g. module reload after
  33. # `hermes update`) otherwise fail with ModuleNotFoundError for hermes_time et al.
  34. sys.path.insert(0, str(Path(__file__).parent.parent))
  35. from cron.worker_bootstrap import WORKER_MARKER, finish_worker_boot
  36. from hermes_constants import get_hermes_home, hermes_home_key
  37. from hermes_cli.observability.shared_metrics_gateway import note_cron_execution, note_cron_skipped
  38. from cron.env_settings import cron_env_setting
  39. from hermes_cli._subprocess_compat import windows_hide_flags
  40. from hermes_cli.config import (
  41. load_config, load_config_readonly)
  42. from hermes_cli.config_backend import ConfigBackendUnavailable
  43. from hermes_cli.fallback_config import get_fallback_chain, scoped_fallback_chain
  44. from hermes_time import now as _hermes_now, safe_strftime
  45. from agent.interrupt_compat import request_hard_interrupt
  46. from agent.delegation_context import (
  47. enter_non_dispatcher_owned_context, exit_non_dispatcher_owned_context)
  48. from agent.memory_provider import ctx_bound
  49. from agent.session_activity import AwakeIdleMeter
  50. from agent.turn_failure_copy import is_max_iteration_handoff
  51. logger = logging.getLogger(__name__)
  52. def _set_cron_session_title(session_db, session_id, base_title):
  53. """Persist a non-blank, unique title for a finished cron session; returns it (None if unset).
  54. Runs BEFORE end_session()/close() so no write races the close. Duplicate title (unique-index
  55. ValueError) -> get_next_title_in_lineage(); if unavailable, raise rather than end up untitled.
  56. Centralizes the title write so the cron finally block can guarantee a non-blank, unique title is
  57. persisted before end_session()/close() tear the connection down (issues #50535, #50536, #50537):
  58. - #50535: never leaves the session blank. base_title already carries a cron-id fallback for nameless
  59. jobs; this also guards a failed write. Recover by appending a #N suffix via get_next_title_in_lineage()
  60. when supported, instead of swallowing the error and ending up untitled. - #50536: this runs
  61. synchronously in the cron finally block ahead of the session close, so no in-flight title write can race
  62. the close.
  63. """
  64. if not session_db or not session_id:
  65. return None
  66. title = (base_title or "").strip()
  67. if not title:
  68. return None
  69. try:
  70. session_db.set_session_title(session_id, title)
  71. return title
  72. except ValueError:
  73. # Unique-title collision: fall back to the next lineage title (base #2, #3, ...).
  74. next_title_fn = getattr(session_db, "get_next_title_in_lineage", None)
  75. if next_title_fn is None:
  76. raise
  77. deduped = next_title_fn(title)
  78. if not deduped or deduped == title:
  79. raise
  80. session_db.set_session_title(session_id, deduped)
  81. return deduped
  82. def _job_route_pinned(job: dict) -> bool:
  83. """True when the job carries its own provider, model or endpoint. Unpinned jobs store none of
  84. these (they follow the main model at fire time), so any value is an explicit operator pin."""
  85. return any(isinstance(job.get(k), str) and job[k].strip() for k in ("provider", "model", "base_url"))
  86. def _job_fallback_chain(job: dict, cfg: Any) -> Optional[list]:
  87. """The fallback chain this job may walk, at credential resolution AND mid-run (#100437).
  88. A pinned job never borrows the global ``fallback_providers`` chain, the same rule a pinned
  89. ``delegate_task`` child follows (``scoped_fallback_chain``): a chain entry is a different
  90. provider and usually a different model, which is exactly what the pin ruled out. Same-provider
  91. credential-pool rotation is not the chain and still applies. Unpinned jobs inherit the chain.
  92. """
  93. return scoped_fallback_chain(
  94. get_fallback_chain(cfg), None, pinned=_job_route_pinned(job), owner="cron job")
  95. def _fallback_chain_phrase(job: Optional[dict] = None) -> str:
  96. """Backup-provider clause for a provider-failure notice: "pinned, no fallback" vs "the backups
  97. failed too" vs "none configured" (most installs). Fails open to "the backups failed too" if
  98. config can't be read — never crash delivery.
  99. """
  100. if job is not None and _job_route_pinned(job):
  101. return (
  102. "This job is pinned to its own provider/model, so it does not fall back to "
  103. f"`fallback_providers`; `hermes cron edit {job.get('id')} --unpin` lets it follow the "
  104. "main model and its fallback chain."
  105. )
  106. try:
  107. cfg = load_config() or {}
  108. chain = get_fallback_chain(cfg)
  109. except Exception:
  110. return "No backup provider succeeded either."
  111. if chain:
  112. return "No backup provider succeeded either."
  113. return (
  114. "No backup provider is configured — add one with `hermes fallback add`, "
  115. "or set a cron-wide default via `cron.model` + `cron.model_provider` in config.yaml."
  116. )
  117. def _failure_streak_nudge(job: dict) -> str:
  118. """Review nudge when a recurring job keeps failing, else "". The failure message is delivered
  119. BEFORE mark_job_run records this run, hence stored ``failure_streak`` + 1. Threshold:
  120. ``cron.failure_nudge_threshold`` (default 3, 0 disables)."""
  121. schedule_kind = (job.get("schedule") or {}).get("kind")
  122. if schedule_kind not in {"cron", "interval"}:
  123. return ""
  124. try:
  125. cfg = load_config() or {}
  126. threshold = int(
  127. ((cfg.get("cron") or {}) if isinstance(cfg, dict) else {}).get(
  128. "failure_nudge_threshold", 3
  129. )
  130. )
  131. except Exception:
  132. threshold = 3
  133. if threshold <= 0:
  134. return ""
  135. streak = int(job.get("failure_streak") or 0) + 1 # +1 = this run
  136. if streak < threshold:
  137. return ""
  138. job_ref = job.get("name") or job.get("id") or "this job"
  139. return (
  140. f"\nThis job has failed {streak} runs in a row — worth a review. "
  141. f"Fix its prompt/config, or pause it with `hermes cron pause {job_ref}` "
  142. "(resume/remove also available) to stop the noise."
  143. )
  144. def _detect_gateway_code_skew() -> tuple[str, str] | None:
  145. """Boot-vs-disk revision skew for THIS process, or None. Test seam over
  146. ``gateway.code_skew.detect_code_skew``; a broken import must never take delivery down."""
  147. try:
  148. from gateway.code_skew import detect_code_skew
  149. return detect_code_skew()
  150. except Exception:
  151. return None
  152. def _current_gateway_code_sha() -> str | None:
  153. """Full revision currently on disk; kept separate from display-shortened skew labels."""
  154. try:
  155. from gateway.code_skew import current_code_sha
  156. return current_code_sha()
  157. except Exception:
  158. return None
  159. class CronTickYielded(RuntimeError):
  160. """A stale-code ticker yielded this tick to a fresh gateway.
  161. Raised by ``tick()`` BEFORE the tick lock when boot fingerprint ≠ disk, this process does NOT
  162. own the runtime lock and its live holder reports the disk revision — the stale process must
  163. stay out of the dispatch race (contention would starve the fresh ticker). Skew ``None`` never
  164. yields (fail open). Raised, not returned, so ``record_ticker_error`` sees it and ``hermes cron
  165. status`` isn't green.
  166. """
  167. def __init__(self, boot_rev: str, disk_rev: str) -> None:
  168. self.boot_rev = boot_rev
  169. self.disk_rev = disk_rev
  170. super().__init__(
  171. f"Cron tick yielded to a fresh gateway process (stale code: "
  172. f"booted on {boot_rev}, disk is at {disk_rev})"
  173. )
  174. _STALE_YIELD_RE = re.compile(r"stale code: booted on (\S+), disk is at (\S+)\)")
  175. def stale_code_yield_labels(recorded_error: str | None) -> tuple[str, str] | None:
  176. """``(boot_rev, disk_rev)`` when a persisted ``ticker_last_error`` is a ``CronTickYielded``.
  177. ``hermes cron status`` runs in another process and only sees the marker text; a yielding
  178. ticker still refreshes its heartbeat, so this is the one signal that separates "stale code,
  179. firing nothing" from a healthy loop (#117275).
  180. """
  181. if not recorded_error or not recorded_error.startswith(CronTickYielded.__name__):
  182. return None
  183. match = _STALE_YIELD_RE.search(recorded_error)
  184. return (match.group(1), match.group(2)) if match else None
  185. def _should_yield_tick_to_fresh_gateway() -> tuple[str, str] | None:
  186. """``(boot_rev, disk_rev)`` when THIS profile's tick must yield to a fresher gateway, else None.
  187. Yields only when ALL hold: code skew, this process is not the cron owner for the profile being
  188. ticked, and another live host gateway whose served set covers that profile reports a fresh
  189. heartbeat on the disk revision. Every probe failure returns None — yielding is a certainty
  190. claim, never a guess.
  191. The question is asked PER PROFILE. One process multiplexes every profile, so "do I own the
  192. runtime lock" (process-global, stamped at boot for the launch home) cannot answer it: a
  193. stale multiplexer that holds the lock must still keep ticking the profiles nobody else ticks,
  194. and must still yield the ones a fresher gateway serves.
  195. ``gateway_state.json`` is per-HOME and last-writer-wins, not per-process: during a
  196. ``--replace`` takeover both the stale and the fresh gateway stamp it, so which one this
  197. predicate reads is write-order dependent. The pid equality in ``live_gateway_ticking`` is what
  198. binds the record to the current lock holder; the takeover window itself fails open (no yield).
  199. """
  200. skew = _detect_gateway_code_skew()
  201. if skew is None:
  202. return None
  203. disk_sha = _current_gateway_code_sha()
  204. if disk_sha is None:
  205. return None
  206. try:
  207. from cron.scheduler_ownership import live_gateway_ticking, owns_cron_tick_for
  208. home = _get_hermes_home()
  209. if owns_cron_tick_for(home):
  210. return None
  211. holder_status = live_gateway_ticking(home)
  212. if holder_status is None or holder_status.get("code_sha") != disk_sha:
  213. return None
  214. except Exception:
  215. return None
  216. return skew
  217. def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str:
  218. """One-line failure notice for chat delivery (full details stay in the run output).
  219. Deterministic scheduler/script shapes are matched first (their text can contain "timed out"
  220. and would otherwise be blamed on the model service); everything else goes through the shared
  221. ``classify_api_error`` verdict and the copy table in ``scheduler_failure_copy``."""
  222. from cron.scheduler_failure_copy import (
  223. classify_cron_failure_reason, generic_failure_notice, inactivity_notice,
  224. provider_failure_notice, script_timeout_notice)
  225. job_name = job.get("name") or job.get("id") or "cron job"
  226. job_id = job.get("id") or job_name
  227. text = (error or "unknown error").strip()
  228. lower = text.lower()
  229. # Script runner contract ("Script timed out after {n}s: {path}") — also for agent jobs with a
  230. # context script. Must precede provider classification so it never claims a model failure.
  231. # See #78503, #82460.
  232. if lower.startswith("script timed out"):
  233. return script_timeout_notice(job_name, job_id)
  234. # Scheduler inactivity watchdog ("idle for {n}s (limit {m}s)"): the job's OWN tool call went
  235. # quiet, no model service involved. Its text may still contain "timed out", so it must be
  236. # recognised before the classifier (field-reported: a stuck `terminal` call was blamed on the
  237. # provider and the operator debugged the wrong system).
  238. if re.search(r"idle for \d+s\s*\(limit \d+s\)", lower):
  239. return inactivity_notice(job_name, job_id)
  240. # no_agent jobs never reach a model, so provider errors are structurally impossible for them:
  241. # gate on job MODE before classifying, or a script's own wording ("429", "timed out") would
  242. # blame the wrong subsystem.
  243. if not job.get("no_agent"):
  244. notice = provider_failure_notice(
  245. job_name, job_id, classify_cron_failure_reason(text),
  246. backup_provider_phrase=_fallback_chain_phrase(job), provider=job.get("provider"))
  247. if notice is not None:
  248. return notice
  249. # Strip exception wrappers; bound input first so a multi-KB blob can't slow the regexes.
  250. cleaned = re.sub(r"^(RuntimeError|Exception|ValueError|HTTPStatusError):\s*", "", text[:2000])
  251. cleaned = re.sub(r"\s+", " ", cleaned).strip().rstrip(".")
  252. if len(cleaned) > 180:
  253. cleaned = cleaned[:177].rstrip() + "..."
  254. message = generic_failure_notice(job_name, job_id, cleaned)
  255. # Import-class failures (#95294 part 3): a long-lived gateway whose checkout was updated
  256. # underneath it (interrupted `hermes update`, manual git pull) serves MIXED modules and every
  257. # agent cron job dies with `cannot import name X`. The error reads like a code bug, so APPEND
  258. # cause + fix — never replace the raw error, which carries the failing symbol. Fail-safe: skew
  259. # is None on non-git/no-fingerprint; no_agent jobs excluded (a fresh subprocess resolves
  260. # imports against disk, so its ImportError is the script's own problem).
  261. if not job.get("no_agent") and re.search(
  262. r"cannot import name|modulenotfounderror|importerror", lower
  263. ):
  264. try:
  265. skew = _detect_gateway_code_skew()
  266. except Exception:
  267. skew = None # delivery must never die on a diagnostics probe
  268. if skew is not None:
  269. boot_rev, disk_rev = skew
  270. message += (
  271. f" Likely cause: the gateway is running stale code (booted "
  272. f"on {boot_rev}, disk is at {disk_rev}) — run "
  273. "`hermes gateway restart` to fix it."
  274. )
  275. return message
  276. DEFAULT_FAILURE_REPEAT_ALERT_HOURS = 6.0
  277. def _failure_repeat_alert_hours() -> float:
  278. """``cron.failure_repeat_alert_hours``: how long an ``alerted`` incident stays silent before one
  279. reminder ping. ``0`` (or negative) re-alerts on every failing run (the pre-gate behaviour)."""
  280. from cron.jobs import _cron_config_number
  281. return _cron_config_number("failure_repeat_alert_hours", DEFAULT_FAILURE_REPEAT_ALERT_HOURS, float)
  282. def _repeat_alert_withheld(incident: dict) -> bool:
  283. """An ``alerted`` incident withholds the per-run ping until the reminder cooldown has elapsed
  284. since its last ping. A missing/unparseable ``alerted_at`` (ledger written before the column
  285. existed) delivers rather than swallowing an alert."""
  286. hours = _failure_repeat_alert_hours()
  287. if hours <= 0:
  288. return False
  289. alerted_at = incident.get("alerted_at")
  290. if not alerted_at:
  291. return False
  292. try:
  293. from cron.jobs import _elapsed_seconds, _ensure_aware
  294. last = _ensure_aware(datetime.fromisoformat(str(alerted_at)))
  295. except (TypeError, ValueError):
  296. return False
  297. return _elapsed_seconds(_hermes_now(), last) < hours * 3600
  298. def _upsert_incident_for_failure(
  299. job: dict, error: str, *, output_file: Optional[Any] = None
  300. ) -> tuple[bool, Optional[str]]:
  301. """Record a durable failure incident (grouped by job + error signature). Returns
  302. ``(withheld, incident_id)``; withheld=True when the signature's incident is already ``closed``
  303. (operator ack) or ``alerted`` inside the ``cron.failure_repeat_alert_hours`` cooldown (a ping
  304. already went out) -> suppress the per-run ping. Store errors log at debug; the caller delivers
  305. as if none existed."""
  306. try:
  307. from cron.incidents import get_incident, upsert_incident
  308. incident_id, _is_new = upsert_incident(
  309. job["id"], str(error or ""), job_name=job.get("name"), output_file=output_file)
  310. incident = get_incident(incident_id)
  311. state = incident.get("state") if incident else None
  312. withheld = state == "closed" or (state == "alerted" and _repeat_alert_withheld(incident))
  313. return withheld, incident_id
  314. except Exception as exc:
  315. logger.debug(
  316. "Incident store unavailable for job %s (delivery unaffected): %s",
  317. job["id"], exc)
  318. return False, None
  319. def _resolve_incidents_for_recovered_job(job: dict) -> None:
  320. """Best-effort: a successful run marks the job's open incidents ``resolved`` (never touches an
  321. operator ``closed`` ack). Store errors log at debug; delivery is unaffected."""
  322. try:
  323. from cron.incidents import close_incidents_for_recovered_job
  324. close_incidents_for_recovered_job(job["id"])
  325. except Exception as exc:
  326. logger.debug("Incident store unavailable for job %s (delivery unaffected): %s", job["id"], exc)
  327. def _mark_incident_alerted(incident_id: Optional[str]) -> None:
  328. """Best-effort: mark incident ``alerted`` (no-op for closed; never resurrects an acked one)."""
  329. if not incident_id:
  330. return
  331. try:
  332. from cron.incidents import set_incident_state
  333. set_incident_state(incident_id, "alerted")
  334. except Exception as exc:
  335. logger.debug("Failed marking incident %s alerted: %s", incident_id, exc)
  336. class CronPromptInjectionBlocked(Exception):
  337. """Raised by _build_job_prompt when the assembled prompt (incl. runtime-loaded skill content,
  338. unseen by create-time scanning) trips the injection scanner; run_job turns it into a clean
  339. "job blocked" delivery.
  340. Assembled-prompt scanning (including loaded skill content) plugs the gap from #3968: create-time
  341. scanning only covers the user-supplied prompt field; skill content loaded at runtime was never scanned,
  342. so a malicious skill could carry an injection payload that reached the non-interactive (auto-approve)
  343. cron agent.
  344. """
  345. def _resolve_cron_disabled_toolsets(cfg: dict) -> list[str]:
  346. """Toolsets a cron-spawned agent must never receive: ``messaging``/``clarify`` always
  347. (interactive); ``cronjob`` by default (loop prevention, not a security boundary —
  348. ``cron.allow_agent_scheduling: true`` lifts only that); ``agent.disabled_toolsets`` layered on
  349. top so per-job ``enabled_toolsets`` cannot widen past config.yaml's denylist.
  350. See #25752.
  351. """
  352. cron_cfg = (cfg or {}).get("cron") or {}
  353. if cron_cfg.get("allow_agent_scheduling"):
  354. disabled = ["messaging", "clarify"]
  355. else:
  356. disabled = ["cronjob", "messaging", "clarify"]
  357. agent_cfg = (cfg or {}).get("agent") or {}
  358. from agent.skill_utils import parse_config_string_list
  359. user_disabled = parse_config_string_list(agent_cfg.get("disabled_toolsets"))
  360. for name in user_disabled:
  361. name = str(name).strip()
  362. if name and name not in disabled:
  363. disabled.append(name)
  364. return disabled
  365. def _merge_mcp_into_per_job_toolsets(per_job: list[str], cfg: dict) -> list[str]:
  366. """Layer enabled MCP servers onto a per-job ``enabled_toolsets`` allowlist (else a per-job list
  367. silently drops every MCP server). Mirrors ``_get_platform_tools``: ``no_mcp`` sentinel -> none
  368. (stripped); any MCP server already listed -> allowlist, add nothing; else union all enabled."""
  369. result = [t for t in per_job if t != "no_mcp"]
  370. if "no_mcp" in per_job:
  371. return result
  372. # lazy: avoid heavy hermes_cli import at module load; shares MCP-membership with gateway/CLI
  373. from hermes_cli.tools_config import enabled_mcp_server_names
  374. enabled_mcp = enabled_mcp_server_names(cfg)
  375. # A job names an MCP server by its TOOLSET name (``mcp-<server>``) or by the bare alias the
  376. # registry registers for it; compare against BOTH, or a job that named one server never matched
  377. # and the fallback below unioned EVERY enabled server into its allowlist (#141468).
  378. # A server that is CONFIGURED but disabled still counts as named: it is the user's selection, it
  379. # stays off, and it must not fall through to every enabled server either - the same rule the
  380. # interactive resolver follows (hermes_cli.tools_config._merge_mcp_servers).
  381. configured = cfg.get("mcp_servers") if isinstance(cfg.get("mcp_servers"), dict) else {}
  382. known = enabled_mcp | {str(name) for name in configured}
  383. if set(result) & (known | {f"mcp-{name}" for name in known}):
  384. return result
  385. for name in sorted(enabled_mcp):
  386. if name not in result:
  387. result.append(name)
  388. return result
  389. def _resolve_cron_enabled_toolsets(job: dict, cfg: dict) -> list[str]:
  390. """Toolset list for a cron job. Precedence: per-job ``enabled_toolsets`` (+ MCP merge) >
  391. ``cron`` platform config (``_get_platform_tools``, which strips _DEFAULT_OFF_TOOLSETS so fresh
  392. installs run without ``moa``). A lookup failure fails CLOSED: the run errors out.
  393. 1. Per-job ``enabled_toolsets`` (set via ``cronjob`` tool on create/update). Keeps the agent's
  394. job-scoped toolset override intact — #6130. Enabled MCP servers are layered on per
  395. ``_merge_mcp_into_per_job_toolsets`` so a native-toolset allowlist does not silently strip MCP tools.
  396. An explicitly-set EMPTY list is a zero-toolset allowlist, not a clear to the platform default —
  397. it is falsy, so it must be compared with ``is not None``, else it fell through to the config
  398. default and widened an unattended job back to every toolset (#82010). 2.
  399. Mirrors gateway behavior (``_get_platform_tools(cfg, platform_key)``) so users can gate cron toolsets
  400. globally without recreating every job. 3. Never ``None``: AIAgent reads ``None`` as "every
  401. toolset", so an unreadable ``platform_toolsets.cron`` restriction would hand an unattended job
  402. the full default set (#111380). The raise reaches ``run_job``'s failure path, which records the
  403. error on the job and opens an incident, so the operator sees it instead of a widened run.
  404. """
  405. per_job = job.get("enabled_toolsets")
  406. if per_job is not None:
  407. if not per_job:
  408. # Explicit zero: no toolsets at all — no MCP merge either, else every enabled server
  409. # would ride back in and widen the allowlist the operator just emptied (#82010).
  410. return []
  411. return _merge_mcp_into_per_job_toolsets(list(per_job), cfg or {})
  412. try:
  413. from hermes_cli.tools_config import _get_platform_tools # lazy: avoid heavy import at cron module load
  414. return sorted(_get_platform_tools(cfg or {}, "cron"))
  415. except Exception as exc:
  416. raise RuntimeError(
  417. "Cron toolset resolution failed, so this run was refused rather than given every "
  418. f"tool. Check `platform_toolsets.cron` in config.yaml (`hermes cron doctor`): {exc}"
  419. ) from exc
  420. def _resolve_job_reasoning_config(job: dict, cfg: dict, model: str) -> dict | None:
  421. """Effective reasoning config for a cron run. A per-job ``reasoning_effort`` pin beats global
  422. and per-model config and is model-independent by design (also governs an auth-fallback swap);
  423. clamping stays with provider transports. An unparseable pin warns and falls back to config."""
  424. from hermes_constants import parse_reasoning_effort, resolve_reasoning_config
  425. pinned = job.get("reasoning_effort")
  426. if pinned is not None:
  427. parsed = parse_reasoning_effort(pinned)
  428. if parsed is not None:
  429. logger.info("Job '%s': using per-job reasoning_effort '%s'", job.get("id", "?"), pinned)
  430. return parsed
  431. logger.warning(
  432. "Job '%s': invalid stored reasoning_effort %r — ignoring the pin "
  433. "and falling back to config resolution. Fix with `cronjob "
  434. "action=update job_id=%s reasoning_effort=<level>` (valid: none, "
  435. "minimal, low, medium, high, xhigh, max, ultra).",
  436. job.get("id", "?"),
  437. pinned,
  438. job.get("id", "?"))
  439. return resolve_reasoning_config(cfg if isinstance(cfg, dict) else {}, str(model))
  440. from cron.jobs import (
  441. _ensure_cron_dir, advance_next_runs, claim_dispatch, claim_job_for_fire, fire_claim_fence,
  442. clear_run_claim, get_due_jobs, heartbeat_fire_claim, heartbeat_run_claim, mark_job_run,
  443. save_job_output, self_removal_delivery_allowed, self_removal_delivery_scope, use_cron_store)
  444. from cron import store_health
  445. from cron.execution_identity import enter_cron_execution, exit_cron_execution
  446. from cron.executions import (
  447. FIRER_FINISHED_STATES, _TERMINAL_STATES, HANDOFF_ADOPTION_GRACE_SECONDS, create_execution, finish_execution,
  448. get_execution, mark_execution_handoff_pending, mark_execution_running,
  449. recover_interrupted_executions, settle_unstarted_execution, terminalize_dead_owner)
  450. from cron.scheduler_liveness import ExecutionProgressStamper, _inactivity_watchdog_loop
  451. from cron.scheduler_bookkeeping import _classify_delivery_outcome
  452. from cron.cancellation import (CancelRequest, _CancelEventLike, _CombinedCancelEvent, cancel_reason,
  453. clear_cancel, finish_lost_execution,
  454. run_job_unless_cancelled as _run_job_unless_cancelled)
  455. # Response marker that suppresses delivery (output is still saved locally for audit).
  456. SILENT_MARKER = "[SILENT]"
  457. # Agent-declared failure marker for cron runs. Unlike SILENT, it is deliberately strict so a
  458. # report that merely quotes the token cannot turn a healthy run into a failed one.
  459. CRON_FAILURE_MARKER = "[CRON_FAILURE]"
  460. def _cron_failure_marker_error(text: str) -> Optional[str]:
  461. """Return failure evidence when an agent response declares a cron failure.
  462. Only the exact, standalone first line is control text. The caller keeps the complete response
  463. in the saved run output while routing this evidence through normal failure bookkeeping.
  464. """
  465. lines = (text or "").splitlines()
  466. if not lines or lines[0].rstrip() != CRON_FAILURE_MARKER:
  467. return None
  468. evidence = "\n".join(lines[1:]).strip()
  469. return evidence or "Cron agent reported failure."
  470. def _is_cron_silence_response(text: str) -> bool:
  471. """True when a cron final response should suppress delivery: ``[SILENT]`` (or SILENT /
  472. NO_REPLY / NO REPLY) as the whole response OR its own first/last line — NOT mid-sentence.
  473. Shares the webhook-lane matcher in :mod:`gateway.response_filters` so the two cannot drift.
  474. Recognizes the bracketed ``[SILENT]`` sentinel (whole-response, first line, or last line) plus the
  475. bracketless ``SILENT`` / ``NO_REPLY`` / ``NO REPLY`` variants the model emits when it drops the brackets
  476. (#51438, #46917). Whitespace-trimmed and case-insensitive. A token buried mid-sentence is treated as
  477. real content and delivered.
  478. """
  479. from gateway.response_filters import is_autonomous_silence_response
  480. return is_autonomous_silence_response(text)
  481. # Persistent pool for parallel cron jobs: tick() submits and returns; long jobs never block it.
  482. # Keyed by profile home: one host gateway multiplexes every profile, and ``max_parallel_jobs`` is a
  483. # per-profile config key — a single process-global pool is sized by whichever profile ticked first
  484. # and then imposes that limit on all the others.
  485. _parallel_pools: dict[str, concurrent.futures.ThreadPoolExecutor] = {}
  486. _parallel_pool_max_workers: dict[str, Optional[int]] = {}
  487. def _inflight_key(job_id: str, home: Optional[Path | str] = None) -> tuple:
  488. """``(home key, job id)`` — the identity of one in-flight cron run.
  489. ONE gateway process ticks every profile, so a job id alone is not unique: two profiles
  490. routinely carry identically named jobs (``daily-brief``) and sharing a key made the second
  491. profile's run read as a duplicate of the first and get skipped. ``home`` defaults to the
  492. active cron scope's home, which the ticker binds per profile for the whole tick.
  493. """
  494. return (hermes_home_key(home) if home is not None else hermes_home_key(_get_hermes_home()), job_id)
  495. # Home key -> the real home Path that produced it. ``hermes_home_key`` normcases (it lower-cases on
  496. # Windows), so ``Path(key[0])`` is a case-folded path that matches nothing else on disk; bookkeeping
  497. # that needs the profile home reads it here instead of reconstructing it from the key.
  498. _inflight_home_paths: dict[str, Path] = {}
  499. def _remember_inflight_home(home: Path) -> Path:
  500. """Record ``home`` under its key so a claim can be mapped back to a usable path."""
  501. _inflight_home_paths[hermes_home_key(home)] = home
  502. return home
  503. def _inflight_home_path(home_key: str) -> Path:
  504. """Real home Path for an in-flight key's home half (falls back to the key itself)."""
  505. return _inflight_home_paths.get(home_key) or Path(home_key)
  506. # In-flight state below is keyed by ``_inflight_key``; the public accessors still report bare job
  507. # ids so host-wide consumers (shutdown drain, idle-exit, metrics) see the union across profiles.
  508. _running_job_ids: set = set()
  509. _running_fire_owners: dict[tuple, dict[object, tuple[Optional[str], Path]]] = {}
  510. # Parent gateway threads synchronously waiting on restart-safe scope workers.
  511. # Shutdown must not misclassify these as ownerless in-process runs: the tool
  512. # process sweep cannot reach the worker's transient scope.
  513. _restart_safe_waiter_job_ids: set = set()
  514. # in-flight key -> pid of the restart-safe external worker executing it (absent for in-process
  515. # runs), so a drain observer can name the process holding the gateway open.
  516. _running_worker_pids: dict[tuple, int] = {}
  517. # In-flight keys whose external worker runs in its OWN transient systemd scope
  518. # (``GatewayChildDispatch.mode == "scoped"``). Only that worker outlives a gateway restart: a
  519. # systemd stop kills the service cgroup, so a ``degraded`` direct subprocess (no reachable user
  520. # bus) dies with the gateway even though it is external. The gateway's restart after-turn wait
  521. # skips exactly this set — see ``get_restart_wait_cron_counts``.
  522. _scope_isolated_job_ids: set = set()
  523. _running_lock = threading.Lock()
  524. # Per in-flight id: time.time() claim instant + the future owning its release (``_FUTURE_PENDING``
  525. # until pool.submit returns). Past-allowance with no live future = leak; the sweep force-releases.
  526. _running_since: dict = {}
  527. # in-flight key -> stale-inflight allowance (s), resolved once per run by get_wedged_job_ids.
  528. _running_allowance_s: dict = {}
  529. _running_futures: dict = {}
  530. # Installed in ``_running_futures`` at claim time so a sweep landing before ``pool.submit`` returns
  531. # never sees ``missing`` and releases a claim about to get its future.
  532. _FUTURE_PENDING = object()
  533. # Identity tokens fence cleanup and late future attachment after recovery.
  534. _running_registration_owners: dict = {}
  535. # Forced-release count/history for ``get_inflight_guard_stats()``; mirrored to JSONL for probes.
  536. _forced_release_count: int = 0
  537. _forced_releases: list = []
  538. _FORCED_RELEASE_HISTORY = 20
  539. # Stale-allowance floor (minutes); per-job allowance is max(2 * interval, this).
  540. _INFLIGHT_MIN_ALLOWANCE_MINUTES = 30.0
  541. # Execution tokens (``_running_fire_owners`` identity keys) force-interrupted at shutdown; see
  542. # ``mark_running_jobs_interrupted``. ``run_one_job`` checks its OWN token before writing
  543. # ``last_status`` so a still-running agent thread can't overwrite "interrupted" with a false "ok".
  544. # Token keying scopes the flag to one execution (recurring jobs reuse IDs); legacy paths without a
  545. # fire owner fall back to the bare job ID.
  546. # ``run_one_job``'s own completion path checks its OWN token before writing ``last_status`` so a cron agent
  547. # thread that keeps running in-process after its tool was killed out from under it — and produces a
  548. # plausible-looking final response from truncated output — can never overwrite the interrupted status with a
  549. # false "ok" (#60432). Token keying keeps an interruption scoped to that exact execution: a later run of the
  550. # same job ID (recurring jobs reuse the ID every fire) must not inherit the stale flag.
  551. _interrupted_job_ids: set = set()
  552. def get_running_job_ids() -> frozenset[str]:
  553. """Thread-safe snapshot of executing job IDs (dispatch until ``_process_job`` returns). Read by
  554. the gateway shutdown drain, otherwise blind to cron work (runs outside ``_running_agents``).
  555. _drain_active_agents``) reads this to treat in-flight cron work as active the same way it already treats
  556. in-flight chat sessions via ``_running_agents`` — cron jobs run through their own thread pool here,
  557. entirely outside that dict, so without this the drain is structurally blind to them (#60432).
  558. """
  559. with _running_lock:
  560. return frozenset(key[1] for key in _running_job_ids | _running_fire_owners.keys())
  561. def get_running_job_details() -> list[dict]:
  562. """Per in-flight job: ``{"job_id", "elapsed_s", "worker_pid", "restart_safe"}`` (``worker_pid``
  563. None for in-process runs; ``restart_safe`` True when the worker owns its own systemd scope). The
  564. drain wait publishes this so ``hermes update`` can say WHICH job it is waiting on."""
  565. now = time.time()
  566. with _running_lock:
  567. return [
  568. {"job_id": key[1],
  569. "elapsed_s": round(now - _running_since[key], 1) if key in _running_since else None,
  570. "worker_pid": _running_worker_pids.get(key),
  571. "restart_safe": key in _scope_isolated_job_ids}
  572. for key in sorted(_running_job_ids | _running_fire_owners.keys())
  573. ]
  574. def get_wedged_job_ids() -> frozenset[str]:
  575. """In-flight job IDs older than their stale-inflight allowance (``max(2 * interval,
  576. cron.inflight_max_minutes)``) — the scheduler's own definition of a claim that can no longer be
  577. making progress. ``sweep_stale_inflight`` cannot release these while the worker thread is still
  578. alive (a delivery blocked on a dead transport, #115469), so the gateway restart drain reads this to
  579. skip them the way it skips wedged chat turns; restart is their remedy.
  580. """
  581. return frozenset(key[1] for key in _wedged_inflight_keys())
  582. def _wedged_inflight_keys() -> frozenset[tuple]:
  583. """In-flight keys behind :func:`get_wedged_job_ids`, before the projection to bare job IDs."""
  584. now = time.time()
  585. with _running_lock:
  586. ages = {key: now - started for key, started in _running_since.items() if key in _running_job_ids}
  587. allowances = {key: _running_allowance_s[key] for key in ages if key in _running_allowance_s}
  588. if not ages:
  589. return frozenset()
  590. floor_seconds = _inflight_min_allowance_minutes() * 60.0
  591. unresolved = [key for key in ages if key not in allowances]
  592. if unresolved:
  593. # One jobs.json parse per run, not per tick per job: the restart drain polls this every
  594. # 0.1 s on the event loop for the whole wait, and get_job() re-reads the file each call.
  595. # ``load_jobs`` reads the ACTIVE scope's store, so only claims from that home can have
  596. # their interval resolved; a claim from another profile falls back to the floor.
  597. by_id: dict = {}
  598. with contextlib.suppress(Exception):
  599. from cron.jobs import load_jobs
  600. by_id = {j.get("id"): j for j in load_jobs()}
  601. local_home = hermes_home_key(_get_hermes_home())
  602. with _running_lock:
  603. for key in unresolved:
  604. allowance = floor_seconds
  605. local_job = by_id.get(key[1]) if key[0] == local_home else None
  606. interval_minutes = _job_interval_minutes(local_job or {})
  607. if interval_minutes:
  608. allowance = max(allowance, 2.0 * interval_minutes * 60.0)
  609. allowances[key] = allowance
  610. if key in _running_job_ids: # released meanwhile -> don't resurrect the entry
  611. _running_allowance_s[key] = allowance
  612. return frozenset(
  613. key for key, age in ages.items() if age >= max(allowances[key], floor_seconds))
  614. def get_restart_wait_cron_counts() -> dict:
  615. """In-flight cron runs split for the gateway restart wait: ``awaitable``, ``wedged`` and
  616. ``restart_safe`` (scoped external worker, not wedged). The three are disjoint and sum to the
  617. number of in-flight runs.
  618. A ``scoped`` worker runs in a transient user scope outside the gateway cgroup, so a systemd
  619. stop/restart cannot reach it: the shutdown drain deliberately leaves it unmarked
  620. (``mark_running_jobs_interrupted``) and its final send rides the durable delivery queue, drained
  621. by whichever gateway is live next. Holding the restart wait for it only keeps the gateway in
  622. ``draining`` (refusing new turns) for up to ``agent.restart_after_turn_timeout``. A ``degraded``
  623. worker (external subprocess, no reachable user bus) stays in the gateway cgroup and dies with a
  624. systemd stop, so it stays awaitable.
  625. Counted per in-flight KEY, not per bare job ID: one multiplexed gateway can run the same job ID
  626. in two profiles, and projecting to bare IDs first lets one profile's wedged or scoped run hide
  627. the other profile's live run, which the restart would then kill (e.g. a scoped ``daily-brief``
  628. in profile A next to a degraded ``daily-brief`` in profile B).
  629. """
  630. wedged = _wedged_inflight_keys()
  631. with _running_lock:
  632. inflight = _running_job_ids | _running_fire_owners.keys()
  633. restart_safe = (_scope_isolated_job_ids & inflight) - wedged
  634. wedged = wedged & inflight
  635. return {
  636. "awaitable": len(inflight - wedged - restart_safe),
  637. "wedged": len(wedged),
  638. "restart_safe": len(restart_safe),
  639. }
  640. def is_job_running(job_id: str, home: Optional[Path | str] = None) -> bool:
  641. """True when THIS process has an in-flight run of ``job_id`` FOR ``home`` (default: the active
  642. cron scope's home).
  643. The bare-id accessors above report the host-wide union so the shutdown drain and metrics see
  644. every profile's work. Liveness consumers must not: one process ticks every profile, so a
  645. ``daily-brief`` running in profile A would otherwise report profile B's idle ``daily-brief``
  646. as running and keep B's stale one-shot alive as a "possibly live run".
  647. """
  648. key = _inflight_key(job_id, home)
  649. with _running_lock:
  650. return key in _running_job_ids or key in _running_fire_owners
  651. def _record_external_cron_worker(job_id: str, pid: Any, *, scope_isolated: bool) -> None:
  652. """Record the external worker holding *job_id* so a drain observer can name it, and whether
  653. its scope isolates it from this process.
  654. ``scope_isolated`` is the strong form of restart-safety: only a ``scoped`` dispatch
  655. (``GatewayChildDispatch``) puts the worker in its own transient systemd scope, outside the
  656. gateway cgroup. A ``degraded`` worker is still external but shares the cgroup and dies with a
  657. systemd stop, so it must NOT be listed as restart-safe. Called once per accepted handoff;
  658. both maps are cleared by ``release_running_job`` and by the worker waiter's ``finally``.
  659. """
  660. key = _inflight_key(job_id)
  661. with _running_lock:
  662. with contextlib.suppress(TypeError, ValueError):
  663. _running_worker_pids[key] = int(pid)
  664. if scope_isolated:
  665. _scope_isolated_job_ids.add(key)
  666. def try_register_running_job(job_id: str, *, owner=None, future=_FUTURE_PENDING) -> bool:
  667. """Atomically add ``job_id`` to the in-flight set; False (caller must skip) if already mid-run.
  668. Single dedupe owner for ticker + manual runs (the fire claim's 300s TTL is outlived by real
  669. jobs). Callers MUST pair success with ``release_running_job`` in a ``finally``.
  670. Dispatchers pass a unique ``owner`` back on release. Direct workers register a running
  671. ``future`` atomically because no executor submission will replace the pending sentinel.
  672. This is the single dedupe owner shared by the ticker's ``_submit_with_guard`` and manual runs
  673. (``tools/cronjob_tools``): the fire claim alone cannot prevent a double-fire because its TTL (300s) is
  674. routinely outlived by real jobs, after which a manual ``cronjob(action='run')`` would claim successfully
  675. and run the same job concurrently (idea from #53395 by @izumi0uu).
  676. Registration also makes the run visible to ``get_running_job_ids`` (the gateway shutdown drain, #60432)
  677. and ``mark_running_jobs_interrupted``. Dedupe is per PROFILE: the key carries the active cron
  678. scope's home, so one multiplexing process never treats two profiles' same-named jobs as one.
  679. """
  680. from hermes_cli.backend_retirement import retirement
  681. key = _inflight_key(job_id, _remember_inflight_home(_get_hermes_home()))
  682. with retirement.work() as admitted, _running_lock:
  683. if not admitted or key in _running_job_ids:
  684. return False
  685. _running_job_ids.add(key)
  686. # Same critical section as the add: no window where an in-flight id lacks an age the sweep
  687. # can bound. Sentinel is replaced by the real future once ``pool.submit`` returns.
  688. _running_since[key] = time.time()
  689. _running_futures[key] = future
  690. _running_registration_owners[key] = owner
  691. return True
  692. def release_running_job(
  693. job_id: str, home: Optional[Path | str] = None, *, owner=None,
  694. ) -> None:
  695. """Remove the registration unless an explicit ``owner`` has been replaced.
  696. ``home`` MUST be passed by any caller that does not run inside the same cron scope the claim
  697. was registered under. The scope is a ContextVar the ticker binds per profile: a pool worker
  698. releasing in a ``finally`` outside ``ctx.run`` resolves the default (launch) home instead, so
  699. the discard misses the real key and every secondary profile's claim leaks.
  700. """
  701. key = _inflight_key(job_id, home)
  702. with _running_lock:
  703. if owner is not None and _running_registration_owners.get(key) is not owner:
  704. return
  705. _running_registration_owners.pop(key, None)
  706. _running_job_ids.discard(key)
  707. _running_since.pop(key, None)
  708. _running_allowance_s.pop(key, None)
  709. _running_futures.pop(key, None)
  710. _running_worker_pids.pop(key, None)
  711. _scope_isolated_job_ids.discard(key)
  712. def _inflight_min_allowance_minutes() -> float:
  713. """Stale allowance floor (min): ``cron.inflight_max_minutes``, else env escape hatch/default."""
  714. with contextlib.suppress(Exception):
  715. _ucfg = load_config() or {}
  716. _cfg_val = (
  717. _ucfg.get("cron", {}) if isinstance(_ucfg, dict) else {}
  718. ).get("inflight_max_minutes")
  719. if _cfg_val is not None:
  720. val = float(_cfg_val)
  721. if val > 0:
  722. return val
  723. raw = cron_env_setting("HERMES_CRON_INFLIGHT_MAX_MINUTES").strip()
  724. if raw:
  725. try:
  726. val = float(raw)
  727. if val > 0:
  728. return val
  729. except (ValueError, TypeError):
  730. logger.warning(
  731. "Invalid HERMES_CRON_INFLIGHT_MAX_MINUTES=%r; using default %s",
  732. raw,
  733. _INFLIGHT_MIN_ALLOWANCE_MINUTES)
  734. return _INFLIGHT_MIN_ALLOWANCE_MINUTES
  735. # expr -> minutes; cadence never changes, so avoid re-evaluating croniter every tick.
  736. _cron_interval_cache: dict = {}
  737. def _cron_interval_minutes(expr: str) -> Optional[float]:
  738. """Cron expression cadence (gap between next two fires) in minutes; None -> floor allowance."""
  739. if expr in _cron_interval_cache:
  740. return _cron_interval_cache[expr]
  741. result = None
  742. ok = False
  743. with contextlib.suppress(Exception):
  744. from cron.jobs import _ensure_croniter
  745. ok = _ensure_croniter()
  746. if ok:
  747. from cron.jobs import croniter as _croniter
  748. from datetime import datetime
  749. base = datetime.now()
  750. it = _croniter(expr, base)
  751. first = it.get_next(datetime)
  752. second = it.get_next(datetime)
  753. gap = (second - first).total_seconds() / 60.0
  754. result = gap if gap > 0 else None
  755. # Cache a real cadence or a bad-expr None (croniter loaded, expr invalid: stable). Skip the
  756. # None from a transient croniter ImportError so it can't pin the floor allowance for the
  757. # process lifetime once the import recovers.
  758. if result is not None or ok:
  759. _cron_interval_cache[expr] = result
  760. return result
  761. def _job_interval_minutes(job: dict) -> Optional[float]:
  762. """Best-effort job interval in minutes (None if unknown / one-shot -> floor). ``schedule`` is
  763. persisted as a parsed dict; the string path is only a fallback for programmatic callers."""
  764. with contextlib.suppress(Exception):
  765. schedule = job.get("schedule")
  766. if isinstance(schedule, str) and schedule.strip():
  767. from cron.jobs import parse_schedule
  768. schedule = parse_schedule(schedule) or {}
  769. if isinstance(schedule, dict):
  770. kind = schedule.get("kind")
  771. if kind == "interval":
  772. minutes = schedule.get("minutes")
  773. return float(minutes) if minutes else None
  774. if kind == "cron":
  775. return _cron_interval_minutes(str(schedule.get("expr") or ""))
  776. return None
  777. def get_inflight_guard_stats() -> dict:
  778. """Probe-visible snapshot; non-zero ``forced_releases`` means a job wedged and was recovered."""
  779. now = time.time()
  780. with _running_lock:
  781. return {
  782. "running": sorted(key[1] for key in _running_job_ids),
  783. "running_ages_seconds": {
  784. key[1]: round(now - started, 1)
  785. for key, started in _running_since.items()
  786. },
  787. "forced_releases": _forced_release_count,
  788. "recent_forced_releases": list(_forced_releases)}
  789. def _record_forced_release(job_id: str, name: str, age_seconds: float, allowance_seconds: float) -> None:
  790. """Persist a countable signal for one forced release (best-effort)."""
  791. entry = {
  792. "job_id": job_id,
  793. "name": name,
  794. "age_seconds": round(age_seconds, 1),
  795. "allowance_seconds": round(allowance_seconds, 1),
  796. "at": _hermes_now().isoformat()}
  797. with _running_lock:
  798. _forced_releases.append(entry)
  799. del _forced_releases[:-_FORCED_RELEASE_HISTORY]
  800. try:
  801. path = _get_hermes_home() / "cron" / "inflight_forced_releases.jsonl"
  802. _ensure_cron_dir(path.parent)
  803. with open(path, "a", encoding="utf-8") as fh:
  804. fh.write(json.dumps(entry) + "\n")
  805. except Exception as e: # never let telemetry break a tick
  806. logger.debug("Could not append forced-release record: %s", e)
  807. def _latest_executions_for_releasable_claims() -> dict:
  808. """Latest durable execution per releasable-looking claim (missing/pending/done future), one
  809. indexed query so the healthy path pays no DB work. Snapshot under _running_lock — iterating
  810. the set while try_register/release mutate it raises RuntimeError.
  811. Scoped to the ACTIVE profile's claims: the executions ledger it queries is that home's.
  812. """
  813. local_home = hermes_home_key(_get_hermes_home())
  814. with _running_lock:
  815. claim_futures = {
  816. key[1]: _running_futures.get(key)
  817. for key in _running_job_ids if key[0] == local_home
  818. }
  819. candidates = [
  820. job_id for job_id, fut in claim_futures.items()
  821. if fut is None or fut is _FUTURE_PENDING or fut.done()
  822. ]
  823. if not candidates:
  824. return {}
  825. try:
  826. from cron.executions import latest_executions as _latest_execs
  827. return _latest_execs(candidates)
  828. except Exception:
  829. return {}
  830. def _row_belongs_to_claim(row: dict, claim_started: float) -> bool:
  831. """True when the ledger row was claimed at/after this in-memory claim. An older terminal row is
  832. the PREVIOUS run's (try_register->create_execution window); releasing on it would
  833. double-dispatch. Unparseable timestamps fail closed (the age path still bounds the claim)."""
  834. claimed_at = row.get("claimed_at")
  835. if not claimed_at:
  836. return False
  837. try:
  838. from cron.jobs import _ensure_aware as _ensure_aware_ts
  839. row_ts = _ensure_aware_ts(datetime.fromisoformat(claimed_at))
  840. return row_ts.timestamp() >= claim_started
  841. except (ValueError, TypeError, OSError):
  842. return False
  843. def _record_stale_release(job: dict, job_id: str, age: float, allowance: float, fut, reason: str) -> None:
  844. """WARNING log + probe record for one forced release, then ``last_error`` unless the ledger
  845. already holds the run's real outcome or the job has a finite repeat budget."""
  846. name = job.get("name") or job_id
  847. future_state = "pending" if fut is _FUTURE_PENDING else "missing" if fut is None else "finished"
  848. logger.warning(
  849. "cron.inflight.forced_release event=forced_release reason=%s job='%s' "
  850. "id=%s age=%.0fs allowance=%.0fs future=%s — stale in-flight claim "
  851. "released; the job was skipping every fire with 'already running'",
  852. reason, name, job_id, age, allowance, future_state)
  853. _record_forced_release(job_id, name, age, allowance)
  854. # Ledger already records how the run ended: mark_job_run here would clobber an honest
  855. # ok status with a synthetic failure or double-write a failure.
  856. if reason == "ledger-terminal":
  857. return
  858. # Age release may lack a ledger row, so last_error is how it surfaces. But a forced release
  859. # is NOT a real run: never consume a finite repeat budget or let mark_job_run auto-delete.
  860. repeat = job.get("repeat") or {}
  861. if isinstance(repeat, dict) and repeat.get("times") is not None:
  862. logger.warning(
  863. "cron.inflight.forced_release.job_untouched job='%s' id=%s — "
  864. "finite-repeat job released without mark_job_run (repeat budget "
  865. "preserved); row left in place so it re-fires normally",
  866. name, job_id)
  867. return
  868. try:
  869. mark_job_run(
  870. job_id, False,
  871. f"Stale in-flight claim force-released after {age / 60:.1f}m "
  872. f"(allowance {allowance / 60:.1f}m); previous run never released "
  873. f"the scheduler in-flight guard")
  874. except Exception as e:
  875. logger.warning("Could not record forced release for job %s: %s", job_id, e)
  876. def sweep_stale_inflight(due_jobs: Optional[list] = None) -> list:
  877. """Force-release in-flight claims that can no longer be making progress; returns released ids.
  878. Stale = older than ``max(2 * interval, floor)`` AND (no live future — submit path hung before
  879. ``pool.submit`` returned — or finished without discarding the id) — or the claim's OWN ledger
  880. row is terminal regardless of age. Each release logs WARNING ``event=forced_release``, bumps
  881. the probe counter, mirrors JSONL, and writes ``last_error``.
  882. """
  883. global _forced_release_count
  884. by_id = {j.get("id"): j for j in (due_jobs or []) if isinstance(j, dict)}
  885. floor_seconds = _inflight_min_allowance_minutes() * 60.0
  886. now = time.time()
  887. stale: list = []
  888. from cron.executions import _TERMINAL_STATES as _terminal_states
  889. _latest = _latest_executions_for_releasable_claims()
  890. _local_home = hermes_home_key(_get_hermes_home())
  891. # Compute intervals OUTSIDE _running_lock so croniter doesn't block try_register/release.
  892. _intervals = {jid: _job_interval_minutes(j) for jid, j in by_id.items()}
  893. with _running_lock:
  894. for key in [k for k in _running_job_ids if k[0] == _local_home]:
  895. job_id = key[1]
  896. started = _running_since.get(key)
  897. if started is None:
  898. # Claim predates this guard — adopt it; sweepable one allowance from now.
  899. _running_since[key] = now
  900. continue
  901. age = now - started
  902. interval_minutes = _intervals.get(job_id)
  903. allowance = floor_seconds
  904. if interval_minutes:
  905. allowance = max(allowance, 2.0 * interval_minutes * 60.0)
  906. fut = _running_futures.get(key)
  907. if fut is _FUTURE_PENDING:
  908. # Submit path hung before ``pool.submit`` returned — the wedge class; release it.
  909. pass
  910. elif fut is not None and not fut.done():
  911. continue # genuinely still executing
  912. # Ledger reconciliation: a terminal row belonging to THIS claim proves it stale even
  913. # inside its age allowance. Row must be this claim's, else a recurring job's previous
  914. # run would double-dispatch a fresh claim.
  915. latest = _latest.get(job_id)
  916. if (
  917. latest is not None
  918. and latest.get("status") in _terminal_states
  919. and _row_belongs_to_claim(latest, started)
  920. ):
  921. reason = "ledger-terminal"
  922. elif age >= allowance:
  923. reason = "age"
  924. else:
  925. continue
  926. _running_registration_owners.pop(key, None)
  927. _running_job_ids.discard(key)
  928. _running_since.pop(key, None)
  929. _running_allowance_s.pop(key, None)
  930. _running_futures.pop(key, None)
  931. _forced_release_count += 1
  932. stale.append((job_id, age, allowance, fut, reason))
  933. for job_id, age, allowance, fut, _reason in stale:
  934. _record_stale_release(by_id.get(job_id) or {}, job_id, age, allowance, fut, _reason)
  935. return [s[0] for s in stale]
  936. def mark_running_jobs_interrupted(
  937. reason: str, *, only_owners: Optional[set] = None,
  938. ) -> list:
  939. """Best-effort: mark every in-flight cron job interrupted; returns the job IDs marked.
  940. Called by gateway shutdown right after ``process_registry.kill_all()``: a job whose tool was
  941. killed must never report success. ``only_owners`` (``(job_id, fire_owner)`` pairs) restricts
  942. marking. Tokens go into ``_interrupted_job_ids`` BEFORE ``last_status`` is written so
  943. ``run_one_job`` sees them.
  944. """
  945. with _running_lock:
  946. restart_safe_waiters = set(_restart_safe_waiter_job_ids)
  947. active_fires = [
  948. (token, key, owner, profile_home)
  949. for key, executions in _running_fire_owners.items()
  950. if key not in restart_safe_waiters
  951. for token, (owner, profile_home) in executions.items()
  952. ]
  953. if only_owners is not None:
  954. active_fires = [fire for fire in active_fires if (fire[1][1], fire[2]) in only_owners]
  955. registered_keys = {key for _t, key, _o, _p in active_fires}
  956. if only_owners is None:
  957. # The key's home half IS the profile home this claim belongs to — the only record of
  958. # it for a claim that never reached ``_running_fire_owners``. Read the real Path back
  959. # from the key, never ``Path(key[0])``: the key is normcased.
  960. active_fires.extend(
  961. (None, key, None, _inflight_home_path(key[0]))
  962. for key in (
  963. _running_job_ids - registered_keys - restart_safe_waiters
  964. )
  965. )
  966. _interrupted_job_ids.update(
  967. token if token is not None else key
  968. for token, key, _owner, _profile_home in active_fires
  969. )
  970. marked = []
  971. for _token, key, fire_owner, profile_home in active_fires:
  972. job_id = key[1]
  973. if not fire_owner:
  974. logger.warning(
  975. "Job '%s' interrupted before its durable fire owner was registered; "
  976. "leaving persisted state untouched",
  977. job_id)
  978. # Still report it: shutdown uses the returned IDs for the interrupted-cron notice. The
  979. # in-memory flag WAS recorded above; only the persisted last_status write is skipped.
  980. # See #82232.
  981. marked.append(job_id)
  982. continue
  983. try:
  984. with use_cron_store(profile_home):
  985. if mark_job_run(
  986. job_id, False, reason, expected_fire_owner=fire_owner):
  987. marked.append(job_id)
  988. except Exception as e:
  989. logger.warning("Failed to mark job %s interrupted: %s", job_id, e)
  990. return marked
  991. def _is_interrupted(job_id: str, token: Optional[object] = None) -> bool:
  992. """Non-destructive peek: has shutdown marked THIS execution interrupted? Used before deciding
  993. what to deliver; does not clear the flag (the authoritative pre-``last_status`` check needs it).
  994. ``token`` scopes to one execution so a fresh run reusing the job ID isn't poisoned."""
  995. key = _inflight_key(job_id)
  996. with _running_lock:
  997. if token is not None and token in _interrupted_job_ids:
  998. return True
  999. return key in _interrupted_job_ids
  1000. def _consume_interrupted_flag(job_id: str, token: Optional[object] = None) -> bool:
  1001. """Return True and clear the flag if shutdown marked THIS execution interrupted. Called right
  1002. before ``last_status`` is written; consuming stops the flag leaking into a later run."""
  1003. key = _inflight_key(job_id)
  1004. with _running_lock:
  1005. hit = False
  1006. if token is not None and token in _interrupted_job_ids:
  1007. _interrupted_job_ids.discard(token)
  1008. hit = True
  1009. if key in _interrupted_job_ids:
  1010. _interrupted_job_ids.discard(key)
  1011. hit = True
  1012. return hit
  1013. def _cron_inactivity_seconds() -> float:
  1014. """Parse HERMES_CRON_TIMEOUT (seconds). 0 = unlimited; bad input = 600. Shared by the
  1015. inactivity monitor and the cwd-lock bound so they can't drift: the lock bound must stay >= the
  1016. inactivity limit or waiters fail while a healthy holder runs."""
  1017. raw = cron_env_setting("HERMES_CRON_TIMEOUT").strip()
  1018. if not raw:
  1019. return 600.0
  1020. try:
  1021. return float(raw)
  1022. except (ValueError, TypeError):
  1023. logger.warning("Invalid HERMES_CRON_TIMEOUT=%r; using default 600s", raw)
  1024. return 600.0
  1025. def _get_parallel_pool(max_workers: Optional[int]) -> concurrent.futures.ThreadPoolExecutor:
  1026. """Return (or create) the persistent parallel pool for the ACTIVE profile.
  1027. Pools are per profile home because ``cron.max_parallel_jobs`` is a per-profile config key:
  1028. a single process-global pool is created by whichever profile the multiplexing gateway ticks
  1029. first and then silently imposes that profile's limit on every other profile.
  1030. """
  1031. home = hermes_home_key(_remember_inflight_home(_get_hermes_home()))
  1032. pool = _parallel_pools.get(home)
  1033. if pool is None or _parallel_pool_max_workers.get(home) != max_workers:
  1034. if pool is not None:
  1035. pool.shutdown(wait=False, cancel_futures=False)
  1036. pool = concurrent.futures.ThreadPoolExecutor(
  1037. max_workers=max_workers, thread_name_prefix="cron-parallel")
  1038. _parallel_pools[home] = pool
  1039. _parallel_pool_max_workers[home] = max_workers
  1040. return pool
  1041. def discard_parallel_pools(home_keys) -> None:
  1042. """Drop the parallel pools of homes this process no longer ticks.
  1043. Pools are per home and used to live until ``atexit``: a host that serves many profiles — or
  1044. churns them — accumulated one ThreadPoolExecutor and its live ``cron-parallel`` threads per
  1045. home ever ticked, and a deleted profile's pool was never reclaimed. ``wait=False`` so the
  1046. ticker is never blocked; queued work still drains before the executor dies.
  1047. """
  1048. for key in list(home_keys):
  1049. pool = _parallel_pools.pop(key, None)
  1050. _parallel_pool_max_workers.pop(key, None)
  1051. if pool is not None:
  1052. pool.shutdown(wait=False, cancel_futures=False)
  1053. def _shutdown_parallel_pool() -> None:
  1054. """Shut down every profile's persistent pool on process exit."""
  1055. for pool in list(_parallel_pools.values()):
  1056. pool.shutdown(wait=True, cancel_futures=False)
  1057. _parallel_pools.clear()
  1058. _parallel_pool_max_workers.clear()
  1059. atexit.register(_shutdown_parallel_pool)
  1060. # Per-fire usage audit log; resolves via _get_hermes_home() so profile-scoped paths work.
  1061. def _usage_audit_path() -> Path:
  1062. return _get_hermes_home() / "cron" / "usage_audit.jsonl"
  1063. def _utcnow_iso_ms() -> str:
  1064. """RFC3339 UTC timestamp with millisecond precision and 'Z' suffix."""
  1065. now = datetime.now(UTC)
  1066. return now.strftime("%Y-%m-%dT%H:%M:%S.") + f"{now.microsecond // 1000:03d}Z"
  1067. def _write_usage_audit(record: dict) -> None:
  1068. """Append one JSONL line to cron/usage_audit.jsonl. NEVER raises — a logger bug must not
  1069. break cron jobs (the whole write is inside one try)."""
  1070. try:
  1071. path = _usage_audit_path()
  1072. _ensure_cron_dir(path.parent)
  1073. line = json.dumps(record, ensure_ascii=False)
  1074. with open(path, "a", encoding="utf-8") as f:
  1075. f.write(line + "\n")
  1076. except Exception as e:
  1077. logger.warning("usage_audit write failed: %s", e)
  1078. def _interpreter_shutting_down(exc: Optional[BaseException] = None) -> bool:
  1079. """True when the interpreter is finalizing (tick fired during gateway teardown): concurrent.
  1080. futures/asyncio refuse new work, so delivery attempts only pollute errors.log — callers skip
  1081. with a warning. ``exc`` lets an already-raised scheduling error count as a shutdown signal.
  1082. A cron tick can fire while the gateway is tearing down — SIGTERM from ``hermes update`` / ``hermes
  1083. gateway stop`` / systemd restart, or an OOM-kill. Once finalization starts, ``concurrent.futures``
  1084. refuses new work with ``RuntimeError: cannot schedule new futures after interpreter shutdown`` and
  1085. asyncio's default executor is gone, so *any* attempt to schedule delivery (live-adapter,
  1086. ``asyncio.run``, or a fresh pool) is doomed and only pollutes ``errors.log`` with a traceback. See
  1087. #55924, #58720.
  1088. """
  1089. from tools.interpreter_shutdown import interpreter_shutting_down
  1090. return interpreter_shutting_down(exc)
  1091. # Module override hook for tests / emergency monkeypatches.
  1092. _hermes_home: Path | None = None
  1093. def _get_hermes_home() -> Path:
  1094. """Hermes home at call time (honouring the test override). Cron is per-profile: never freeze
  1095. this at import or anchor it at the shared default root — either breaks profile isolation.
  1096. Cron is per-profile by design (#4707): the in-process ticker runs inside a profile-scoped gateway, so
  1097. resolving the active HERMES_HOME at call time means a profile's jobs are stored AND executed under that
  1098. profile's home (its .env, config.yaml, scripts, skills).
  1099. """
  1100. return _hermes_home or get_hermes_home()
  1101. def _get_lock_paths() -> tuple[Path, Path]:
  1102. """Resolve cron lock paths at call time so profile/env changes are honored."""
  1103. hermes_home = _get_hermes_home()
  1104. lock_dir = hermes_home / "cron"
  1105. return lock_dir, lock_dir / ".tick.lock"
  1106. # Errnos that mean "another ticker (or manual tick) holds the tick lock", as opposed to a real failure
  1107. # opening/locking the file. Everything else — most importantly EMFILE/ENFILE (fd exhaustion, #87644) and
  1108. # EACCES on open() — must be surfaced, never swallowed as lock contention.
  1109. def _is_lock_contention_errno(err: OSError) -> bool:
  1110. """True when *err* from the lock syscall means another ticker holds the lock (POSIX flock:
  1111. EWOULDBLOCK/EAGAIN, EACCES on some NFS; msvcrt.locking: EACCES/EDEADLK). Everything else —
  1112. notably EMFILE/ENFILE and EACCES on open() — must be surfaced, never swallowed as contention."""
  1113. if err.errno is None:
  1114. return False
  1115. if fcntl is not None:
  1116. return err.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES)
  1117. if msvcrt is not None:
  1118. return err.errno in (errno.EACCES, errno.EDEADLK)
  1119. return False
  1120. def _is_fd_exhaustion_text(text: str) -> bool:
  1121. """Text half of _is_fd_exhaustion (shared with the CLI hint)."""
  1122. lowered = text.lower()
  1123. return "too many open files" in lowered or "emfile" in lowered
  1124. def _is_fd_exhaustion(exc: BaseException) -> bool:
  1125. """True when *exc* indicates fd exhaustion: EMFILE/ENFILE errno, or the "Too many open files"
  1126. wording for wrapped exceptions (load_jobs wraps the OSError in a RuntimeError).
  1127. See #87644.
  1128. """
  1129. if isinstance(exc, OSError) and exc.errno in (errno.EMFILE, errno.ENFILE):
  1130. return True
  1131. return _is_fd_exhaustion_text(str(exc))
  1132. def _reclaim_fds_best_effort() -> None:
  1133. """Best-effort fd reclamation: gc.collect() closes file objects stuck in reference cycles;
  1134. apply_nofile_soft_limit() raises the RLIMIT_NOFILE soft limit for headroom. Never raises.
  1135. The cron FD-leak family (#60859, #79742, #80792) leaks descriptors from abandoned workers/sessions. Two
  1136. safe, idempotent levers:
  1137. """
  1138. with contextlib.suppress(Exception):
  1139. import gc
  1140. gc.collect()
  1141. with contextlib.suppress(Exception):
  1142. from hermes_cli.resource_limits import apply_nofile_soft_limit
  1143. apply_nofile_soft_limit(None)
  1144. def drain_delivery_queue(adapters, loop) -> int:
  1145. """Send queued worker results through this gateway's live adapters."""
  1146. from cron.delivery_queue import _path, drain
  1147. # Only restart-safe workers create the queue file. Every gateway (macOS,
  1148. # Windows, launchd, Docker) runs this housekeeping tick, so skip the sqlite
  1149. # open/create entirely until a worker has actually queued something.
  1150. if not _path().exists():
  1151. return 0
  1152. def send(queued_job, queued_content, queued_for_failure):
  1153. error = _deliver_result(
  1154. queued_job, queued_content, adapters=adapters, loop=loop,
  1155. for_failure=queued_for_failure,
  1156. )
  1157. # The ping left the process: book the incident alerted. A notice every target hid
  1158. # (warning notifications suppressed) is a durable disposition, not an alert.
  1159. if (not error and queued_for_failure
  1160. and not queued_job.get("_notification_all_targets_suppressed")):
  1161. _mark_incident_alerted(queued_job.get("_failure_incident_id"))
  1162. return error
  1163. return drain(send)
  1164. _DEFAULT_SCRIPT_TIMEOUT = 3600 # seconds (1 hour)
  1165. # Backward-compatible module override used by tests and emergency monkeypatches.
  1166. _SCRIPT_TIMEOUT = _DEFAULT_SCRIPT_TIMEOUT
  1167. _RUN_CLAIM_HEARTBEAT_SECONDS = 60.0
  1168. _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS = _RUN_CLAIM_HEARTBEAT_SECONDS * 3
  1169. # Pause before re-sampling a fire-claim heartbeat miss; a genuinely re-owned claim misses twice.
  1170. _FIRE_CLAIM_MISS_CONFIRM_SECONDS = 1.0
  1171. def _cron_cleanup_timeout_seconds() -> float:
  1172. """Return the wall-clock bound for cron post-run cleanup."""
  1173. default = 10.0
  1174. try:
  1175. from hermes_cli.config import load_config
  1176. cfg = load_config() or {}
  1177. cron_cfg = cfg.get("cron", {}) if isinstance(cfg, dict) else {}
  1178. configured = cron_cfg.get("cleanup_timeout_seconds")
  1179. if configured is not None:
  1180. timeout = float(configured)
  1181. if timeout >= 0:
  1182. return timeout
  1183. except Exception as exc:
  1184. logger.debug("Failed to load cron cleanup timeout from config: %s", exc)
  1185. return default
  1186. def _run_cron_cleanup_with_timeout(
  1187. cleanup, *, job_id: str, label: str, timeout_seconds: Optional[float] = None,
  1188. ) -> bool:
  1189. """Run fallible post-run cleanup without permanently wedging a cron ID."""
  1190. timeout = (_cron_cleanup_timeout_seconds() if timeout_seconds is None else float(timeout_seconds))
  1191. if timeout <= 0:
  1192. try:
  1193. cleanup()
  1194. return True
  1195. except (Exception, KeyboardInterrupt) as exc:
  1196. logger.debug("Job '%s': %s failed: %s", job_id, label, exc)
  1197. return False
  1198. done = threading.Event()
  1199. error: list[BaseException] = []
  1200. def _runner() -> None:
  1201. try:
  1202. cleanup()
  1203. except BaseException as exc:
  1204. error.append(exc)
  1205. finally:
  1206. done.set()
  1207. # Daemon thread is deliberate: unlike ThreadPoolExecutor workers it is not joined at interpreter
  1208. # exit if cleanup never returns, so the gateway can still shut down.
  1209. worker = threading.Thread(
  1210. target=ctx_bound(_runner), name=f"cron-cleanup-{job_id}", daemon=True)
  1211. worker.start()
  1212. if not done.wait(timeout):
  1213. logger.error(
  1214. "Job '%s': %s exceeded %.1fs; abandoning cleanup so future runs remain dispatchable",
  1215. job_id,
  1216. label,
  1217. timeout)
  1218. return False
  1219. if error:
  1220. logger.debug("Job '%s': %s failed: %s", job_id, label, error[0])
  1221. return False
  1222. return True
  1223. class _BoundedCronSessionDB:
  1224. """Proxy SessionDB cleanup calls through the cron cleanup timeout; after the first failure or
  1225. timeout all later calls fail immediately (a damaged connection leaks at most one worker)."""
  1226. def __init__(self, session_db, job_id: str):
  1227. self._session_db = session_db
  1228. self._job_id = job_id
  1229. self._disabled = False
  1230. def __getattr__(self, name):
  1231. target = getattr(self._session_db, name)
  1232. if not callable(target):
  1233. return target
  1234. def _bounded(*args, **kwargs):
  1235. if self._disabled:
  1236. raise RuntimeError("session finalization disabled after prior cleanup failure")
  1237. result = {}
  1238. def _call():
  1239. try:
  1240. result["value"] = target(*args, **kwargs)
  1241. except BaseException as exc:
  1242. result["error"] = exc
  1243. raise
  1244. ok = _run_cron_cleanup_with_timeout(
  1245. _call, job_id=self._job_id, label=f"session finalization ({name})")
  1246. if not ok:
  1247. error = result.get("error")
  1248. if error is not None:
  1249. raise error
  1250. # No error yet not complete == timeout: disable so later steps fail fast.
  1251. self._disabled = True
  1252. raise TimeoutError(f"session finalization method {name} timed out")
  1253. return result.get("value")
  1254. return _bounded
  1255. def _resolve_job_workdir(job: dict, job_id: str) -> Optional[str]:
  1256. """Configured job workdir, or None when unset / no longer a directory (logged)."""
  1257. workdir = (job.get("workdir") or "").strip() or None
  1258. if workdir and not Path(workdir).is_dir():
  1259. logger.warning(
  1260. "Job '%s': configured workdir %r no longer exists — running without it",
  1261. job_id, workdir)
  1262. return None
  1263. return workdir
  1264. def _run_no_agent_job(
  1265. job: dict, job_id: str, job_name: str, cancel_event,
  1266. ) -> tuple[bool, str, str, Optional[str]]:
  1267. """no_agent short-circuit — the script IS the job (no AIAgent, no tokens). stdout → delivered
  1268. verbatim; empty stdout or wakeAgent=false → silent success; non-zero exit/timeout → error alert.
  1269. """
  1270. # Load .env first so auto-delivery can resolve *_HOME_CHANNEL: the agent path's per-run dotenv
  1271. # reload never runs for no_agent jobs. Does not override existing values.
  1272. try:
  1273. from hermes_cli.env_loader import load_hermes_dotenv
  1274. load_hermes_dotenv(hermes_home=_get_hermes_home())
  1275. except Exception:
  1276. logger.debug("Job '%s': no_agent .env reload failed", job_id, exc_info=True)
  1277. script_path = job.get("script")
  1278. # Legacy/hand-edited no_agent job without a script: pause it, or it re-fires every tick.
  1279. if not str(script_path or "").strip():
  1280. from cron.jobs_invariants import NO_AGENT_WITHOUT_SCRIPT_ERROR
  1281. return _block_and_pause_job(job_id, job_name, NO_AGENT_WITHOUT_SCRIPT_ERROR)
  1282. # Pass workdir as subprocess cwd; never os.chdir() (leaks into concurrent gateway sessions).
  1283. _job_workdir = _resolve_job_workdir(job, job_id)
  1284. try:
  1285. ok, output = _run_job_script_with_claim_heartbeat(
  1286. job, script_path, workdir=_job_workdir, cancel_event=cancel_event)
  1287. except Exception as exc:
  1288. logger.exception("Job '%s': script execution raised unexpectedly", job_id)
  1289. ok, output = False, f"Script execution failed: {exc}"
  1290. now_iso = _hermes_now().strftime("%Y-%m-%d %H:%M:%S")
  1291. header = _job_doc_header(job_name, job_id, now_iso, "no_agent (script)")
  1292. if not ok:
  1293. # Deliver the error: a silently broken watchdog is the worst-case outcome.
  1294. alert = (
  1295. f"⚠ Cron watchdog '{job_name}' script failed\n\n"
  1296. f"{output}\n\n"
  1297. f"Time: {now_iso}"
  1298. )
  1299. return False, f"{header}**Status:** script failed\n\n{output}\n", alert, output
  1300. # wakeAgent=false is a silent signal, same as empty stdout.
  1301. if not _parse_wake_gate(output):
  1302. logger.info("Job '%s' (no_agent): wakeAgent=false gate — silent run", job_id)
  1303. return True, f"{header}**Status:** silent (wakeAgent=false)\n", SILENT_MARKER, None
  1304. if not output.strip():
  1305. logger.info("Job '%s' (no_agent): empty stdout — silent run", job_id)
  1306. return True, f"{header}**Status:** silent (empty output)\n", SILENT_MARKER, None
  1307. return True, f"{header}\n---\n\n{output}\n", output, None
  1308. def _apply_monitor_gate(
  1309. job: dict, job_id: str, job_name: str, extra_prompt: Optional[str],
  1310. ) -> tuple[Optional[tuple], Optional[str], Optional[str]]:
  1311. """Monitor gate (hash-suppressed change detection). Must run BEFORE any agent machinery so an
  1312. unchanged tick costs no LLM/delivery. Returns ``(early_result | None, extra_prompt,
  1313. monitor_context)``. Monitor context is runtime data and must remain distinct from a
  1314. user-authored ``extra_prompt`` so the prompt scanner keeps its strict user-input boundary.
  1315. """
  1316. from cron.monitor import check_monitor, job_has_monitor
  1317. if not job_has_monitor(job):
  1318. return None, extra_prompt, None
  1319. _mon = check_monitor(job)
  1320. _mon_now = _hermes_now().strftime("%Y-%m-%d %H:%M:%S")
  1321. header = _job_doc_header(job_name, job_id, _mon_now, "monitor")
  1322. if not _mon.ok:
  1323. # Source failure is an ERROR, never a change: alert so a broken monitor can't silently
  1324. # stop watching. Stored hash untouched.
  1325. logger.error("Job '%s': monitor source failed: %s", job_id, _mon.error)
  1326. _mon_alert = (
  1327. f"⚠ Cron monitor '{job_name}' source failed\n\n"
  1328. f"{_mon.error}\n\n"
  1329. f"Time: {_mon_now}"
  1330. )
  1331. return (
  1332. False, f"{header}**Status:** monitor source failed\n\n{_mon.error}\n", _mon_alert, _mon.error,
  1333. ), extra_prompt, None
  1334. if not _mon.changed:
  1335. # Unchanged: silent no_change tick (ledger doc kept; SILENT_MARKER blocks delivery).
  1336. logger.info("Job '%s': monitor output unchanged — suppressing agent run", job_id)
  1337. return (
  1338. True, f"{header}**Status:** no_change (agent run suppressed)\n", SILENT_MARKER, None,
  1339. ), extra_prompt, None
  1340. # Changed (or first run): pass monitor output through the runtime-data seam. Keep any manual
  1341. # per-run prompt separate: it remains user input and is therefore still strict-scanned.
  1342. return None, extra_prompt, _mon.context_block
  1343. @dataclass
  1344. class _CronJobConfig:
  1345. """Config-derived inputs for one agent-backed cron run."""
  1346. cfg: dict
  1347. model: str
  1348. model_cfg: Any
  1349. cron_default_provider: str
  1350. def _load_cron_job_config(job: dict, job_id: str, job_name: str) -> _CronJobConfig:
  1351. """Load config.yaml and resolve the run's model: per-job pin > cron.model (fleet default) >
  1352. the main agent model (config ``model:``, then HERMES_MODEL) > the claimed welcome trial's. Re-read
  1353. every tick (no cache) so ``hermes cron edit --model`` and ``hermes model`` both apply next tick."""
  1354. from hermes_cli.anon_auth import trial_model
  1355. model = job.get("model") or cron_env_setting("HERMES_MODEL") or ""
  1356. _cron_default_provider = ""
  1357. _cfg: dict = {}
  1358. _model_cfg: Any = {}
  1359. try:
  1360. from hermes_cli.config_backend import config_exists
  1361. from hermes_cli.config_effective import load_user_config_effective
  1362. _cfg_path = str(_get_hermes_home() / "config.yaml")
  1363. if config_exists(_cfg_path):
  1364. _cfg = load_user_config_effective(Path(_cfg_path))
  1365. # Coerce null to {} so a falsy default never clobbers a resolved env value.
  1366. _model_cfg = _cfg.get("model") or {}
  1367. _cron_cfg_for_model = _cfg.get("cron") or {}
  1368. _cron_default_model = ""
  1369. if isinstance(_cron_cfg_for_model, dict):
  1370. _cron_default_model = str(_cron_cfg_for_model.get("model") or "").strip()
  1371. _cron_default_provider = str(_cron_cfg_for_model.get("model_provider") or "").strip()
  1372. if not job.get("model"):
  1373. if _cron_default_model:
  1374. model = _cron_default_model
  1375. else:
  1376. # The main agent model: ``model: <name>`` shorthand or ``model.default``.
  1377. _main = _model_cfg if isinstance(_model_cfg, str) else (
  1378. _model_cfg.get("default") or _model_cfg.get("model") or _model_cfg.get("name")
  1379. if isinstance(_model_cfg, dict) else "")
  1380. model = str(_main or "").strip() or model
  1381. except Exception as e: # ConfigBackendUnavailable is not one: run_job fails this fire, never defaults
  1382. logger.warning("Job '%s': failed to load config.yaml, using defaults: %s", job_id, e)
  1383. model = (model if isinstance(model, str) else "").strip() or trial_model()
  1384. # Fail fast: an empty model otherwise reaches the provider as an opaque 400 (#23979).
  1385. if not model:
  1386. raise RuntimeError(
  1387. f"Cron job '{job_name}' has no model configured "
  1388. f"(job.model={job.get('model')!r}, "
  1389. f"HERMES_MODEL={cron_env_setting('HERMES_MODEL')!r}, "
  1390. "config.yaml model.default missing or empty). "
  1391. f"Set a per-job model via "
  1392. f"`hermes cron edit {job_id} --model <name>` or set a "
  1393. "default with `hermes model <name>`."
  1394. )
  1395. with contextlib.suppress(Exception):
  1396. from hermes_constants import apply_ipv4_preference
  1397. _net_cfg = _cfg.get("network", {})
  1398. if isinstance(_net_cfg, dict) and _net_cfg.get("force_ipv4"):
  1399. apply_ipv4_preference(force=True)
  1400. return _CronJobConfig(_cfg, model, _model_cfg, _cron_default_provider)
  1401. def _load_prefill_messages(cfg: dict, job_id: str) -> Optional[list]:
  1402. """Prefill messages from env or config.yaml (top-level key canonical; agent.* is legacy)."""
  1403. agent_cfg = cfg.get("agent", {}) if isinstance(cfg.get("agent", {}), dict) else {}
  1404. prefill_file = (
  1405. cron_env_setting("HERMES_PREFILL_MESSAGES_FILE")
  1406. or cfg.get("prefill_messages_file", "")
  1407. or agent_cfg.get("prefill_messages_file", "")
  1408. )
  1409. if not prefill_file:
  1410. return None
  1411. pfpath = Path(prefill_file).expanduser()
  1412. if not pfpath.is_absolute():
  1413. pfpath = _get_hermes_home() / pfpath
  1414. if not pfpath.exists():
  1415. return None
  1416. try:
  1417. with open(pfpath, "r", encoding="utf-8-sig") as _pf:
  1418. prefill_messages = json.load(_pf)
  1419. return prefill_messages if isinstance(prefill_messages, list) else None
  1420. except Exception as e:
  1421. logger.warning("Job '%s': failed to parse prefill messages file '%s': %s", job_id, pfpath, e)
  1422. return None
  1423. def _preflight_or_block(job: dict, job_id: str, job_name: str, cfg: dict) -> Optional[tuple]:
  1424. """Pre-dispatch config validation: refuse unrunnable jobs (missing key, unready skill,
  1425. unconfigured delivery) BEFORE AIAgent is built. run_one_job keys off BLOCKED_CONFIG_MARKER to
  1426. record blocked_config and alert once (`preflight_alerted` bit). Must run after the wake gate so
  1427. silent ticks stay silent. Opt-out: `cron.preflight: false`. Returns failure tuple or None.
  1428. """
  1429. # --------------------------------------------------------------- Pre-dispatch configuration validation
  1430. # (T1-26). A job whose configuration cannot possibly produce a successful run — missing provider API key
  1431. # (no fallback chain), unready attached skill, unconfigured delivery platform — is refused HERE, before
  1432. # AIAgent is constructed and before the resolution below can feed a doomed runtime into it, so a
  1433. # misconfigured job never burns an LLM call. run_one_job keys off the BLOCKED_CONFIG_MARKER in the
  1434. # returned error to record last_status='blocked_config' and alert exactly once (dedup persisted via the
  1435. # job's `preflight_alerted` bit — the #73506 alert-once shape).
  1436. _pf_reason = None
  1437. try:
  1438. if _cron_preflight_enabled(cfg):
  1439. _pf_reason = _preflight_job_config(job, cfg)
  1440. if not _pf_reason and job.get("preflight_alerted"):
  1441. # Config healthy again: clear alert-once marker so a future break re-alerts.
  1442. with contextlib.suppress(Exception):
  1443. from cron.jobs import clear_preflight_alerted
  1444. clear_preflight_alerted(job_id)
  1445. except Exception:
  1446. # Fail open: the validator must never take down a runnable job.
  1447. logger.debug("Job '%s': preflight validation errored — failing open", job_id, exc_info=True)
  1448. _pf_reason = None
  1449. if not _pf_reason:
  1450. return None
  1451. return _blocked_config_result(job_id, job_name, _pf_reason)
  1452. def _blocked_config_result(job_id: str, job_name: str, _pf_reason: str) -> tuple:
  1453. """The ``blocked_config`` failure tuple for *_pf_reason*, alerting once per job."""
  1454. logger.warning(
  1455. "Job '%s' (ID: %s): BLOCKED by pre-dispatch config validation — %s (no LLM call was made)",
  1456. job_name, job_id, _pf_reason)
  1457. already_alerted = False
  1458. try:
  1459. from cron.jobs import mark_preflight_alerted
  1460. already_alerted = mark_preflight_alerted(job_id)
  1461. except Exception:
  1462. logger.debug("Job '%s': could not persist preflight alert marker", job_id, exc_info=True)
  1463. marker = BLOCKED_CONFIG_SILENT_MARKER if already_alerted else BLOCKED_CONFIG_MARKER
  1464. blocked_doc = (
  1465. f"# Cron Job: {job_name}\n\n"
  1466. f"**Job ID:** {job_id}\n"
  1467. f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
  1468. f"**Status:** BLOCKED (configuration)\n\n"
  1469. "The pre-run configuration check found a problem, so the agent did not run "
  1470. "(nothing was charged).\n\n"
  1471. f"**Reason:** {_pf_reason}\n\n"
  1472. "Hermes tries again at the next scheduled time and clears this state on the first healthy "
  1473. "run; this alert is not repeated. Check with `hermes cron doctor`. Set `cron.preflight: "
  1474. "false` in config.yaml to disable this check."
  1475. )
  1476. return False, blocked_doc, "", f"{marker} {_pf_reason}"
  1477. def _resolve_job_runtime(job: dict, job_id: str, jc: _CronJobConfig) -> tuple[dict, str]:
  1478. """Resolve the runtime, walking the fallback chain on auth/transient-network errors. Returns
  1479. ``(runtime, model)``; provider+model swap atomically (never swap only the provider while keeping
  1480. a paid primary model). Provider precedence: per-job pin > cron.model_provider > persisted
  1481. global config (None lets resolve_runtime_provider read it). A pinned job has no chain here
  1482. (``_job_fallback_chain``): its resolve failure is the job's failure."""
  1483. from hermes_cli.runtime_provider import (
  1484. resolve_runtime_provider, format_runtime_provider_error)
  1485. from hermes_cli.auth import AuthError
  1486. model = jc.model
  1487. requested = job.get("provider") or jc.cron_default_provider or None
  1488. try:
  1489. # Do NOT pass HERMES_INFERENCE_PROVIDER as `requested`: it would override persisted config
  1490. # and resurrect stale providers for unpinned jobs.
  1491. runtime_kwargs = {
  1492. "requested": requested,
  1493. # api_mode must derive from the model actually run, not the stale persisted default.
  1494. "target_model": model,
  1495. }
  1496. if job.get("base_url"):
  1497. runtime_kwargs["explicit_base_url"] = job.get("base_url")
  1498. return resolve_runtime_provider(**runtime_kwargs), model
  1499. except Exception as resolve_exc:
  1500. # Walk the fallback chain on AuthError AND transient network/DNS failures (e.g. during
  1501. # OAuth refresh); anything else re-raises.
  1502. is_auth = isinstance(resolve_exc, AuthError)
  1503. is_transient_net = _is_transient_provider_resolve_error(resolve_exc)
  1504. if not (is_auth or is_transient_net):
  1505. raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc
  1506. chain = _job_fallback_chain(job, jc.cfg) or []
  1507. logger.warning(
  1508. "Job '%s': primary provider resolve failed (%s: %s), %s",
  1509. job_id, "auth" if is_auth else "transient network", resolve_exc,
  1510. "trying fallback" if chain else (
  1511. "not falling back: the job is pinned" if _job_route_pinned(job) else "no fallback configured"))
  1512. for entry in chain:
  1513. if not isinstance(entry, dict):
  1514. continue
  1515. fb_provider = str(entry.get("provider") or "").strip()
  1516. fb_model = str(entry.get("model") or "").strip()
  1517. if not fb_provider or not fb_model:
  1518. continue
  1519. try:
  1520. from hermes_cli.fallback_config import effective_runtime_provider, resolve_entry_api_key
  1521. fb_kwargs = {"requested": fb_provider, "target_model": fb_model}
  1522. if entry.get("base_url"):
  1523. fb_kwargs["explicit_base_url"] = entry["base_url"]
  1524. fb_api_key = resolve_entry_api_key(entry)
  1525. if fb_api_key:
  1526. fb_kwargs["explicit_api_key"] = fb_api_key
  1527. runtime = resolve_runtime_provider(**fb_kwargs)
  1528. # Named custom entries resolve to the bare "custom" billing class; keep the configured
  1529. # identity so job sessions record the provider name (#98739).
  1530. runtime["provider"] = effective_runtime_provider(entry, runtime)
  1531. logger.info(
  1532. "Job '%s': fallback resolved to %s model %s",
  1533. job_id, runtime.get("provider"), fb_model)
  1534. # Delivered with the job output (#74349): a cron agent has no status rail, so the
  1535. # switch would otherwise stay in the scheduler log only. run_job pops it.
  1536. from hermes_cli.fallback_config import pre_agent_fallback_notice
  1537. runtime["_fallback_notice"] = pre_agent_fallback_notice(
  1538. requested or (jc.model_cfg.get("provider") if isinstance(jc.model_cfg, dict) else ""),
  1539. model, runtime.get("provider"), fb_model)
  1540. return runtime, fb_model
  1541. except Exception as fb_exc:
  1542. logger.debug("Job '%s': fallback %s failed: %s", job_id, fb_provider, fb_exc)
  1543. raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc
  1544. def _load_credential_pool(runtime: dict, job_id: str):
  1545. runtime_provider = str(runtime.get("provider") or "").strip().lower()
  1546. if not runtime_provider:
  1547. return None
  1548. try:
  1549. from agent.credential_pool import load_pool
  1550. pool = load_pool(runtime_provider)
  1551. if pool.has_credentials():
  1552. logger.info(
  1553. "Job '%s': loaded credential pool for provider %s with %d entries",
  1554. job_id, runtime_provider, len(pool.entries()))
  1555. return pool
  1556. except Exception as e:
  1557. logger.debug("Job '%s': failed to load credential pool for %s: %s", job_id, runtime_provider, e)
  1558. return None
  1559. def _init_cron_mcp_tools(job_id: str) -> None:
  1560. """Register MCP servers for the agent's tool registry. Idempotent across ticks; non-fatal so a
  1561. broken MCP server never kills a working job."""
  1562. try:
  1563. # Initialize MCP servers so configured mcp_servers are available to the agent's tool registry before
  1564. # AIAgent is constructed. Without this, cron jobs never saw any MCP tools — only the gateway / CLI
  1565. # paths called discover_mcp_tools() at startup. Idempotent: subsequent ticks short-circuit on
  1566. # already-connected servers inside register_mcp_servers(). Non-fatal on failure: a broken MCP server
  1567. # shouldn't kill an otherwise-working cron job. See #4219.
  1568. from tools.mcp_tool_discovery import discover_mcp_tools
  1569. _mcp_tools = discover_mcp_tools()
  1570. if _mcp_tools:
  1571. logger.info("Job '%s': %d MCP tool(s) available", job_id, len(_mcp_tools))
  1572. except Exception as _mcp_exc:
  1573. logger.warning("Job '%s': MCP initialization failed (non-fatal): %s", job_id, _mcp_exc)
  1574. def _open_cron_session_db(job: dict):
  1575. """Bound store acquisition separately from the conversation watchdog.
  1576. Canonical runs borrow the owner's exact store and fail closed on acquisition
  1577. errors. Only the legacy non-owner helper path may return no session store.
  1578. """
  1579. # Initialize the SQLite session store so cron job messages are persisted and discoverable via
  1580. # session_search (same pattern as gateway/run.py) — only now, after every early-return path (wake-gate,
  1581. # prompt validation, drift skip) has passed, so a gated run never opens state.db just to abandon the
  1582. # handle (#96290). Bounded with its own timeout (separate from HERMES_CRON_TIMEOUT, which only watches
  1583. # the agent's run_conversation below): SessionDB.__init__ opens/migrates state.db synchronously and has
  1584. # no timeout of its own against a wedged sqlite3.connect (e.g. a stale flock left by a crashed sibling
  1585. # process). An unbounded hang here would wedge the job's worker thread, so the init is bounded and a
  1586. # timeout proceeds without a session store instead of blocking the run forever.
  1587. from gateway.session_cron import current_execution
  1588. owner = current_execution()
  1589. from agent.runtime_session_store import WorkerPersistenceError, is_worker_process
  1590. if owner is None and is_worker_process():
  1591. # Do not turn an unsupported worker assignment into the legacy None
  1592. # fallback: that would run billed inference without durable persistence.
  1593. raise WorkerPersistenceError('worker_cron_registration_required')
  1594. _session_db_timeout = _get_session_db_timeout()
  1595. try:
  1596. from hermes_state_registry import acquire
  1597. def acquire_store():
  1598. if owner is None:
  1599. return acquire()
  1600. db = acquire(Path(owner[0].db.db_path))
  1601. if db is not owner[0].db:
  1602. from hermes_state_registry import release_or_close
  1603. release_or_close(db)
  1604. raise RuntimeError('cron requires the canonical owner store')
  1605. return db
  1606. if _session_db_timeout <= 0:
  1607. return acquire_store()
  1608. _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
  1609. # Copy the context so a profile run resolves ITS OWN home/state.db on the worker thread
  1610. # instead of the process-global default.
  1611. _session_db_context = contextvars.copy_context()
  1612. _session_db_future = _session_db_pool.submit(_session_db_context.run, acquire_store)
  1613. try:
  1614. return _session_db_future.result(timeout=_session_db_timeout)
  1615. except concurrent.futures.TimeoutError:
  1616. # The abandoned worker may still finish; close its late result or its SQLite FDs leak.
  1617. # The worker is abandoned (shutdown below doesn't wait for it). If SessionDB() later completes
  1618. # inside it, the future's result would be orphaned and its SQLite FDs (.db, WAL, SHM) leak until
  1619. # process exit. Register a done-callback that retrieves and closes any eventual late result
  1620. # (#72782).
  1621. from cron.scheduler_detached_worker import _close_late_session_db_result
  1622. _session_db_future.add_done_callback(_close_late_session_db_result)
  1623. raise
  1624. finally:
  1625. # Abandon a wedged connect() rather than blocking shutdown on it.
  1626. _session_db_pool.shutdown(wait=False)
  1627. except concurrent.futures.TimeoutError:
  1628. if owner is not None:
  1629. raise TimeoutError("Canonical cron session store acquisition timed out")
  1630. logger.error(
  1631. "Job '%s': SessionDB init did not return within %.0fs — proceeding "
  1632. "without a session store for this run instead of blocking it forever",
  1633. job.get("id", "?"), _session_db_timeout)
  1634. except Exception as e:
  1635. if owner is not None:
  1636. raise
  1637. logger.debug("Job '%s': SQLite session store not available: %s", job.get("id", "?"), e)
  1638. return None
  1639. def _raise_inactivity_timeout(agent, job_name: str, limit_s: float) -> None:
  1640. """Log the agent's last activity, hard-interrupt it and raise TimeoutError."""
  1641. _activity = {}
  1642. if hasattr(agent, "get_activity_summary"):
  1643. with contextlib.suppress(Exception):
  1644. _activity = agent.get_activity_summary()
  1645. _last_desc = _activity.get("last_activity_desc", "unknown")
  1646. _secs_ago = _activity.get("seconds_since_activity", 0)
  1647. logger.error(
  1648. "Job '%s' idle for %.0fs (inactivity limit %.0fs) "
  1649. "| last_activity=%s | iteration=%s/%s | tool=%s",
  1650. job_name, _secs_ago, limit_s,
  1651. _last_desc, _activity.get("api_call_count", 0), _activity.get("max_iterations", 0),
  1652. _activity.get("current_tool") or "none")
  1653. request_hard_interrupt(agent, "Cron job timed out (inactivity)", tool_reason="cron inactivity watchdog")
  1654. raise TimeoutError(
  1655. f"Cron job '{job_name}' idle for "
  1656. f"{int(_secs_ago)}s (limit {int(limit_s)}s) "
  1657. f"— last activity: {_last_desc}")
  1658. def _run_agent_with_watchdog(
  1659. agent, prompt: str, job: dict, job_id: str, job_name: str, task_id: str, cancel_event,
  1660. worker_state: Optional[dict] = None,
  1661. ) -> dict:
  1662. """Run ``agent.run_conversation`` on a worker thread under the inactivity (not wall-clock)
  1663. watchdog: default 600s, override HERMES_CRON_TIMEOUT, 0 = unlimited."""
  1664. _cron_timeout = _cron_inactivity_seconds()
  1665. _cron_inactivity_limit = _cron_timeout if _cron_timeout > 0 else None
  1666. _POLL_INTERVAL = 5.0
  1667. # Heartbeat the one-shot run_claim while alive: without it a long run looks like a dead owner
  1668. # and gets re-dispatched / stale-removed out from under the live run.
  1669. # Keep the one-shot run_claim fresh while the run is alive (#62002): the claim TTL is a dead-owner
  1670. # detector, but without a heartbeat a run that legitimately outlives it (stream stall, laptop asleep
  1671. # mid-run) is indistinguishable from a dead tick — another process re-dispatches it and get_due_jobs
  1672. # stale-removes the job record out from under the live run. Refreshing the claim from this monitor keeps
  1673. # "expired claim" meaning "owner died".
  1674. _job_schedule = job.get("schedule")
  1675. _is_oneshot = isinstance(_job_schedule, dict) and _job_schedule.get("kind") == "once"
  1676. _run_claim = job.get("run_claim")
  1677. _run_claim_owner = str(_run_claim.get("by") or "") if isinstance(_run_claim, dict) else ""
  1678. _last_claim_heartbeat = time.monotonic()
  1679. def _abort_if_fire_claim_lost() -> None:
  1680. if cancel_event is None or not cancel_event.is_set():
  1681. return
  1682. reason = cancel_reason(cancel_event)
  1683. if agent is not None and hasattr(agent, "interrupt"):
  1684. agent.interrupt(f"Cron run stopped: {reason}")
  1685. raise RuntimeError(f"Cron job '{job_name}' was stopped: {reason}")
  1686. def _heartbeat_run_claim_if_due():
  1687. nonlocal _last_claim_heartbeat
  1688. if not _is_oneshot or not _run_claim_owner:
  1689. return
  1690. _mono = time.monotonic()
  1691. if _mono - _last_claim_heartbeat < _RUN_CLAIM_HEARTBEAT_SECONDS:
  1692. return
  1693. _last_claim_heartbeat = _mono
  1694. try:
  1695. heartbeat_run_claim(job_id, expected_owner=_run_claim_owner)
  1696. except Exception:
  1697. logger.debug("Job '%s': run_claim heartbeat failed", job_name, exc_info=True)
  1698. # Establish suspend accounting before the worker can stamp its first activity.
  1699. _awake_idle = AwakeIdleMeter()
  1700. _cron_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
  1701. # Carry scheduler-scoped ContextVar state (e.g. env passthrough) into the worker thread.
  1702. _cron_context = contextvars.copy_context()
  1703. _cron_future = _cron_pool.submit(
  1704. _cron_context.run, agent.run_conversation, prompt, task_id=task_id)
  1705. if worker_state is not None:
  1706. worker_state["future"] = _cron_future
  1707. _inactivity_timeout = False
  1708. _watch_stop = threading.Event()
  1709. def _idle_seconds() -> float:
  1710. if not hasattr(agent, "get_activity_summary"):
  1711. return 0.0
  1712. try:
  1713. _act = agent.get_activity_summary()
  1714. return float(_act.get("seconds_since_activity", 0.0) or 0.0)
  1715. except Exception:
  1716. return 0.0
  1717. def _watch_inactivity() -> None:
  1718. nonlocal _inactivity_timeout
  1719. if _cron_inactivity_limit is None:
  1720. return
  1721. if _inactivity_watchdog_loop(
  1722. get_idle_seconds=_idle_seconds, limit_s=_cron_inactivity_limit, poll_s=_POLL_INTERVAL,
  1723. stop=_watch_stop, future_done=_cron_future.done, meter=_awake_idle):
  1724. _inactivity_timeout = True
  1725. _watch_thread = threading.Thread(
  1726. target=_watch_inactivity, name=f"cron-inactivity-{str(job_id)[:8]}", daemon=True)
  1727. # Ledger progress stamps: the stale-claim sweep measures silence since the last stamp.
  1728. _progress = ExecutionProgressStamper(
  1729. str(job.get("execution_id") or ""), job_name, idle_seconds=_idle_seconds,
  1730. every_seconds=_RUN_CLAIM_HEARTBEAT_SECONDS)
  1731. try:
  1732. if _cron_inactivity_limit is not None:
  1733. # Separate daemon thread so a hung get_activity_summary can't stop the limit firing.
  1734. # Daemon thread: kernel ``Event.wait`` timeout, independent of the ``run_job`` thread. A blocked
  1735. # loop / hung ``get_activity_summary`` on this thread can no longer keep the 600s inactivity
  1736. # limit from firing (#94285).
  1737. _watch_thread.start()
  1738. if _cron_inactivity_limit is None and not _is_oneshot and cancel_event is None:
  1739. result = _cron_future.result()
  1740. else:
  1741. result = None
  1742. while True:
  1743. done, _ = concurrent.futures.wait({_cron_future}, timeout=_POLL_INTERVAL)
  1744. if done:
  1745. _abort_if_fire_claim_lost()
  1746. result = _cron_future.result()
  1747. break
  1748. if _inactivity_timeout:
  1749. break
  1750. _abort_if_fire_claim_lost()
  1751. _heartbeat_run_claim_if_due()
  1752. _progress.tick()
  1753. except Exception:
  1754. _cron_pool.shutdown(wait=False, cancel_futures=True)
  1755. raise
  1756. finally:
  1757. _watch_stop.set()
  1758. _cron_pool.shutdown(wait=False, cancel_futures=True)
  1759. if _inactivity_timeout:
  1760. _raise_inactivity_timeout(agent, job_name, _cron_inactivity_limit)
  1761. if not isinstance(result, dict):
  1762. raise RuntimeError(
  1763. f"agent.run_conversation returned {type(result).__name__} instead of dict: {result!r}"
  1764. )
  1765. return result
  1766. def _final_response_from_result(result: dict, job_id: str, job_name: str, AIAgent) -> str:
  1767. """Deliverable final response from a ``run_conversation`` result. Raises RuntimeError on
  1768. `failed=True`/`completed=False`: the error text may sit in `final_response` and would otherwise
  1769. be delivered as the reply with the job marked ok."""
  1770. # If the agent itself reported failure (e.g. all retries exhausted on API errors, model abort, mid-run
  1771. # interrupt), do not silently mark the job as successful. run_agent populates
  1772. # `failed=True`/`completed=False` on these paths and may put the error into `final_response`, which
  1773. # would otherwise be delivered as if it were the agent's reply and the job's `last_status` set to "ok".
  1774. # Raise so the except handler below builds the proper failure tuple. (issue #17855)
  1775. turn_exit_reason = str(result.get("turn_exit_reason") or "")
  1776. final_response_text = (result.get("final_response") or "").strip()
  1777. max_iteration_summary = is_max_iteration_handoff(result)
  1778. if result.get("failed") is True or (result.get("completed") is False and not max_iteration_summary):
  1779. raise RuntimeError(result.get("error") or final_response_text or "agent reported failure")
  1780. if max_iteration_summary:
  1781. logger.warning(
  1782. "Job '%s' reached the iteration limit but produced a final fallback response; "
  1783. "delivering the response instead of failing the cron run",
  1784. job_name)
  1785. final_response = result.get("final_response", "") or ""
  1786. # Repair model-mangled computer_use media paths before delivery (fail-open, as in gateway).
  1787. if final_response:
  1788. from gateway.media_repair import repair_explicit_computer_use_media_paths
  1789. final_response = repair_explicit_computer_use_media_paths(
  1790. final_response, result.get("messages", []))
  1791. if final_response.strip() == "(No response generated)":
  1792. final_response = ""
  1793. # The "⚠️ No reply" turn-completion explainer would be delivered as a cron warning; detect it
  1794. # via the same formatter and treat as empty so cron stays silent on abnormal empty turns.
  1795. if final_response.strip() and turn_exit_reason:
  1796. # Render every persistence-cause variant or cause-refined text slips through.
  1797. _explainer_variants = []
  1798. try:
  1799. from hermes_state_errors import PERSISTENCE_ERROR_CAUSES as _causes
  1800. except Exception:
  1801. _causes = ("locked", "disk", "unknown")
  1802. # The finalizer fills the model name into the explainer; render with the same name (and
  1803. # the bare form) or the comparison below misses and the warning is delivered.
  1804. _model = str(result.get("model") or "")
  1805. for _cause in (None, *_causes):
  1806. for _kwargs in ({"model": _model}, {}):
  1807. try:
  1808. _variant = AIAgent._format_turn_completion_explanation(turn_exit_reason, _cause, **_kwargs)
  1809. except TypeError:
  1810. try:
  1811. _variant = AIAgent._format_turn_completion_explanation(turn_exit_reason)
  1812. except Exception:
  1813. _variant = ""
  1814. except Exception:
  1815. _variant = ""
  1816. if _variant:
  1817. _explainer_variants.append(_variant.strip())
  1818. if final_response.strip() in _explainer_variants:
  1819. logger.info(
  1820. "Job '%s': abnormal empty turn (%s) — suppressing explainer for cron delivery",
  1821. job_id, turn_exit_reason)
  1822. final_response = ""
  1823. return final_response
  1824. def _finalize_cron_session(session_db, agent, job_id: str, job_name: str, cron_session_id: str,
  1825. workdir: Optional[str] = None) -> None:
  1826. """Title, classify, end and release the cron session after the agent turn has returned."""
  1827. # Bound every DB op so storage failure cannot hold the dispatch guard.
  1828. _session_db = _BoundedCronSessionDB(session_db, job_id)
  1829. # Compression may have rotated the run onto a continuation: finalize that, not the stale cron
  1830. # id. SessionDB lineage is authoritative; agent.session_id is only a fail-safe.
  1831. _final_cron_session_id = cron_session_id
  1832. try:
  1833. _compression_tip = _session_db.get_compression_tip(cron_session_id)
  1834. if _compression_tip:
  1835. _final_cron_session_id = _compression_tip
  1836. except (Exception, KeyboardInterrupt) as e:
  1837. with contextlib.suppress((Exception, KeyboardInterrupt)):
  1838. _agent_session_id = getattr(agent, "session_id", None)
  1839. # CLI (single-process) path: the approval contextvar is only bound during gateway/TUI turns and
  1840. # HERMES_SESSION_KEY is not in the CLI environment, so the key resolves empty here. Since #64240
  1841. # the CLI drains completions through a positive-ownership filter keyed on the durable
  1842. # AIAgent.session_id — an empty session_key would fail closed and the CLI could never claim its
  1843. # own completions, while a restored foreign event with an empty key could leak into any
  1844. # unfiltered consumer (#64484). Stamp the parent's durable session id instead; compression
  1845. # rotations are handled on the drain side via resolve_resume_session_id lineage resolution.
  1846. if _agent_session_id:
  1847. _final_cron_session_id = _agent_session_id
  1848. logger.debug("Job '%s': failed to resolve cron compression tip: %s", job_id, e)
  1849. # Title must persist BEFORE end_session()/close(). Run-time suffix keeps it unique against the
  1850. # sessions.title index; the fallbacks below guarantee a non-blank title.
  1851. try:
  1852. # Title the cron session from the job (name -> id) and PERSIST it BEFORE end_session()/close() tear
  1853. # the connection down, so the close can never run over an in-flight title write (#50536).
  1854. _title_base = " ".join(job_name.split())[:60].strip() or f"cron {job_id}"
  1855. _cron_title = f"{_title_base} · {safe_strftime(_hermes_now(), '%b %d %H:%M')}"
  1856. if not _set_cron_session_title(_session_db, _final_cron_session_id, _cron_title):
  1857. _set_cron_session_title(_session_db, _final_cron_session_id, f"cron {job_id}")
  1858. except (Exception, KeyboardInterrupt) as e:
  1859. logger.debug("Job '%s': failed to set cron session title: %s", job_id, e)
  1860. # Never leave the session untitled.
  1861. # Try the next free title in the lineage, then a bare id-stamped title. See #50535.
  1862. for _fallback in (
  1863. getattr(_session_db, "get_next_title_in_lineage", lambda b: b)(f"cron {job_id}"),
  1864. f"cron {job_id} {_final_cron_session_id[-6:]}"):
  1865. try:
  1866. if _set_cron_session_title(_session_db, _final_cron_session_id, _fallback):
  1867. break
  1868. except (Exception, KeyboardInterrupt):
  1869. continue
  1870. # Book cron_complete only when the last row is a real assistant reply ([SILENT] counts). Only a
  1871. # POSITIVELY recognized bad status downgrades (keep tuple in sync with
  1872. # session_lifecycle_statuses); unknown values / probe failures fail OPEN.
  1873. # Verified completion booking (#93820): the run may only be recorded as cron_complete when the session's
  1874. # LAST message row is a real assistant reply — a plain answer or the [SILENT] sentinel (both are
  1875. # assistant-text rows, so both classify as 'complete'). A turn that died after a tool call,
  1876. # mid-API-wait, or without any assistant text leaves the last row as a tool result / pending call / user
  1877. # prompt and must not surface as a healthy run. session_lifecycle_statuses is the existing cost-bounded
  1878. # classifier for exactly this shape. Only a POSITIVELY recognized pathological status (see the status
  1879. # vocabulary in hermes_state's session_lifecycle_statuses docstring — keep the tuple below in sync when
  1880. # it grows) downgrades the booking: an unknown value (newer classifier shape, test doubles) keeps the
  1881. # historical reason, and so does a failed probe — the booking itself is FAIL-OPEN on probe errors,
  1882. # because classification is best-effort metadata and must not mislabel a healthy run.
  1883. _end_reason = "cron_complete"
  1884. try:
  1885. _statuses = _session_db.session_lifecycle_statuses([_final_cron_session_id])
  1886. _lifecycle = _statuses.get(_final_cron_session_id)
  1887. if _lifecycle in ("interrupted", "error", "empty"):
  1888. _end_reason = "cron_incomplete_no_output"
  1889. logger.warning(
  1890. "Job '%s': session ended without a final assistant "
  1891. "message (lifecycle=%s) — booking run as %s",
  1892. job_id, _lifecycle, _end_reason)
  1893. except (Exception, KeyboardInterrupt) as e:
  1894. logger.debug("Job '%s': session lifecycle classification failed: %s", job_id, e)
  1895. # Stamp the job's workdir on the row BEFORE end_session (title-write ordering, #50536): the
  1896. # sidebar groups by cwd prefix and nothing else writes a cron row's cwd (#108205).
  1897. if workdir:
  1898. try:
  1899. _session_db.update_session_cwd(_final_cron_session_id, workdir)
  1900. except (Exception, KeyboardInterrupt) as e:
  1901. logger.debug("Job '%s': failed to stamp workdir on session row: %s", job_id, e, exc_info=True)
  1902. try:
  1903. _session_db.end_session(_final_cron_session_id, _end_reason)
  1904. # The scheduler owns cron-session finalization. AIAgent.close() also
  1905. # finalizes owned session rows by default; once the shared SessionDB is
  1906. # released below, that second end_session() would reopen the just-closed
  1907. # SQLite handle (#94736). The reason is durably booked, so disarm only the
  1908. # agent's redundant row-finalization; its resource teardown still runs in
  1909. # _teardown_cron_agent.
  1910. if agent is not None:
  1911. agent._end_session_on_close = False
  1912. except (Exception, KeyboardInterrupt) as e:
  1913. logger.debug("Job '%s': failed to end session: %s", job_id, e)
  1914. try:
  1915. from hermes_state_registry import release_or_close
  1916. release_or_close(_session_db)
  1917. except (Exception, KeyboardInterrupt) as e:
  1918. logger.debug("Job '%s': failed to close SQLite session store: %s", job_id, e)
  1919. def _normalize_newlines(text: str) -> str:
  1920. """CRLF/CR -> LF, matching what text-mode reads of the archive return."""
  1921. return text.replace("\r\n", "\n").replace("\r", "\n")
  1922. def _run_doc_header(job: dict, title: str, job_id: str, prompt: str, *, prompt_stamp: str = "") -> str:
  1923. """Header of the persisted run document (title, ids, schedule, prompt)."""
  1924. return (
  1925. f"# Cron Job: {title}\n\n"
  1926. f"**Job ID:** {job_id}\n"
  1927. f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
  1928. f"**Schedule:** {job.get('schedule_display', 'N/A')}\n\n"
  1929. f"{prompt_stamp}{_PROMPT_HEADING}{prompt}{_PROMPT_SEPARATOR}"
  1930. )
  1931. _RunResult = tuple[bool, str, str, Optional[str]]
  1932. def _prepare_job_prompt(
  1933. job: dict, job_id: str, job_name: str, extra_prompt: Optional[str], cancel_event,
  1934. ) -> tuple[Optional[_RunResult], Optional[str]]:
  1935. """Run every pre-agent gate and build the prompt. Returns ``(early_result, prompt)``: an early
  1936. result short-circuits ``run_job`` (no_agent job, empty payload, monitor gate, wake gate,
  1937. injection block, empty prompt); otherwise ``prompt`` is set."""
  1938. # Fail closed on a corrupt config.yaml: defaults would let auto-detection bill a provider the
  1939. # user never chose. no_agent jobs are exempt. Escape hatch: HERMES_IGNORE_USER_CONFIG=1.
  1940. if not job.get("no_agent"):
  1941. from hermes_cli.config import InvalidUserConfigError, require_parseable_user_config
  1942. try:
  1943. require_parseable_user_config()
  1944. except InvalidUserConfigError as exc:
  1945. logger.error("Job '%s': refusing to run — %s", job_id, exc)
  1946. return (False, f"# Cron Job: {job_name}\n\nError: {exc}\n", "", str(exc)), None
  1947. # no_agent short-circuits BEFORE importing run_agent / opening SessionDB.
  1948. if job.get("no_agent"):
  1949. return _run_no_agent_job(job, job_id, job_name, cancel_event), None
  1950. # Legacy / hand-edited job with nothing to run: pause it instead of waking the LLM every fire.
  1951. from cron.jobs import EMPTY_PAYLOAD_ERROR, job_payload_is_empty
  1952. if job_payload_is_empty(job):
  1953. return _block_and_pause_job(job_id, job_name, EMPTY_PAYLOAD_ERROR), None
  1954. _early, extra_prompt, monitor_context = _apply_monitor_gate(job, job_id, job_name, extra_prompt)
  1955. if _early is not None:
  1956. return _early, None
  1957. # Wake-gate: run the pre-check script BEFORE building the prompt; its result is passed into
  1958. # _build_job_prompt so the script runs only once.
  1959. # NOTE: the SQLite session store used to be initialized here, BEFORE the wake-gate and prompt-validation
  1960. # early returns below. Every gated run (``wakeAgent: false``, blocked prompt) opened state.db and
  1961. # returned without reaching the finally that closes it, relying on GC to release the handle. Init now
  1962. # happens inside the main try, right before the agent is constructed — after every early-return path
  1963. # (#96290).
  1964. prerun_script = None
  1965. script_path = job.get("script")
  1966. if script_path:
  1967. prerun_script = _run_job_script_with_claim_heartbeat(
  1968. job,
  1969. script_path,
  1970. workdir=_resolve_job_workdir(job, job_id),
  1971. cancel_event=cancel_event,
  1972. )
  1973. _ran_ok, _script_output = prerun_script
  1974. if _ran_ok and not _parse_wake_gate(_script_output):
  1975. logger.info("Job '%s' (ID: %s): wakeAgent=false, skipping agent run", job_name, job_id)
  1976. note_cron_skipped(job)
  1977. silent_doc = (
  1978. f"# Cron Job: {job_name}\n\n"
  1979. f"**Job ID:** {job_id}\n"
  1980. f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n\n"
  1981. "Script gate returned `wakeAgent=false` — agent skipped.\n"
  1982. )
  1983. return (True, silent_doc, SILENT_MARKER, None), None
  1984. try:
  1985. prompt = _build_job_prompt(
  1986. job, prerun_script=prerun_script, extra_prompt=extra_prompt,
  1987. runtime_data_prompt=monitor_context,
  1988. )
  1989. except CronPromptInjectionBlocked as block_exc:
  1990. # Injection scanner tripped: refuse this tick and tell the operator WHY.
  1991. logger.warning(
  1992. "Job '%s' (ID: %s): blocked by prompt-injection scanner — %s", job_name, job_id, block_exc,
  1993. )
  1994. blocked_doc = (
  1995. f"# Cron Job: {job_name}\n\n"
  1996. f"**Job ID:** {job_id}\n"
  1997. f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
  1998. f"**Status:** BLOCKED\n\n"
  1999. "The assembled prompt (user prompt + loaded skill content) tripped "
  2000. "the cron injection scanner and the agent was NOT run.\n\n"
  2001. f"**Scanner result:** {block_exc}\n\n"
  2002. "Audit the skill(s) attached to this job for prompt-injection "
  2003. "payloads or invisible-unicode markers. If the skill is legitimate "
  2004. "and the match is a false positive, rephrase the content to avoid "
  2005. "the threat pattern (`tools/cronjob_tools.py::_CRON_THREAT_PATTERNS`)."
  2006. )
  2007. return (False, blocked_doc, "", str(block_exc)), None
  2008. if prompt is None:
  2009. logger.info("Job '%s': script produced no output, skipping AI call.", job_name)
  2010. note_cron_skipped(job)
  2011. return (True, "", SILENT_MARKER, None), None
  2012. return None, prompt
  2013. _CRON_DELIVERY_VARS = (
  2014. "HERMES_CRON_AUTO_DELIVER_PLATFORM",
  2015. "HERMES_CRON_AUTO_DELIVER_CHAT_ID",
  2016. "HERMES_CRON_AUTO_DELIVER_THREAD_ID")
  2017. class _CronRunScope:
  2018. """Per-run ContextVar / tool-cwd scope for ``run_job`` (ContextVars, not os.environ, so
  2019. parallel jobs don't clobber each other). Construct before the try, ``enter()`` as its first
  2020. statement, ``exit()`` in the finally — every setter here has a matching reset there.
  2021. HERMES_SESSION_* are deliberately NOT seeded from job["origin"]: it is delivery metadata, not
  2022. a sender, and terminal/tts/skills/send_message tools would act as if the origin user were
  2023. driving the agent. Delivery reads job["origin"] / HERMES_CRON_AUTO_DELIVER_* directly.
  2024. """
  2025. def __init__(self, job: dict, job_id: str, execution_id: Optional[str]):
  2026. from gateway.session_context import set_session_vars, _VAR_MAP
  2027. from tools.terminal_tool import record_session_cwd
  2028. self._var_map = _VAR_MAP
  2029. # Resolve workdir BEFORE set_session_vars so it owns the _SESSION_CWD set/clear.
  2030. self.workdir = _resolve_job_workdir(job, job_id)
  2031. self._ctx_tokens = set_session_vars(
  2032. platform="",
  2033. chat_id="",
  2034. chat_name="",
  2035. # Cron can't receive completions after its turn; async delegation output could
  2036. # otherwise route to an unrelated chat via the ambient session key => inline delegation.
  2037. # We clear the HERMES_SESSION_* routing keys just below, so an async delegation's completion
  2038. # event carries session_key="" — _enrich_async_delegation_routing cannot resolve it and
  2039. # _inject_watch_notification drops it ("no routing metadata"). And by the time a child finishes,
  2040. # run_job has already shipped the job's final response via _deliver_result; there is no turn
  2041. # left to re-enter. (Worse, get_current_session_key() can fall back to the ambient os.environ
  2042. # HERMES_SESSION_KEY, which risks routing a cron subagent's output into an unrelated user chat.)
  2043. # Declaring the channel stateless routes delegate_task to its existing inline/synchronous path,
  2044. # so results return within the job's own turn. See declare_stateless_channel(). Upstream:
  2045. # #53027, #63142.
  2046. async_delivery=False,
  2047. cwd=self.workdir or "",
  2048. )
  2049. for name in _CRON_DELIVERY_VARS:
  2050. _VAR_MAP[name].set("")
  2051. # Workdir binds to the per-run task id (tool-layer cwd authority) instead of mutating
  2052. # global TERMINAL_CWD; _SESSION_CWD above remains the prompt/context-file authority.
  2053. self.task_id = f"cron:{job_id}:{execution_id or job.get('execution_id') or uuid.uuid4().hex}"
  2054. if self.workdir:
  2055. record_session_cwd(self.task_id, self.workdir)
  2056. self._cron_session_var = _VAR_MAP["HERMES_CRON_SESSION"]
  2057. self._cron_session_token = None
  2058. self._non_dispatcher_token = None
  2059. self._spend_token = None
  2060. def enter(self) -> None:
  2061. # Scope cron approval policy; exit() RESETS via token (pinning "" would suppress the legacy
  2062. # os.environ fallback used by standalone entrypoints/tests).
  2063. self._cron_session_token = self._cron_session_var.set("1")
  2064. # Mark NOT the kanban worker: a worker's cronjob(action="run") lands here with
  2065. # HERMES_KANBAN_TASK in env, and an unrelated job could close the worker's task. Must be a
  2066. # ContextVar, NOT an os.environ clear (env is shared with the worker heartbeat and
  2067. # concurrent jobs); copy_context() carries it into the agent thread.
  2068. self._non_dispatcher_token = enter_non_dispatcher_owned_context()
  2069. # Each run is its own MCP spend "chat" (its approval session key is "" for every job).
  2070. from tools.mcp_tool_costs import bind_run
  2071. self._spend_token = bind_run(self.task_id)
  2072. def exit(self) -> None:
  2073. from gateway.session_context import clear_session_vars
  2074. from tools.terminal_tool import clear_session_cwd
  2075. clear_session_cwd(self.task_id)
  2076. clear_session_vars(self._ctx_tokens) # also clears _SESSION_CWD
  2077. if self._cron_session_token is not None:
  2078. self._cron_session_var.reset(self._cron_session_token)
  2079. if self._non_dispatcher_token is not None:
  2080. exit_non_dispatcher_owned_context(self._non_dispatcher_token)
  2081. if self._spend_token is not None:
  2082. from tools.mcp_tool_costs import unbind_run
  2083. unbind_run(self._spend_token)
  2084. for name in _CRON_DELIVERY_VARS:
  2085. self._var_map[name].set("")
  2086. def _reload_dotenv_and_publish_delivery_target(job: dict) -> None:
  2087. """Re-read .env for this run and publish the auto-deliver target into the session ContextVars."""
  2088. # Reset the secret-source cache FIRST or a Bitwarden/BSM-backed secret is never re-resolved
  2089. # (only the placeholder reloads -> 401s).
  2090. from hermes_cli.env_loader import load_hermes_dotenv, reset_secret_source_cache
  2091. from gateway.session_context import _VAR_MAP
  2092. reset_secret_source_cache(_get_hermes_home())
  2093. load_hermes_dotenv(hermes_home=_get_hermes_home())
  2094. delivery_target = _resolve_delivery_target(job)
  2095. if delivery_target:
  2096. _VAR_MAP["HERMES_CRON_AUTO_DELIVER_PLATFORM"].set(delivery_target["platform"])
  2097. _VAR_MAP["HERMES_CRON_AUTO_DELIVER_CHAT_ID"].set(str(delivery_target["chat_id"]))
  2098. _VAR_MAP["HERMES_CRON_AUTO_DELIVER_THREAD_ID"].set(
  2099. "" if delivery_target.get("thread_id") is None else str(delivery_target["thread_id"])
  2100. )
  2101. @dataclass
  2102. class _CronAgentSetup:
  2103. """Everything ``AIAgent(...)`` needs that is resolved from job + config (or a preflight block)."""
  2104. blocked: Optional[_RunResult] = None
  2105. model: str = ""
  2106. runtime: dict = None
  2107. prefill_messages: Any = None
  2108. max_iterations: Any = None
  2109. reasoning_config: Any = None
  2110. fallback_model: Any = None
  2111. credential_pool: Any = None
  2112. fallback_notice: Optional[str] = None
  2113. def _resolve_cron_agent_setup(job: dict, job_id: str, job_name: str, jc) -> _CronAgentSetup:
  2114. """Resolve model/runtime/reasoning/pool for the run, in the original gate order: exfil guard ->
  2115. preflight (may block) -> runtime (+ fallback chain) -> credential pool -> MCP."""
  2116. _cfg = jc.cfg
  2117. setup = _CronAgentSetup(model=jc.model)
  2118. setup.prefill_messages = _load_prefill_messages(_cfg, job_id)
  2119. # resolve_turn_limit() honors none/unlimited (sys.maxsize) and explicit 0 / null.
  2120. from hermes_cli.config import resolve_turn_limit as _resolve_turn_limit
  2121. _mt = _cfg.get("agent", {}).get("max_turns")
  2122. if _mt is None:
  2123. _mt = _cfg.get("max_turns")
  2124. setup.max_iterations = _resolve_turn_limit(_mt)
  2125. # Runtime backstop (CWE-200/522): fail closed BEFORE resolution on a provider/base_url pair
  2126. # that would ship a stored credential off-host; hand-written jobs bypass create-time checks.
  2127. _guard_job_credential_exfil(job)
  2128. setup.blocked = _preflight_or_block(job, job_id, job_name, _cfg)
  2129. if setup.blocked is not None:
  2130. return setup
  2131. setup.runtime, setup.model = _resolve_job_runtime(job, job_id, jc)
  2132. setup.fallback_notice = setup.runtime.pop("_fallback_notice", None)
  2133. setup.reasoning_config = _resolve_job_reasoning_config(
  2134. job, _cfg if isinstance(_cfg, dict) else {}, str(setup.model)
  2135. )
  2136. # Mid-run provider ladder: same rule as resolution above, so a pinned job cannot be swapped
  2137. # onto the global chain by a 5xx/429 either.
  2138. setup.fallback_model = _job_fallback_chain(job, _cfg)
  2139. setup.credential_pool = _load_credential_pool(setup.runtime, job_id)
  2140. # MCP servers must be registered before AIAgent is constructed.
  2141. _init_cron_mcp_tools(job_id)
  2142. # Only now can a requested MCP toolset be judged: its alias is process-global but its tools live
  2143. # in this profile's registry overlay, and quiet_mode hides the empty resolution (#109050).
  2144. if _cron_preflight_enabled(_cfg):
  2145. _mcp_reason = _empty_requested_mcp_toolsets(job, _cfg)
  2146. if _mcp_reason:
  2147. setup.blocked = _blocked_config_result(job_id, job_name, _mcp_reason)
  2148. return setup
  2149. def _construct_cron_agent(AIAgent, job: dict, _cfg: dict, setup: _CronAgentSetup, *, workdir, session_id, session_db):
  2150. runtime = setup.runtime
  2151. pr = _cfg.get("provider_routing") or {}
  2152. return AIAgent(
  2153. model=setup.model,
  2154. api_key=runtime.get("api_key"),
  2155. base_url=runtime.get("base_url"),
  2156. provider=runtime.get("provider"),
  2157. requested_provider=runtime.get("requested_provider"),
  2158. api_mode=runtime.get("api_mode"),
  2159. request_overrides=runtime.get("request_overrides"),
  2160. acp_command=runtime.get("command"),
  2161. acp_args=runtime.get("args"),
  2162. max_iterations=setup.max_iterations,
  2163. reasoning_config=setup.reasoning_config,
  2164. prefill_messages=setup.prefill_messages,
  2165. fallback_model=setup.fallback_model,
  2166. credential_pool=setup.credential_pool,
  2167. providers_allowed=pr.get("only"),
  2168. providers_ignored=pr.get("ignore"),
  2169. providers_order=pr.get("order"),
  2170. provider_sort=pr.get("sort"),
  2171. openrouter_min_coding_score=(_cfg.get("openrouter") or {}).get("min_coding_score"),
  2172. enabled_toolsets=_resolve_cron_enabled_toolsets(job, _cfg),
  2173. disabled_toolsets=_resolve_cron_disabled_toolsets(_cfg),
  2174. quiet_mode=True,
  2175. # Project context files only with a configured workdir; SOUL.md always.
  2176. skip_context_files=not bool(workdir),
  2177. load_soul_identity=True,
  2178. skip_memory=False,
  2179. skip_background_review=True, # Cron has no human-in-the-loop need for skill/memory review forks (~30K tok/event)
  2180. platform="cron",
  2181. session_id=session_id,
  2182. session_db=session_db,
  2183. )
  2184. class _FireAudit:
  2185. """One usage_audit.jsonl line per fire (created once the agent exists; fire id + start clock)."""
  2186. def __init__(self, job: dict, job_id: str, model: str):
  2187. self.job, self.job_id, self.model = job, job_id, model
  2188. self.fire_id = uuid.uuid4().hex
  2189. self.t_start = time.monotonic()
  2190. def write(self, result: dict, error: Optional[str]) -> None:
  2191. _write_usage_audit({
  2192. "ts": _utcnow_iso_ms(),
  2193. "job_id": self.job_id,
  2194. "fire_id": self.fire_id,
  2195. "prompt_tokens": result.get("prompt_tokens"),
  2196. "completion_tokens": result.get("completion_tokens"),
  2197. "total_tokens": result.get("total_tokens"),
  2198. "response_silent": bool(result.get("response_silent")),
  2199. "deliver_target": self.job.get("deliver"),
  2200. "model": self.model or None,
  2201. "duration_ms": int((time.monotonic() - self.t_start) * 1000),
  2202. "error": error})
  2203. def run_job(
  2204. job: dict, *, defer_agent_teardown: Optional[list] = None, extra_prompt: Optional[str] = None,
  2205. cancel_event: Optional[_CancelEventLike] = None, execution_id: Optional[str] = None,
  2206. ) -> tuple[bool, str, str, Optional[str]]:
  2207. """Execute a single cron job. Returns (success, full_output_doc, final_response, error).
  2208. ``defer_agent_teardown``: if a list, the live agent is appended instead of torn down; the caller
  2209. MUST call ``_teardown_cron_agent(agent)`` AFTER delivery (a torn-down async client can't
  2210. deliver). ``extra_prompt``: per-fire context, never persisted.
  2211. ``defer_agent_teardown``: when a caller passes a list, ``run_job`` skips the agent's async-resource
  2212. teardown (``agent.close()`` + ``cleanup_stale_async_clients()``) in its ``finally`` block and instead
  2213. appends the live agent to that list. The caller is then responsible for calling
  2214. ``_teardown_cron_agent(agent)`` AFTER it has delivered the result. This closes the ordering window in
  2215. #58720 where delivery ran against a torn-down async client (defense-in-depth alongside the
  2216. interpreter-shutdown guard). When ``None`` (the default) teardown happens inline as before, so every
  2217. existing caller is unchanged.
  2218. ``extra_prompt``: optional per-run context from ``cronjob(action='run', prompt=...)`` (#57331). Appended
  2219. to the stored prompt for this fire only — never persisted to the job definition.
  2220. """
  2221. from gateway.session_cron import current_execution
  2222. owner_execution = current_execution()
  2223. if owner_execution is not None and owner_execution[2:] != (job['id'], execution_id):
  2224. owner_execution = None
  2225. if not job.get("no_agent") and owner_execution is None:
  2226. from cron.scheduler_authority import run_canonical_job
  2227. return run_canonical_job(job, extra_prompt=extra_prompt, cancel_event=cancel_event,
  2228. execution_id=execution_id)
  2229. job_id = job["id"]
  2230. job_name = str(job.get("name") or job.get("prompt") or job_id or "cron job")
  2231. early, prompt = _prepare_job_prompt(job, job_id, job_name, extra_prompt, cancel_event)
  2232. if early is not None:
  2233. return early
  2234. from run_agent import AIAgent
  2235. _cron_session_id = (owner_execution[1] if owner_execution is not None else
  2236. f"cron_{job_id}_{_hermes_now().strftime('%Y%m%d_%H%M%S')}")
  2237. logger.info("Running job '%s' (ID: %s)", job_name, job_id)
  2238. logger.info("Prompt: %s", prompt[:100])
  2239. agent = None
  2240. model = ""
  2241. _session_db = None
  2242. _audit: Optional[_FireAudit] = None
  2243. _worker_state: dict = {}
  2244. scope = _CronRunScope(job, job_id, execution_id)
  2245. try:
  2246. scope.enter()
  2247. if scope.workdir:
  2248. logger.info("Job '%s': using task-scoped workdir %s", job_id, scope.workdir)
  2249. _reload_dotenv_and_publish_delivery_target(job)
  2250. jc = _load_cron_job_config(job, job_id, job_name)
  2251. _cfg = jc.cfg
  2252. model = jc.model
  2253. setup = _resolve_cron_agent_setup(job, job_id, job_name, jc)
  2254. if setup.blocked is not None:
  2255. return setup.blocked
  2256. model = setup.model
  2257. # Open state.db only after every early-return gate has passed.
  2258. _session_db = _open_cron_session_db(job)
  2259. agent = _construct_cron_agent(
  2260. AIAgent, job, _cfg, setup, workdir=scope.workdir, session_id=_cron_session_id,
  2261. session_db=_session_db)
  2262. _audit = _FireAudit(job, job_id, model)
  2263. result = _run_agent_with_watchdog(
  2264. agent, prompt, job, job_id, job_name, scope.task_id, cancel_event,
  2265. worker_state=_worker_state)
  2266. final_response = _final_response_from_result(result, job_id, job_name, AIAgent)
  2267. if (setup.fallback_notice and final_response.strip() and not _is_cron_silence_response(final_response)
  2268. and _cron_failure_marker_error(final_response) is None):
  2269. # Pre-agent provider switch (#74349) rides with the delivered report; silence and the
  2270. # agent-declared failure marker keep their first-line/whole-response contract.
  2271. final_response = f"{setup.fallback_notice}\n\n{final_response}"
  2272. # Keep final_response clean for delivery logic (empty = no delivery).
  2273. logged_response = final_response if final_response else "(No response generated)"
  2274. # Text-file reads normalize newlines; count the same characters the
  2275. # context_from reader sees so quoted markers can never become boundaries.
  2276. framed_prompt = _normalize_newlines(prompt)
  2277. logged_response = _normalize_newlines(logged_response)
  2278. output = (
  2279. _run_doc_header(job, job_name, job_id, framed_prompt,
  2280. prompt_stamp=f"{_PROMPT_FRAME}{len(framed_prompt)}\n")
  2281. + f"{_RESPONSE_FRAME}{len(logged_response)}\n{_RESPONSE_HEADING}{logged_response}{_RESPONSE_TERMINATOR}"
  2282. )
  2283. logger.info("Job '%s' completed successfully", job_name)
  2284. _audit.write(dict(result, response_silent=_is_cron_silence_response(final_response or "")), None)
  2285. return True, output, final_response, None
  2286. except (Exception, ConfigBackendUnavailable) as e: # the plane being down fails THIS fire only
  2287. error_msg = f"{type(e).__name__}: {e!s}"
  2288. logger.exception("Job '%s' failed: %s", job_name, error_msg)
  2289. # Cowork-style unreachable-model re-run (cron/unreachable_retry.py): flag failures where
  2290. # the model was never reached (transient network/DNS, zero API calls) so the bookkeeping
  2291. # tail can schedule a bounded automatic re-run instead of waiting a full period.
  2292. try:
  2293. from cron.unreachable_retry import is_model_unreachable_failure
  2294. if is_model_unreachable_failure(e, agent):
  2295. job["_model_unreachable"] = True
  2296. # Provider usage window closed for a known duration (cron/quota_hold.py): flag it so the
  2297. # bookkeeping tail parks the job past the window instead of re-firing into it (#89376).
  2298. from cron.quota_hold import hold_seconds_from_failure
  2299. _hold_s = hold_seconds_from_failure(e)
  2300. if _hold_s:
  2301. job["_quota_hold_seconds"] = _hold_s
  2302. except Exception: # classification must never mask the real failure
  2303. logger.debug("Job '%s': unreachable-failure classification failed", job_id)
  2304. # No audit row when we failed before the agent existed; the audit write must never raise.
  2305. if _audit is not None:
  2306. _audit.write({}, error_msg)
  2307. from cron.scheduler_diagnostics import format_run_error
  2308. output = (
  2309. _run_doc_header(job, f"{job_name} (FAILED)", job_id, prompt)
  2310. + format_run_error(e)
  2311. )
  2312. return False, output, "", error_msg
  2313. finally:
  2314. from cron.scheduler_detached_worker import defer_teardown_to_running_worker
  2315. _worker_teardown_deferred = defer_teardown_to_running_worker(
  2316. _worker_state.get("future"), _session_db, agent, job_id, job_name, _cron_session_id,
  2317. workdir=scope.workdir)
  2318. scope.exit()
  2319. if _session_db and not _worker_teardown_deferred:
  2320. _finalize_cron_session(_session_db, agent, job_id, job_name, _cron_session_id,
  2321. workdir=scope.workdir)
  2322. # Tear down the ephemeral agent or the gateway leaks fds per tick (EMFILE). With deferred
  2323. # teardown, hand the live agent back: delivery needs a live async client.
  2324. # Release subprocesses, terminal sandboxes, browser daemons, and the main OpenAI/httpx client held
  2325. # by this ephemeral cron agent. Without this, a gateway that ticks cron every N minutes leaks fds
  2326. # per job until it hits EMFILE (#10200 / "too many open files"). When the caller opted to defer
  2327. # teardown (passed a list), hand the live agent back instead of closing it here — delivery must run
  2328. # against a live async client, and the caller tears down afterwards (#58720).
  2329. if not _worker_teardown_deferred:
  2330. if defer_agent_teardown is not None:
  2331. if agent is not None:
  2332. defer_agent_teardown.append(agent)
  2333. else:
  2334. _teardown_cron_agent(agent, job_id)
  2335. def _teardown_cron_agent(
  2336. agent, job_id: str, *, timeout_seconds: Optional[float] = None
  2337. ) -> None:
  2338. """Release an ephemeral cron agent's async resources within a hard bound (this runs outside the
  2339. inactivity watchdog). Shared by ``run_job``'s finally and deferred post-delivery teardown.
  2340. Split out of ``run_job``'s ``finally`` so a caller that defers teardown (to deliver first — #58720) can
  2341. invoke the identical cleanup AFTER delivery. The timeout matters because this executes after
  2342. ``run_conversation`` has returned, outside the agent inactivity watchdog.
  2343. """
  2344. def _cleanup_agent() -> None:
  2345. try:
  2346. if agent is not None:
  2347. agent.close()
  2348. except (Exception, KeyboardInterrupt) as e:
  2349. logger.debug("Job '%s': failed to close agent resources: %s", job_id, e)
  2350. # Worker-thread event loop dies with the executor; reap httpx clients cached under it.
  2351. try:
  2352. from agent.auxiliary_client import cleanup_stale_async_clients
  2353. cleanup_stale_async_clients()
  2354. except Exception as e:
  2355. logger.debug("Job '%s': failed to reap stale auxiliary clients: %s", job_id, e)
  2356. _run_cron_cleanup_with_timeout(
  2357. _cleanup_agent, job_id=job_id, label="agent resource teardown",
  2358. timeout_seconds=timeout_seconds)
  2359. def _run_with_fire_claim_heartbeat(job: dict, run) -> bool:
  2360. """Run ``run`` while keeping this job's owned durable fire claim fresh."""
  2361. claim = job.get("fire_claim")
  2362. owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
  2363. if not owner:
  2364. return run(None)
  2365. job_id = str(job.get("id") or "")
  2366. stop = threading.Event()
  2367. lost_ownership = threading.Event()
  2368. def _finish_unstarted(error: str) -> None:
  2369. execution_id = job.get("execution_id")
  2370. if not execution_id:
  2371. return
  2372. try:
  2373. finish_lost_execution(execution_id, error)
  2374. except Exception:
  2375. logger.warning(
  2376. "Job '%s': failed to close unstarted execution ledger row",
  2377. job_id,
  2378. exc_info=True)
  2379. try:
  2380. owns_fire_claim = heartbeat_fire_claim(job_id, expected_owner=owner)
  2381. except Exception:
  2382. logger.warning("Job '%s': initial fire_claim validation failed", job_id, exc_info=True)
  2383. _finish_unstarted("Fire claim ownership could not be validated before execution started.")
  2384. return True
  2385. if owns_fire_claim is False:
  2386. logger.warning("Job '%s': fire claim ownership was already lost before execution", job_id)
  2387. _finish_unstarted("Fire claim ownership lost before execution started.")
  2388. return True
  2389. def _heartbeat_loop() -> None:
  2390. last_confirmed = time.monotonic()
  2391. while not stop.wait(_RUN_CLAIM_HEARTBEAT_SECONDS):
  2392. try:
  2393. if not heartbeat_fire_claim(job_id, expected_owner=owner):
  2394. if self_removal_delivery_allowed(job_id):
  2395. # Record dropped by this run; nothing left to keep fresh.
  2396. continue
  2397. # One miss is a sample, not a verdict (#113357): a latch cancels the live
  2398. # agent run and ends the lease refresh, so confirm before acting on it.
  2399. if stop.wait(_FIRE_CLAIM_MISS_CONFIRM_SECONDS) or heartbeat_fire_claim(
  2400. job_id, expected_owner=owner):
  2401. last_confirmed = time.monotonic()
  2402. continue
  2403. lost_ownership.set()
  2404. logger.warning(
  2405. "Job '%s': fire claim ownership lost; interrupting stale run",
  2406. job_id)
  2407. return
  2408. last_confirmed = time.monotonic()
  2409. except Exception:
  2410. logger.debug("Job '%s': fire_claim heartbeat failed", job_id, exc_info=True)
  2411. if (
  2412. time.monotonic() - last_confirmed
  2413. >= _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS
  2414. ):
  2415. lost_ownership.set()
  2416. logger.warning(
  2417. "Job '%s': fire_claim could not be renewed within %.1fs; "
  2418. "interrupting uncertain run",
  2419. job_id,
  2420. _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS)
  2421. return
  2422. heartbeat_thread = _start_heartbeat_thread(
  2423. _heartbeat_loop, "cron-fire-claim-heartbeat",
  2424. lambda: logger.warning(
  2425. "Job '%s': could not start fire_claim heartbeat", job_id, exc_info=True))
  2426. if heartbeat_thread is None:
  2427. _finish_unstarted("Fire claim heartbeat could not be started; execution was not run.")
  2428. return True
  2429. try:
  2430. return run(lost_ownership)
  2431. finally:
  2432. stop.set()
  2433. heartbeat_thread.join(timeout=1.0)
  2434. def run_one_job(
  2435. job: dict, *, adapters=None, loop=None, verbose: bool = False,
  2436. extra_prompt: Optional[str] = None, cancel_event: Optional[_CancelEventLike] = None,
  2437. ) -> bool:
  2438. """Run ONE due job end-to-end: execute → save output → deliver → mark. Shared by the built-in
  2439. ticker and external providers' ``fire_due``; does NOT decide due-ness or acquire the initial
  2440. claim (callers use the store CAS) but keeps it alive. True if processed (a job failure is
  2441. recorded via ``mark_job_run``), False only if processing raised. ``cancel_event``: optional
  2442. transport-level cancel (dashboard drain)."""
  2443. # Every gateway path (built-in scheduler, external providers, and direct
  2444. # API fires) crosses this seam. Ensure the detached worker has a durable
  2445. # attempt to adopt before any launch can occur.
  2446. if not job.get("execution_id"):
  2447. execution = create_execution(
  2448. job["id"], source="direct", scheduled_instant=job.get("_scheduled_instant"))
  2449. job["execution_id"] = execution["id"]
  2450. execution_id = str(job["execution_id"])
  2451. note_cron_execution(job)
  2452. external_owner = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id
  2453. if not external_owner:
  2454. try:
  2455. handed_off = _launch_external_cron_worker(job)
  2456. except Exception as handoff_error:
  2457. # Arbitrary waiter errors must not claim dispatch failure or resend
  2458. # a worker's notice. A recovered unknown execution is different:
  2459. # its worker never committed a terminal outcome.
  2460. post_handoff = isinstance(handoff_error, _ExternalWorkerPostHandoffError)
  2461. stage = "failed after handoff" if post_handoff else "dispatch failed"
  2462. error = f"Restart-safe cron worker {stage}: {handoff_error}"
  2463. if post_handoff:
  2464. from cron.scheduler_worker_failure import record_unknown_worker_outcome
  2465. if record_unknown_worker_outcome(job, error=str(handoff_error), adapters=adapters, loop=loop):
  2466. return True
  2467. logger.error("Job '%s': %s", job["id"], error)
  2468. claim = job.get("fire_claim")
  2469. owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
  2470. delivery_error = delivery_outcome = None
  2471. try:
  2472. # A pre-handoff dispatch failure is a job failure like any other: it
  2473. # must open an incident and leave through the job's failure lane
  2474. # (#123401). Without this the outage is silent — no cron_incidents
  2475. # row, no ping — while executions.db keeps piling up failed rows.
  2476. if not post_handoff:
  2477. # Returns before _run_one_job_body / record_unknown_worker_outcome install the scope; get_secret fails closed without one under multiplex.
  2478. with _fire_secret_scope():
  2479. delivery_error, delivery_outcome = _deliver_crash_failure(
  2480. job, error, adapters=adapters, loop=loop)
  2481. from cron.unreachable_retry import is_retry_run
  2482. mark_job_run(
  2483. job["id"],
  2484. False,
  2485. error,
  2486. delivery_error=delivery_error,
  2487. **({"expected_fire_owner": owner} if owner else {}),
  2488. # A ladder re-run's occurrence already counted toward repeat.
  2489. **({"ladder_rung": True} if is_retry_run(job) else {}),
  2490. )
  2491. finally:
  2492. finish_execution(
  2493. execution_id, success=False, error=error,
  2494. delivery_outcome=delivery_outcome)
  2495. return True
  2496. if handed_off:
  2497. from cron.scheduler_worker_failure import record_unknown_worker_outcome
  2498. record_unknown_worker_outcome(job, adapters=adapters, loop=loop)
  2499. return True
  2500. if extra_prompt is None:
  2501. # Gateway-forwarded manual run stamps its prompt on the job via trigger_job; the fire that
  2502. # consumes the manual occurrence picks it up here. Single-fire: mark_job_run clears it.
  2503. _stamped = job.get("manual_run_prompt")
  2504. if _stamped and job.get("manual_run_at"):
  2505. extra_prompt = str(_stamped)
  2506. claim = job.get("fire_claim")
  2507. fire_owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
  2508. execution_token = object()
  2509. profile_home = _get_hermes_home().resolve()
  2510. _fire_key = _inflight_key(job["id"])
  2511. with _running_lock:
  2512. _running_fire_owners.setdefault(_fire_key, {})[execution_token] = (
  2513. fire_owner or None, profile_home)
  2514. try:
  2515. with self_removal_delivery_scope(job["id"]):
  2516. return _run_with_fire_claim_heartbeat(
  2517. job,
  2518. lambda lost_ownership: _run_one_job_body(
  2519. job,
  2520. adapters=adapters,
  2521. loop=loop,
  2522. verbose=verbose,
  2523. extra_prompt=extra_prompt,
  2524. claim_lost=lost_ownership,
  2525. transport_cancel=cancel_event,
  2526. execution_token=execution_token))
  2527. finally:
  2528. with _running_lock:
  2529. executions = _running_fire_owners.get(_fire_key)
  2530. if executions is not None:
  2531. executions.pop(execution_token, None)
  2532. if not executions:
  2533. _running_fire_owners.pop(_fire_key, None)
  2534. _OWNERSHIP_LOST_INTERRUPTED = "Interrupted by shutdown before terminal completion."
  2535. def _record_fire_ownership_lost(
  2536. job: dict, fire_owner: Optional[str], execution_id: str,
  2537. ) -> None:
  2538. """Bookkeeping after fire-claim ownership loss. A transport-level cancel (dashboard drain) is
  2539. not a real loss — we still own the claim, so record the interruption via the owner-fenced
  2540. terminal write instead of leaving fire_claim/last_status stale; otherwise discard.
  2541. An interrupted ladder re-run re-ran an occurrence that already counted, so its terminal
  2542. write must not spend another repeat slot (same as every other terminal path)."""
  2543. from cron.unreachable_retry import is_retry_run
  2544. job_id = job["id"]
  2545. if fire_owner is not None and heartbeat_fire_claim(job_id, expected_owner=fire_owner):
  2546. mark_job_run(
  2547. job_id, False, _OWNERSHIP_LOST_INTERRUPTED, expected_fire_owner=fire_owner,
  2548. **({"ladder_rung": True} if is_retry_run(job) else {}),
  2549. )
  2550. finish_execution(execution_id, success=False, error=_OWNERSHIP_LOST_INTERRUPTED)
  2551. else:
  2552. finish_lost_execution(
  2553. execution_id, "Fire claim ownership lost; stale result was discarded.")
  2554. def _compose_run_delivery(
  2555. job: dict, *, success: bool, error, final_response: str, output_file,
  2556. agent_declared: bool = False,
  2557. ) -> tuple[str, bool, bool, bool, Optional[str]]:
  2558. """Text to deliver for a finished run. Returns ``(deliver_content, blocked_config,
  2559. silent_alert, incident_acked, failure_incident_id)``; ``silent_alert``: an alert-once marker
  2560. says the operator was already told, deliver nothing. ``agent_declared``: *error* is the
  2561. agent's own ``[CRON_FAILURE]`` evidence, delivered verbatim."""
  2562. err = str(error) if error else ""
  2563. # Failed jobs always deliver, except blocked-config runs, which alert exactly ONCE.
  2564. blocked_config_silent = BLOCKED_CONFIG_SILENT_MARKER in err
  2565. blocked_config = blocked_config_silent or BLOCKED_CONFIG_MARKER in err
  2566. incident_acked = False
  2567. failure_incident_id = None
  2568. if blocked_config and not success:
  2569. # Bypass the generic failure summarizer (its auth/timeout heuristics would mislabel this).
  2570. _pf_text = re.sub(r"\[blocked_config[^\]]*\]\s*", "", err).strip()
  2571. from cron.scheduler_failure_copy import blocked_config_notice
  2572. deliver_content = blocked_config_notice(job.get("name") or job["id"], _pf_text)
  2573. elif success:
  2574. deliver_content = final_response
  2575. _resolve_incidents_for_recovered_job(job)
  2576. else:
  2577. # Record the job+error signature once; withhold the per-run ping while the operator
  2578. # already acked it (closed) or was already told (alerted, inside the reminder cooldown).
  2579. # Best-effort: a ledger failure never breaks delivery.
  2580. incident_acked, failure_incident_id = _upsert_incident_for_failure(
  2581. job, error or "", output_file=output_file
  2582. )
  2583. if incident_acked:
  2584. deliver_content = ""
  2585. elif agent_declared:
  2586. # The agent already diagnosed the failure in prose; the summarizer's substring
  2587. # heuristics would re-diagnose it ("timed out" -> blame the model service, "401" ->
  2588. # "sign in again") and attach the wrong remediation. Deliver the evidence as-is.
  2589. from cron.scheduler_failure_copy import generic_failure_notice
  2590. deliver_content = generic_failure_notice(
  2591. job.get("name") or job["id"], job["id"], err.strip().rstrip("."),
  2592. ) + _failure_streak_nudge(job)
  2593. else:
  2594. from cron.quota_hold import hold_notice
  2595. deliver_content = (
  2596. _summarize_cron_failure_for_delivery(job, error) + _failure_streak_nudge(job)
  2597. # The one alert on entering a provider-window hold says so (#89376).
  2598. + hold_notice(job, job.get("_quota_hold_seconds"))
  2599. )
  2600. return deliver_content, blocked_config, blocked_config_silent, incident_acked, failure_incident_id
  2601. class _FireClaimLostDuringSideEffect(Exception):
  2602. """Raised inside a side-effect fence when the durable fire claim is no longer ours."""
  2603. class _FireOwnership:
  2604. """Fire-claim ownership checks for one run (``owner`` is None when the job carries no claim)."""
  2605. def __init__(
  2606. self, job: dict, claim_lost: Optional[_CancelEventLike] = None,
  2607. transport_cancel: Optional[_CancelEventLike] = None,
  2608. operator_cancel: Optional[_CancelEventLike] = None,
  2609. ):
  2610. self.job = job
  2611. # Two handles: the heartbeat's raw ``lost_ownership`` event and the caller's transport
  2612. # ``cancel_event``. A sampled miss latches on the raw one only — the combined view's
  2613. # ``set()`` would propagate into the transport event this run does not own (#105861).
  2614. self.claim_lost = claim_lost
  2615. self.transport_cancel = transport_cancel
  2616. self.operator_cancel = operator_cancel
  2617. # ``operator_cancel`` (``hermes cron cancel``) is a THIRD source, deliberately not folded
  2618. # into ``transport_cancel``: only a transport cancel counts as claim loss for bookkeeping,
  2619. # while an operator cancel is a real outcome this run records itself.
  2620. # What ``run_job`` receives as its ``cancel_event``: any source cancels the run.
  2621. self.cancel_event: Optional[_CancelEventLike] = (
  2622. _CombinedCancelEvent(claim_lost, transport_cancel, operator_cancel)
  2623. if claim_lost is not None or transport_cancel is not None or operator_cancel is not None
  2624. else None)
  2625. claim = job.get("fire_claim")
  2626. self.owner = str(claim.get("by") or "") if isinstance(claim, dict) else None
  2627. def transport_cancelled(self) -> bool:
  2628. return self.transport_cancel is not None and self.transport_cancel.is_set()
  2629. def operator_cancelled(self) -> bool:
  2630. return self.operator_cancel is not None and self.operator_cancel.is_set()
  2631. def side_effect_fence(self):
  2632. if self.owner is None:
  2633. return contextlib.nullcontext(True)
  2634. return fire_claim_fence(self.job["id"], expected_owner=self.owner)
  2635. def lost(self) -> bool:
  2636. if self.transport_cancelled():
  2637. return True
  2638. if self.owner is None:
  2639. return False
  2640. if self_removal_delivery_allowed(self.job["id"]):
  2641. # The run deleted its own record; there is no claim left to re-resolve.
  2642. return False
  2643. # The heartbeat's latched miss is one sample, not the verdict (#113357): the store is
  2644. # re-checked here, and a claim that still validates keeps the run's real outcome. The
  2645. # owner-fenced mark_job_run / fire_claim_fence remain the authority for side effects.
  2646. latched = self.claim_lost is not None and self.claim_lost.is_set()
  2647. try:
  2648. if heartbeat_fire_claim(self.job["id"], expected_owner=self.owner):
  2649. return False
  2650. except Exception:
  2651. logger.debug(
  2652. "Job '%s': fire_claim ownership validation failed", self.job["id"], exc_info=True)
  2653. # Unreachable store after a latched miss stays fail-closed; a first miss is best-effort.
  2654. return latched
  2655. if self.claim_lost is not None:
  2656. self.claim_lost.set()
  2657. return True
  2658. @dataclass
  2659. class _RunDelivery:
  2660. """Mutable outcome of the save/compose/deliver phase, read back by the bookkeeping tail."""
  2661. job: dict
  2662. success: bool
  2663. error: Optional[str]
  2664. delivery_attempted: bool = False
  2665. delivery_error: Optional[str] = None
  2666. should_deliver: bool = False
  2667. unresolved_origin: bool = False
  2668. blocked_config: bool = False
  2669. # True when ``error`` is the agent's own ``[CRON_FAILURE]`` evidence rather than a runtime
  2670. # error string, so composition must not run it through the provider-error heuristics.
  2671. agent_declared: bool = False
  2672. incident_acked: bool = False
  2673. failure_incident_id: Optional[str] = None
  2674. side_effect_ownership_lost: bool = False
  2675. def _save_compose_deliver(
  2676. d: _RunDelivery, fence: _FireOwnership, final_response: str, output: str, *,
  2677. adapters, loop, verbose: bool, execution_token,
  2678. ) -> None:
  2679. """Save output, compose the notice and deliver it (both side effects run under the fire-claim
  2680. fence; a lost claim raises ``_FireClaimLostDuringSideEffect`` for the caller)."""
  2681. job = d.job
  2682. with fence.side_effect_fence() as owns_output:
  2683. if not owns_output:
  2684. raise _FireClaimLostDuringSideEffect
  2685. # remove_job() already deleted this job's output dir; saving would re-create an orphan.
  2686. output_file = (
  2687. None if self_removal_delivery_allowed(job["id"])
  2688. else save_job_output(job["id"], output))
  2689. if verbose and output_file is not None:
  2690. logger.info("Output saved to: %s", output_file)
  2691. # A shutdown-killed tool subprocess can leave a plausible final_response from truncated
  2692. # output; force the honest "interrupted" failure path. Peek-only (consumed later).
  2693. if d.success and _is_interrupted(job["id"], execution_token):
  2694. d.success = False
  2695. d.error = (
  2696. "Interrupted by gateway shutdown before the run finished "
  2697. "(tool subprocess was killed mid-flight)."
  2698. )
  2699. (
  2700. deliver_content, d.blocked_config, _silent_alert, d.incident_acked, d.failure_incident_id,
  2701. ) = _compose_run_delivery(
  2702. job, success=d.success, error=d.error, final_response=final_response,
  2703. output_file=output_file, agent_declared=d.agent_declared)
  2704. # Whitespace-only == empty: skip delivery; the guard below marks it a soft failure.
  2705. d.should_deliver = bool(deliver_content.strip()) and not _silent_alert
  2706. if d.should_deliver and not d.success and job.get("_model_unreachable"):
  2707. # The model was never reached and a bounded automatic re-run will be scheduled
  2708. # (cron/unreachable_retry.py): hold the failure notice — the re-run either
  2709. # delivers the real result or, once the ladder is exhausted, the next failure
  2710. # alerts normally. Mirrors Cowork's silent 5/15/30-minute re-runs.
  2711. from cron.unreachable_retry import will_retry
  2712. if will_retry(job):
  2713. d.should_deliver = False
  2714. logger.info(
  2715. "Job '%s': suppressing failure notice — automatic re-run pending", job["id"])
  2716. # Not a substring check: bare "SILENT"/"NO_REPLY" or a report quoting "[SILENT]" must
  2717. # not be swallowed; bracketed-prefix / trailing-line tolerance is kept.
  2718. if d.should_deliver and d.success and _is_cron_silence_response(deliver_content):
  2719. # Cron silence suppression — see _is_cron_silence_response. Replaces the old `SILENT_MARKER in
  2720. # ...upper()` substring check, which both leaked bracketless near-markers ("SILENT" / "NO_REPLY")
  2721. # and wrongly swallowed a real report that merely quoted "[SILENT]" mid-sentence (#51438, #46917).
  2722. logger.info("Job '%s': agent returned %s — skipping delivery", job["id"], SILENT_MARKER)
  2723. d.should_deliver = False
  2724. if d.should_deliver and fence.lost():
  2725. d.should_deliver = False
  2726. logger.warning("Job '%s': skipping delivery after fire claim ownership loss", job["id"])
  2727. if not d.should_deliver:
  2728. return
  2729. d.unresolved_origin = (
  2730. _normalize_deliver_value(_delivery_lane_value(job, for_failure=not d.success)) == "origin"
  2731. and not _resolve_delivery_targets(job, for_failure=not d.success)
  2732. )
  2733. execution_id = job.get('execution_id')
  2734. if execution_id and not job.get('no_agent'):
  2735. from cron.delivery_queue import enqueue
  2736. # Nothing to send: an origin-less job is ``not_configured``, never a queued "delivery".
  2737. if d.unresolved_origin or _normalize_deliver_value(
  2738. _delivery_lane_value(job, for_failure=not d.success)) == "local":
  2739. return
  2740. queued_job = dict(job)
  2741. if d.failure_incident_id:
  2742. queued_job["_failure_incident_id"] = d.failure_incident_id
  2743. # The queue row IS the send: publish it under the same fence as a direct send, so a claim
  2744. # stolen after the lost() sample above cannot leave this stale fire's notice queued.
  2745. with fence.side_effect_fence() as owns_delivery:
  2746. if not owns_delivery:
  2747. raise _FireClaimLostDuringSideEffect
  2748. queued = enqueue(execution_id, queued_job, deliver_content, for_failure=not d.success)
  2749. # The queue owns this send even if subsequent bookkeeping fails.
  2750. d.delivery_attempted = True
  2751. job['last_delivery_queued'] = {'canonical': {'status': queued['status'], 'execution_id': execution_id}}
  2752. return
  2753. try:
  2754. with fence.side_effect_fence() as owns_delivery:
  2755. if not owns_delivery:
  2756. raise _FireClaimLostDuringSideEffect
  2757. d.delivery_attempted = True
  2758. d.delivery_error = _deliver_result(
  2759. job,
  2760. deliver_content,
  2761. adapters=adapters,
  2762. loop=loop,
  2763. # Failure summaries (and drift/blocked-config alerts composed into deliver_content
  2764. # on the failure path) honor the job's failure_deliver override (NS-788).
  2765. for_failure=not d.success,
  2766. )
  2767. except Exception as de:
  2768. if isinstance(de, _FireClaimLostDuringSideEffect):
  2769. raise
  2770. d.delivery_error = str(de)
  2771. logger.error("Delivery failed for job %s: %s", job["id"], de)
  2772. def _deliver_crash_failure(
  2773. job: dict, err_text: str, *, adapters, loop,
  2774. ) -> tuple[Optional[str], str]:
  2775. """Failure notice for a run that raised out of run_job. Returns (delivery_error, outcome)."""
  2776. normalized_deliver = _normalize_deliver_value(_delivery_lane_value(job, for_failure=True))
  2777. # Same ack gate as the normal failure delivery: acked signatures stay silent here too.
  2778. incident_acked, failure_incident_id = _upsert_incident_for_failure(job, err_text)
  2779. if incident_acked:
  2780. return None, "suppressed_acked"
  2781. delivery_error = None
  2782. try:
  2783. delivery_error = _deliver_result(
  2784. job,
  2785. # Same text as the normal failure delivery: this run also counts toward
  2786. # failure_streak, so the nudge must leave through here too.
  2787. _summarize_cron_failure_for_delivery(job, err_text) + _failure_streak_nudge(job),
  2788. adapters=adapters,
  2789. loop=loop,
  2790. for_failure=True,
  2791. )
  2792. except Exception as delivery_exc:
  2793. delivery_error = str(delivery_exc)
  2794. logger.error("Delivery failed for job %s: %s", job["id"], delivery_exc)
  2795. unresolved_origin = bool(
  2796. not delivery_error
  2797. and normalized_deliver == "origin"
  2798. and not _resolve_delivery_targets(job, for_failure=True)
  2799. )
  2800. delivery_outcome = _classify_delivery_outcome(
  2801. delivery_error=delivery_error, should_deliver=True, unresolved_origin=unresolved_origin,
  2802. normalized_deliver=normalized_deliver, incident_acked=False, success=False,
  2803. delivery_queued=job.get("last_delivery_queued"),
  2804. notification_suppressed=bool(job.get("_notification_all_targets_suppressed")))
  2805. if delivery_outcome in ("delivered", "not_configured"):
  2806. _mark_incident_alerted(failure_incident_id)
  2807. return delivery_error, delivery_outcome
  2808. def _install_fire_secret_scope() -> tuple[contextvars.Token, Optional[contextvars.Token]]:
  2809. """Install the firing profile's secret scope for the span ``_run_one_job_body`` runs, delivery
  2810. included, and return the tokens ``_reset_fire_secret_scope`` needs.
  2811. Hydrate the profile's external secret sources BEFORE freezing the scope — the order
  2812. gateway/run.py and the external cron worker already use: ``build_profile_secret_scope`` only
  2813. READS the per-home source map, so a scope frozen first would carry no vault-backed value.
  2814. For a fire routed to a profile other than the process's own (``routed_profile_fire`` on the
  2815. fire's home — every entry point: ticker, dashboard Run now, CLI) also run under multiplex semantics — for
  2816. exactly this span and no wider. The desktop backend ticks every local profile from a process
  2817. that is not a multiplexer, so nothing else isolates that fire; and switching the context on
  2818. here rather than at the tick means no read is ever fail-closed without a scope to read — the
  2819. restart-safe handoff in ``run_one_job`` runs before this and keeps today's semantics (#107692).
  2820. """
  2821. from agent.secret_scope import set_multiplex_context, set_secret_scope
  2822. from cron.scheduler_provider import routed_profile_fire
  2823. from hermes_cli.env_loader import hydrate_profile_secret_sources
  2824. from tui_gateway.launch_profile_policy import owner_secret_scope
  2825. home = Path(_get_hermes_home())
  2826. hydrate_profile_secret_sources(home)
  2827. scope_token = set_secret_scope(owner_secret_scope(home), profile_home=str(home))
  2828. context_token = set_multiplex_context(True) if routed_profile_fire() else None
  2829. return scope_token, context_token
  2830. def _reset_fire_secret_scope(tokens: tuple[contextvars.Token, Optional[contextvars.Token]]) -> None:
  2831. """Undo ``_install_fire_secret_scope`` — the context first, so multiplex semantics never
  2832. outlive the scope they depend on."""
  2833. from agent.secret_scope import reset_multiplex_context, reset_secret_scope
  2834. scope_token, context_token = tokens
  2835. if context_token is not None:
  2836. reset_multiplex_context(context_token)
  2837. reset_secret_scope(scope_token)
  2838. @contextlib.contextmanager
  2839. def _fire_secret_scope():
  2840. """``_install_fire_secret_scope`` for the ``with`` block, reset on the way out."""
  2841. tokens = _install_fire_secret_scope()
  2842. try:
  2843. yield
  2844. finally:
  2845. _reset_fire_secret_scope(tokens)
  2846. def _start_owned_run(job: dict, execution_id: str) -> Optional[contextvars.Token]:
  2847. """Win the run's ``claimed`` → ``running`` CAS and bind its cron identity; ``None`` when the run
  2848. lost ownership first. A restart-safe worker already won it by adopting the row."""
  2849. if os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id:
  2850. record = get_execution(execution_id)
  2851. elif (record := mark_execution_running(execution_id)) is None:
  2852. return None
  2853. return enter_cron_execution(job, execution_id, record or {})
  2854. def _apply_agent_failure_marker(job: dict, success: bool, error, final_response):
  2855. """An agent can finish its own turn after a delegated child has failed. Let it explicitly
  2856. declare that semantic failure so the existing failure path updates status, streaks, ledger,
  2857. and notification routing instead of recording a false healthy result.
  2858. Returns ``(success, error, agent_declared)``."""
  2859. if success and not job.get("no_agent"):
  2860. marker_error = _cron_failure_marker_error(final_response)
  2861. if marker_error is not None:
  2862. return False, marker_error, True
  2863. return success, error, False
  2864. def _run_one_job_body(
  2865. job: dict, *, adapters=None, loop=None, verbose: bool = False,
  2866. extra_prompt: Optional[str] = None, claim_lost: Optional[_CancelEventLike] = None,
  2867. transport_cancel: Optional[_CancelEventLike] = None,
  2868. execution_token: Optional[object] = None,
  2869. ) -> bool:
  2870. execution_id = job.get("execution_id")
  2871. if not execution_id:
  2872. execution_id = create_execution(
  2873. job["id"], source="direct", scheduled_instant=job.get("_scheduled_instant"))["id"]
  2874. job = dict(job, execution_id=execution_id)
  2875. # The execution id is resolved BEFORE the fence so ``hermes cron cancel`` can join the same
  2876. # cancel seam: the marker it writes is polled by whichever process owns this attempt.
  2877. fence = _FireOwnership(job, claim_lost, transport_cancel, CancelRequest(execution_id))
  2878. fire_owner = fence.owner
  2879. _side_effect_fence = fence.side_effect_fence
  2880. _fire_claim_ownership_lost = fence.lost
  2881. delivery_attempted = False
  2882. delivery_error = None
  2883. _fire_scope_tokens = None
  2884. _terminal_scope_token = None
  2885. _identity_token = None
  2886. try:
  2887. # get_secret() fails closed outside a scope; the ticker thread has none. Delivery adapters
  2888. # resolve credentials, so the scope must span delivery too (reset in the outer finally) —
  2889. # including the crash notice in the except below when claim_dispatch or
  2890. # mark_execution_running raises, so install it before either.
  2891. _fire_scope_tokens = _install_fire_secret_scope()
  2892. # Commit a finite one-shot's dispatch BEFORE its side effect so a tick dying mid-run cannot
  2893. # re-fire it forever on restart. No-op for recurring/infinite jobs (at-most-times).
  2894. # This lives here in the shared body so BOTH the built-in ticker and the external provider (Chronos
  2895. # fire_due) get at-most-times semantics. See #38758.
  2896. if not claim_dispatch(job["id"]):
  2897. logger.info(
  2898. "Job '%s': one-shot dispatch limit reached — skipping",
  2899. job.get("name", job["id"]))
  2900. finish_execution(
  2901. execution_id, success=False,
  2902. error="Dispatch claim rejected; execution was not started.")
  2903. return True # not an error — already handled/removed
  2904. # Claimed durably before dispatch; becomes running only right before the actual run.
  2905. # Detached workers transition to running while adopting; in-process paths must win the
  2906. # claimed->running CAS here before any user script or agent side effect may begin.
  2907. # The identity plugins read via ctx.current_cron_execution() is bound only on that win.
  2908. _identity_token = _start_owned_run(job, execution_id)
  2909. if _identity_token is None:
  2910. logger.warning("Cron job %s lost execution ownership before start; skipping", job["id"])
  2911. return True
  2912. # Bind the firing profile's COMPLETE terminal policy for this fire — agent build, run, delivery
  2913. # and bookkeeping (gateway/run.py _profile_runtime_scope does the same per turn) — else the
  2914. # ticker reads process-global TERMINAL_* env another profile pinned (#68559, #98581). A
  2915. # malformed policy installs a refusal scope: terminal execution raises instead.
  2916. from tools.terminal_scope import (
  2917. install_profile_terminal_scope)
  2918. _terminal_scope_token = install_profile_terminal_scope(_get_hermes_home())
  2919. # Defer agent teardown until AFTER delivery: closing first races the live send against a
  2920. # torn-down async client (#58720). run_job hands the agent back via this list instead.
  2921. _deferred_agents: list = []
  2922. def _teardown_deferred() -> None:
  2923. # run_job's finally still hands back the agent when it raises; tear it down here so a failed run
  2924. # never leaks its async resources (#10200), then re-raise into the outer handler. BaseException
  2925. # (not just Exception) so a KeyboardInterrupt/SystemExit mid-run still triggers teardown before
  2926. # propagating.
  2927. # Tear down the deferred agent(s) now that save + delivery have run (or raised). Must happen on
  2928. # every path so cron agents never leak their subprocesses/clients (#10200).
  2929. for _deferred_agent in _deferred_agents:
  2930. _teardown_cron_agent(_deferred_agent, job["id"])
  2931. _run_kwargs = {
  2932. "defer_agent_teardown": _deferred_agents,
  2933. "extra_prompt": extra_prompt,
  2934. "execution_id": execution_id}
  2935. if fence.cancel_event is not None:
  2936. _run_kwargs["cancel_event"] = fence.cancel_event
  2937. outcome = _run_job_unless_cancelled(
  2938. run_job, job, _run_kwargs, _teardown_deferred, fence, execution_id, fire_owner)
  2939. if outcome is None: # an operator cancel stopped it and is already booked
  2940. return True
  2941. success, output, final_response, error = outcome
  2942. if _fire_claim_ownership_lost():
  2943. _teardown_deferred()
  2944. _record_fire_ownership_lost(job, fire_owner, execution_id)
  2945. return True
  2946. success, error, agent_declared = _apply_agent_failure_marker(job, success, error, final_response)
  2947. # Agent is still live through delivery; wrap ALL of save/compose/deliver in try/finally so a
  2948. # raise anywhere still tears the deferred agent down.
  2949. d = _RunDelivery(job=job, success=success, error=error, agent_declared=agent_declared)
  2950. try:
  2951. _save_compose_deliver(
  2952. d, fence, final_response, output, adapters=adapters, loop=loop, verbose=verbose,
  2953. execution_token=execution_token)
  2954. except _FireClaimLostDuringSideEffect:
  2955. d.side_effect_ownership_lost = True
  2956. finally:
  2957. delivery_attempted, delivery_error = d.delivery_attempted, d.delivery_error
  2958. # Every path must tear down deferred agent(s) so they never leak subprocesses/clients.
  2959. _teardown_deferred()
  2960. if d.side_effect_ownership_lost:
  2961. # The claim died inside a side-effect fence: the side effect did NOT complete.
  2962. _record_fire_ownership_lost(job, fire_owner, execution_id)
  2963. return True
  2964. from cron.scheduler_bookkeeping import fail_empty_response
  2965. fail_empty_response(d, final_response)
  2966. if _fire_claim_ownership_lost():
  2967. # #105861: the claim check is one sample; a miss AFTER a completed delivery must not
  2968. # overwrite the delivered run's terminal status — ok, or a failure whose notice already
  2969. # left with its real error — so fall through to finish_completed_run, whose owner-fenced
  2970. # mark_job_run is authoritative either way. An explicit transport cancel stays fail-closed.
  2971. transport_cancelled = fence.transport_cancelled()
  2972. if d.delivery_attempted and not d.delivery_error and not transport_cancelled:
  2973. logger.warning(
  2974. "Job '%s': fire claim ownership lost after completed delivery; "
  2975. "recording the delivered run's terminal status",
  2976. job["id"])
  2977. else:
  2978. if transport_cancelled:
  2979. logger.warning(
  2980. "Job '%s': transport cancellation arrived before terminal completion; "
  2981. "recording the interrupted run",
  2982. job["id"])
  2983. _record_fire_ownership_lost(job, fire_owner, execution_id)
  2984. return True
  2985. if _consume_interrupted_flag(job["id"], execution_token):
  2986. from cron.scheduler_bookkeeping import finish_interrupted_run
  2987. finish_interrupted_run(job, execution_id, delivery_error)
  2988. return True
  2989. from cron.scheduler_bookkeeping import finish_completed_run
  2990. return finish_completed_run(d, fire_owner, execution_id)
  2991. except BaseException as e:
  2992. from cron.scheduler_authority import CronExecutionUnknown
  2993. if isinstance(e, CronExecutionUnknown):
  2994. from cron.executions import give_up_execution
  2995. from cron.scheduler_authority import pause_for_unknown
  2996. pause_for_unknown(job["id"], execution_id, str(e))
  2997. give_up_execution(execution_id, reason=str(e))
  2998. logger.error("Cron %s paused with unverified admission: %s", job["id"], e)
  2999. return False
  3000. # BaseException, not Exception: CancelledError/KeyboardInterrupt/SystemExit propagate here.
  3001. # Without mark_job_run(False) a finite one-shot is wedged: claim_dispatch consumed
  3002. # repeat.completed but last_run_at is never written. Record first, then re-raise
  3003. # non-Exception. Owner fencing still applies.
  3004. # BaseException, not Exception (#73973): the inner run_job handler re-raises CancelledError /
  3005. # KeyboardInterrupt / SystemExit after agent teardown, and none of those are Exception subclasses.
  3006. # If they escape without mark_job_run(False), a finite one-shot is left wedged — claim_dispatch()
  3007. # already consumed repeat.completed, but last_run_at is never written, so the job sits in state
  3008. # "scheduled" until the run-claim TTL expires and the dispatch-limit guard removes it with no output
  3009. # and no error. Owner fencing still applies: a stale worker must not record over a replacement claim
  3010. # owner.
  3011. _err_text = str(e) or type(e).__name__
  3012. # A config backend outage (a SystemExit by design) fails this fire, never the ticker.
  3013. contained = isinstance(e, (Exception, ConfigBackendUnavailable))
  3014. logger.error(
  3015. "Error processing job %s: %s",
  3016. job["id"],
  3017. _err_text,
  3018. exc_info=(type(e), e, e.__traceback__))
  3019. delivery_outcome = "suppressed"
  3020. # Owner fencing: a stale worker whose claim was taken over (or transport-cancelled) must not
  3021. # send a failure alert on top of the replacement run's; fall through to fenced bookkeeping.
  3022. if (
  3023. contained
  3024. and not delivery_attempted
  3025. and not isinstance(e, _FireClaimLostDuringSideEffect)
  3026. and not _fire_claim_ownership_lost()
  3027. ):
  3028. delivery_error, delivery_outcome = _deliver_crash_failure(
  3029. job, _err_text, adapters=adapters, loop=loop)
  3030. try:
  3031. if (
  3032. not _consume_interrupted_flag(job["id"], execution_token)
  3033. and not self_removal_delivery_allowed(job["id"]) # no record left to mark
  3034. ):
  3035. mark_kwargs = {}
  3036. if fire_owner is not None:
  3037. mark_kwargs["expected_fire_owner"] = fire_owner
  3038. if contained:
  3039. mark_kwargs["delivery_error"] = delivery_error
  3040. from cron.unreachable_retry import is_retry_run
  3041. if is_retry_run(job):
  3042. # A crashed ladder re-run: its occurrence already counted toward repeat.
  3043. mark_kwargs["ladder_rung"] = True
  3044. mark_job_run(job["id"], False, _err_text, **mark_kwargs)
  3045. except Exception as record_err:
  3046. # Never let bookkeeping mask the original interruption.
  3047. logger.error("Failed to record interrupted run for job %s: %s", job["id"], record_err)
  3048. try:
  3049. finish_execution(
  3050. execution_id, success=False, error=_err_text, delivery_outcome=delivery_outcome)
  3051. except Exception as record_err:
  3052. logger.error("Failed to finish execution record for job %s: %s", job["id"], record_err)
  3053. if not contained:
  3054. raise
  3055. return False
  3056. finally:
  3057. # Function-level on purpose: must scope delivery, deferred teardown, claim-loss handling and
  3058. # bookkeeping — not just run_job. Do not move into the run block's finally.
  3059. # The attempt is over either way, so its cancel marker goes with it — an unread marker
  3060. # would otherwise sit in the cancels dir until the TTL prune.
  3061. clear_cancel(execution_id)
  3062. exit_cron_execution(_identity_token)
  3063. if _fire_scope_tokens is not None:
  3064. _reset_fire_secret_scope(_fire_scope_tokens)
  3065. if _terminal_scope_token is not None:
  3066. from tools.terminal_scope import reset_terminal_scope
  3067. reset_terminal_scope(_terminal_scope_token)
  3068. def _wait_for_external_cron_worker_body(
  3069. process: subprocess.Popen,
  3070. *,
  3071. execution_id: str,
  3072. stderr_path: Optional[Path] = None,
  3073. ) -> bool:
  3074. """Preserve ``run_one_job``'s synchronous contract after handoff.
  3075. The worker owns the durable execution and survives this gateway process.
  3076. The caller nevertheless waits while it remains alive so manual/background
  3077. callers do not release their in-process guard or report stale job state.
  3078. A gateway replacement may kill this waiter; it does not kill the scoped
  3079. worker or change its ledger ownership.
  3080. """
  3081. def _is_terminal() -> bool:
  3082. current = get_execution(execution_id)
  3083. return bool(current and current.get("status") in _TERMINAL_STATES)
  3084. # The worker commits its terminal row before its process exits, so exit is
  3085. # the correct wakeup. Each ledger read opens a connection and re-runs
  3086. # schema init; polling it at 50ms for an hours-long agent run is ~72k
  3087. # opens/hour of pure contention with the worker's own writes.
  3088. while True:
  3089. try:
  3090. returncode = process.wait(timeout=1.0)
  3091. except subprocess.TimeoutExpired:
  3092. if not _is_terminal():
  3093. continue
  3094. # Recovery can win between the timeout and this ledger read. If
  3095. # the worker has since exited, consume its diagnostics below before
  3096. # cleanup; only a still-live terminal worker needs background reaping.
  3097. returncode = process.poll()
  3098. if returncode is None:
  3099. from cron.scheduler_detached_worker import reap_terminal_worker_in_background
  3100. reap_terminal_worker_in_background(process)
  3101. return True
  3102. # The worker can commit its terminal row and exit between the first
  3103. # read and wait(). Re-read the exact attempt before declaring that
  3104. # it died without terminalizing.
  3105. current = get_execution(execution_id)
  3106. if current and current.get("status") in FIRER_FINISHED_STATES:
  3107. return True
  3108. # If the adopted worker died without terminalizing, its owner is
  3109. # now provably gone. Recover to ``unknown`` rather than routing the
  3110. # exception through the pre-handoff dispatch-failure path, which
  3111. # would falsely assert that no side effect could have happened.
  3112. #
  3113. # Record the cause THIS waiter observed instead of letting the blanket
  3114. # sweep stamp "Scheduler restarted ...", and then RAISE: the sweep leaves
  3115. # the row terminal, so returning success here made run_one_job skip
  3116. # mark_job_run entirely and the job's fire claim outlived its lost
  3117. # execution, refusing the next manual fire with "Job is already being
  3118. # fired" (#128509). Raising routes the existing post-handoff bookkeeping
  3119. # instead: retire the claim and report the uncertain run, not a pre-dispatch failure.
  3120. from cron.scheduler_worker_failure import external_worker_exited_reason
  3121. reason = external_worker_exited_reason(returncode)
  3122. if not terminalize_dead_owner(execution_id, reason=reason):
  3123. recover_interrupted_executions()
  3124. # A concurrent sweep may have won the unknown transition. Still
  3125. # surface this waiter's exit code and stderr instead of discarding them.
  3126. current = get_execution(execution_id)
  3127. if current and current.get("status") in FIRER_FINISHED_STATES:
  3128. return True
  3129. stderr_tail = ""
  3130. if stderr_path is not None:
  3131. from cron.scheduler_diagnostics import external_worker_stderr_tail
  3132. stderr_tail = external_worker_stderr_tail(stderr_path)
  3133. raise RuntimeError(f"{reason}{stderr_tail}")
  3134. class _ExternalWorkerPostHandoffError(RuntimeError):
  3135. """The waiter failed after the worker was spawned and may own the execution."""
  3136. def _wait_for_external_cron_worker(
  3137. process: subprocess.Popen,
  3138. *,
  3139. execution_id: str,
  3140. job_id: Optional[str] = None,
  3141. handoff_files: tuple[Path, ...] = (),
  3142. stderr_path: Optional[Path] = None,
  3143. ) -> bool:
  3144. try:
  3145. return _wait_for_external_cron_worker_body(
  3146. process, execution_id=execution_id, stderr_path=stderr_path
  3147. )
  3148. except Exception as wait_error:
  3149. raise _ExternalWorkerPostHandoffError(str(wait_error)) from wait_error
  3150. finally:
  3151. if job_id is not None:
  3152. _key = _inflight_key(job_id)
  3153. with _running_lock:
  3154. _restart_safe_waiter_job_ids.discard(_key)
  3155. _scope_isolated_job_ids.discard(_key)
  3156. # The execution is terminal or its worker is dead: nobody will read a
  3157. # payload or acknowledgement left behind by a late/unread handoff.
  3158. for stale in handoff_files:
  3159. try:
  3160. stale.unlink(missing_ok=True)
  3161. except OSError:
  3162. pass
  3163. def _launch_external_cron_worker(job: dict) -> bool:
  3164. """Launch *job* outside the managed gateway process when required.
  3165. Returns ``False`` outside a managed systemd gateway (in-process path). In
  3166. managed topology the job always goes to an external worker with the #101940
  3167. ownership handoff: in a transient user scope, or — when no user D-Bus
  3168. session exists and ``cron.require_restart_safe_scope`` is false — as a
  3169. direct subprocess (process separation kept, cgroup isolation lost).
  3170. A fire routed to a profile other than the process's own is multiplexed at THIS boundary too.
  3171. ``run_one_job`` switches the context on in ``_install_fire_secret_scope``, which runs AFTER
  3172. this handoff, so a routed desktop fire on the managed path serialized ``multiplex_active=False``
  3173. and built the worker environment with the launch profile's residue and no scrub (review on
  3174. f5f88d5058). Enable it for exactly this span; the worker then re-establishes it from the payload.
  3175. """
  3176. execution_id = str(job["execution_id"])
  3177. job_id = str(job["id"])
  3178. handoff_dir = _get_hermes_home() / "cron" / "external-workers"
  3179. payload_path = handoff_dir / f"{execution_id}.json"
  3180. ack_path = handoff_dir / f"{execution_id}.ready"
  3181. # Captured so a worker that dies before its acknowledgement can name the cause (#112729).
  3182. stderr_path = handoff_dir / f"{execution_id}.stderr"
  3183. command = [
  3184. sys.executable,
  3185. "-m",
  3186. "cron.scheduler",
  3187. "--external-worker-file",
  3188. str(payload_path),
  3189. "--ack-file",
  3190. str(ack_path),
  3191. ]
  3192. from agent.secret_scope import is_multiplex_active
  3193. from cron.scheduler_provider import routed_profile_fire
  3194. from tools.environments.local import build_subprocess_env, strip_launch_profile_env
  3195. from tools.process_registry import (
  3196. restart_safe_gateway_child_argv,
  3197. scoped_spawn_lost_user_bus,
  3198. systemd_user_bus_env,
  3199. )
  3200. # A fire routed to another profile is multiplexed at this handoff even when the process flag is
  3201. # off (the desktop ticker is not a multiplexer): the payload says so, and the worker env is built
  3202. # under that context so the scrub and the launch-residue strip both apply. The context is set
  3203. # for exactly the env build; nothing else here reads it.
  3204. multiplex_active = is_multiplex_active() or routed_profile_fire()
  3205. try:
  3206. require_restart_safe_scope = bool(
  3207. (load_config_readonly().get("cron") or {}).get("require_restart_safe_scope", False)
  3208. )
  3209. except Exception:
  3210. require_restart_safe_scope = False
  3211. dispatch = restart_safe_gateway_child_argv(
  3212. command,
  3213. unit_suffix=f"cron-{job_id}-exec-{execution_id}",
  3214. require_restart_safe_scope=require_restart_safe_scope,
  3215. )
  3216. if dispatch.mode == "in_process":
  3217. return False
  3218. if mark_execution_handoff_pending(execution_id) is None:
  3219. raise RuntimeError(
  3220. "cron execution claim changed before external worker handoff"
  3221. )
  3222. _ensure_cron_dir(handoff_dir)
  3223. try:
  3224. handoff_dir.chmod(0o700)
  3225. except OSError:
  3226. pass
  3227. fd = os.open(payload_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
  3228. try:
  3229. with os.fdopen(fd, "w", encoding="utf-8") as payload_file:
  3230. json.dump(
  3231. {
  3232. "job": job,
  3233. "profile_home": str(_get_hermes_home().resolve()),
  3234. "multiplex_active": multiplex_active,
  3235. },
  3236. payload_file,
  3237. )
  3238. payload_file.flush()
  3239. os.fsync(payload_file.fileno())
  3240. except BaseException:
  3241. payload_path.unlink(missing_ok=True)
  3242. raise
  3243. profile_home = _get_hermes_home().resolve()
  3244. # Same hydrate -> scope -> (routed) multiplex-context install the in-process fire uses, for exactly
  3245. # the env build; the helper's reset order keeps the context from outliving its scope.
  3246. with _fire_secret_scope():
  3247. # No restore_managed_env here: the worker re-runs load_hermes_dotenv -> _apply_managed_env at
  3248. # import, and strip_launch_profile_env leaves managed keys in place.
  3249. worker_env = strip_launch_profile_env(build_subprocess_env(
  3250. scrub_secrets=multiplex_active,
  3251. inherit_profile_home=True,
  3252. extra={"HERMES_HOME": str(profile_home)},
  3253. ))
  3254. worker_env = systemd_user_bus_env(worker_env)
  3255. # Unattended worker: the gateway sets HERMES_EXEC_ASK at startup (interactive launches set
  3256. # the other two), and an inherited presence var makes every env-fallback consumer in the
  3257. # child (`_is_interactive_cli`, sudo prompting, `check_cronjob_requirements`) believe a
  3258. # human is present to answer (#110932).
  3259. for _presence_var in (
  3260. "HERMES_INTERACTIVE",
  3261. "HERMES_GATEWAY_SESSION",
  3262. "HERMES_EXEC_ASK",
  3263. ):
  3264. worker_env.pop(_presence_var, None)
  3265. # `-m cron.scheduler` has no hermes_cli.main bootstrap; pin this checkout explicitly
  3266. # (PYTHONSAFEPATH / stale editable mapping, #112729), hand the child the committed
  3267. # dependency generation (#122222), and mark it so its own entry runs the PM dependency
  3268. # boot. See cron/scheduler_worker_env.py and cron/worker_bootstrap.py.
  3269. from cron.scheduler_worker_env import pin_hermes_tree_on_pythonpath
  3270. repo_root = Path(__file__).resolve().parent.parent
  3271. worker_env = pin_hermes_tree_on_pythonpath(worker_env, repo_root)
  3272. worker_env[WORKER_MARKER] = "1"
  3273. try:
  3274. stderr_fd = os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
  3275. try:
  3276. process = subprocess.Popen(
  3277. dispatch.argv,
  3278. cwd=str(repo_root),
  3279. env=worker_env,
  3280. stdin=subprocess.DEVNULL,
  3281. stdout=subprocess.DEVNULL,
  3282. stderr=stderr_fd,
  3283. # detached-spawn: ok — restart-safe cron worker, must survive a gateway restart (#101940)
  3284. start_new_session=True,
  3285. creationflags=windows_hide_flags(),
  3286. )
  3287. finally:
  3288. os.close(stderr_fd)
  3289. except BaseException:
  3290. payload_path.unlink(missing_ok=True)
  3291. stderr_path.unlink(missing_ok=True)
  3292. raise
  3293. with _running_lock:
  3294. _restart_safe_waiter_job_ids.add(_inflight_key(job_id))
  3295. # Same window the dead-owner recovery ledger grants a pending handoff: a cold
  3296. # worker start (imports + secret hydration) measures ~10-12s in the field, and
  3297. # a dispatch deadline shorter than the adoption grace made the two guards
  3298. # around one handoff disagree.
  3299. deadline = time.monotonic() + HANDOFF_ADOPTION_GRACE_SECONDS
  3300. while time.monotonic() < deadline:
  3301. if ack_path.exists():
  3302. try:
  3303. acknowledgement = json.loads(ack_path.read_text(encoding="utf-8-sig"))
  3304. except Exception:
  3305. logger.exception(
  3306. "Cron external worker %s published an unreadable acknowledgement; "
  3307. "treating handoff as ownership-uncertain",
  3308. execution_id,
  3309. )
  3310. return _wait_for_external_cron_worker(
  3311. process,
  3312. execution_id=execution_id,
  3313. job_id=job_id,
  3314. handoff_files=(payload_path, stderr_path),
  3315. stderr_path=stderr_path,
  3316. )
  3317. finally:
  3318. ack_path.unlink(missing_ok=True)
  3319. if (
  3320. not isinstance(acknowledgement, dict)
  3321. or acknowledgement.get("execution_id") != execution_id
  3322. ):
  3323. logger.error(
  3324. "Cron external worker acknowledgement mismatch for %s; "
  3325. "treating handoff as ownership-uncertain",
  3326. execution_id,
  3327. )
  3328. return _wait_for_external_cron_worker(
  3329. process,
  3330. execution_id=execution_id,
  3331. job_id=job_id,
  3332. handoff_files=(payload_path, stderr_path),
  3333. stderr_path=stderr_path,
  3334. )
  3335. logger.info(
  3336. "Cron job '%s' handed to restart-safe worker pid=%s execution=%s dispatch=%s",
  3337. job_id,
  3338. acknowledgement.get("pid"),
  3339. execution_id,
  3340. dispatch.mode,
  3341. )
  3342. _record_external_cron_worker(
  3343. job_id,
  3344. acknowledgement.get("pid") or process.pid,
  3345. scope_isolated=dispatch.mode == "scoped",
  3346. )
  3347. return _wait_for_external_cron_worker(
  3348. process,
  3349. execution_id=execution_id,
  3350. job_id=job_id,
  3351. handoff_files=(payload_path, stderr_path),
  3352. stderr_path=stderr_path,
  3353. )
  3354. returncode = process.poll()
  3355. if returncode is not None:
  3356. with _running_lock:
  3357. _restart_safe_waiter_job_ids.discard(_inflight_key(job_id))
  3358. payload_path.unlink(missing_ok=True)
  3359. from cron.scheduler_diagnostics import external_worker_stderr_tail
  3360. stderr_tail = external_worker_stderr_tail(stderr_path)
  3361. stderr_path.unlink(missing_ok=True)
  3362. if dispatch.mode == "scoped" and scoped_spawn_lost_user_bus(worker_env):
  3363. # systemd-run itself failed before any worker ran, so the captured stderr
  3364. # holds nothing useful: name the cause, not the exit code.
  3365. raise RuntimeError(
  3366. "restart-safe systemd scope could not be created: the user D-Bus session at "
  3367. f"/run/user/{os.getuid()}/bus disappeared after the gateway started. On a " # windows-footgun: ok — scoped dispatch exists only on Linux
  3368. "system-level service install, run `sudo loginctl enable-linger <gateway-user>`; "
  3369. "the next fire dispatches without scope isolation."
  3370. )
  3371. raise RuntimeError(
  3372. f"cron external worker exited before ownership acknowledgement "
  3373. f"(exit {returncode}){stderr_tail}"
  3374. )
  3375. time.sleep(0.05)
  3376. # The child may have adopted the durable row just before publishing its
  3377. # acknowledgement. Never fall back to in-process execution on an uncertain
  3378. # handoff: that could duplicate side effects. The execution owner/dead-owner
  3379. # recovery ledger remains the authority.
  3380. logger.warning(
  3381. "Cron external worker for job '%s' did not acknowledge within %.0fs; "
  3382. "leaving the durable execution claim untouched",
  3383. job_id,
  3384. HANDOFF_ADOPTION_GRACE_SECONDS,
  3385. )
  3386. return _wait_for_external_cron_worker(
  3387. process,
  3388. execution_id=execution_id,
  3389. job_id=job_id,
  3390. handoff_files=(payload_path, ack_path, stderr_path),
  3391. stderr_path=stderr_path,
  3392. )
  3393. def _run_external_worker_payload(payload_path: Path, ack_path: Path) -> bool:
  3394. """Adopt and execute one gateway-dispatched cron payload.
  3395. The execution row is created by the gateway before spawn, then transferred
  3396. here before the ready acknowledgement is published. No side effect runs
  3397. unless that durable ownership transfer succeeds.
  3398. """
  3399. try:
  3400. payload = json.loads(payload_path.read_text(encoding="utf-8-sig"))
  3401. job = payload["job"]
  3402. profile_home = Path(payload["profile_home"]).resolve()
  3403. execution_id = str(job["execution_id"])
  3404. except Exception:
  3405. logger.exception("Cron external worker could not load payload %s", payload_path)
  3406. return False
  3407. finally:
  3408. try:
  3409. payload_path.unlink(missing_ok=True)
  3410. except OSError:
  3411. pass
  3412. from agent.secret_scope import (
  3413. build_profile_secret_scope,
  3414. is_multiplex_active,
  3415. reset_secret_scope,
  3416. set_multiplex_active,
  3417. set_secret_scope,
  3418. )
  3419. from cron.executions import adopt_claimed_execution
  3420. from hermes_cli.env_loader import hydrate_profile_secret_sources
  3421. from hermes_constants import (
  3422. reset_hermes_home_override,
  3423. set_hermes_home_override,
  3424. )
  3425. previous_multiplex = is_multiplex_active()
  3426. home_token = secret_token = None
  3427. try:
  3428. home_token = set_hermes_home_override(profile_home)
  3429. multiplex_active = bool(payload.get("multiplex_active", False))
  3430. set_multiplex_active(multiplex_active)
  3431. # Plugin secret sources (``ctx.register_secret_source()``) only exist after plugin
  3432. # discovery; this process starts with the builtin registry alone, so hydrating without it
  3433. # silently dropped every plugin-sourced credential (#121929). Runs under the home override
  3434. # so a multiplexed worker loads the OWNING profile's plugins, not the launch profile's.
  3435. from hermes_cli.plugins import discover_plugins
  3436. discover_plugins()
  3437. hydrate_profile_secret_sources(profile_home)
  3438. secret_token = set_secret_scope(build_profile_secret_scope(profile_home), profile_home=str(profile_home))
  3439. # This process never ran gateway startup, so the OWNING profile's ``hooks:`` block (shell
  3440. # hooks + ``hooks.outbound``) was never registered and cron sessions silently lost it
  3441. # (#131764). Same helper the gateway uses per profile; it must run inside the secret scope
  3442. # because ``secret_env`` targets resolve through ``get_secret`` (raises unscoped under multiplex).
  3443. from gateway.run_startup import GatewayStartupMixin
  3444. GatewayStartupMixin._register_config_hooks(
  3445. "Cron external worker: config hook registration failed", level=logging.WARNING)
  3446. with use_cron_store(profile_home):
  3447. if adopt_claimed_execution(execution_id) is None:
  3448. logger.error(
  3449. "Cron external worker refused execution %s: durable ownership "
  3450. "could not be established",
  3451. execution_id,
  3452. )
  3453. return False
  3454. try:
  3455. ack_path.parent.mkdir(parents=True, exist_ok=True)
  3456. # Publish via write-to-temp + atomic rename. Writing ack_path in place
  3457. # (the old approach) let O_CREAT make the empty file visible to the
  3458. # scheduler's exists()-then-read polling loop before the JSON body was
  3459. # written, occasionally handing it a 0-byte file and a JSONDecodeError.
  3460. # os.replace() is a single atomic syscall on the same filesystem, so
  3461. # readers only ever see the file fully absent or fully written.
  3462. ack_tmp_path = ack_path.with_name(f"{ack_path.name}.tmp{os.getpid()}")
  3463. fd = os.open(ack_tmp_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
  3464. try:
  3465. with os.fdopen(fd, "w", encoding="utf-8") as ack_file:
  3466. json.dump({"pid": os.getpid(), "execution_id": execution_id}, ack_file)
  3467. ack_file.flush()
  3468. os.fsync(ack_file.fileno())
  3469. os.replace(ack_tmp_path, ack_path)
  3470. except BaseException:
  3471. ack_tmp_path.unlink(missing_ok=True)
  3472. raise
  3473. except Exception:
  3474. logger.exception(
  3475. "Cron external worker could not publish ready acknowledgement for %s",
  3476. execution_id,
  3477. )
  3478. return False
  3479. old_external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER")
  3480. os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = execution_id
  3481. try:
  3482. completed = run_one_job(job, adapters=None, loop=None, verbose=False)
  3483. # Successful return: the worker recorded its outcome. If it raises,
  3484. # retain fd 2's pathname until the waiter consumes the traceback.
  3485. with contextlib.suppress(OSError):
  3486. ack_path.with_suffix(".stderr").unlink(missing_ok=True)
  3487. return completed
  3488. finally:
  3489. if old_external_execution is None:
  3490. os.environ.pop("_HERMES_CRON_EXTERNAL_WORKER", None)
  3491. else:
  3492. os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = old_external_execution
  3493. finally:
  3494. if secret_token is not None:
  3495. reset_secret_scope(secret_token)
  3496. set_multiplex_active(previous_multiplex)
  3497. if home_token is not None:
  3498. reset_hermes_home_override(home_token)
  3499. def _notify_provider_jobs_changed() -> None:
  3500. """Best-effort: tell the active scheduler provider the job set changed. Call AFTER a successful
  3501. store mutation so an external provider can re-provision/cancel the one-shot; no-op for the
  3502. built-in. Kept out of cron/jobs.py (import cycle). Never raises."""
  3503. try:
  3504. from cron.scheduler_provider import resolve_cron_scheduler
  3505. resolve_cron_scheduler().on_jobs_changed()
  3506. except Exception as e:
  3507. logger.debug("on_jobs_changed notify failed: %s", e)
  3508. class CronSchedulerRegistrationError(RuntimeError):
  3509. """A job was persisted but its first external trigger was not registered."""
  3510. def __init__(self, job: dict, cause: Exception) -> None:
  3511. self.job = job
  3512. self.cause = cause
  3513. super().__init__(
  3514. f"Cron job '{job['id']}' was saved, but its first scheduler "
  3515. f"registration failed ({type(cause).__name__}). Do not create a "
  3516. "duplicate. Pause/resume or update the job to retry registration."
  3517. )
  3518. def user_message(self) -> str:
  3519. """Human-facing variant for chat/CLI surfaces (no exception class name)."""
  3520. label = self.job.get("name") or self.job["id"]
  3521. return (
  3522. f"Saved cron job '{label}', but couldn't register it with the "
  3523. "external scheduler yet. The job is kept — don't re-create it; "
  3524. "pause/resume or edit it (e.g. via /cron) to retry registration."
  3525. )
  3526. def to_dict(self) -> dict:
  3527. """Return the public partial-failure contract without provider details."""
  3528. return {
  3529. "error": str(self),
  3530. "job_id": self.job["id"],
  3531. "job_saved": True,
  3532. "scheduler_registered": False,
  3533. "retry_create": False}
  3534. def create_job_with_scheduler_registration(**kwargs) -> dict:
  3535. """Persist one job and register its first trigger with the active provider."""
  3536. from cron.jobs import create_job
  3537. from cron.scheduler_provider import resolve_cron_scheduler
  3538. job = create_job(**kwargs)
  3539. if not job.get("enabled", True):
  3540. return job
  3541. try:
  3542. resolve_cron_scheduler().register_job(job)
  3543. except Exception as exc:
  3544. raise CronSchedulerRegistrationError(job, exc) from exc
  3545. return job
  3546. # Dead-owner reap is throttled (opens the executions ledger). Tests may reset
  3547. # _last_dead_owner_reap_at to {} to force a reap next tick.
  3548. # Dead-owner claim reclaim throttle (#86721): recover_interrupted_executions opens the executions ledger, so
  3549. # the per-tick reap is rate-limited rather than run on every idle 60s cycle.
  3550. # The throttle is keyed by profile home: under multiplex_profiles the ticker
  3551. # ticks every profile each cycle, and a process-global slot would let the
  3552. # first profile starve all the others.
  3553. _DEAD_OWNER_REAP_INTERVAL_SECONDS = 300.0
  3554. _last_dead_owner_reap_at: dict[str, float] = {}
  3555. # Worktree prune throttle: the cron tick is the only reliably periodic process on gateway boxes.
  3556. _WORKTREE_MAINTENANCE_INTERVAL_SECONDS = 6 * 3600.0
  3557. _last_worktree_maintenance_at: Optional[dict] = None # profile home -> monotonic stamp
  3558. _worktree_maintenance_lock = threading.Lock()
  3559. def _worktree_maintenance_repos() -> list[str]:
  3560. """Repos whose ``.worktrees/`` to keep pruned: the hermes checkout, job workdir repo roots and
  3561. every repo this profile created a worktree in (``worktree_registry``), filtered to those that
  3562. actually have a ``.worktrees/`` dir."""
  3563. repos: set = set()
  3564. with contextlib.suppress(Exception):
  3565. from hermes_cli.worktree_registry import registered_repos
  3566. repos.update(registered_repos())
  3567. # Hermes source checkout (git installs only; wheel installs have no .git).
  3568. with contextlib.suppress(Exception):
  3569. install_root = Path(__file__).resolve().parent.parent
  3570. if (install_root / ".git").exists():
  3571. repos.add(str(install_root))
  3572. with contextlib.suppress(Exception):
  3573. from cron.jobs import load_jobs
  3574. for job in load_jobs():
  3575. workdir = str(job.get("workdir") or "").strip()
  3576. if not workdir or not Path(workdir).is_dir():
  3577. continue
  3578. try:
  3579. probe = subprocess.run(
  3580. ["git", "rev-parse", "--show-toplevel"],
  3581. capture_output=True, text=True, encoding="utf-8",
  3582. errors="replace", timeout=5, cwd=workdir, check=False)
  3583. if probe.returncode == 0 and probe.stdout.strip():
  3584. repos.add(probe.stdout.strip())
  3585. except Exception:
  3586. continue
  3587. return [r for r in sorted(repos) if (Path(r) / ".worktrees").is_dir()]
  3588. def _maybe_run_worktree_maintenance() -> None:
  3589. """Throttled worktree prune from the cron tick, on a daemon thread so the tick never waits on
  3590. git. Same pruner as ``hermes -w`` startup (idle dirty/unpushed trees archived before removal,
  3591. live trees untouched). Discovery is throttled per profile home, the prune once per repo across
  3592. served profiles, and ``cron.scheduler_worktree_maintenance`` runs one sweep at a time per
  3593. process. Runs in a copy of the caller's context (profile-home override); never raises."""
  3594. global _last_worktree_maintenance_at
  3595. import contextvars
  3596. from hermes_constants import get_hermes_home
  3597. home = str(get_hermes_home())
  3598. now = time.monotonic()
  3599. with _worktree_maintenance_lock:
  3600. stamps = _last_worktree_maintenance_at or {}
  3601. last = stamps.get(home)
  3602. if last is not None and now - last < _WORKTREE_MAINTENANCE_INTERVAL_SECONDS:
  3603. return
  3604. _last_worktree_maintenance_at = {**stamps, home: now}
  3605. def _run() -> None:
  3606. from cron.scheduler_worktree_maintenance import run_sweep
  3607. run_sweep(_worktree_maintenance_repos,
  3608. interval_seconds=_WORKTREE_MAINTENANCE_INTERVAL_SECONDS)
  3609. threading.Thread(target=contextvars.copy_context().run, args=(_run,), name="cron-worktree-prune",
  3610. daemon=True).start()
  3611. def _acquire_tick_lock(lock_file):
  3612. """Open + non-blocking lock the tick file (fcntl / msvcrt). Returns the fd, or None on genuine
  3613. contention. A real OSError (esp. EMFILE/ENFILE) must NOT pass as contention — the scheduler
  3614. would look healthy while no job runs — so it is re-raised for the ticker to record a FAILED
  3615. tick."""
  3616. lock_fd = None
  3617. try:
  3618. # Cross-platform file locking: fcntl on Unix, msvcrt on Windows. Only genuine lock contention
  3619. # (another ticker holds the lock) skips the tick silently. A real OSError — most importantly
  3620. # EMFILE/ENFILE from fd exhaustion — must NOT be swallowed as "another instance holds the lock":
  3621. # that previously made the scheduler appear healthy (tick returned 0, heartbeat recorded success)
  3622. # while no job ever ran again (#87644).
  3623. lock_fd = open(lock_file, "w", encoding="utf-8")
  3624. if fcntl:
  3625. fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  3626. elif msvcrt:
  3627. msvcrt.locking(lock_fd.fileno(), msvcrt.LK_NBLCK, 1)
  3628. return lock_fd
  3629. except OSError as exc:
  3630. if lock_fd is not None:
  3631. with contextlib.suppress(OSError):
  3632. lock_fd.close()
  3633. if _is_lock_contention_errno(exc):
  3634. logger.info("Tick skipped — another instance holds the lock")
  3635. return None
  3636. if _is_fd_exhaustion(exc):
  3637. # fd reclamation is the ticker loop's job (scheduler_provider.py); here would double it.
  3638. logger.error(
  3639. "Cron tick could not acquire tick lock: %s — scheduler will "
  3640. "attempt fd reclamation and retry with backoff",
  3641. exc)
  3642. elif exc.errno not in store_health.UNWRITABLE_ERRNOS: # those degrade the store (tick caller)
  3643. logger.error("Cron tick could not acquire tick lock: %s", exc)
  3644. raise
  3645. def _release_tick_lock(lock_fd) -> None:
  3646. if fcntl:
  3647. with contextlib.suppress((OSError, IOError)):
  3648. fcntl.flock(lock_fd, fcntl.LOCK_UN)
  3649. elif msvcrt:
  3650. with contextlib.suppress((OSError, IOError)):
  3651. msvcrt.locking(lock_fd.fileno(), msvcrt.LK_UNLCK, 1)
  3652. lock_fd.close()
  3653. def _maybe_reap_dead_owners() -> None:
  3654. """Dead-owner reclaim: a run that died mid-flight would leave its row 'claimed' forever. Rows
  3655. whose owner process is proved gone are released (_owner_is_live), as are rows whose live owner
  3656. holds a claim older than the derived stale bound (the process is not killed). Throttled."""
  3657. # Dead-owner claim reclaim (#86721): execution rows carry their owner pid + process start time, but
  3658. # recovery previously ran only at scheduler STARTUP. A one-shot `hermes cron run` that claimed a job and
  3659. # died mid-run (its runner thread lived in the exiting CLI process) left the row 'claimed' forever while
  3660. # the long-lived gateway ticker kept running — blocking every future run of that job. Reap provably-dead
  3661. # owners periodically so stale claims auto-clear without a gateway restart. Throttled so idle 60s ticks
  3662. # don't pay a ledger connection every cycle (#33612).
  3663. _reap_key = hermes_home_key(_get_hermes_home())
  3664. _reap_now = time.monotonic()
  3665. _last_reap = _last_dead_owner_reap_at.get(_reap_key)
  3666. if (
  3667. _last_reap is not None
  3668. and _reap_now - _last_reap < _DEAD_OWNER_REAP_INTERVAL_SECONDS
  3669. ):
  3670. return
  3671. _last_dead_owner_reap_at[_reap_key] = _reap_now
  3672. try:
  3673. from cron.executions import recover_interrupted_executions
  3674. _reclaimed = recover_interrupted_executions()
  3675. if _reclaimed:
  3676. logger.warning(
  3677. "Reclaimed %d cron execution(s) whose owner process died "
  3678. "before reaching a terminal state (marked unknown)",
  3679. _reclaimed)
  3680. except Exception as _reap_exc:
  3681. logger.debug("Dead-owner execution reclaim failed: %s", _reap_exc)
  3682. def _sweep_stale_inflight_for_tick(due_jobs: list) -> None:
  3683. """Bound the in-flight set BEFORE the dedup guard so a leaked claim is force-released now
  3684. rather than eating every later fire until restart. Skipped when nothing is in flight."""
  3685. _local_home = hermes_home_key(_get_hermes_home())
  3686. if not any(key[0] == _local_home for key in _running_job_ids):
  3687. return
  3688. _sweep_jobs = due_jobs
  3689. with contextlib.suppress(Exception):
  3690. _inflight_ids = {key[1] for key in _running_job_ids if key[0] == _local_home}
  3691. _due_ids = {j.get("id") for j in due_jobs if isinstance(j, dict)}
  3692. if not _inflight_ids <= _due_ids:
  3693. from cron.jobs import load_jobs as _load_all_jobs
  3694. _sweep_jobs = _load_all_jobs()
  3695. try:
  3696. sweep_stale_inflight(_sweep_jobs)
  3697. except Exception as e:
  3698. logger.warning("Stale in-flight sweep failed: %s", e)
  3699. def _resolve_max_parallel_workers() -> Optional[int]:
  3700. """Max workers: env > config.yaml > unbounded (HERMES_CRON_MAX_PARALLEL=1 restores serial)."""
  3701. try:
  3702. _env_par = cron_env_setting("HERMES_CRON_MAX_PARALLEL").strip()
  3703. if _env_par:
  3704. return int(_env_par) or None
  3705. except (ValueError, TypeError):
  3706. logger.warning("Invalid HERMES_CRON_MAX_PARALLEL value; defaulting to unbounded")
  3707. with contextlib.suppress(Exception):
  3708. _ucfg = load_config() or {}
  3709. _cfg_par = (_ucfg.get("cron", {}) if isinstance(_ucfg, dict) else {}).get("max_parallel_jobs")
  3710. if _cfg_par is not None:
  3711. return int(_cfg_par) or None
  3712. return None
  3713. def _sweep_mcp_orphans() -> None:
  3714. """Reap MCP stdio orphans (only PIDs flagged by tools.mcp_tool._run_stdio's finally block);
  3715. run AFTER jobs finish so live sessions are never touched."""
  3716. try:
  3717. from tools.mcp_tool_lifecycle import _kill_orphaned_mcp_children
  3718. _kill_orphaned_mcp_children()
  3719. except Exception as _e:
  3720. logger.debug("Post-tick MCP orphan cleanup failed: %s", _e)
  3721. def _process_due_job(job: dict, adapters, loop, verbose: bool) -> bool:
  3722. """Run one due job via the shared ``run_one_job`` body."""
  3723. # Claim only when the worker actually starts, so a queued lease can't expire first.
  3724. try:
  3725. claimed = claim_job_for_fire(job["id"], return_job=True)
  3726. except OSError as exc:
  3727. store_health.note_unwritable(exc, f"skipped job '{job.get('name') or job['id']}'", "claim", [job])
  3728. settle_unstarted_execution(
  3729. job["execution_id"], job["id"], f"Cron store unwritable; not started: {exc}")
  3730. return False
  3731. except BaseException as exc: # settle first, then surface the real error
  3732. settle_unstarted_execution(
  3733. job["execution_id"], job["id"], f"Fire claim failed: {type(exc).__name__}: {exc}")
  3734. raise
  3735. if not claimed:
  3736. finish_execution(
  3737. job["execution_id"], success=False, error="Fire claim lost; execution was not started.")
  3738. return True
  3739. # CAS returns the persisted record; bool fallback only for older test doubles.
  3740. claimed_job = dict(claimed) if isinstance(claimed, dict) else dict(job)
  3741. claimed_job["execution_id"] = job["execution_id"]
  3742. claimed_job["_scheduled_instant"] = job.get("_scheduled_instant")
  3743. return run_one_job(claimed_job, adapters=adapters, loop=loop, verbose=verbose)
  3744. def _submit_with_guard(job: dict, pool: concurrent.futures.ThreadPoolExecutor, process_job):
  3745. """Submit with the in-flight dedup guard; None if a prior tick's run is still in flight.
  3746. Running-set membership is released in the worker's finally."""
  3747. job_id = job["id"]
  3748. job_label = job.get("name", job_id)
  3749. def _clear_run_claim_best_effort() -> None:
  3750. """Best-effort claim cleanup on dispatch-failure paths. Only one-shots carry a run_claim;
  3751. clear_run_claim takes _jobs_lock + full load/save and can raise on degraded paths
  3752. (shutdown, EMFILE) — a claim expiring at TTL beats crashing the tick.
  3753. Only one-shot jobs carry a ``run_claim`` (stamped by get_due_jobs, #59229), so recurring jobs skip
  3754. the call entirely — clear_run_claim acquires _jobs_lock (blocking cross-process flock) and does a
  3755. full load_jobs read, and the dispatch-failure paths fire exactly when the process can least afford N
  3756. pointless lock/read round-trips (interpreter shutdown, EMFILE). clear_run_claim itself does
  3757. load_jobs/save_jobs file I/O; on those degraded paths it can raise, and these early-exits exist
  3758. precisely to skip cleanly — a stale claim expiring at the TTL is a better outcome than crashing the
  3759. tick (#86522).
  3760. """
  3761. _schedule = job.get("schedule")
  3762. if not (isinstance(_schedule, dict) and _schedule.get("kind") == "once"):
  3763. return
  3764. try:
  3765. clear_run_claim(job_id)
  3766. except Exception as claim_err:
  3767. logger.warning(
  3768. "Could not clear run_claim for job '%s' after dispatch "
  3769. "failure: %s (claim will expire at TTL)",
  3770. job_label, claim_err)
  3771. def _not_dispatched_shutdown() -> None:
  3772. logger.warning("Job '%s' not dispatched — interpreter is shutting down", job_label)
  3773. # During interpreter shutdown pool.submit raises; skip — the job fires on the next tick.
  3774. # If the interpreter is finalizing (gateway SIGTERM / restart / OOM), scheduling any new delivery is
  3775. # futile — asyncio.run and a fresh ThreadPoolExecutor both raise "cannot schedule new futures after
  3776. # interpreter shutdown". Skip gracefully with a warning rather than emitting an ERROR traceback on every
  3777. # restart-race (#58720, #55924).
  3778. # A tick can race gateway teardown: once the interpreter is finalizing, ``pool.submit`` raises "cannot
  3779. # schedule new futures after interpreter shutdown" and crashes the tick. Skip cleanly — the job stays
  3780. # due and will fire on the next healthy tick (#58720, #55924).
  3781. if _interpreter_shutting_down():
  3782. _not_dispatched_shutdown()
  3783. _clear_run_claim_best_effort()
  3784. return None
  3785. registration_owner = object()
  3786. if not try_register_running_job(job_id, owner=registration_owner):
  3787. logger.info("Job '%s' already running — skipping", job_label)
  3788. return None
  3789. # The home the claim was registered under. The pool worker's ``finally`` runs OUTSIDE
  3790. # ``ctx.run``, where the per-profile cron scope is not bound, so releasing without it would
  3791. # discard the LAUNCH home's key and leak every secondary profile's claim.
  3792. _claim_home = _get_hermes_home()
  3793. # Record the attempt before dispatch; recovery marks abandoned rows unknown (no retry).
  3794. execution = None
  3795. try:
  3796. execution = create_execution(
  3797. job_id, source="builtin", scheduled_instant=job.get("_scheduled_instant"))
  3798. dispatched_job = dict(job, execution_id=execution["id"])
  3799. note_cron_execution(dispatched_job)
  3800. _ctx = contextvars.copy_context()
  3801. except Exception as execution_err:
  3802. # Release the claim so the next tick retries instead of wedging "already running".
  3803. release_running_job(job_id, home=_claim_home, owner=registration_owner)
  3804. _clear_run_claim_best_effort()
  3805. if execution is not None: # the receipt was persisted; a later step failed
  3806. settle_unstarted_execution(execution["id"], job_id, (
  3807. f"Dispatch preparation failed: {type(execution_err).__name__}: {execution_err}"))
  3808. logger.exception("Job '%s' not dispatched: %s failed: %s", job_label,
  3809. "execution creation" if execution is None else "dispatch preparation",
  3810. execution_err)
  3811. return None
  3812. def _run_and_release(j=dispatched_job, ctx=_ctx, home=_claim_home):
  3813. try:
  3814. return ctx.run(process_job, j)
  3815. finally:
  3816. release_running_job(j["id"], home=home, owner=registration_owner)
  3817. try:
  3818. fut = pool.submit(_run_and_release)
  3819. except Exception as submit_err:
  3820. release_running_job(job_id, home=_claim_home, owner=registration_owner)
  3821. _clear_run_claim_best_effort()
  3822. finish_execution(
  3823. execution["id"], success=False, error=f"Executor dispatch failed: {submit_err}")
  3824. if isinstance(submit_err, RuntimeError) and _interpreter_shutting_down(submit_err):
  3825. _not_dispatched_shutdown()
  3826. else:
  3827. logger.error("Job '%s' not dispatched: %s", job_label, submit_err)
  3828. return None
  3829. with _running_lock:
  3830. _submit_key = _inflight_key(job_id, _claim_home)
  3831. if _running_registration_owners.get(_submit_key) is registration_owner:
  3832. _running_futures[_submit_key] = fut
  3833. return fut
  3834. def _sweep_mcp_orphans_when_all_done(futures: list) -> None:
  3835. """Async (gateway ticker) mode: sweep via a done-callback after the LAST job completes; sweep
  3836. inline when nothing was dispatched (all skipped / no due jobs)."""
  3837. if not futures:
  3838. _sweep_mcp_orphans()
  3839. return
  3840. _remaining = [len(futures)]
  3841. def _on_done(_f: concurrent.futures.Future) -> None:
  3842. _remaining[0] -= 1
  3843. with contextlib.suppress(Exception):
  3844. _exc = _f.exception()
  3845. if _exc is not None:
  3846. logger.error(
  3847. "Cron job future failed in async mode: %s", _exc,
  3848. exc_info=(type(_exc), _exc, _exc.__traceback__))
  3849. if _remaining[0] <= 0:
  3850. _sweep_mcp_orphans()
  3851. for _f in futures:
  3852. _f.add_done_callback(_on_done)
  3853. from cron.scheduler_tick import tick
  3854. # ---------------------------------------------------------------------------
  3855. # Split modules. Imported at the bottom (import cycle: they late-bind ``cron.scheduler`` as
  3856. # ``_sched``). Only names this module itself calls; everything else lives in the split module.
  3857. # ---------------------------------------------------------------------------
  3858. from cron.scheduler_delivery import (
  3859. _deliver_result, _delivery_lane_value, _normalize_deliver_value, _resolve_delivery_target,
  3860. _resolve_delivery_targets,
  3861. )
  3862. from cron.scheduler_script import (
  3863. _get_session_db_timeout, _run_job_script_with_claim_heartbeat, _start_heartbeat_thread,
  3864. )
  3865. from cron.scheduler_prompt import (
  3866. _PROMPT_FRAME, _PROMPT_HEADING, _PROMPT_SEPARATOR, _RESPONSE_FRAME, _RESPONSE_HEADING,
  3867. _RESPONSE_TERMINATOR, _block_and_pause_job, _build_job_prompt, _guard_job_credential_exfil,
  3868. _job_doc_header, _parse_wake_gate,
  3869. )
  3870. from cron.scheduler_preflight import (
  3871. BLOCKED_CONFIG_MARKER, BLOCKED_CONFIG_SILENT_MARKER, _cron_preflight_enabled,
  3872. _empty_requested_mcp_toolsets, _is_transient_provider_resolve_error, _preflight_job_config,
  3873. )
  3874. # `python -m cron.scheduler` entry: MUST stay below the split-module imports so the worker /
  3875. # tick paths see every name they need.
  3876. if __name__ == "__main__":
  3877. if "--external-worker-file" in sys.argv:
  3878. finish_worker_boot() # may relaunch: before the payload is read and the ack published
  3879. import argparse
  3880. parser = argparse.ArgumentParser(add_help=False)
  3881. parser.add_argument("--external-worker-file", type=Path, required=True)
  3882. parser.add_argument("--ack-file", type=Path, required=True)
  3883. args = parser.parse_args()
  3884. # The gateway spawns this worker with stdout on DEVNULL and stderr on a
  3885. # capture file it only reads back if we die before the ack; without a
  3886. # log handler every adoption/ack failure below would otherwise be
  3887. # invisible to the persistent log.
  3888. try:
  3889. from hermes_logging import setup_logging
  3890. setup_logging(hermes_home=_get_hermes_home(), mode="cron")
  3891. except Exception:
  3892. pass
  3893. raise SystemExit(
  3894. 0 if _run_external_worker_payload(args.external_worker_file, args.ack_file) else 1
  3895. )
  3896. tick(verbose=True, headless=True)