| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416 |
- # health: allow FILE_LINES -- security fix for #82010: distinguish an explicitly-empty toolset allowlist (fail closed, nothing allowed) from an absent one (no restriction); the added lines are minimal fail-closed branches at this existing chokepoint. Also #141468: match the `mcp-<server>` toolset spelling so a narrow job's MCP allowlist is not widened into every enabled server — a minimal branch at the same chokepoint.
- """Cron job scheduler: tick() runs due jobs (gateway calls it every 60s from a background thread).
- A file lock (~/.hermes/cron/.tick.lock) keeps overlapping processes to one tick at a time.
- """
- import atexit
- import concurrent.futures
- import contextlib
- import contextvars
- import errno
- import json
- import logging
- import os
- import re
- import subprocess
- import sys
- import threading
- import time
- import uuid
- from dataclasses import dataclass
- from datetime import datetime, timezone, UTC
- # fcntl is Unix-only; Windows uses msvcrt
- try:
- import fcntl
- except ImportError:
- fcntl = None
- try:
- import msvcrt
- except ImportError:
- msvcrt = None
- from pathlib import Path
- from typing import Any, Callable, Dict, List, Optional, Union
- # Must precede repo-level imports: standalone invocations (e.g. module reload after
- # `hermes update`) otherwise fail with ModuleNotFoundError for hermes_time et al.
- sys.path.insert(0, str(Path(__file__).parent.parent))
- from cron.worker_bootstrap import WORKER_MARKER, finish_worker_boot
- from hermes_constants import get_hermes_home, hermes_home_key
- from hermes_cli.observability.shared_metrics_gateway import note_cron_execution, note_cron_skipped
- from cron.env_settings import cron_env_setting
- from hermes_cli._subprocess_compat import windows_hide_flags
- from hermes_cli.config import (
- load_config, load_config_readonly)
- from hermes_cli.config_backend import ConfigBackendUnavailable
- from hermes_cli.fallback_config import get_fallback_chain, scoped_fallback_chain
- from hermes_time import now as _hermes_now, safe_strftime
- from agent.interrupt_compat import request_hard_interrupt
- from agent.delegation_context import (
- enter_non_dispatcher_owned_context, exit_non_dispatcher_owned_context)
- from agent.memory_provider import ctx_bound
- from agent.session_activity import AwakeIdleMeter
- from agent.turn_failure_copy import is_max_iteration_handoff
- logger = logging.getLogger(__name__)
- def _set_cron_session_title(session_db, session_id, base_title):
- """Persist a non-blank, unique title for a finished cron session; returns it (None if unset).
- Runs BEFORE end_session()/close() so no write races the close. Duplicate title (unique-index
- ValueError) -> get_next_title_in_lineage(); if unavailable, raise rather than end up untitled.
- Centralizes the title write so the cron finally block can guarantee a non-blank, unique title is
- persisted before end_session()/close() tear the connection down (issues #50535, #50536, #50537):
- - #50535: never leaves the session blank. base_title already carries a cron-id fallback for nameless
- jobs; this also guards a failed write. Recover by appending a #N suffix via get_next_title_in_lineage()
- when supported, instead of swallowing the error and ending up untitled. - #50536: this runs
- synchronously in the cron finally block ahead of the session close, so no in-flight title write can race
- the close.
- """
- if not session_db or not session_id:
- return None
- title = (base_title or "").strip()
- if not title:
- return None
- try:
- session_db.set_session_title(session_id, title)
- return title
- except ValueError:
- # Unique-title collision: fall back to the next lineage title (base #2, #3, ...).
- next_title_fn = getattr(session_db, "get_next_title_in_lineage", None)
- if next_title_fn is None:
- raise
- deduped = next_title_fn(title)
- if not deduped or deduped == title:
- raise
- session_db.set_session_title(session_id, deduped)
- return deduped
- def _job_route_pinned(job: dict) -> bool:
- """True when the job carries its own provider, model or endpoint. Unpinned jobs store none of
- these (they follow the main model at fire time), so any value is an explicit operator pin."""
- return any(isinstance(job.get(k), str) and job[k].strip() for k in ("provider", "model", "base_url"))
- def _job_fallback_chain(job: dict, cfg: Any) -> Optional[list]:
- """The fallback chain this job may walk, at credential resolution AND mid-run (#100437).
- A pinned job never borrows the global ``fallback_providers`` chain, the same rule a pinned
- ``delegate_task`` child follows (``scoped_fallback_chain``): a chain entry is a different
- provider and usually a different model, which is exactly what the pin ruled out. Same-provider
- credential-pool rotation is not the chain and still applies. Unpinned jobs inherit the chain.
- """
- return scoped_fallback_chain(
- get_fallback_chain(cfg), None, pinned=_job_route_pinned(job), owner="cron job")
- def _fallback_chain_phrase(job: Optional[dict] = None) -> str:
- """Backup-provider clause for a provider-failure notice: "pinned, no fallback" vs "the backups
- failed too" vs "none configured" (most installs). Fails open to "the backups failed too" if
- config can't be read — never crash delivery.
- """
- if job is not None and _job_route_pinned(job):
- return (
- "This job is pinned to its own provider/model, so it does not fall back to "
- f"`fallback_providers`; `hermes cron edit {job.get('id')} --unpin` lets it follow the "
- "main model and its fallback chain."
- )
- try:
- cfg = load_config() or {}
- chain = get_fallback_chain(cfg)
- except Exception:
- return "No backup provider succeeded either."
- if chain:
- return "No backup provider succeeded either."
- return (
- "No backup provider is configured — add one with `hermes fallback add`, "
- "or set a cron-wide default via `cron.model` + `cron.model_provider` in config.yaml."
- )
- def _failure_streak_nudge(job: dict) -> str:
- """Review nudge when a recurring job keeps failing, else "". The failure message is delivered
- BEFORE mark_job_run records this run, hence stored ``failure_streak`` + 1. Threshold:
- ``cron.failure_nudge_threshold`` (default 3, 0 disables)."""
- schedule_kind = (job.get("schedule") or {}).get("kind")
- if schedule_kind not in {"cron", "interval"}:
- return ""
- try:
- cfg = load_config() or {}
- threshold = int(
- ((cfg.get("cron") or {}) if isinstance(cfg, dict) else {}).get(
- "failure_nudge_threshold", 3
- )
- )
- except Exception:
- threshold = 3
- if threshold <= 0:
- return ""
- streak = int(job.get("failure_streak") or 0) + 1 # +1 = this run
- if streak < threshold:
- return ""
- job_ref = job.get("name") or job.get("id") or "this job"
- return (
- f"\nThis job has failed {streak} runs in a row — worth a review. "
- f"Fix its prompt/config, or pause it with `hermes cron pause {job_ref}` "
- "(resume/remove also available) to stop the noise."
- )
- def _detect_gateway_code_skew() -> tuple[str, str] | None:
- """Boot-vs-disk revision skew for THIS process, or None. Test seam over
- ``gateway.code_skew.detect_code_skew``; a broken import must never take delivery down."""
- try:
- from gateway.code_skew import detect_code_skew
- return detect_code_skew()
- except Exception:
- return None
- def _current_gateway_code_sha() -> str | None:
- """Full revision currently on disk; kept separate from display-shortened skew labels."""
- try:
- from gateway.code_skew import current_code_sha
- return current_code_sha()
- except Exception:
- return None
- class CronTickYielded(RuntimeError):
- """A stale-code ticker yielded this tick to a fresh gateway.
- Raised by ``tick()`` BEFORE the tick lock when boot fingerprint ≠ disk, this process does NOT
- own the runtime lock and its live holder reports the disk revision — the stale process must
- stay out of the dispatch race (contention would starve the fresh ticker). Skew ``None`` never
- yields (fail open). Raised, not returned, so ``record_ticker_error`` sees it and ``hermes cron
- status`` isn't green.
- """
- def __init__(self, boot_rev: str, disk_rev: str) -> None:
- self.boot_rev = boot_rev
- self.disk_rev = disk_rev
- super().__init__(
- f"Cron tick yielded to a fresh gateway process (stale code: "
- f"booted on {boot_rev}, disk is at {disk_rev})"
- )
- _STALE_YIELD_RE = re.compile(r"stale code: booted on (\S+), disk is at (\S+)\)")
- def stale_code_yield_labels(recorded_error: str | None) -> tuple[str, str] | None:
- """``(boot_rev, disk_rev)`` when a persisted ``ticker_last_error`` is a ``CronTickYielded``.
- ``hermes cron status`` runs in another process and only sees the marker text; a yielding
- ticker still refreshes its heartbeat, so this is the one signal that separates "stale code,
- firing nothing" from a healthy loop (#117275).
- """
- if not recorded_error or not recorded_error.startswith(CronTickYielded.__name__):
- return None
- match = _STALE_YIELD_RE.search(recorded_error)
- return (match.group(1), match.group(2)) if match else None
- def _should_yield_tick_to_fresh_gateway() -> tuple[str, str] | None:
- """``(boot_rev, disk_rev)`` when THIS profile's tick must yield to a fresher gateway, else None.
- Yields only when ALL hold: code skew, this process is not the cron owner for the profile being
- ticked, and another live host gateway whose served set covers that profile reports a fresh
- heartbeat on the disk revision. Every probe failure returns None — yielding is a certainty
- claim, never a guess.
- The question is asked PER PROFILE. One process multiplexes every profile, so "do I own the
- runtime lock" (process-global, stamped at boot for the launch home) cannot answer it: a
- stale multiplexer that holds the lock must still keep ticking the profiles nobody else ticks,
- and must still yield the ones a fresher gateway serves.
- ``gateway_state.json`` is per-HOME and last-writer-wins, not per-process: during a
- ``--replace`` takeover both the stale and the fresh gateway stamp it, so which one this
- predicate reads is write-order dependent. The pid equality in ``live_gateway_ticking`` is what
- binds the record to the current lock holder; the takeover window itself fails open (no yield).
- """
- skew = _detect_gateway_code_skew()
- if skew is None:
- return None
- disk_sha = _current_gateway_code_sha()
- if disk_sha is None:
- return None
- try:
- from cron.scheduler_ownership import live_gateway_ticking, owns_cron_tick_for
- home = _get_hermes_home()
- if owns_cron_tick_for(home):
- return None
- holder_status = live_gateway_ticking(home)
- if holder_status is None or holder_status.get("code_sha") != disk_sha:
- return None
- except Exception:
- return None
- return skew
- def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str:
- """One-line failure notice for chat delivery (full details stay in the run output).
- Deterministic scheduler/script shapes are matched first (their text can contain "timed out"
- and would otherwise be blamed on the model service); everything else goes through the shared
- ``classify_api_error`` verdict and the copy table in ``scheduler_failure_copy``."""
- from cron.scheduler_failure_copy import (
- classify_cron_failure_reason, generic_failure_notice, inactivity_notice,
- provider_failure_notice, script_timeout_notice)
- job_name = job.get("name") or job.get("id") or "cron job"
- job_id = job.get("id") or job_name
- text = (error or "unknown error").strip()
- lower = text.lower()
- # Script runner contract ("Script timed out after {n}s: {path}") — also for agent jobs with a
- # context script. Must precede provider classification so it never claims a model failure.
- # See #78503, #82460.
- if lower.startswith("script timed out"):
- return script_timeout_notice(job_name, job_id)
- # Scheduler inactivity watchdog ("idle for {n}s (limit {m}s)"): the job's OWN tool call went
- # quiet, no model service involved. Its text may still contain "timed out", so it must be
- # recognised before the classifier (field-reported: a stuck `terminal` call was blamed on the
- # provider and the operator debugged the wrong system).
- if re.search(r"idle for \d+s\s*\(limit \d+s\)", lower):
- return inactivity_notice(job_name, job_id)
- # no_agent jobs never reach a model, so provider errors are structurally impossible for them:
- # gate on job MODE before classifying, or a script's own wording ("429", "timed out") would
- # blame the wrong subsystem.
- if not job.get("no_agent"):
- notice = provider_failure_notice(
- job_name, job_id, classify_cron_failure_reason(text),
- backup_provider_phrase=_fallback_chain_phrase(job), provider=job.get("provider"))
- if notice is not None:
- return notice
- # Strip exception wrappers; bound input first so a multi-KB blob can't slow the regexes.
- cleaned = re.sub(r"^(RuntimeError|Exception|ValueError|HTTPStatusError):\s*", "", text[:2000])
- cleaned = re.sub(r"\s+", " ", cleaned).strip().rstrip(".")
- if len(cleaned) > 180:
- cleaned = cleaned[:177].rstrip() + "..."
- message = generic_failure_notice(job_name, job_id, cleaned)
- # Import-class failures (#95294 part 3): a long-lived gateway whose checkout was updated
- # underneath it (interrupted `hermes update`, manual git pull) serves MIXED modules and every
- # agent cron job dies with `cannot import name X`. The error reads like a code bug, so APPEND
- # cause + fix — never replace the raw error, which carries the failing symbol. Fail-safe: skew
- # is None on non-git/no-fingerprint; no_agent jobs excluded (a fresh subprocess resolves
- # imports against disk, so its ImportError is the script's own problem).
- if not job.get("no_agent") and re.search(
- r"cannot import name|modulenotfounderror|importerror", lower
- ):
- try:
- skew = _detect_gateway_code_skew()
- except Exception:
- skew = None # delivery must never die on a diagnostics probe
- if skew is not None:
- boot_rev, disk_rev = skew
- message += (
- f" Likely cause: the gateway is running stale code (booted "
- f"on {boot_rev}, disk is at {disk_rev}) — run "
- "`hermes gateway restart` to fix it."
- )
- return message
- DEFAULT_FAILURE_REPEAT_ALERT_HOURS = 6.0
- def _failure_repeat_alert_hours() -> float:
- """``cron.failure_repeat_alert_hours``: how long an ``alerted`` incident stays silent before one
- reminder ping. ``0`` (or negative) re-alerts on every failing run (the pre-gate behaviour)."""
- from cron.jobs import _cron_config_number
- return _cron_config_number("failure_repeat_alert_hours", DEFAULT_FAILURE_REPEAT_ALERT_HOURS, float)
- def _repeat_alert_withheld(incident: dict) -> bool:
- """An ``alerted`` incident withholds the per-run ping until the reminder cooldown has elapsed
- since its last ping. A missing/unparseable ``alerted_at`` (ledger written before the column
- existed) delivers rather than swallowing an alert."""
- hours = _failure_repeat_alert_hours()
- if hours <= 0:
- return False
- alerted_at = incident.get("alerted_at")
- if not alerted_at:
- return False
- try:
- from cron.jobs import _elapsed_seconds, _ensure_aware
- last = _ensure_aware(datetime.fromisoformat(str(alerted_at)))
- except (TypeError, ValueError):
- return False
- return _elapsed_seconds(_hermes_now(), last) < hours * 3600
- def _upsert_incident_for_failure(
- job: dict, error: str, *, output_file: Optional[Any] = None
- ) -> tuple[bool, Optional[str]]:
- """Record a durable failure incident (grouped by job + error signature). Returns
- ``(withheld, incident_id)``; withheld=True when the signature's incident is already ``closed``
- (operator ack) or ``alerted`` inside the ``cron.failure_repeat_alert_hours`` cooldown (a ping
- already went out) -> suppress the per-run ping. Store errors log at debug; the caller delivers
- as if none existed."""
- try:
- from cron.incidents import get_incident, upsert_incident
- incident_id, _is_new = upsert_incident(
- job["id"], str(error or ""), job_name=job.get("name"), output_file=output_file)
- incident = get_incident(incident_id)
- state = incident.get("state") if incident else None
- withheld = state == "closed" or (state == "alerted" and _repeat_alert_withheld(incident))
- return withheld, incident_id
- except Exception as exc:
- logger.debug(
- "Incident store unavailable for job %s (delivery unaffected): %s",
- job["id"], exc)
- return False, None
- def _resolve_incidents_for_recovered_job(job: dict) -> None:
- """Best-effort: a successful run marks the job's open incidents ``resolved`` (never touches an
- operator ``closed`` ack). Store errors log at debug; delivery is unaffected."""
- try:
- from cron.incidents import close_incidents_for_recovered_job
- close_incidents_for_recovered_job(job["id"])
- except Exception as exc:
- logger.debug("Incident store unavailable for job %s (delivery unaffected): %s", job["id"], exc)
- def _mark_incident_alerted(incident_id: Optional[str]) -> None:
- """Best-effort: mark incident ``alerted`` (no-op for closed; never resurrects an acked one)."""
- if not incident_id:
- return
- try:
- from cron.incidents import set_incident_state
- set_incident_state(incident_id, "alerted")
- except Exception as exc:
- logger.debug("Failed marking incident %s alerted: %s", incident_id, exc)
- class CronPromptInjectionBlocked(Exception):
- """Raised by _build_job_prompt when the assembled prompt (incl. runtime-loaded skill content,
- unseen by create-time scanning) trips the injection scanner; run_job turns it into a clean
- "job blocked" delivery.
- Assembled-prompt scanning (including loaded skill content) plugs the gap from #3968: create-time
- scanning only covers the user-supplied prompt field; skill content loaded at runtime was never scanned,
- so a malicious skill could carry an injection payload that reached the non-interactive (auto-approve)
- cron agent.
- """
- def _resolve_cron_disabled_toolsets(cfg: dict) -> list[str]:
- """Toolsets a cron-spawned agent must never receive: ``messaging``/``clarify`` always
- (interactive); ``cronjob`` by default (loop prevention, not a security boundary —
- ``cron.allow_agent_scheduling: true`` lifts only that); ``agent.disabled_toolsets`` layered on
- top so per-job ``enabled_toolsets`` cannot widen past config.yaml's denylist.
- See #25752.
- """
- cron_cfg = (cfg or {}).get("cron") or {}
- if cron_cfg.get("allow_agent_scheduling"):
- disabled = ["messaging", "clarify"]
- else:
- disabled = ["cronjob", "messaging", "clarify"]
- agent_cfg = (cfg or {}).get("agent") or {}
- from agent.skill_utils import parse_config_string_list
- user_disabled = parse_config_string_list(agent_cfg.get("disabled_toolsets"))
- for name in user_disabled:
- name = str(name).strip()
- if name and name not in disabled:
- disabled.append(name)
- return disabled
- def _merge_mcp_into_per_job_toolsets(per_job: list[str], cfg: dict) -> list[str]:
- """Layer enabled MCP servers onto a per-job ``enabled_toolsets`` allowlist (else a per-job list
- silently drops every MCP server). Mirrors ``_get_platform_tools``: ``no_mcp`` sentinel -> none
- (stripped); any MCP server already listed -> allowlist, add nothing; else union all enabled."""
- result = [t for t in per_job if t != "no_mcp"]
- if "no_mcp" in per_job:
- return result
- # lazy: avoid heavy hermes_cli import at module load; shares MCP-membership with gateway/CLI
- from hermes_cli.tools_config import enabled_mcp_server_names
- enabled_mcp = enabled_mcp_server_names(cfg)
- # A job names an MCP server by its TOOLSET name (``mcp-<server>``) or by the bare alias the
- # registry registers for it; compare against BOTH, or a job that named one server never matched
- # and the fallback below unioned EVERY enabled server into its allowlist (#141468).
- # A server that is CONFIGURED but disabled still counts as named: it is the user's selection, it
- # stays off, and it must not fall through to every enabled server either - the same rule the
- # interactive resolver follows (hermes_cli.tools_config._merge_mcp_servers).
- configured = cfg.get("mcp_servers") if isinstance(cfg.get("mcp_servers"), dict) else {}
- known = enabled_mcp | {str(name) for name in configured}
- if set(result) & (known | {f"mcp-{name}" for name in known}):
- return result
- for name in sorted(enabled_mcp):
- if name not in result:
- result.append(name)
- return result
- def _resolve_cron_enabled_toolsets(job: dict, cfg: dict) -> list[str]:
- """Toolset list for a cron job. Precedence: per-job ``enabled_toolsets`` (+ MCP merge) >
- ``cron`` platform config (``_get_platform_tools``, which strips _DEFAULT_OFF_TOOLSETS so fresh
- installs run without ``moa``). A lookup failure fails CLOSED: the run errors out.
- 1. Per-job ``enabled_toolsets`` (set via ``cronjob`` tool on create/update). Keeps the agent's
- job-scoped toolset override intact — #6130. Enabled MCP servers are layered on per
- ``_merge_mcp_into_per_job_toolsets`` so a native-toolset allowlist does not silently strip MCP tools.
- An explicitly-set EMPTY list is a zero-toolset allowlist, not a clear to the platform default —
- it is falsy, so it must be compared with ``is not None``, else it fell through to the config
- default and widened an unattended job back to every toolset (#82010). 2.
- Mirrors gateway behavior (``_get_platform_tools(cfg, platform_key)``) so users can gate cron toolsets
- globally without recreating every job. 3. Never ``None``: AIAgent reads ``None`` as "every
- toolset", so an unreadable ``platform_toolsets.cron`` restriction would hand an unattended job
- the full default set (#111380). The raise reaches ``run_job``'s failure path, which records the
- error on the job and opens an incident, so the operator sees it instead of a widened run.
- """
- per_job = job.get("enabled_toolsets")
- if per_job is not None:
- if not per_job:
- # Explicit zero: no toolsets at all — no MCP merge either, else every enabled server
- # would ride back in and widen the allowlist the operator just emptied (#82010).
- return []
- return _merge_mcp_into_per_job_toolsets(list(per_job), cfg or {})
- try:
- from hermes_cli.tools_config import _get_platform_tools # lazy: avoid heavy import at cron module load
- return sorted(_get_platform_tools(cfg or {}, "cron"))
- except Exception as exc:
- raise RuntimeError(
- "Cron toolset resolution failed, so this run was refused rather than given every "
- f"tool. Check `platform_toolsets.cron` in config.yaml (`hermes cron doctor`): {exc}"
- ) from exc
- def _resolve_job_reasoning_config(job: dict, cfg: dict, model: str) -> dict | None:
- """Effective reasoning config for a cron run. A per-job ``reasoning_effort`` pin beats global
- and per-model config and is model-independent by design (also governs an auth-fallback swap);
- clamping stays with provider transports. An unparseable pin warns and falls back to config."""
- from hermes_constants import parse_reasoning_effort, resolve_reasoning_config
- pinned = job.get("reasoning_effort")
- if pinned is not None:
- parsed = parse_reasoning_effort(pinned)
- if parsed is not None:
- logger.info("Job '%s': using per-job reasoning_effort '%s'", job.get("id", "?"), pinned)
- return parsed
- logger.warning(
- "Job '%s': invalid stored reasoning_effort %r — ignoring the pin "
- "and falling back to config resolution. Fix with `cronjob "
- "action=update job_id=%s reasoning_effort=<level>` (valid: none, "
- "minimal, low, medium, high, xhigh, max, ultra).",
- job.get("id", "?"),
- pinned,
- job.get("id", "?"))
- return resolve_reasoning_config(cfg if isinstance(cfg, dict) else {}, str(model))
- from cron.jobs import (
- _ensure_cron_dir, advance_next_runs, claim_dispatch, claim_job_for_fire, fire_claim_fence,
- clear_run_claim, get_due_jobs, heartbeat_fire_claim, heartbeat_run_claim, mark_job_run,
- save_job_output, self_removal_delivery_allowed, self_removal_delivery_scope, use_cron_store)
- from cron import store_health
- from cron.execution_identity import enter_cron_execution, exit_cron_execution
- from cron.executions import (
- FIRER_FINISHED_STATES, _TERMINAL_STATES, HANDOFF_ADOPTION_GRACE_SECONDS, create_execution, finish_execution,
- get_execution, mark_execution_handoff_pending, mark_execution_running,
- recover_interrupted_executions, settle_unstarted_execution, terminalize_dead_owner)
- from cron.scheduler_liveness import ExecutionProgressStamper, _inactivity_watchdog_loop
- from cron.scheduler_bookkeeping import _classify_delivery_outcome
- from cron.cancellation import (CancelRequest, _CancelEventLike, _CombinedCancelEvent, cancel_reason,
- clear_cancel, finish_lost_execution,
- run_job_unless_cancelled as _run_job_unless_cancelled)
- # Response marker that suppresses delivery (output is still saved locally for audit).
- SILENT_MARKER = "[SILENT]"
- # Agent-declared failure marker for cron runs. Unlike SILENT, it is deliberately strict so a
- # report that merely quotes the token cannot turn a healthy run into a failed one.
- CRON_FAILURE_MARKER = "[CRON_FAILURE]"
- def _cron_failure_marker_error(text: str) -> Optional[str]:
- """Return failure evidence when an agent response declares a cron failure.
- Only the exact, standalone first line is control text. The caller keeps the complete response
- in the saved run output while routing this evidence through normal failure bookkeeping.
- """
- lines = (text or "").splitlines()
- if not lines or lines[0].rstrip() != CRON_FAILURE_MARKER:
- return None
- evidence = "\n".join(lines[1:]).strip()
- return evidence or "Cron agent reported failure."
- def _is_cron_silence_response(text: str) -> bool:
- """True when a cron final response should suppress delivery: ``[SILENT]`` (or SILENT /
- NO_REPLY / NO REPLY) as the whole response OR its own first/last line — NOT mid-sentence.
- Shares the webhook-lane matcher in :mod:`gateway.response_filters` so the two cannot drift.
- Recognizes the bracketed ``[SILENT]`` sentinel (whole-response, first line, or last line) plus the
- bracketless ``SILENT`` / ``NO_REPLY`` / ``NO REPLY`` variants the model emits when it drops the brackets
- (#51438, #46917). Whitespace-trimmed and case-insensitive. A token buried mid-sentence is treated as
- real content and delivered.
- """
- from gateway.response_filters import is_autonomous_silence_response
- return is_autonomous_silence_response(text)
- # Persistent pool for parallel cron jobs: tick() submits and returns; long jobs never block it.
- # Keyed by profile home: one host gateway multiplexes every profile, and ``max_parallel_jobs`` is a
- # per-profile config key — a single process-global pool is sized by whichever profile ticked first
- # and then imposes that limit on all the others.
- _parallel_pools: dict[str, concurrent.futures.ThreadPoolExecutor] = {}
- _parallel_pool_max_workers: dict[str, Optional[int]] = {}
- def _inflight_key(job_id: str, home: Optional[Path | str] = None) -> tuple:
- """``(home key, job id)`` — the identity of one in-flight cron run.
- ONE gateway process ticks every profile, so a job id alone is not unique: two profiles
- routinely carry identically named jobs (``daily-brief``) and sharing a key made the second
- profile's run read as a duplicate of the first and get skipped. ``home`` defaults to the
- active cron scope's home, which the ticker binds per profile for the whole tick.
- """
- return (hermes_home_key(home) if home is not None else hermes_home_key(_get_hermes_home()), job_id)
- # Home key -> the real home Path that produced it. ``hermes_home_key`` normcases (it lower-cases on
- # Windows), so ``Path(key[0])`` is a case-folded path that matches nothing else on disk; bookkeeping
- # that needs the profile home reads it here instead of reconstructing it from the key.
- _inflight_home_paths: dict[str, Path] = {}
- def _remember_inflight_home(home: Path) -> Path:
- """Record ``home`` under its key so a claim can be mapped back to a usable path."""
- _inflight_home_paths[hermes_home_key(home)] = home
- return home
- def _inflight_home_path(home_key: str) -> Path:
- """Real home Path for an in-flight key's home half (falls back to the key itself)."""
- return _inflight_home_paths.get(home_key) or Path(home_key)
- # In-flight state below is keyed by ``_inflight_key``; the public accessors still report bare job
- # ids so host-wide consumers (shutdown drain, idle-exit, metrics) see the union across profiles.
- _running_job_ids: set = set()
- _running_fire_owners: dict[tuple, dict[object, tuple[Optional[str], Path]]] = {}
- # Parent gateway threads synchronously waiting on restart-safe scope workers.
- # Shutdown must not misclassify these as ownerless in-process runs: the tool
- # process sweep cannot reach the worker's transient scope.
- _restart_safe_waiter_job_ids: set = set()
- # in-flight key -> pid of the restart-safe external worker executing it (absent for in-process
- # runs), so a drain observer can name the process holding the gateway open.
- _running_worker_pids: dict[tuple, int] = {}
- # In-flight keys whose external worker runs in its OWN transient systemd scope
- # (``GatewayChildDispatch.mode == "scoped"``). Only that worker outlives a gateway restart: a
- # systemd stop kills the service cgroup, so a ``degraded`` direct subprocess (no reachable user
- # bus) dies with the gateway even though it is external. The gateway's restart after-turn wait
- # skips exactly this set — see ``get_restart_wait_cron_counts``.
- _scope_isolated_job_ids: set = set()
- _running_lock = threading.Lock()
- # Per in-flight id: time.time() claim instant + the future owning its release (``_FUTURE_PENDING``
- # until pool.submit returns). Past-allowance with no live future = leak; the sweep force-releases.
- _running_since: dict = {}
- # in-flight key -> stale-inflight allowance (s), resolved once per run by get_wedged_job_ids.
- _running_allowance_s: dict = {}
- _running_futures: dict = {}
- # Installed in ``_running_futures`` at claim time so a sweep landing before ``pool.submit`` returns
- # never sees ``missing`` and releases a claim about to get its future.
- _FUTURE_PENDING = object()
- # Identity tokens fence cleanup and late future attachment after recovery.
- _running_registration_owners: dict = {}
- # Forced-release count/history for ``get_inflight_guard_stats()``; mirrored to JSONL for probes.
- _forced_release_count: int = 0
- _forced_releases: list = []
- _FORCED_RELEASE_HISTORY = 20
- # Stale-allowance floor (minutes); per-job allowance is max(2 * interval, this).
- _INFLIGHT_MIN_ALLOWANCE_MINUTES = 30.0
- # Execution tokens (``_running_fire_owners`` identity keys) force-interrupted at shutdown; see
- # ``mark_running_jobs_interrupted``. ``run_one_job`` checks its OWN token before writing
- # ``last_status`` so a still-running agent thread can't overwrite "interrupted" with a false "ok".
- # Token keying scopes the flag to one execution (recurring jobs reuse IDs); legacy paths without a
- # fire owner fall back to the bare job ID.
- # ``run_one_job``'s own completion path checks its OWN token before writing ``last_status`` so a cron agent
- # thread that keeps running in-process after its tool was killed out from under it — and produces a
- # plausible-looking final response from truncated output — can never overwrite the interrupted status with a
- # false "ok" (#60432). Token keying keeps an interruption scoped to that exact execution: a later run of the
- # same job ID (recurring jobs reuse the ID every fire) must not inherit the stale flag.
- _interrupted_job_ids: set = set()
- def get_running_job_ids() -> frozenset[str]:
- """Thread-safe snapshot of executing job IDs (dispatch until ``_process_job`` returns). Read by
- the gateway shutdown drain, otherwise blind to cron work (runs outside ``_running_agents``).
- _drain_active_agents``) reads this to treat in-flight cron work as active the same way it already treats
- in-flight chat sessions via ``_running_agents`` — cron jobs run through their own thread pool here,
- entirely outside that dict, so without this the drain is structurally blind to them (#60432).
- """
- with _running_lock:
- return frozenset(key[1] for key in _running_job_ids | _running_fire_owners.keys())
- def get_running_job_details() -> list[dict]:
- """Per in-flight job: ``{"job_id", "elapsed_s", "worker_pid", "restart_safe"}`` (``worker_pid``
- None for in-process runs; ``restart_safe`` True when the worker owns its own systemd scope). The
- drain wait publishes this so ``hermes update`` can say WHICH job it is waiting on."""
- now = time.time()
- with _running_lock:
- return [
- {"job_id": key[1],
- "elapsed_s": round(now - _running_since[key], 1) if key in _running_since else None,
- "worker_pid": _running_worker_pids.get(key),
- "restart_safe": key in _scope_isolated_job_ids}
- for key in sorted(_running_job_ids | _running_fire_owners.keys())
- ]
- def get_wedged_job_ids() -> frozenset[str]:
- """In-flight job IDs older than their stale-inflight allowance (``max(2 * interval,
- cron.inflight_max_minutes)``) — the scheduler's own definition of a claim that can no longer be
- making progress. ``sweep_stale_inflight`` cannot release these while the worker thread is still
- alive (a delivery blocked on a dead transport, #115469), so the gateway restart drain reads this to
- skip them the way it skips wedged chat turns; restart is their remedy.
- """
- return frozenset(key[1] for key in _wedged_inflight_keys())
- def _wedged_inflight_keys() -> frozenset[tuple]:
- """In-flight keys behind :func:`get_wedged_job_ids`, before the projection to bare job IDs."""
- now = time.time()
- with _running_lock:
- ages = {key: now - started for key, started in _running_since.items() if key in _running_job_ids}
- allowances = {key: _running_allowance_s[key] for key in ages if key in _running_allowance_s}
- if not ages:
- return frozenset()
- floor_seconds = _inflight_min_allowance_minutes() * 60.0
- unresolved = [key for key in ages if key not in allowances]
- if unresolved:
- # One jobs.json parse per run, not per tick per job: the restart drain polls this every
- # 0.1 s on the event loop for the whole wait, and get_job() re-reads the file each call.
- # ``load_jobs`` reads the ACTIVE scope's store, so only claims from that home can have
- # their interval resolved; a claim from another profile falls back to the floor.
- by_id: dict = {}
- with contextlib.suppress(Exception):
- from cron.jobs import load_jobs
- by_id = {j.get("id"): j for j in load_jobs()}
- local_home = hermes_home_key(_get_hermes_home())
- with _running_lock:
- for key in unresolved:
- allowance = floor_seconds
- local_job = by_id.get(key[1]) if key[0] == local_home else None
- interval_minutes = _job_interval_minutes(local_job or {})
- if interval_minutes:
- allowance = max(allowance, 2.0 * interval_minutes * 60.0)
- allowances[key] = allowance
- if key in _running_job_ids: # released meanwhile -> don't resurrect the entry
- _running_allowance_s[key] = allowance
- return frozenset(
- key for key, age in ages.items() if age >= max(allowances[key], floor_seconds))
- def get_restart_wait_cron_counts() -> dict:
- """In-flight cron runs split for the gateway restart wait: ``awaitable``, ``wedged`` and
- ``restart_safe`` (scoped external worker, not wedged). The three are disjoint and sum to the
- number of in-flight runs.
- A ``scoped`` worker runs in a transient user scope outside the gateway cgroup, so a systemd
- stop/restart cannot reach it: the shutdown drain deliberately leaves it unmarked
- (``mark_running_jobs_interrupted``) and its final send rides the durable delivery queue, drained
- by whichever gateway is live next. Holding the restart wait for it only keeps the gateway in
- ``draining`` (refusing new turns) for up to ``agent.restart_after_turn_timeout``. A ``degraded``
- worker (external subprocess, no reachable user bus) stays in the gateway cgroup and dies with a
- systemd stop, so it stays awaitable.
- Counted per in-flight KEY, not per bare job ID: one multiplexed gateway can run the same job ID
- in two profiles, and projecting to bare IDs first lets one profile's wedged or scoped run hide
- the other profile's live run, which the restart would then kill (e.g. a scoped ``daily-brief``
- in profile A next to a degraded ``daily-brief`` in profile B).
- """
- wedged = _wedged_inflight_keys()
- with _running_lock:
- inflight = _running_job_ids | _running_fire_owners.keys()
- restart_safe = (_scope_isolated_job_ids & inflight) - wedged
- wedged = wedged & inflight
- return {
- "awaitable": len(inflight - wedged - restart_safe),
- "wedged": len(wedged),
- "restart_safe": len(restart_safe),
- }
- def is_job_running(job_id: str, home: Optional[Path | str] = None) -> bool:
- """True when THIS process has an in-flight run of ``job_id`` FOR ``home`` (default: the active
- cron scope's home).
- The bare-id accessors above report the host-wide union so the shutdown drain and metrics see
- every profile's work. Liveness consumers must not: one process ticks every profile, so a
- ``daily-brief`` running in profile A would otherwise report profile B's idle ``daily-brief``
- as running and keep B's stale one-shot alive as a "possibly live run".
- """
- key = _inflight_key(job_id, home)
- with _running_lock:
- return key in _running_job_ids or key in _running_fire_owners
- def _record_external_cron_worker(job_id: str, pid: Any, *, scope_isolated: bool) -> None:
- """Record the external worker holding *job_id* so a drain observer can name it, and whether
- its scope isolates it from this process.
- ``scope_isolated`` is the strong form of restart-safety: only a ``scoped`` dispatch
- (``GatewayChildDispatch``) puts the worker in its own transient systemd scope, outside the
- gateway cgroup. A ``degraded`` worker is still external but shares the cgroup and dies with a
- systemd stop, so it must NOT be listed as restart-safe. Called once per accepted handoff;
- both maps are cleared by ``release_running_job`` and by the worker waiter's ``finally``.
- """
- key = _inflight_key(job_id)
- with _running_lock:
- with contextlib.suppress(TypeError, ValueError):
- _running_worker_pids[key] = int(pid)
- if scope_isolated:
- _scope_isolated_job_ids.add(key)
- def try_register_running_job(job_id: str, *, owner=None, future=_FUTURE_PENDING) -> bool:
- """Atomically add ``job_id`` to the in-flight set; False (caller must skip) if already mid-run.
- Single dedupe owner for ticker + manual runs (the fire claim's 300s TTL is outlived by real
- jobs). Callers MUST pair success with ``release_running_job`` in a ``finally``.
- Dispatchers pass a unique ``owner`` back on release. Direct workers register a running
- ``future`` atomically because no executor submission will replace the pending sentinel.
- This is the single dedupe owner shared by the ticker's ``_submit_with_guard`` and manual runs
- (``tools/cronjob_tools``): the fire claim alone cannot prevent a double-fire because its TTL (300s) is
- routinely outlived by real jobs, after which a manual ``cronjob(action='run')`` would claim successfully
- and run the same job concurrently (idea from #53395 by @izumi0uu).
- Registration also makes the run visible to ``get_running_job_ids`` (the gateway shutdown drain, #60432)
- and ``mark_running_jobs_interrupted``. Dedupe is per PROFILE: the key carries the active cron
- scope's home, so one multiplexing process never treats two profiles' same-named jobs as one.
- """
- from hermes_cli.backend_retirement import retirement
- key = _inflight_key(job_id, _remember_inflight_home(_get_hermes_home()))
- with retirement.work() as admitted, _running_lock:
- if not admitted or key in _running_job_ids:
- return False
- _running_job_ids.add(key)
- # Same critical section as the add: no window where an in-flight id lacks an age the sweep
- # can bound. Sentinel is replaced by the real future once ``pool.submit`` returns.
- _running_since[key] = time.time()
- _running_futures[key] = future
- _running_registration_owners[key] = owner
- return True
- def release_running_job(
- job_id: str, home: Optional[Path | str] = None, *, owner=None,
- ) -> None:
- """Remove the registration unless an explicit ``owner`` has been replaced.
- ``home`` MUST be passed by any caller that does not run inside the same cron scope the claim
- was registered under. The scope is a ContextVar the ticker binds per profile: a pool worker
- releasing in a ``finally`` outside ``ctx.run`` resolves the default (launch) home instead, so
- the discard misses the real key and every secondary profile's claim leaks.
- """
- key = _inflight_key(job_id, home)
- with _running_lock:
- if owner is not None and _running_registration_owners.get(key) is not owner:
- return
- _running_registration_owners.pop(key, None)
- _running_job_ids.discard(key)
- _running_since.pop(key, None)
- _running_allowance_s.pop(key, None)
- _running_futures.pop(key, None)
- _running_worker_pids.pop(key, None)
- _scope_isolated_job_ids.discard(key)
- def _inflight_min_allowance_minutes() -> float:
- """Stale allowance floor (min): ``cron.inflight_max_minutes``, else env escape hatch/default."""
- with contextlib.suppress(Exception):
- _ucfg = load_config() or {}
- _cfg_val = (
- _ucfg.get("cron", {}) if isinstance(_ucfg, dict) else {}
- ).get("inflight_max_minutes")
- if _cfg_val is not None:
- val = float(_cfg_val)
- if val > 0:
- return val
- raw = cron_env_setting("HERMES_CRON_INFLIGHT_MAX_MINUTES").strip()
- if raw:
- try:
- val = float(raw)
- if val > 0:
- return val
- except (ValueError, TypeError):
- logger.warning(
- "Invalid HERMES_CRON_INFLIGHT_MAX_MINUTES=%r; using default %s",
- raw,
- _INFLIGHT_MIN_ALLOWANCE_MINUTES)
- return _INFLIGHT_MIN_ALLOWANCE_MINUTES
- # expr -> minutes; cadence never changes, so avoid re-evaluating croniter every tick.
- _cron_interval_cache: dict = {}
- def _cron_interval_minutes(expr: str) -> Optional[float]:
- """Cron expression cadence (gap between next two fires) in minutes; None -> floor allowance."""
- if expr in _cron_interval_cache:
- return _cron_interval_cache[expr]
- result = None
- ok = False
- with contextlib.suppress(Exception):
- from cron.jobs import _ensure_croniter
- ok = _ensure_croniter()
- if ok:
- from cron.jobs import croniter as _croniter
- from datetime import datetime
- base = datetime.now()
- it = _croniter(expr, base)
- first = it.get_next(datetime)
- second = it.get_next(datetime)
- gap = (second - first).total_seconds() / 60.0
- result = gap if gap > 0 else None
- # Cache a real cadence or a bad-expr None (croniter loaded, expr invalid: stable). Skip the
- # None from a transient croniter ImportError so it can't pin the floor allowance for the
- # process lifetime once the import recovers.
- if result is not None or ok:
- _cron_interval_cache[expr] = result
- return result
- def _job_interval_minutes(job: dict) -> Optional[float]:
- """Best-effort job interval in minutes (None if unknown / one-shot -> floor). ``schedule`` is
- persisted as a parsed dict; the string path is only a fallback for programmatic callers."""
- with contextlib.suppress(Exception):
- schedule = job.get("schedule")
- if isinstance(schedule, str) and schedule.strip():
- from cron.jobs import parse_schedule
- schedule = parse_schedule(schedule) or {}
- if isinstance(schedule, dict):
- kind = schedule.get("kind")
- if kind == "interval":
- minutes = schedule.get("minutes")
- return float(minutes) if minutes else None
- if kind == "cron":
- return _cron_interval_minutes(str(schedule.get("expr") or ""))
- return None
- def get_inflight_guard_stats() -> dict:
- """Probe-visible snapshot; non-zero ``forced_releases`` means a job wedged and was recovered."""
- now = time.time()
- with _running_lock:
- return {
- "running": sorted(key[1] for key in _running_job_ids),
- "running_ages_seconds": {
- key[1]: round(now - started, 1)
- for key, started in _running_since.items()
- },
- "forced_releases": _forced_release_count,
- "recent_forced_releases": list(_forced_releases)}
- def _record_forced_release(job_id: str, name: str, age_seconds: float, allowance_seconds: float) -> None:
- """Persist a countable signal for one forced release (best-effort)."""
- entry = {
- "job_id": job_id,
- "name": name,
- "age_seconds": round(age_seconds, 1),
- "allowance_seconds": round(allowance_seconds, 1),
- "at": _hermes_now().isoformat()}
- with _running_lock:
- _forced_releases.append(entry)
- del _forced_releases[:-_FORCED_RELEASE_HISTORY]
- try:
- path = _get_hermes_home() / "cron" / "inflight_forced_releases.jsonl"
- _ensure_cron_dir(path.parent)
- with open(path, "a", encoding="utf-8") as fh:
- fh.write(json.dumps(entry) + "\n")
- except Exception as e: # never let telemetry break a tick
- logger.debug("Could not append forced-release record: %s", e)
- def _latest_executions_for_releasable_claims() -> dict:
- """Latest durable execution per releasable-looking claim (missing/pending/done future), one
- indexed query so the healthy path pays no DB work. Snapshot under _running_lock — iterating
- the set while try_register/release mutate it raises RuntimeError.
- Scoped to the ACTIVE profile's claims: the executions ledger it queries is that home's.
- """
- local_home = hermes_home_key(_get_hermes_home())
- with _running_lock:
- claim_futures = {
- key[1]: _running_futures.get(key)
- for key in _running_job_ids if key[0] == local_home
- }
- candidates = [
- job_id for job_id, fut in claim_futures.items()
- if fut is None or fut is _FUTURE_PENDING or fut.done()
- ]
- if not candidates:
- return {}
- try:
- from cron.executions import latest_executions as _latest_execs
- return _latest_execs(candidates)
- except Exception:
- return {}
- def _row_belongs_to_claim(row: dict, claim_started: float) -> bool:
- """True when the ledger row was claimed at/after this in-memory claim. An older terminal row is
- the PREVIOUS run's (try_register->create_execution window); releasing on it would
- double-dispatch. Unparseable timestamps fail closed (the age path still bounds the claim)."""
- claimed_at = row.get("claimed_at")
- if not claimed_at:
- return False
- try:
- from cron.jobs import _ensure_aware as _ensure_aware_ts
- row_ts = _ensure_aware_ts(datetime.fromisoformat(claimed_at))
- return row_ts.timestamp() >= claim_started
- except (ValueError, TypeError, OSError):
- return False
- def _record_stale_release(job: dict, job_id: str, age: float, allowance: float, fut, reason: str) -> None:
- """WARNING log + probe record for one forced release, then ``last_error`` unless the ledger
- already holds the run's real outcome or the job has a finite repeat budget."""
- name = job.get("name") or job_id
- future_state = "pending" if fut is _FUTURE_PENDING else "missing" if fut is None else "finished"
- logger.warning(
- "cron.inflight.forced_release event=forced_release reason=%s job='%s' "
- "id=%s age=%.0fs allowance=%.0fs future=%s — stale in-flight claim "
- "released; the job was skipping every fire with 'already running'",
- reason, name, job_id, age, allowance, future_state)
- _record_forced_release(job_id, name, age, allowance)
- # Ledger already records how the run ended: mark_job_run here would clobber an honest
- # ok status with a synthetic failure or double-write a failure.
- if reason == "ledger-terminal":
- return
- # Age release may lack a ledger row, so last_error is how it surfaces. But a forced release
- # is NOT a real run: never consume a finite repeat budget or let mark_job_run auto-delete.
- repeat = job.get("repeat") or {}
- if isinstance(repeat, dict) and repeat.get("times") is not None:
- logger.warning(
- "cron.inflight.forced_release.job_untouched job='%s' id=%s — "
- "finite-repeat job released without mark_job_run (repeat budget "
- "preserved); row left in place so it re-fires normally",
- name, job_id)
- return
- try:
- mark_job_run(
- job_id, False,
- f"Stale in-flight claim force-released after {age / 60:.1f}m "
- f"(allowance {allowance / 60:.1f}m); previous run never released "
- f"the scheduler in-flight guard")
- except Exception as e:
- logger.warning("Could not record forced release for job %s: %s", job_id, e)
- def sweep_stale_inflight(due_jobs: Optional[list] = None) -> list:
- """Force-release in-flight claims that can no longer be making progress; returns released ids.
- Stale = older than ``max(2 * interval, floor)`` AND (no live future — submit path hung before
- ``pool.submit`` returned — or finished without discarding the id) — or the claim's OWN ledger
- row is terminal regardless of age. Each release logs WARNING ``event=forced_release``, bumps
- the probe counter, mirrors JSONL, and writes ``last_error``.
- """
- global _forced_release_count
- by_id = {j.get("id"): j for j in (due_jobs or []) if isinstance(j, dict)}
- floor_seconds = _inflight_min_allowance_minutes() * 60.0
- now = time.time()
- stale: list = []
- from cron.executions import _TERMINAL_STATES as _terminal_states
- _latest = _latest_executions_for_releasable_claims()
- _local_home = hermes_home_key(_get_hermes_home())
- # Compute intervals OUTSIDE _running_lock so croniter doesn't block try_register/release.
- _intervals = {jid: _job_interval_minutes(j) for jid, j in by_id.items()}
- with _running_lock:
- for key in [k for k in _running_job_ids if k[0] == _local_home]:
- job_id = key[1]
- started = _running_since.get(key)
- if started is None:
- # Claim predates this guard — adopt it; sweepable one allowance from now.
- _running_since[key] = now
- continue
- age = now - started
- interval_minutes = _intervals.get(job_id)
- allowance = floor_seconds
- if interval_minutes:
- allowance = max(allowance, 2.0 * interval_minutes * 60.0)
- fut = _running_futures.get(key)
- if fut is _FUTURE_PENDING:
- # Submit path hung before ``pool.submit`` returned — the wedge class; release it.
- pass
- elif fut is not None and not fut.done():
- continue # genuinely still executing
- # Ledger reconciliation: a terminal row belonging to THIS claim proves it stale even
- # inside its age allowance. Row must be this claim's, else a recurring job's previous
- # run would double-dispatch a fresh claim.
- latest = _latest.get(job_id)
- if (
- latest is not None
- and latest.get("status") in _terminal_states
- and _row_belongs_to_claim(latest, started)
- ):
- reason = "ledger-terminal"
- elif age >= allowance:
- reason = "age"
- else:
- continue
- _running_registration_owners.pop(key, None)
- _running_job_ids.discard(key)
- _running_since.pop(key, None)
- _running_allowance_s.pop(key, None)
- _running_futures.pop(key, None)
- _forced_release_count += 1
- stale.append((job_id, age, allowance, fut, reason))
- for job_id, age, allowance, fut, _reason in stale:
- _record_stale_release(by_id.get(job_id) or {}, job_id, age, allowance, fut, _reason)
- return [s[0] for s in stale]
- def mark_running_jobs_interrupted(
- reason: str, *, only_owners: Optional[set] = None,
- ) -> list:
- """Best-effort: mark every in-flight cron job interrupted; returns the job IDs marked.
- Called by gateway shutdown right after ``process_registry.kill_all()``: a job whose tool was
- killed must never report success. ``only_owners`` (``(job_id, fire_owner)`` pairs) restricts
- marking. Tokens go into ``_interrupted_job_ids`` BEFORE ``last_status`` is written so
- ``run_one_job`` sees them.
- """
- with _running_lock:
- restart_safe_waiters = set(_restart_safe_waiter_job_ids)
- active_fires = [
- (token, key, owner, profile_home)
- for key, executions in _running_fire_owners.items()
- if key not in restart_safe_waiters
- for token, (owner, profile_home) in executions.items()
- ]
- if only_owners is not None:
- active_fires = [fire for fire in active_fires if (fire[1][1], fire[2]) in only_owners]
- registered_keys = {key for _t, key, _o, _p in active_fires}
- if only_owners is None:
- # The key's home half IS the profile home this claim belongs to — the only record of
- # it for a claim that never reached ``_running_fire_owners``. Read the real Path back
- # from the key, never ``Path(key[0])``: the key is normcased.
- active_fires.extend(
- (None, key, None, _inflight_home_path(key[0]))
- for key in (
- _running_job_ids - registered_keys - restart_safe_waiters
- )
- )
- _interrupted_job_ids.update(
- token if token is not None else key
- for token, key, _owner, _profile_home in active_fires
- )
- marked = []
- for _token, key, fire_owner, profile_home in active_fires:
- job_id = key[1]
- if not fire_owner:
- logger.warning(
- "Job '%s' interrupted before its durable fire owner was registered; "
- "leaving persisted state untouched",
- job_id)
- # Still report it: shutdown uses the returned IDs for the interrupted-cron notice. The
- # in-memory flag WAS recorded above; only the persisted last_status write is skipped.
- # See #82232.
- marked.append(job_id)
- continue
- try:
- with use_cron_store(profile_home):
- if mark_job_run(
- job_id, False, reason, expected_fire_owner=fire_owner):
- marked.append(job_id)
- except Exception as e:
- logger.warning("Failed to mark job %s interrupted: %s", job_id, e)
- return marked
- def _is_interrupted(job_id: str, token: Optional[object] = None) -> bool:
- """Non-destructive peek: has shutdown marked THIS execution interrupted? Used before deciding
- what to deliver; does not clear the flag (the authoritative pre-``last_status`` check needs it).
- ``token`` scopes to one execution so a fresh run reusing the job ID isn't poisoned."""
- key = _inflight_key(job_id)
- with _running_lock:
- if token is not None and token in _interrupted_job_ids:
- return True
- return key in _interrupted_job_ids
- def _consume_interrupted_flag(job_id: str, token: Optional[object] = None) -> bool:
- """Return True and clear the flag if shutdown marked THIS execution interrupted. Called right
- before ``last_status`` is written; consuming stops the flag leaking into a later run."""
- key = _inflight_key(job_id)
- with _running_lock:
- hit = False
- if token is not None and token in _interrupted_job_ids:
- _interrupted_job_ids.discard(token)
- hit = True
- if key in _interrupted_job_ids:
- _interrupted_job_ids.discard(key)
- hit = True
- return hit
- def _cron_inactivity_seconds() -> float:
- """Parse HERMES_CRON_TIMEOUT (seconds). 0 = unlimited; bad input = 600. Shared by the
- inactivity monitor and the cwd-lock bound so they can't drift: the lock bound must stay >= the
- inactivity limit or waiters fail while a healthy holder runs."""
- raw = cron_env_setting("HERMES_CRON_TIMEOUT").strip()
- if not raw:
- return 600.0
- try:
- return float(raw)
- except (ValueError, TypeError):
- logger.warning("Invalid HERMES_CRON_TIMEOUT=%r; using default 600s", raw)
- return 600.0
- def _get_parallel_pool(max_workers: Optional[int]) -> concurrent.futures.ThreadPoolExecutor:
- """Return (or create) the persistent parallel pool for the ACTIVE profile.
- Pools are per profile home because ``cron.max_parallel_jobs`` is a per-profile config key:
- a single process-global pool is created by whichever profile the multiplexing gateway ticks
- first and then silently imposes that profile's limit on every other profile.
- """
- home = hermes_home_key(_remember_inflight_home(_get_hermes_home()))
- pool = _parallel_pools.get(home)
- if pool is None or _parallel_pool_max_workers.get(home) != max_workers:
- if pool is not None:
- pool.shutdown(wait=False, cancel_futures=False)
- pool = concurrent.futures.ThreadPoolExecutor(
- max_workers=max_workers, thread_name_prefix="cron-parallel")
- _parallel_pools[home] = pool
- _parallel_pool_max_workers[home] = max_workers
- return pool
- def discard_parallel_pools(home_keys) -> None:
- """Drop the parallel pools of homes this process no longer ticks.
- Pools are per home and used to live until ``atexit``: a host that serves many profiles — or
- churns them — accumulated one ThreadPoolExecutor and its live ``cron-parallel`` threads per
- home ever ticked, and a deleted profile's pool was never reclaimed. ``wait=False`` so the
- ticker is never blocked; queued work still drains before the executor dies.
- """
- for key in list(home_keys):
- pool = _parallel_pools.pop(key, None)
- _parallel_pool_max_workers.pop(key, None)
- if pool is not None:
- pool.shutdown(wait=False, cancel_futures=False)
- def _shutdown_parallel_pool() -> None:
- """Shut down every profile's persistent pool on process exit."""
- for pool in list(_parallel_pools.values()):
- pool.shutdown(wait=True, cancel_futures=False)
- _parallel_pools.clear()
- _parallel_pool_max_workers.clear()
- atexit.register(_shutdown_parallel_pool)
- # Per-fire usage audit log; resolves via _get_hermes_home() so profile-scoped paths work.
- def _usage_audit_path() -> Path:
- return _get_hermes_home() / "cron" / "usage_audit.jsonl"
- def _utcnow_iso_ms() -> str:
- """RFC3339 UTC timestamp with millisecond precision and 'Z' suffix."""
- now = datetime.now(UTC)
- return now.strftime("%Y-%m-%dT%H:%M:%S.") + f"{now.microsecond // 1000:03d}Z"
- def _write_usage_audit(record: dict) -> None:
- """Append one JSONL line to cron/usage_audit.jsonl. NEVER raises — a logger bug must not
- break cron jobs (the whole write is inside one try)."""
- try:
- path = _usage_audit_path()
- _ensure_cron_dir(path.parent)
- line = json.dumps(record, ensure_ascii=False)
- with open(path, "a", encoding="utf-8") as f:
- f.write(line + "\n")
- except Exception as e:
- logger.warning("usage_audit write failed: %s", e)
- def _interpreter_shutting_down(exc: Optional[BaseException] = None) -> bool:
- """True when the interpreter is finalizing (tick fired during gateway teardown): concurrent.
- futures/asyncio refuse new work, so delivery attempts only pollute errors.log — callers skip
- with a warning. ``exc`` lets an already-raised scheduling error count as a shutdown signal.
- A cron tick can fire while the gateway is tearing down — SIGTERM from ``hermes update`` / ``hermes
- gateway stop`` / systemd restart, or an OOM-kill. Once finalization starts, ``concurrent.futures``
- refuses new work with ``RuntimeError: cannot schedule new futures after interpreter shutdown`` and
- asyncio's default executor is gone, so *any* attempt to schedule delivery (live-adapter,
- ``asyncio.run``, or a fresh pool) is doomed and only pollutes ``errors.log`` with a traceback. See
- #55924, #58720.
- """
- from tools.interpreter_shutdown import interpreter_shutting_down
- return interpreter_shutting_down(exc)
- # Module override hook for tests / emergency monkeypatches.
- _hermes_home: Path | None = None
- def _get_hermes_home() -> Path:
- """Hermes home at call time (honouring the test override). Cron is per-profile: never freeze
- this at import or anchor it at the shared default root — either breaks profile isolation.
- Cron is per-profile by design (#4707): the in-process ticker runs inside a profile-scoped gateway, so
- resolving the active HERMES_HOME at call time means a profile's jobs are stored AND executed under that
- profile's home (its .env, config.yaml, scripts, skills).
- """
- return _hermes_home or get_hermes_home()
- def _get_lock_paths() -> tuple[Path, Path]:
- """Resolve cron lock paths at call time so profile/env changes are honored."""
- hermes_home = _get_hermes_home()
- lock_dir = hermes_home / "cron"
- return lock_dir, lock_dir / ".tick.lock"
- # Errnos that mean "another ticker (or manual tick) holds the tick lock", as opposed to a real failure
- # opening/locking the file. Everything else — most importantly EMFILE/ENFILE (fd exhaustion, #87644) and
- # EACCES on open() — must be surfaced, never swallowed as lock contention.
- def _is_lock_contention_errno(err: OSError) -> bool:
- """True when *err* from the lock syscall means another ticker holds the lock (POSIX flock:
- EWOULDBLOCK/EAGAIN, EACCES on some NFS; msvcrt.locking: EACCES/EDEADLK). Everything else —
- notably EMFILE/ENFILE and EACCES on open() — must be surfaced, never swallowed as contention."""
- if err.errno is None:
- return False
- if fcntl is not None:
- return err.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES)
- if msvcrt is not None:
- return err.errno in (errno.EACCES, errno.EDEADLK)
- return False
- def _is_fd_exhaustion_text(text: str) -> bool:
- """Text half of _is_fd_exhaustion (shared with the CLI hint)."""
- lowered = text.lower()
- return "too many open files" in lowered or "emfile" in lowered
- def _is_fd_exhaustion(exc: BaseException) -> bool:
- """True when *exc* indicates fd exhaustion: EMFILE/ENFILE errno, or the "Too many open files"
- wording for wrapped exceptions (load_jobs wraps the OSError in a RuntimeError).
- See #87644.
- """
- if isinstance(exc, OSError) and exc.errno in (errno.EMFILE, errno.ENFILE):
- return True
- return _is_fd_exhaustion_text(str(exc))
- def _reclaim_fds_best_effort() -> None:
- """Best-effort fd reclamation: gc.collect() closes file objects stuck in reference cycles;
- apply_nofile_soft_limit() raises the RLIMIT_NOFILE soft limit for headroom. Never raises.
- The cron FD-leak family (#60859, #79742, #80792) leaks descriptors from abandoned workers/sessions. Two
- safe, idempotent levers:
- """
- with contextlib.suppress(Exception):
- import gc
- gc.collect()
- with contextlib.suppress(Exception):
- from hermes_cli.resource_limits import apply_nofile_soft_limit
- apply_nofile_soft_limit(None)
- def drain_delivery_queue(adapters, loop) -> int:
- """Send queued worker results through this gateway's live adapters."""
- from cron.delivery_queue import _path, drain
- # Only restart-safe workers create the queue file. Every gateway (macOS,
- # Windows, launchd, Docker) runs this housekeeping tick, so skip the sqlite
- # open/create entirely until a worker has actually queued something.
- if not _path().exists():
- return 0
- def send(queued_job, queued_content, queued_for_failure):
- error = _deliver_result(
- queued_job, queued_content, adapters=adapters, loop=loop,
- for_failure=queued_for_failure,
- )
- # The ping left the process: book the incident alerted. A notice every target hid
- # (warning notifications suppressed) is a durable disposition, not an alert.
- if (not error and queued_for_failure
- and not queued_job.get("_notification_all_targets_suppressed")):
- _mark_incident_alerted(queued_job.get("_failure_incident_id"))
- return error
- return drain(send)
- _DEFAULT_SCRIPT_TIMEOUT = 3600 # seconds (1 hour)
- # Backward-compatible module override used by tests and emergency monkeypatches.
- _SCRIPT_TIMEOUT = _DEFAULT_SCRIPT_TIMEOUT
- _RUN_CLAIM_HEARTBEAT_SECONDS = 60.0
- _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS = _RUN_CLAIM_HEARTBEAT_SECONDS * 3
- # Pause before re-sampling a fire-claim heartbeat miss; a genuinely re-owned claim misses twice.
- _FIRE_CLAIM_MISS_CONFIRM_SECONDS = 1.0
- def _cron_cleanup_timeout_seconds() -> float:
- """Return the wall-clock bound for cron post-run cleanup."""
- default = 10.0
- try:
- from hermes_cli.config import load_config
- cfg = load_config() or {}
- cron_cfg = cfg.get("cron", {}) if isinstance(cfg, dict) else {}
- configured = cron_cfg.get("cleanup_timeout_seconds")
- if configured is not None:
- timeout = float(configured)
- if timeout >= 0:
- return timeout
- except Exception as exc:
- logger.debug("Failed to load cron cleanup timeout from config: %s", exc)
- return default
- def _run_cron_cleanup_with_timeout(
- cleanup, *, job_id: str, label: str, timeout_seconds: Optional[float] = None,
- ) -> bool:
- """Run fallible post-run cleanup without permanently wedging a cron ID."""
- timeout = (_cron_cleanup_timeout_seconds() if timeout_seconds is None else float(timeout_seconds))
- if timeout <= 0:
- try:
- cleanup()
- return True
- except (Exception, KeyboardInterrupt) as exc:
- logger.debug("Job '%s': %s failed: %s", job_id, label, exc)
- return False
- done = threading.Event()
- error: list[BaseException] = []
- def _runner() -> None:
- try:
- cleanup()
- except BaseException as exc:
- error.append(exc)
- finally:
- done.set()
- # Daemon thread is deliberate: unlike ThreadPoolExecutor workers it is not joined at interpreter
- # exit if cleanup never returns, so the gateway can still shut down.
- worker = threading.Thread(
- target=ctx_bound(_runner), name=f"cron-cleanup-{job_id}", daemon=True)
- worker.start()
- if not done.wait(timeout):
- logger.error(
- "Job '%s': %s exceeded %.1fs; abandoning cleanup so future runs remain dispatchable",
- job_id,
- label,
- timeout)
- return False
- if error:
- logger.debug("Job '%s': %s failed: %s", job_id, label, error[0])
- return False
- return True
- class _BoundedCronSessionDB:
- """Proxy SessionDB cleanup calls through the cron cleanup timeout; after the first failure or
- timeout all later calls fail immediately (a damaged connection leaks at most one worker)."""
- def __init__(self, session_db, job_id: str):
- self._session_db = session_db
- self._job_id = job_id
- self._disabled = False
- def __getattr__(self, name):
- target = getattr(self._session_db, name)
- if not callable(target):
- return target
- def _bounded(*args, **kwargs):
- if self._disabled:
- raise RuntimeError("session finalization disabled after prior cleanup failure")
- result = {}
- def _call():
- try:
- result["value"] = target(*args, **kwargs)
- except BaseException as exc:
- result["error"] = exc
- raise
- ok = _run_cron_cleanup_with_timeout(
- _call, job_id=self._job_id, label=f"session finalization ({name})")
- if not ok:
- error = result.get("error")
- if error is not None:
- raise error
- # No error yet not complete == timeout: disable so later steps fail fast.
- self._disabled = True
- raise TimeoutError(f"session finalization method {name} timed out")
- return result.get("value")
- return _bounded
- def _resolve_job_workdir(job: dict, job_id: str) -> Optional[str]:
- """Configured job workdir, or None when unset / no longer a directory (logged)."""
- workdir = (job.get("workdir") or "").strip() or None
- if workdir and not Path(workdir).is_dir():
- logger.warning(
- "Job '%s': configured workdir %r no longer exists — running without it",
- job_id, workdir)
- return None
- return workdir
- def _run_no_agent_job(
- job: dict, job_id: str, job_name: str, cancel_event,
- ) -> tuple[bool, str, str, Optional[str]]:
- """no_agent short-circuit — the script IS the job (no AIAgent, no tokens). stdout → delivered
- verbatim; empty stdout or wakeAgent=false → silent success; non-zero exit/timeout → error alert.
- """
- # Load .env first so auto-delivery can resolve *_HOME_CHANNEL: the agent path's per-run dotenv
- # reload never runs for no_agent jobs. Does not override existing values.
- try:
- from hermes_cli.env_loader import load_hermes_dotenv
- load_hermes_dotenv(hermes_home=_get_hermes_home())
- except Exception:
- logger.debug("Job '%s': no_agent .env reload failed", job_id, exc_info=True)
- script_path = job.get("script")
- # Legacy/hand-edited no_agent job without a script: pause it, or it re-fires every tick.
- if not str(script_path or "").strip():
- from cron.jobs_invariants import NO_AGENT_WITHOUT_SCRIPT_ERROR
- return _block_and_pause_job(job_id, job_name, NO_AGENT_WITHOUT_SCRIPT_ERROR)
- # Pass workdir as subprocess cwd; never os.chdir() (leaks into concurrent gateway sessions).
- _job_workdir = _resolve_job_workdir(job, job_id)
- try:
- ok, output = _run_job_script_with_claim_heartbeat(
- job, script_path, workdir=_job_workdir, cancel_event=cancel_event)
- except Exception as exc:
- logger.exception("Job '%s': script execution raised unexpectedly", job_id)
- ok, output = False, f"Script execution failed: {exc}"
- now_iso = _hermes_now().strftime("%Y-%m-%d %H:%M:%S")
- header = _job_doc_header(job_name, job_id, now_iso, "no_agent (script)")
- if not ok:
- # Deliver the error: a silently broken watchdog is the worst-case outcome.
- alert = (
- f"⚠ Cron watchdog '{job_name}' script failed\n\n"
- f"{output}\n\n"
- f"Time: {now_iso}"
- )
- return False, f"{header}**Status:** script failed\n\n{output}\n", alert, output
- # wakeAgent=false is a silent signal, same as empty stdout.
- if not _parse_wake_gate(output):
- logger.info("Job '%s' (no_agent): wakeAgent=false gate — silent run", job_id)
- return True, f"{header}**Status:** silent (wakeAgent=false)\n", SILENT_MARKER, None
- if not output.strip():
- logger.info("Job '%s' (no_agent): empty stdout — silent run", job_id)
- return True, f"{header}**Status:** silent (empty output)\n", SILENT_MARKER, None
- return True, f"{header}\n---\n\n{output}\n", output, None
- def _apply_monitor_gate(
- job: dict, job_id: str, job_name: str, extra_prompt: Optional[str],
- ) -> tuple[Optional[tuple], Optional[str], Optional[str]]:
- """Monitor gate (hash-suppressed change detection). Must run BEFORE any agent machinery so an
- unchanged tick costs no LLM/delivery. Returns ``(early_result | None, extra_prompt,
- monitor_context)``. Monitor context is runtime data and must remain distinct from a
- user-authored ``extra_prompt`` so the prompt scanner keeps its strict user-input boundary.
- """
- from cron.monitor import check_monitor, job_has_monitor
- if not job_has_monitor(job):
- return None, extra_prompt, None
- _mon = check_monitor(job)
- _mon_now = _hermes_now().strftime("%Y-%m-%d %H:%M:%S")
- header = _job_doc_header(job_name, job_id, _mon_now, "monitor")
- if not _mon.ok:
- # Source failure is an ERROR, never a change: alert so a broken monitor can't silently
- # stop watching. Stored hash untouched.
- logger.error("Job '%s': monitor source failed: %s", job_id, _mon.error)
- _mon_alert = (
- f"⚠ Cron monitor '{job_name}' source failed\n\n"
- f"{_mon.error}\n\n"
- f"Time: {_mon_now}"
- )
- return (
- False, f"{header}**Status:** monitor source failed\n\n{_mon.error}\n", _mon_alert, _mon.error,
- ), extra_prompt, None
- if not _mon.changed:
- # Unchanged: silent no_change tick (ledger doc kept; SILENT_MARKER blocks delivery).
- logger.info("Job '%s': monitor output unchanged — suppressing agent run", job_id)
- return (
- True, f"{header}**Status:** no_change (agent run suppressed)\n", SILENT_MARKER, None,
- ), extra_prompt, None
- # Changed (or first run): pass monitor output through the runtime-data seam. Keep any manual
- # per-run prompt separate: it remains user input and is therefore still strict-scanned.
- return None, extra_prompt, _mon.context_block
- @dataclass
- class _CronJobConfig:
- """Config-derived inputs for one agent-backed cron run."""
- cfg: dict
- model: str
- model_cfg: Any
- cron_default_provider: str
- def _load_cron_job_config(job: dict, job_id: str, job_name: str) -> _CronJobConfig:
- """Load config.yaml and resolve the run's model: per-job pin > cron.model (fleet default) >
- the main agent model (config ``model:``, then HERMES_MODEL) > the claimed welcome trial's. Re-read
- every tick (no cache) so ``hermes cron edit --model`` and ``hermes model`` both apply next tick."""
- from hermes_cli.anon_auth import trial_model
- model = job.get("model") or cron_env_setting("HERMES_MODEL") or ""
- _cron_default_provider = ""
- _cfg: dict = {}
- _model_cfg: Any = {}
- try:
- from hermes_cli.config_backend import config_exists
- from hermes_cli.config_effective import load_user_config_effective
- _cfg_path = str(_get_hermes_home() / "config.yaml")
- if config_exists(_cfg_path):
- _cfg = load_user_config_effective(Path(_cfg_path))
- # Coerce null to {} so a falsy default never clobbers a resolved env value.
- _model_cfg = _cfg.get("model") or {}
- _cron_cfg_for_model = _cfg.get("cron") or {}
- _cron_default_model = ""
- if isinstance(_cron_cfg_for_model, dict):
- _cron_default_model = str(_cron_cfg_for_model.get("model") or "").strip()
- _cron_default_provider = str(_cron_cfg_for_model.get("model_provider") or "").strip()
- if not job.get("model"):
- if _cron_default_model:
- model = _cron_default_model
- else:
- # The main agent model: ``model: <name>`` shorthand or ``model.default``.
- _main = _model_cfg if isinstance(_model_cfg, str) else (
- _model_cfg.get("default") or _model_cfg.get("model") or _model_cfg.get("name")
- if isinstance(_model_cfg, dict) else "")
- model = str(_main or "").strip() or model
- except Exception as e: # ConfigBackendUnavailable is not one: run_job fails this fire, never defaults
- logger.warning("Job '%s': failed to load config.yaml, using defaults: %s", job_id, e)
- model = (model if isinstance(model, str) else "").strip() or trial_model()
- # Fail fast: an empty model otherwise reaches the provider as an opaque 400 (#23979).
- if not model:
- raise RuntimeError(
- f"Cron job '{job_name}' has no model configured "
- f"(job.model={job.get('model')!r}, "
- f"HERMES_MODEL={cron_env_setting('HERMES_MODEL')!r}, "
- "config.yaml model.default missing or empty). "
- f"Set a per-job model via "
- f"`hermes cron edit {job_id} --model <name>` or set a "
- "default with `hermes model <name>`."
- )
- with contextlib.suppress(Exception):
- from hermes_constants import apply_ipv4_preference
- _net_cfg = _cfg.get("network", {})
- if isinstance(_net_cfg, dict) and _net_cfg.get("force_ipv4"):
- apply_ipv4_preference(force=True)
- return _CronJobConfig(_cfg, model, _model_cfg, _cron_default_provider)
- def _load_prefill_messages(cfg: dict, job_id: str) -> Optional[list]:
- """Prefill messages from env or config.yaml (top-level key canonical; agent.* is legacy)."""
- agent_cfg = cfg.get("agent", {}) if isinstance(cfg.get("agent", {}), dict) else {}
- prefill_file = (
- cron_env_setting("HERMES_PREFILL_MESSAGES_FILE")
- or cfg.get("prefill_messages_file", "")
- or agent_cfg.get("prefill_messages_file", "")
- )
- if not prefill_file:
- return None
- pfpath = Path(prefill_file).expanduser()
- if not pfpath.is_absolute():
- pfpath = _get_hermes_home() / pfpath
- if not pfpath.exists():
- return None
- try:
- with open(pfpath, "r", encoding="utf-8-sig") as _pf:
- prefill_messages = json.load(_pf)
- return prefill_messages if isinstance(prefill_messages, list) else None
- except Exception as e:
- logger.warning("Job '%s': failed to parse prefill messages file '%s': %s", job_id, pfpath, e)
- return None
- def _preflight_or_block(job: dict, job_id: str, job_name: str, cfg: dict) -> Optional[tuple]:
- """Pre-dispatch config validation: refuse unrunnable jobs (missing key, unready skill,
- unconfigured delivery) BEFORE AIAgent is built. run_one_job keys off BLOCKED_CONFIG_MARKER to
- record blocked_config and alert once (`preflight_alerted` bit). Must run after the wake gate so
- silent ticks stay silent. Opt-out: `cron.preflight: false`. Returns failure tuple or None.
- """
- # --------------------------------------------------------------- Pre-dispatch configuration validation
- # (T1-26). A job whose configuration cannot possibly produce a successful run — missing provider API key
- # (no fallback chain), unready attached skill, unconfigured delivery platform — is refused HERE, before
- # AIAgent is constructed and before the resolution below can feed a doomed runtime into it, so a
- # misconfigured job never burns an LLM call. run_one_job keys off the BLOCKED_CONFIG_MARKER in the
- # returned error to record last_status='blocked_config' and alert exactly once (dedup persisted via the
- # job's `preflight_alerted` bit — the #73506 alert-once shape).
- _pf_reason = None
- try:
- if _cron_preflight_enabled(cfg):
- _pf_reason = _preflight_job_config(job, cfg)
- if not _pf_reason and job.get("preflight_alerted"):
- # Config healthy again: clear alert-once marker so a future break re-alerts.
- with contextlib.suppress(Exception):
- from cron.jobs import clear_preflight_alerted
- clear_preflight_alerted(job_id)
- except Exception:
- # Fail open: the validator must never take down a runnable job.
- logger.debug("Job '%s': preflight validation errored — failing open", job_id, exc_info=True)
- _pf_reason = None
- if not _pf_reason:
- return None
- return _blocked_config_result(job_id, job_name, _pf_reason)
- def _blocked_config_result(job_id: str, job_name: str, _pf_reason: str) -> tuple:
- """The ``blocked_config`` failure tuple for *_pf_reason*, alerting once per job."""
- logger.warning(
- "Job '%s' (ID: %s): BLOCKED by pre-dispatch config validation — %s (no LLM call was made)",
- job_name, job_id, _pf_reason)
- already_alerted = False
- try:
- from cron.jobs import mark_preflight_alerted
- already_alerted = mark_preflight_alerted(job_id)
- except Exception:
- logger.debug("Job '%s': could not persist preflight alert marker", job_id, exc_info=True)
- marker = BLOCKED_CONFIG_SILENT_MARKER if already_alerted else BLOCKED_CONFIG_MARKER
- blocked_doc = (
- f"# Cron Job: {job_name}\n\n"
- f"**Job ID:** {job_id}\n"
- f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
- f"**Status:** BLOCKED (configuration)\n\n"
- "The pre-run configuration check found a problem, so the agent did not run "
- "(nothing was charged).\n\n"
- f"**Reason:** {_pf_reason}\n\n"
- "Hermes tries again at the next scheduled time and clears this state on the first healthy "
- "run; this alert is not repeated. Check with `hermes cron doctor`. Set `cron.preflight: "
- "false` in config.yaml to disable this check."
- )
- return False, blocked_doc, "", f"{marker} {_pf_reason}"
- def _resolve_job_runtime(job: dict, job_id: str, jc: _CronJobConfig) -> tuple[dict, str]:
- """Resolve the runtime, walking the fallback chain on auth/transient-network errors. Returns
- ``(runtime, model)``; provider+model swap atomically (never swap only the provider while keeping
- a paid primary model). Provider precedence: per-job pin > cron.model_provider > persisted
- global config (None lets resolve_runtime_provider read it). A pinned job has no chain here
- (``_job_fallback_chain``): its resolve failure is the job's failure."""
- from hermes_cli.runtime_provider import (
- resolve_runtime_provider, format_runtime_provider_error)
- from hermes_cli.auth import AuthError
- model = jc.model
- requested = job.get("provider") or jc.cron_default_provider or None
- try:
- # Do NOT pass HERMES_INFERENCE_PROVIDER as `requested`: it would override persisted config
- # and resurrect stale providers for unpinned jobs.
- runtime_kwargs = {
- "requested": requested,
- # api_mode must derive from the model actually run, not the stale persisted default.
- "target_model": model,
- }
- if job.get("base_url"):
- runtime_kwargs["explicit_base_url"] = job.get("base_url")
- return resolve_runtime_provider(**runtime_kwargs), model
- except Exception as resolve_exc:
- # Walk the fallback chain on AuthError AND transient network/DNS failures (e.g. during
- # OAuth refresh); anything else re-raises.
- is_auth = isinstance(resolve_exc, AuthError)
- is_transient_net = _is_transient_provider_resolve_error(resolve_exc)
- if not (is_auth or is_transient_net):
- raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc
- chain = _job_fallback_chain(job, jc.cfg) or []
- logger.warning(
- "Job '%s': primary provider resolve failed (%s: %s), %s",
- job_id, "auth" if is_auth else "transient network", resolve_exc,
- "trying fallback" if chain else (
- "not falling back: the job is pinned" if _job_route_pinned(job) else "no fallback configured"))
- for entry in chain:
- if not isinstance(entry, dict):
- continue
- fb_provider = str(entry.get("provider") or "").strip()
- fb_model = str(entry.get("model") or "").strip()
- if not fb_provider or not fb_model:
- continue
- try:
- from hermes_cli.fallback_config import effective_runtime_provider, resolve_entry_api_key
- fb_kwargs = {"requested": fb_provider, "target_model": fb_model}
- if entry.get("base_url"):
- fb_kwargs["explicit_base_url"] = entry["base_url"]
- fb_api_key = resolve_entry_api_key(entry)
- if fb_api_key:
- fb_kwargs["explicit_api_key"] = fb_api_key
- runtime = resolve_runtime_provider(**fb_kwargs)
- # Named custom entries resolve to the bare "custom" billing class; keep the configured
- # identity so job sessions record the provider name (#98739).
- runtime["provider"] = effective_runtime_provider(entry, runtime)
- logger.info(
- "Job '%s': fallback resolved to %s model %s",
- job_id, runtime.get("provider"), fb_model)
- # Delivered with the job output (#74349): a cron agent has no status rail, so the
- # switch would otherwise stay in the scheduler log only. run_job pops it.
- from hermes_cli.fallback_config import pre_agent_fallback_notice
- runtime["_fallback_notice"] = pre_agent_fallback_notice(
- requested or (jc.model_cfg.get("provider") if isinstance(jc.model_cfg, dict) else ""),
- model, runtime.get("provider"), fb_model)
- return runtime, fb_model
- except Exception as fb_exc:
- logger.debug("Job '%s': fallback %s failed: %s", job_id, fb_provider, fb_exc)
- raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc
- def _load_credential_pool(runtime: dict, job_id: str):
- runtime_provider = str(runtime.get("provider") or "").strip().lower()
- if not runtime_provider:
- return None
- try:
- from agent.credential_pool import load_pool
- pool = load_pool(runtime_provider)
- if pool.has_credentials():
- logger.info(
- "Job '%s': loaded credential pool for provider %s with %d entries",
- job_id, runtime_provider, len(pool.entries()))
- return pool
- except Exception as e:
- logger.debug("Job '%s': failed to load credential pool for %s: %s", job_id, runtime_provider, e)
- return None
- def _init_cron_mcp_tools(job_id: str) -> None:
- """Register MCP servers for the agent's tool registry. Idempotent across ticks; non-fatal so a
- broken MCP server never kills a working job."""
- try:
- # Initialize MCP servers so configured mcp_servers are available to the agent's tool registry before
- # AIAgent is constructed. Without this, cron jobs never saw any MCP tools — only the gateway / CLI
- # paths called discover_mcp_tools() at startup. Idempotent: subsequent ticks short-circuit on
- # already-connected servers inside register_mcp_servers(). Non-fatal on failure: a broken MCP server
- # shouldn't kill an otherwise-working cron job. See #4219.
- from tools.mcp_tool_discovery import discover_mcp_tools
- _mcp_tools = discover_mcp_tools()
- if _mcp_tools:
- logger.info("Job '%s': %d MCP tool(s) available", job_id, len(_mcp_tools))
- except Exception as _mcp_exc:
- logger.warning("Job '%s': MCP initialization failed (non-fatal): %s", job_id, _mcp_exc)
- def _open_cron_session_db(job: dict):
- """Bound store acquisition separately from the conversation watchdog.
- Canonical runs borrow the owner's exact store and fail closed on acquisition
- errors. Only the legacy non-owner helper path may return no session store.
- """
- # Initialize the SQLite session store so cron job messages are persisted and discoverable via
- # session_search (same pattern as gateway/run.py) — only now, after every early-return path (wake-gate,
- # prompt validation, drift skip) has passed, so a gated run never opens state.db just to abandon the
- # handle (#96290). Bounded with its own timeout (separate from HERMES_CRON_TIMEOUT, which only watches
- # the agent's run_conversation below): SessionDB.__init__ opens/migrates state.db synchronously and has
- # no timeout of its own against a wedged sqlite3.connect (e.g. a stale flock left by a crashed sibling
- # process). An unbounded hang here would wedge the job's worker thread, so the init is bounded and a
- # timeout proceeds without a session store instead of blocking the run forever.
- from gateway.session_cron import current_execution
- owner = current_execution()
- from agent.runtime_session_store import WorkerPersistenceError, is_worker_process
- if owner is None and is_worker_process():
- # Do not turn an unsupported worker assignment into the legacy None
- # fallback: that would run billed inference without durable persistence.
- raise WorkerPersistenceError('worker_cron_registration_required')
- _session_db_timeout = _get_session_db_timeout()
- try:
- from hermes_state_registry import acquire
- def acquire_store():
- if owner is None:
- return acquire()
- db = acquire(Path(owner[0].db.db_path))
- if db is not owner[0].db:
- from hermes_state_registry import release_or_close
- release_or_close(db)
- raise RuntimeError('cron requires the canonical owner store')
- return db
- if _session_db_timeout <= 0:
- return acquire_store()
- _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
- # Copy the context so a profile run resolves ITS OWN home/state.db on the worker thread
- # instead of the process-global default.
- _session_db_context = contextvars.copy_context()
- _session_db_future = _session_db_pool.submit(_session_db_context.run, acquire_store)
- try:
- return _session_db_future.result(timeout=_session_db_timeout)
- except concurrent.futures.TimeoutError:
- # The abandoned worker may still finish; close its late result or its SQLite FDs leak.
- # The worker is abandoned (shutdown below doesn't wait for it). If SessionDB() later completes
- # inside it, the future's result would be orphaned and its SQLite FDs (.db, WAL, SHM) leak until
- # process exit. Register a done-callback that retrieves and closes any eventual late result
- # (#72782).
- from cron.scheduler_detached_worker import _close_late_session_db_result
- _session_db_future.add_done_callback(_close_late_session_db_result)
- raise
- finally:
- # Abandon a wedged connect() rather than blocking shutdown on it.
- _session_db_pool.shutdown(wait=False)
- except concurrent.futures.TimeoutError:
- if owner is not None:
- raise TimeoutError("Canonical cron session store acquisition timed out")
- logger.error(
- "Job '%s': SessionDB init did not return within %.0fs — proceeding "
- "without a session store for this run instead of blocking it forever",
- job.get("id", "?"), _session_db_timeout)
- except Exception as e:
- if owner is not None:
- raise
- logger.debug("Job '%s': SQLite session store not available: %s", job.get("id", "?"), e)
- return None
- def _raise_inactivity_timeout(agent, job_name: str, limit_s: float) -> None:
- """Log the agent's last activity, hard-interrupt it and raise TimeoutError."""
- _activity = {}
- if hasattr(agent, "get_activity_summary"):
- with contextlib.suppress(Exception):
- _activity = agent.get_activity_summary()
- _last_desc = _activity.get("last_activity_desc", "unknown")
- _secs_ago = _activity.get("seconds_since_activity", 0)
- logger.error(
- "Job '%s' idle for %.0fs (inactivity limit %.0fs) "
- "| last_activity=%s | iteration=%s/%s | tool=%s",
- job_name, _secs_ago, limit_s,
- _last_desc, _activity.get("api_call_count", 0), _activity.get("max_iterations", 0),
- _activity.get("current_tool") or "none")
- request_hard_interrupt(agent, "Cron job timed out (inactivity)", tool_reason="cron inactivity watchdog")
- raise TimeoutError(
- f"Cron job '{job_name}' idle for "
- f"{int(_secs_ago)}s (limit {int(limit_s)}s) "
- f"— last activity: {_last_desc}")
- def _run_agent_with_watchdog(
- agent, prompt: str, job: dict, job_id: str, job_name: str, task_id: str, cancel_event,
- worker_state: Optional[dict] = None,
- ) -> dict:
- """Run ``agent.run_conversation`` on a worker thread under the inactivity (not wall-clock)
- watchdog: default 600s, override HERMES_CRON_TIMEOUT, 0 = unlimited."""
- _cron_timeout = _cron_inactivity_seconds()
- _cron_inactivity_limit = _cron_timeout if _cron_timeout > 0 else None
- _POLL_INTERVAL = 5.0
- # Heartbeat the one-shot run_claim while alive: without it a long run looks like a dead owner
- # and gets re-dispatched / stale-removed out from under the live run.
- # Keep the one-shot run_claim fresh while the run is alive (#62002): the claim TTL is a dead-owner
- # detector, but without a heartbeat a run that legitimately outlives it (stream stall, laptop asleep
- # mid-run) is indistinguishable from a dead tick — another process re-dispatches it and get_due_jobs
- # stale-removes the job record out from under the live run. Refreshing the claim from this monitor keeps
- # "expired claim" meaning "owner died".
- _job_schedule = job.get("schedule")
- _is_oneshot = isinstance(_job_schedule, dict) and _job_schedule.get("kind") == "once"
- _run_claim = job.get("run_claim")
- _run_claim_owner = str(_run_claim.get("by") or "") if isinstance(_run_claim, dict) else ""
- _last_claim_heartbeat = time.monotonic()
- def _abort_if_fire_claim_lost() -> None:
- if cancel_event is None or not cancel_event.is_set():
- return
- reason = cancel_reason(cancel_event)
- if agent is not None and hasattr(agent, "interrupt"):
- agent.interrupt(f"Cron run stopped: {reason}")
- raise RuntimeError(f"Cron job '{job_name}' was stopped: {reason}")
- def _heartbeat_run_claim_if_due():
- nonlocal _last_claim_heartbeat
- if not _is_oneshot or not _run_claim_owner:
- return
- _mono = time.monotonic()
- if _mono - _last_claim_heartbeat < _RUN_CLAIM_HEARTBEAT_SECONDS:
- return
- _last_claim_heartbeat = _mono
- try:
- heartbeat_run_claim(job_id, expected_owner=_run_claim_owner)
- except Exception:
- logger.debug("Job '%s': run_claim heartbeat failed", job_name, exc_info=True)
- # Establish suspend accounting before the worker can stamp its first activity.
- _awake_idle = AwakeIdleMeter()
- _cron_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
- # Carry scheduler-scoped ContextVar state (e.g. env passthrough) into the worker thread.
- _cron_context = contextvars.copy_context()
- _cron_future = _cron_pool.submit(
- _cron_context.run, agent.run_conversation, prompt, task_id=task_id)
- if worker_state is not None:
- worker_state["future"] = _cron_future
- _inactivity_timeout = False
- _watch_stop = threading.Event()
- def _idle_seconds() -> float:
- if not hasattr(agent, "get_activity_summary"):
- return 0.0
- try:
- _act = agent.get_activity_summary()
- return float(_act.get("seconds_since_activity", 0.0) or 0.0)
- except Exception:
- return 0.0
- def _watch_inactivity() -> None:
- nonlocal _inactivity_timeout
- if _cron_inactivity_limit is None:
- return
- if _inactivity_watchdog_loop(
- get_idle_seconds=_idle_seconds, limit_s=_cron_inactivity_limit, poll_s=_POLL_INTERVAL,
- stop=_watch_stop, future_done=_cron_future.done, meter=_awake_idle):
- _inactivity_timeout = True
- _watch_thread = threading.Thread(
- target=_watch_inactivity, name=f"cron-inactivity-{str(job_id)[:8]}", daemon=True)
- # Ledger progress stamps: the stale-claim sweep measures silence since the last stamp.
- _progress = ExecutionProgressStamper(
- str(job.get("execution_id") or ""), job_name, idle_seconds=_idle_seconds,
- every_seconds=_RUN_CLAIM_HEARTBEAT_SECONDS)
- try:
- if _cron_inactivity_limit is not None:
- # Separate daemon thread so a hung get_activity_summary can't stop the limit firing.
- # Daemon thread: kernel ``Event.wait`` timeout, independent of the ``run_job`` thread. A blocked
- # loop / hung ``get_activity_summary`` on this thread can no longer keep the 600s inactivity
- # limit from firing (#94285).
- _watch_thread.start()
- if _cron_inactivity_limit is None and not _is_oneshot and cancel_event is None:
- result = _cron_future.result()
- else:
- result = None
- while True:
- done, _ = concurrent.futures.wait({_cron_future}, timeout=_POLL_INTERVAL)
- if done:
- _abort_if_fire_claim_lost()
- result = _cron_future.result()
- break
- if _inactivity_timeout:
- break
- _abort_if_fire_claim_lost()
- _heartbeat_run_claim_if_due()
- _progress.tick()
- except Exception:
- _cron_pool.shutdown(wait=False, cancel_futures=True)
- raise
- finally:
- _watch_stop.set()
- _cron_pool.shutdown(wait=False, cancel_futures=True)
- if _inactivity_timeout:
- _raise_inactivity_timeout(agent, job_name, _cron_inactivity_limit)
- if not isinstance(result, dict):
- raise RuntimeError(
- f"agent.run_conversation returned {type(result).__name__} instead of dict: {result!r}"
- )
- return result
- def _final_response_from_result(result: dict, job_id: str, job_name: str, AIAgent) -> str:
- """Deliverable final response from a ``run_conversation`` result. Raises RuntimeError on
- `failed=True`/`completed=False`: the error text may sit in `final_response` and would otherwise
- be delivered as the reply with the job marked ok."""
- # If the agent itself reported failure (e.g. all retries exhausted on API errors, model abort, mid-run
- # interrupt), do not silently mark the job as successful. run_agent populates
- # `failed=True`/`completed=False` on these paths and may put the error into `final_response`, which
- # would otherwise be delivered as if it were the agent's reply and the job's `last_status` set to "ok".
- # Raise so the except handler below builds the proper failure tuple. (issue #17855)
- turn_exit_reason = str(result.get("turn_exit_reason") or "")
- final_response_text = (result.get("final_response") or "").strip()
- max_iteration_summary = is_max_iteration_handoff(result)
- if result.get("failed") is True or (result.get("completed") is False and not max_iteration_summary):
- raise RuntimeError(result.get("error") or final_response_text or "agent reported failure")
- if max_iteration_summary:
- logger.warning(
- "Job '%s' reached the iteration limit but produced a final fallback response; "
- "delivering the response instead of failing the cron run",
- job_name)
- final_response = result.get("final_response", "") or ""
- # Repair model-mangled computer_use media paths before delivery (fail-open, as in gateway).
- if final_response:
- from gateway.media_repair import repair_explicit_computer_use_media_paths
- final_response = repair_explicit_computer_use_media_paths(
- final_response, result.get("messages", []))
- if final_response.strip() == "(No response generated)":
- final_response = ""
- # The "⚠️ No reply" turn-completion explainer would be delivered as a cron warning; detect it
- # via the same formatter and treat as empty so cron stays silent on abnormal empty turns.
- if final_response.strip() and turn_exit_reason:
- # Render every persistence-cause variant or cause-refined text slips through.
- _explainer_variants = []
- try:
- from hermes_state_errors import PERSISTENCE_ERROR_CAUSES as _causes
- except Exception:
- _causes = ("locked", "disk", "unknown")
- # The finalizer fills the model name into the explainer; render with the same name (and
- # the bare form) or the comparison below misses and the warning is delivered.
- _model = str(result.get("model") or "")
- for _cause in (None, *_causes):
- for _kwargs in ({"model": _model}, {}):
- try:
- _variant = AIAgent._format_turn_completion_explanation(turn_exit_reason, _cause, **_kwargs)
- except TypeError:
- try:
- _variant = AIAgent._format_turn_completion_explanation(turn_exit_reason)
- except Exception:
- _variant = ""
- except Exception:
- _variant = ""
- if _variant:
- _explainer_variants.append(_variant.strip())
- if final_response.strip() in _explainer_variants:
- logger.info(
- "Job '%s': abnormal empty turn (%s) — suppressing explainer for cron delivery",
- job_id, turn_exit_reason)
- final_response = ""
- return final_response
- def _finalize_cron_session(session_db, agent, job_id: str, job_name: str, cron_session_id: str,
- workdir: Optional[str] = None) -> None:
- """Title, classify, end and release the cron session after the agent turn has returned."""
- # Bound every DB op so storage failure cannot hold the dispatch guard.
- _session_db = _BoundedCronSessionDB(session_db, job_id)
- # Compression may have rotated the run onto a continuation: finalize that, not the stale cron
- # id. SessionDB lineage is authoritative; agent.session_id is only a fail-safe.
- _final_cron_session_id = cron_session_id
- try:
- _compression_tip = _session_db.get_compression_tip(cron_session_id)
- if _compression_tip:
- _final_cron_session_id = _compression_tip
- except (Exception, KeyboardInterrupt) as e:
- with contextlib.suppress((Exception, KeyboardInterrupt)):
- _agent_session_id = getattr(agent, "session_id", None)
- # CLI (single-process) path: the approval contextvar is only bound during gateway/TUI turns and
- # HERMES_SESSION_KEY is not in the CLI environment, so the key resolves empty here. Since #64240
- # the CLI drains completions through a positive-ownership filter keyed on the durable
- # AIAgent.session_id — an empty session_key would fail closed and the CLI could never claim its
- # own completions, while a restored foreign event with an empty key could leak into any
- # unfiltered consumer (#64484). Stamp the parent's durable session id instead; compression
- # rotations are handled on the drain side via resolve_resume_session_id lineage resolution.
- if _agent_session_id:
- _final_cron_session_id = _agent_session_id
- logger.debug("Job '%s': failed to resolve cron compression tip: %s", job_id, e)
- # Title must persist BEFORE end_session()/close(). Run-time suffix keeps it unique against the
- # sessions.title index; the fallbacks below guarantee a non-blank title.
- try:
- # Title the cron session from the job (name -> id) and PERSIST it BEFORE end_session()/close() tear
- # the connection down, so the close can never run over an in-flight title write (#50536).
- _title_base = " ".join(job_name.split())[:60].strip() or f"cron {job_id}"
- _cron_title = f"{_title_base} · {safe_strftime(_hermes_now(), '%b %d %H:%M')}"
- if not _set_cron_session_title(_session_db, _final_cron_session_id, _cron_title):
- _set_cron_session_title(_session_db, _final_cron_session_id, f"cron {job_id}")
- except (Exception, KeyboardInterrupt) as e:
- logger.debug("Job '%s': failed to set cron session title: %s", job_id, e)
- # Never leave the session untitled.
- # Try the next free title in the lineage, then a bare id-stamped title. See #50535.
- for _fallback in (
- getattr(_session_db, "get_next_title_in_lineage", lambda b: b)(f"cron {job_id}"),
- f"cron {job_id} {_final_cron_session_id[-6:]}"):
- try:
- if _set_cron_session_title(_session_db, _final_cron_session_id, _fallback):
- break
- except (Exception, KeyboardInterrupt):
- continue
- # Book cron_complete only when the last row is a real assistant reply ([SILENT] counts). Only a
- # POSITIVELY recognized bad status downgrades (keep tuple in sync with
- # session_lifecycle_statuses); unknown values / probe failures fail OPEN.
- # Verified completion booking (#93820): the run may only be recorded as cron_complete when the session's
- # LAST message row is a real assistant reply — a plain answer or the [SILENT] sentinel (both are
- # assistant-text rows, so both classify as 'complete'). A turn that died after a tool call,
- # mid-API-wait, or without any assistant text leaves the last row as a tool result / pending call / user
- # prompt and must not surface as a healthy run. session_lifecycle_statuses is the existing cost-bounded
- # classifier for exactly this shape. Only a POSITIVELY recognized pathological status (see the status
- # vocabulary in hermes_state's session_lifecycle_statuses docstring — keep the tuple below in sync when
- # it grows) downgrades the booking: an unknown value (newer classifier shape, test doubles) keeps the
- # historical reason, and so does a failed probe — the booking itself is FAIL-OPEN on probe errors,
- # because classification is best-effort metadata and must not mislabel a healthy run.
- _end_reason = "cron_complete"
- try:
- _statuses = _session_db.session_lifecycle_statuses([_final_cron_session_id])
- _lifecycle = _statuses.get(_final_cron_session_id)
- if _lifecycle in ("interrupted", "error", "empty"):
- _end_reason = "cron_incomplete_no_output"
- logger.warning(
- "Job '%s': session ended without a final assistant "
- "message (lifecycle=%s) — booking run as %s",
- job_id, _lifecycle, _end_reason)
- except (Exception, KeyboardInterrupt) as e:
- logger.debug("Job '%s': session lifecycle classification failed: %s", job_id, e)
- # Stamp the job's workdir on the row BEFORE end_session (title-write ordering, #50536): the
- # sidebar groups by cwd prefix and nothing else writes a cron row's cwd (#108205).
- if workdir:
- try:
- _session_db.update_session_cwd(_final_cron_session_id, workdir)
- except (Exception, KeyboardInterrupt) as e:
- logger.debug("Job '%s': failed to stamp workdir on session row: %s", job_id, e, exc_info=True)
- try:
- _session_db.end_session(_final_cron_session_id, _end_reason)
- # The scheduler owns cron-session finalization. AIAgent.close() also
- # finalizes owned session rows by default; once the shared SessionDB is
- # released below, that second end_session() would reopen the just-closed
- # SQLite handle (#94736). The reason is durably booked, so disarm only the
- # agent's redundant row-finalization; its resource teardown still runs in
- # _teardown_cron_agent.
- if agent is not None:
- agent._end_session_on_close = False
- except (Exception, KeyboardInterrupt) as e:
- logger.debug("Job '%s': failed to end session: %s", job_id, e)
- try:
- from hermes_state_registry import release_or_close
- release_or_close(_session_db)
- except (Exception, KeyboardInterrupt) as e:
- logger.debug("Job '%s': failed to close SQLite session store: %s", job_id, e)
- def _normalize_newlines(text: str) -> str:
- """CRLF/CR -> LF, matching what text-mode reads of the archive return."""
- return text.replace("\r\n", "\n").replace("\r", "\n")
- def _run_doc_header(job: dict, title: str, job_id: str, prompt: str, *, prompt_stamp: str = "") -> str:
- """Header of the persisted run document (title, ids, schedule, prompt)."""
- return (
- f"# Cron Job: {title}\n\n"
- f"**Job ID:** {job_id}\n"
- f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
- f"**Schedule:** {job.get('schedule_display', 'N/A')}\n\n"
- f"{prompt_stamp}{_PROMPT_HEADING}{prompt}{_PROMPT_SEPARATOR}"
- )
- _RunResult = tuple[bool, str, str, Optional[str]]
- def _prepare_job_prompt(
- job: dict, job_id: str, job_name: str, extra_prompt: Optional[str], cancel_event,
- ) -> tuple[Optional[_RunResult], Optional[str]]:
- """Run every pre-agent gate and build the prompt. Returns ``(early_result, prompt)``: an early
- result short-circuits ``run_job`` (no_agent job, empty payload, monitor gate, wake gate,
- injection block, empty prompt); otherwise ``prompt`` is set."""
- # Fail closed on a corrupt config.yaml: defaults would let auto-detection bill a provider the
- # user never chose. no_agent jobs are exempt. Escape hatch: HERMES_IGNORE_USER_CONFIG=1.
- if not job.get("no_agent"):
- from hermes_cli.config import InvalidUserConfigError, require_parseable_user_config
- try:
- require_parseable_user_config()
- except InvalidUserConfigError as exc:
- logger.error("Job '%s': refusing to run — %s", job_id, exc)
- return (False, f"# Cron Job: {job_name}\n\nError: {exc}\n", "", str(exc)), None
- # no_agent short-circuits BEFORE importing run_agent / opening SessionDB.
- if job.get("no_agent"):
- return _run_no_agent_job(job, job_id, job_name, cancel_event), None
- # Legacy / hand-edited job with nothing to run: pause it instead of waking the LLM every fire.
- from cron.jobs import EMPTY_PAYLOAD_ERROR, job_payload_is_empty
- if job_payload_is_empty(job):
- return _block_and_pause_job(job_id, job_name, EMPTY_PAYLOAD_ERROR), None
- _early, extra_prompt, monitor_context = _apply_monitor_gate(job, job_id, job_name, extra_prompt)
- if _early is not None:
- return _early, None
- # Wake-gate: run the pre-check script BEFORE building the prompt; its result is passed into
- # _build_job_prompt so the script runs only once.
- # NOTE: the SQLite session store used to be initialized here, BEFORE the wake-gate and prompt-validation
- # early returns below. Every gated run (``wakeAgent: false``, blocked prompt) opened state.db and
- # returned without reaching the finally that closes it, relying on GC to release the handle. Init now
- # happens inside the main try, right before the agent is constructed — after every early-return path
- # (#96290).
- prerun_script = None
- script_path = job.get("script")
- if script_path:
- prerun_script = _run_job_script_with_claim_heartbeat(
- job,
- script_path,
- workdir=_resolve_job_workdir(job, job_id),
- cancel_event=cancel_event,
- )
- _ran_ok, _script_output = prerun_script
- if _ran_ok and not _parse_wake_gate(_script_output):
- logger.info("Job '%s' (ID: %s): wakeAgent=false, skipping agent run", job_name, job_id)
- note_cron_skipped(job)
- silent_doc = (
- f"# Cron Job: {job_name}\n\n"
- f"**Job ID:** {job_id}\n"
- f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n\n"
- "Script gate returned `wakeAgent=false` — agent skipped.\n"
- )
- return (True, silent_doc, SILENT_MARKER, None), None
- try:
- prompt = _build_job_prompt(
- job, prerun_script=prerun_script, extra_prompt=extra_prompt,
- runtime_data_prompt=monitor_context,
- )
- except CronPromptInjectionBlocked as block_exc:
- # Injection scanner tripped: refuse this tick and tell the operator WHY.
- logger.warning(
- "Job '%s' (ID: %s): blocked by prompt-injection scanner — %s", job_name, job_id, block_exc,
- )
- blocked_doc = (
- f"# Cron Job: {job_name}\n\n"
- f"**Job ID:** {job_id}\n"
- f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
- f"**Status:** BLOCKED\n\n"
- "The assembled prompt (user prompt + loaded skill content) tripped "
- "the cron injection scanner and the agent was NOT run.\n\n"
- f"**Scanner result:** {block_exc}\n\n"
- "Audit the skill(s) attached to this job for prompt-injection "
- "payloads or invisible-unicode markers. If the skill is legitimate "
- "and the match is a false positive, rephrase the content to avoid "
- "the threat pattern (`tools/cronjob_tools.py::_CRON_THREAT_PATTERNS`)."
- )
- return (False, blocked_doc, "", str(block_exc)), None
- if prompt is None:
- logger.info("Job '%s': script produced no output, skipping AI call.", job_name)
- note_cron_skipped(job)
- return (True, "", SILENT_MARKER, None), None
- return None, prompt
- _CRON_DELIVERY_VARS = (
- "HERMES_CRON_AUTO_DELIVER_PLATFORM",
- "HERMES_CRON_AUTO_DELIVER_CHAT_ID",
- "HERMES_CRON_AUTO_DELIVER_THREAD_ID")
- class _CronRunScope:
- """Per-run ContextVar / tool-cwd scope for ``run_job`` (ContextVars, not os.environ, so
- parallel jobs don't clobber each other). Construct before the try, ``enter()`` as its first
- statement, ``exit()`` in the finally — every setter here has a matching reset there.
- HERMES_SESSION_* are deliberately NOT seeded from job["origin"]: it is delivery metadata, not
- a sender, and terminal/tts/skills/send_message tools would act as if the origin user were
- driving the agent. Delivery reads job["origin"] / HERMES_CRON_AUTO_DELIVER_* directly.
- """
- def __init__(self, job: dict, job_id: str, execution_id: Optional[str]):
- from gateway.session_context import set_session_vars, _VAR_MAP
- from tools.terminal_tool import record_session_cwd
- self._var_map = _VAR_MAP
- # Resolve workdir BEFORE set_session_vars so it owns the _SESSION_CWD set/clear.
- self.workdir = _resolve_job_workdir(job, job_id)
- self._ctx_tokens = set_session_vars(
- platform="",
- chat_id="",
- chat_name="",
- # Cron can't receive completions after its turn; async delegation output could
- # otherwise route to an unrelated chat via the ambient session key => inline delegation.
- # We clear the HERMES_SESSION_* routing keys just below, so an async delegation's completion
- # event carries session_key="" — _enrich_async_delegation_routing cannot resolve it and
- # _inject_watch_notification drops it ("no routing metadata"). And by the time a child finishes,
- # run_job has already shipped the job's final response via _deliver_result; there is no turn
- # left to re-enter. (Worse, get_current_session_key() can fall back to the ambient os.environ
- # HERMES_SESSION_KEY, which risks routing a cron subagent's output into an unrelated user chat.)
- # Declaring the channel stateless routes delegate_task to its existing inline/synchronous path,
- # so results return within the job's own turn. See declare_stateless_channel(). Upstream:
- # #53027, #63142.
- async_delivery=False,
- cwd=self.workdir or "",
- )
- for name in _CRON_DELIVERY_VARS:
- _VAR_MAP[name].set("")
- # Workdir binds to the per-run task id (tool-layer cwd authority) instead of mutating
- # global TERMINAL_CWD; _SESSION_CWD above remains the prompt/context-file authority.
- self.task_id = f"cron:{job_id}:{execution_id or job.get('execution_id') or uuid.uuid4().hex}"
- if self.workdir:
- record_session_cwd(self.task_id, self.workdir)
- self._cron_session_var = _VAR_MAP["HERMES_CRON_SESSION"]
- self._cron_session_token = None
- self._non_dispatcher_token = None
- self._spend_token = None
- def enter(self) -> None:
- # Scope cron approval policy; exit() RESETS via token (pinning "" would suppress the legacy
- # os.environ fallback used by standalone entrypoints/tests).
- self._cron_session_token = self._cron_session_var.set("1")
- # Mark NOT the kanban worker: a worker's cronjob(action="run") lands here with
- # HERMES_KANBAN_TASK in env, and an unrelated job could close the worker's task. Must be a
- # ContextVar, NOT an os.environ clear (env is shared with the worker heartbeat and
- # concurrent jobs); copy_context() carries it into the agent thread.
- self._non_dispatcher_token = enter_non_dispatcher_owned_context()
- # Each run is its own MCP spend "chat" (its approval session key is "" for every job).
- from tools.mcp_tool_costs import bind_run
- self._spend_token = bind_run(self.task_id)
- def exit(self) -> None:
- from gateway.session_context import clear_session_vars
- from tools.terminal_tool import clear_session_cwd
- clear_session_cwd(self.task_id)
- clear_session_vars(self._ctx_tokens) # also clears _SESSION_CWD
- if self._cron_session_token is not None:
- self._cron_session_var.reset(self._cron_session_token)
- if self._non_dispatcher_token is not None:
- exit_non_dispatcher_owned_context(self._non_dispatcher_token)
- if self._spend_token is not None:
- from tools.mcp_tool_costs import unbind_run
- unbind_run(self._spend_token)
- for name in _CRON_DELIVERY_VARS:
- self._var_map[name].set("")
- def _reload_dotenv_and_publish_delivery_target(job: dict) -> None:
- """Re-read .env for this run and publish the auto-deliver target into the session ContextVars."""
- # Reset the secret-source cache FIRST or a Bitwarden/BSM-backed secret is never re-resolved
- # (only the placeholder reloads -> 401s).
- from hermes_cli.env_loader import load_hermes_dotenv, reset_secret_source_cache
- from gateway.session_context import _VAR_MAP
- reset_secret_source_cache(_get_hermes_home())
- load_hermes_dotenv(hermes_home=_get_hermes_home())
- delivery_target = _resolve_delivery_target(job)
- if delivery_target:
- _VAR_MAP["HERMES_CRON_AUTO_DELIVER_PLATFORM"].set(delivery_target["platform"])
- _VAR_MAP["HERMES_CRON_AUTO_DELIVER_CHAT_ID"].set(str(delivery_target["chat_id"]))
- _VAR_MAP["HERMES_CRON_AUTO_DELIVER_THREAD_ID"].set(
- "" if delivery_target.get("thread_id") is None else str(delivery_target["thread_id"])
- )
- @dataclass
- class _CronAgentSetup:
- """Everything ``AIAgent(...)`` needs that is resolved from job + config (or a preflight block)."""
- blocked: Optional[_RunResult] = None
- model: str = ""
- runtime: dict = None
- prefill_messages: Any = None
- max_iterations: Any = None
- reasoning_config: Any = None
- fallback_model: Any = None
- credential_pool: Any = None
- fallback_notice: Optional[str] = None
- def _resolve_cron_agent_setup(job: dict, job_id: str, job_name: str, jc) -> _CronAgentSetup:
- """Resolve model/runtime/reasoning/pool for the run, in the original gate order: exfil guard ->
- preflight (may block) -> runtime (+ fallback chain) -> credential pool -> MCP."""
- _cfg = jc.cfg
- setup = _CronAgentSetup(model=jc.model)
- setup.prefill_messages = _load_prefill_messages(_cfg, job_id)
- # resolve_turn_limit() honors none/unlimited (sys.maxsize) and explicit 0 / null.
- from hermes_cli.config import resolve_turn_limit as _resolve_turn_limit
- _mt = _cfg.get("agent", {}).get("max_turns")
- if _mt is None:
- _mt = _cfg.get("max_turns")
- setup.max_iterations = _resolve_turn_limit(_mt)
- # Runtime backstop (CWE-200/522): fail closed BEFORE resolution on a provider/base_url pair
- # that would ship a stored credential off-host; hand-written jobs bypass create-time checks.
- _guard_job_credential_exfil(job)
- setup.blocked = _preflight_or_block(job, job_id, job_name, _cfg)
- if setup.blocked is not None:
- return setup
- setup.runtime, setup.model = _resolve_job_runtime(job, job_id, jc)
- setup.fallback_notice = setup.runtime.pop("_fallback_notice", None)
- setup.reasoning_config = _resolve_job_reasoning_config(
- job, _cfg if isinstance(_cfg, dict) else {}, str(setup.model)
- )
- # Mid-run provider ladder: same rule as resolution above, so a pinned job cannot be swapped
- # onto the global chain by a 5xx/429 either.
- setup.fallback_model = _job_fallback_chain(job, _cfg)
- setup.credential_pool = _load_credential_pool(setup.runtime, job_id)
- # MCP servers must be registered before AIAgent is constructed.
- _init_cron_mcp_tools(job_id)
- # Only now can a requested MCP toolset be judged: its alias is process-global but its tools live
- # in this profile's registry overlay, and quiet_mode hides the empty resolution (#109050).
- if _cron_preflight_enabled(_cfg):
- _mcp_reason = _empty_requested_mcp_toolsets(job, _cfg)
- if _mcp_reason:
- setup.blocked = _blocked_config_result(job_id, job_name, _mcp_reason)
- return setup
- def _construct_cron_agent(AIAgent, job: dict, _cfg: dict, setup: _CronAgentSetup, *, workdir, session_id, session_db):
- runtime = setup.runtime
- pr = _cfg.get("provider_routing") or {}
- return AIAgent(
- model=setup.model,
- api_key=runtime.get("api_key"),
- base_url=runtime.get("base_url"),
- provider=runtime.get("provider"),
- requested_provider=runtime.get("requested_provider"),
- api_mode=runtime.get("api_mode"),
- request_overrides=runtime.get("request_overrides"),
- acp_command=runtime.get("command"),
- acp_args=runtime.get("args"),
- max_iterations=setup.max_iterations,
- reasoning_config=setup.reasoning_config,
- prefill_messages=setup.prefill_messages,
- fallback_model=setup.fallback_model,
- credential_pool=setup.credential_pool,
- providers_allowed=pr.get("only"),
- providers_ignored=pr.get("ignore"),
- providers_order=pr.get("order"),
- provider_sort=pr.get("sort"),
- openrouter_min_coding_score=(_cfg.get("openrouter") or {}).get("min_coding_score"),
- enabled_toolsets=_resolve_cron_enabled_toolsets(job, _cfg),
- disabled_toolsets=_resolve_cron_disabled_toolsets(_cfg),
- quiet_mode=True,
- # Project context files only with a configured workdir; SOUL.md always.
- skip_context_files=not bool(workdir),
- load_soul_identity=True,
- skip_memory=False,
- skip_background_review=True, # Cron has no human-in-the-loop need for skill/memory review forks (~30K tok/event)
- platform="cron",
- session_id=session_id,
- session_db=session_db,
- )
- class _FireAudit:
- """One usage_audit.jsonl line per fire (created once the agent exists; fire id + start clock)."""
- def __init__(self, job: dict, job_id: str, model: str):
- self.job, self.job_id, self.model = job, job_id, model
- self.fire_id = uuid.uuid4().hex
- self.t_start = time.monotonic()
- def write(self, result: dict, error: Optional[str]) -> None:
- _write_usage_audit({
- "ts": _utcnow_iso_ms(),
- "job_id": self.job_id,
- "fire_id": self.fire_id,
- "prompt_tokens": result.get("prompt_tokens"),
- "completion_tokens": result.get("completion_tokens"),
- "total_tokens": result.get("total_tokens"),
- "response_silent": bool(result.get("response_silent")),
- "deliver_target": self.job.get("deliver"),
- "model": self.model or None,
- "duration_ms": int((time.monotonic() - self.t_start) * 1000),
- "error": error})
- def run_job(
- job: dict, *, defer_agent_teardown: Optional[list] = None, extra_prompt: Optional[str] = None,
- cancel_event: Optional[_CancelEventLike] = None, execution_id: Optional[str] = None,
- ) -> tuple[bool, str, str, Optional[str]]:
- """Execute a single cron job. Returns (success, full_output_doc, final_response, error).
- ``defer_agent_teardown``: if a list, the live agent is appended instead of torn down; the caller
- MUST call ``_teardown_cron_agent(agent)`` AFTER delivery (a torn-down async client can't
- deliver). ``extra_prompt``: per-fire context, never persisted.
- ``defer_agent_teardown``: when a caller passes a list, ``run_job`` skips the agent's async-resource
- teardown (``agent.close()`` + ``cleanup_stale_async_clients()``) in its ``finally`` block and instead
- appends the live agent to that list. The caller is then responsible for calling
- ``_teardown_cron_agent(agent)`` AFTER it has delivered the result. This closes the ordering window in
- #58720 where delivery ran against a torn-down async client (defense-in-depth alongside the
- interpreter-shutdown guard). When ``None`` (the default) teardown happens inline as before, so every
- existing caller is unchanged.
- ``extra_prompt``: optional per-run context from ``cronjob(action='run', prompt=...)`` (#57331). Appended
- to the stored prompt for this fire only — never persisted to the job definition.
- """
- from gateway.session_cron import current_execution
- owner_execution = current_execution()
- if owner_execution is not None and owner_execution[2:] != (job['id'], execution_id):
- owner_execution = None
- if not job.get("no_agent") and owner_execution is None:
- from cron.scheduler_authority import run_canonical_job
- return run_canonical_job(job, extra_prompt=extra_prompt, cancel_event=cancel_event,
- execution_id=execution_id)
- job_id = job["id"]
- job_name = str(job.get("name") or job.get("prompt") or job_id or "cron job")
- early, prompt = _prepare_job_prompt(job, job_id, job_name, extra_prompt, cancel_event)
- if early is not None:
- return early
- from run_agent import AIAgent
- _cron_session_id = (owner_execution[1] if owner_execution is not None else
- f"cron_{job_id}_{_hermes_now().strftime('%Y%m%d_%H%M%S')}")
- logger.info("Running job '%s' (ID: %s)", job_name, job_id)
- logger.info("Prompt: %s", prompt[:100])
- agent = None
- model = ""
- _session_db = None
- _audit: Optional[_FireAudit] = None
- _worker_state: dict = {}
- scope = _CronRunScope(job, job_id, execution_id)
- try:
- scope.enter()
- if scope.workdir:
- logger.info("Job '%s': using task-scoped workdir %s", job_id, scope.workdir)
- _reload_dotenv_and_publish_delivery_target(job)
- jc = _load_cron_job_config(job, job_id, job_name)
- _cfg = jc.cfg
- model = jc.model
- setup = _resolve_cron_agent_setup(job, job_id, job_name, jc)
- if setup.blocked is not None:
- return setup.blocked
- model = setup.model
- # Open state.db only after every early-return gate has passed.
- _session_db = _open_cron_session_db(job)
- agent = _construct_cron_agent(
- AIAgent, job, _cfg, setup, workdir=scope.workdir, session_id=_cron_session_id,
- session_db=_session_db)
- _audit = _FireAudit(job, job_id, model)
- result = _run_agent_with_watchdog(
- agent, prompt, job, job_id, job_name, scope.task_id, cancel_event,
- worker_state=_worker_state)
- final_response = _final_response_from_result(result, job_id, job_name, AIAgent)
- if (setup.fallback_notice and final_response.strip() and not _is_cron_silence_response(final_response)
- and _cron_failure_marker_error(final_response) is None):
- # Pre-agent provider switch (#74349) rides with the delivered report; silence and the
- # agent-declared failure marker keep their first-line/whole-response contract.
- final_response = f"{setup.fallback_notice}\n\n{final_response}"
- # Keep final_response clean for delivery logic (empty = no delivery).
- logged_response = final_response if final_response else "(No response generated)"
- # Text-file reads normalize newlines; count the same characters the
- # context_from reader sees so quoted markers can never become boundaries.
- framed_prompt = _normalize_newlines(prompt)
- logged_response = _normalize_newlines(logged_response)
- output = (
- _run_doc_header(job, job_name, job_id, framed_prompt,
- prompt_stamp=f"{_PROMPT_FRAME}{len(framed_prompt)}\n")
- + f"{_RESPONSE_FRAME}{len(logged_response)}\n{_RESPONSE_HEADING}{logged_response}{_RESPONSE_TERMINATOR}"
- )
- logger.info("Job '%s' completed successfully", job_name)
- _audit.write(dict(result, response_silent=_is_cron_silence_response(final_response or "")), None)
- return True, output, final_response, None
- except (Exception, ConfigBackendUnavailable) as e: # the plane being down fails THIS fire only
- error_msg = f"{type(e).__name__}: {e!s}"
- logger.exception("Job '%s' failed: %s", job_name, error_msg)
- # Cowork-style unreachable-model re-run (cron/unreachable_retry.py): flag failures where
- # the model was never reached (transient network/DNS, zero API calls) so the bookkeeping
- # tail can schedule a bounded automatic re-run instead of waiting a full period.
- try:
- from cron.unreachable_retry import is_model_unreachable_failure
- if is_model_unreachable_failure(e, agent):
- job["_model_unreachable"] = True
- # Provider usage window closed for a known duration (cron/quota_hold.py): flag it so the
- # bookkeeping tail parks the job past the window instead of re-firing into it (#89376).
- from cron.quota_hold import hold_seconds_from_failure
- _hold_s = hold_seconds_from_failure(e)
- if _hold_s:
- job["_quota_hold_seconds"] = _hold_s
- except Exception: # classification must never mask the real failure
- logger.debug("Job '%s': unreachable-failure classification failed", job_id)
- # No audit row when we failed before the agent existed; the audit write must never raise.
- if _audit is not None:
- _audit.write({}, error_msg)
- from cron.scheduler_diagnostics import format_run_error
- output = (
- _run_doc_header(job, f"{job_name} (FAILED)", job_id, prompt)
- + format_run_error(e)
- )
- return False, output, "", error_msg
- finally:
- from cron.scheduler_detached_worker import defer_teardown_to_running_worker
- _worker_teardown_deferred = defer_teardown_to_running_worker(
- _worker_state.get("future"), _session_db, agent, job_id, job_name, _cron_session_id,
- workdir=scope.workdir)
- scope.exit()
- if _session_db and not _worker_teardown_deferred:
- _finalize_cron_session(_session_db, agent, job_id, job_name, _cron_session_id,
- workdir=scope.workdir)
- # Tear down the ephemeral agent or the gateway leaks fds per tick (EMFILE). With deferred
- # teardown, hand the live agent back: delivery needs a live async client.
- # Release subprocesses, terminal sandboxes, browser daemons, and the main OpenAI/httpx client held
- # by this ephemeral cron agent. Without this, a gateway that ticks cron every N minutes leaks fds
- # per job until it hits EMFILE (#10200 / "too many open files"). When the caller opted to defer
- # teardown (passed a list), hand the live agent back instead of closing it here — delivery must run
- # against a live async client, and the caller tears down afterwards (#58720).
- if not _worker_teardown_deferred:
- if defer_agent_teardown is not None:
- if agent is not None:
- defer_agent_teardown.append(agent)
- else:
- _teardown_cron_agent(agent, job_id)
- def _teardown_cron_agent(
- agent, job_id: str, *, timeout_seconds: Optional[float] = None
- ) -> None:
- """Release an ephemeral cron agent's async resources within a hard bound (this runs outside the
- inactivity watchdog). Shared by ``run_job``'s finally and deferred post-delivery teardown.
- Split out of ``run_job``'s ``finally`` so a caller that defers teardown (to deliver first — #58720) can
- invoke the identical cleanup AFTER delivery. The timeout matters because this executes after
- ``run_conversation`` has returned, outside the agent inactivity watchdog.
- """
- def _cleanup_agent() -> None:
- try:
- if agent is not None:
- agent.close()
- except (Exception, KeyboardInterrupt) as e:
- logger.debug("Job '%s': failed to close agent resources: %s", job_id, e)
- # Worker-thread event loop dies with the executor; reap httpx clients cached under it.
- try:
- from agent.auxiliary_client import cleanup_stale_async_clients
- cleanup_stale_async_clients()
- except Exception as e:
- logger.debug("Job '%s': failed to reap stale auxiliary clients: %s", job_id, e)
- _run_cron_cleanup_with_timeout(
- _cleanup_agent, job_id=job_id, label="agent resource teardown",
- timeout_seconds=timeout_seconds)
- def _run_with_fire_claim_heartbeat(job: dict, run) -> bool:
- """Run ``run`` while keeping this job's owned durable fire claim fresh."""
- claim = job.get("fire_claim")
- owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
- if not owner:
- return run(None)
- job_id = str(job.get("id") or "")
- stop = threading.Event()
- lost_ownership = threading.Event()
- def _finish_unstarted(error: str) -> None:
- execution_id = job.get("execution_id")
- if not execution_id:
- return
- try:
- finish_lost_execution(execution_id, error)
- except Exception:
- logger.warning(
- "Job '%s': failed to close unstarted execution ledger row",
- job_id,
- exc_info=True)
- try:
- owns_fire_claim = heartbeat_fire_claim(job_id, expected_owner=owner)
- except Exception:
- logger.warning("Job '%s': initial fire_claim validation failed", job_id, exc_info=True)
- _finish_unstarted("Fire claim ownership could not be validated before execution started.")
- return True
- if owns_fire_claim is False:
- logger.warning("Job '%s': fire claim ownership was already lost before execution", job_id)
- _finish_unstarted("Fire claim ownership lost before execution started.")
- return True
- def _heartbeat_loop() -> None:
- last_confirmed = time.monotonic()
- while not stop.wait(_RUN_CLAIM_HEARTBEAT_SECONDS):
- try:
- if not heartbeat_fire_claim(job_id, expected_owner=owner):
- if self_removal_delivery_allowed(job_id):
- # Record dropped by this run; nothing left to keep fresh.
- continue
- # One miss is a sample, not a verdict (#113357): a latch cancels the live
- # agent run and ends the lease refresh, so confirm before acting on it.
- if stop.wait(_FIRE_CLAIM_MISS_CONFIRM_SECONDS) or heartbeat_fire_claim(
- job_id, expected_owner=owner):
- last_confirmed = time.monotonic()
- continue
- lost_ownership.set()
- logger.warning(
- "Job '%s': fire claim ownership lost; interrupting stale run",
- job_id)
- return
- last_confirmed = time.monotonic()
- except Exception:
- logger.debug("Job '%s': fire_claim heartbeat failed", job_id, exc_info=True)
- if (
- time.monotonic() - last_confirmed
- >= _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS
- ):
- lost_ownership.set()
- logger.warning(
- "Job '%s': fire_claim could not be renewed within %.1fs; "
- "interrupting uncertain run",
- job_id,
- _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS)
- return
- heartbeat_thread = _start_heartbeat_thread(
- _heartbeat_loop, "cron-fire-claim-heartbeat",
- lambda: logger.warning(
- "Job '%s': could not start fire_claim heartbeat", job_id, exc_info=True))
- if heartbeat_thread is None:
- _finish_unstarted("Fire claim heartbeat could not be started; execution was not run.")
- return True
- try:
- return run(lost_ownership)
- finally:
- stop.set()
- heartbeat_thread.join(timeout=1.0)
- def run_one_job(
- job: dict, *, adapters=None, loop=None, verbose: bool = False,
- extra_prompt: Optional[str] = None, cancel_event: Optional[_CancelEventLike] = None,
- ) -> bool:
- """Run ONE due job end-to-end: execute → save output → deliver → mark. Shared by the built-in
- ticker and external providers' ``fire_due``; does NOT decide due-ness or acquire the initial
- claim (callers use the store CAS) but keeps it alive. True if processed (a job failure is
- recorded via ``mark_job_run``), False only if processing raised. ``cancel_event``: optional
- transport-level cancel (dashboard drain)."""
- # Every gateway path (built-in scheduler, external providers, and direct
- # API fires) crosses this seam. Ensure the detached worker has a durable
- # attempt to adopt before any launch can occur.
- if not job.get("execution_id"):
- execution = create_execution(
- job["id"], source="direct", scheduled_instant=job.get("_scheduled_instant"))
- job["execution_id"] = execution["id"]
- execution_id = str(job["execution_id"])
- note_cron_execution(job)
- external_owner = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id
- if not external_owner:
- try:
- handed_off = _launch_external_cron_worker(job)
- except Exception as handoff_error:
- # Arbitrary waiter errors must not claim dispatch failure or resend
- # a worker's notice. A recovered unknown execution is different:
- # its worker never committed a terminal outcome.
- post_handoff = isinstance(handoff_error, _ExternalWorkerPostHandoffError)
- stage = "failed after handoff" if post_handoff else "dispatch failed"
- error = f"Restart-safe cron worker {stage}: {handoff_error}"
- if post_handoff:
- from cron.scheduler_worker_failure import record_unknown_worker_outcome
- if record_unknown_worker_outcome(job, error=str(handoff_error), adapters=adapters, loop=loop):
- return True
- logger.error("Job '%s': %s", job["id"], error)
- claim = job.get("fire_claim")
- owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
- delivery_error = delivery_outcome = None
- try:
- # A pre-handoff dispatch failure is a job failure like any other: it
- # must open an incident and leave through the job's failure lane
- # (#123401). Without this the outage is silent — no cron_incidents
- # row, no ping — while executions.db keeps piling up failed rows.
- if not post_handoff:
- # Returns before _run_one_job_body / record_unknown_worker_outcome install the scope; get_secret fails closed without one under multiplex.
- with _fire_secret_scope():
- delivery_error, delivery_outcome = _deliver_crash_failure(
- job, error, adapters=adapters, loop=loop)
- from cron.unreachable_retry import is_retry_run
- mark_job_run(
- job["id"],
- False,
- error,
- delivery_error=delivery_error,
- **({"expected_fire_owner": owner} if owner else {}),
- # A ladder re-run's occurrence already counted toward repeat.
- **({"ladder_rung": True} if is_retry_run(job) else {}),
- )
- finally:
- finish_execution(
- execution_id, success=False, error=error,
- delivery_outcome=delivery_outcome)
- return True
- if handed_off:
- from cron.scheduler_worker_failure import record_unknown_worker_outcome
- record_unknown_worker_outcome(job, adapters=adapters, loop=loop)
- return True
- if extra_prompt is None:
- # Gateway-forwarded manual run stamps its prompt on the job via trigger_job; the fire that
- # consumes the manual occurrence picks it up here. Single-fire: mark_job_run clears it.
- _stamped = job.get("manual_run_prompt")
- if _stamped and job.get("manual_run_at"):
- extra_prompt = str(_stamped)
- claim = job.get("fire_claim")
- fire_owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
- execution_token = object()
- profile_home = _get_hermes_home().resolve()
- _fire_key = _inflight_key(job["id"])
- with _running_lock:
- _running_fire_owners.setdefault(_fire_key, {})[execution_token] = (
- fire_owner or None, profile_home)
- try:
- with self_removal_delivery_scope(job["id"]):
- return _run_with_fire_claim_heartbeat(
- job,
- lambda lost_ownership: _run_one_job_body(
- job,
- adapters=adapters,
- loop=loop,
- verbose=verbose,
- extra_prompt=extra_prompt,
- claim_lost=lost_ownership,
- transport_cancel=cancel_event,
- execution_token=execution_token))
- finally:
- with _running_lock:
- executions = _running_fire_owners.get(_fire_key)
- if executions is not None:
- executions.pop(execution_token, None)
- if not executions:
- _running_fire_owners.pop(_fire_key, None)
- _OWNERSHIP_LOST_INTERRUPTED = "Interrupted by shutdown before terminal completion."
- def _record_fire_ownership_lost(
- job: dict, fire_owner: Optional[str], execution_id: str,
- ) -> None:
- """Bookkeeping after fire-claim ownership loss. A transport-level cancel (dashboard drain) is
- not a real loss — we still own the claim, so record the interruption via the owner-fenced
- terminal write instead of leaving fire_claim/last_status stale; otherwise discard.
- An interrupted ladder re-run re-ran an occurrence that already counted, so its terminal
- write must not spend another repeat slot (same as every other terminal path)."""
- from cron.unreachable_retry import is_retry_run
- job_id = job["id"]
- if fire_owner is not None and heartbeat_fire_claim(job_id, expected_owner=fire_owner):
- mark_job_run(
- job_id, False, _OWNERSHIP_LOST_INTERRUPTED, expected_fire_owner=fire_owner,
- **({"ladder_rung": True} if is_retry_run(job) else {}),
- )
- finish_execution(execution_id, success=False, error=_OWNERSHIP_LOST_INTERRUPTED)
- else:
- finish_lost_execution(
- execution_id, "Fire claim ownership lost; stale result was discarded.")
- def _compose_run_delivery(
- job: dict, *, success: bool, error, final_response: str, output_file,
- agent_declared: bool = False,
- ) -> tuple[str, bool, bool, bool, Optional[str]]:
- """Text to deliver for a finished run. Returns ``(deliver_content, blocked_config,
- silent_alert, incident_acked, failure_incident_id)``; ``silent_alert``: an alert-once marker
- says the operator was already told, deliver nothing. ``agent_declared``: *error* is the
- agent's own ``[CRON_FAILURE]`` evidence, delivered verbatim."""
- err = str(error) if error else ""
- # Failed jobs always deliver, except blocked-config runs, which alert exactly ONCE.
- blocked_config_silent = BLOCKED_CONFIG_SILENT_MARKER in err
- blocked_config = blocked_config_silent or BLOCKED_CONFIG_MARKER in err
- incident_acked = False
- failure_incident_id = None
- if blocked_config and not success:
- # Bypass the generic failure summarizer (its auth/timeout heuristics would mislabel this).
- _pf_text = re.sub(r"\[blocked_config[^\]]*\]\s*", "", err).strip()
- from cron.scheduler_failure_copy import blocked_config_notice
- deliver_content = blocked_config_notice(job.get("name") or job["id"], _pf_text)
- elif success:
- deliver_content = final_response
- _resolve_incidents_for_recovered_job(job)
- else:
- # Record the job+error signature once; withhold the per-run ping while the operator
- # already acked it (closed) or was already told (alerted, inside the reminder cooldown).
- # Best-effort: a ledger failure never breaks delivery.
- incident_acked, failure_incident_id = _upsert_incident_for_failure(
- job, error or "", output_file=output_file
- )
- if incident_acked:
- deliver_content = ""
- elif agent_declared:
- # The agent already diagnosed the failure in prose; the summarizer's substring
- # heuristics would re-diagnose it ("timed out" -> blame the model service, "401" ->
- # "sign in again") and attach the wrong remediation. Deliver the evidence as-is.
- from cron.scheduler_failure_copy import generic_failure_notice
- deliver_content = generic_failure_notice(
- job.get("name") or job["id"], job["id"], err.strip().rstrip("."),
- ) + _failure_streak_nudge(job)
- else:
- from cron.quota_hold import hold_notice
- deliver_content = (
- _summarize_cron_failure_for_delivery(job, error) + _failure_streak_nudge(job)
- # The one alert on entering a provider-window hold says so (#89376).
- + hold_notice(job, job.get("_quota_hold_seconds"))
- )
- return deliver_content, blocked_config, blocked_config_silent, incident_acked, failure_incident_id
- class _FireClaimLostDuringSideEffect(Exception):
- """Raised inside a side-effect fence when the durable fire claim is no longer ours."""
- class _FireOwnership:
- """Fire-claim ownership checks for one run (``owner`` is None when the job carries no claim)."""
- def __init__(
- self, job: dict, claim_lost: Optional[_CancelEventLike] = None,
- transport_cancel: Optional[_CancelEventLike] = None,
- operator_cancel: Optional[_CancelEventLike] = None,
- ):
- self.job = job
- # Two handles: the heartbeat's raw ``lost_ownership`` event and the caller's transport
- # ``cancel_event``. A sampled miss latches on the raw one only — the combined view's
- # ``set()`` would propagate into the transport event this run does not own (#105861).
- self.claim_lost = claim_lost
- self.transport_cancel = transport_cancel
- self.operator_cancel = operator_cancel
- # ``operator_cancel`` (``hermes cron cancel``) is a THIRD source, deliberately not folded
- # into ``transport_cancel``: only a transport cancel counts as claim loss for bookkeeping,
- # while an operator cancel is a real outcome this run records itself.
- # What ``run_job`` receives as its ``cancel_event``: any source cancels the run.
- self.cancel_event: Optional[_CancelEventLike] = (
- _CombinedCancelEvent(claim_lost, transport_cancel, operator_cancel)
- if claim_lost is not None or transport_cancel is not None or operator_cancel is not None
- else None)
- claim = job.get("fire_claim")
- self.owner = str(claim.get("by") or "") if isinstance(claim, dict) else None
- def transport_cancelled(self) -> bool:
- return self.transport_cancel is not None and self.transport_cancel.is_set()
- def operator_cancelled(self) -> bool:
- return self.operator_cancel is not None and self.operator_cancel.is_set()
- def side_effect_fence(self):
- if self.owner is None:
- return contextlib.nullcontext(True)
- return fire_claim_fence(self.job["id"], expected_owner=self.owner)
- def lost(self) -> bool:
- if self.transport_cancelled():
- return True
- if self.owner is None:
- return False
- if self_removal_delivery_allowed(self.job["id"]):
- # The run deleted its own record; there is no claim left to re-resolve.
- return False
- # The heartbeat's latched miss is one sample, not the verdict (#113357): the store is
- # re-checked here, and a claim that still validates keeps the run's real outcome. The
- # owner-fenced mark_job_run / fire_claim_fence remain the authority for side effects.
- latched = self.claim_lost is not None and self.claim_lost.is_set()
- try:
- if heartbeat_fire_claim(self.job["id"], expected_owner=self.owner):
- return False
- except Exception:
- logger.debug(
- "Job '%s': fire_claim ownership validation failed", self.job["id"], exc_info=True)
- # Unreachable store after a latched miss stays fail-closed; a first miss is best-effort.
- return latched
- if self.claim_lost is not None:
- self.claim_lost.set()
- return True
- @dataclass
- class _RunDelivery:
- """Mutable outcome of the save/compose/deliver phase, read back by the bookkeeping tail."""
- job: dict
- success: bool
- error: Optional[str]
- delivery_attempted: bool = False
- delivery_error: Optional[str] = None
- should_deliver: bool = False
- unresolved_origin: bool = False
- blocked_config: bool = False
- # True when ``error`` is the agent's own ``[CRON_FAILURE]`` evidence rather than a runtime
- # error string, so composition must not run it through the provider-error heuristics.
- agent_declared: bool = False
- incident_acked: bool = False
- failure_incident_id: Optional[str] = None
- side_effect_ownership_lost: bool = False
- def _save_compose_deliver(
- d: _RunDelivery, fence: _FireOwnership, final_response: str, output: str, *,
- adapters, loop, verbose: bool, execution_token,
- ) -> None:
- """Save output, compose the notice and deliver it (both side effects run under the fire-claim
- fence; a lost claim raises ``_FireClaimLostDuringSideEffect`` for the caller)."""
- job = d.job
- with fence.side_effect_fence() as owns_output:
- if not owns_output:
- raise _FireClaimLostDuringSideEffect
- # remove_job() already deleted this job's output dir; saving would re-create an orphan.
- output_file = (
- None if self_removal_delivery_allowed(job["id"])
- else save_job_output(job["id"], output))
- if verbose and output_file is not None:
- logger.info("Output saved to: %s", output_file)
- # A shutdown-killed tool subprocess can leave a plausible final_response from truncated
- # output; force the honest "interrupted" failure path. Peek-only (consumed later).
- if d.success and _is_interrupted(job["id"], execution_token):
- d.success = False
- d.error = (
- "Interrupted by gateway shutdown before the run finished "
- "(tool subprocess was killed mid-flight)."
- )
- (
- deliver_content, d.blocked_config, _silent_alert, d.incident_acked, d.failure_incident_id,
- ) = _compose_run_delivery(
- job, success=d.success, error=d.error, final_response=final_response,
- output_file=output_file, agent_declared=d.agent_declared)
- # Whitespace-only == empty: skip delivery; the guard below marks it a soft failure.
- d.should_deliver = bool(deliver_content.strip()) and not _silent_alert
- if d.should_deliver and not d.success and job.get("_model_unreachable"):
- # The model was never reached and a bounded automatic re-run will be scheduled
- # (cron/unreachable_retry.py): hold the failure notice — the re-run either
- # delivers the real result or, once the ladder is exhausted, the next failure
- # alerts normally. Mirrors Cowork's silent 5/15/30-minute re-runs.
- from cron.unreachable_retry import will_retry
- if will_retry(job):
- d.should_deliver = False
- logger.info(
- "Job '%s': suppressing failure notice — automatic re-run pending", job["id"])
- # Not a substring check: bare "SILENT"/"NO_REPLY" or a report quoting "[SILENT]" must
- # not be swallowed; bracketed-prefix / trailing-line tolerance is kept.
- if d.should_deliver and d.success and _is_cron_silence_response(deliver_content):
- # Cron silence suppression — see _is_cron_silence_response. Replaces the old `SILENT_MARKER in
- # ...upper()` substring check, which both leaked bracketless near-markers ("SILENT" / "NO_REPLY")
- # and wrongly swallowed a real report that merely quoted "[SILENT]" mid-sentence (#51438, #46917).
- logger.info("Job '%s': agent returned %s — skipping delivery", job["id"], SILENT_MARKER)
- d.should_deliver = False
- if d.should_deliver and fence.lost():
- d.should_deliver = False
- logger.warning("Job '%s': skipping delivery after fire claim ownership loss", job["id"])
- if not d.should_deliver:
- return
- d.unresolved_origin = (
- _normalize_deliver_value(_delivery_lane_value(job, for_failure=not d.success)) == "origin"
- and not _resolve_delivery_targets(job, for_failure=not d.success)
- )
- execution_id = job.get('execution_id')
- if execution_id and not job.get('no_agent'):
- from cron.delivery_queue import enqueue
- # Nothing to send: an origin-less job is ``not_configured``, never a queued "delivery".
- if d.unresolved_origin or _normalize_deliver_value(
- _delivery_lane_value(job, for_failure=not d.success)) == "local":
- return
- queued_job = dict(job)
- if d.failure_incident_id:
- queued_job["_failure_incident_id"] = d.failure_incident_id
- # The queue row IS the send: publish it under the same fence as a direct send, so a claim
- # stolen after the lost() sample above cannot leave this stale fire's notice queued.
- with fence.side_effect_fence() as owns_delivery:
- if not owns_delivery:
- raise _FireClaimLostDuringSideEffect
- queued = enqueue(execution_id, queued_job, deliver_content, for_failure=not d.success)
- # The queue owns this send even if subsequent bookkeeping fails.
- d.delivery_attempted = True
- job['last_delivery_queued'] = {'canonical': {'status': queued['status'], 'execution_id': execution_id}}
- return
- try:
- with fence.side_effect_fence() as owns_delivery:
- if not owns_delivery:
- raise _FireClaimLostDuringSideEffect
- d.delivery_attempted = True
- d.delivery_error = _deliver_result(
- job,
- deliver_content,
- adapters=adapters,
- loop=loop,
- # Failure summaries (and drift/blocked-config alerts composed into deliver_content
- # on the failure path) honor the job's failure_deliver override (NS-788).
- for_failure=not d.success,
- )
- except Exception as de:
- if isinstance(de, _FireClaimLostDuringSideEffect):
- raise
- d.delivery_error = str(de)
- logger.error("Delivery failed for job %s: %s", job["id"], de)
- def _deliver_crash_failure(
- job: dict, err_text: str, *, adapters, loop,
- ) -> tuple[Optional[str], str]:
- """Failure notice for a run that raised out of run_job. Returns (delivery_error, outcome)."""
- normalized_deliver = _normalize_deliver_value(_delivery_lane_value(job, for_failure=True))
- # Same ack gate as the normal failure delivery: acked signatures stay silent here too.
- incident_acked, failure_incident_id = _upsert_incident_for_failure(job, err_text)
- if incident_acked:
- return None, "suppressed_acked"
- delivery_error = None
- try:
- delivery_error = _deliver_result(
- job,
- # Same text as the normal failure delivery: this run also counts toward
- # failure_streak, so the nudge must leave through here too.
- _summarize_cron_failure_for_delivery(job, err_text) + _failure_streak_nudge(job),
- adapters=adapters,
- loop=loop,
- for_failure=True,
- )
- except Exception as delivery_exc:
- delivery_error = str(delivery_exc)
- logger.error("Delivery failed for job %s: %s", job["id"], delivery_exc)
- unresolved_origin = bool(
- not delivery_error
- and normalized_deliver == "origin"
- and not _resolve_delivery_targets(job, for_failure=True)
- )
- delivery_outcome = _classify_delivery_outcome(
- delivery_error=delivery_error, should_deliver=True, unresolved_origin=unresolved_origin,
- normalized_deliver=normalized_deliver, incident_acked=False, success=False,
- delivery_queued=job.get("last_delivery_queued"),
- notification_suppressed=bool(job.get("_notification_all_targets_suppressed")))
- if delivery_outcome in ("delivered", "not_configured"):
- _mark_incident_alerted(failure_incident_id)
- return delivery_error, delivery_outcome
- def _install_fire_secret_scope() -> tuple[contextvars.Token, Optional[contextvars.Token]]:
- """Install the firing profile's secret scope for the span ``_run_one_job_body`` runs, delivery
- included, and return the tokens ``_reset_fire_secret_scope`` needs.
- Hydrate the profile's external secret sources BEFORE freezing the scope — the order
- gateway/run.py and the external cron worker already use: ``build_profile_secret_scope`` only
- READS the per-home source map, so a scope frozen first would carry no vault-backed value.
- For a fire routed to a profile other than the process's own (``routed_profile_fire`` on the
- fire's home — every entry point: ticker, dashboard Run now, CLI) also run under multiplex semantics — for
- exactly this span and no wider. The desktop backend ticks every local profile from a process
- that is not a multiplexer, so nothing else isolates that fire; and switching the context on
- here rather than at the tick means no read is ever fail-closed without a scope to read — the
- restart-safe handoff in ``run_one_job`` runs before this and keeps today's semantics (#107692).
- """
- from agent.secret_scope import set_multiplex_context, set_secret_scope
- from cron.scheduler_provider import routed_profile_fire
- from hermes_cli.env_loader import hydrate_profile_secret_sources
- from tui_gateway.launch_profile_policy import owner_secret_scope
- home = Path(_get_hermes_home())
- hydrate_profile_secret_sources(home)
- scope_token = set_secret_scope(owner_secret_scope(home), profile_home=str(home))
- context_token = set_multiplex_context(True) if routed_profile_fire() else None
- return scope_token, context_token
- def _reset_fire_secret_scope(tokens: tuple[contextvars.Token, Optional[contextvars.Token]]) -> None:
- """Undo ``_install_fire_secret_scope`` — the context first, so multiplex semantics never
- outlive the scope they depend on."""
- from agent.secret_scope import reset_multiplex_context, reset_secret_scope
- scope_token, context_token = tokens
- if context_token is not None:
- reset_multiplex_context(context_token)
- reset_secret_scope(scope_token)
- @contextlib.contextmanager
- def _fire_secret_scope():
- """``_install_fire_secret_scope`` for the ``with`` block, reset on the way out."""
- tokens = _install_fire_secret_scope()
- try:
- yield
- finally:
- _reset_fire_secret_scope(tokens)
- def _start_owned_run(job: dict, execution_id: str) -> Optional[contextvars.Token]:
- """Win the run's ``claimed`` → ``running`` CAS and bind its cron identity; ``None`` when the run
- lost ownership first. A restart-safe worker already won it by adopting the row."""
- if os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id:
- record = get_execution(execution_id)
- elif (record := mark_execution_running(execution_id)) is None:
- return None
- return enter_cron_execution(job, execution_id, record or {})
- def _apply_agent_failure_marker(job: dict, success: bool, error, final_response):
- """An agent can finish its own turn after a delegated child has failed. Let it explicitly
- declare that semantic failure so the existing failure path updates status, streaks, ledger,
- and notification routing instead of recording a false healthy result.
- Returns ``(success, error, agent_declared)``."""
- if success and not job.get("no_agent"):
- marker_error = _cron_failure_marker_error(final_response)
- if marker_error is not None:
- return False, marker_error, True
- return success, error, False
- def _run_one_job_body(
- job: dict, *, adapters=None, loop=None, verbose: bool = False,
- extra_prompt: Optional[str] = None, claim_lost: Optional[_CancelEventLike] = None,
- transport_cancel: Optional[_CancelEventLike] = None,
- execution_token: Optional[object] = None,
- ) -> bool:
- execution_id = job.get("execution_id")
- if not execution_id:
- execution_id = create_execution(
- job["id"], source="direct", scheduled_instant=job.get("_scheduled_instant"))["id"]
- job = dict(job, execution_id=execution_id)
- # The execution id is resolved BEFORE the fence so ``hermes cron cancel`` can join the same
- # cancel seam: the marker it writes is polled by whichever process owns this attempt.
- fence = _FireOwnership(job, claim_lost, transport_cancel, CancelRequest(execution_id))
- fire_owner = fence.owner
- _side_effect_fence = fence.side_effect_fence
- _fire_claim_ownership_lost = fence.lost
- delivery_attempted = False
- delivery_error = None
- _fire_scope_tokens = None
- _terminal_scope_token = None
- _identity_token = None
- try:
- # get_secret() fails closed outside a scope; the ticker thread has none. Delivery adapters
- # resolve credentials, so the scope must span delivery too (reset in the outer finally) —
- # including the crash notice in the except below when claim_dispatch or
- # mark_execution_running raises, so install it before either.
- _fire_scope_tokens = _install_fire_secret_scope()
- # Commit a finite one-shot's dispatch BEFORE its side effect so a tick dying mid-run cannot
- # re-fire it forever on restart. No-op for recurring/infinite jobs (at-most-times).
- # This lives here in the shared body so BOTH the built-in ticker and the external provider (Chronos
- # fire_due) get at-most-times semantics. See #38758.
- if not claim_dispatch(job["id"]):
- logger.info(
- "Job '%s': one-shot dispatch limit reached — skipping",
- job.get("name", job["id"]))
- finish_execution(
- execution_id, success=False,
- error="Dispatch claim rejected; execution was not started.")
- return True # not an error — already handled/removed
- # Claimed durably before dispatch; becomes running only right before the actual run.
- # Detached workers transition to running while adopting; in-process paths must win the
- # claimed->running CAS here before any user script or agent side effect may begin.
- # The identity plugins read via ctx.current_cron_execution() is bound only on that win.
- _identity_token = _start_owned_run(job, execution_id)
- if _identity_token is None:
- logger.warning("Cron job %s lost execution ownership before start; skipping", job["id"])
- return True
- # Bind the firing profile's COMPLETE terminal policy for this fire — agent build, run, delivery
- # and bookkeeping (gateway/run.py _profile_runtime_scope does the same per turn) — else the
- # ticker reads process-global TERMINAL_* env another profile pinned (#68559, #98581). A
- # malformed policy installs a refusal scope: terminal execution raises instead.
- from tools.terminal_scope import (
- install_profile_terminal_scope)
- _terminal_scope_token = install_profile_terminal_scope(_get_hermes_home())
- # Defer agent teardown until AFTER delivery: closing first races the live send against a
- # torn-down async client (#58720). run_job hands the agent back via this list instead.
- _deferred_agents: list = []
- def _teardown_deferred() -> None:
- # run_job's finally still hands back the agent when it raises; tear it down here so a failed run
- # never leaks its async resources (#10200), then re-raise into the outer handler. BaseException
- # (not just Exception) so a KeyboardInterrupt/SystemExit mid-run still triggers teardown before
- # propagating.
- # Tear down the deferred agent(s) now that save + delivery have run (or raised). Must happen on
- # every path so cron agents never leak their subprocesses/clients (#10200).
- for _deferred_agent in _deferred_agents:
- _teardown_cron_agent(_deferred_agent, job["id"])
- _run_kwargs = {
- "defer_agent_teardown": _deferred_agents,
- "extra_prompt": extra_prompt,
- "execution_id": execution_id}
- if fence.cancel_event is not None:
- _run_kwargs["cancel_event"] = fence.cancel_event
- outcome = _run_job_unless_cancelled(
- run_job, job, _run_kwargs, _teardown_deferred, fence, execution_id, fire_owner)
- if outcome is None: # an operator cancel stopped it and is already booked
- return True
- success, output, final_response, error = outcome
- if _fire_claim_ownership_lost():
- _teardown_deferred()
- _record_fire_ownership_lost(job, fire_owner, execution_id)
- return True
- success, error, agent_declared = _apply_agent_failure_marker(job, success, error, final_response)
- # Agent is still live through delivery; wrap ALL of save/compose/deliver in try/finally so a
- # raise anywhere still tears the deferred agent down.
- d = _RunDelivery(job=job, success=success, error=error, agent_declared=agent_declared)
- try:
- _save_compose_deliver(
- d, fence, final_response, output, adapters=adapters, loop=loop, verbose=verbose,
- execution_token=execution_token)
- except _FireClaimLostDuringSideEffect:
- d.side_effect_ownership_lost = True
- finally:
- delivery_attempted, delivery_error = d.delivery_attempted, d.delivery_error
- # Every path must tear down deferred agent(s) so they never leak subprocesses/clients.
- _teardown_deferred()
- if d.side_effect_ownership_lost:
- # The claim died inside a side-effect fence: the side effect did NOT complete.
- _record_fire_ownership_lost(job, fire_owner, execution_id)
- return True
- from cron.scheduler_bookkeeping import fail_empty_response
- fail_empty_response(d, final_response)
- if _fire_claim_ownership_lost():
- # #105861: the claim check is one sample; a miss AFTER a completed delivery must not
- # overwrite the delivered run's terminal status — ok, or a failure whose notice already
- # left with its real error — so fall through to finish_completed_run, whose owner-fenced
- # mark_job_run is authoritative either way. An explicit transport cancel stays fail-closed.
- transport_cancelled = fence.transport_cancelled()
- if d.delivery_attempted and not d.delivery_error and not transport_cancelled:
- logger.warning(
- "Job '%s': fire claim ownership lost after completed delivery; "
- "recording the delivered run's terminal status",
- job["id"])
- else:
- if transport_cancelled:
- logger.warning(
- "Job '%s': transport cancellation arrived before terminal completion; "
- "recording the interrupted run",
- job["id"])
- _record_fire_ownership_lost(job, fire_owner, execution_id)
- return True
- if _consume_interrupted_flag(job["id"], execution_token):
- from cron.scheduler_bookkeeping import finish_interrupted_run
- finish_interrupted_run(job, execution_id, delivery_error)
- return True
- from cron.scheduler_bookkeeping import finish_completed_run
- return finish_completed_run(d, fire_owner, execution_id)
- except BaseException as e:
- from cron.scheduler_authority import CronExecutionUnknown
- if isinstance(e, CronExecutionUnknown):
- from cron.executions import give_up_execution
- from cron.scheduler_authority import pause_for_unknown
- pause_for_unknown(job["id"], execution_id, str(e))
- give_up_execution(execution_id, reason=str(e))
- logger.error("Cron %s paused with unverified admission: %s", job["id"], e)
- return False
- # BaseException, not Exception: CancelledError/KeyboardInterrupt/SystemExit propagate here.
- # Without mark_job_run(False) a finite one-shot is wedged: claim_dispatch consumed
- # repeat.completed but last_run_at is never written. Record first, then re-raise
- # non-Exception. Owner fencing still applies.
- # BaseException, not Exception (#73973): the inner run_job handler re-raises CancelledError /
- # KeyboardInterrupt / SystemExit after agent teardown, and none of those are Exception subclasses.
- # If they escape without mark_job_run(False), a finite one-shot is left wedged — claim_dispatch()
- # already consumed repeat.completed, but last_run_at is never written, so the job sits in state
- # "scheduled" until the run-claim TTL expires and the dispatch-limit guard removes it with no output
- # and no error. Owner fencing still applies: a stale worker must not record over a replacement claim
- # owner.
- _err_text = str(e) or type(e).__name__
- # A config backend outage (a SystemExit by design) fails this fire, never the ticker.
- contained = isinstance(e, (Exception, ConfigBackendUnavailable))
- logger.error(
- "Error processing job %s: %s",
- job["id"],
- _err_text,
- exc_info=(type(e), e, e.__traceback__))
- delivery_outcome = "suppressed"
- # Owner fencing: a stale worker whose claim was taken over (or transport-cancelled) must not
- # send a failure alert on top of the replacement run's; fall through to fenced bookkeeping.
- if (
- contained
- and not delivery_attempted
- and not isinstance(e, _FireClaimLostDuringSideEffect)
- and not _fire_claim_ownership_lost()
- ):
- delivery_error, delivery_outcome = _deliver_crash_failure(
- job, _err_text, adapters=adapters, loop=loop)
- try:
- if (
- not _consume_interrupted_flag(job["id"], execution_token)
- and not self_removal_delivery_allowed(job["id"]) # no record left to mark
- ):
- mark_kwargs = {}
- if fire_owner is not None:
- mark_kwargs["expected_fire_owner"] = fire_owner
- if contained:
- mark_kwargs["delivery_error"] = delivery_error
- from cron.unreachable_retry import is_retry_run
- if is_retry_run(job):
- # A crashed ladder re-run: its occurrence already counted toward repeat.
- mark_kwargs["ladder_rung"] = True
- mark_job_run(job["id"], False, _err_text, **mark_kwargs)
- except Exception as record_err:
- # Never let bookkeeping mask the original interruption.
- logger.error("Failed to record interrupted run for job %s: %s", job["id"], record_err)
- try:
- finish_execution(
- execution_id, success=False, error=_err_text, delivery_outcome=delivery_outcome)
- except Exception as record_err:
- logger.error("Failed to finish execution record for job %s: %s", job["id"], record_err)
- if not contained:
- raise
- return False
- finally:
- # Function-level on purpose: must scope delivery, deferred teardown, claim-loss handling and
- # bookkeeping — not just run_job. Do not move into the run block's finally.
- # The attempt is over either way, so its cancel marker goes with it — an unread marker
- # would otherwise sit in the cancels dir until the TTL prune.
- clear_cancel(execution_id)
- exit_cron_execution(_identity_token)
- if _fire_scope_tokens is not None:
- _reset_fire_secret_scope(_fire_scope_tokens)
- if _terminal_scope_token is not None:
- from tools.terminal_scope import reset_terminal_scope
- reset_terminal_scope(_terminal_scope_token)
- def _wait_for_external_cron_worker_body(
- process: subprocess.Popen,
- *,
- execution_id: str,
- stderr_path: Optional[Path] = None,
- ) -> bool:
- """Preserve ``run_one_job``'s synchronous contract after handoff.
- The worker owns the durable execution and survives this gateway process.
- The caller nevertheless waits while it remains alive so manual/background
- callers do not release their in-process guard or report stale job state.
- A gateway replacement may kill this waiter; it does not kill the scoped
- worker or change its ledger ownership.
- """
- def _is_terminal() -> bool:
- current = get_execution(execution_id)
- return bool(current and current.get("status") in _TERMINAL_STATES)
- # The worker commits its terminal row before its process exits, so exit is
- # the correct wakeup. Each ledger read opens a connection and re-runs
- # schema init; polling it at 50ms for an hours-long agent run is ~72k
- # opens/hour of pure contention with the worker's own writes.
- while True:
- try:
- returncode = process.wait(timeout=1.0)
- except subprocess.TimeoutExpired:
- if not _is_terminal():
- continue
- # Recovery can win between the timeout and this ledger read. If
- # the worker has since exited, consume its diagnostics below before
- # cleanup; only a still-live terminal worker needs background reaping.
- returncode = process.poll()
- if returncode is None:
- from cron.scheduler_detached_worker import reap_terminal_worker_in_background
- reap_terminal_worker_in_background(process)
- return True
- # The worker can commit its terminal row and exit between the first
- # read and wait(). Re-read the exact attempt before declaring that
- # it died without terminalizing.
- current = get_execution(execution_id)
- if current and current.get("status") in FIRER_FINISHED_STATES:
- return True
- # If the adopted worker died without terminalizing, its owner is
- # now provably gone. Recover to ``unknown`` rather than routing the
- # exception through the pre-handoff dispatch-failure path, which
- # would falsely assert that no side effect could have happened.
- #
- # Record the cause THIS waiter observed instead of letting the blanket
- # sweep stamp "Scheduler restarted ...", and then RAISE: the sweep leaves
- # the row terminal, so returning success here made run_one_job skip
- # mark_job_run entirely and the job's fire claim outlived its lost
- # execution, refusing the next manual fire with "Job is already being
- # fired" (#128509). Raising routes the existing post-handoff bookkeeping
- # instead: retire the claim and report the uncertain run, not a pre-dispatch failure.
- from cron.scheduler_worker_failure import external_worker_exited_reason
- reason = external_worker_exited_reason(returncode)
- if not terminalize_dead_owner(execution_id, reason=reason):
- recover_interrupted_executions()
- # A concurrent sweep may have won the unknown transition. Still
- # surface this waiter's exit code and stderr instead of discarding them.
- current = get_execution(execution_id)
- if current and current.get("status") in FIRER_FINISHED_STATES:
- return True
- stderr_tail = ""
- if stderr_path is not None:
- from cron.scheduler_diagnostics import external_worker_stderr_tail
- stderr_tail = external_worker_stderr_tail(stderr_path)
- raise RuntimeError(f"{reason}{stderr_tail}")
- class _ExternalWorkerPostHandoffError(RuntimeError):
- """The waiter failed after the worker was spawned and may own the execution."""
- def _wait_for_external_cron_worker(
- process: subprocess.Popen,
- *,
- execution_id: str,
- job_id: Optional[str] = None,
- handoff_files: tuple[Path, ...] = (),
- stderr_path: Optional[Path] = None,
- ) -> bool:
- try:
- return _wait_for_external_cron_worker_body(
- process, execution_id=execution_id, stderr_path=stderr_path
- )
- except Exception as wait_error:
- raise _ExternalWorkerPostHandoffError(str(wait_error)) from wait_error
- finally:
- if job_id is not None:
- _key = _inflight_key(job_id)
- with _running_lock:
- _restart_safe_waiter_job_ids.discard(_key)
- _scope_isolated_job_ids.discard(_key)
- # The execution is terminal or its worker is dead: nobody will read a
- # payload or acknowledgement left behind by a late/unread handoff.
- for stale in handoff_files:
- try:
- stale.unlink(missing_ok=True)
- except OSError:
- pass
- def _launch_external_cron_worker(job: dict) -> bool:
- """Launch *job* outside the managed gateway process when required.
- Returns ``False`` outside a managed systemd gateway (in-process path). In
- managed topology the job always goes to an external worker with the #101940
- ownership handoff: in a transient user scope, or — when no user D-Bus
- session exists and ``cron.require_restart_safe_scope`` is false — as a
- direct subprocess (process separation kept, cgroup isolation lost).
- A fire routed to a profile other than the process's own is multiplexed at THIS boundary too.
- ``run_one_job`` switches the context on in ``_install_fire_secret_scope``, which runs AFTER
- this handoff, so a routed desktop fire on the managed path serialized ``multiplex_active=False``
- and built the worker environment with the launch profile's residue and no scrub (review on
- f5f88d5058). Enable it for exactly this span; the worker then re-establishes it from the payload.
- """
- execution_id = str(job["execution_id"])
- job_id = str(job["id"])
- handoff_dir = _get_hermes_home() / "cron" / "external-workers"
- payload_path = handoff_dir / f"{execution_id}.json"
- ack_path = handoff_dir / f"{execution_id}.ready"
- # Captured so a worker that dies before its acknowledgement can name the cause (#112729).
- stderr_path = handoff_dir / f"{execution_id}.stderr"
- command = [
- sys.executable,
- "-m",
- "cron.scheduler",
- "--external-worker-file",
- str(payload_path),
- "--ack-file",
- str(ack_path),
- ]
- from agent.secret_scope import is_multiplex_active
- from cron.scheduler_provider import routed_profile_fire
- from tools.environments.local import build_subprocess_env, strip_launch_profile_env
- from tools.process_registry import (
- restart_safe_gateway_child_argv,
- scoped_spawn_lost_user_bus,
- systemd_user_bus_env,
- )
- # A fire routed to another profile is multiplexed at this handoff even when the process flag is
- # off (the desktop ticker is not a multiplexer): the payload says so, and the worker env is built
- # under that context so the scrub and the launch-residue strip both apply. The context is set
- # for exactly the env build; nothing else here reads it.
- multiplex_active = is_multiplex_active() or routed_profile_fire()
- try:
- require_restart_safe_scope = bool(
- (load_config_readonly().get("cron") or {}).get("require_restart_safe_scope", False)
- )
- except Exception:
- require_restart_safe_scope = False
- dispatch = restart_safe_gateway_child_argv(
- command,
- unit_suffix=f"cron-{job_id}-exec-{execution_id}",
- require_restart_safe_scope=require_restart_safe_scope,
- )
- if dispatch.mode == "in_process":
- return False
- if mark_execution_handoff_pending(execution_id) is None:
- raise RuntimeError(
- "cron execution claim changed before external worker handoff"
- )
- _ensure_cron_dir(handoff_dir)
- try:
- handoff_dir.chmod(0o700)
- except OSError:
- pass
- fd = os.open(payload_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
- try:
- with os.fdopen(fd, "w", encoding="utf-8") as payload_file:
- json.dump(
- {
- "job": job,
- "profile_home": str(_get_hermes_home().resolve()),
- "multiplex_active": multiplex_active,
- },
- payload_file,
- )
- payload_file.flush()
- os.fsync(payload_file.fileno())
- except BaseException:
- payload_path.unlink(missing_ok=True)
- raise
- profile_home = _get_hermes_home().resolve()
- # Same hydrate -> scope -> (routed) multiplex-context install the in-process fire uses, for exactly
- # the env build; the helper's reset order keeps the context from outliving its scope.
- with _fire_secret_scope():
- # No restore_managed_env here: the worker re-runs load_hermes_dotenv -> _apply_managed_env at
- # import, and strip_launch_profile_env leaves managed keys in place.
- worker_env = strip_launch_profile_env(build_subprocess_env(
- scrub_secrets=multiplex_active,
- inherit_profile_home=True,
- extra={"HERMES_HOME": str(profile_home)},
- ))
- worker_env = systemd_user_bus_env(worker_env)
- # Unattended worker: the gateway sets HERMES_EXEC_ASK at startup (interactive launches set
- # the other two), and an inherited presence var makes every env-fallback consumer in the
- # child (`_is_interactive_cli`, sudo prompting, `check_cronjob_requirements`) believe a
- # human is present to answer (#110932).
- for _presence_var in (
- "HERMES_INTERACTIVE",
- "HERMES_GATEWAY_SESSION",
- "HERMES_EXEC_ASK",
- ):
- worker_env.pop(_presence_var, None)
- # `-m cron.scheduler` has no hermes_cli.main bootstrap; pin this checkout explicitly
- # (PYTHONSAFEPATH / stale editable mapping, #112729), hand the child the committed
- # dependency generation (#122222), and mark it so its own entry runs the PM dependency
- # boot. See cron/scheduler_worker_env.py and cron/worker_bootstrap.py.
- from cron.scheduler_worker_env import pin_hermes_tree_on_pythonpath
- repo_root = Path(__file__).resolve().parent.parent
- worker_env = pin_hermes_tree_on_pythonpath(worker_env, repo_root)
- worker_env[WORKER_MARKER] = "1"
- try:
- stderr_fd = os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
- try:
- process = subprocess.Popen(
- dispatch.argv,
- cwd=str(repo_root),
- env=worker_env,
- stdin=subprocess.DEVNULL,
- stdout=subprocess.DEVNULL,
- stderr=stderr_fd,
- # detached-spawn: ok — restart-safe cron worker, must survive a gateway restart (#101940)
- start_new_session=True,
- creationflags=windows_hide_flags(),
- )
- finally:
- os.close(stderr_fd)
- except BaseException:
- payload_path.unlink(missing_ok=True)
- stderr_path.unlink(missing_ok=True)
- raise
- with _running_lock:
- _restart_safe_waiter_job_ids.add(_inflight_key(job_id))
- # Same window the dead-owner recovery ledger grants a pending handoff: a cold
- # worker start (imports + secret hydration) measures ~10-12s in the field, and
- # a dispatch deadline shorter than the adoption grace made the two guards
- # around one handoff disagree.
- deadline = time.monotonic() + HANDOFF_ADOPTION_GRACE_SECONDS
- while time.monotonic() < deadline:
- if ack_path.exists():
- try:
- acknowledgement = json.loads(ack_path.read_text(encoding="utf-8-sig"))
- except Exception:
- logger.exception(
- "Cron external worker %s published an unreadable acknowledgement; "
- "treating handoff as ownership-uncertain",
- execution_id,
- )
- return _wait_for_external_cron_worker(
- process,
- execution_id=execution_id,
- job_id=job_id,
- handoff_files=(payload_path, stderr_path),
- stderr_path=stderr_path,
- )
- finally:
- ack_path.unlink(missing_ok=True)
- if (
- not isinstance(acknowledgement, dict)
- or acknowledgement.get("execution_id") != execution_id
- ):
- logger.error(
- "Cron external worker acknowledgement mismatch for %s; "
- "treating handoff as ownership-uncertain",
- execution_id,
- )
- return _wait_for_external_cron_worker(
- process,
- execution_id=execution_id,
- job_id=job_id,
- handoff_files=(payload_path, stderr_path),
- stderr_path=stderr_path,
- )
- logger.info(
- "Cron job '%s' handed to restart-safe worker pid=%s execution=%s dispatch=%s",
- job_id,
- acknowledgement.get("pid"),
- execution_id,
- dispatch.mode,
- )
- _record_external_cron_worker(
- job_id,
- acknowledgement.get("pid") or process.pid,
- scope_isolated=dispatch.mode == "scoped",
- )
- return _wait_for_external_cron_worker(
- process,
- execution_id=execution_id,
- job_id=job_id,
- handoff_files=(payload_path, stderr_path),
- stderr_path=stderr_path,
- )
- returncode = process.poll()
- if returncode is not None:
- with _running_lock:
- _restart_safe_waiter_job_ids.discard(_inflight_key(job_id))
- payload_path.unlink(missing_ok=True)
- from cron.scheduler_diagnostics import external_worker_stderr_tail
- stderr_tail = external_worker_stderr_tail(stderr_path)
- stderr_path.unlink(missing_ok=True)
- if dispatch.mode == "scoped" and scoped_spawn_lost_user_bus(worker_env):
- # systemd-run itself failed before any worker ran, so the captured stderr
- # holds nothing useful: name the cause, not the exit code.
- raise RuntimeError(
- "restart-safe systemd scope could not be created: the user D-Bus session at "
- f"/run/user/{os.getuid()}/bus disappeared after the gateway started. On a " # windows-footgun: ok — scoped dispatch exists only on Linux
- "system-level service install, run `sudo loginctl enable-linger <gateway-user>`; "
- "the next fire dispatches without scope isolation."
- )
- raise RuntimeError(
- f"cron external worker exited before ownership acknowledgement "
- f"(exit {returncode}){stderr_tail}"
- )
- time.sleep(0.05)
- # The child may have adopted the durable row just before publishing its
- # acknowledgement. Never fall back to in-process execution on an uncertain
- # handoff: that could duplicate side effects. The execution owner/dead-owner
- # recovery ledger remains the authority.
- logger.warning(
- "Cron external worker for job '%s' did not acknowledge within %.0fs; "
- "leaving the durable execution claim untouched",
- job_id,
- HANDOFF_ADOPTION_GRACE_SECONDS,
- )
- return _wait_for_external_cron_worker(
- process,
- execution_id=execution_id,
- job_id=job_id,
- handoff_files=(payload_path, ack_path, stderr_path),
- stderr_path=stderr_path,
- )
- def _run_external_worker_payload(payload_path: Path, ack_path: Path) -> bool:
- """Adopt and execute one gateway-dispatched cron payload.
- The execution row is created by the gateway before spawn, then transferred
- here before the ready acknowledgement is published. No side effect runs
- unless that durable ownership transfer succeeds.
- """
- try:
- payload = json.loads(payload_path.read_text(encoding="utf-8-sig"))
- job = payload["job"]
- profile_home = Path(payload["profile_home"]).resolve()
- execution_id = str(job["execution_id"])
- except Exception:
- logger.exception("Cron external worker could not load payload %s", payload_path)
- return False
- finally:
- try:
- payload_path.unlink(missing_ok=True)
- except OSError:
- pass
- from agent.secret_scope import (
- build_profile_secret_scope,
- is_multiplex_active,
- reset_secret_scope,
- set_multiplex_active,
- set_secret_scope,
- )
- from cron.executions import adopt_claimed_execution
- from hermes_cli.env_loader import hydrate_profile_secret_sources
- from hermes_constants import (
- reset_hermes_home_override,
- set_hermes_home_override,
- )
- previous_multiplex = is_multiplex_active()
- home_token = secret_token = None
- try:
- home_token = set_hermes_home_override(profile_home)
- multiplex_active = bool(payload.get("multiplex_active", False))
- set_multiplex_active(multiplex_active)
- # Plugin secret sources (``ctx.register_secret_source()``) only exist after plugin
- # discovery; this process starts with the builtin registry alone, so hydrating without it
- # silently dropped every plugin-sourced credential (#121929). Runs under the home override
- # so a multiplexed worker loads the OWNING profile's plugins, not the launch profile's.
- from hermes_cli.plugins import discover_plugins
- discover_plugins()
- hydrate_profile_secret_sources(profile_home)
- secret_token = set_secret_scope(build_profile_secret_scope(profile_home), profile_home=str(profile_home))
- # This process never ran gateway startup, so the OWNING profile's ``hooks:`` block (shell
- # hooks + ``hooks.outbound``) was never registered and cron sessions silently lost it
- # (#131764). Same helper the gateway uses per profile; it must run inside the secret scope
- # because ``secret_env`` targets resolve through ``get_secret`` (raises unscoped under multiplex).
- from gateway.run_startup import GatewayStartupMixin
- GatewayStartupMixin._register_config_hooks(
- "Cron external worker: config hook registration failed", level=logging.WARNING)
- with use_cron_store(profile_home):
- if adopt_claimed_execution(execution_id) is None:
- logger.error(
- "Cron external worker refused execution %s: durable ownership "
- "could not be established",
- execution_id,
- )
- return False
- try:
- ack_path.parent.mkdir(parents=True, exist_ok=True)
- # Publish via write-to-temp + atomic rename. Writing ack_path in place
- # (the old approach) let O_CREAT make the empty file visible to the
- # scheduler's exists()-then-read polling loop before the JSON body was
- # written, occasionally handing it a 0-byte file and a JSONDecodeError.
- # os.replace() is a single atomic syscall on the same filesystem, so
- # readers only ever see the file fully absent or fully written.
- ack_tmp_path = ack_path.with_name(f"{ack_path.name}.tmp{os.getpid()}")
- fd = os.open(ack_tmp_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
- try:
- with os.fdopen(fd, "w", encoding="utf-8") as ack_file:
- json.dump({"pid": os.getpid(), "execution_id": execution_id}, ack_file)
- ack_file.flush()
- os.fsync(ack_file.fileno())
- os.replace(ack_tmp_path, ack_path)
- except BaseException:
- ack_tmp_path.unlink(missing_ok=True)
- raise
- except Exception:
- logger.exception(
- "Cron external worker could not publish ready acknowledgement for %s",
- execution_id,
- )
- return False
- old_external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER")
- os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = execution_id
- try:
- completed = run_one_job(job, adapters=None, loop=None, verbose=False)
- # Successful return: the worker recorded its outcome. If it raises,
- # retain fd 2's pathname until the waiter consumes the traceback.
- with contextlib.suppress(OSError):
- ack_path.with_suffix(".stderr").unlink(missing_ok=True)
- return completed
- finally:
- if old_external_execution is None:
- os.environ.pop("_HERMES_CRON_EXTERNAL_WORKER", None)
- else:
- os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = old_external_execution
- finally:
- if secret_token is not None:
- reset_secret_scope(secret_token)
- set_multiplex_active(previous_multiplex)
- if home_token is not None:
- reset_hermes_home_override(home_token)
- def _notify_provider_jobs_changed() -> None:
- """Best-effort: tell the active scheduler provider the job set changed. Call AFTER a successful
- store mutation so an external provider can re-provision/cancel the one-shot; no-op for the
- built-in. Kept out of cron/jobs.py (import cycle). Never raises."""
- try:
- from cron.scheduler_provider import resolve_cron_scheduler
- resolve_cron_scheduler().on_jobs_changed()
- except Exception as e:
- logger.debug("on_jobs_changed notify failed: %s", e)
- class CronSchedulerRegistrationError(RuntimeError):
- """A job was persisted but its first external trigger was not registered."""
- def __init__(self, job: dict, cause: Exception) -> None:
- self.job = job
- self.cause = cause
- super().__init__(
- f"Cron job '{job['id']}' was saved, but its first scheduler "
- f"registration failed ({type(cause).__name__}). Do not create a "
- "duplicate. Pause/resume or update the job to retry registration."
- )
- def user_message(self) -> str:
- """Human-facing variant for chat/CLI surfaces (no exception class name)."""
- label = self.job.get("name") or self.job["id"]
- return (
- f"Saved cron job '{label}', but couldn't register it with the "
- "external scheduler yet. The job is kept — don't re-create it; "
- "pause/resume or edit it (e.g. via /cron) to retry registration."
- )
- def to_dict(self) -> dict:
- """Return the public partial-failure contract without provider details."""
- return {
- "error": str(self),
- "job_id": self.job["id"],
- "job_saved": True,
- "scheduler_registered": False,
- "retry_create": False}
- def create_job_with_scheduler_registration(**kwargs) -> dict:
- """Persist one job and register its first trigger with the active provider."""
- from cron.jobs import create_job
- from cron.scheduler_provider import resolve_cron_scheduler
- job = create_job(**kwargs)
- if not job.get("enabled", True):
- return job
- try:
- resolve_cron_scheduler().register_job(job)
- except Exception as exc:
- raise CronSchedulerRegistrationError(job, exc) from exc
- return job
- # Dead-owner reap is throttled (opens the executions ledger). Tests may reset
- # _last_dead_owner_reap_at to {} to force a reap next tick.
- # Dead-owner claim reclaim throttle (#86721): recover_interrupted_executions opens the executions ledger, so
- # the per-tick reap is rate-limited rather than run on every idle 60s cycle.
- # The throttle is keyed by profile home: under multiplex_profiles the ticker
- # ticks every profile each cycle, and a process-global slot would let the
- # first profile starve all the others.
- _DEAD_OWNER_REAP_INTERVAL_SECONDS = 300.0
- _last_dead_owner_reap_at: dict[str, float] = {}
- # Worktree prune throttle: the cron tick is the only reliably periodic process on gateway boxes.
- _WORKTREE_MAINTENANCE_INTERVAL_SECONDS = 6 * 3600.0
- _last_worktree_maintenance_at: Optional[dict] = None # profile home -> monotonic stamp
- _worktree_maintenance_lock = threading.Lock()
- def _worktree_maintenance_repos() -> list[str]:
- """Repos whose ``.worktrees/`` to keep pruned: the hermes checkout, job workdir repo roots and
- every repo this profile created a worktree in (``worktree_registry``), filtered to those that
- actually have a ``.worktrees/`` dir."""
- repos: set = set()
- with contextlib.suppress(Exception):
- from hermes_cli.worktree_registry import registered_repos
- repos.update(registered_repos())
- # Hermes source checkout (git installs only; wheel installs have no .git).
- with contextlib.suppress(Exception):
- install_root = Path(__file__).resolve().parent.parent
- if (install_root / ".git").exists():
- repos.add(str(install_root))
- with contextlib.suppress(Exception):
- from cron.jobs import load_jobs
- for job in load_jobs():
- workdir = str(job.get("workdir") or "").strip()
- if not workdir or not Path(workdir).is_dir():
- continue
- try:
- probe = subprocess.run(
- ["git", "rev-parse", "--show-toplevel"],
- capture_output=True, text=True, encoding="utf-8",
- errors="replace", timeout=5, cwd=workdir, check=False)
- if probe.returncode == 0 and probe.stdout.strip():
- repos.add(probe.stdout.strip())
- except Exception:
- continue
- return [r for r in sorted(repos) if (Path(r) / ".worktrees").is_dir()]
- def _maybe_run_worktree_maintenance() -> None:
- """Throttled worktree prune from the cron tick, on a daemon thread so the tick never waits on
- git. Same pruner as ``hermes -w`` startup (idle dirty/unpushed trees archived before removal,
- live trees untouched). Discovery is throttled per profile home, the prune once per repo across
- served profiles, and ``cron.scheduler_worktree_maintenance`` runs one sweep at a time per
- process. Runs in a copy of the caller's context (profile-home override); never raises."""
- global _last_worktree_maintenance_at
- import contextvars
- from hermes_constants import get_hermes_home
- home = str(get_hermes_home())
- now = time.monotonic()
- with _worktree_maintenance_lock:
- stamps = _last_worktree_maintenance_at or {}
- last = stamps.get(home)
- if last is not None and now - last < _WORKTREE_MAINTENANCE_INTERVAL_SECONDS:
- return
- _last_worktree_maintenance_at = {**stamps, home: now}
- def _run() -> None:
- from cron.scheduler_worktree_maintenance import run_sweep
- run_sweep(_worktree_maintenance_repos,
- interval_seconds=_WORKTREE_MAINTENANCE_INTERVAL_SECONDS)
- threading.Thread(target=contextvars.copy_context().run, args=(_run,), name="cron-worktree-prune",
- daemon=True).start()
- def _acquire_tick_lock(lock_file):
- """Open + non-blocking lock the tick file (fcntl / msvcrt). Returns the fd, or None on genuine
- contention. A real OSError (esp. EMFILE/ENFILE) must NOT pass as contention — the scheduler
- would look healthy while no job runs — so it is re-raised for the ticker to record a FAILED
- tick."""
- lock_fd = None
- try:
- # Cross-platform file locking: fcntl on Unix, msvcrt on Windows. Only genuine lock contention
- # (another ticker holds the lock) skips the tick silently. A real OSError — most importantly
- # EMFILE/ENFILE from fd exhaustion — must NOT be swallowed as "another instance holds the lock":
- # that previously made the scheduler appear healthy (tick returned 0, heartbeat recorded success)
- # while no job ever ran again (#87644).
- lock_fd = open(lock_file, "w", encoding="utf-8")
- if fcntl:
- fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- elif msvcrt:
- msvcrt.locking(lock_fd.fileno(), msvcrt.LK_NBLCK, 1)
- return lock_fd
- except OSError as exc:
- if lock_fd is not None:
- with contextlib.suppress(OSError):
- lock_fd.close()
- if _is_lock_contention_errno(exc):
- logger.info("Tick skipped — another instance holds the lock")
- return None
- if _is_fd_exhaustion(exc):
- # fd reclamation is the ticker loop's job (scheduler_provider.py); here would double it.
- logger.error(
- "Cron tick could not acquire tick lock: %s — scheduler will "
- "attempt fd reclamation and retry with backoff",
- exc)
- elif exc.errno not in store_health.UNWRITABLE_ERRNOS: # those degrade the store (tick caller)
- logger.error("Cron tick could not acquire tick lock: %s", exc)
- raise
- def _release_tick_lock(lock_fd) -> None:
- if fcntl:
- with contextlib.suppress((OSError, IOError)):
- fcntl.flock(lock_fd, fcntl.LOCK_UN)
- elif msvcrt:
- with contextlib.suppress((OSError, IOError)):
- msvcrt.locking(lock_fd.fileno(), msvcrt.LK_UNLCK, 1)
- lock_fd.close()
- def _maybe_reap_dead_owners() -> None:
- """Dead-owner reclaim: a run that died mid-flight would leave its row 'claimed' forever. Rows
- whose owner process is proved gone are released (_owner_is_live), as are rows whose live owner
- holds a claim older than the derived stale bound (the process is not killed). Throttled."""
- # Dead-owner claim reclaim (#86721): execution rows carry their owner pid + process start time, but
- # recovery previously ran only at scheduler STARTUP. A one-shot `hermes cron run` that claimed a job and
- # died mid-run (its runner thread lived in the exiting CLI process) left the row 'claimed' forever while
- # the long-lived gateway ticker kept running — blocking every future run of that job. Reap provably-dead
- # owners periodically so stale claims auto-clear without a gateway restart. Throttled so idle 60s ticks
- # don't pay a ledger connection every cycle (#33612).
- _reap_key = hermes_home_key(_get_hermes_home())
- _reap_now = time.monotonic()
- _last_reap = _last_dead_owner_reap_at.get(_reap_key)
- if (
- _last_reap is not None
- and _reap_now - _last_reap < _DEAD_OWNER_REAP_INTERVAL_SECONDS
- ):
- return
- _last_dead_owner_reap_at[_reap_key] = _reap_now
- try:
- from cron.executions import recover_interrupted_executions
- _reclaimed = recover_interrupted_executions()
- if _reclaimed:
- logger.warning(
- "Reclaimed %d cron execution(s) whose owner process died "
- "before reaching a terminal state (marked unknown)",
- _reclaimed)
- except Exception as _reap_exc:
- logger.debug("Dead-owner execution reclaim failed: %s", _reap_exc)
- def _sweep_stale_inflight_for_tick(due_jobs: list) -> None:
- """Bound the in-flight set BEFORE the dedup guard so a leaked claim is force-released now
- rather than eating every later fire until restart. Skipped when nothing is in flight."""
- _local_home = hermes_home_key(_get_hermes_home())
- if not any(key[0] == _local_home for key in _running_job_ids):
- return
- _sweep_jobs = due_jobs
- with contextlib.suppress(Exception):
- _inflight_ids = {key[1] for key in _running_job_ids if key[0] == _local_home}
- _due_ids = {j.get("id") for j in due_jobs if isinstance(j, dict)}
- if not _inflight_ids <= _due_ids:
- from cron.jobs import load_jobs as _load_all_jobs
- _sweep_jobs = _load_all_jobs()
- try:
- sweep_stale_inflight(_sweep_jobs)
- except Exception as e:
- logger.warning("Stale in-flight sweep failed: %s", e)
- def _resolve_max_parallel_workers() -> Optional[int]:
- """Max workers: env > config.yaml > unbounded (HERMES_CRON_MAX_PARALLEL=1 restores serial)."""
- try:
- _env_par = cron_env_setting("HERMES_CRON_MAX_PARALLEL").strip()
- if _env_par:
- return int(_env_par) or None
- except (ValueError, TypeError):
- logger.warning("Invalid HERMES_CRON_MAX_PARALLEL value; defaulting to unbounded")
- with contextlib.suppress(Exception):
- _ucfg = load_config() or {}
- _cfg_par = (_ucfg.get("cron", {}) if isinstance(_ucfg, dict) else {}).get("max_parallel_jobs")
- if _cfg_par is not None:
- return int(_cfg_par) or None
- return None
- def _sweep_mcp_orphans() -> None:
- """Reap MCP stdio orphans (only PIDs flagged by tools.mcp_tool._run_stdio's finally block);
- run AFTER jobs finish so live sessions are never touched."""
- try:
- from tools.mcp_tool_lifecycle import _kill_orphaned_mcp_children
- _kill_orphaned_mcp_children()
- except Exception as _e:
- logger.debug("Post-tick MCP orphan cleanup failed: %s", _e)
- def _process_due_job(job: dict, adapters, loop, verbose: bool) -> bool:
- """Run one due job via the shared ``run_one_job`` body."""
- # Claim only when the worker actually starts, so a queued lease can't expire first.
- try:
- claimed = claim_job_for_fire(job["id"], return_job=True)
- except OSError as exc:
- store_health.note_unwritable(exc, f"skipped job '{job.get('name') or job['id']}'", "claim", [job])
- settle_unstarted_execution(
- job["execution_id"], job["id"], f"Cron store unwritable; not started: {exc}")
- return False
- except BaseException as exc: # settle first, then surface the real error
- settle_unstarted_execution(
- job["execution_id"], job["id"], f"Fire claim failed: {type(exc).__name__}: {exc}")
- raise
- if not claimed:
- finish_execution(
- job["execution_id"], success=False, error="Fire claim lost; execution was not started.")
- return True
- # CAS returns the persisted record; bool fallback only for older test doubles.
- claimed_job = dict(claimed) if isinstance(claimed, dict) else dict(job)
- claimed_job["execution_id"] = job["execution_id"]
- claimed_job["_scheduled_instant"] = job.get("_scheduled_instant")
- return run_one_job(claimed_job, adapters=adapters, loop=loop, verbose=verbose)
- def _submit_with_guard(job: dict, pool: concurrent.futures.ThreadPoolExecutor, process_job):
- """Submit with the in-flight dedup guard; None if a prior tick's run is still in flight.
- Running-set membership is released in the worker's finally."""
- job_id = job["id"]
- job_label = job.get("name", job_id)
- def _clear_run_claim_best_effort() -> None:
- """Best-effort claim cleanup on dispatch-failure paths. Only one-shots carry a run_claim;
- clear_run_claim takes _jobs_lock + full load/save and can raise on degraded paths
- (shutdown, EMFILE) — a claim expiring at TTL beats crashing the tick.
- Only one-shot jobs carry a ``run_claim`` (stamped by get_due_jobs, #59229), so recurring jobs skip
- the call entirely — clear_run_claim acquires _jobs_lock (blocking cross-process flock) and does a
- full load_jobs read, and the dispatch-failure paths fire exactly when the process can least afford N
- pointless lock/read round-trips (interpreter shutdown, EMFILE). clear_run_claim itself does
- load_jobs/save_jobs file I/O; on those degraded paths it can raise, and these early-exits exist
- precisely to skip cleanly — a stale claim expiring at the TTL is a better outcome than crashing the
- tick (#86522).
- """
- _schedule = job.get("schedule")
- if not (isinstance(_schedule, dict) and _schedule.get("kind") == "once"):
- return
- try:
- clear_run_claim(job_id)
- except Exception as claim_err:
- logger.warning(
- "Could not clear run_claim for job '%s' after dispatch "
- "failure: %s (claim will expire at TTL)",
- job_label, claim_err)
- def _not_dispatched_shutdown() -> None:
- logger.warning("Job '%s' not dispatched — interpreter is shutting down", job_label)
- # During interpreter shutdown pool.submit raises; skip — the job fires on the next tick.
- # If the interpreter is finalizing (gateway SIGTERM / restart / OOM), scheduling any new delivery is
- # futile — asyncio.run and a fresh ThreadPoolExecutor both raise "cannot schedule new futures after
- # interpreter shutdown". Skip gracefully with a warning rather than emitting an ERROR traceback on every
- # restart-race (#58720, #55924).
- # A tick can race gateway teardown: once the interpreter is finalizing, ``pool.submit`` raises "cannot
- # schedule new futures after interpreter shutdown" and crashes the tick. Skip cleanly — the job stays
- # due and will fire on the next healthy tick (#58720, #55924).
- if _interpreter_shutting_down():
- _not_dispatched_shutdown()
- _clear_run_claim_best_effort()
- return None
- registration_owner = object()
- if not try_register_running_job(job_id, owner=registration_owner):
- logger.info("Job '%s' already running — skipping", job_label)
- return None
- # The home the claim was registered under. The pool worker's ``finally`` runs OUTSIDE
- # ``ctx.run``, where the per-profile cron scope is not bound, so releasing without it would
- # discard the LAUNCH home's key and leak every secondary profile's claim.
- _claim_home = _get_hermes_home()
- # Record the attempt before dispatch; recovery marks abandoned rows unknown (no retry).
- execution = None
- try:
- execution = create_execution(
- job_id, source="builtin", scheduled_instant=job.get("_scheduled_instant"))
- dispatched_job = dict(job, execution_id=execution["id"])
- note_cron_execution(dispatched_job)
- _ctx = contextvars.copy_context()
- except Exception as execution_err:
- # Release the claim so the next tick retries instead of wedging "already running".
- release_running_job(job_id, home=_claim_home, owner=registration_owner)
- _clear_run_claim_best_effort()
- if execution is not None: # the receipt was persisted; a later step failed
- settle_unstarted_execution(execution["id"], job_id, (
- f"Dispatch preparation failed: {type(execution_err).__name__}: {execution_err}"))
- logger.exception("Job '%s' not dispatched: %s failed: %s", job_label,
- "execution creation" if execution is None else "dispatch preparation",
- execution_err)
- return None
- def _run_and_release(j=dispatched_job, ctx=_ctx, home=_claim_home):
- try:
- return ctx.run(process_job, j)
- finally:
- release_running_job(j["id"], home=home, owner=registration_owner)
- try:
- fut = pool.submit(_run_and_release)
- except Exception as submit_err:
- release_running_job(job_id, home=_claim_home, owner=registration_owner)
- _clear_run_claim_best_effort()
- finish_execution(
- execution["id"], success=False, error=f"Executor dispatch failed: {submit_err}")
- if isinstance(submit_err, RuntimeError) and _interpreter_shutting_down(submit_err):
- _not_dispatched_shutdown()
- else:
- logger.error("Job '%s' not dispatched: %s", job_label, submit_err)
- return None
- with _running_lock:
- _submit_key = _inflight_key(job_id, _claim_home)
- if _running_registration_owners.get(_submit_key) is registration_owner:
- _running_futures[_submit_key] = fut
- return fut
- def _sweep_mcp_orphans_when_all_done(futures: list) -> None:
- """Async (gateway ticker) mode: sweep via a done-callback after the LAST job completes; sweep
- inline when nothing was dispatched (all skipped / no due jobs)."""
- if not futures:
- _sweep_mcp_orphans()
- return
- _remaining = [len(futures)]
- def _on_done(_f: concurrent.futures.Future) -> None:
- _remaining[0] -= 1
- with contextlib.suppress(Exception):
- _exc = _f.exception()
- if _exc is not None:
- logger.error(
- "Cron job future failed in async mode: %s", _exc,
- exc_info=(type(_exc), _exc, _exc.__traceback__))
- if _remaining[0] <= 0:
- _sweep_mcp_orphans()
- for _f in futures:
- _f.add_done_callback(_on_done)
- from cron.scheduler_tick import tick
- # ---------------------------------------------------------------------------
- # Split modules. Imported at the bottom (import cycle: they late-bind ``cron.scheduler`` as
- # ``_sched``). Only names this module itself calls; everything else lives in the split module.
- # ---------------------------------------------------------------------------
- from cron.scheduler_delivery import (
- _deliver_result, _delivery_lane_value, _normalize_deliver_value, _resolve_delivery_target,
- _resolve_delivery_targets,
- )
- from cron.scheduler_script import (
- _get_session_db_timeout, _run_job_script_with_claim_heartbeat, _start_heartbeat_thread,
- )
- from cron.scheduler_prompt import (
- _PROMPT_FRAME, _PROMPT_HEADING, _PROMPT_SEPARATOR, _RESPONSE_FRAME, _RESPONSE_HEADING,
- _RESPONSE_TERMINATOR, _block_and_pause_job, _build_job_prompt, _guard_job_credential_exfil,
- _job_doc_header, _parse_wake_gate,
- )
- from cron.scheduler_preflight import (
- BLOCKED_CONFIG_MARKER, BLOCKED_CONFIG_SILENT_MARKER, _cron_preflight_enabled,
- _empty_requested_mcp_toolsets, _is_transient_provider_resolve_error, _preflight_job_config,
- )
- # `python -m cron.scheduler` entry: MUST stay below the split-module imports so the worker /
- # tick paths see every name they need.
- if __name__ == "__main__":
- if "--external-worker-file" in sys.argv:
- finish_worker_boot() # may relaunch: before the payload is read and the ack published
- import argparse
- parser = argparse.ArgumentParser(add_help=False)
- parser.add_argument("--external-worker-file", type=Path, required=True)
- parser.add_argument("--ack-file", type=Path, required=True)
- args = parser.parse_args()
- # The gateway spawns this worker with stdout on DEVNULL and stderr on a
- # capture file it only reads back if we die before the ack; without a
- # log handler every adoption/ack failure below would otherwise be
- # invisible to the persistent log.
- try:
- from hermes_logging import setup_logging
- setup_logging(hermes_home=_get_hermes_home(), mode="cron")
- except Exception:
- pass
- raise SystemExit(
- 0 if _run_external_worker_payload(args.external_worker_file, args.ack_file) else 1
- )
- tick(verbose=True, headless=True)
|