jobs.py 156 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453
  1. # health: allow FILE_LINES -- security fix for #82010: distinguish an explicitly-empty toolset allowlist (fail closed, nothing allowed) from an absent one (no restriction); the added lines are minimal fail-closed branches at this existing chokepoint
  2. """Cron job storage: ~/.hermes/cron/jobs.json; output in
  3. ~/.hermes/cron/output/{job_id}/{timestamp}.md"""
  4. import contextlib
  5. import copy
  6. from contextvars import ContextVar
  7. from dataclasses import dataclass, field
  8. import json
  9. import logging
  10. import shutil
  11. import sys
  12. import threading
  13. import time
  14. import os
  15. import re
  16. import uuid
  17. # Cross-process advisory locking for jobs.json: fcntl (Unix) or msvcrt (Windows). If both are
  18. # absent, _jobs_lock() degrades to in-process locking rather than failing.
  19. try:
  20. import fcntl
  21. except ImportError: # pragma: no cover - non-Unix
  22. fcntl = None
  23. try:
  24. import msvcrt
  25. except ImportError: # pragma: no cover - non-Windows
  26. msvcrt = None
  27. # ``datetime.timezone`` is deliberately NOT imported: the per-job ``timezone`` parameter
  28. # (compute_next_run/create_job) would shadow it, and everything here uses ``UTC`` instead.
  29. from datetime import datetime, timedelta, UTC
  30. from pathlib import Path
  31. from agent.deadline import poll_until
  32. from hermes_constants import get_hermes_home
  33. from cron.constants import CLAIM_TTL_INACTIVITY_HEADROOM, FIRE_CLAIM_SKEW_SECONDS, FIRE_CLAIM_TTL_SECONDS
  34. from cron.env_settings import cron_env_setting
  35. from cron.jobs_invariants import _validate_job_mode_invariants, validate_schedule_knobs
  36. from cron.jobs_model_pin import _main_model_pin
  37. from cron import jobs_time
  38. from cron.scheduler_ownership import _claim_owner_is_dead
  39. from cron import extra_fire, store_health
  40. from typing import Optional, Dict, List, Any, Callable, Set, Tuple, Union, Collection
  41. logger = logging.getLogger(__name__)
  42. from hermes_time import now as _hermes_now
  43. from hermes_time import get_timezone
  44. from hermes_cli.observability.shared_metrics_gateway import record_cron_missed
  45. from utils import atomic_replace, atomic_write_text, fsync_directory, mkstemp_beside
  46. # croniter is imported lazily (slow import, only needed for cron exprs). HAS_CRONITER stays a
  47. # module attribute: a monkeypatched value wins because _ensure_croniter only probes while None
  48. # (and past `_croniter_retry_at`).
  49. croniter = None
  50. HAS_CRONITER: Optional[bool] = None
  51. # Monotonic deadline before the next croniter import probe after an ImportError (not latched).
  52. _CRONITER_RETRY_SECONDS = 60.0
  53. _croniter_retry_at: float = 0.0
  54. def _ensure_croniter() -> bool:
  55. """Import croniter on first use; honor a pre-set HAS_CRONITER override.
  56. An ImportError is NOT latched: caching False would pin a single transient failure
  57. (wrong interpreter, shadowed path) for the whole process lifetime, leaving every
  58. recurring job's next_run_at None until a gateway restart (#127182). Stay None and
  59. re-probe once ``_CRONITER_RETRY_SECONDS`` have passed (so a genuinely absent package does
  60. not cost a finder walk per call on every due-scan tick), and due-scan recovery re-arms the
  61. schedules as soon as the import succeeds. An explicit True/False override always wins."""
  62. global croniter, HAS_CRONITER, _croniter_retry_at
  63. if HAS_CRONITER is None and time.monotonic() >= _croniter_retry_at:
  64. try:
  65. from croniter import croniter as _croniter
  66. croniter = _croniter
  67. HAS_CRONITER = True
  68. except ImportError:
  69. _croniter_retry_at = time.monotonic() + _CRONITER_RETRY_SECONDS
  70. return bool(HAS_CRONITER)
  71. # --- Configuration ---
  72. # Cron is per-profile by design: anchor at get_hermes_home() (active profile home), NOT
  73. # get_default_hermes_root() — the shared root would funnel every profile's jobs into one jobs.json
  74. # and run them under the ticker's HERMES_HOME, leaking config/credentials/skills across profiles.
  75. # Each profile owns its own cron store under its own HERMES_HOME, and a profile-scoped gateway runs that
  76. # profile's jobs under that same HERMES_HOME — so a job authored in profile `coder` lives in
  77. # `~/.hermes/profiles/coder/cron/jobs.json` and executes with `coder`'s `.env`, `config.yaml`, and skills.
  78. # Do NOT change this to the default root: that re-breaks per-profile isolation. See also the dynamic
  79. # `_get_hermes_home()` / `_get_lock_paths()` resolution in cron/scheduler.py. See #4707.
  80. HERMES_DIR = get_hermes_home().resolve()
  81. # Default-profile fallback and compatibility surface for callers/tests. Cross-profile callers must
  82. # scope paths with use_cron_store() instead of mutating these process-wide.
  83. CRON_DIR = HERMES_DIR / "cron"
  84. JOBS_FILE = CRON_DIR / "jobs.json"
  85. # Heartbeat: touched every ticker loop so `hermes cron status` can tell the ticker THREAD is alive,
  86. # not just the gateway PROCESS; success = last tick that completed WITHOUT raising.
  87. # The gateway process and the (separate) ``hermes cron status`` process share it so status can tell whether
  88. # the ticker THREAD is alive, not just whether the gateway PROCESS exists — a ticker that dies silently
  89. # inside a live gateway would otherwise report healthy (#32612, #32895).
  90. TICKER_HEARTBEAT_FILE = CRON_DIR / "ticker_heartbeat"
  91. TICKER_SUCCESS_FILE = CRON_DIR / "ticker_last_success"
  92. # Single source of truth for the ticker interval (scheduler_provider.py) and the staleness
  93. # threshold in `hermes cron status` (hermes_cli/cron.py), so they never drift apart.
  94. TICKER_INTERVAL_SECONDS = 60
  95. # In-process lock for load_jobs→modify→save_jobs cycles; without it, parallel tick threads'
  96. # mark_job_run / advance_next_run calls clobber each other.
  97. _jobs_file_lock = threading.RLock()
  98. _jobs_lock_state = threading.local()
  99. _fire_fence_locks: dict[str, threading.RLock] = {}
  100. _fire_fence_locks_guard = threading.Lock()
  101. _fire_fence_lock_state = threading.local()
  102. # Upper bound on waiting for the cross-process .jobs.lock. Every cron function funnels through
  103. # _jobs_lock(), so blocking forever on a wedged sibling process would freeze the ticker and every
  104. # job. 30s is far above any legitimate critical section yet under one status-alarm threshold.
  105. _JOBS_LOCK_TIMEOUT_SECONDS = 30.0
  106. OUTPUT_DIR = CRON_DIR / "output"
  107. ONESHOT_GRACE_SECONDS = 120
  108. @dataclass(frozen=True)
  109. class _CronStorePaths:
  110. cron_dir: Path
  111. jobs_file: Path
  112. output_dir: Path
  113. @classmethod
  114. def for_dir(cls, cron_dir: Path) -> "_CronStorePaths":
  115. return cls(cron_dir, cron_dir / "jobs.json", cron_dir / "output")
  116. _cron_store_override: ContextVar[Optional[_CronStorePaths]] = ContextVar(
  117. "cron_store_override", default=None)
  118. class _SelfRemovalDelivery:
  119. """Mutable run-local marker shared with the agent's copied ContextVar context."""
  120. def __init__(self, job_id: str):
  121. self.job_id = job_id
  122. self.removed = False
  123. _self_removal_delivery: ContextVar[Optional[_SelfRemovalDelivery]] = ContextVar(
  124. "self_removal_delivery", default=None)
  125. @contextlib.contextmanager
  126. def self_removal_delivery_scope(job_id: str):
  127. """Permit this run's final delivery after it removes its own job record."""
  128. marker = _SelfRemovalDelivery(job_id)
  129. token = _self_removal_delivery.set(marker)
  130. try:
  131. yield marker
  132. finally:
  133. _self_removal_delivery.reset(token)
  134. def self_removal_delivery_allowed(job_id: str) -> bool:
  135. """Whether the active run deleted exactly its own job record and no record has since taken
  136. its id (a replacement record belongs to another owner, so that stays fail-closed)."""
  137. marker = _self_removal_delivery.get()
  138. if marker is None or marker.job_id != job_id or not marker.removed:
  139. return False
  140. return all(item.get("id") != job_id for item in load_jobs())
  141. # Import-time snapshot so deliberate re-pointing of CRON_DIR/JOBS_FILE/OUTPUT_DIR (the documented
  142. # escape hatch for tests/embedders) is distinguishable from the constants merely being stale.
  143. _IMPORT_STORE = _CronStorePaths(CRON_DIR, JOBS_FILE, OUTPUT_DIR)
  144. def _current_cron_store() -> _CronStorePaths:
  145. """Paths pinned to this execution context's profile. Precedence: (1) active use_cron_store()
  146. override; (2) deliberately re-pointed module constants; (3) the ACTIVE profile home via
  147. get_hermes_home(), so re-pointing HERMES_HOME after import uses ITS OWN store rather than the
  148. user's real jobs.json frozen at import; (4) import-time constants."""
  149. override = _cron_store_override.get()
  150. if override is not None:
  151. return override
  152. live_constants = _CronStorePaths(CRON_DIR, JOBS_FILE, OUTPUT_DIR)
  153. if live_constants != _IMPORT_STORE:
  154. return live_constants
  155. home = get_hermes_home().resolve()
  156. if home == HERMES_DIR:
  157. return live_constants
  158. return _CronStorePaths.for_dir(home / "cron")
  159. @contextlib.contextmanager
  160. def use_cron_store(home: str | Path):
  161. """Route cron storage to ``home`` without mutating process globals."""
  162. token = _cron_store_override.set(
  163. _CronStorePaths.for_dir(Path(home).expanduser().resolve() / "cron"))
  164. try:
  165. yield
  166. finally:
  167. _cron_store_override.reset(token)
  168. def get_cron_output_dir() -> Path:
  169. """Return the output directory for the active cron store context."""
  170. return _current_cron_store().output_dir
  171. # Fallback stale-recovery window for a one-shot's running-claim when HERMES_CRON_TIMEOUT=0
  172. # (unlimited, no bound to derive from); also the floor so a tiny timeout can't expire a claim
  173. # mid-run.
  174. ONESHOT_RUN_CLAIM_TTL_SECONDS = 1800
  175. # Derived TTL = inactivity timeout × CLAIM_TTL_INACTIVITY_HEADROOM (cron/constants.py). The TTL
  176. # only recovers a claim left by a tick that DIED mid-run.
  177. _DEFAULT_CRON_INACTIVITY_TIMEOUT = 600.0
  178. def _oneshot_run_claim_ttl_seconds() -> float:
  179. """One-shot running-claim TTL from ``HERMES_CRON_TIMEOUT``: unset/invalid → 600s → 1800s;
  180. ``0`` (unlimited) → the fixed floor; positive N → ``max(N * headroom, floor)``."""
  181. raw = cron_env_setting("HERMES_CRON_TIMEOUT").strip()
  182. try:
  183. timeout = float(raw) if raw else _DEFAULT_CRON_INACTIVITY_TIMEOUT
  184. except (ValueError, TypeError):
  185. timeout = _DEFAULT_CRON_INACTIVITY_TIMEOUT
  186. if timeout <= 0:
  187. return float(ONESHOT_RUN_CLAIM_TTL_SECONDS)
  188. return max(timeout * CLAIM_TTL_INACTIVITY_HEADROOM, float(ONESHOT_RUN_CLAIM_TTL_SECONDS))
  189. def _job_running_in_this_process(job_id: str) -> bool:
  190. """True when the scheduler in THIS process is still running ``job_id``: the run_claim TTL alone
  191. cannot distinguish "claiming tick died" from "alive but slow". Lazy import: scheduler imports
  192. us.
  193. Direct liveness signal for stale-entry recovery (#62002): the run_claim TTL alone cannot distinguish
  194. "the claiming tick died" from "the run is alive but slow" — a run stalled on network I/O (or a laptop
  195. that slept mid-run) legitimately outlives the TTL. The in-process ticker and the run share this process,
  196. so the scheduler's running set settles the common single-gateway case without any claim-age guesswork.
  197. Home-scoped: one process ticks every profile, so the host-wide union of bare job ids would let
  198. profile A's running ``daily-brief`` keep profile B's stale one-shot alive forever.
  199. """
  200. try:
  201. from cron.scheduler import is_job_running
  202. return is_job_running(job_id)
  203. except Exception:
  204. logger.warning(
  205. "Cron running-set liveness check failed for job %r; keeping the "
  206. "entry to avoid deleting a possibly live one-shot run",
  207. job_id, exc_info=True)
  208. return True
  209. def _jobs_lock_file() -> Path:
  210. """Return the advisory lock path for the current cron directory."""
  211. return _current_cron_store().cron_dir / ".jobs.lock"
  212. def _acquire_flock(lock_fd, timeout: float) -> Optional[bool]:
  213. """Bounded exclusive lock: True when acquired, False on timeout, None when no backend exists. A
  214. blocking flock(LOCK_EX) taken under the in-process lock would let a wedged sibling freeze
  215. EVERY cron function forever, so poll LOCK_NB against a deadline; the caller picks the
  216. degraded mode."""
  217. if fcntl is not None:
  218. def _try_flock() -> bool:
  219. try:
  220. fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  221. return True
  222. except OSError:
  223. return False
  224. return poll_until(_try_flock, timeout, 0.1)
  225. if msvcrt is not None:
  226. msvcrt.locking(lock_fd.fileno(), msvcrt.LK_LOCK, 1)
  227. return True
  228. return None
  229. def _release_flock(lock_fd) -> None:
  230. """Unlock (best effort) and close a lock file opened for ``_acquire_flock``."""
  231. try:
  232. if fcntl is not None:
  233. fcntl.flock(lock_fd, fcntl.LOCK_UN)
  234. elif msvcrt is not None:
  235. msvcrt.locking(lock_fd.fileno(), msvcrt.LK_UNLCK, 1)
  236. except OSError:
  237. pass
  238. finally:
  239. lock_fd.close()
  240. @contextlib.contextmanager
  241. def _jobs_lock():
  242. """Serialize a load_jobs→modify→save_jobs critical section: in-process RLock (parallel tick
  243. threads) plus a cross-process flock on ``<cron dir>/.jobs.lock`` (gateway vs. CLI writes —
  244. otherwise a `cron pause` could be clobbered and keep firing). Nested calls in one thread
  245. reuse the held lock. Without a flock backend, or on flock timeout (logged loudly), it
  246. degrades to in-process-only locking: a briefly torn cross-process write beats a dead
  247. scheduler."""
  248. depth = getattr(_jobs_lock_state, "depth", 0)
  249. if depth:
  250. _jobs_lock_state.depth = depth + 1
  251. try:
  252. yield
  253. finally:
  254. _jobs_lock_state.depth -= 1
  255. return
  256. with _jobs_file_lock:
  257. _jobs_lock_state.depth = 1
  258. # jobs.json stamp as of this section's load_jobs(): lets _save_jobs_unlocked skip the
  259. # shrink-merge parse when the file provably hasn't changed. Reset on entry/exit so stale
  260. # stamps from unlocked loads or prior sections can never suppress a needed merge.
  261. # See #80703.
  262. _jobs_lock_state.load_stamp = None
  263. lock_fd = None
  264. try:
  265. try:
  266. ensure_dirs()
  267. lock_fd = open(_jobs_lock_file(), "a+", encoding="utf-8-sig")
  268. lock_fd.seek(0)
  269. if _acquire_flock(lock_fd, _JOBS_LOCK_TIMEOUT_SECONDS) is False:
  270. logger.error(
  271. "Timed out after %.0fs waiting for the cron "
  272. "jobs lock (%s) — another process is holding "
  273. "it. Proceeding with in-process locking only "
  274. "so the scheduler stays alive (#60703).",
  275. _JOBS_LOCK_TIMEOUT_SECONDS, _jobs_lock_file())
  276. with contextlib.suppress(OSError):
  277. lock_fd.close()
  278. lock_fd = None
  279. except OSError as e:
  280. # A locking failure must never take down cron writes — in-process lock still held.
  281. logger.warning("jobs.json cross-process lock unavailable (%s); "
  282. "proceeding with in-process lock only", e)
  283. try:
  284. yield
  285. finally:
  286. if lock_fd is not None:
  287. _release_flock(lock_fd)
  288. finally:
  289. _jobs_lock_state.depth = 0
  290. _jobs_lock_state.load_stamp = None
  291. @contextlib.contextmanager
  292. def _fire_job_lock(job_id: str):
  293. """Serialize one job's owner mutations and external side effects. Unlike the global jobs lock
  294. this may be held across network delivery; scoped to one profile + job so unrelated jobs keep
  295. progressing. Fails closed when cross-process locking is unavailable."""
  296. cron_dir = _current_cron_store().cron_dir
  297. lock_key = f"{cron_dir.resolve()}::{job_id}"
  298. with _fire_fence_locks_guard:
  299. local_lock = _fire_fence_locks.setdefault(lock_key, threading.RLock())
  300. if not local_lock.acquire(timeout=_JOBS_LOCK_TIMEOUT_SECONDS):
  301. logger.error("Timed out waiting for local fire fence %s; failing closed", lock_key)
  302. yield False
  303. return
  304. held_locks = _fire_fence_lock_state.__dict__.setdefault("held", {})
  305. if lock_key in held_locks:
  306. try:
  307. yield held_locks[lock_key]
  308. finally:
  309. local_lock.release()
  310. return
  311. try:
  312. ensure_dirs()
  313. lock_path = cron_dir / f".fire-{uuid.uuid5(uuid.NAMESPACE_URL, lock_key).hex}.lock"
  314. lock_fd = None
  315. acquired = False
  316. try:
  317. lock_fd = open(lock_path, "a+", encoding="utf-8")
  318. lock_fd.seek(0)
  319. result = _acquire_flock(lock_fd, _JOBS_LOCK_TIMEOUT_SECONDS)
  320. if result is None: # pragma: no cover - supported platforms provide one backend
  321. logger.error("No cross-process lock backend for cron fire fence")
  322. elif not result:
  323. logger.error("Timed out waiting for fire fence %s; failing closed", lock_path)
  324. acquired = bool(result)
  325. except OSError as exc:
  326. logger.error("Cron fire fence unavailable for %s: %s", job_id, exc)
  327. held_locks[lock_key] = acquired
  328. try:
  329. yield acquired
  330. finally:
  331. held_locks.pop(lock_key, None)
  332. if lock_fd is not None:
  333. if acquired:
  334. _release_flock(lock_fd)
  335. else:
  336. lock_fd.close()
  337. finally:
  338. local_lock.release()
  339. def _under_fire_fence(job_id: str, fn: Callable[[], Any]) -> Any:
  340. """Run ``fn()`` holding the job's fire fence; False (fail closed) when it can't be acquired."""
  341. with _fire_job_lock(job_id) as acquired:
  342. if not acquired:
  343. return False
  344. return fn()
  345. @contextlib.contextmanager
  346. def fire_claim_fence(job_id: str, *, expected_owner: str):
  347. """Hold a per-job fence while an owner performs an external side effect. A missing record
  348. is accepted only for the active run that removed this exact job (#111039)."""
  349. with _fire_job_lock(job_id) as acquired:
  350. if not acquired:
  351. yield False
  352. return
  353. with _jobs_lock():
  354. job = next((item for item in load_jobs() if item.get("id") == job_id), None)
  355. claim = job.get("fire_claim") if isinstance(job, dict) else None
  356. owns_claim = isinstance(claim, dict) and claim.get("by") == expected_owner
  357. if job is None:
  358. owns_claim = self_removal_delivery_allowed(job_id)
  359. yield owns_claim
  360. # Fields that must never change after creation: ``id`` is a path component under OUTPUT_DIR, so an
  361. # update could leak ``../escape``/absolute/nested values into output writes/deletes.
  362. _IMMUTABLE_JOB_FIELDS = frozenset({"id"})
  363. def _job_output_dir(job_id: str) -> Path:
  364. """Resolve a job's output directory, rejecting any path-escape attempt (``..``, absolute
  365. paths, separators): only a single safe path component is accepted."""
  366. text = str(job_id or "").strip()
  367. if (
  368. not text or text in {".", ".."} or "/" in text or "\\" in text
  369. or Path(text).is_absolute() or Path(text).drive
  370. ):
  371. raise ValueError(f"Invalid cron job id for output path: {job_id!r}")
  372. return _current_cron_store().output_dir / text
  373. def _normalize_skill_list(skill: Optional[str] = None, skills: Optional[Any] = None) -> list[str]:
  374. """Normalize legacy/single-skill and multi-skill inputs into a unique ordered list."""
  375. if skills is None:
  376. raw_items = [skill] if skill else []
  377. elif isinstance(skills, str):
  378. raw_items = [skills]
  379. else:
  380. raw_items = list(skills)
  381. normalized: list[str] = []
  382. for item in raw_items:
  383. text = str(item or "").strip()
  384. if text and text not in normalized:
  385. normalized.append(text)
  386. return normalized
  387. def _apply_skill_fields(job: dict[str, Any]) -> dict[str, Any]:
  388. """Return a job dict with canonical `skills` and legacy `skill` fields aligned."""
  389. normalized = dict(job)
  390. skills = _normalize_skill_list(normalized.get("skill"), normalized.get("skills"))
  391. normalized["skills"] = skills
  392. normalized["skill"] = skills[0] if skills else None
  393. return normalized
  394. def _coerce_job_text(value: Any, fallback: str = "") -> str:
  395. """Coerce legacy/hand-edited nullable cron fields to strings for readers."""
  396. return fallback if value is None else str(value)
  397. # Fields whose presence in an update can turn a runnable job into an empty one.
  398. _PAYLOAD_FIELDS = frozenset({"prompt", "script", "skill", "skills", "no_agent"})
  399. EMPTY_PAYLOAD_ERROR = (
  400. "Cron job has nothing to run: the prompt is blank and no script or "
  401. "skill(s) are set. Provide a prompt, a script, or at least one skill."
  402. )
  403. def job_payload_is_empty(job: dict[str, Any]) -> bool:
  404. """True when a job record has nothing runnable (blank prompt, no script, no skills) AND at
  405. least one payload field is explicitly present. ``no_agent`` already requires a script."""
  406. if _coerce_job_text(job.get("prompt")).strip() or _coerce_job_text(job.get("script")).strip():
  407. return False
  408. if _normalize_skill_list(job.get("skill"), job.get("skills")):
  409. return False
  410. return any(k in job for k in ("prompt", "script", "skill", "skills"))
  411. def _schedule_display_for_job(job: dict[str, Any]) -> str:
  412. display = _coerce_job_text(job.get("schedule_display")).strip()
  413. if display:
  414. return display
  415. schedule = job.get("schedule")
  416. if isinstance(schedule, dict):
  417. for key in ("display", "value", "expr", "run_at"):
  418. text = _coerce_job_text(schedule.get(key)).strip()
  419. if text:
  420. return text
  421. elif schedule is not None:
  422. return str(schedule)
  423. return "?"
  424. def _normalize_job_record(job: dict[str, Any]) -> dict[str, Any]:
  425. """Read-safe job shape: legacy/hand-edited records may have nullable ``prompt``, ``name``,
  426. ``schedule_display``. Storage is untouched; consumers never crash on formatting."""
  427. normalized = _apply_skill_fields(job)
  428. job_id = normalized["id"] = _coerce_job_text(normalized.get("id"), "unknown")
  429. prompt = normalized["prompt"] = _coerce_job_text(normalized.get("prompt"))
  430. name = _coerce_job_text(normalized.get("name")).strip()
  431. if not name:
  432. label_source = (
  433. prompt
  434. or (normalized["skills"][0] if normalized.get("skills") else "")
  435. or _coerce_job_text(normalized.get("script")).strip()
  436. or job_id
  437. or "cron job"
  438. )
  439. name = label_source[:50].strip() or "cron job"
  440. normalized["name"] = name
  441. normalized["schedule_display"] = _schedule_display_for_job(normalized)
  442. # Derived from the scheduler-honoured ``enabled`` flag so a half-paused record cannot render
  443. # "paused" while still firing. See effective_job_state().
  444. normalized["state"] = effective_job_state(normalized)
  445. return normalized
  446. def _has_pause_marker(job: dict[str, Any]) -> bool:
  447. """True when the record carries any operator-facing pause signal."""
  448. return _coerce_job_text(job.get("state")).strip() == "paused" or bool(job.get("paused_at"))
  449. def is_job_runnable(job: dict[str, Any]) -> bool:
  450. """True iff the scheduler may fire this job: ``enabled`` plus pause markers as a second gate so
  451. a contradictory half-paused record never fires even before self-heal runs."""
  452. return bool(job.get("enabled", True)) and not _has_pause_marker(job)
  453. def effective_job_state(job: dict[str, Any]) -> str:
  454. """Operator-facing state derived from ``enabled``: an enabled job must never display as paused
  455. (list looked frozen while jobs kept firing). Terminal states are preserved regardless."""
  456. stored = _coerce_job_text(job.get("state")).strip()
  457. if stored in {"completed", "error"}:
  458. return stored
  459. if not job.get("enabled", True):
  460. if _has_pause_marker(job) or stored == "paused":
  461. return "paused"
  462. return stored or "paused"
  463. # enabled=true is authoritative: never claim paused
  464. if stored == "paused" or job.get("paused_at"):
  465. return "scheduled"
  466. return stored or "scheduled"
  467. def is_terminal_job(job: dict[str, Any]) -> bool:
  468. """Return whether a job record is in a terminal scheduler state."""
  469. return job.get("state") in {"completed", "error"}
  470. def _is_recoverable_error_job(job: dict[str, Any]) -> bool:
  471. """True for a recurring job stuck in ``state=error`` (set ONLY when ``compute_next_run()`` fails
  472. for a cron/interval job: croniter missing, malformed schedule). Such a job still has future
  473. occurrences once the issue resolves, so treating it as terminal would block due-scan self-heal,
  474. pre-advance, dispatch claim and ``resume_job`` — wedging it forever.
  475. Unlike ``state=completed`` (a one-shot that genuinely has no more occurrences, ever), an error-state
  476. recurring job still has a schedule with future occurrences once the underlying issue resolves — it is
  477. stuck pending a ``next_run_at`` recompute, not truly done. See #16265.
  478. """
  479. return (
  480. job.get("state") == "error"
  481. and (job.get("schedule") or {}).get("kind") in {"cron", "interval"}
  482. )
  483. def _secure_dir(path: Path):
  484. """Owner-only (0700) via the shared helper, so cron/ and cron/output honor the same managed/
  485. container/HERMES_HOME_MODE rules as the rest of HERMES_HOME (#10757)."""
  486. from hermes_cli.config import _secure_dir as _shared_secure_dir
  487. _shared_secure_dir(path)
  488. def _secure_file(path: Path):
  489. """Owner-only (0600) via the shared helper (managed/container skip included)."""
  490. from hermes_cli.config import _secure_file as _shared_secure_file
  491. _shared_secure_file(path)
  492. def _preserve_file_ownership(path: Path, before: Optional[os.stat_result]) -> None:
  493. """Restore a rewritten file's previous owner (POSIX, root writer only): atomic replace makes the
  494. file owned by the writer's euid, so a root CLI write (e.g. ``docker exec``) against the
  495. unprivileged gateway's store would flip jobs.json to root:root 0600 and lock the ticker out."""
  496. if before is None or os.name != "posix":
  497. return
  498. geteuid = getattr(os, "geteuid", None)
  499. getegid = getattr(os, "getegid", None)
  500. if geteuid is None or getegid is None:
  501. return
  502. try:
  503. euid = geteuid()
  504. if euid != 0 or (before.st_uid, before.st_gid) == (euid, getegid()):
  505. return # unprivileged writer, or already ours before the rewrite
  506. os.chown(path, before.st_uid, before.st_gid)
  507. except OSError as e:
  508. logger.warning(
  509. "Could not restore ownership of %s to uid=%s gid=%s after rewrite: %s "
  510. "— if the gateway runs as a different user, its cron ticker may now "
  511. "be locked out (see issue #68483).",
  512. path, before.st_uid, before.st_gid, e)
  513. def _is_named_profile_path(path: Path) -> bool:
  514. """True if *path* is under ``<hermes_home>/profiles/<name>/`` (default/custom homes are not).
  515. Checks the resolved path (symlinked parents) and the raw path (symlinked profile homes)."""
  516. with contextlib.suppress(OSError, RuntimeError):
  517. if "profiles" in path.resolve().parts:
  518. return True
  519. return "profiles" in path.parts
  520. def _ensure_cron_dir(cron_dir: Path) -> None:
  521. """Create a cron directory without resurrecting a deleted profile home: a stale multiplex
  522. scheduler may still hold a deleted profile's path, so named profiles use ``parents=False`` and
  523. fail closed. Default/custom homes keep ``parents=True`` so first-run creation works."""
  524. if _is_named_profile_path(cron_dir):
  525. cron_dir.mkdir(exist_ok=True)
  526. return
  527. cron_dir.mkdir(parents=True, exist_ok=True)
  528. def ensure_dirs():
  529. """Ensure cron directories exist with secure permissions."""
  530. store = _current_cron_store()
  531. _ensure_cron_dir(store.cron_dir)
  532. _ensure_cron_dir(store.output_dir)
  533. _secure_dir(store.cron_dir)
  534. _secure_dir(store.output_dir)
  535. # --- Schedule Parsing ---
  536. def normalize_repeat_value(repeat: Any) -> Optional[int]:
  537. """Coerce a repeat value (int or user-facing string) into ``Optional[int]``:
  538. ``'forever'``-family -> None, ``'once'``-family -> 1, numeric -> int, 0/negative -> None,
  539. else ValueError.
  540. The tool schema exposes ``repeat`` as an integer, but agents and users legitimately pass the user-facing
  541. strings ``'forever'``/``'once'`` or numeric strings (``'3'``). Uncoerced strings previously died with
  542. ``'<=' not supported between instances of 'str' and 'int'`` at create
  543. (#66824/#64520/#7142/#71987/#95706) and were stored raw by update paths, breaking ``mark_job_run``
  544. later.
  545. """
  546. if repeat is None:
  547. return None
  548. if isinstance(repeat, str):
  549. repeat_str = repeat.strip().lower()
  550. if repeat_str in ("forever", "infinite", "inf", "none", ""):
  551. return None
  552. if repeat_str in ("once", "one", "1x"):
  553. return 1
  554. try:
  555. repeat = int(repeat_str)
  556. except ValueError:
  557. raise ValueError(
  558. f"Invalid repeat value {repeat!r}: use an integer, "
  559. f"'forever', or 'once'."
  560. )
  561. return None if repeat <= 0 else int(repeat)
  562. _DURATION_MULTIPLIERS = {'m': 1, 'h': 60, 'd': 1440}
  563. def parse_duration(s: str) -> int:
  564. """Parse a duration into minutes: "30m" → 30, "2h" → 120, "1d" → 1440, bare "hour" → 60."""
  565. s = s.strip().lower()
  566. match = re.match(r'^(\d*)\s*(m|min|mins|minute|minutes|h|hr|hrs|hour|hours|d|day|days)$', s)
  567. if not match:
  568. raise ValueError(
  569. f"Invalid duration: '{s}'. Use format like '30m', '2h', '1d', "
  570. "or a bare unit like 'hour' (defaults to 1).")
  571. value = int(match.group(1)) if match.group(1) else 1
  572. return value * _DURATION_MULTIPLIERS[match.group(2)[0]]
  573. # Day-spec phrases for "every monday 9am" / "every day at 9am". Cron weekday numbering is
  574. # 0=Sunday … 6=Saturday (croniter's default).
  575. _WEEKDAY_TO_CRON_DOW = {
  576. "sunday": "0", "sun": "0",
  577. "monday": "1", "mon": "1",
  578. "tuesday": "2", "tue": "2", "tues": "2",
  579. "wednesday": "3", "wed": "3", "weds": "3",
  580. "thursday": "4", "thu": "4", "thur": "4", "thurs": "4",
  581. "friday": "5", "fri": "5",
  582. "saturday": "6", "sat": "6",
  583. }
  584. # Keyword day-specs that expand to a cron weekday field.
  585. _DAYSPEC_TO_CRON_DOW = {
  586. "day": "*", "daily": "*", "everyday": "*",
  587. "weekday": "1-5", "weekdays": "1-5",
  588. "weekend": "0,6", "weekends": "0,6",
  589. }
  590. def _parse_clock_time(text: str) -> Optional[tuple]:
  591. """Parse ``9am``/``9:30am``/``14:00``/``7`` (bare 24h hour)/``noon``/``midnight`` into a
  592. 24-hour ``(hour, minute)`` tuple, or None when unrecognized."""
  593. t = text.strip().lower().replace(" ", "")
  594. if not t:
  595. return None
  596. if t in ("noon", "midday"):
  597. return (12, 0)
  598. if t == "midnight":
  599. return (0, 0)
  600. match = re.match(r'^(\d{1,2})(?::(\d{2}))?(am|pm)?$', t)
  601. if not match:
  602. return None
  603. hour = int(match.group(1))
  604. minute = int(match.group(2) or 0)
  605. meridiem = match.group(3)
  606. if meridiem:
  607. if not 1 <= hour <= 12:
  608. return None
  609. hour = hour % 12 + (12 if meridiem == "pm" else 0)
  610. if hour > 23 or minute > 59:
  611. return None
  612. return (hour, minute)
  613. def _natural_every_to_cron(rest: str) -> Optional[str]:
  614. """Convert ``<when> [at] <time>`` ("monday 9am", "weekday at 9am", "monday, wednesday at 9am")
  615. to a 5-field cron expr, or None so ``parse_schedule`` can fall back to the interval path."""
  616. tokens = rest.lower().replace(",", " ").split()
  617. if not tokens:
  618. return None
  619. # Leading day tokens: a keyword spec ("weekdays") or a comma/"and"-separated weekday list.
  620. dow = _DAYSPEC_TO_CRON_DOW.get(tokens[0])
  621. idx = 1
  622. if dow is None:
  623. days = []
  624. idx = len(tokens)
  625. for i, tok in enumerate(tokens):
  626. if tok == "and":
  627. continue
  628. mapped = _WEEKDAY_TO_CRON_DOW.get(tok)
  629. if mapped is None:
  630. idx = i
  631. break
  632. if mapped not in days:
  633. days.append(mapped)
  634. if not days:
  635. return None
  636. dow = ",".join(days)
  637. time_tokens = tokens[idx:]
  638. if time_tokens and time_tokens[0] == "at": # optional separator: "every day at 9am"
  639. time_tokens = time_tokens[1:]
  640. if not time_tokens:
  641. return None
  642. parsed = _parse_clock_time(" ".join(time_tokens))
  643. if parsed is None:
  644. return None
  645. hour, minute = parsed
  646. return f"{minute} {hour} * * {dow}"
  647. def _cron_schedule(
  648. expr: str, display: str, missing_croniter: str, invalid_label: str
  649. ) -> dict[str, Any]:
  650. """Validate a cron expression with croniter and build the stored schedule dict."""
  651. if not _ensure_croniter():
  652. raise ValueError(f"{missing_croniter} Install with: pip install croniter")
  653. try:
  654. croniter(expr)
  655. except Exception as e:
  656. raise ValueError(f"Invalid {invalid_label} '{display}': {e}")
  657. return {"kind": "cron", "expr": expr, "display": display}
  658. def _interval_schedule(minutes: int) -> dict[str, Any]:
  659. return {"kind": "interval", "minutes": minutes, "display": f"every {minutes}m"}
  660. def parse_schedule(schedule: str) -> dict[str, Any]:
  661. """Parse a schedule string into ``{"kind": "once"|"interval"|"cron", ...}`` with ``run_at`` /
  662. ``minutes`` / ``expr``. "30m" and "every 30m" are recurring intervals; "every monday 9am" and
  663. "0 9 * * *" are cron; an ISO timestamp is once."""
  664. schedule = schedule.strip()
  665. original = schedule
  666. schedule_lower = schedule.lower()
  667. # Natural day/time phrase → cron ("every monday 9am", or sans prefix "weekdays at 9am");
  668. # any other "every X" → recurring interval.
  669. is_every = schedule_lower.startswith("every ")
  670. rest = schedule[6:].strip() if is_every else schedule_lower
  671. cron_expr = _natural_every_to_cron(rest)
  672. # Reuse the same helper — the phrase shape is identical without the "every " prefix. See #51975.
  673. if cron_expr is not None:
  674. example = "every monday 9am" if is_every else "weekdays at 9am"
  675. return _cron_schedule(
  676. cron_expr, original,
  677. f"Weekday/time schedules like '{example}' require the 'croniter' package.", "schedule")
  678. if is_every:
  679. return _interval_schedule(parse_duration(rest))
  680. # Cron expression (5-6 fields). Letters are allowed so named months/weekdays (JAN-DEC, MON-FRI)
  681. # reach croniter, which supports them.
  682. parts = schedule.split()
  683. if len(parts) >= 5 and all(re.match(r'^[A-Za-z\d\*\-,/]+$', p) for p in parts[:5]):
  684. return _cron_schedule(
  685. schedule, schedule, "Cron expressions require 'croniter' package.", "cron expression")
  686. # ISO timestamp (contains T or looks like date)
  687. if 'T' in schedule or re.match(r'^\d{4}-\d{2}-\d{2}', schedule):
  688. try:
  689. dt = datetime.fromisoformat(schedule)
  690. # Naive timestamps become aware in the CONFIGURED Hermes timezone (not server-local):
  691. # the due-check compares against hermes_time.now().
  692. # Make naive timestamps timezone-aware at parse time so the stored value doesn't depend on the
  693. # system timezone matching at check time. UTC) while now() runs in Asia/Kolkata, the stored
  694. # instant would land hours off from the user's wall-clock intent — far enough that one-shots
  695. # never become due and recurring jobs fire at the wrong time. Using the configured zone makes
  696. # "20:07" mean 20:07 on the same clock the scheduler checks against (#51021).
  697. if dt.tzinfo is None:
  698. dt = dt.replace(tzinfo=_hermes_now().tzinfo)
  699. return {
  700. "kind": "once",
  701. "run_at": dt.isoformat(),
  702. "display": f"once at {dt.strftime('%Y-%m-%d %H:%M')}"
  703. }
  704. except ValueError as e:
  705. raise ValueError(f"Invalid timestamp '{schedule}': {e}")
  706. # "in 30m"/"in 2h" is the explicit one-shot-by-duration form; a bare duration ("30m") is a
  707. # RECURRING interval per the documented tool contract.
  708. if schedule_lower.startswith("in "):
  709. duration_str = schedule[3:].strip()
  710. try:
  711. minutes = parse_duration(duration_str)
  712. except ValueError:
  713. raise ValueError(
  714. f"Invalid duration '{duration_str}' after 'in '. Use e.g. 'in 30m', 'in 2h'.")
  715. now = _hermes_now()
  716. # Durations measure elapsed time, not wall-clock hours across a DST transition.
  717. run_at = (now.astimezone(UTC) + timedelta(minutes=minutes)).astimezone(now.tzinfo)
  718. return {"kind": "once", "run_at": run_at.isoformat(), "display": f"once in {duration_str}"}
  719. with contextlib.suppress(ValueError):
  720. return _interval_schedule(parse_duration(schedule))
  721. raise ValueError(
  722. f"Invalid schedule '{original}'. Use:\n"
  723. f" - Interval: '30m', 'every 30m', 'every 2h' (recurring)\n"
  724. f" - One-shot delay: 'in 30m', 'in 2h' (fires once)\n"
  725. f" - Weekly/daily: 'every monday 9am', 'weekdays at 9am' (recurring)\n"
  726. f" - Cron: '0 9 * * *' (cron expression)\n"
  727. f" - Timestamp: '2026-02-03T14:00:00' (one-shot at time)"
  728. )
  729. def _ensure_aware(dt: datetime, target_tz: Optional[Any] = None) -> datetime:
  730. """Aware datetime in *target_tz*, defaulting to the configured Hermes timezone. Legacy naive
  731. values are read as *system-local* wall time (what created them) then converted, preserving
  732. ordering across timezone changes and avoiding false not-due results.
  733. The due scan passes the JOB's pinned zone here: every downstream cron predicate compares wall
  734. clocks, so normalizing a pinned job's instant into the profile zone makes a due occurrence
  735. look still-in-the-future and the repair path re-anchors past it (job lost, never fired)."""
  736. target_tz = target_tz or _hermes_now().tzinfo
  737. if dt.tzinfo is None:
  738. return dt.replace(tzinfo=datetime.now().astimezone().tzinfo).astimezone(target_tz)
  739. return dt.astimezone(target_tz)
  740. def _elapsed_seconds(later: datetime, earlier: datetime) -> float:
  741. """Return elapsed seconds between aware instants, independent of wall time."""
  742. return (later.astimezone(UTC) - earlier.astimezone(UTC)).total_seconds()
  743. def _instant_after(left: datetime, right: datetime) -> bool:
  744. """Whether *left* is a later absolute instant than *right*."""
  745. return left.astimezone(UTC) > right.astimezone(UTC)
  746. def _instant_at_or_before(left: datetime, right: datetime) -> bool:
  747. """Whether *left* is at or before *right* as an absolute instant."""
  748. return left.astimezone(UTC) <= right.astimezone(UTC)
  749. def _instant_before(left: datetime, right: datetime) -> bool:
  750. """Whether *left* is an earlier absolute instant than *right*."""
  751. return left.astimezone(UTC) < right.astimezone(UTC)
  752. def _seconds_after(dt: datetime, seconds: float) -> datetime:
  753. """*dt* plus real *seconds*, in *dt*'s zone. Aware ``+ timedelta`` is wall-clock arithmetic
  754. that drops ``fold``, so inside a fall-back hour it lands an hour off."""
  755. return (dt.astimezone(UTC) + timedelta(seconds=seconds)).astimezone(dt.tzinfo)
  756. def _parse_aware(value: Any, zone: Optional[Any] = None) -> Optional[datetime]:
  757. """``_ensure_aware(datetime.fromisoformat(value), zone)``, or None when *value* is not a
  758. parseable ISO string."""
  759. try:
  760. return _ensure_aware(datetime.fromisoformat(value), zone)
  761. except Exception:
  762. return None
  763. def _timezone_offset_mismatch(stored: datetime, current: datetime) -> bool:
  764. """True when a stored aware timestamp uses a different UTC offset. Naive values return False:
  765. they are normalized by ``_ensure_aware`` and intentionally never take the offset-repair path."""
  766. if stored.tzinfo is None or current.tzinfo is None:
  767. return False
  768. return stored.utcoffset() != current.utcoffset()
  769. def _stored_wall_clock_is_future(stored: datetime, current: datetime) -> bool:
  770. """True when the stored local wall-clock time has not arrived yet. Cron expresses wall-clock
  771. intent; after a timezone change an old offset can make a future run look due (21:00+10 →
  772. 13:00+02). Comparing naive wall clocks separates that from a genuine miss."""
  773. return stored.replace(tzinfo=None) > current.replace(tzinfo=None)
  774. def _recoverable_oneshot_run_at(
  775. schedule: dict[str, Any], now: datetime, *, last_run_at: Optional[str] = None,
  776. zone: Optional[Any] = None,
  777. ) -> Optional[str]:
  778. """One-shot run time if still eligible: a small grace window covers jobs created just after
  779. their minute; once run, a one-shot is never eligible again. *zone* is the job's pinned zone
  780. (an offset-less ``run_at`` is that zone's wall clock)."""
  781. if not isinstance(schedule, dict) or schedule.get("kind") != "once" or last_run_at:
  782. return None
  783. run_at = schedule.get("run_at")
  784. run_at_dt = jobs_time.oneshot_instant(run_at, zone) if run_at else None
  785. if run_at_dt is not None and _elapsed_seconds(now, run_at_dt) <= ONESHOT_GRACE_SECONDS:
  786. return run_at
  787. return None
  788. _MIN_GRACE_SECONDS = 120
  789. _MAX_GRACE_SECONDS = 7200
  790. def _compute_grace_seconds(schedule: dict, grace_minutes: Any = None) -> int:
  791. """How late a job can be and still catch up rather than fast-forward: half the period, clamped
  792. to [120s, 2h], so daily jobs catch up but frequent jobs fast-forward quickly.
  793. A per-job ``grace_minutes`` overrides that window (0 = never catch up), floored at the
  794. 120s tick tolerance: the ticker always fires a few seconds after the slot, so a 0s window
  795. would classify every on-time run as missed."""
  796. override = jobs_time.grace_seconds_override(grace_minutes)
  797. if override is not None:
  798. return max(override, _MIN_GRACE_SECONDS)
  799. period_seconds = _schedule_cadence_seconds(schedule)
  800. if not period_seconds:
  801. return _MIN_GRACE_SECONDS
  802. return max(_MIN_GRACE_SECONDS, min(int(period_seconds) // 2, _MAX_GRACE_SECONDS))
  803. # A recurring dispatch within this many seconds of schedule renders "on time": a busy once-a-minute
  804. # ticker can slip a couple of minutes — normal cadence, not gateway downtime.
  805. # See #99879.
  806. _LATE_DISPATCH_TOLERANCE_SECONDS = 300
  807. def _classify_dispatch_lateness(lateness_seconds: float, grace_seconds: int) -> str:
  808. """``on_time`` (within ticker slack), ``late`` (within the catch-up grace window), or
  809. ``catch_up`` (beyond grace; accumulated misses skipped, executed once now)."""
  810. if lateness_seconds > grace_seconds:
  811. return "catch_up"
  812. if lateness_seconds > _LATE_DISPATCH_TOLERANCE_SECONDS:
  813. return "late"
  814. return "on_time"
  815. # Recovery counter for recurring jobs wedged in stale ``last_status == "error"`` with a future
  816. # next_run_at.
  817. _persisted_error_recoveries: int = 0
  818. # Bounded in-memory history kept by every probe-visible fire-path counter.
  819. _TELEMETRY_RECENT_HISTORY = 20
  820. _persisted_error_recoveries_recent: list = []
  821. def _job_is_stale_error_recurring(
  822. job: dict[str, Any], schedule: dict[str, Any], now: datetime,
  823. ) -> bool:
  824. """True when a recurring job (caller-checked) is wedged in a stale persisted error state:
  825. ``last_status == "error"``, not running in this process (never re-arm a live run underneath
  826. itself), and ``last_run_at`` older than ``cadence + grace`` (a job merely erroring-and-retrying
  827. on schedule stays fresh and is not flagged).
  828. Condition (all must hold): * it has NOT successfully re-fired within its natural cadence — its
  829. ``last_run_at`` is older than ``cadence + grace``, so this is not a normal transient-error retry that
  830. will fire on its own soon, it is a job that has been sitting errored for a full period with no recovery;
  831. See #62002.
  832. """
  833. if job.get("last_status") != "error":
  834. return False
  835. from cron.quota_hold import hold_active
  836. if hold_active(job, now):
  837. return False # deliberately parked past a provider usage window, not wedged (#89376)
  838. if _job_running_in_this_process(str(job.get("id") or "")):
  839. return False
  840. # A fresh fire_claim means the job is running in ANOTHER process sharing this
  841. # store, not wedged; re-arming it here would only claim-fight the live run.
  842. if _claim_is_live(job.get("fire_claim"), now, FIRE_CLAIM_TTL_SECONDS):
  843. return False
  844. last_run = job.get("last_run_at")
  845. last_run_dt = _parse_aware(last_run) if last_run else None
  846. if last_run_dt is None:
  847. return False
  848. age_seconds = _elapsed_seconds(now, last_run_dt)
  849. if age_seconds < 0:
  850. return False
  851. grace = _compute_grace_seconds(schedule)
  852. cadence_seconds = _schedule_cadence_seconds(schedule)
  853. if cadence_seconds is None:
  854. # Unknown cadence: fall back to the grace window, never re-arming anything younger than it.
  855. cadence_seconds = grace
  856. return age_seconds > (cadence_seconds + grace)
  857. # Per-expr cache for _schedule_cadence_seconds' croniter measurements.
  858. _cron_cadence_cache: dict[str, Optional[float]] = {}
  859. def _schedule_cadence_seconds(schedule: dict[str, Any]) -> Optional[float]:
  860. """Approximate schedule period in seconds, or None (croniter missing / malformed expr). Cron
  861. results are cached per expr because this runs under ``_jobs_lock`` every tick; the gap can vary
  862. with base time for irregular exprs, acceptable for a staleness *threshold*."""
  863. if not isinstance(schedule, dict):
  864. return None
  865. kind = schedule.get("kind")
  866. if kind == "interval":
  867. minutes = schedule.get("minutes")
  868. try:
  869. return float(minutes) * 60.0 if minutes else None
  870. except (TypeError, ValueError):
  871. return None
  872. if kind != "cron" or not _ensure_croniter():
  873. return None
  874. expr = schedule.get("expr")
  875. if not expr:
  876. return None
  877. if expr in _cron_cadence_cache:
  878. return _cron_cadence_cache[expr]
  879. try:
  880. it = croniter(expr, _hermes_now())
  881. first = it.get_next(datetime)
  882. gap = (it.get_next(datetime) - first).total_seconds()
  883. result = gap if gap > 0 else None
  884. except Exception:
  885. result = None
  886. # Hard bound so deleted/edited exprs can't grow the cache unboundedly in a long-lived gateway.
  887. if len(_cron_cadence_cache) >= 256:
  888. _cron_cadence_cache.clear()
  889. _cron_cadence_cache[expr] = result
  890. return result
  891. def _append_telemetry_record(filename: str, entry: dict[str, Any], recent: list) -> None:
  892. """Record ``entry`` in the bounded ``recent`` list and append to ``<cron_dir>/<filename>``
  893. (best effort — telemetry must never break a tick). Counters stay module-level ints per
  894. metric because tests reset them by name."""
  895. recent.append(entry)
  896. del recent[:-_TELEMETRY_RECENT_HISTORY]
  897. try:
  898. path = _current_cron_store().cron_dir / filename
  899. _ensure_cron_dir(path.parent)
  900. with open(path, "a", encoding="utf-8") as fh:
  901. fh.write(json.dumps(entry) + "\n")
  902. except Exception as exc:
  903. logger.debug("Could not append %s record: %s", filename, exc)
  904. def _record_persisted_error_recovery(job: dict[str, Any], previous_next_run: str) -> None:
  905. """Persist a countable, probe-visible signal for one stale-error re-arm."""
  906. global _persisted_error_recoveries
  907. entry = {
  908. "job_id": job.get("id"),
  909. "name": job.get("name") or job.get("id"),
  910. "previous_next_run_at": previous_next_run,
  911. "rearmed_at": _hermes_now().isoformat(),
  912. }
  913. _persisted_error_recoveries += 1
  914. _append_telemetry_record(
  915. "persisted_error_recoveries.jsonl", entry, _persisted_error_recoveries_recent)
  916. def get_persisted_error_recovery_stats() -> dict[str, Any]:
  917. """Probe-visible snapshot of persisted-error recoveries."""
  918. return {
  919. "persisted_error_recoveries": _persisted_error_recoveries,
  920. "recent": list(_persisted_error_recoveries_recent),
  921. }
  922. def _cron_next_run_matches_expr(schedule: dict[str, Any], next_run_dt: datetime) -> bool:
  923. """Whether ``next_run_dt`` is an occurrence of the schedule's current expr (detects a
  924. hand-edited ``schedule.expr`` whose stored ``next_run_at`` came from the old one).
  925. Best-effort: anything uncheckable (non-cron, no expr, no croniter, malformed) reports a
  926. match.
  927. A direct ``jobs.json`` edit can change ``schedule.expr`` while leaving the stored ``next_run_at``
  928. computed under the *old* expression (#93049). The stored instant is stale exactly when it is not an
  929. occurrence of the current expression. Validation is best-effort: anything that cannot be checked
  930. (non-cron kind, missing expr, croniter unavailable, malformed input) reports a match so the fire path
  931. keeps its existing semantics.
  932. """
  933. if schedule.get("kind") != "cron":
  934. return True
  935. expr = schedule.get("expr")
  936. if not expr or not _ensure_croniter() or croniter is None:
  937. return True
  938. try:
  939. # Last occurrence at-or-before the instant: base one second past it so an exact hit is
  940. # included, then compare at second granularity (croniter is second-precision).
  941. prev = croniter(str(expr), next_run_dt + timedelta(seconds=1)).get_prev(datetime)
  942. return abs((prev - next_run_dt).total_seconds()) < 1.0
  943. except Exception:
  944. return True
  945. # Classifications for a due cron instant NOT on the current expr (see
  946. # _classify_stale_cron_next_run).
  947. STALE_CRON_MATCH = "match"
  948. STALE_CRON_TIMEZONE_MIGRATION = "timezone_migration"
  949. STALE_CRON_EXPR_EDIT = "expr_edit"
  950. def _classify_stale_cron_next_run(
  951. schedule: dict[str, Any], raw_next_run_dt: datetime, next_run_dt: datetime,
  952. ) -> str:
  953. """Explain WHY a stored ``next_run_at`` misses the current cron lattice; the causes need
  954. opposite actions. ``expr_edit``: a hand edit changed ``schedule.expr`` — re-anchor WITHOUT
  955. firing. ``timezone_migration``: only the offset representation changed (legacy UTC rows
  956. normalized into the profile tz); treating it as an edit would skip a due, never-fired
  957. occurrence. Discriminator: normalization moved the wall clock AND the stored instant's OWN
  958. wall clock is a legal occurrence (when offsets agree a genuine expr edit can never be misread
  959. as a migration).
  960. * ``expr_edit`` — a direct ``jobs.json`` edit changed ``schedule.expr`` while leaving ``next_run_at``
  961. computed under the old one (#93049). Upgrading from a UTC-scheduling build to one that honours the
  962. profile timezone leaves legacy rows like ``2026-09-02T04:00:00+00:00`` for ``0 4 * * *``; normalizing to
  963. Europe/Brussels turns that into ``06:00+02``, which the expression excludes. Treating it as a stale edit
  964. re-anchored to tomorrow and silently skipped a due occurrence that had never fired.
  965. """
  966. if _cron_next_run_matches_expr(schedule, next_run_dt):
  967. return STALE_CRON_MATCH
  968. wall_clock_shifted = raw_next_run_dt.replace(tzinfo=None) != next_run_dt.replace(tzinfo=None)
  969. if wall_clock_shifted and _cron_next_run_matches_expr(schedule, raw_next_run_dt):
  970. return STALE_CRON_TIMEZONE_MIGRATION
  971. return STALE_CRON_EXPR_EDIT
  972. # Offset-migration catch-ups on the fire path: climbing after a deploy = draining; steady = TZ
  973. # churn.
  974. _timezone_migration_catchups: int = 0
  975. _timezone_migration_catchups_recent: list = []
  976. def _record_timezone_migration_catchup(
  977. job: dict[str, Any], raw_next_run_dt: datetime, next_run_dt: datetime,
  978. ) -> None:
  979. """Persist a countable signal for one offset-migration catch-up fire."""
  980. global _timezone_migration_catchups
  981. entry = {
  982. "job_id": job.get("id"),
  983. "name": job.get("name") or job.get("id"),
  984. "expr": (job.get("schedule") or {}).get("expr"),
  985. "stored_next_run_at": raw_next_run_dt.isoformat(),
  986. "normalized_next_run_at": next_run_dt.isoformat(),
  987. "fired_at": _hermes_now().isoformat(),
  988. }
  989. _timezone_migration_catchups += 1
  990. _append_telemetry_record(
  991. "timezone_migration_catchups.jsonl", entry, _timezone_migration_catchups_recent)
  992. def get_timezone_migration_catchup_stats() -> dict[str, Any]:
  993. """Probe-visible snapshot of offset-migration catch-up fires."""
  994. return {
  995. "timezone_migration_catchups": _timezone_migration_catchups,
  996. "recent": list(_timezone_migration_catchups_recent),
  997. }
  998. def compute_next_run(
  999. schedule: dict[str, Any], last_run_at: Optional[str] = None, *, timezone: Any = None,
  1000. ) -> Optional[str]:
  1001. """Compute the next run time for a schedule as an ISO string, or None if no more runs.
  1002. *timezone* is the job's pinned IANA zone (a job record or a bare zone name); None inherits
  1003. the profile's configured zone. Every caller that advances a job's ``next_run_at`` must pass
  1004. it, or a pinned job is computed in the wrong zone and jumps a slot."""
  1005. now = _hermes_now()
  1006. if not isinstance(schedule, dict):
  1007. return None
  1008. kind = schedule.get("kind")
  1009. job_zone = jobs_time.zone_for_job(timezone)
  1010. if kind == "once":
  1011. return _recoverable_oneshot_run_at(
  1012. schedule, now, last_run_at=last_run_at, zone=job_zone)
  1013. # Recurring kinds anchor on last_run_at so a restart doesn't re-anchor the schedule.
  1014. base_time = (_parse_aware(last_run_at) if last_run_at else None) or now
  1015. if kind == "interval":
  1016. minutes = schedule.get("minutes")
  1017. if minutes is None:
  1018. return None
  1019. # Add in UTC so an interval keeps its duration when the profile's UTC offset changes.
  1020. next_run = base_time.astimezone(UTC) + timedelta(minutes=minutes)
  1021. return next_run.astimezone(base_time.tzinfo).isoformat()
  1022. if kind == "cron":
  1023. expr = schedule.get("expr")
  1024. if not expr:
  1025. return None
  1026. if not _ensure_croniter():
  1027. logger.warning(
  1028. "Cannot compute next run for cron schedule %r: 'croniter' is "
  1029. "not installed. croniter is a core dependency as of v0.9.x; "
  1030. "reinstall hermes-agent or run 'pip install croniter' in your runtime env "
  1031. "(interpreter: %s).",
  1032. expr, sys.executable)
  1033. return None
  1034. # Anchor cron matching to the CONFIGURED IANA timezone's WALL CLOCK,
  1035. # not to the UTC offset carried by ``base_time``. croniter ignores
  1036. # the tzinfo on its start time and uses the start's UTC offset as its
  1037. # working offset, so a ``last_run_at`` stored in UTC (+00:00) would
  1038. # push the next fire to 09:00 UTC instead of 09:00 local, and DST
  1039. # transition days (spring-forward / fall-back) would land one hour off
  1040. # (08:00 or 10:00). Render the base as the configured zone's naive
  1041. # wall clock for croniter, then re-attach the zone to the result, so
  1042. # the wall-clock hour stays correct every calendar day, including DST
  1043. # boundaries (morning-routine 09:00 America/Toronto).
  1044. # Fall back to the base's own zone only when nothing is configured.
  1045. zone = job_zone or get_timezone() or base_time.tzinfo
  1046. base_wall = base_time.astimezone(zone).replace(tzinfo=None)
  1047. it = croniter(expr, base_wall)
  1048. # Strictly-after guard for the DST fall-back hour (qwen-code#11723 class):
  1049. # attaching the zone to a naive wall clock resolves the repeated autumn hour
  1050. # to its EARLIER occurrence (fold=0), so a base inside the second occurrence
  1051. # got a "next run" up to an hour in the PAST — the fire path would re-fire
  1052. # immediately and re-anchor, looping. Try both folds of each candidate wall
  1053. # clock and return the earliest instant strictly after the base; a repeated
  1054. # hour has two instants, so two candidates always suffice.
  1055. base_ts = base_time.timestamp()
  1056. next_wall = it.get_next(datetime)
  1057. for _ in range(2):
  1058. for fold in (0, 1):
  1059. candidate = next_wall.replace(tzinfo=zone, fold=fold)
  1060. if candidate.timestamp() > base_ts:
  1061. return candidate.isoformat()
  1062. next_wall = it.get_next(datetime)
  1063. return next_wall.replace(tzinfo=zone).isoformat()
  1064. return None
  1065. # --- Ticker heartbeat (liveness signal for `hermes cron status`) ---
  1066. def _write_marker(name: str, text: str, tmp_prefix: str) -> None:
  1067. """Atomic (never torn) best-effort marker write; failures swallowed so markers never break the
  1068. tick."""
  1069. try:
  1070. ensure_dirs()
  1071. atomic_write_text(_current_cron_store().cron_dir / name, text, tmp_prefix=tmp_prefix, mode=0o600)
  1072. except Exception:
  1073. pass
  1074. def record_ticker_heartbeat(success: bool = False) -> None:
  1075. """Record ticker liveness (+ last-success marker when ``success``) so `cron status` can tell
  1076. "alive but failing" from "firing"; scoped per profile store.
  1077. The ticker calls this once per loop iteration. ``success=True`` additionally bumps the *last successful
  1078. tick* marker. We track two distinct signals so `hermes cron status` can tell a thread that is merely
  1079. *alive and looping* (heartbeat fresh, success stale) from one that is actually *firing jobs* (both
  1080. fresh) — a ticker stuck failing every tick would otherwise keep the plain heartbeat fresh and falsely
  1081. report healthy (#32612, #32895).
  1082. Resolution uses ``_current_cron_store()`` so the heartbeat is correctly scoped to the active profile's
  1083. store — critical under multiplex_profiles where each profile needs its own liveness signal (#69377).
  1084. """
  1085. # ``<epoch> <pid>``: a killed ticker's last stamp reads fresh for ~3 minutes, so a reader with no
  1086. # other proof of the scheduler (the in-process serve/Desktop ticker) checks the writer is alive.
  1087. _write_marker("ticker_heartbeat", f"{time.time()} {os.getpid()}", ".hb_")
  1088. if success:
  1089. _write_marker("ticker_last_success", str(time.time()), ".hb_")
  1090. def _read_marker_fields(name: str) -> list[str]:
  1091. try:
  1092. return (_current_cron_store().cron_dir / name).read_text(encoding="utf-8-sig").split()
  1093. except Exception:
  1094. return []
  1095. def _epoch_file_age(name: str) -> Optional[float]:
  1096. """Seconds since the epoch stamp stored in ``<cron_dir>/<name>``; None = missing/unreadable."""
  1097. try:
  1098. return max(0.0, time.time() - float(_read_marker_fields(name)[0]))
  1099. except Exception:
  1100. return None
  1101. def ticker_heartbeat_writer_alive() -> bool:
  1102. """True when the process that wrote this store's ticker heartbeat is still running. A legacy
  1103. bare-epoch stamp names no writer and is NOT proof of a live scheduler by itself."""
  1104. fields = _read_marker_fields("ticker_heartbeat")
  1105. try:
  1106. from hermes_cli._subprocess_compat import pid_exists_stdlib
  1107. return len(fields) >= 2 and pid_exists_stdlib(int(fields[1]))
  1108. except Exception:
  1109. return False
  1110. def get_ticker_heartbeat_age() -> Optional[float]:
  1111. """Seconds since the ticker loop last iterated; None = missing/unreadable ("cannot determine",
  1112. not "dead").
  1113. Resolution uses ``_current_cron_store()`` so the heartbeat is correctly scoped to the active profile —
  1114. critical under multiplex_profiles where ``hermes cron status`` must report per-profile liveness
  1115. (#69377).
  1116. """
  1117. return _epoch_file_age("ticker_heartbeat")
  1118. def get_ticker_success_age() -> Optional[float]:
  1119. """Seconds since the ticker last completed a tick WITHOUT raising, or None.
  1120. Resolution uses ``_current_cron_store()`` so the heartbeat is correctly scoped to the active profile —
  1121. critical under multiplex_profiles where ``hermes cron status`` must report per-profile liveness
  1122. (#69377).
  1123. """
  1124. return _epoch_file_age("ticker_last_success")
  1125. def record_ticker_error(message: str) -> None:
  1126. """Persist the latest tick failure so `cron status` (another process) can show WHY, not just
  1127. staleness."""
  1128. _write_marker("ticker_last_error", f"{time.time()}\n{message.strip()}\n", ".terr_")
  1129. def clear_ticker_error() -> None:
  1130. """Remove the last-tick-error marker after a successful tick. Best-effort."""
  1131. with contextlib.suppress(OSError):
  1132. (_current_cron_store().cron_dir / "ticker_last_error").unlink()
  1133. def get_ticker_last_error() -> Optional[str]:
  1134. """Return the most recent recorded tick error message, or None."""
  1135. try:
  1136. raw = (_current_cron_store().cron_dir / "ticker_last_error").read_text(encoding="utf-8-sig")
  1137. except Exception:
  1138. return None
  1139. lines = raw.splitlines()
  1140. if len(lines) < 2:
  1141. return None
  1142. return "\n".join(lines[1:]).strip() or None
  1143. # --- Job CRUD Operations ---
  1144. def _parse_jobs_file(jobs_file: Path) -> tuple[Any, bool]:
  1145. """Tolerant jobs.json parse -> ``(data, used_strict_fallback)``: utf-8-sig absorbs a BOM, strict
  1146. failure retries with ``strict=False``. IO/fallback errors propagate (caller decides repair vs
  1147. bail)."""
  1148. with open(jobs_file, "r", encoding="utf-8-sig") as f:
  1149. raw = f.read()
  1150. try:
  1151. return json.loads(raw), False
  1152. except json.JSONDecodeError:
  1153. return json.loads(raw, strict=False), True
  1154. def load_jobs() -> list[dict[str, Any]]:
  1155. """Load all jobs from storage."""
  1156. jobs_file = _current_cron_store().jobs_file
  1157. ensure_dirs()
  1158. # Stamp BEFORE reading (fail-safe, see _record_load_stamp): a racing write then forces the
  1159. # merge.
  1160. pre_read_stamp = _jobs_file_stamp(jobs_file)
  1161. if not jobs_file.exists():
  1162. _record_load_stamp(None)
  1163. return []
  1164. try:
  1165. data, _strict_retry = _parse_jobs_file(jobs_file)
  1166. except OSError as e:
  1167. logger.error("IOError reading jobs.json: %s", e)
  1168. raise RuntimeError(f"Failed to read cron database: {e}") from e
  1169. except Exception as e:
  1170. logger.error("Failed to auto-repair jobs.json: %s", e)
  1171. raise RuntimeError(f"Cron database corrupted and unrepairable: {e}") from e
  1172. # Accept the canonical dict, or a bare list (auto-repair); any other top-level shape is
  1173. # corruption. Repair details are logged only by the locked pass (an unlocked pass re-runs
  1174. # under the lock below), so each warning is emitted once per repair.
  1175. repair = "had invalid control characters" if _strict_retry else None
  1176. notes: list[str] = []
  1177. unmergeable = False # disk shape _peek_jobs_unlocked cannot read: only a replace save fixes it
  1178. if isinstance(data, dict):
  1179. jobs = data.get("jobs", [])
  1180. if isinstance(jobs, dict):
  1181. # ID-keyed map from external tools: flatten (inline "id" wins, else the key), skip junk.
  1182. # _peek_jobs_unlocked deliberately does NOT flatten, so a merging save refuses this
  1183. # shape; the repair below rewrites it with replace=True.
  1184. skipped = [k for k, v in jobs.items() if not isinstance(v, dict)]
  1185. if skipped:
  1186. notes.append(f"Skipping {len(skipped):d} non-dict entr{('y' if len(skipped) == 1 else 'ies')} in id-keyed jobs map: {', '.join(map(repr, skipped))}")
  1187. jobs = [{**v, "id": v.get("id") or k} for k, v in jobs.items() if isinstance(v, dict)]
  1188. repair = "id-keyed jobs map flattened to list"
  1189. unmergeable = True
  1190. elif not isinstance(jobs, list):
  1191. notes.append(f"Replacing invalid jobs.json 'jobs' field ({type(jobs).__name__}) with an empty list")
  1192. jobs = []
  1193. repair = "invalid jobs field replaced with list"
  1194. unmergeable = True
  1195. elif isinstance(data, list):
  1196. jobs = data
  1197. repair = "bare list wrapped as dict"
  1198. else:
  1199. raise RuntimeError(
  1200. f"Cron database corrupted: expected {{'jobs': [...]}}, got {type(data).__name__}")
  1201. junk = [j for j in jobs if not isinstance(j, dict)]
  1202. if junk:
  1203. # Every reader and the due scan index records as dicts: one junk entry would crash the
  1204. # whole tick and freeze every healthy sibling job, so skip it like the id-keyed map does.
  1205. # Types only: the raw values are arbitrary file content and must not reach the logs.
  1206. notes.append(f"Skipping {len(junk)} non-object entr{'y' if len(junk) == 1 else 'ies'} in jobs.json (types: {', '.join(sorted({type(j).__name__ for j in junk}))})")
  1207. jobs = [j for j in jobs if isinstance(j, dict)]
  1208. repair = repair or "non-object entries dropped"
  1209. for job in jobs:
  1210. # A hand-edited "completed" that is not a non-negative int (null, "2", 1.0, -5, Infinity)
  1211. # would crash every counter reader (None += 1, "2" + 1), render as "None/3" / "2.0/3", or
  1212. # grant extra runs; normalize it once here so readers can trust a non-negative int.
  1213. # OverflowError: json.loads turns Infinity / 1e999 into float inf, and int(inf) raises.
  1214. rep = job.get("repeat")
  1215. if isinstance(rep, dict) and "completed" in rep and (
  1216. type(rep["completed"]) is not int or rep["completed"] < 0):
  1217. try:
  1218. rep["completed"] = max(int(rep["completed"]), 0)
  1219. except (TypeError, ValueError, OverflowError):
  1220. rep["completed"] = 0
  1221. repair = repair or "invalid repeat.completed normalized"
  1222. # Persist even an empty result, or an all-junk store repeats the repair on every tick.
  1223. if repair:
  1224. if not getattr(_jobs_lock_state, "depth", 0):
  1225. # An unlocked snapshot may predate a locked writer's update to a job it already holds
  1226. # (the shrink-merge only restores missing ids), so re-read and repair under the lock.
  1227. with _jobs_lock():
  1228. return load_jobs()
  1229. for note in notes:
  1230. logger.warning("%s", note)
  1231. # Keep the shrink-merge (a degraded-lock sibling's create may have landed since the read)
  1232. # unless disk is STILL a shape the merge would refuse: a sibling may have rewritten it.
  1233. try:
  1234. save_jobs(jobs, replace=unmergeable and _peek_jobs_unlocked() is None)
  1235. logger.warning("Auto-repaired jobs.json (%s)", repair)
  1236. except OSError as exc: # repaired in memory; the next load retries the persist
  1237. store_health.note_unwritable(exc, f"jobs.json repair ({repair}) not persisted", "load")
  1238. _record_load_stamp(pre_read_stamp)
  1239. return jobs
  1240. def _peek_jobs_unlocked() -> Optional[list[dict[str, Any]]]:
  1241. """Repair-free read under ``_jobs_lock()``: ``[]`` if missing, ``None`` if corrupt (never
  1242. shrink-merge against an unknown baseline). Never saves — that would recurse."""
  1243. jobs_file = _current_cron_store().jobs_file
  1244. if not jobs_file.exists():
  1245. return []
  1246. try:
  1247. data, _ = _parse_jobs_file(jobs_file)
  1248. except Exception:
  1249. return None
  1250. if isinstance(data, dict):
  1251. jobs = data.get("jobs", [])
  1252. return jobs if isinstance(jobs, list) else None
  1253. return data if isinstance(data, list) else None
  1254. def _jobs_file_stamp(jobs_file: Path) -> Optional[tuple[int, int, int]]:
  1255. """Shrink-merge fast-path stamp ``(mtime_ns, size, ino)``; ``None`` if unstatable. ``st_ino`` is
  1256. included because every writer uses mkstemp+rename, so a same-size write in one mtime quantum
  1257. can't false-match."""
  1258. try:
  1259. st = jobs_file.stat()
  1260. return (st.st_mtime_ns, st.st_size, st.st_ino)
  1261. except OSError:
  1262. return None
  1263. def _record_load_stamp(stamp: Optional[tuple[int, int, int]]) -> None:
  1264. """Remember jobs.json's stamp for the enclosing _jobs_lock() section (no-op outside one) so the
  1265. save path can skip the shrink-merge when disk provably hasn't changed. Capture it BEFORE
  1266. reading: a mid-read sibling then mismatches (fail-safe); stamping after would certify an
  1267. unseen write.
  1268. Stamping after the read would let that sibling's write be certified as "seen" without being in the
  1269. loaded payload, wrongly suppressing the recovery. See #80703.
  1270. """
  1271. if getattr(_jobs_lock_state, "depth", 0):
  1272. _jobs_lock_state.load_stamp = stamp
  1273. def _unmerged_disk_jobs(
  1274. jobs: list[dict[str, Any]], removed_ids: Optional[Collection[str]]
  1275. ) -> list[dict[str, Any]]:
  1276. """On-disk jobs missing from *jobs* and not intentionally removed. Stamp match => nothing
  1277. landed, return without parsing; unreadable store => raise RuntimeError (fail closed: writing
  1278. over an unknown baseline would silently drop every job in it)."""
  1279. stamp = getattr(_jobs_lock_state, "load_stamp", None)
  1280. if stamp is not None and _jobs_file_stamp(_current_cron_store().jobs_file) == stamp:
  1281. return []
  1282. disk_jobs = _peek_jobs_unlocked()
  1283. if disk_jobs is None:
  1284. raise RuntimeError(
  1285. f"Cron database corrupted; refusing to overwrite {_current_cron_store().jobs_file}")
  1286. seen = {str(j["id"]) for j in jobs if isinstance(j, dict) and j.get("id")}
  1287. seen |= {str(i) for i in (removed_ids or ()) if i}
  1288. recovered: list[dict[str, Any]] = []
  1289. for disk_job in disk_jobs:
  1290. if not isinstance(disk_job, dict) or not disk_job.get("id"):
  1291. continue
  1292. disk_id = str(disk_job["id"])
  1293. if disk_id not in seen:
  1294. recovered.append(disk_job)
  1295. seen.add(disk_id)
  1296. return recovered
  1297. def _merge_unexpected_disk_jobs(
  1298. jobs: list[dict[str, Any]], *, removed_ids: Optional[Collection[str]] = None,
  1299. ) -> list[dict[str, Any]]:
  1300. """*jobs* plus on-disk jobs absent from the payload (under the degraded flock-timeout path a
  1301. stale writer would otherwise clobber concurrent creates). Deletes pass ``removed_ids``; never
  1302. mutates *jobs*."""
  1303. recovered = _unmerged_disk_jobs(jobs, removed_ids)
  1304. if not recovered:
  1305. return jobs
  1306. logger.warning(
  1307. "Preserved %d cron job(s) present on disk but missing from the "
  1308. "in-memory save payload (concurrent create under degraded lock "
  1309. "or stale writer) (#80624): %s",
  1310. len(recovered), [j.get("id") for j in recovered])
  1311. return jobs + recovered
  1312. def _unlink_quiet(path: Optional[str]) -> None:
  1313. if path is not None:
  1314. with contextlib.suppress(OSError):
  1315. os.unlink(path)
  1316. def _stage_jobs_payload(jobs_file: Path, jobs: list[dict[str, Any]]) -> str:
  1317. """Serialize the store payload to a fsynced temp file beside the resolved *jobs_file*; return its path."""
  1318. fd, tmp_path = mkstemp_beside(jobs_file, suffix=".tmp", prefix=".jobs_")
  1319. try:
  1320. with os.fdopen(fd, "w", encoding="utf-8") as f:
  1321. json.dump(
  1322. {"jobs": jobs, "updated_at": _hermes_now().isoformat()},
  1323. f, indent=2, ensure_ascii=False)
  1324. f.flush()
  1325. os.fsync(f.fileno())
  1326. except BaseException:
  1327. _unlink_quiet(tmp_path)
  1328. raise
  1329. return tmp_path
  1330. _SAVE_JOBS_MERGE_ATTEMPTS = 5
  1331. def _save_jobs_unlocked(
  1332. jobs: list[dict[str, Any]], *, removed_ids: Optional[Collection[str]] = None,
  1333. replace: bool = False,
  1334. ):
  1335. """Save all jobs; caller must hold _jobs_lock(). ``removed_ids`` = intentional deletes;
  1336. ``replace=True`` skips the shrink-merge guard and the corrupt-store refusal (wholesale
  1337. rewrite for tests, disaster recovery and load_jobs' auto-repair of unmergeable shapes)."""
  1338. jobs_file = _current_cron_store().jobs_file
  1339. ensure_dirs()
  1340. # Owner snapshot BEFORE replace so a root writer can hand the file back to the gateway user.
  1341. _stat_before = None
  1342. for probe in (jobs_file, jobs_file.parent):
  1343. with contextlib.suppress(OSError):
  1344. _stat_before = os.stat(probe)
  1345. break
  1346. # Shrink-merge loop: merge, stage, re-peek, repeat; the last attempt writes without a re-peek.
  1347. tmp_path = None
  1348. try:
  1349. for attempt in range(_SAVE_JOBS_MERGE_ATTEMPTS + 1):
  1350. if not replace:
  1351. jobs = _merge_unexpected_disk_jobs(jobs, removed_ids=removed_ids)
  1352. tmp_path = _stage_jobs_payload(jobs_file, jobs)
  1353. # Verify-after-stage: a sibling landing during serialization forces another merge round.
  1354. if (
  1355. not replace
  1356. and attempt < _SAVE_JOBS_MERGE_ATTEMPTS
  1357. and _unmerged_disk_jobs(jobs, removed_ids)
  1358. ):
  1359. _unlink_quiet(tmp_path)
  1360. tmp_path = None
  1361. continue
  1362. # fsync the directory the rename actually landed in (atomic_replace resolves symlinks).
  1363. replaced = Path(atomic_replace(tmp_path, jobs_file))
  1364. tmp_path = None
  1365. fsync_directory(replaced.parent)
  1366. _secure_file(jobs_file)
  1367. _preserve_file_ownership(jobs_file, _stat_before)
  1368. # Invalidate (never refresh) the stamp: a refresh would let a nested save certify disk
  1369. # against an OUTER caller's stale payload. Later saves take the full merge (fail-safe).
  1370. _record_load_stamp(None)
  1371. store_health.note_writable(jobs_file.parent)
  1372. return
  1373. except BaseException:
  1374. _unlink_quiet(tmp_path)
  1375. raise
  1376. def save_jobs(
  1377. jobs: list[dict[str, Any]], *, removed_ids: Optional[Collection[str]] = None,
  1378. replace: bool = False,
  1379. ):
  1380. """Save all jobs under the lock; see ``_save_jobs_unlocked`` for ``removed_ids``/``replace``."""
  1381. with _jobs_lock():
  1382. _save_jobs_unlocked(jobs, removed_ids=removed_ids, replace=replace)
  1383. _MISSING = object()
  1384. def _with_job(
  1385. job_id: Any, fn: Callable[[list[dict[str, Any]], int, dict[str, Any]], Any], missing: Any = None
  1386. ) -> Any:
  1387. """Run ``fn(jobs, i, job)`` on the first match under ``_jobs_lock()``; ``fn`` saves. *missing*
  1388. if none."""
  1389. with _jobs_lock():
  1390. jobs = load_jobs()
  1391. for i, job in enumerate(jobs):
  1392. if job.get("id") == job_id:
  1393. return fn(jobs, i, job)
  1394. return missing
  1395. def _complete_job_record(job: dict[str, Any]) -> None:
  1396. """Retire *job* in place as a terminal completion (record kept for `cronjob list`)."""
  1397. job.update(enabled=False, state="completed", next_run_at=None)
  1398. def _activate_job_record(job: dict[str, Any]) -> None:
  1399. """Clear pause markers in place so *job* is runnable again."""
  1400. job.update(enabled=True, state="scheduled", paused_at=None, paused_reason=None)
  1401. def _normalize_workdir(workdir: Optional[str]) -> Optional[str]:
  1402. """Workdir -> absolute path, or None when empty. ``~`` expands; relative paths are rejected
  1403. (cron runs detached from any cwd); must be an existing dir now but is deliberately NOT
  1404. re-checked at run time (scheduler falls back with a warning). ValueError when invalid."""
  1405. if workdir is None:
  1406. return None
  1407. raw = str(workdir).strip()
  1408. if not raw:
  1409. return None
  1410. expanded = Path(raw).expanduser()
  1411. if not expanded.is_absolute():
  1412. raise ValueError(
  1413. f"Cron workdir must be an absolute path (got {raw!r}). "
  1414. f"Cron jobs run detached from any shell cwd, so relative paths are ambiguous.")
  1415. resolved = expanded.resolve()
  1416. if not resolved.exists():
  1417. raise ValueError(f"Cron workdir does not exist: {resolved}")
  1418. if not resolved.is_dir():
  1419. raise ValueError(f"Cron workdir is not a directory: {resolved}")
  1420. return str(resolved)
  1421. def _normalize_job_optional_text(
  1422. value: Any, *, strip_trailing_slash: bool = False
  1423. ) -> Optional[str]:
  1424. if not isinstance(value, str):
  1425. return None
  1426. return (value.strip().rstrip("/") if strip_trailing_slash else value.strip()) or None
  1427. def _normalize_base_url(value: Any) -> Optional[str]:
  1428. return _normalize_job_optional_text(value, strip_trailing_slash=True)
  1429. def _normalize_str_list(items: Any) -> Optional[list[str]]:
  1430. """Non-blank stripped items of *items*, or None when nothing remains."""
  1431. return [str(j).strip() for j in items if str(j).strip()] or None
  1432. def _normalize_enabled_toolsets(value: Any) -> Optional[list[str]]:
  1433. """Per-job toolset allowlist. An explicitly-set EMPTY list means 'nothing allowed' and must
  1434. persist as [] — normalizing it to None would read back as 'no restriction' (#82010). Absent
  1435. or None stays None (the unrestricted default); a non-list raises before anything is stored."""
  1436. if value is None:
  1437. return None
  1438. if not isinstance(value, list):
  1439. raise ValueError("enabled_toolsets must be a list of toolset names (or [] for none).")
  1440. return [str(j).strip() for j in value if str(j).strip()]
  1441. def _normalize_context_from(value: Any) -> Optional[list[str]]:
  1442. """Accept a job id or a list of ids; anything else is None."""
  1443. if isinstance(value, str):
  1444. value = [value]
  1445. return _normalize_str_list(value) if isinstance(value, list) else None
  1446. def _normalize_failure_deliver(value: Any) -> Optional[str]:
  1447. """failure_deliver shares deliver's value grammar; flatten str/list like the tool layer's
  1448. _normalize_deliver_param for direct create_job callers. Semantic validation happens at
  1449. resolution time via the shared deliver path."""
  1450. if isinstance(value, (list, tuple)):
  1451. return ",".join(str(p).strip() for p in value if str(p).strip()) or None
  1452. return _normalize_job_optional_text(value)
  1453. def _normalize_reasoning_effort(value: Any) -> Optional[str]:
  1454. """Spelling-only validation via the shared parser (cron knob never stricter/looser than
  1455. config.yaml); model capability is deliberately NOT checked (model unknowable at create time,
  1456. transports clamp at send time). None for unset, lowercase level, or ValueError."""
  1457. if value is None:
  1458. return None
  1459. text = str(value).strip().lower()
  1460. if not text:
  1461. return None
  1462. from hermes_constants import parse_reasoning_effort
  1463. if parse_reasoning_effort(text) is None:
  1464. raise ValueError(
  1465. f"Invalid reasoning_effort {value!r}. Valid levels: "
  1466. "none, minimal, low, medium, high, xhigh, max, ultra "
  1467. "(empty string clears the override).")
  1468. if text in {"false", "disabled"}:
  1469. return "none"
  1470. return text
  1471. # Normalizers for create_job (all fields) / update_job (present fields). Invalid values raise BEFORE
  1472. # storing.
  1473. def _normalize_catch_up(value: Any) -> Optional[bool]:
  1474. """``catch_up``: True runs one catch-up for missed slots, False skips them, None = default."""
  1475. if value is None or isinstance(value, bool):
  1476. return value
  1477. raise ValueError("catch_up must be true, false, or null (use the default).")
  1478. _CREATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {
  1479. "model": _normalize_job_optional_text,
  1480. "provider": _normalize_job_optional_text,
  1481. "base_url": _normalize_base_url,
  1482. "script": _normalize_job_optional_text,
  1483. "monitor_script": _normalize_job_optional_text,
  1484. "monitor_url": _normalize_job_optional_text,
  1485. "enabled_toolsets": _normalize_enabled_toolsets,
  1486. "workdir": _normalize_workdir,
  1487. "no_agent": bool,
  1488. "context_from": _normalize_context_from,
  1489. "failure_deliver": _normalize_failure_deliver,
  1490. "interpreter": _normalize_job_optional_text,
  1491. "timezone": jobs_time.normalize_timezone,
  1492. "grace_minutes": jobs_time.normalize_grace_minutes,
  1493. "catch_up": _normalize_catch_up,
  1494. }
  1495. _UPDATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {
  1496. # [] is an explicit zero-tool allowlist and must survive the update path as [] too (#82010).
  1497. "enabled_toolsets": _normalize_enabled_toolsets,
  1498. "workdir": lambda v: None if v in {None, "", False} else _normalize_workdir(v),
  1499. "monitor_script": _normalize_job_optional_text,
  1500. "monitor_url": _normalize_job_optional_text,
  1501. "interpreter": _normalize_job_optional_text,
  1502. "reasoning_effort": _normalize_reasoning_effort,
  1503. # ``0`` is a real grace value (never catch up), so this normalizer must never use an
  1504. # ``in {None, "", False}`` test — ``0 == False`` would silently clear it.
  1505. "timezone": jobs_time.normalize_timezone,
  1506. "grace_minutes": jobs_time.normalize_grace_minutes,
  1507. "catch_up": _normalize_catch_up,
  1508. }
  1509. def _oneshot_past_grace_error(run_at: Any) -> ValueError:
  1510. return ValueError(
  1511. f"Requested one-shot time {run_at} is more than "
  1512. f"{ONESHOT_GRACE_SECONDS}s in the past and cannot be scheduled.")
  1513. def _next_run_or_reject_past_oneshot(
  1514. parsed_schedule: dict[str, Any], label: str, fallback_run_at: Any, what: str,
  1515. timezone: Any = None,
  1516. ) -> Optional[str]:
  1517. """``compute_next_run`` that raises (after a warning log) for a one-shot outside the grace
  1518. window, so a ghost job with ``next_run_at=None`` can never be stored."""
  1519. next_run_at = compute_next_run(parsed_schedule, timezone=timezone)
  1520. if parsed_schedule.get("kind") == "once" and next_run_at is None:
  1521. run_at = parsed_schedule.get("run_at") or fallback_run_at
  1522. logger.warning(
  1523. "Rejecting one-shot cron job %s'%s': run_at %s is outside the %ss grace window",
  1524. what, label, run_at, ONESHOT_GRACE_SECONDS)
  1525. raise _oneshot_past_grace_error(run_at)
  1526. return next_run_at
  1527. def create_job(
  1528. prompt: Optional[str],
  1529. schedule: str,
  1530. name: Optional[str] = None,
  1531. repeat: Optional[int] = None,
  1532. deliver: Optional[str] = None,
  1533. origin: Optional[dict[str, Any]] = None,
  1534. skill: Optional[str] = None,
  1535. skills: Optional[list[str]] = None,
  1536. model: Optional[str] = None,
  1537. provider: Optional[str] = None,
  1538. base_url: Optional[str] = None,
  1539. script: Optional[str] = None,
  1540. context_from: Optional[str | list[str]] = None,
  1541. enabled_toolsets: Optional[list[str]] = None,
  1542. workdir: Optional[str] = None,
  1543. no_agent: bool = False,
  1544. attach_to_session: Optional[bool] = None,
  1545. monitor_script: Optional[str] = None,
  1546. monitor_url: Optional[str] = None,
  1547. reasoning_effort: Optional[str] = None,
  1548. failure_deliver: Optional[str] = None,
  1549. catch_up: Optional[bool] = None,
  1550. paused: bool = False,
  1551. paused_reason: Optional[str] = None,
  1552. pinned: bool = False,
  1553. interpreter: Optional[str] = None,
  1554. timezone: Optional[str] = None,
  1555. grace_minutes: Optional[int] = None,
  1556. ) -> dict[str, Any]:
  1557. """Create a new cron job and return the stored record.
  1558. deliver defaults to "origin" when ``origin`` is given, else "local"; repeat None = forever.
  1559. script: stdout is injected as prompt context, or with ``no_agent=True`` IS the job (stdout
  1560. delivered verbatim, requires ``script``). context_from: job id(s) whose latest output is
  1561. injected. workdir: absolute cwd for tools/scripts. monitor_script/monitor_url: cheap monitor
  1562. source run FIRST each tick; unchanged output suppresses the agent run (mutually exclusive,
  1563. incompatible with ``no_agent``). reasoning_effort: per-job pin; capability NOT validated.
  1564. interpreter: absolute/``~`` Python for ``.py`` script/monitor_script, validated at run time
  1565. (a venv can be rebuilt or moved after creation). timezone: IANA key pinning the schedule's
  1566. wall clock (None/'' = inherit the profile zone). grace_minutes: catch-up window override
  1567. (0 = never catch up)."""
  1568. if not isinstance(paused, bool):
  1569. raise ValueError("paused must be a boolean.")
  1570. if paused_reason is not None and not isinstance(paused_reason, str):
  1571. raise ValueError("paused_reason must be a string.")
  1572. if paused_reason is not None and not paused:
  1573. raise ValueError("paused_reason requires paused=True.")
  1574. parsed_schedule = parse_schedule(schedule)
  1575. # Normalize repeat: treat 0 or negative values as None (infinite). String forms
  1576. # ('forever'/'once'/numeric) coerce via normalize_repeat_value — the shared chokepoint with update paths
  1577. # (#66824/#64520/#7142/#71987/#95706).
  1578. repeat = normalize_repeat_value(repeat)
  1579. if parsed_schedule["kind"] == "once" and repeat is None:
  1580. repeat = 1
  1581. if deliver is None:
  1582. deliver = "origin" if origin else "local"
  1583. job_id = uuid.uuid4().hex[:12]
  1584. now = _hermes_now().isoformat()
  1585. raw = locals()
  1586. f = {key: norm(raw[key]) for key, norm in _CREATE_FIELD_NORMALIZERS.items()}
  1587. normalized_skills = _normalize_skill_list(skill, skills)
  1588. normalized_attach = attach_to_session if isinstance(attach_to_session, bool) else None
  1589. normalized_reasoning_effort = _normalize_reasoning_effort(reasoning_effort)
  1590. _validate_job_mode_invariants(f["monitor_script"], f["monitor_url"], f["no_agent"], f["script"])
  1591. validate_schedule_knobs(parsed_schedule, f["timezone"], f["grace_minutes"])
  1592. prompt_text = _coerce_job_text(prompt).strip()
  1593. if not prompt_text and not f["script"] and not normalized_skills:
  1594. raise ValueError(EMPTY_PAYLOAD_ERROR)
  1595. # Reject gateway-lifecycle commands (respawn loops) here, not just in the CLI: covers the tool.
  1596. from cron.lifecycle_guard import check_gateway_lifecycle
  1597. check_gateway_lifecycle(prompt_text, f["script"])
  1598. label_source = (
  1599. prompt_text
  1600. or (normalized_skills[0] if normalized_skills else None)
  1601. or (f["script"] if f["no_agent"] else None)
  1602. or "cron job"
  1603. )
  1604. name = name or label_source[:50].strip()
  1605. if pinned and not f["model"]:
  1606. f["provider"], f["model"] = _main_model_pin()
  1607. next_run_at = _next_run_or_reject_past_oneshot(
  1608. parsed_schedule, name, schedule, "", f["timezone"])
  1609. job = {
  1610. "id": job_id,
  1611. "name": name,
  1612. "prompt": prompt_text,
  1613. "skills": normalized_skills,
  1614. "skill": normalized_skills[0] if normalized_skills else None,
  1615. "model": f["model"],
  1616. "provider": f["provider"],
  1617. "base_url": f["base_url"],
  1618. "script": f["script"],
  1619. "no_agent": f["no_agent"],
  1620. "monitor_script": f["monitor_script"],
  1621. "monitor_url": f["monitor_url"],
  1622. "monitor_state": None,
  1623. "context_from": f["context_from"],
  1624. "schedule": parsed_schedule,
  1625. "schedule_display": parsed_schedule.get("display", schedule),
  1626. "repeat": {"times": repeat, "completed": 0}, # times None = forever
  1627. "enabled": not paused,
  1628. "state": "paused" if paused else "scheduled",
  1629. "paused_at": now if paused else None,
  1630. "paused_reason": ((paused_reason or "").strip() or "Created paused; awaiting operator approval.") if paused else None,
  1631. "created_at": now,
  1632. "next_run_at": None if paused else next_run_at,
  1633. "last_run_at": None,
  1634. "last_status": None,
  1635. "last_error": None,
  1636. "last_delivery_error": None,
  1637. # Targets acked without message_id/raw_response (accepted but UNVERIFIED).
  1638. "last_delivery_unverified": None,
  1639. # Most recent failed run ({at, detail}); NOT cleared by a later success (#118354).
  1640. "last_failure": None,
  1641. "failure_streak": 0,
  1642. "deliver": deliver,
  1643. "origin": origin, # Tracks where job was created for "origin" delivery
  1644. "enabled_toolsets": f["enabled_toolsets"],
  1645. "workdir": f["workdir"],
  1646. }
  1647. # Optional keys are persisted only when explicitly set: an absent key falls back to global
  1648. # config (attach/reasoning) or to ``deliver`` (failure_deliver), byte-identical to pre-feature
  1649. # jobs.
  1650. for key, value in (
  1651. ("attach_to_session", normalized_attach), ("reasoning_effort", normalized_reasoning_effort),
  1652. ("failure_deliver", f["failure_deliver"]), ("interpreter", f["interpreter"]),
  1653. ("timezone", f["timezone"]), ("grace_minutes", f["grace_minutes"]),
  1654. ("catch_up", f["catch_up"]),
  1655. ):
  1656. if value is not None:
  1657. job[key] = value
  1658. with _jobs_lock():
  1659. save_jobs(load_jobs() + [job])
  1660. return job
  1661. def get_job(job_id: str) -> Optional[dict[str, Any]]:
  1662. """Get a job by ID."""
  1663. job = next((j for j in load_jobs() if j["id"] == job_id), None)
  1664. return _normalize_job_record(job) if job is not None else None
  1665. class AmbiguousJobReference(LookupError):
  1666. """Raised when a job name matches more than one job."""
  1667. def __init__(self, ref: str, matches: list[dict[str, Any]]):
  1668. self.ref = ref
  1669. self.matches = matches
  1670. ids = ", ".join(m["id"] for m in matches)
  1671. super().__init__(
  1672. f"Job name '{ref}' is ambiguous — matches {len(matches)} jobs: {ids}. "
  1673. f"Use the job ID instead.")
  1674. def resolve_job_ref(ref: str) -> Optional[dict[str, Any]]:
  1675. """Resolve an ID or name to a job record. Exact ID wins, then case-insensitive name; an
  1676. ambiguous name raises AmbiguousJobReference rather than silently picking one."""
  1677. if not ref:
  1678. return None
  1679. jobs = load_jobs()
  1680. by_id = next((j for j in jobs if j["id"] == ref), None)
  1681. if by_id is not None:
  1682. return _normalize_job_record(by_id)
  1683. ref_lower = ref.lower()
  1684. name_matches = [j for j in jobs if (j.get("name") or "").lower() == ref_lower]
  1685. if not name_matches:
  1686. return None
  1687. if len(name_matches) > 1:
  1688. raise AmbiguousJobReference(ref, [_normalize_job_record(j) for j in name_matches])
  1689. return _normalize_job_record(name_matches[0])
  1690. def list_jobs(include_disabled: bool = False) -> list[dict[str, Any]]:
  1691. """List all jobs, optionally including disabled ones."""
  1692. jobs = [_normalize_job_record(j) for j in load_jobs()]
  1693. if not include_disabled:
  1694. jobs = [j for j in jobs if j.get("enabled", True)]
  1695. try:
  1696. from cron.executions import latest_executions, live_executions
  1697. ids = [job.get("id", "") for job in jobs]
  1698. latest, live = latest_executions(ids), live_executions(ids)
  1699. except Exception:
  1700. latest, live = {}, {}
  1701. for job in jobs:
  1702. job["latest_execution"] = latest.get(job.get("id", ""))
  1703. job["live_execution"] = live.get(job.get("id", ""))
  1704. return jobs
  1705. def _reject_terminal_activation(job: dict[str, Any], updated: dict[str, Any], job_id: str) -> None:
  1706. """A genuinely terminal job cannot be reactivated through update_job (use cron resume)."""
  1707. if (
  1708. is_terminal_job(job)
  1709. and not _is_recoverable_error_job(job)
  1710. and (
  1711. updated.get("state") not in {"completed", "error"}
  1712. or updated.get("enabled") is True
  1713. or updated.get("next_run_at") is not None
  1714. )
  1715. ):
  1716. raise ValueError(
  1717. f"Cannot activate terminal cron job '{job.get('name', job_id)}' "
  1718. "through update_job; use cron resume --run-now or --at.")
  1719. def _apply_pin_update(job: dict[str, Any], updates: dict[str, Any]) -> None:
  1720. """``pinned`` is not stored; it rewrites the per-job pin. ``True`` with no explicit model locks
  1721. the main agent's current provider+model onto the job, ``False`` releases both so the job follows
  1722. the main model again (an explicit ``model`` in the same update wins over either)."""
  1723. if "pinned" not in updates:
  1724. return
  1725. pinned = bool(updates.pop("pinned"))
  1726. if "model" in updates:
  1727. return
  1728. if pinned:
  1729. if not _normalize_job_optional_text(job.get("model")):
  1730. updates["provider"], updates["model"] = _main_model_pin()
  1731. else:
  1732. updates["provider"], updates["model"] = None, None
  1733. def _normalize_job_updates(job: dict[str, Any], updates: dict[str, Any]) -> None:
  1734. """Normalize updates in place like create_job; invalid values raise BEFORE the merge. ``repeat``
  1735. accepts the stored dict or a bare value (coerced, completed counter preserved)."""
  1736. for key, norm in _UPDATE_FIELD_NORMALIZERS.items():
  1737. if key in updates:
  1738. updates[key] = norm(updates[key])
  1739. if "repeat" in updates:
  1740. _rp = updates["repeat"]
  1741. completed = (job.get("repeat") or {}).get("completed", 0)
  1742. if isinstance(_rp, dict):
  1743. _rp = dict(_rp)
  1744. _rp["times"] = normalize_repeat_value(_rp.get("times"))
  1745. _rp.setdefault("completed", completed)
  1746. updates["repeat"] = _rp
  1747. else:
  1748. updates["repeat"] = {"times": normalize_repeat_value(_rp), "completed": completed}
  1749. def _rederive_repeat_for_schedule_change(
  1750. job: dict[str, Any], updates: dict[str, Any]
  1751. ) -> None:
  1752. """Re-derive the ``repeat`` default when a schedule update flips the kind.
  1753. ``create_job`` derives it from the schedule kind (once -> 1, recurring -> forever); the update
  1754. path must honour the same contract, otherwise a one-shot turned recurring keeps its ``times=1``
  1755. budget and retires after one fire, while a recurring job turned one-shot never completes. An
  1756. explicit ``repeat`` in the same update wins; a same-kind schedule edit leaves ``repeat`` alone.
  1757. """
  1758. if "schedule" not in updates or "repeat" in updates:
  1759. return
  1760. new_schedule = updates["schedule"]
  1761. if isinstance(new_schedule, str):
  1762. new_schedule = parse_schedule(new_schedule)
  1763. updates["schedule"] = new_schedule
  1764. old_kind = (job.get("schedule") or {}).get("kind")
  1765. new_kind = new_schedule.get("kind")
  1766. if old_kind == new_kind:
  1767. return
  1768. repeat = dict(job.get("repeat") or {})
  1769. times = repeat.get("times")
  1770. if new_kind == "once" and times is None:
  1771. repeat["times"] = 1
  1772. elif new_kind != "once" and old_kind == "once" and times == 1:
  1773. repeat["times"] = None
  1774. else:
  1775. return
  1776. repeat.setdefault("completed", 0)
  1777. updates["repeat"] = repeat
  1778. def _apply_schedule_update(updated: dict[str, Any], updates: dict[str, Any], job_id: str) -> None:
  1779. """Parse a string schedule, refresh ``schedule_display`` and (unless paused) ``next_run_at``."""
  1780. updated_schedule = updated["schedule"]
  1781. if isinstance(updated_schedule, str):
  1782. updated_schedule = parse_schedule(updated_schedule)
  1783. updated["schedule"] = updated_schedule
  1784. updated["schedule_display"] = updates.get(
  1785. "schedule_display", updated_schedule.get("display", updated.get("schedule_display")))
  1786. if updated.get("state") != "paused":
  1787. updated["next_run_at"] = _next_run_or_reject_past_oneshot(
  1788. updated_schedule, updated.get("name", job_id), updated_schedule, "update ",
  1789. updated.get("timezone"))
  1790. def _fill_missing_next_run(updated: dict[str, Any]) -> None:
  1791. """An enabled, unpaused record must never persist without ``next_run_at`` (it would never fire).
  1792. """
  1793. if (
  1794. not updated.get("enabled", True)
  1795. or updated.get("state") == "paused"
  1796. or updated.get("next_run_at")
  1797. ):
  1798. return
  1799. next_run = compute_next_run(updated["schedule"], timezone=updated.get("timezone"))
  1800. if next_run is None and updated["schedule"].get("kind") == "once":
  1801. run_at = updated["schedule"].get("run_at", "unknown")
  1802. raise ValueError(
  1803. f"Requested one-shot time {run_at} is in the past "
  1804. f"(grace window: {ONESHOT_GRACE_SECONDS}s) and cannot be scheduled.")
  1805. updated["next_run_at"] = next_run
  1806. def update_job(job_id: str, updates: dict[str, Any]) -> Optional[dict[str, Any]]:
  1807. """Update a job by ID, refreshing derived schedule fields when needed."""
  1808. # ``id`` is a path component under OUTPUT_DIR — changing it would leak path-escape values.
  1809. bad_fields = _IMMUTABLE_JOB_FIELDS.intersection(updates or {})
  1810. if bad_fields:
  1811. raise ValueError(f"Cron job field(s) cannot be updated: {', '.join(sorted(bad_fields))}")
  1812. def apply(jobs, i, job):
  1813. _rederive_repeat_for_schedule_change(job, updates)
  1814. _normalize_job_updates(job, updates)
  1815. _apply_pin_update(job, updates)
  1816. updated = _apply_skill_fields({**job, **updates})
  1817. _reject_terminal_activation(job, updated, job_id)
  1818. # Re-check on the MERGED record; scoped to changed fields so legacy records keep loading.
  1819. if {"monitor_script", "monitor_url", "no_agent", "script"}.intersection(updates):
  1820. _validate_job_mode_invariants(
  1821. updated.get("monitor_script") or None,
  1822. updated.get("monitor_url") or None,
  1823. bool(updated.get("no_agent")),
  1824. _normalize_job_optional_text(updated.get("script")))
  1825. if any(k in updates for k in _PAYLOAD_FIELDS) and job_payload_is_empty(updated):
  1826. raise ValueError(EMPTY_PAYLOAD_ERROR)
  1827. if {"timezone", "grace_minutes"}.intersection(updates):
  1828. validate_schedule_knobs(
  1829. updated.get("schedule") or {}, updated.get("timezone"),
  1830. updated.get("grace_minutes"))
  1831. jobs_time.prune_inherited_knobs(updated)
  1832. if "schedule" in updates:
  1833. _apply_schedule_update(updated, updates, job_id)
  1834. # next_run_at now follows the new schedule; a stale quota_hold_until would only shield
  1835. # the record from the stale-error re-arm while no longer describing where it is
  1836. # parked. The next fire re-parks (with a fresh notice) if the window is still closed.
  1837. from cron.quota_hold import clear_state as _clear_quota_hold
  1838. _clear_quota_hold(updated)
  1839. elif "timezone" in updates and updated.get("state") != "paused":
  1840. # The instant on file was computed in the OLD zone; leaving it would make the new
  1841. # zone fire it at the wrong wall clock (or skip it). Only overwrite when the new
  1842. # zone can compute one (a past one-shot keeps its recorded instant).
  1843. recomputed = compute_next_run(
  1844. updated["schedule"], timezone=updated.get("timezone"))
  1845. if recomputed is not None:
  1846. updated["next_run_at"] = recomputed
  1847. if {"schedule", "next_run_at", "enabled", "state", "timezone"}.intersection(updates):
  1848. # An explicit schedule/lifecycle rewrite supersedes any occurrence the dispatcher
  1849. # left unclaimed — pause/resume/edit must not resurrect a slot from before the edit.
  1850. updated.pop("pending_slot", None)
  1851. _fill_missing_next_run(updated)
  1852. _reject_terminal_activation(job, updated, job_id)
  1853. jobs[i] = updated
  1854. save_jobs(jobs)
  1855. return _normalize_job_record(updated)
  1856. return _with_job(job_id, apply)
  1857. def pause_job(job_id: str, reason: Optional[str] = None) -> Optional[dict[str, Any]]:
  1858. """Pause a job without deleting it. Accepts a job ID or name."""
  1859. job = resolve_job_ref(job_id)
  1860. if not job:
  1861. return None
  1862. return update_job(job["id"], {
  1863. "enabled": False,
  1864. "state": "paused",
  1865. "paused_at": _hermes_now().isoformat(),
  1866. "paused_reason": reason,
  1867. })
  1868. def resume_job(job_id: str) -> Optional[dict[str, Any]]:
  1869. """Resume a paused job. Accepts a job ID or name.
  1870. A recurring job paused across one of its slots must not lose that slot silently: the stored
  1871. ``next_run_at`` (already past) survives resume as the due instant, and the ordinary late /
  1872. catch-up / ``cron.catch_up_missed`` policy in the due scan decides what happens to it — one
  1873. fire or a logged skip, never a silent re-anchor past it (#113603). One-shots and future
  1874. instants recompute from now as before.
  1875. """
  1876. job = resolve_job_ref(job_id)
  1877. if not job:
  1878. return None
  1879. stored_next = job.get("next_run_at")
  1880. stored_dt = _parse_aware(stored_next) if stored_next else None
  1881. if (
  1882. job["schedule"].get("kind") in {"cron", "interval"}
  1883. and stored_dt is not None
  1884. and _instant_at_or_before(stored_dt, _hermes_now())
  1885. ):
  1886. next_run_at = stored_next
  1887. logger.info(
  1888. "Job '%s' resumed with occurrence %s that elapsed while paused kept due; the next "
  1889. "tick fires it (late/catch-up) or logs the skip.",
  1890. job.get("name", job["id"]), stored_next)
  1891. else:
  1892. next_run_at = compute_next_run(job["schedule"], timezone=job.get("timezone"))
  1893. if next_run_at is None and job["schedule"].get("kind") == "once":
  1894. run_at = job["schedule"].get("run_at", "unknown")
  1895. raise ValueError(
  1896. f"Cannot resume: one-shot time {run_at} is in the past "
  1897. f"(grace window: {ONESHOT_GRACE_SECONDS}s) and will never fire.")
  1898. return update_job(job["id"], {
  1899. "enabled": True,
  1900. "state": "scheduled",
  1901. "paused_at": None,
  1902. "paused_reason": None,
  1903. "next_run_at": next_run_at,
  1904. })
  1905. def trigger_job(job_id: str, extra_prompt: Optional[str] = None) -> Optional[dict[str, Any]]:
  1906. """Schedule a job for the next tick (ID or name). ``extra_prompt`` is stamped as
  1907. ``manual_run_prompt`` for that single fire only; ``mark_job_run`` clears it."""
  1908. job = resolve_job_ref(job_id)
  1909. if not job:
  1910. return None
  1911. if is_terminal_job(job):
  1912. name = job.get("name", job_id)
  1913. raise ValueError(
  1914. f"Cannot run: job '{name}' is {job.get('state')} (terminal). "
  1915. f"Create a new occurrence with 'hermes cron resume {name} "
  1916. "--run-now' or '--at <ISO-8601>'.")
  1917. manual_run_at = _hermes_now().isoformat()
  1918. return update_job(job["id"], {
  1919. "enabled": True,
  1920. "state": "scheduled",
  1921. "paused_at": None,
  1922. "paused_reason": None,
  1923. "next_run_at": manual_run_at,
  1924. # Run-now intent, so cron expression/TZ repair guards don't treat it as stale state; the
  1925. # overwritten slot comes back at claim time (an extra fire, cron/extra_fire.py).
  1926. "manual_run_at": manual_run_at, extra_fire.RESTORE_KEY: extra_fire.restore_target(job),
  1927. "manual_run_prompt": (extra_prompt or None),
  1928. })
  1929. def _claim_is_live(claim: Any, now: datetime, ttl_seconds: float) -> bool:
  1930. """True for a well-formed claim aged within ``[0, ttl)`` whose owner is not provably dead:
  1931. future-dated (clock/TZ skew) or malformed claims count as stale so they can never wedge a
  1932. job, and a same-host owner pid that has exited releases the claim immediately instead of
  1933. after the TTL (a killed ``hermes cron run`` otherwise blocks the next manual run for the
  1934. full window with "already being fired")."""
  1935. if not isinstance(claim, dict) or not claim.get("at"):
  1936. return False
  1937. claimed_at = _parse_aware(claim["at"])
  1938. if claimed_at is None or not (0 <= _elapsed_seconds(now, claimed_at) < ttl_seconds):
  1939. return False
  1940. return not _claim_owner_is_dead(claim)
  1941. _REARM_RECURRING_ERROR = (
  1942. "Cannot re-arm recurring jobs: re-arm is one-shot-only; use plain resume or cron run."
  1943. )
  1944. def rearm_oneshot(job_id: str, run_at: Any) -> Optional[dict[str, Any]]:
  1945. """Re-arm a completed one-shot as an explicit new occurrence."""
  1946. job_ref = resolve_job_ref(job_id)
  1947. if not job_ref:
  1948. return None
  1949. if isinstance(run_at, datetime):
  1950. run_at = run_at.isoformat()
  1951. parsed_schedule = parse_schedule(str(run_at))
  1952. if parsed_schedule.get("kind") != "once":
  1953. raise ValueError(_REARM_RECURRING_ERROR)
  1954. next_run_at = compute_next_run(parsed_schedule, timezone=job_ref.get("timezone"))
  1955. if next_run_at is None:
  1956. raise _oneshot_past_grace_error(parsed_schedule.get("run_at") or run_at)
  1957. def apply(jobs, _i, job):
  1958. now = _hermes_now()
  1959. if _claim_is_live(job.get("run_claim"), now, _oneshot_run_claim_ttl_seconds()):
  1960. raise ValueError("Cannot re-arm one-shot over a live run claim.")
  1961. if _claim_is_live(job.get("fire_claim"), now, FIRE_CLAIM_TTL_SECONDS):
  1962. raise ValueError("Cannot re-arm one-shot over a live fire claim.")
  1963. if job.get("schedule", {}).get("kind") != "once":
  1964. raise ValueError(_REARM_RECURRING_ERROR)
  1965. repeat = job.get("repeat") or {}
  1966. repeat["completed"] = 0
  1967. job.update(
  1968. schedule=parsed_schedule, schedule_display=parsed_schedule.get("display", str(run_at)),
  1969. repeat=repeat, run_claim=None, fire_claim=None)
  1970. _activate_job_record(job)
  1971. job["next_run_at"] = next_run_at
  1972. save_jobs(jobs)
  1973. return _normalize_job_record(job)
  1974. return _with_job(job_ref["id"], apply)
  1975. def remove_job(job_id: str) -> bool:
  1976. """Remove a job by ID or name."""
  1977. job = resolve_job_ref(job_id)
  1978. if not job:
  1979. return False
  1980. canonical_id = job["id"]
  1981. with _jobs_lock():
  1982. jobs = load_jobs()
  1983. original_len = len(jobs)
  1984. jobs = [j for j in jobs if j["id"] != canonical_id]
  1985. if len(jobs) == original_len:
  1986. return False
  1987. # Resolve BEFORE saving so a legacy unsafe ID fails closed without a half-applied removal.
  1988. job_output_dir = _job_output_dir(canonical_id)
  1989. save_jobs(jobs, removed_ids={canonical_id})
  1990. marker = _self_removal_delivery.get()
  1991. if marker is not None and marker.job_id == canonical_id:
  1992. marker.removed = True
  1993. else: # someone else removed it: its live run must stop, not burn tokens to a discard
  1994. cancel_job_run(canonical_id, reason="job removed while running")
  1995. if job_output_dir.exists():
  1996. shutil.rmtree(job_output_dir)
  1997. try:
  1998. from cron.notepad import clear_notepad
  1999. clear_notepad(canonical_id)
  2000. except Exception:
  2001. logger.debug("Failed to clear notepad for removed job %s", canonical_id, exc_info=True)
  2002. # Prune the fire-fence lock entry so the registry doesn't grow monotonically.
  2003. _fence_key = f"{_current_cron_store().cron_dir.resolve()}::{canonical_id}"
  2004. with _fire_fence_locks_guard:
  2005. _fire_fence_locks.pop(_fence_key, None)
  2006. return True
  2007. def cancel_job_run(job_id: str, reason: Optional[str] = None) -> dict[str, Any]:
  2008. """Stop the job's live run (if any); its schedule is untouched. See ``cron.cancellation``."""
  2009. from cron.cancellation import cancel_run
  2010. return cancel_run(job_id, reason=reason)
  2011. def _set_alert_flag(job_id: str, field: str, value: bool) -> bool:
  2012. """Set/clear a persisted alert-dedup marker (alert exactly once until the condition heals;
  2013. survives restarts) and return the PRIOR value. Field: ``preflight_alerted`` (blocked config).
  2014. The marker records that the operator was already alerted about this job's condition, so the scheduler
  2015. alerts exactly once and stays silent on subsequent ticks until the condition heals (same alert-once
  2016. shape as the dead-pin auto-pause in #73506).
  2017. """
  2018. def apply(jobs, _i, job):
  2019. prior = bool(job.get(field))
  2020. if value:
  2021. job[field] = True
  2022. else:
  2023. job.pop(field, None)
  2024. if prior != value:
  2025. save_jobs(jobs)
  2026. return prior
  2027. return _with_job(job_id, apply, False)
  2028. def mark_preflight_alerted(job_id: str) -> bool:
  2029. """Mark the job as preflight-alerted; return True if it already was."""
  2030. return _set_alert_flag(job_id, "preflight_alerted", True)
  2031. def clear_preflight_alerted(job_id: str) -> None:
  2032. """Clear the preflight alert-dedup marker (config validates again)."""
  2033. _set_alert_flag(job_id, "preflight_alerted", False)
  2034. def note_fire_forward_failure(job_id: str, detail: str) -> bool:
  2035. """Durably record (as ``last_fire_error``) that a scheduled fire could not be handed to the
  2036. runner — written by the dashboard fire webhook when the loopback forward fails. Without it
  2037. the miss is invisible (no execution row, last_status only covers started runs); mark_job_run
  2038. clears it."""
  2039. def apply(jobs, _i, job):
  2040. job["last_fire_error"] = {
  2041. "at": _hermes_now().isoformat(), "detail": str(detail or "")[:500]}
  2042. save_jobs(jobs)
  2043. return True
  2044. return _with_job(job_id, apply, False)
  2045. def _record_run_outcome(
  2046. job: dict[str, Any], success: bool, error: Optional[str], delivery_error: Optional[str],
  2047. status: Optional[str], now: str,
  2048. ) -> None:
  2049. """Stamp one completed run onto *job*: status fields, failure streak, alert markers, claims."""
  2050. job["last_run_at"] = now
  2051. job.pop("manual_run_at", None)
  2052. job.pop("manual_run_restore_at", None)
  2053. # The transient manual-run context is single-fire: the run that just completed consumed it.
  2054. job.pop("manual_run_prompt", None)
  2055. delivery_failed = isinstance(delivery_error, str) and bool(delivery_error.strip())
  2056. job["last_status"] = status or (
  2057. "error" if not success else ("delivery_failed" if delivery_failed else "ok"))
  2058. cancelled = status == "cancelled" # an operator stop is not a failure
  2059. job["last_error"] = None if success or cancelled else error
  2060. if success:
  2061. # Healthy run: drop the alert-once dedup markers so a FUTURE break re-alerts, and clear
  2062. # the forward-failure stamp so it only describes CURRENT auto-fire health.
  2063. job.pop("preflight_alerted", None)
  2064. job.pop("last_fire_error", None)
  2065. job["failure_streak"] = 0
  2066. elif not cancelled:
  2067. # Consecutive agent-failure streak; delivery failures do NOT count
  2068. # (scheduler._failure_streak_nudge).
  2069. job["failure_streak"] = int(job.get("failure_streak") or 0) + 1
  2070. # Sticky last-failure stamp (#118354): the next success resets last_status and
  2071. # failure_streak, which erases the only job-level trace that a run ever failed —
  2072. # a monitor sampling jobs.json then sees a permanently green job. last_failure
  2073. # survives success (latest failure wins); the recency window is the consumer's
  2074. # call. Delivery failures keep their own sticky last_delivery_error.
  2075. job["last_failure"] = {"at": now, "detail": error or (status or "run failed")}
  2076. job["last_delivery_error"] = delivery_error
  2077. # Clear both claims: the run is over, so the job is claimable again.
  2078. job["fire_claim"] = None
  2079. job.pop("pending_slot", None)
  2080. if job.get("run_claim") is not None: # keep key absence for legacy records
  2081. job["run_claim"] = None
  2082. def _advance_after_run(job: dict[str, Any], now: str, *, ladder_rung: bool = False) -> None:
  2083. """Bump ``repeat.completed`` and recompute ``next_run_at``; retire the record as a terminal
  2084. completion when the repeat limit is reached or a one-shot has no further run.
  2085. ``ladder_rung``: this run re-ran an occurrence that already counted (an unreachable-model
  2086. re-run, ``cron.unreachable_retry.is_retry_run``), so ``repeat.completed`` is left as is."""
  2087. # If no next run, decide whether this is terminal completion (one-shot) or a transient failure
  2088. # (recurring schedule couldn't compute — e.g. 'croniter' missing from the runtime env). Recurring jobs
  2089. # must NEVER be silently disabled: that turns a missing runtime dep into "job completed" and the user's
  2090. # schedule quietly goes off. See issue #16265.
  2091. kind = job.get("schedule", {}).get("kind")
  2092. # One-shot dispatch-limit guard (issue #38758): a finite one-shot claimed via claim_dispatch() but whose
  2093. # tick died before mark_job_run could remove it will have completed >= times while still looking due
  2094. # (last_run_at was never written, so the recovery helper re-armed it). Remove it instead of re-firing.
  2095. repeat = job.get("repeat")
  2096. if repeat:
  2097. times = repeat.get("times")
  2098. finite = times is not None and times > 0
  2099. completed = repeat.get("completed", 0)
  2100. # Count each occurrence once. A ladder re-run's occurrence already counted, and finite
  2101. # one-shots were pre-claimed by claim_dispatch() (completed already incremented); every
  2102. # other run, recurring or direct, gets the increment.
  2103. if not ladder_rung and not (kind == "once" and finite and completed > 0):
  2104. completed += 1
  2105. repeat["completed"] = completed
  2106. if finite and completed >= times:
  2107. # Limit reached: retain a terminal record instead of popping it, so the status just
  2108. # written stays inspectable in `cronjob list`; the retention sweep prunes it later.
  2109. _complete_job_record(job)
  2110. return
  2111. job["next_run_at"] = compute_next_run(
  2112. job["schedule"], now, timezone=job.get("timezone"))
  2113. if job["next_run_at"] is not None:
  2114. if job.get("state") != "paused":
  2115. job["state"] = "scheduled"
  2116. elif kind in {"cron", "interval"}:
  2117. # Recurring: transient failure (e.g. croniter missing) — disabling it would turn a missing
  2118. # dep into "job completed" and silently drop the schedule.
  2119. job["state"] = "error"
  2120. if not job.get("last_error"):
  2121. job["last_error"] = (
  2122. "Failed to compute next run for recurring schedule (is the 'croniter' package "
  2123. "installed in the gateway's Python env?)")
  2124. logger.error(
  2125. "Job '%s' (%s) could not compute next_run_at; "
  2126. "leaving enabled and marking state=error so the job is not silently disabled.",
  2127. job.get("name", job.get("id", "?")), kind)
  2128. else:
  2129. _complete_job_record(job) # one-shot: terminal completion
  2130. def mark_job_run(
  2131. job_id: str,
  2132. success: bool,
  2133. error: Optional[str] = None,
  2134. delivery_error: Optional[str] = None,
  2135. status: Optional[str] = None,
  2136. *,
  2137. expected_fire_owner: Optional[str] = None,
  2138. execution_id: Optional[str] = None,
  2139. model_unreachable: bool = False,
  2140. ladder_rung: bool = False,
  2141. quota_hold_seconds: Optional[float] = None,
  2142. recover_consumed_fire: bool = False,
  2143. ) -> bool:
  2144. """Mark a job as run: update last_run_at/last_status, bump completed, recompute next_run_at,
  2145. and retire the record as a terminal completion when the repeat limit is reached. A manual run
  2146. (an extra fire, ``cron/extra_fire.py``) bumps nothing and keeps the pending occurrence.
  2147. ``delivery_error`` is separate from the agent error: agent succeeded but delivery failed records
  2148. ``last_status = "delivery_failed"`` (never "ok") while ``failure_streak`` is left alone. An
  2149. explicit ``status`` (e.g. "blocked_config") overrides the derived value. False when the fence
  2150. can't be taken, the job is missing, or ``expected_fire_owner`` no longer holds the fire claim.
  2151. ``model_unreachable``: this failed run never reached the model (transient network/DNS error,
  2152. zero API calls). Recurring jobs then get a bounded automatic re-run — ``next_run_at`` is pulled
  2153. earlier per ``cron.unreachable_retry.RETRY_DELAYS_SECONDS`` — instead of waiting a full period
  2154. (Cowork-style; see cron/unreachable_retry.py). ``ladder_rung``: this run IS one of those
  2155. re-runs (``is_retry_run``); its occurrence already counted, so ``repeat.completed`` is not
  2156. bumped again.
  2157. ``quota_hold_seconds``: the provider said it stays closed for this long (a quota 429 with
  2158. ``retry after <N>s``). Recurring jobs are parked through the window instead of re-firing into
  2159. it on every tick. ``recover_consumed_fire`` lets a scheduled sparse cron recover its
  2160. consumed fire when the provider reopens; manual runs retain the natural schedule
  2161. (cron/quota_hold.py, #89376).
  2162. """
  2163. def apply(jobs, _i, job):
  2164. if execution_id is not None and execution_id in job.get("canonical_completions", []):
  2165. return True
  2166. if expected_fire_owner is not None:
  2167. claim = job.get("fire_claim")
  2168. if not isinstance(claim, dict) or claim.get("by") != expected_fire_owner:
  2169. logger.warning(
  2170. "mark_job_run: job_id %s fire claim owner changed; discarding stale completion",
  2171. job_id)
  2172. return False
  2173. now = _hermes_now().isoformat()
  2174. extra = extra_fire.is_extra(job) # read before the outcome clears the claim
  2175. held = extra_fire.held_markers(job) if extra else {}
  2176. _record_run_outcome(job, success, error, delivery_error, status, now)
  2177. job.update(held) # a run-now queued mid-run is its own fire, not this one
  2178. if execution_id is not None: # retire ids recovery can no longer replay; latest stays last
  2179. from cron.scheduler_authority import replayable_completions as live
  2180. job["canonical_completions"] = [*live(job_id, job.get("canonical_completions")), execution_id]
  2181. if extra: # a manual run: repeat budget and pending occurrence untouched
  2182. extra_fire.settle(job, now)
  2183. else:
  2184. _advance_after_run(job, now, ladder_rung=ladder_rung)
  2185. from cron import quota_hold
  2186. from cron.unreachable_retry import clear_state, plan_retry
  2187. if not success and model_unreachable and not is_terminal_job(job):
  2188. plan_retry(job)
  2189. else:
  2190. # Any run that reached the model (either outcome) resets the re-run ladder.
  2191. clear_state(job)
  2192. if not success and quota_hold_seconds and not is_terminal_job(job):
  2193. quota_hold.plan_hold(job, quota_hold_seconds, recover_consumed_fire=recover_consumed_fire)
  2194. else:
  2195. quota_hold.clear_state(job)
  2196. save_jobs(jobs)
  2197. return True
  2198. def locked():
  2199. found = _with_job(job_id, apply, missing=_MISSING)
  2200. if found is _MISSING:
  2201. logger.warning("mark_job_run: job_id %s not found, skipping save", job_id)
  2202. return False
  2203. return found
  2204. return _under_fire_fence(job_id, locked)
  2205. def claim_dispatch(job_id: str) -> bool:
  2206. """Atomically claim a finite one-shot dispatch BEFORE execution: ``repeat.completed`` is bumped
  2207. and persisted under the jobs lock so a tick dying mid-execution cannot lose the dispatch
  2208. (*at-most-times* instead of *at-least-once*). True if the caller may run the job; False when
  2209. the limit is already reached. Only ``kind == "once"`` with ``repeat.times > 0`` is claimed.
  2210. Increments ``repeat.completed`` under the cross-process jobs lock and persists the claim immediately, so
  2211. that if the tick dies mid-execution (gateway kill, OOM, segfault, hard-timeout) the dispatch is not
  2212. lost. This converts finite one-shot jobs from *at-least-once* to *at-most-times* semantics — a job that
  2213. self-destructs fires at most ``repeat.times`` times instead of infinitely (issue #38758).
  2214. """
  2215. def apply(jobs, i, job):
  2216. repeat = job.get("repeat") or {}
  2217. times = repeat.get("times")
  2218. # Recurring jobs use advance_next_run(); no/infinite repeat limit always dispatches.
  2219. if job.get("schedule", {}).get("kind") != "once" or times is None or times <= 0:
  2220. return True
  2221. if extra_fire.is_extra(job): # a manual run leaves the scheduled fire's budget alone
  2222. return True
  2223. completed = repeat.get("completed", 0)
  2224. label = job.get("name", job.get("id", "?"))
  2225. if completed >= times:
  2226. if job.get("last_run_at") is not None:
  2227. # A prior run completed normally (mark_job_run raced this tick). Retain the terminal
  2228. # record, as mark_job_run's repeat-limit branch does, instead of deleting the
  2229. # status.
  2230. _complete_job_record(job)
  2231. save_jobs(jobs)
  2232. logger.info(
  2233. "Job '%s': dispatch limit reached (%d/%d) — marking completed",
  2234. label, completed, times)
  2235. return False
  2236. # A prior tick claimed the dispatch then died — a genuinely wedged claim. Remove it so
  2237. # it stops appearing due, leaving an operator-visible diagnostic.
  2238. jobs.pop(i)
  2239. # See #73973.
  2240. save_jobs(jobs, removed_ids={job_id})
  2241. from cron.jobs_oneshot_diagnostics import write_wedged
  2242. write_wedged(job)
  2243. logger.info(
  2244. "Job '%s': dispatch limit reached (%d/%d) — removing", label, completed, times)
  2245. return False
  2246. # Claim this dispatch before the side effect runs.
  2247. repeat["completed"] = completed + 1
  2248. save_jobs(jobs)
  2249. logger.debug("Job '%s': claimed dispatch %d/%d", label, repeat["completed"], times)
  2250. return True
  2251. claimed = _with_job(job_id, apply, missing=_MISSING)
  2252. if claimed is _MISSING:
  2253. logger.debug(
  2254. "claim_dispatch: job_id %s not in store — proceeding without claim "
  2255. "(handed-in job dict; nothing to persist a claim against)",
  2256. job_id)
  2257. return True
  2258. return claimed
  2259. def _refresh_claim(jobs: list[dict[str, Any]], claim: Any, expected_owner: str) -> bool:
  2260. """Compare-and-refresh a claim's ``at`` stamp; False unless *expected_owner* still holds it."""
  2261. if not isinstance(claim, dict) or claim.get("by") != expected_owner:
  2262. return False
  2263. claim["at"] = _hermes_now().isoformat()
  2264. save_jobs(jobs)
  2265. return True
  2266. def heartbeat_run_claim(job_id: str, *, expected_owner: str) -> bool:
  2267. """Refresh a one-shot's ``run_claim`` timestamp while its run is alive, so an expired claim
  2268. really means the claiming process died. Compare-and-refresh on ``expected_owner`` stops a stale
  2269. runner from extending a claim another process has since taken over.
  2270. Called periodically from the scheduler's run monitor (#62002) so a legitimately long run keeps its claim
  2271. fresh: an expired claim then really does mean "the claiming process died", and neither another process's
  2272. tick nor this process's own next tick will re-dispatch or stale-remove the job while the run is in
  2273. flight. mark_job_run() clears the claim on completion.
  2274. """
  2275. def apply(jobs, _i, job):
  2276. if job.get("schedule", {}).get("kind") != "once":
  2277. return False
  2278. return _refresh_claim(jobs, job.get("run_claim"), expected_owner)
  2279. return _with_job(job_id, apply, False)
  2280. def clear_run_claim(job_id: str) -> bool:
  2281. """Clear a one-shot's ``run_claim`` when dispatch itself fails: such a job never reaches
  2282. mark_job_run, so the stale claim would block re-dispatch until the TTL expires.
  2283. Calling this on every early-exit path restores the "the job stays due and will fire on the next healthy
  2284. tick" invariant that the scheduler comment promises (#86522).
  2285. """
  2286. def apply(jobs, _i, job):
  2287. claim = job.get("run_claim")
  2288. if job.get("schedule", {}).get("kind") != "once" or claim in (None, {"outage": True}):
  2289. return False # recurring, already cleared, or already reduced to the outage marker
  2290. # Keep the outage marker (no "at": a stale claim, so it re-dispatches) or the skipped
  2291. # one-shot, past its grace, is retired as missed on the next scan.
  2292. job["run_claim"] = {"outage": True} if isinstance(claim, dict) and claim.get("outage") else None
  2293. save_jobs(jobs)
  2294. return True
  2295. return _with_job(job_id, apply, False)
  2296. def advance_next_runs(job_ids) -> int:
  2297. """Batch form of :func:`advance_next_run`: one load + at most one save for the whole due set;
  2298. one-shot/unknown ids are skipped. Returns the count advanced. Persisted once at the end, so a
  2299. crash mid-batch re-fires the whole set on restart rather than a prefix (sub-10ms window)."""
  2300. ids = set(job_ids)
  2301. if not ids:
  2302. return 0
  2303. with _jobs_lock():
  2304. jobs = load_jobs()
  2305. now = _hermes_now().isoformat()
  2306. advanced = 0
  2307. for job in jobs:
  2308. if (
  2309. job["id"] not in ids
  2310. or (is_terminal_job(job) and not _is_recoverable_error_job(job))
  2311. or job.get("schedule", {}).get("kind") not in {"cron", "interval"}
  2312. or extra_fire.queued(job) # the claim restores its slot; advancing erases the intent
  2313. ):
  2314. continue
  2315. new_next = compute_next_run(
  2316. job["schedule"], now, timezone=job.get("timezone"))
  2317. if new_next and new_next != job.get("next_run_at"):
  2318. job["next_run_at"] = new_next
  2319. advanced += 1
  2320. if advanced:
  2321. save_jobs(jobs)
  2322. return advanced
  2323. def advance_next_run(job_id: str) -> bool:
  2324. """Advance a recurring job's next_run_at BEFORE run_job() so a mid-run crash cannot re-fire it
  2325. on restart (at-most-once for recurring jobs — one missed run beats a crash-loop burst).
  2326. One-shots are left unchanged so they can retry. Returns True if next_run_at was advanced."""
  2327. # >= 1 (not == 1): duplicate ids in a corrupted file all advance; still report the advance.
  2328. return advance_next_runs([job_id]) >= 1
  2329. def _machine_id() -> str:
  2330. """Claim attribution/debugging id (NOT correctness — that comes from the file lock and the
  2331. fresh-claim check): ``HERMES_MACHINE_ID`` if set, else hostname:pid."""
  2332. explicit = os.getenv("HERMES_MACHINE_ID", "").strip()
  2333. if explicit:
  2334. return explicit
  2335. try:
  2336. import socket
  2337. host = socket.gethostname()
  2338. except Exception:
  2339. host = "unknown"
  2340. return f"{host}:{os.getpid()}"
  2341. def claim_job_for_fire(
  2342. job_id: str, *, claim_ttl_seconds: int = FIRE_CLAIM_TTL_SECONDS, force: bool = False,
  2343. manual: bool = False, return_job: bool = False,
  2344. ) -> bool | dict[str, Any]:
  2345. """Atomically claim a job for one external 'fire' (multi-machine at-most-once); True iff THIS
  2346. caller won (``CronScheduler.fire_due``: exactly one of N replicas runs a job). Under the
  2347. fence + file lock: reject missing/terminal/paused jobs unless ``force`` (explicit manual
  2348. fire, which also resumes the job atomically; external callbacks must leave it false so a
  2349. stale callback cannot resurrect a paused job). ``manual`` = off-tick run-now without the
  2350. resume: no occurrence stamp, so the still-pending ``next_run_at`` slot is not skipped. Lose if a claim younger than
  2351. ``claim_ttl_seconds`` exists (the TTL lets another fire reclaim after a crash; mark_job_run
  2352. clears the claim). Otherwise stamp ``fire_claim`` and, for recurring jobs, advance
  2353. ``next_run_at`` so a stale re-delivery cannot re-fire."""
  2354. def apply(jobs, _i, job):
  2355. if is_terminal_job(job) and not _is_recoverable_error_job(job):
  2356. return False
  2357. # Both enabled and pause markers must clear — a half-paused record must not claim. ``force``
  2358. # (Trigger-now on a paused job) bypasses the gate and atomically resumes the job below.
  2359. if not force and not is_job_runnable(job):
  2360. return False
  2361. now = _hermes_now()
  2362. if _claim_is_live(job.get("fire_claim"), now, claim_ttl_seconds):
  2363. return False # someone holds a fresh claim
  2364. from cron.occurrences import completed_occurrence, scheduled_instant
  2365. # ``manual`` (an off-tick run-now) must NOT stamp an occurrence identity: outside a
  2366. # scheduler tick ``next_run_at`` is the NEXT occurrence, not the one being run, so
  2367. # stamping it would make completed_occurrence() skip that slot when it arrives.
  2368. manual_fire = force or manual or job.get("manual_run_at") == job.get("next_run_at")
  2369. instant = None if manual_fire else scheduled_instant(job.get("next_run_at"))
  2370. # A scheduled tick only ever fires when now >= next_run_at
  2371. # (_evaluate_due_job returns False while the stored occurrence is still
  2372. # in the future), so a claim arriving BEFORE the stored next occurrence
  2373. # cannot be the tick that owns it — it is a manual / dashboard / webhook
  2374. # fire and must stay occurrence-free. Binding it would make run_one_job
  2375. # stamp that FUTURE instant completed in the ledger: later manual fires
  2376. # are then refused ("Job is already being fired by the scheduler") and
  2377. # the scheduled tick dedupe-skips its real delivery (2026-09-08 live:
  2378. # a manual run at 19:53 consumed the next day's 19:00 occurrence).
  2379. # A claim within FIRE_CLAIM_SKEW_SECONDS of the slot is the fire for that slot
  2380. # (provider clock skew); dropping its identity would leave the slot unrecorded, so
  2381. # mark_job_run recomputes the same cron slot and the misfire backstop runs it twice.
  2382. if (instant is not None
  2383. and datetime.fromisoformat(instant) - now >= timedelta(seconds=FIRE_CLAIM_SKEW_SECONDS)):
  2384. instant = None
  2385. if instant and completed_occurrence(job, instant):
  2386. if job.get("schedule", {}).get("kind") in {"cron", "interval"}:
  2387. nxt = compute_next_run(
  2388. job["schedule"], now.isoformat(), timezone=job.get("timezone"))
  2389. if nxt:
  2390. job["next_run_at"] = nxt
  2391. save_jobs(jobs)
  2392. return False
  2393. if force:
  2394. _activate_job_record(job)
  2395. # Per-acquisition token: a process may legitimately reclaim its own stale lease, and the
  2396. # previous runner must not heartbeat the new claim merely because hostname + PID match.
  2397. job["fire_claim"] = {"at": now.isoformat(), "by": f"{_machine_id()}:{uuid.uuid4().hex}"}
  2398. # Claimed: the occurrence is now owned by a run (its ledger row + fire claim carry it).
  2399. job.pop("pending_slot", None)
  2400. if slot := extra_fire.pending_slot(job, now, manual_fire):
  2401. extra_fire.stamp_claim(job, slot) # off-schedule: the pending occurrence stays
  2402. elif job.get("schedule", {}).get("kind") in {"cron", "interval"}:
  2403. nxt = compute_next_run(
  2404. job["schedule"], now.isoformat(), timezone=job.get("timezone"))
  2405. if nxt:
  2406. job["next_run_at"] = nxt
  2407. save_jobs(jobs)
  2408. return dict(copy.deepcopy(job), _scheduled_instant=instant) if return_job else True
  2409. return _under_fire_fence(job_id, lambda: _with_job(job_id, apply, False))
  2410. def heartbeat_fire_claim(job_id: str, *, expected_owner: str) -> bool:
  2411. """Refresh an active ``fire_claim`` without extending another owner's lease: an execution may
  2412. outlive the TTL, and the owner check stops a stale runner from refreshing a recovered claim.
  2413. Deliberately NOT under ``_under_fire_fence``: the run thread holds the per-job fence across
  2414. delivery, and the fence's reentrancy is per thread, so the heartbeat thread would time out and
  2415. report a false ownership loss. ``_jobs_lock`` already serializes the compare-and-refresh."""
  2416. def apply(jobs, _i, job):
  2417. return _refresh_claim(jobs, job.get("fire_claim"), expected_owner)
  2418. return _with_job(job_id, apply, False)
  2419. # Completed one-shots are retained in jobs.json (final status stays inspectable) and pruned by
  2420. # _sweep_completed_oneshots once they age out.
  2421. COMPLETED_ONESHOT_RETENTION_DAYS = 7
  2422. def _cron_config_number(key: str, default: Any, cast: Callable[[Any], Any]) -> Any:
  2423. """Read ``cron.<key>`` from config as *cast*, falling back to *default* on any failure."""
  2424. try:
  2425. from hermes_cli.config import load_config
  2426. cfg = load_config() or {}
  2427. cron_cfg = cfg.get("cron", {}) if isinstance(cfg, dict) else {}
  2428. return cast(cron_cfg.get(key, default))
  2429. except Exception:
  2430. return cast(default)
  2431. def _completed_oneshot_retention_days() -> float:
  2432. """``cron.completed_retention_days``; non-positive disables the sweep (records kept forever)."""
  2433. return _cron_config_number("completed_retention_days", COMPLETED_ONESHOT_RETENTION_DAYS, float)
  2434. def _sweep_completed_oneshots(
  2435. raw_jobs: list[dict[str, Any]], now: datetime, *, removed_ids: Optional[set[str]] = None,
  2436. ) -> bool:
  2437. """Prune completed one-shot records past retention (in place; True when anything was removed).
  2438. Removed ids go into *removed_ids* so save_jobs's shrink-merge guard allows the delete. Age is
  2439. measured from ``last_run_at``; a record without a parseable one is kept (never guess into
  2440. deletion)."""
  2441. retention_days = _completed_oneshot_retention_days()
  2442. if retention_days <= 0:
  2443. return False
  2444. cutoff = now - timedelta(days=retention_days)
  2445. removed = False
  2446. for rj in list(raw_jobs):
  2447. try:
  2448. if rj.get("state") != "completed":
  2449. continue
  2450. schedule = rj.get("schedule")
  2451. if (schedule.get("kind") if isinstance(schedule, dict) else None) != "once":
  2452. continue
  2453. last_run = rj.get("last_run_at")
  2454. last_run_dt = _parse_aware(last_run) if isinstance(last_run, str) else None
  2455. if last_run_dt is None or last_run_dt >= cutoff:
  2456. continue
  2457. raw_jobs.remove(rj)
  2458. removed = True
  2459. rid = rj.get("id")
  2460. if removed_ids is not None and rid:
  2461. removed_ids.add(str(rid))
  2462. logger.info(
  2463. "Job '%s': pruning completed one-shot record (finished %s, retention %.1f days)",
  2464. rj.get("name", rj.get("id", "?")), last_run, retention_days)
  2465. except Exception:
  2466. logger.debug(
  2467. "Retention sweep skipped malformed job record %r", rj.get("id", "?"), exc_info=True)
  2468. return removed
  2469. # --- Due scan ---
  2470. def get_due_jobs() -> list[dict[str, Any]]:
  2471. """Return all jobs due now. A recurring job more than one period stale (gateway down, or a run
  2472. overran the interval) has its backlog collapsed — next_run_at fast-forwards so nothing
  2473. burst-fires — but still fires ONCE now (via mark_job_run, consuming one ``repeat.times`` run),
  2474. avoiding the perpetual-defer loop for runs longer than interval + grace.
  2475. This prevents the perpetual-defer loop (#33315) where a job whose runtime exceeds ``interval + grace``
  2476. would be skipped forever.
  2477. """
  2478. with _jobs_lock():
  2479. return _get_due_jobs_locked()
  2480. @dataclass
  2481. class _DueScan:
  2482. """Mutable state threaded through one due scan: the raw store records plus what to persist."""
  2483. raw_jobs: list[dict[str, Any]]
  2484. now: datetime
  2485. needs_save: bool = False
  2486. removed: set[str] = field(default_factory=set)
  2487. # Miss/catch-up metrics, emitted once the save that persists the scan's repairs lands: an
  2488. # unwritable store re-finds the same miss every tick, so counting before the save over-counts.
  2489. on_saved: list[Callable[[], None]] = field(default_factory=list)
  2490. def find(self, job_id: Any) -> Optional[dict[str, Any]]:
  2491. return next((rj for rj in self.raw_jobs if rj["id"] == job_id), None)
  2492. def persist(self, job_id: Any, **fields: Any) -> None:
  2493. """Write *fields* onto the raw record for *job_id* and flag a save (no-op if missing)."""
  2494. rj = self.find(job_id)
  2495. if rj is not None:
  2496. rj.update(fields)
  2497. self.needs_save = True
  2498. def retire(self, job_id: Any) -> None:
  2499. """Drop the raw record for *job_id* as an intentional removal."""
  2500. rj = self.find(job_id)
  2501. if rj is not None:
  2502. self.raw_jobs.remove(rj)
  2503. self.removed.add(str(job_id))
  2504. self.needs_save = True
  2505. def _normalize_due_scan_records(raw_jobs: list[dict[str, Any]]) -> bool:
  2506. """Repair malformed store records in place BEFORE the due scan keys off them: a missing ``id``
  2507. (older writers used ``job_id``), non-dict ``schedule``, or non-ISO timestamp used to abort the
  2508. whole scan before save_jobs(), freezing the scheduler in a fast-forward loop."""
  2509. changed = False
  2510. for rj in raw_jobs:
  2511. if not rj.get("id"):
  2512. rj["id"] = rj.pop("job_id", None) or uuid.uuid4().hex[:12]
  2513. changed = True
  2514. if not isinstance(rj.get("schedule"), dict):
  2515. rj["schedule"] = {}
  2516. changed = True
  2517. for key in ("next_run_at", "last_run_at"):
  2518. value = rj.get(key)
  2519. if value is not None and _parse_aware(value) is None:
  2520. rj.pop(key, None) # the "no next_run_at" path recomputes
  2521. changed = True
  2522. return changed
  2523. def _self_disable_half_paused(job: dict[str, Any], scan: _DueScan) -> None:
  2524. """Self-heal enabled=true with pause markers: the operator believes the job is frozen while the
  2525. scheduler would still fire it. Force enabled=false so listings are honest; logged loudly since
  2526. pause_job sets both fields atomically, so this should be rare."""
  2527. jid = job.get("id")
  2528. logger.error(
  2529. "Job '%s' (%s) has pause markers while enabled=true; "
  2530. "self-disabling so it cannot fire (pause must be authoritative).",
  2531. job.get("name", jid), jid)
  2532. rj = scan.find(jid)
  2533. if rj is None:
  2534. return
  2535. rj.update(enabled=False, state="paused")
  2536. if not rj.get("paused_at"):
  2537. rj["paused_at"] = scan.now.isoformat()
  2538. if not rj.get("paused_reason"):
  2539. rj["paused_reason"] = "auto-disabled: enabled+paused contradiction"
  2540. scan.needs_save = True
  2541. def _recover_missing_next_run(job: dict[str, Any], scan: _DueScan) -> Optional[str]:
  2542. """Recompute and persist a missing ``next_run_at``; None when unrecoverable. One-shots use the
  2543. grace window; recurring jobs get here after a direct jobs.json edit bypassed add_job() or a
  2544. transient croniter ImportError left them in state 'error' (#127182) — re-armed ones go back
  2545. to 'scheduled', otherwise they would be silently skipped forever."""
  2546. schedule = job.get("schedule", {})
  2547. kind = schedule.get("kind")
  2548. zone = jobs_time.zone_for_job(job)
  2549. recovered_next = _recoverable_oneshot_run_at(
  2550. schedule, scan.now, last_run_at=job.get("last_run_at"), zone=zone)
  2551. recovery_kind = "one-shot" if recovered_next else None
  2552. if not recovered_next and kind in {"cron", "interval"}:
  2553. recovered_next = compute_next_run(
  2554. schedule, scan.now.isoformat(), timezone=job.get("timezone"))
  2555. if recovered_next:
  2556. recovery_kind = kind
  2557. if not recovered_next:
  2558. return None
  2559. logger.info(
  2560. "Job '%s' had no next_run_at; recovering %s run at %s",
  2561. job.get("name", job.get("id", "?")), recovery_kind, recovered_next)
  2562. fields: dict[str, Any] = {"next_run_at": recovered_next}
  2563. if _is_recoverable_error_job(job):
  2564. fields["state"] = "scheduled"
  2565. job.update(fields)
  2566. scan.persist(job["id"], **fields)
  2567. return recovered_next
  2568. @dataclass
  2569. class _DueJob:
  2570. """One candidate under evaluation: its record, schedule and the stored next_run in raw/aware
  2571. form."""
  2572. job: dict[str, Any]
  2573. scan: _DueScan
  2574. next_run: str # stored ISO string, compared string-exact against manual_run_at
  2575. raw_next_run_dt: datetime # as stored (may carry a pre-migration offset)
  2576. next_run_dt: datetime # normalized to the job's pinned tz, else the configured one
  2577. @property
  2578. def schedule(self) -> dict[str, Any]:
  2579. return self.job.get("schedule", {})
  2580. @property
  2581. def kind(self) -> Optional[str]:
  2582. return self.schedule.get("kind")
  2583. @property
  2584. def label(self) -> Any:
  2585. return self.job.get("name", self.job.get("id", "?"))
  2586. def recompute_next(self) -> Optional[str]:
  2587. return compute_next_run(
  2588. self.schedule, self.scan.now.isoformat(), timezone=self.job.get("timezone"))
  2589. def _repair_timezone_shifted_cron(d: _DueJob) -> bool:
  2590. """Repair a cron job whose stored offset no longer matches now's (TZ migration).
  2591. next_run_at is an absolute instant but the expr means local wall clock, so a TZ change can make
  2592. it look due hours early. If the stored wall clock is still in the future, recompute so we fire
  2593. at the intended local time. True when re-anchored (caller skips this tick). TRADE-OFF: a DST
  2594. offset change meeting the same conditions SKIPS the pending occurrence; accepted as rare."""
  2595. now = d.scan.now
  2596. if not (
  2597. _instant_at_or_before(d.next_run_dt, now)
  2598. and _timezone_offset_mismatch(d.raw_next_run_dt, now)
  2599. and _stored_wall_clock_is_future(d.raw_next_run_dt, now)
  2600. ):
  2601. return False
  2602. new_next = d.recompute_next()
  2603. if not new_next:
  2604. return False
  2605. logger.info(
  2606. "Job '%s' next_run_at offset changed (%s -> %s). "
  2607. "Recomputing cron run to preserve local wall-clock intent: %s",
  2608. d.label, d.raw_next_run_dt.utcoffset(), now.utcoffset(), new_next)
  2609. d.scan.persist(d.job["id"], next_run_at=new_next)
  2610. return True
  2611. def _rearm_stale_error_recurring(d: _DueJob) -> datetime:
  2612. """Re-arm a recurring job wedged in persisted last_status=error; returns the effective
  2613. next_run_dt.
  2614. Such a job errored, mark_job_run parked next_run_at in the future, and nothing re-dispatched it
  2615. (the in-memory stale-claim sweep cannot see it). Interval jobs re-arm to now (always a legal
  2616. fire); cron jobs re-arm to the next LEGAL occurrence, since re-arming to now would fire at times
  2617. the expression excludes. A correctly-parked cron value is left as-is.
  2618. """
  2619. now = d.scan.now
  2620. if not (
  2621. d.kind in ("cron", "interval")
  2622. and _instant_after(d.next_run_dt, now)
  2623. and _job_is_stale_error_recurring(d.job, d.schedule, now)
  2624. ):
  2625. return d.next_run_dt
  2626. if d.kind == "interval":
  2627. recovered_next = now.isoformat()
  2628. recovered_next_dt: Optional[datetime] = now
  2629. else:
  2630. recovered_next = d.recompute_next()
  2631. recovered_next_dt = _parse_aware(recovered_next) if recovered_next else None
  2632. if not (
  2633. recovered_next and recovered_next_dt is not None
  2634. and _instant_before(recovered_next_dt, d.next_run_dt)
  2635. ):
  2636. return d.next_run_dt
  2637. jid = d.job.get("id")
  2638. logger.warning(
  2639. "cron.persisted_error.recovered job='%s' id=%s — recurring "
  2640. "job wedged in stale last_status=error without re-firing for "
  2641. "a full cadence; re-arming next_run_at to %s so it re-dispatches without force-run/resume",
  2642. d.job.get("name", jid), jid, recovered_next)
  2643. _record_persisted_error_recovery(d.job, d.next_run)
  2644. d.job["next_run_at"] = recovered_next
  2645. d.scan.persist(jid, next_run_at=recovered_next)
  2646. return recovered_next_dt
  2647. def _reanchor_stale_cron(d: _DueJob) -> bool:
  2648. """Stale-schedule guard for a due cron instant; True when re-anchored without firing.
  2649. A direct edit of schedule.expr leaves next_run_at on the old lattice, so re-anchor first (from
  2650. the current expr, so this converges). Some instants are off the lattice on purpose and
  2651. authorize one fire instead: an offset-representation migration that would otherwise swallow
  2652. a never-fired occurrence, and the failure-path planners' parked instants (a quota recovery
  2653. before a sparse cron's next natural occurrence, an unreachable-model retry rung). All fall
  2654. through to fire ONCE (at-most-once holds: completion rewrites the instant)."""
  2655. stale_class = _classify_stale_cron_next_run(d.schedule, d.raw_next_run_dt, d.next_run_dt)
  2656. if stale_class == STALE_CRON_EXPR_EDIT:
  2657. from cron.quota_hold import is_recovery_fire
  2658. from cron.unreachable_retry import is_retry_fire
  2659. if is_recovery_fire(d.job, d.next_run) or is_retry_fire(d.job, d.next_run):
  2660. logger.info(
  2661. "cron.off_lattice_fire job='%s' id=%s expr=%r at=%s",
  2662. d.label, d.job.get("id"), d.schedule.get("expr"), d.next_run)
  2663. return False
  2664. new_next = d.recompute_next()
  2665. logger.info(
  2666. "Job '%s' next_run_at %s does not match its current "
  2667. "cron expression %r (direct jobs.json edit?); re-anchoring to %s without firing.",
  2668. d.label, d.next_run, d.schedule.get("expr"), new_next)
  2669. if new_next:
  2670. d.scan.persist(d.job["id"], next_run_at=new_next)
  2671. return True
  2672. if stale_class == STALE_CRON_TIMEZONE_MIGRATION:
  2673. logger.warning(
  2674. "cron.timezone_migration.catch_up job='%s' id=%s expr=%r "
  2675. "stored=%s normalized=%s — stored next_run_at carries a "
  2676. "pre-migration UTC offset (%s, now %s) and is a legal "
  2677. "occurrence at its own wall clock; firing the due run instead of re-anchoring past it.",
  2678. d.label, d.job.get("id"), d.schedule.get("expr"), d.next_run,
  2679. d.next_run_dt.isoformat(), d.raw_next_run_dt.utcoffset(), d.scan.now.utcoffset())
  2680. _record_timezone_migration_catchup(d.job, d.raw_next_run_dt, d.next_run_dt)
  2681. return False
  2682. def _oneshot_dispatch_limit_reached(job: dict[str, Any], scan: _DueScan) -> bool:
  2683. """One-shot dispatch-limit guard; True when the job must not fire this tick.
  2684. A finite one-shot claimed via claim_dispatch() whose tick died before mark_job_run has
  2685. completed >= times while still looking due. Remove it instead of re-firing — unless THIS
  2686. process is still running it (a run outliving the run_claim TTL is slow, not stale)."""
  2687. repeat = job.get("repeat") or {}
  2688. times = repeat.get("times")
  2689. completed = repeat.get("completed", 0)
  2690. if times is None or times <= 0 or completed < times:
  2691. return False
  2692. name = job.get("name", job.get("id", "?"))
  2693. # A live run must never have its job record deleted underneath it (#62002): a run that outlives the
  2694. # run_claim TTL (stream stall, laptop asleep mid-run) satisfies the same completed >= times +
  2695. # expired-claim condition as a dead tick, but mark_job_run() still needs the record to land last_run_at
  2696. # / last_status / last_delivery_error. If this process is still running the job, it is slow, not stale —
  2697. # keep the entry and skip.
  2698. if _job_running_in_this_process(job.get("id", "")):
  2699. logger.info(
  2700. "Job '%s': dispatch limit reached (%d/%d) but its run is still in flight in this "
  2701. "process — keeping entry",
  2702. name, completed, times)
  2703. return True
  2704. if job.get("last_run_at") is not None:
  2705. # A record with last_run_at completed a real run and was re-armed without a budget reset
  2706. # (old build or hand edit) — not the dead-tick case; warn so the removal leaves a trace.
  2707. logger.warning(
  2708. "Job '%s': one-shot dispatch limit reached (%d/%d) on a record that already completed "
  2709. "a run (last_run_at=%s) — removing it WITHOUT firing. This record was re-armed "
  2710. "without a budget reset (pre-#93615 store or hand edit); re-run it with "
  2711. "'hermes cron resume <job> --run-now' (#93524).",
  2712. name, completed, times, job.get("last_run_at"))
  2713. else:
  2714. logger.info(
  2715. "Job '%s': one-shot dispatch limit reached (%d/%d) — removing stale due entry",
  2716. name, completed, times)
  2717. scan.retire(job["id"])
  2718. # The claimed run never completed here by definition — leave an operator-visible diagnostic.
  2719. from cron.jobs_oneshot_diagnostics import write_wedged
  2720. write_wedged(job)
  2721. return True
  2722. def _restore_unclaimed_slot(job: dict[str, Any], scan: _DueScan) -> Optional[str]:
  2723. """Put an occurrence the dispatcher advanced past but never claimed back on the schedule
  2724. (#107485); returns the restored ``next_run_at`` or None. Restored ONCE: the stamp is dropped
  2725. here, so the slot then meets the ordinary late / fast-forward / ``cron.catch_up_missed``
  2726. policy like any other overdue instant — never a replay of every missed slot."""
  2727. from cron.occurrences import unclaimed_pending_slot
  2728. slot = unclaimed_pending_slot(job, scan.now)
  2729. if slot is None:
  2730. return None
  2731. logger.warning(
  2732. "Job '%s' (%s): occurrence %s was taken off the schedule but never claimed "
  2733. "(scheduler stopped before dispatch); restoring it as the due instant (was %s).",
  2734. job.get("name", job.get("id")), job.get("id"), slot, job.get("next_run_at"))
  2735. job["next_run_at"] = slot
  2736. job.pop("pending_slot", None)
  2737. rj = scan.find(job["id"])
  2738. if rj is not None:
  2739. rj.pop("pending_slot", None)
  2740. scan.persist(job["id"], next_run_at=slot)
  2741. return slot
  2742. def _evaluate_due_job(job: dict[str, Any], scan: _DueScan, run_claim_ttl: float) -> bool:
  2743. """Decide whether one enabled, non-terminal job fires this tick, persisting any repairs.
  2744. Ordering matters: recover missing next_run_at, repair timezone shifts, re-arm stale-error
  2745. recurring jobs; then once due: re-anchor stale cron instants, fast-forward missed recurring
  2746. runs, retire/guard one-shots, and finally stamp the run claim / dispatch record."""
  2747. now = scan.now
  2748. # Cross-process guard: another process's live one-shot run_claim (younger than TTL) — do NOT
  2749. # re-dispatch. Malformed/future-dated claims (clock/TZ skew) count as stale, never eternally
  2750. # fresh.
  2751. if (
  2752. job.get("schedule", {}).get("kind") == "once"
  2753. and _claim_is_live(job.get("run_claim"), now, run_claim_ttl)
  2754. ):
  2755. return False
  2756. next_run = _restore_unclaimed_slot(job, scan) or job.get("next_run_at") or _recover_missing_next_run(job, scan)
  2757. if not next_run:
  2758. return False
  2759. # A pinned job's stored instant must be normalized into ITS zone: the cron repair and
  2760. # stale-lattice predicates below compare wall clocks, so the profile zone would make a due
  2761. # occurrence look future-dated and re-anchor past it (the job then never fires).
  2762. zone = jobs_time.zone_for_job(job)
  2763. raw_next_run_dt = datetime.fromisoformat(next_run)
  2764. d = _DueJob(job, scan, next_run, raw_next_run_dt, _ensure_aware(raw_next_run_dt, zone))
  2765. kind = d.kind
  2766. recurring = kind in {"cron", "interval"}
  2767. # Intentionally string-exact on raw stored values: trigger_job stamps the SAME isoformat string
  2768. # into both fields, and any rewrite of next_run_at (edit, re-anchor, fire-claim advance) must
  2769. # invalidate the marker. Do not "fix" this with _ensure_aware normalization.
  2770. manual_run = job.get("manual_run_at") == next_run
  2771. from cron.occurrences import completed_occurrence, scheduled_instant
  2772. if not manual_run and completed_occurrence(job, next_run):
  2773. new_next = d.recompute_next() if recurring else None
  2774. if new_next:
  2775. scan.persist(job["id"], next_run_at=new_next)
  2776. return False
  2777. # The offset-repair path exists to rescue jobs that FOLLOW the profile zone across a zone
  2778. # change. A pinned job's instant legitimately carries its own offset, so repair can only
  2779. # misfire on it — skip it entirely.
  2780. if kind == "cron" and not manual_run and zone is None and _repair_timezone_shifted_cron(d):
  2781. return False
  2782. d.next_run_dt = _rearm_stale_error_recurring(d)
  2783. if _instant_after(d.next_run_dt, now):
  2784. return False
  2785. # Only the dispatch snapshot carries this field; never infer it from a later stamp.
  2786. job["_scheduled_instant"] = None if manual_run else scheduled_instant(job.get("next_run_at"))
  2787. if not manual_run and kind == "cron" and _reanchor_stale_cron(d):
  2788. return False
  2789. grace = _compute_grace_seconds(d.schedule, job.get("grace_minutes"))
  2790. from cron.missed_runs import fast_forward_missed_recurring, retire_expired_oneshot
  2791. if not manual_run and recurring and fast_forward_missed_recurring(d, grace):
  2792. return False
  2793. if kind == "once":
  2794. if retire_expired_oneshot(d) or _oneshot_dispatch_limit_reached(job, scan):
  2795. return False
  2796. # Durably claim the one-shot for the DURATION of its run: a second scheduler process on the
  2797. # same HERMES_HOME must not re-dispatch it while in flight, and advancing next_run_at by a
  2798. # fixed window is not enough for a run that outlives a tick. The other process sees the
  2799. # fresh claim and skips; mark_job_run() clears it. The TTL only covers a tick that DIES.
  2800. claim = {"at": now.isoformat(), "by": _machine_id()}
  2801. if _elapsed_seconds(now, d.next_run_dt) > ONESHOT_GRACE_SECONDS: # only an outage lets it through
  2802. claim["outage"] = True
  2803. job["run_claim"] = claim
  2804. scan.persist(job["id"], run_claim=claim)
  2805. # Missed-run visibility: persist scheduled-vs-actual timing so separate CLI processes can show a
  2806. # late catch-up. Recurring only — expired one-shots were retired above; manual triggers aren't
  2807. # late.
  2808. if not manual_run and recurring:
  2809. lateness = max(0.0, _elapsed_seconds(now, d.next_run_dt))
  2810. # See #99879.
  2811. dispatch_stamp = {
  2812. "scheduled_at": next_run,
  2813. "dispatched_at": now.isoformat(),
  2814. "lateness_seconds": round(lateness, 1),
  2815. "kind": _classify_dispatch_lateness(lateness, grace),
  2816. }
  2817. job["last_dispatch"] = dispatch_stamp
  2818. # The tick advances next_run_at past this occurrence before any fire claim exists; the
  2819. # stamp survives a process death in that window so the slot is restored, not lost.
  2820. from cron.occurrences import pending_slot_stamp
  2821. scan.persist(
  2822. job["id"], last_dispatch=dispatch_stamp,
  2823. pending_slot=pending_slot_stamp(next_run, now))
  2824. return True
  2825. def _get_due_jobs_locked() -> list[dict[str, Any]]:
  2826. """Inner implementation of get_due_jobs(); must be called with _jobs_lock held."""
  2827. raw_jobs = load_jobs()
  2828. scan = _DueScan(raw_jobs, _hermes_now())
  2829. scan.needs_save = _normalize_due_scan_records(raw_jobs)
  2830. jobs = [_apply_skill_fields(j) for j in copy.deepcopy(raw_jobs)]
  2831. # One-shot run-claim TTL, resolved once per scan (see _oneshot_run_claim_ttl_seconds).
  2832. run_claim_ttl = _oneshot_run_claim_ttl_seconds()
  2833. # Retention sweep: completed one-shots are kept for inspection but must not accumulate forever.
  2834. if _sweep_completed_oneshots(raw_jobs, scan.now, removed_ids=scan.removed):
  2835. scan.needs_save = True
  2836. jobs = [j for j in jobs if scan.find(j.get("id")) is not None]
  2837. due = []
  2838. for job in jobs:
  2839. # Per-job containment: one malformed record must never abort the whole scan. Normalization
  2840. # above repairs known shapes; this catches FUTURE variants so healthy siblings still
  2841. # run/persist.
  2842. try:
  2843. if is_terminal_job(job) and not _is_recoverable_error_job(job):
  2844. continue
  2845. if not job.get("enabled", True):
  2846. continue
  2847. if _has_pause_marker(job):
  2848. _self_disable_half_paused(job, scan)
  2849. continue
  2850. if _evaluate_due_job(job, scan, run_claim_ttl):
  2851. due.append(job)
  2852. except Exception:
  2853. logger.exception(
  2854. "Skipping malformed cron job %r during due scan",
  2855. job.get("name") or job.get("id") or "?")
  2856. if scan.needs_save:
  2857. try:
  2858. save_jobs(raw_jobs, removed_ids=scan.removed or None)
  2859. except OSError as exc: # repairs live in memory; the next tick retries the persist
  2860. store_health.note_unwritable(exc, "due-scan repairs not persisted", "scan")
  2861. return due
  2862. # Only now is a skipped one-shot claimed or durably due on disk: normal grace applies again.
  2863. store_health.end_recovery_window(_current_cron_store().cron_dir)
  2864. for emit in scan.on_saved:
  2865. emit()
  2866. return due
  2867. # --- Run output ---
  2868. # Per-run output files (`cron/output/<job>/<timestamp>.md`) are capped so a frequent job can't fill
  2869. # the disk.
  2870. # Unlike the quick-snapshot store (`hermes_cli.backup`, capped at 20) it had no retention, so a
  2871. # frequently-scheduled job on a long-running deploy accumulated one file per run forever and could fill the
  2872. # disk (#52383). Keep the most recent N files per job; a non-positive value disables pruning (opt-out).
  2873. _CRON_OUTPUT_DEFAULT_KEEP = 50
  2874. def _cron_output_keep() -> int:
  2875. """Per-job output-file retention cap (``cron.output_retention``)."""
  2876. return _cron_config_number("output_retention", _CRON_OUTPUT_DEFAULT_KEEP, int)
  2877. def _prune_job_output(job_output_dir: Path, keep: int) -> int:
  2878. """Remove the oldest ``*.md`` run-output files beyond *keep*; returns count deleted. Filenames
  2879. are timestamps, so a reverse lexical sort is newest-first. Non-positive *keep* disables pruning;
  2880. failures are swallowed so they can never break output saving."""
  2881. if keep <= 0:
  2882. return 0
  2883. try:
  2884. files = sorted(
  2885. (f for f in job_output_dir.glob("*.md") if f.is_file()),
  2886. key=lambda f: f.name, reverse=True)
  2887. except OSError:
  2888. return 0
  2889. deleted = 0
  2890. for stale in files[keep:]:
  2891. try:
  2892. stale.unlink()
  2893. deleted += 1
  2894. except OSError as exc:
  2895. logger.debug("Failed to prune cron output %s: %s", stale.name, exc)
  2896. return deleted
  2897. def save_job_output(job_id: str, output: str):
  2898. """Save job output to file."""
  2899. ensure_dirs()
  2900. job_output_dir = _job_output_dir(job_id)
  2901. _ensure_cron_dir(job_output_dir)
  2902. _secure_dir(job_output_dir)
  2903. output_file = job_output_dir / f"{_hermes_now().strftime('%Y-%m-%d_%H-%M-%S')}.md"
  2904. atomic_write_text(output_file, output, tmp_prefix=".output_", mode=0o600)
  2905. _secure_file(output_file)
  2906. # Bound per-job output growth so long-running deploys don't fill the disk (#52383).
  2907. _prune_job_output(job_output_dir, _cron_output_keep())
  2908. return output_file
  2909. # --- Skill reference rewriting (curator integration) ---
  2910. def _canonical_skill_ref(raw: Any) -> str:
  2911. """Reduce a job skill reference (possibly an absolute path) to the bare name the curator matches
  2912. on, resolving as the scheduler does — otherwise a path-referencing job's skill looks
  2913. unreferenced and gets archived. Falls back to plain cleanup so a broken import can never lose
  2914. a name."""
  2915. value = str(raw or "").strip()
  2916. if not value:
  2917. return ""
  2918. try:
  2919. from agent.skill_utils import normalize_skill_lookup_name
  2920. value = normalize_skill_lookup_name(value) or value
  2921. except Exception:
  2922. logger.debug("referenced_skill_names: could not normalize skill ref %r", raw, exc_info=True)
  2923. return value.strip().lstrip("/")
  2924. def referenced_skill_names() -> set[str]:
  2925. """Skill names referenced by ANY cron job, deliberately including paused/disabled ones (resuming
  2926. must still find them); the curator protects these from inactivity archival. Canonicalized as the
  2927. scheduler does, so absolute paths are protected too. A corrupt store yields an empty set."""
  2928. try:
  2929. jobs = load_jobs()
  2930. except Exception:
  2931. logger.debug("referenced_skill_names: failed to load cron jobs", exc_info=True)
  2932. return set()
  2933. return {
  2934. cleaned
  2935. for job in jobs
  2936. if isinstance(job, dict)
  2937. for name in _normalize_skill_list(job.get("skill"), job.get("skills"))
  2938. if (cleaned := _canonical_skill_ref(name))
  2939. }
  2940. def rewrite_skill_refs(
  2941. consolidated: Optional[dict[str, str]] = None, pruned: Optional[list[str]] = None,
  2942. ) -> dict[str, Any]:
  2943. """Rewrite cron job skill references after a curator consolidation pass (a job listing a
  2944. consolidated/pruned skill would otherwise run without it). Consolidated names map to their
  2945. umbrella target without duplication, pruned names are dropped, ordering is preserved, and the
  2946. legacy ``skill`` field is realigned. Returns ``{"rewrites": [{job_id, job_name, before,
  2947. after, mapped, dropped}, ...], "jobs_updated": N, "jobs_scanned": M}``. Load/save exceptions
  2948. propagate."""
  2949. consolidated = dict(consolidated or {})
  2950. # A skill listed in both wins as "consolidated" — it has a target, the more useful outcome.
  2951. pruned_set = set(pruned or []) - set(consolidated.keys())
  2952. if not consolidated and not pruned_set:
  2953. return {"rewrites": [], "jobs_updated": 0, "jobs_scanned": 0}
  2954. with _jobs_lock():
  2955. jobs = load_jobs()
  2956. rewrites: list[dict[str, Any]] = []
  2957. for job in jobs:
  2958. skills_before = _normalize_skill_list(job.get("skill"), job.get("skills"))
  2959. if not skills_before:
  2960. continue
  2961. mapped: dict[str, str] = {}
  2962. dropped: list[str] = []
  2963. new_skills: list[str] = []
  2964. for name in skills_before:
  2965. if name in consolidated:
  2966. target = consolidated[name]
  2967. mapped[name] = target
  2968. if target and target not in new_skills:
  2969. new_skills.append(target)
  2970. elif name in pruned_set:
  2971. dropped.append(name)
  2972. elif name not in new_skills:
  2973. new_skills.append(name)
  2974. if not mapped and not dropped:
  2975. continue
  2976. job["skills"] = new_skills
  2977. job["skill"] = new_skills[0] if new_skills else None
  2978. rewrites.append({
  2979. "job_id": job.get("id"),
  2980. "job_name": job.get("name") or job.get("id"),
  2981. "before": list(skills_before),
  2982. "after": list(new_skills),
  2983. "mapped": mapped,
  2984. "dropped": dropped,
  2985. })
  2986. if rewrites:
  2987. save_jobs(jobs)
  2988. logger.info("Curator rewrote skill references in %d cron job(s)", len(rewrites))
  2989. return {"rewrites": rewrites, "jobs_updated": len(rewrites), "jobs_scanned": len(jobs)}