session_cron.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321
  1. """Trusted cron producer and owner execution of the existing scheduler contract.
  2. Public callers name a persisted job, never supply agent policy or arbitrary jobs.
  3. The ordinary admission ledger owns execution; the scheduler owns final delivery.
  4. """
  5. import asyncio
  6. from contextvars import ContextVar
  7. from dataclasses import asdict, replace
  8. import json
  9. import os
  10. from pathlib import Path
  11. import threading
  12. from gateway.session_contract import SessionActor, SessionRef
  13. from hermes_state_keys import CRON_PRINCIPAL_ID, cron_request_id
  14. from hermes_state_runtime import RuntimeStoreError, admit_session_input, get_session_admission
  15. _execution: ContextVar[tuple | None] = ContextVar('cron_owner_execution', default=None)
  16. _owners = {}
  17. def bind_owner(authority):
  18. home = Path(authority.db.db_path).resolve().parent
  19. _owners[home] = (os.getpid(), authority, asyncio.get_running_loop())
  20. def unbind_owner(authority):
  21. home = Path(authority.db.db_path).resolve().parent
  22. bound = _owners.get(home)
  23. if bound is not None and bound[1] is authority:
  24. del _owners[home]
  25. def owner_for_home(home):
  26. bound = _owners.get(Path(home).resolve())
  27. if bound is not None and bound[0] == os.getpid() and not bound[2].is_closed():
  28. return bound[1:]
  29. return None
  30. async def _admit_local_delivery(authority, chat_id, text, metadata):
  31. from gateway.config import Platform
  32. from gateway.session_authorities import owner_scope
  33. from gateway.session_automation import admit_route_wake
  34. from gateway.session_local_recovery import local_adapter_map, restore_local_session
  35. from gateway.session_route_recovery import mountable, restore_routed_session
  36. with owner_scope(authority):
  37. live = authority.sessions.get(chat_id)
  38. if live is None and mountable(authority, chat_id):
  39. # A messaging conversation a reminder set from the app names (gateway/session_detached.py),
  40. # not mounted (a restart): it mounts off the chat, or on it when the chat names it again.
  41. restore_routed_session(authority, chat_id)
  42. elif live is None or live.source is None or live.source.platform == Platform.LOCAL:
  43. restore_local_session(authority, chat_id) # a chat closed before this fire, as for completions
  44. source = authority.sessions[chat_id].source
  45. # Off the chat its app viewers take it (as a note); back on the chat (its /resume) it is that
  46. # conversation's own turn there.
  47. adapter = (local_adapter_map(authority)[Platform.LOCAL] if source.platform == Platform.LOCAL
  48. else authority.runner._adapter_for_source(source))
  49. return await admit_route_wake(authority.runner, adapter, source, text, 'cron', metadata)
  50. def deliver_to_local_chat(home, chat_id, text, *, diagnostic=False):
  51. """Admit a cron result into local chat *chat_id*'s FIFO; at the head it is written into the chat's
  52. history as a note, never a turn (``session_completion_notes``). A local chat has no push transport
  53. (``LocalSessionAdapter.send`` only acknowledges a turn's own output), so only the owner in this
  54. process can deliver. Called off the owner loop; None once admitted, else the error."""
  55. owner = owner_for_home(home)
  56. if owner is None:
  57. return f"local chat {chat_id}: no gateway in this process owns its session"
  58. authority, loop = owner
  59. metadata = {'notification_category': 'diagnostic'} if diagnostic else {}
  60. try:
  61. admitted = asyncio.run_coroutine_threadsafe(
  62. _admit_local_delivery(authority, chat_id, text, metadata), loop).result(timeout=60)
  63. except (RuntimeStoreError, KeyError, TimeoutError) as exc: # unknown chat, refused admission, stalled owner
  64. return f"local chat {chat_id}: delivery not confirmed: {exc!r}"
  65. return None if admitted else f"local chat {chat_id}: delivery refused: the chat has no committed route"
  66. def current_execution():
  67. return _execution.get()
  68. def bind_execution(authority, session_id, job_id, execution_id):
  69. """Bind this context's cron admission ``(authority, session_id, job_id, execution_id)``, the
  70. layout ``current_execution()`` readers index; returns the ``ContextVar`` reset token."""
  71. return _execution.set((authority, session_id, job_id, execution_id))
  72. def _actor(authority, actor):
  73. if actor is None:
  74. return SessionActor(CRON_PRINCIPAL_ID, authority.profile_id,
  75. frozenset({'session:create', 'session:submit', 'session:read', 'session:control'}), 'cron-ticker')
  76. if actor.profile_id != authority.profile_id:
  77. raise RuntimeStoreError('profile_mismatch')
  78. if not {'session:create', 'session:submit', 'session:control'} <= actor.capabilities:
  79. raise RuntimeStoreError('permission_denied')
  80. # Cron fires belong to the profile scheduler, not the transport that observes
  81. # them. The same execution ID from ticker and CLI must dedupe together.
  82. return SessionActor(CRON_PRINCIPAL_ID, authority.profile_id, actor.capabilities, actor.transport_id)
  83. def _create(authority, actor, params):
  84. from cron.jobs import get_job
  85. from gateway.session_local_recovery import local_identity, restore_local_session
  86. from gateway.session_policy import build_policy, bind_launch_key
  87. from gateway.run import _load_gateway_config
  88. from cron.scheduler import _load_cron_job_config
  89. from gateway.config import Platform
  90. from gateway.session import SessionSource, SessionEntry
  91. from gateway.session_lifecycle import _now
  92. if set(params) != {'job_id', 'request_id', 'extra_prompt'}:
  93. raise RuntimeStoreError('invalid_params')
  94. jid, request_id, extra = params['job_id'], params['request_id'], params['extra_prompt']
  95. if (not isinstance(jid, str) or not jid or not isinstance(request_id, str)
  96. or not request_id or len(request_id) > 256 or (extra is not None and not isinstance(extra, str))):
  97. raise RuntimeStoreError('invalid_params')
  98. execution_id, request_id = request_id, cron_request_id(jid, request_id)
  99. sid = local_identity(authority.profile_id, actor.subject, request_id)
  100. if authority.db.get_session(sid) is not None:
  101. return restore_local_session(authority, sid), request_id
  102. if ':' in execution_id:
  103. # Fresh admissions only: ``cron:<job>:<execution>`` then splits at its one ``:`` after the
  104. # job. Ledger ids are hex, so only a hand-made id has one; an identity admitted before this
  105. # guard keeps its exact retry above, and run history files it by its stored policy.
  106. raise RuntimeStoreError('invalid_params')
  107. job = get_job(jid)
  108. if job is None or job.get('no_agent'):
  109. raise RuntimeStoreError('not_found')
  110. config = _load_gateway_config()
  111. private = {}
  112. launch = {'source': 'cli', 'request_id': request_id, 'toolsets': []}
  113. if job.get('workdir'):
  114. launch['cwd'] = job['workdir']
  115. policy = build_policy(launch, config, private_secrets=private)
  116. policy = replace(policy, source='cron', platform='cron', model=_load_cron_job_config(job, jid, job.get('name') or jid).model,
  117. request_json=json.dumps({'cron_job': job, 'extra_prompt': extra, 'request_id': request_id}))
  118. policy = bind_launch_key(authority, sid, policy, None, config_secrets=private)
  119. from gateway.session_local_recovery import local_source
  120. source = local_source(authority, sid, actor.subject)
  121. route = authority.runner.session_store._generate_session_key(source)
  122. now = _now()
  123. entry = SessionEntry(route, sid, now, now, origin=source, platform=Platform.LOCAL)
  124. # The staged creation receipt: ``operation('submit')`` commits it off the owner loop.
  125. return {'profile_id': authority.profile_id, 'principal_id': actor.subject, 'request_id': request_id,
  126. 'session_id': sid, 'route': route, 'entry': entry.to_dict(), 'policy': asdict(policy)}, request_id
  127. # What a message typed into a scheduled job's session settles with (see ``execute``); its text is
  128. # ``gateway.errors.cron_session_read_only``, rendered in the owning profile's language.
  129. CRON_SESSION_READ_ONLY_SURFACE = {'layer': 'gateway', 'code': 'cron_session_read_only', 'retryable': False}
  130. # Run-side verdicts run_job stamps on the job dict for the firing scheduler's bookkeeping tail
  131. # (cron/quota_hold.py parks past a closed usage window; cron/unreachable_retry.py re-runs a
  132. # never-reached model). They cross the authority boundary inside the cron result.
  133. RUN_JOB_FLAGS = ('_model_unreachable', '_quota_hold_seconds')
  134. async def operation(authority, name, params, actor=None):
  135. actor = _actor(authority, actor)
  136. from gateway.session_authorities import require_serving
  137. require_serving(authority) # released ownership: nothing here is ours to touch
  138. # Only new work is fenced by a drain. ``recover``/``status``/``cancel`` observe or stop work
  139. # already admitted, which the drain is waiting for: refusing them makes the firer give up
  140. # (CronExecutionUnknown pauses the job) on every restart that overlaps a cron run.
  141. if name == 'recover':
  142. from gateway.session_local_recovery import local_identity
  143. if (set(params) != {'job_id', 'request_id', 'extra_prompt'}
  144. or any(not isinstance(params[k], str) or not params[k] for k in ('job_id', 'request_id'))
  145. or (params['extra_prompt'] is not None and not isinstance(params['extra_prompt'], str))):
  146. raise RuntimeStoreError('invalid_params')
  147. request_id = cron_request_id(params['job_id'], params['request_id'])
  148. sid = local_identity(authority.profile_id, actor.subject, request_id)
  149. from hermes_state_runtime import list_session_admissions
  150. rows = list_session_admissions(authority.db, session_id=sid, pending_only=False)
  151. from gateway.owner_identity import same_owner
  152. row = next((r for r in rows if r['request_id'] == request_id and same_owner(r['principal_id'], actor.subject)), None)
  153. if row is None:
  154. return {'status': 'missing', 'result': None}
  155. state = await operation(authority, 'status', {'session_id': sid, 'admission_id': row['admission_id']}, actor)
  156. from gateway.session_local_recovery import restore_local_session
  157. restore_local_session(authority, sid)
  158. from gateway.config import Platform
  159. from gateway.session_local_recovery import local_adapter_map
  160. policy = local_adapter_map(authority)[Platform.LOCAL].policies[sid]
  161. frozen = json.loads(policy.request_json)
  162. if frozen['extra_prompt'] != params['extra_prompt']:
  163. raise RuntimeStoreError('admission_conflict')
  164. state['job'] = frozen['cron_job']
  165. # The receipt a firer whose submit answer was lost polls, exactly as if it had arrived.
  166. state.update(session_id=sid, admission_id=row['admission_id'])
  167. return state
  168. if name == 'submit':
  169. authority._require_admission_open()
  170. from gateway.session_authorities import owner_scope
  171. # The job id, jobs file and gateway config belong to the OWNING profile.
  172. with owner_scope(authority):
  173. ref, request_id = _create(authority, actor, params)
  174. from functools import partial
  175. from gateway.session_runtime_workers import tracked_write
  176. if isinstance(ref, dict):
  177. # The creation commit waits on the writer off the owner loop, like session.create (N50).
  178. from gateway.session_local_recovery import restore_local_session
  179. from hermes_state_local import commit_local_session
  180. await tracked_write(authority, partial(commit_local_session, authority.db, epoch=authority.epoch, receipt=ref))
  181. ref = restore_local_session(authority, ref['session_id'])
  182. receipt = await tracked_write(authority, partial(
  183. admit_session_input, authority.db, epoch=authority.epoch, principal_id=actor.subject,
  184. session_id=ref.session_id, request_id=request_id, payload={'text': params['extra_prompt'] or ''},
  185. _authorize_write=authority._admission_gate()), then=partial(authority._admitted, ref, None), ordered=True,
  186. after=partial(authority._schedule_admitted, ref))
  187. return {'session_id': ref.session_id, 'admission_id': receipt.admission_id}
  188. if set(params) != {'session_id', 'admission_id'} or name not in {'status', 'cancel'}:
  189. raise RuntimeStoreError('invalid_params')
  190. ref = SessionRef(authority.profile_id, params['session_id'])
  191. authority.authorize(actor, ref, 'session:submit')
  192. row = get_session_admission(authority.db, admission_id=params['admission_id'])
  193. from gateway.owner_identity import same_owner
  194. if row is None or row['target_session_id'] != ref.session_id or not same_owner(row['principal_id'], actor.subject):
  195. raise RuntimeStoreError('permission_denied')
  196. if name == 'cancel':
  197. if row['status'] == 'queued':
  198. row = await authority.cancel_or_claimed(actor, ref, row['admission_id']) or {'status': 'terminal'}
  199. if row['status'] == 'started':
  200. # The claim commits before execute() registers its event; latching the
  201. # cancellation on the admission here means that window cannot lose it.
  202. _cancellations(authority).setdefault(row['admission_id'], threading.Event()).set()
  203. return {'ok': True}
  204. result, flags = None, {}
  205. if row['status'] == 'terminal':
  206. from gateway.session_results import admission_result
  207. saved = admission_result(authority.db, row['admission_id'])
  208. result = saved['result'].get('cron_result') if saved else None
  209. flags = (saved['result'].get('cron_job_flags') or {}) if saved else {}
  210. if result is None:
  211. result = [False, '', '', row.get('outcome') or 'unknown_execution']
  212. from cron.scheduler_liveness import owner_progress
  213. return {'status': row['status'], 'result': result, 'job_flags': flags,
  214. 'progress_at': owner_progress.get(row['request_id'].rsplit(':', 1)[-1])}
  215. def _cancellations(authority):
  216. cancellations = getattr(authority, '_cron_cancellations', None)
  217. if cancellations is None:
  218. cancellations = authority._cron_cancellations = {}
  219. return cancellations
  220. async def rpc(connection, name, params):
  221. return await operation(connection.authority, name, params, connection.actor)
  222. def scheduler_execution_id(job_id, request_id):
  223. """The scheduler's execution id this admission carries: ``_create`` admits under
  224. ``cron:<job>:<request_id>`` and the ticker's ``request_id`` IS its ``cron/executions`` row id."""
  225. return request_id.removeprefix(cron_request_id(job_id, ''))
  226. def _run_identified(run_job, job, execution_id, **kwargs):
  227. """``run_job`` inside this fire's cron identity (``ctx.current_cron_execution()``).
  228. The scheduler binds it in the FIRING thread, but the owner executes on its own loop, whose
  229. context never saw that binding; re-bind it here from the same ledger row, in the profile scope
  230. the caller entered (so the row and the profile name are this profile's)."""
  231. from cron.execution_identity import enter_cron_execution, exit_cron_execution
  232. from cron.executions import get_execution
  233. token = enter_cron_execution(job, execution_id, get_execution(execution_id) or {})
  234. try:
  235. return run_job(job, execution_id=execution_id, **kwargs)
  236. finally:
  237. exit_cron_execution(token)
  238. async def execute(authority, ref, row, policy):
  239. from cron.scheduler import run_job
  240. from gateway.run import _profile_runtime_scope
  241. data = json.loads(policy.request_json)
  242. if (row['payload'] != {'text': data['extra_prompt'] or ''}
  243. or row['request_id'] != data['request_id']
  244. or row['principal_id'] != authority.sessions[ref.session_id].source.user_id):
  245. # A cron session only runs its own frozen fire; anything else typed into it fails with a
  246. # descriptor the viewer can render (and offer a new chat for), never a generic failure.
  247. from agent.i18n import t
  248. text = t('gateway.errors.cron_session_read_only')
  249. authority.pending_results[row['admission_id']] = {'result': {
  250. 'final_response': '', 'messages': [], 'failed': True, 'completed': False,
  251. 'error': text, 'error_surface': dict(CRON_SESSION_READ_ONLY_SURFACE)},
  252. 'usage': {}}
  253. return text
  254. cancellations = _cancellations(authority)
  255. cancel = cancellations.setdefault(row['admission_id'], threading.Event())
  256. execution_id = scheduler_execution_id(data['cron_job']['id'], row['request_id'])
  257. # The frozen snapshot predates the fire's ledger row; the run's progress stamps need its id.
  258. job = dict(data['cron_job'], execution_id=execution_id)
  259. token = bind_execution(authority, ref.session_id, job['id'], execution_id)
  260. try:
  261. from cron.cancellation import CancelRequest, _CombinedCancelEvent
  262. db_path = await asyncio.to_thread(Path(authority.db.db_path).resolve)
  263. with _profile_runtime_scope(db_path.parent):
  264. # The forwarded ``cron.cancel`` carries no reason, so the operator's marker is read here
  265. # first: a stopped agent run reports "cancelled on request", not a lost fire claim.
  266. stop = _CombinedCancelEvent(CancelRequest(execution_id), cancel)
  267. result = await asyncio.to_thread(_run_identified, run_job, job, execution_id,
  268. extra_prompt=data['extra_prompt'], cancel_event=stop)
  269. # run_job stamps run-side verdicts on ITS job copy (quota hold, unreachable model); the
  270. # firing scheduler's bookkeeping reads them off its own dict, so they ride the result.
  271. flags = {key: job[key] for key in RUN_JOB_FLAGS if key in job}
  272. authority.pending_results[row['admission_id']] = {
  273. 'result': {'final_response': result[2], 'cron_result': list(result), 'cron_job_flags': flags,
  274. 'failed': not result[0], 'completed': result[0]}, 'usage': {}}
  275. if not result[0]:
  276. raise RuntimeError(result[3] or 'cron execution failed')
  277. return result[2]
  278. finally:
  279. from cron.scheduler_liveness import owner_progress
  280. owner_progress.pop(execution_id, None)
  281. _execution.reset(token)
  282. cancellations.pop(row['admission_id'], None)