scheduler_authority.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230
  1. """Transport-only cron execution; output/delivery remain with the firing scheduler."""
  2. import asyncio
  3. import hashlib
  4. import json
  5. import uuid
  6. class CronExecutionUnknown(RuntimeError):
  7. """The owner may have accepted this fire; never book it as failed or re-fire."""
  8. def journal_path(job_id, request_id):
  9. from hermes_constants import get_hermes_home
  10. key = hashlib.sha256(json.dumps([job_id, request_id]).encode()).hexdigest()
  11. return get_hermes_home() / 'cron' / 'admissions' / (key + '.json')
  12. def _pause_identity(paused):
  13. """The exact pause this recovery uncertainty caused, so only that pause is ever lifted."""
  14. return {'paused_at': paused.get('paused_at'), 'reason': paused.get('paused_reason')} if paused else True
  15. def replayable_completions(job_id, completions):
  16. """The recorded completion ids whose admission journal still exists: only journal recovery
  17. can replay a completion into ``mark_job_run``, so an id without one is a dead tombstone."""
  18. return [eid for eid in completions or () if journal_path(job_id, eid).exists()]
  19. def pause_for_unknown(job_id, execution_id, reason):
  20. """The firer pauses the job for this fire's unverified admission and records that exact pause
  21. on the fire's journal, so the receipt that later proves the run terminal can lift it."""
  22. from cron.jobs import pause_job
  23. from utils import atomic_json_write
  24. paused = pause_job(job_id, reason=reason)
  25. journal = journal_path(job_id, execution_id)
  26. if paused and journal.exists():
  27. record = json.loads(journal.read_text(encoding='utf-8-sig'))
  28. record['pause_recorded'] = _pause_identity(paused)
  29. atomic_json_write(journal, record, mode=0o600, fsync_dir=True)
  30. def _lift_pause(job_id, pause):
  31. """A verified terminal receipt resolved the uncertainty that paused the schedule: resume it iff
  32. the job is still held by exactly that pause (an operator's own pause/resume is never touched)."""
  33. from cron.jobs import get_job, resume_job
  34. job = get_job(job_id) if isinstance(pause, dict) else None
  35. if (job is not None and job.get('state') == 'paused'
  36. and (job.get('paused_at'), job.get('paused_reason')) == (pause['paused_at'], pause['reason'])):
  37. try:
  38. resume_job(job_id)
  39. except ValueError: # a one-shot whose instant has passed stays paused for the operator
  40. pass
  41. def run_canonical_job(job, *, extra_prompt=None, cancel_event=None, execution_id=None):
  42. from gateway.session_cron import owner_for_home, operation
  43. from hermes_cli.gateway_client import connect_gateway
  44. from hermes_constants import get_hermes_home
  45. from utils import atomic_json_write
  46. request_id = execution_id or job.get('execution_id')
  47. if not request_id:
  48. request_id = job['execution_id'] = uuid.uuid4().hex
  49. params = {'job_id': job['id'], 'request_id': request_id,
  50. 'extra_prompt': extra_prompt}
  51. root = get_hermes_home() / 'cron' / 'admissions'
  52. journal = journal_path(params['job_id'], params['request_id'])
  53. record = {'params': params, 'receipt': None}
  54. if journal.exists():
  55. record = json.loads(journal.read_text(encoding='utf-8-sig'))
  56. if record['params'] != params:
  57. raise CronExecutionUnknown('cron admission identity conflict')
  58. attempted = journal.exists()
  59. owner = owner_for_home(get_hermes_home())
  60. def save():
  61. root.mkdir(parents=True, exist_ok=True)
  62. atomic_json_write(journal, record, mode=0o600, fsync_dir=True)
  63. async def observe(call):
  64. nonlocal attempted
  65. attempted = True
  66. save()
  67. try:
  68. receipt = await call('submit', params)
  69. except Exception as exc:
  70. from hermes_cli.gateway_client import GatewayRPCError
  71. # A returned owner exception is definite; a transport disconnect is not. Even an
  72. # owner error after commit must retain the admitted identity, so verify its absence.
  73. if owner is not None or isinstance(exc, GatewayRPCError):
  74. state = await call('recover', params)
  75. if state['status'] == 'missing':
  76. error = f'{type(exc).__name__}: {exc}'
  77. result = [False, f'# Cron Job: {job["id"]} (FAILED)\n\n{error}\n', '', error]
  78. record['refusal'] = result
  79. record['job'] = dict(job)
  80. save()
  81. return tuple(result)
  82. if state['status'] not in {'queued', 'started', 'terminal'}:
  83. raise
  84. # Admitted before the answer was lost: observe it through the recovered receipt.
  85. receipt = {'session_id': state['session_id'], 'admission_id': state['admission_id']}
  86. else:
  87. raise
  88. record['receipt'] = receipt
  89. save()
  90. delay, progress = .1, None
  91. while True:
  92. if cancel_event is not None and cancel_event.is_set():
  93. await call('cancel', receipt)
  94. state = await call('status', receipt)
  95. if state.get('progress_at') not in (None, progress):
  96. # The owner's agent is still working: refresh OUR ledger row (the owner's stamp is
  97. # fenced to the row's owner), once per owner stamp, so the stale-claim sweep measures
  98. # the run's silence. A wedged owner stops stamping and the sweep still fires.
  99. from cron.executions import touch_execution_progress
  100. progress = state['progress_at']
  101. await asyncio.to_thread(touch_execution_progress, request_id)
  102. if state['status'] == 'terminal':
  103. job.update(state.get('job_flags') or {})
  104. return tuple(state['result'])
  105. if state['status'] == 'unknown':
  106. raise CronExecutionUnknown('unknown_execution: cron admission was not replayed')
  107. await asyncio.sleep(delay)
  108. delay = min(delay * 2, 2.0)
  109. async def remote():
  110. async with connect_gateway() as client:
  111. return await observe(lambda op, data: client.rpc('cron.' + op, **data))
  112. try:
  113. if owner is not None:
  114. authority, loop = owner
  115. try:
  116. current = asyncio.get_running_loop()
  117. except RuntimeError:
  118. current = None
  119. if current is loop:
  120. raise RuntimeError('cron synchronous execution must run off the owner event loop')
  121. return asyncio.run_coroutine_threadsafe(
  122. observe(lambda op, data: operation(authority, op, data)), loop).result()
  123. return asyncio.run(remote())
  124. except Exception as exc:
  125. if attempted:
  126. raise CronExecutionUnknown(f'Cron execution unverified; reconcile {journal}: {exc}') from exc
  127. error = f'{type(exc).__name__}: {exc}'
  128. return False, f'# Cron Job: {job["id"]} (FAILED)\n\n{error}\n', '', error
  129. def reconcile_pending(*, allow_connect=True):
  130. """Observe prepared fires; never submit missing or interrupted work.
  131. With allow_connect=False, a headless tick leaves durable receipts for the next
  132. live owner instead of ensuring or spawning a gateway.
  133. """
  134. from gateway.session_cron import owner_for_home, operation
  135. from hermes_constants import get_hermes_home
  136. from hermes_cli.gateway_client import connect_gateway
  137. from cron.executions import FIRER_FINISHED_STATES, execution_owner_live, get_execution
  138. from cron.jobs import get_job, pause_job
  139. from cron.scheduler import (_RunDelivery, _FireOwnership, _apply_agent_failure_marker,
  140. _save_compose_deliver)
  141. from cron.scheduler_bookkeeping import fail_empty_response, finish_completed_run
  142. from utils import atomic_json_write
  143. import logging
  144. root = get_hermes_home() / 'cron' / 'admissions'
  145. for journal in sorted(root.glob('*.json')):
  146. try:
  147. record = json.loads(journal.read_text(encoding='utf-8-sig'))
  148. params = record['params']
  149. if journal != journal_path(params['job_id'], params['request_id']):
  150. raise ValueError('cron journal identity conflict')
  151. if execution_owner_live(params['request_id'], params['job_id']):
  152. # Its firer (this process included) still owns the row and its tail: re-saving the
  153. # output or re-marking the job on every tick would duplicate its side effects.
  154. continue
  155. row = get_execution(params['request_id'])
  156. if row is not None and row['job_id'] == params['job_id'] and row['status'] in FIRER_FINISHED_STATES:
  157. # The firer booked this attempt's terminal outcome itself (ownership lost, shutdown
  158. # interrupt, a crashed tail): the receipt is not a result to save or deliver again.
  159. journal.unlink(missing_ok=True)
  160. continue
  161. if 'refusal' in record:
  162. state = {'status': 'terminal', 'result': record['refusal'], 'job': record['job']}
  163. else:
  164. owner = owner_for_home(get_hermes_home())
  165. if owner is None and not allow_connect:
  166. continue
  167. async def observe():
  168. if owner is not None:
  169. return await operation(owner[0], 'recover', params)
  170. async with connect_gateway() as client:
  171. return await client.rpc('cron.recover', **params)
  172. if owner is not None:
  173. state = asyncio.run_coroutine_threadsafe(observe(), owner[1]).result(timeout=20)
  174. else:
  175. state = asyncio.run(observe())
  176. if state['status'] == 'missing':
  177. # The owner holds no admission for this fire: a definite non-admission, booked as
  178. # the same failed run the in-submit refusal books. Only ``unknown`` pauses.
  179. current = get_job(params['job_id'])
  180. if current is None:
  181. journal.unlink(missing_ok=True)
  182. continue
  183. error = 'Cron admission missing: the owner never admitted this fire'
  184. state = {'status': 'terminal', 'job': current,
  185. 'result': [False, f'# Cron Job: {params["job_id"]} (FAILED)\n\n{error}\n', '', error]}
  186. if state['status'] != 'terminal':
  187. if state['status'] == 'unknown' and not record.get('pause_recorded'):
  188. reason = 'Canonical cron unknown; no automatic re-execution'
  189. record['pause_recorded'] = _pause_identity(pause_job(params['job_id'], reason=reason))
  190. atomic_json_write(journal, record, mode=0o600, fsync_dir=True)
  191. continue
  192. success, output, answer, error = state['result']
  193. job = dict(state['job'], **(state.get('job_flags') or {}))
  194. job['execution_id'] = params['request_id']
  195. # The same result verdicts as the ordinary tail: an agent-declared [CRON_FAILURE] and a
  196. # blank answer are failures, whichever process books them.
  197. success, error, declared = _apply_agent_failure_marker(job, success, error, answer)
  198. delivery = _RunDelivery(job, success, error, agent_declared=declared)
  199. # The canonical receipt supplies the result; a dead scheduler's fire claim does not
  200. # authorize replay and must not prevent finishing its durable bookkeeping.
  201. fence = _FireOwnership(dict(job, fire_claim=None))
  202. _save_compose_deliver(delivery, fence, answer, output, adapters=None, loop=None,
  203. verbose=False, execution_token=None)
  204. fail_empty_response(delivery, answer)
  205. if finish_completed_run(delivery, None, params['request_id'], recovered=True):
  206. _lift_pause(params['job_id'], record.get('pause_recorded'))
  207. except Exception:
  208. logging.getLogger(__name__).warning('Cron receipt recovery deferred: %s', journal, exc_info=True)