kanban_owner_recovery.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297
  1. """Read-only canonical admission fences for dispatcher reclaim paths."""
  2. from contextlib import closing
  3. import json
  4. from pathlib import Path
  5. import sqlite3
  6. import time
  7. import psutil
  8. def _owner_receipt(conn, task_id):
  9. """The parsed ``owner_admitted`` receipt of the task's CURRENT run, or None.
  10. The receipt is the owner's pointer to its own admission ledger. No receipt means the run
  11. never reached an owner (a plain dispatcher worker): nothing about it is ambiguous.
  12. """
  13. marker = conn.execute(
  14. "SELECT e.payload FROM task_events e JOIN tasks t ON t.current_run_id=e.run_id "
  15. "WHERE t.id=? AND e.task_id=t.id AND e.kind='owner_admitted' ORDER BY e.id DESC LIMIT 1",
  16. (task_id,)).fetchone()
  17. if marker is None:
  18. return None
  19. try:
  20. receipt = json.loads(marker['payload'])
  21. except (ValueError, TypeError):
  22. return {}
  23. return receipt if isinstance(receipt, dict) else {}
  24. def _admission_row(receipt):
  25. """``(status, outcome)`` from the owner ledger ``receipt`` names, or None when unreadable.
  26. None is "the owner said nothing we can read" — never "the owner said the attempt ended".
  27. """
  28. if not receipt:
  29. return None
  30. try:
  31. with closing(sqlite3.connect(Path(receipt['db']).as_uri() + '?mode=ro', uri=True)) as owner:
  32. row = owner.execute('SELECT status,outcome FROM session_admissions WHERE target_session_id=? AND request_id=?',
  33. (receipt['session_id'], receipt['request_id'])).fetchone()
  34. except (OSError, KeyError, TypeError, ValueError, sqlite3.Error):
  35. return None
  36. return (row[0], row[1]) if row is not None else None
  37. def owner_reclaim_paused(conn, task_id):
  38. """Missing/ambiguous owner receipts never authorize another attempt."""
  39. receipt = _owner_receipt(conn, task_id)
  40. if receipt is None:
  41. return False
  42. try:
  43. row = _admission_row(receipt)
  44. if row is not None and row[0] == 'terminal':
  45. # Interrupted still pauses the generic reclaim paths; owner_attempt_provably_over decides
  46. # when nothing can still be running it.
  47. return row[1] == 'interrupted'
  48. if row is None or row[0] in {'queued', 'unknown'}:
  49. return True
  50. from hermes_cli.process_identity import worker_birth
  51. return worker_birth(psutil.Process(receipt['pid'])) != receipt['birth']
  52. except (OSError, ValueError, KeyError, TypeError, psutil.Error):
  53. return True
  54. def _latest_run_event(conn, task_id, kind):
  55. """The newest ``kind`` payload on the task's CURRENT run, or ``{}``.
  56. A payload that cannot be read is not a payload that says the attempt is over, so
  57. callers treat ``{}`` as "no evidence" rather than as an absence.
  58. """
  59. row = conn.execute(
  60. "SELECT e.payload FROM task_events e JOIN tasks t ON t.current_run_id=e.run_id "
  61. "WHERE t.id=? AND e.task_id=t.id AND e.kind=? ORDER BY e.id DESC LIMIT 1",
  62. (task_id, kind)).fetchone()
  63. try:
  64. payload = json.loads(row['payload']) if row else {}
  65. except (ValueError, TypeError):
  66. return {}
  67. return payload if isinstance(payload, dict) else {}
  68. def _host_local_lock(lock) -> bool:
  69. """True when ``lock`` was issued by this host — the only pids we can judge."""
  70. from hermes_cli import kanban_db as kb
  71. return bool(lock) and str(lock).startswith(kb._host_prefix())
  72. def bound_interpreter_gone(conn, task_id):
  73. """True when the owner bound an interpreter to the task's current run and it is dead."""
  74. bound = _latest_run_event(conn, task_id, 'worker_bound')
  75. if not bound:
  76. return False
  77. from hermes_cli.kanban_db_dispatch import _worker_alive
  78. return not _worker_alive(bound.get('pid'), bound.get('started_at'))
  79. def _owner_process_gone(receipt) -> bool:
  80. """True when the receipt's own pid is gone, or its number now belongs to another process.
  81. The identity rule ``owner_reclaim_paused`` already applies: a live pid whose ``birth``
  82. differs is a recycled number, not the owner. A zombie keeps its birth until reaped but has
  83. already exited, so liveness goes through the dispatcher's zombie-aware ``_pid_alive``. A live pid we cannot compare (no ``birth``
  84. recorded) and an unreadable one are never proof of death.
  85. """
  86. try:
  87. pid = int(receipt['pid'])
  88. except (KeyError, TypeError, ValueError):
  89. return False
  90. from hermes_cli import kanban_db as kb
  91. if not kb._pid_alive(pid):
  92. return True # absent, or an exited zombie still waiting to be reaped
  93. try:
  94. from hermes_cli.process_identity import worker_birth
  95. current = worker_birth(psutil.Process(pid))
  96. except psutil.NoSuchProcess:
  97. return True
  98. except psutil.Error:
  99. return False
  100. birth = receipt.get('birth')
  101. return birth is not None and current != birth
  102. def _run_worker_pids(conn, task_row, task_id):
  103. """Every worker pid recorded for the task's current run, with its spawn fingerprint."""
  104. recorded = [(task_row['worker_pid'], task_row['worker_started_at'])]
  105. for kind in ('spawned', 'worker_bound'):
  106. payload = _latest_run_event(conn, task_id, kind)
  107. recorded.append((payload.get('pid'), payload.get('started_at')))
  108. return [(int(pid), fingerprint) for pid, fingerprint in recorded if pid]
  109. def owner_attempt_provably_over(conn, task_id) -> bool:
  110. """True when an unresolved owner receipt cannot still be executing this attempt.
  111. ``owner_reclaim_paused`` is about effects: an admission that may still be in flight is
  112. never repeated. It must not also mean the run keeps its claim forever. A paused run whose
  113. owner died, or whose owner settled the admission after losing its worker, may never have
  114. written a ``worker_bound`` event, so ``bound_interpreter_gone`` cannot speak for it. This
  115. asks the question that matters — can anything still be executing this attempt? — from
  116. evidence the board already keeps:
  117. * the run never bound an interpreter (a bound run is the crash sweep's to book);
  118. * every worker pid recorded for the run (``tasks.worker_pid``, its ``spawned`` and
  119. ``worker_bound`` events) is dead or recycled;
  120. * and the owner's own admission is SETTLED (``status='terminal'``) with at least one
  121. recorded pid, or the host-local claim has EXPIRED with no heartbeat inside
  122. ``DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS`` and the admission is settled or (pids
  123. recorded) the owner's process is gone.
  124. A settled admission outranks the owner's liveness: the owner is usually the long-lived
  125. gateway, alive by design, and it settles a lost managed worker's admission without
  126. extending the claim.
  127. Missing evidence is never proof: no receipt, a live pid whose identity cannot be
  128. compared, no recorded pid inside the claim's TTL, or an unreadable clock all return False
  129. and the pause holds.
  130. """
  131. from hermes_cli import kanban_db as kb
  132. from hermes_cli.kanban_db_dispatch import _worker_alive
  133. if _latest_run_event(conn, task_id, 'worker_bound'):
  134. return False # a bound run is the crash sweep's to book (bound_interpreter_gone)
  135. receipt = _owner_receipt(conn, task_id)
  136. if receipt is None:
  137. return False # no receipt: this pause did not come from an owner attempt
  138. row = conn.execute(
  139. "SELECT claim_lock, claim_expires, worker_pid, worker_started_at, last_heartbeat_at "
  140. "FROM tasks WHERE id=?", (task_id,)).fetchone()
  141. if row is None or not _host_local_lock(row['claim_lock']):
  142. return False
  143. pids = _run_worker_pids(conn, row, task_id)
  144. if any(_worker_alive(pid, fingerprint) for pid, fingerprint in pids):
  145. return False
  146. admission = _admission_row(receipt)
  147. settled = admission is not None and admission[0] == 'terminal'
  148. if settled and pids:
  149. return True
  150. now = int(time.time())
  151. expires = kb._to_epoch(row['claim_expires'])
  152. if expires is None or expires >= now:
  153. return False
  154. heartbeat = kb._to_epoch(row['last_heartbeat_at'])
  155. if heartbeat is not None and now - heartbeat <= kb.DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS:
  156. return False
  157. # A lapsed claim: a settled admission is proof; an unsettled one needs recorded pids and a dead owner.
  158. return settled or (bool(pids) and _owner_process_gone(receipt))
  159. def owner_attempt_note(conn, task_id, lost=None) -> str:
  160. """One clause naming why the owner says the attempt ended, or ``""``.
  161. The reclaim paths put this in the run error so the board says what happened instead of
  162. a bare ``stale_lock=``: a managed worker that died (its owner recorded ``worker_lost``,
  163. passed as ``lost`` when the caller already read it) is a different fault from a worker
  164. that never started.
  165. """
  166. if lost is None:
  167. lost = _latest_run_event(conn, task_id, 'worker_lost')
  168. reason = str(lost.get('reason') or '')
  169. if reason:
  170. return f"managed worker lost: {reason[:80]}"
  171. receipt = _owner_receipt(conn, task_id)
  172. if receipt is None:
  173. return ""
  174. admission = _admission_row(receipt)
  175. if admission is not None and admission[0] == 'terminal':
  176. return f"owner admission settled {admission[1] or 'terminal'}"
  177. return ""
  178. def reclaim_error(conn, task_id, claim_lock) -> str:
  179. """A stale-lock reclaim's run error, naming how the attempt ended when the owner recorded it."""
  180. note = owner_attempt_note(conn, task_id)
  181. return f"stale_lock={claim_lock}" + (f" [{note}]" if note else "")
  182. def apply_attempt_note(conn, task_id, pid, dead) -> None:
  183. """Fold the owner's own record of how the attempt ended into ONE dead worker's booking.
  184. A bound interpreter whose ``worker_result`` says exit 0 after its managed execution was
  185. lost reads as a protocol violation. The owner's ``worker_lost`` evidence outranks it: the
  186. turn never finished, so it books as a crash and charges the breaker, and the
  187. protocol-violation streak (which reads the persisted payload) does not count it.
  188. """
  189. lost = _latest_run_event(conn, task_id, "worker_lost")
  190. note = owner_attempt_note(conn, task_id, lost)
  191. if not note:
  192. return
  193. dead.event_payload["attempt_note"] = note
  194. if lost:
  195. dead.event_payload["worker_lost"] = lost
  196. if dead.protocol_violation:
  197. from hermes_cli.kanban_db_dispatch import _PROTOCOL_VIOLATION_ERROR
  198. dead.protocol_violation = False
  199. dead.event_payload.pop("protocol_violation", None)
  200. dead.event_payload["exit_kind"] = dead.kind
  201. dead.event_kind = "crashed"
  202. # Keep the worker's last output that _classify_dead_worker appended.
  203. dead.error_text = (f"pid {pid} exited cleanly after its managed worker was lost."
  204. + dead.error_text.removeprefix(_PROTOCOL_VIOLATION_ERROR))
  205. # Before the worker's own output: the stored error is capped, and the output may be long.
  206. from hermes_cli.kanban_db_dispatch import WORKER_OUTPUT_SEP
  207. head, sep, output = dead.error_text.partition(WORKER_OUTPUT_SEP)
  208. dead.error_text = f"{head} [{note}]{sep}{output}"
  209. def owner_reclaim_blocked(conn, task_id, *, crash_sweep=False) -> bool:
  210. """True when a reclaim path must leave this task alone.
  211. The pause protects the effects of an attempt that may still be in flight, and stops
  212. holding the claim as soon as nothing can still be executing it. A run whose bound
  213. interpreter is dead stays the crash sweep's alone (``crash_sweep=True``), so it is booked
  214. ``crashed`` and re-queued once rather than ``reclaimed`` by whichever path ticks first. A
  215. run that never bound one is released by any path once ``owner_attempt_provably_over``:
  216. the shape that used to be unreclaimable forever, because ``bound_interpreter_gone`` was the
  217. only override and it needs the ``worker_bound`` event that run never wrote.
  218. """
  219. if not owner_reclaim_paused(conn, task_id):
  220. return False
  221. if crash_sweep and bound_interpreter_gone(conn, task_id):
  222. return False
  223. return not owner_attempt_provably_over(conn, task_id)
  224. def owner_reclaim_blocked_after_kill(conn, task_id, *, signalled: bool) -> bool:
  225. """The under-lock re-check for a reclaim path, after its termination step.
  226. When this host just signalled the run's worker (``signalled``), killing it takes its bound
  227. interpreter down too (``_watch_submitter``) and the owner settles the admission ``unknown``:
  228. a pause this path caused itself, which would leave a timed-out run for the crash sweep to
  229. misbook ``crashed``. Only an interpreter still LIVE (one that bound after the first look)
  230. keeps the claim. Without a local signal (a foreign-host claim) the bound interpreter's pid
  231. means nothing here, so a "dead" reading is no proof and the plain gate applies.
  232. """
  233. return owner_reclaim_blocked(conn, task_id, crash_sweep=signalled)
  234. def managed_run_exit_code(conn, task_id, worker_pid, claim_lock):
  235. """The exit code a managed run's authority recorded for the task's CURRENT run, else None.
  236. Managed interpreters are children of the authority, not the dispatcher, so their result is
  237. read from the run's ``worker_result`` event (it survives a dispatcher restart and cannot be
  238. reaped here). It speaks for the run only under the same claim and from a pid of this run:
  239. the dispatcher's own worker, or the owner-side interpreter ``worker_bound`` recorded under
  240. that claim — ``session_kanban._record_worker_result`` writes the interpreter's pid, never the
  241. spawned submitter's ``tasks.worker_pid``."""
  242. def latest(kind):
  243. row = conn.execute(
  244. "SELECT e.payload FROM task_events e JOIN tasks t ON t.current_run_id=e.run_id "
  245. "WHERE t.id=? AND e.task_id=t.id AND e.kind=? ORDER BY e.id DESC LIMIT 1", (task_id, kind)).fetchone()
  246. try:
  247. payload = json.loads(row['payload']) if row else {}
  248. except ValueError:
  249. return {}
  250. return payload if isinstance(payload, dict) else {}
  251. result, bound = latest('worker_result'), latest('worker_bound')
  252. pids = {worker_pid, *([bound.get('pid')] if bound.get('claim_lock') == claim_lock else [])}
  253. return result.get('exit_code') if result.get('claim_lock') == claim_lock and result.get('pid') in pids else None