kanban_worker_client.py 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194
  1. """Dispatcher subprocess: submit one claim to its profile owner; never run an agent."""
  2. # Spawned as a bare ``sys.executable -m``: in a PM install that interpreter carries no dependencies
  3. # until hermes_bootstrap selects the committed environment, so it must be the first import.
  4. try:
  5. import hermes_bootstrap
  6. except ModuleNotFoundError as exc:
  7. if exc.name != "hermes_bootstrap":
  8. raise
  9. import asyncio
  10. from dataclasses import dataclass
  11. import os
  12. from pathlib import Path
  13. import sys
  14. import time
  15. from typing import Optional
  16. # A worker that loses its owner (a dropped socket, a restart, `hermes update` draining it or a new
  17. # owner still starting) waits for it instead of exiting: the condition is the host's, not the card's,
  18. # and a failed exit here spends the card's failure budget and lets the owner end the turn. Asking
  19. # again is safe: ``kanban.run`` is idempotent per attempt and ``prompt.receipt`` is a read.
  20. # ``OWNER_WAIT_S`` is the longest the owner may stay silent between answers before the worker
  21. # gives up (the first ``kanban.run`` answer has its own wait, ``admission_answer_budget``).
  22. OWNER_WAIT_S = 180.0
  23. _TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'})
  24. # In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a
  25. # retired authority's successor holds the admission the next connection reads.
  26. _TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'})
  27. # Control-socket answers a restarting owner gives (an ``inaccessible`` owner's reason, from
  28. # discovery or the ticket fetch): a reset or half-written answer, or a replaced owner refusing a
  29. # stale instance. Unsafe nodes or peers, a refused authorization, a bad pointer or a failed start
  30. # stay final.
  31. _RESTART_CONTROL_REASONS = frozenset({'invalid_control_peer', 'incomplete_control_response',
  32. 'invalid_control_response'})
  33. # Past the board lock wait: the owner's session setup and receipt before and after the write.
  34. ADMISSION_ANSWER_MARGIN_S = 60.0
  35. def admission_answer_budget():
  36. """Seconds to wait for ``kanban.run``'s answer: the longest the owner's admission write can
  37. wait on the board lock (``write_txn`` retries ``BEGIN IMMEDIATE`` ``_BUSY_MAX_RETRIES`` times,
  38. each under the busy timeout) plus ``ADMISSION_ANSWER_MARGIN_S``."""
  39. from hermes_cli import kanban_db_connect
  40. lock_wait = (kanban_db_connect._BUSY_MAX_RETRIES + 1) * kanban_db_connect._resolve_busy_timeout_ms() / 1000
  41. return lock_wait + ADMISSION_ANSWER_MARGIN_S
  42. @dataclass
  43. class _Claim:
  44. """What the owner has said about this attempt, carried across connections."""
  45. answered: float # monotonic time of the owner's last answer
  46. session_id: Optional[str] = None
  47. admission_id: Optional[str] = None
  48. def silent_for(self):
  49. return time.monotonic() - self.answered
  50. async def run(params, board_db):
  51. from agent.retry_utils import jittered_backoff
  52. from hermes_cli.gateway_client import connect_gateway, GatewayClientError, outside_restart_wait
  53. from websockets.exceptions import WebSocketException
  54. if os.environ.get('HERMES_TUI_GATEWAY_URL'):
  55. raise GatewayClientError('Kanban requires the assigned local profile owner')
  56. # A lost socket is not a lost attempt, and exiting here would let ``_watch_submitter`` end the
  57. # owner's in-flight turn, so the next connection resumes whatever the last one reached. The
  58. # wait is bounded from the owner's last answer: an owner that keeps answering keeps the
  59. # worker, and one that stays silent for ``OWNER_WAIT_S`` fails the worker honestly.
  60. claim = _Claim(answered=time.monotonic())
  61. failures = 0
  62. while True:
  63. seen = claim.answered
  64. try:
  65. # Never into a restart's after-turn wait: it cut this socket so the turn's submitter
  66. # waits for the NEXT owner, and a worker polling its receipt inside the wait keeps the
  67. # turn it waits on alive, holding `hermes update` until restart_after_turn_timeout.
  68. with outside_restart_wait():
  69. async with connect_gateway() as client:
  70. # kanban.run subscribes this socket to the card's session, but only receipts
  71. # matter here: discard the turn's events, or they fill the client's bounded
  72. # queue and its reader stops, with the next receipt behind it.
  73. drain = asyncio.create_task(_discard(client.events))
  74. try:
  75. receipt = await _await_receipt(client, params, board_db, claim)
  76. finally:
  77. drain.cancel()
  78. # A restart also surfaces as a refused or reset dial, a failed handshake, or an answer
  79. # that never came.
  80. except (GatewayClientError, WebSocketException, OSError, TimeoutError) as exc:
  81. if not _restartable(exc):
  82. raise
  83. if claim.silent_for() >= OWNER_WAIT_S:
  84. # WebSocket errors can embed remote bodies; only our own messages are quoted.
  85. status = getattr(getattr(exc, 'response', None), 'status_code', None)
  86. detail = exc if isinstance(exc, GatewayClientError) else type(exc).__name__ + (
  87. f' (HTTP {status})' if status else '')
  88. raise GatewayClientError(f'Kanban owner silent for {claim.silent_for():.0f}s: {detail}') from exc
  89. else:
  90. return _exit_code(receipt, params, board_db)
  91. failures = 1 if claim.answered != seen else failures + 1
  92. # Jittered (1-2 s first, up to 16 s): workers of one dead owner do not all redial, and so
  93. # start a replacement, in the same instant.
  94. await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0, jitter_ratio=1.0))
  95. def _restartable(exc):
  96. """False when *exc* is the owner's own final answer; True for a restarting owner or a lost transport."""
  97. from hermes_cli.gateway_client import GatewayRPCError, GatewayUnavailableError
  98. if isinstance(exc, GatewayRPCError):
  99. return str(exc) in _TRANSIENT_RPC_REASONS
  100. if isinstance(exc, GatewayUnavailableError):
  101. return exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS
  102. return True
  103. async def _discard(events):
  104. while True:
  105. await events.get()
  106. async def _await_receipt(client, params, board_db, claim):
  107. """Submit the claim (or, on a later connection, re-read the admission it got) and return its
  108. settled receipt, recording each owner answer on *claim*."""
  109. if claim.admission_id is None:
  110. db = str(await asyncio.to_thread(Path(board_db).resolve))
  111. # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker
  112. # in one board write transaction. The board (the cross-profile dispatch bus) can make that
  113. # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose
  114. # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the
  115. # admission was left ``unknown``). The wait is still finite: once the marker commits,
  116. # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner
  117. # that stalls after admitting is bounded by this wait and ``OWNER_WAIT_S``.
  118. accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db))
  119. receipt = accepted['receipt']
  120. claim.session_id, claim.admission_id = accepted['session_id'], receipt['admission_id']
  121. print(f'Session: {claim.session_id}', file=sys.stderr)
  122. else:
  123. receipt = await _receipt(client, claim)
  124. claim.answered = time.monotonic()
  125. while receipt['status'] not in {'terminal', 'unknown'}:
  126. await asyncio.sleep(.2)
  127. receipt = await _receipt(client, claim)
  128. claim.answered = time.monotonic()
  129. return receipt
  130. def _exit_code(receipt, params, board_db):
  131. from hermes_cli.gateway_client import GatewayClientError
  132. if receipt['status'] == 'unknown':
  133. raise GatewayClientError('Kanban execution unknown; retained by owner')
  134. from gateway.session_kanban import worker_result
  135. result = worker_result(board_db, params)
  136. # The attempt's reply ends the log, as a one-shot worker's did: crash diagnostics quote it.
  137. if result.get('last_output'):
  138. print(result['last_output'])
  139. return result.get('exit_code', 1)
  140. async def _receipt(client, claim):
  141. return await client.rpc('prompt.receipt', session_id=claim.session_id, admission_id=claim.admission_id)
  142. def main():
  143. # The dead-worker sweep books this process's exit from the trailer in its log (it never
  144. # reaps us), so every exit path writes it — the same contract as a quiet one-shot worker.
  145. from hermes_cli.quiet_single_query import KANBAN_WORKER_EXIT_TRAILER
  146. rc = 1
  147. try:
  148. rc = _submit()
  149. finally:
  150. print(f'\n{KANBAN_WORKER_EXIT_TRAILER}{int(rc)}', file=sys.stderr, flush=True)
  151. return rc
  152. def _submit():
  153. try:
  154. params = {'board': os.environ['HERMES_KANBAN_BOARD'], 'task_id': os.environ['HERMES_KANBAN_TASK'],
  155. 'run_id': int(os.environ['HERMES_KANBAN_RUN_ID']), 'claim_lock': os.environ['HERMES_KANBAN_CLAIM_LOCK']}
  156. board_db = os.environ['HERMES_KANBAN_DB']
  157. # Owner startup must never inherit this worker's task/tool/goal identity.
  158. for key in list(os.environ):
  159. # The busy timeout is a host tuning knob, not task identity: the answer budget reads it.
  160. if key.startswith('HERMES_KANBAN_') and key not in ('HERMES_KANBAN_HOME', 'HERMES_KANBAN_BUSY_TIMEOUT_MS'):
  161. os.environ.pop(key)
  162. for key in ('HERMES_SESSION_SOURCE', 'HERMES_TENANT', 'TERMINAL_CWD'):
  163. os.environ.pop(key, None)
  164. return asyncio.run(run(params, board_db))
  165. except (KeyError, ValueError, OSError) as exc:
  166. print(f'Kanban owner refused: {exc}', file=sys.stderr)
  167. return 1
  168. if __name__ == '__main__':
  169. raise SystemExit(main())