"""Dispatcher subprocess: submit one claim to its profile owner; never run an agent.""" # Spawned as a bare ``sys.executable -m``: in a PM install that interpreter carries no dependencies # until hermes_bootstrap selects the committed environment, so it must be the first import. try: import hermes_bootstrap except ModuleNotFoundError as exc: if exc.name != "hermes_bootstrap": raise import asyncio from dataclasses import dataclass import os from pathlib import Path import sys import time from typing import Optional # A worker that loses its owner (a dropped socket, a restart, `hermes update` draining it or a new # owner still starting) waits for it instead of exiting: the condition is the host's, not the card's, # and a failed exit here spends the card's failure budget and lets the owner end the turn. Asking # again is safe: ``kanban.run`` is idempotent per attempt and ``prompt.receipt`` is a read. # ``OWNER_WAIT_S`` is the longest the owner may stay silent between answers before the worker # gives up (the first ``kanban.run`` answer has its own wait, ``admission_answer_budget``). OWNER_WAIT_S = 180.0 _TRANSIENT_OWNER_STATES = frozenset({'draining', 'starting'}) # In-band owner refusals that leave the attempt intact: a draining owner admits nothing new, and a # retired authority's successor holds the admission the next connection reads. _TRANSIENT_RPC_REASONS = frozenset({'runtime_draining', 'runtime_retired'}) # Control-socket answers a restarting owner gives (an ``inaccessible`` owner's reason, from # discovery or the ticket fetch): a reset or half-written answer, or a replaced owner refusing a # stale instance. Unsafe nodes or peers, a refused authorization, a bad pointer or a failed start # stay final. _RESTART_CONTROL_REASONS = frozenset({'invalid_control_peer', 'incomplete_control_response', 'invalid_control_response'}) # Past the board lock wait: the owner's session setup and receipt before and after the write. ADMISSION_ANSWER_MARGIN_S = 60.0 def admission_answer_budget(): """Seconds to wait for ``kanban.run``'s answer: the longest the owner's admission write can wait on the board lock (``write_txn`` retries ``BEGIN IMMEDIATE`` ``_BUSY_MAX_RETRIES`` times, each under the busy timeout) plus ``ADMISSION_ANSWER_MARGIN_S``.""" from hermes_cli import kanban_db_connect lock_wait = (kanban_db_connect._BUSY_MAX_RETRIES + 1) * kanban_db_connect._resolve_busy_timeout_ms() / 1000 return lock_wait + ADMISSION_ANSWER_MARGIN_S @dataclass class _Claim: """What the owner has said about this attempt, carried across connections.""" answered: float # monotonic time of the owner's last answer session_id: Optional[str] = None admission_id: Optional[str] = None def silent_for(self): return time.monotonic() - self.answered async def run(params, board_db): from agent.retry_utils import jittered_backoff from hermes_cli.gateway_client import connect_gateway, GatewayClientError, outside_restart_wait from websockets.exceptions import WebSocketException if os.environ.get('HERMES_TUI_GATEWAY_URL'): raise GatewayClientError('Kanban requires the assigned local profile owner') # A lost socket is not a lost attempt, and exiting here would let ``_watch_submitter`` end the # owner's in-flight turn, so the next connection resumes whatever the last one reached. The # wait is bounded from the owner's last answer: an owner that keeps answering keeps the # worker, and one that stays silent for ``OWNER_WAIT_S`` fails the worker honestly. claim = _Claim(answered=time.monotonic()) failures = 0 while True: seen = claim.answered try: # Never into a restart's after-turn wait: it cut this socket so the turn's submitter # waits for the NEXT owner, and a worker polling its receipt inside the wait keeps the # turn it waits on alive, holding `hermes update` until restart_after_turn_timeout. with outside_restart_wait(): async with connect_gateway() as client: # kanban.run subscribes this socket to the card's session, but only receipts # matter here: discard the turn's events, or they fill the client's bounded # queue and its reader stops, with the next receipt behind it. drain = asyncio.create_task(_discard(client.events)) try: receipt = await _await_receipt(client, params, board_db, claim) finally: drain.cancel() # A restart also surfaces as a refused or reset dial, a failed handshake, or an answer # that never came. except (GatewayClientError, WebSocketException, OSError, TimeoutError) as exc: if not _restartable(exc): raise if claim.silent_for() >= OWNER_WAIT_S: # WebSocket errors can embed remote bodies; only our own messages are quoted. status = getattr(getattr(exc, 'response', None), 'status_code', None) detail = exc if isinstance(exc, GatewayClientError) else type(exc).__name__ + ( f' (HTTP {status})' if status else '') raise GatewayClientError(f'Kanban owner silent for {claim.silent_for():.0f}s: {detail}') from exc else: return _exit_code(receipt, params, board_db) failures = 1 if claim.answered != seen else failures + 1 # Jittered (1-2 s first, up to 16 s): workers of one dead owner do not all redial, and so # start a replacement, in the same instant. await asyncio.sleep(jittered_backoff(failures, base_delay=1.0, max_delay=8.0, jitter_ratio=1.0)) def _restartable(exc): """False when *exc* is the owner's own final answer; True for a restarting owner or a lost transport.""" from hermes_cli.gateway_client import GatewayRPCError, GatewayUnavailableError if isinstance(exc, GatewayRPCError): return str(exc) in _TRANSIENT_RPC_REASONS if isinstance(exc, GatewayUnavailableError): return exc.state in _TRANSIENT_OWNER_STATES or exc.reason_code in _RESTART_CONTROL_REASONS return True async def _discard(events): while True: await events.get() async def _await_receipt(client, params, board_db, claim): """Submit the claim (or, on a later connection, re-read the admission it got) and return its settled receipt, recording each owner answer on *claim*.""" if claim.admission_id is None: db = str(await asyncio.to_thread(Path(board_db).resolve)) # The owner answers only after it checks the claim and writes its ``owner_admitted`` marker # in one board write transaction. The board (the cross-profile dispatch bus) can make that # wait out its busy timeout, so the 30 s default left the owner admitting an attempt whose # submitter had exited (the later-bound worker's submitter-liveness watcher ended it, and the # admission was left ``unknown``). The wait is still finite: once the marker commits, # ``enforce_max_runtime`` and TTL reclaim skip the run (``owner_reclaim_paused``), so an owner # that stalls after admitting is bounded by this wait and ``OWNER_WAIT_S``. accepted = await client.rpc('kanban.run', _timeout=admission_answer_budget(), **dict(params, db=db)) receipt = accepted['receipt'] claim.session_id, claim.admission_id = accepted['session_id'], receipt['admission_id'] print(f'Session: {claim.session_id}', file=sys.stderr) else: receipt = await _receipt(client, claim) claim.answered = time.monotonic() while receipt['status'] not in {'terminal', 'unknown'}: await asyncio.sleep(.2) receipt = await _receipt(client, claim) claim.answered = time.monotonic() return receipt def _exit_code(receipt, params, board_db): from hermes_cli.gateway_client import GatewayClientError if receipt['status'] == 'unknown': raise GatewayClientError('Kanban execution unknown; retained by owner') from gateway.session_kanban import worker_result result = worker_result(board_db, params) # The attempt's reply ends the log, as a one-shot worker's did: crash diagnostics quote it. if result.get('last_output'): print(result['last_output']) return result.get('exit_code', 1) async def _receipt(client, claim): return await client.rpc('prompt.receipt', session_id=claim.session_id, admission_id=claim.admission_id) def main(): # The dead-worker sweep books this process's exit from the trailer in its log (it never # reaps us), so every exit path writes it — the same contract as a quiet one-shot worker. from hermes_cli.quiet_single_query import KANBAN_WORKER_EXIT_TRAILER rc = 1 try: rc = _submit() finally: print(f'\n{KANBAN_WORKER_EXIT_TRAILER}{int(rc)}', file=sys.stderr, flush=True) return rc def _submit(): try: params = {'board': os.environ['HERMES_KANBAN_BOARD'], 'task_id': os.environ['HERMES_KANBAN_TASK'], 'run_id': int(os.environ['HERMES_KANBAN_RUN_ID']), 'claim_lock': os.environ['HERMES_KANBAN_CLAIM_LOCK']} board_db = os.environ['HERMES_KANBAN_DB'] # Owner startup must never inherit this worker's task/tool/goal identity. for key in list(os.environ): # The busy timeout is a host tuning knob, not task identity: the answer budget reads it. if key.startswith('HERMES_KANBAN_') and key not in ('HERMES_KANBAN_HOME', 'HERMES_KANBAN_BUSY_TIMEOUT_MS'): os.environ.pop(key) for key in ('HERMES_SESSION_SOURCE', 'HERMES_TENANT', 'TERMINAL_CWD'): os.environ.pop(key, None) return asyncio.run(run(params, board_db)) except (KeyError, ValueError, OSError) as exc: print(f'Kanban owner refused: {exc}', file=sys.stderr) return 1 if __name__ == '__main__': raise SystemExit(main())