runtime_session_store.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497
  1. """Explicit worker persistence facade; never opens the canonical SQLite store.
  2. This first operation family is deliberately NOT a complete SessionDB substitute.
  3. No existing cron/child/compute consumer is switched until its inventory is covered.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. import contextlib
  8. import contextvars
  9. from concurrent.futures import ThreadPoolExecutor
  10. import json
  11. import logging
  12. import os
  13. from pathlib import Path
  14. import queue
  15. import sys
  16. import threading
  17. from hermes_dir_acl import make_user_dir
  18. logger = logging.getLogger(__name__)
  19. _worker_process = False
  20. def is_worker_process():
  21. """A successful self-registration makes this interpreter compute-only.
  22. Process-scoped (not ContextVar): import-time recovery and fresh helper threads
  23. must not regain owner-ledger access. This is cooperative runtime ownership,
  24. not OS confinement or an authorization credential.
  25. """
  26. return _worker_process
  27. def apply_row_annotations(messages, annotations):
  28. """Mirror the owner's in-place row stamps onto the worker's dicts (None = the owner popped it)."""
  29. for message, annotation in zip(messages, annotations, strict=True):
  30. for key, value in annotation.items():
  31. if value is None:
  32. message.pop(key, None)
  33. else:
  34. message[key] = value
  35. class WorkerPersistenceError(RuntimeError):
  36. pass
  37. def _caller_summary(limit=8):
  38. """module:function:line of the frames above this store (its own frames skipped), never
  39. locals or argument values: best-effort callers swallow the guard's raise (#139029)."""
  40. frame, frames = sys._getframe(2), []
  41. while frame is not None and len(frames) < limit:
  42. module = frame.f_globals.get('__name__', '?')
  43. if not module.startswith('agent.runtime_session_'):
  44. frames.append(f'{module}:{frame.f_code.co_name}:{frame.f_lineno}')
  45. frame = frame.f_back
  46. return ' <- '.join(frames)
  47. def call_off_event_loop(fn, *args, **kwargs):
  48. """Call ``fn`` here, or on a worker thread when this thread is running an event loop.
  49. For synchronous best-effort writers that can run inside a callback Relay drives on its private
  50. loop (provider/tool callbacks): there ``WorkerRPC`` refuses the call (``synchronous_rpc_on_event_loop``,
  51. sticky for the rest of the turn; #139029). The callback already blocks that loop, so waiting for the
  52. thread costs it nothing; the context is copied so session-scoped state survives the hop, and errors
  53. re-raise here. Async code awaits ``asyncio.to_thread`` instead.
  54. """
  55. try:
  56. asyncio.get_running_loop()
  57. except RuntimeError:
  58. return fn(*args, **kwargs)
  59. context = contextvars.copy_context()
  60. with ThreadPoolExecutor(max_workers=1, thread_name_prefix='worker-persist-off-loop') as pool:
  61. return pool.submit(context.run, fn, *args, **kwargs).result()
  62. class _OwnerLink:
  63. """One owner WebSocket and the only thread that reads it. Events the owner broadcasts to every
  64. transport are discarded as they arrive, idle or not: left queued, websockets' bounded receive
  65. buffer pauses socket reads, which also stalls pongs and the close handshake. The reply to the
  66. one outstanding request id goes to its caller; a closed connection goes to it as the exception."""
  67. def __init__(self, ws):
  68. self.ws = ws
  69. self.awaiting = None
  70. self.replies = queue.SimpleQueue()
  71. self.reader = threading.Thread(target=self._read, name='worker-rpc-reader', daemon=True)
  72. self.reader.start()
  73. def _read(self):
  74. try:
  75. while True:
  76. try:
  77. frame = json.loads(self.ws.recv())
  78. except ValueError:
  79. continue
  80. if isinstance(frame, dict) and self.awaiting is not None and frame.get('id') == self.awaiting:
  81. self.replies.put(frame)
  82. except Exception as exc: # health: allow BLE001 -- handed to the caller, which raises it (ConnectionClosed included)
  83. self.replies.put(exc)
  84. def send(self, request_id, message):
  85. self.awaiting = request_id
  86. self.ws.send(message)
  87. def receive(self, timeout):
  88. try:
  89. reply = self.replies.get(timeout=timeout)
  90. except queue.Empty:
  91. raise TimeoutError('worker_receipt_timeout') from None
  92. if isinstance(reply, BaseException):
  93. raise reply
  94. return reply
  95. class WorkerRPC:
  96. """Bounded authenticated calls to an existing owner; never starts a daemon.
  97. No owner-loop synchronous self-RPC or SQLite fallback is permitted.
  98. One connection serves every call (serialized by ``lock``, correlated by id); it is
  99. discovered, identified and ticketed on connect exactly as each call once was.
  100. """
  101. def __init__(self, home):
  102. self.home = Path(home).resolve()
  103. self.lock = threading.Lock()
  104. self.endpoint = None
  105. self.link = None
  106. self.request_id = 0
  107. def _connection(self):
  108. if self.link is not None:
  109. if self.link.reader.is_alive():
  110. return self.link
  111. # The owner closed it while idle, before any send: redial with a fresh ticket.
  112. self._drop()
  113. from hermes_cli.gateway_client import _session_ticket, gateway_ws_target
  114. from hermes_cli.gateway_runtime_discovery import query_identify
  115. from websockets.sync.client import connect
  116. # A served secondary has no socket; its multiplexer's descriptor names it.
  117. from hermes_cli.gateway_runtime import discover_gateway_endpoint
  118. discovery = discover_gateway_endpoint(self.home, timeout=5)
  119. if discovery.state == 'ready' and discovery.endpoint is not None:
  120. self.endpoint = discovery.endpoint
  121. elif discovery.state != 'draining' or self.endpoint is None:
  122. raise WorkerPersistenceError('owner_unavailable')
  123. from hermes_cli.gateway_runtime import control_home_for
  124. endpoint = self.endpoint
  125. descriptor = query_identify(control_home_for(self.home, endpoint), timeout=5)
  126. if descriptor.get('pid') == os.getpid():
  127. raise WorkerPersistenceError('synchronous_self_rpc')
  128. ticket = _session_ticket(self.home, endpoint, purpose='worker-adoption')
  129. url, protocols = gateway_ws_target(endpoint, ticket)
  130. # The owner is loopback, like connect_gateway's peer: an inherited HTTPS_PROXY must not carry it.
  131. # No library keepalive: its pong deadline would run independently of (and could fail) a call
  132. # inside its receipt budget; that budget and the observed close are the liveness checks.
  133. self.link = _OwnerLink(connect(url, subprotocols=protocols, open_timeout=5, close_timeout=1,
  134. max_size=8 * 1024 * 1024, proxy=None, ping_interval=None))
  135. return self.link
  136. def _drop(self):
  137. link, self.link = self.link, None
  138. if link is not None:
  139. with contextlib.suppress(OSError): # a broken socket is already gone
  140. link.ws.close()
  141. def close(self):
  142. with self.lock:
  143. self._drop()
  144. def __call__(self, method, **params):
  145. try:
  146. asyncio.get_running_loop()
  147. except RuntimeError:
  148. pass
  149. else:
  150. logger.warning('synchronous_rpc_on_event_loop: %s on thread %s; caller: %s',
  151. method, threading.current_thread().name, _caller_summary())
  152. raise WorkerPersistenceError('synchronous_rpc_on_event_loop')
  153. from websockets.exceptions import ConnectionClosed
  154. with self.lock:
  155. self.request_id += 1
  156. request_id = self.request_id
  157. message = json.dumps({'jsonrpc': '2.0', 'id': request_id, 'method': method, 'params': params})
  158. try:
  159. link = self._connection()
  160. try:
  161. link.send(request_id, message)
  162. except ConnectionClosed:
  163. # websockets raises this from send only before a complete frame is written (closed
  164. # on entry, or the socket write failed): the owner closed the link after the idle
  165. # check and never saw this request, so redial and send it once more.
  166. self._drop()
  167. link = self._connection()
  168. link.send(request_id, message)
  169. response = link.receive(20)
  170. except BaseException:
  171. # Outcome unknown once sent: surface the error and never resend; the next call redials.
  172. self._drop()
  173. raise
  174. if 'error' in response:
  175. raise WorkerPersistenceError(response['error'].get('message', 'persistence_failed'))
  176. if method in ('worker.register', 'worker.adopt') and params.get('pid') == os.getpid():
  177. global _worker_process
  178. _worker_process = True
  179. return response['result']
  180. from agent.runtime_session_compression import RuntimeSessionCompressionMixin
  181. from agent.runtime_session_lifecycle import RuntimeSessionLifecycleMixin
  182. class RuntimeSessionStore(RuntimeSessionCompressionMixin, RuntimeSessionLifecycleMixin):
  183. """Synchronous durable results with a private byte-bounded retry journal.
  184. A failed write remains pending, and failure is sticky until explicit retry or
  185. adoption. Local queue acceptance is never returned as canonical commit.
  186. Pending journals live outside age-pruned cache trees.
  187. """
  188. # The process's async-delegation ledger belongs to the adopted execution, never a reserved child.
  189. binds_delegation_ledger = True
  190. def __init__(self, rpc, scope, outbox_dir, adopted=None, *, max_bytes=4 * 1024 * 1024):
  191. self.rpc = rpc
  192. self.scope = dict(scope)
  193. self.max_bytes = max_bytes
  194. self.lock = threading.RLock()
  195. self.failure = None
  196. self.path = Path(outbox_dir) / 'pending.json'
  197. make_user_dir(self.path.parent, mode=0o700, parents=True, exist_ok=True)
  198. if self.path.parent.is_symlink() or self.path.is_symlink():
  199. raise WorkerPersistenceError('unsafe_outbox')
  200. os.chmod(self.path.parent, 0o700)
  201. from gateway.status import _try_acquire_file_lock
  202. lock_path = self.path.parent / 'owner.lock'
  203. if lock_path.is_symlink():
  204. raise WorkerPersistenceError('unsafe_outbox')
  205. self._outbox_owner = lock_path.open('a+', encoding='utf-8')
  206. os.chmod(lock_path, 0o600)
  207. if not _try_acquire_file_lock(self._outbox_owner):
  208. self._outbox_owner.close()
  209. raise WorkerPersistenceError('outbox_in_use')
  210. try:
  211. if self.path.exists():
  212. if self.path.stat().st_size > max_bytes:
  213. raise WorkerPersistenceError('outbox_full')
  214. self.journal = json.loads(self.path.read_text(encoding="utf-8-sig"))
  215. reconciled = self.journal['scope'] != self.scope
  216. if reconciled:
  217. self._reconcile_adopted(adopted)
  218. else:
  219. reconciled = False
  220. self.journal = {'scope': self.scope, 'next_sequence': 1, 'pending': []}
  221. self._save(self.journal)
  222. if self.journal['pending']:
  223. self.retry_pending()
  224. if reconciled and self.scope != dict(scope):
  225. # Replay must land the journal exactly on the adopted assignment (a pending
  226. # publication moves it to the child); anything else is not this successor's outbox.
  227. raise WorkerPersistenceError('outbox_scope_mismatch')
  228. if is_worker_process() and self.binds_delegation_ledger:
  229. from tools.async_delegation_worker import bind_worker_delegation_store
  230. bind_worker_delegation_store(self)
  231. except Exception:
  232. self._outbox_owner.close()
  233. raise
  234. def _reconcile_adopted(self, adopted):
  235. """Carry a retained outbox across verified owner replacement.
  236. ``adopted`` is the separately authenticated ``worker.adopt`` result for the requested
  237. scope; never self-adopt. Only the owner epoch (forward) and the physical target may
  238. differ from the journal (a same-epoch restart can still find the target rotated).
  239. Pending entries replay under the journal's own target with the adopted epoch, so an
  240. exact lost-ACK retry resolves its original receipt instead of creating a second one;
  241. the constructor then requires the replayed scope to equal the adopted one.
  242. """
  243. old, new = self.journal['scope'], self.scope
  244. fixed = set(new) - {'epoch', 'session_id'}
  245. if (not isinstance(adopted, dict) or set(old) != set(new)
  246. or any(old[key] != new[key] for key in fixed)
  247. or adopted.get('execution_id') != new.get('execution_id')
  248. or adopted.get('generation') != new.get('generation')
  249. or adopted.get('session_id') != new.get('session_id')
  250. or adopted.get('owner_epoch') != new.get('epoch') or adopted.get('status') != 'running'
  251. or type(old.get('epoch')) is not int or type(new.get('epoch')) is not int
  252. or old['epoch'] > new['epoch']
  253. # Only a pending publication can still move the target; replay must prove it.
  254. or (old['session_id'] != new['session_id'] and not any(
  255. entry['operation'] == 'compression.publish' for entry in self.journal['pending']))):
  256. raise WorkerPersistenceError('outbox_scope_mismatch')
  257. candidate = dict(self.journal, scope=dict(old, epoch=new['epoch']))
  258. self._save(candidate)
  259. self.scope = candidate['scope']
  260. self.journal = candidate
  261. def _save(self, journal):
  262. if self._outbox_owner.closed:
  263. raise WorkerPersistenceError('outbox_closed')
  264. encoded = json.dumps(journal, ensure_ascii=True, allow_nan=False, separators=(',', ':')).encode()
  265. if len(encoded) > self.max_bytes:
  266. self.failure = 'outbox_full'
  267. raise WorkerPersistenceError(self.failure)
  268. from utils import atomic_write_bytes
  269. # Owner-only journal (the dir is 0700 too); the rename and its directory entry are durable.
  270. atomic_write_bytes(self.path, encoded, tmp_prefix='.pending-', mode=0o600, fsync_dir=True)
  271. def _session(self, session_id):
  272. if session_id != self.scope['session_id']:
  273. raise WorkerPersistenceError('permission_denied')
  274. def _apply(self, operation, payload):
  275. with self.lock:
  276. if self.failure:
  277. raise WorkerPersistenceError(self.failure)
  278. if self.journal['pending']:
  279. raise WorkerPersistenceError('pending_receipt')
  280. entry = {'sequence': self.journal['next_sequence'], 'operation': operation, 'payload': payload}
  281. candidate = json.loads(json.dumps(self.journal))
  282. candidate['pending'].append(json.loads(json.dumps(entry, allow_nan=False)))
  283. candidate['next_sequence'] += 1
  284. try:
  285. self._save(candidate)
  286. except Exception as exc:
  287. self.failure = str(exc)
  288. raise
  289. self.journal = candidate
  290. return self.retry_pending()[0]
  291. def retry_pending(self):
  292. with self.lock:
  293. results = []
  294. try:
  295. while self.journal['pending']:
  296. entry = self.journal['pending'][0]
  297. result = self.rpc('worker.persist', **self.scope, **entry)
  298. candidate = dict(self.journal, pending=self.journal['pending'][1:])
  299. candidate = self._compression_receipt_journal(candidate, result)
  300. self._save(candidate)
  301. self.scope = candidate['scope']
  302. self.journal = candidate
  303. results.append(result)
  304. except Exception as exc:
  305. self.failure = str(exc)
  306. raise
  307. self.failure = None
  308. return results
  309. def adopt(self, epoch):
  310. """Install only a separately verified worker.adopt result; never self-adopt."""
  311. with self.lock:
  312. candidate = dict(self.journal, scope=dict(self.scope, epoch=epoch))
  313. self._save(candidate)
  314. self.scope = candidate['scope']
  315. self.journal = candidate
  316. self.failure = None
  317. def append_messages_batch(self, session_id, messages, compression_lock_holder=None,
  318. turn_lease_holder=None, chunk_rows=None, turn_lease_ttl_seconds=300.0):
  319. self._session(session_id)
  320. if chunk_rows is not None:
  321. raise WorkerPersistenceError('unsupported_operation')
  322. if compression_lock_holder is not None or turn_lease_ttl_seconds != 300.0:
  323. return self._append_compression_messages(session_id, messages, compression_lock_holder,
  324. turn_lease_holder, turn_lease_ttl_seconds)
  325. result = self._apply('transcript.append', {'messages': messages, 'turn_lease_holder': turn_lease_holder})
  326. apply_row_annotations(messages, result['annotations'])
  327. return result['count']
  328. def try_acquire_session_turn_lease(self, session_id, holder, *, ttl_seconds=300.0, patience_s=None):
  329. self._session(session_id)
  330. return self._apply('turn.acquire', {'holder': holder, 'ttl_seconds': ttl_seconds})['value']
  331. def acquire_session_turn_lease(self, session_id, holder, *, ttl_seconds=300.0,
  332. wait_seconds=1800.0, poll_interval_seconds=1.0, on_wait=None,
  333. wait_notice_interval_seconds=15.0, should_abort=None, acquire_patience_s=0.5,
  334. on_contended=None):
  335. # Reuse only the local polling orchestrator, not the SQLite mixin surface. Same signature
  336. # as the owner's (``on_contended``: a busy-database notice; the RPC try never raises it).
  337. from hermes_state_compression import SessionCompressionMixin
  338. return SessionCompressionMixin.acquire_session_turn_lease(self, session_id, holder,
  339. ttl_seconds=ttl_seconds, wait_seconds=wait_seconds,
  340. poll_interval_seconds=poll_interval_seconds, on_wait=on_wait,
  341. wait_notice_interval_seconds=wait_notice_interval_seconds,
  342. should_abort=should_abort, acquire_patience_s=acquire_patience_s,
  343. on_contended=on_contended)
  344. def refresh_session_turn_lease(self, session_id, holder, *, ttl_seconds=300.0, patience_s=None):
  345. self._session(session_id)
  346. return self._apply('turn.renew', {'holder': holder, 'ttl_seconds': ttl_seconds})['value']
  347. def release_session_turn_lease(self, session_id, holder):
  348. # DurableTurnLease retains its admission target across physical rotation.
  349. self._apply('turn.cleanup', {'target': session_id, 'holder': holder})
  350. def queue_token_counts(self, session_id, **usage):
  351. self._session(session_id)
  352. self._apply('usage.main', usage)
  353. def record_auxiliary_usage(self, session_id, task, **usage):
  354. self._session(session_id)
  355. self._apply('usage.auxiliary', dict(usage, task=task))
  356. def flush_token_counts(self, timeout=5.0):
  357. with self.lock:
  358. if self.failure:
  359. raise WorkerPersistenceError(self.failure)
  360. if self.journal['pending']:
  361. raise WorkerPersistenceError('pending_receipt')
  362. return True
  363. def get_session_title(self, session_id):
  364. return self.get_session(session_id).get('title')
  365. # Transcript-tail reads the agent core makes on its session_db (failed-turn boundary, in-place
  366. # compaction coverage); a missing one raised AttributeError inside a best-effort guard.
  367. def latest_conversation_role(self, session_id):
  368. self._session(session_id)
  369. return self._apply('session.tail_role', {})['value']
  370. def get_message_role(self, session_id, row_id):
  371. self._session(session_id)
  372. return self._apply('session.row_role', {'row_id': int(row_id)})['value']
  373. def get_compression_failure_cooldown_row(self, session_id):
  374. row = self.get_session(session_id)
  375. return {'session_exists': True, 'cooldown_until': row.get('compression_failure_cooldown_until'),
  376. 'error': row.get('compression_failure_error')}
  377. def get_compression_failure_cooldown(self, session_id):
  378. import time
  379. now = time.time()
  380. row = self.get_compression_failure_cooldown_row(session_id)
  381. deadline = row['cooldown_until']
  382. if deadline is None or float(deadline) <= now:
  383. return None
  384. return {'cooldown_until': float(deadline), 'remaining_seconds': float(deadline) - now,
  385. 'error': row['error']}
  386. def _session_number(self, session_id, column, cast, zero):
  387. row = self.get_session(session_id)
  388. try:
  389. return max(zero, cast(row.get(column) or zero))
  390. except (TypeError, ValueError):
  391. return zero
  392. def get_compression_fallback_streak(self, session_id):
  393. return self._session_number(session_id, 'compression_fallback_streak', int, 0)
  394. def get_compression_ineffective_count(self, session_id):
  395. return self._session_number(session_id, 'compression_ineffective_count', int, 0)
  396. def get_compression_recovery_deadline(self, session_id):
  397. return self._session_number(session_id, 'compression_recovery_deadline', float, 0.0)
  398. def get_session_model_config_value(self, session_id, key, default=None):
  399. from hermes_state_sessions import _parse_model_config
  400. return _parse_model_config(self.get_session(session_id).get('model_config')).get(key, default)
  401. def update_conversation_model_config_value(self, session_id, key, update):
  402. # A worker owns exactly its session row, so the read-modify-write is that row's; the owner applies
  403. # the patch in its own transaction (``None`` removes the key, as on SessionDB).
  404. self._session(session_id)
  405. self.patch_session_model_config(session_id, {key: update(self.get_session_model_config_value(session_id, key))})
  406. return session_id
  407. def session_ids_with_model_config_key(self, key):
  408. # The scan a worker can answer: its own session (it may read no other row).
  409. sid = self.scope['session_id']
  410. return [sid] if self.get_session_model_config_value(sid, key) is not None else []
  411. def update_system_prompt(self, session_id, system_prompt):
  412. self._session(session_id)
  413. self._apply('session.prompt', {'system_prompt': system_prompt})
  414. def patch_session_model_config(self, session_id, patch):
  415. self._session(session_id)
  416. self._apply('session.sidecars', {'patch': patch})
  417. def update_session_tool_names(self, session_id, pin):
  418. # The tools[] freeze pin ({"version", "tools"[, "delegation_backend"]}; None clears): without it every
  419. # fresh worker re-probes check_fns and a config flip between turns silently forks the cached prefix.
  420. self._session(session_id)
  421. self._apply('session.tools', {'tool_names': pin})
  422. def finish(self):
  423. return self._apply('execution.finish', {})
  424. def close(self):
  425. with self.lock:
  426. try:
  427. self.flush_token_counts()
  428. finally:
  429. self._outbox_owner.close()