bot_relay.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668
  1. """Bot Mode cross-connection relay — connections ARE the peer set.
  2. Gateway-side half of the relay letting agents on ANY Desktop-connected gateway
  3. message agents on ANY other. Plain file plumbing under ``<root>/bot_relay/`` —
  4. no network; the Desktop owns every socket: ``roster.json`` (union roster of
  5. agents on OTHER connections, pushed via ``bot_relay.roster.sync``), ``outbox/``
  6. (envelopes queued by ``message_agent``, drained via ``bot_relay.outbox.drain``),
  7. ``replies/`` (one JSON per envelope via ``bot_relay.reply``; a waiter spawned at
  8. send time watches it so the reply wakes the sender like a local DM).
  9. Public helpers never raise, except ``enqueue_envelope`` → ``EnvelopeRefusedError``
  10. when the target is definitively offline (fail fast instead of queueing a DM nobody will drain).
  11. """
  12. from __future__ import annotations
  13. import contextlib
  14. import json
  15. import logging
  16. import os
  17. import re
  18. import shlex
  19. import shutil
  20. import sys
  21. import time
  22. import uuid
  23. from pathlib import Path
  24. from typing import Any, Iterator, Mapping, Optional
  25. from tools.bot_mode_probe import _default_home, _hermes_root, alias_forms
  26. from utils import atomic_json_write, unlink_files_older_than
  27. logger = logging.getLogger(__name__)
  28. RELAY_DIR_NAME = "bot_relay"
  29. ROSTER_FILE = "roster.json"
  30. OUTBOX_DIR = "outbox"
  31. CLAIMED_DIR = "claimed"
  32. REPLIES_DIR = "replies"
  33. LOCKS_DIR = "locks"
  34. # Config fallbacks (real knobs: ``bot_mode.turn_wait_seconds`` / ``bot_mode.envelope_ttl_seconds``).
  35. TURN_WAIT_SECONDS_FALLBACK = 120
  36. DEFAULT_ENVELOPE_TTL_SECONDS = 900 # older envelopes are refused at drain with 'queued_expired'
  37. # Per-attempt turn budget and attempt ceiling of one relayed delivery: the owner admits the turn and, when it
  38. # settles failed with a transient error, ONE retry under a derived identity (``gateway.session_bot._maybe_retry``).
  39. TURN_ATTEMPT_TIMEOUT_SECONDS = 600
  40. TURN_MAX_ATTEMPTS = 2 # first admission + the owner's one transient-failure retry (gateway.session_bot)
  41. # Mirrors RELAY_DELIVER_TIMEOUT_MS in apps/desktop/src/plugins/hermes-bots/relay-budget.ts; both test suites pin it.
  42. DESKTOP_DELIVER_SETTLEMENT_MARGIN_SECONDS = 180
  43. DESKTOP_DELIVER_TIMEOUT_SECONDS = (
  44. TURN_WAIT_SECONDS_FALLBACK + TURN_ATTEMPT_TIMEOUT_SECONDS * TURN_MAX_ATTEMPTS + DESKTOP_DELIVER_SETTLEMENT_MARGIN_SECONDS
  45. )
  46. # A claimed envelope is replayed on every drain until its reply lands (``_replay_unanswered``); the
  47. # budgets below only bound how long the sender-side waiter keeps watching. The longest a LIVE delivery
  48. # can be in flight without a reply on disk is the Desktop's own deliver deadline (it posts a
  49. # ``delivery_timeout`` reply when that passes), so a second delivery of a replayed envelope can start
  50. # no earlier than that plus posting headroom. tests/tools/test_bot_relay.py pins the order.
  51. REOFFER_AFTER_SECONDS = DESKTOP_DELIVER_TIMEOUT_SECONDS + 60
  52. # The waiter must outlive the Desktop's timeout reply for the first delivery AND for one replayed
  53. # delivery: replay window + a whole deliver budget + posting headroom.
  54. REPLY_WAIT_SECONDS = REOFFER_AFTER_SECONDS + DESKTOP_DELIVER_TIMEOUT_SECONDS + 60
  55. # Envelopes/replies older than this are stale artifacts (Desktop closed) and are swept.
  56. STALE_AFTER_SECONDS = 6 * 3600
  57. # Only a recent roster is authoritative for the fail-fast offline check: the
  58. # Desktop re-pushes roster.sync on connection-state changes.
  59. ROSTER_FRESH_SECONDS = 600
  60. class EnvelopeRefusedError(RuntimeError):
  61. """``enqueue_envelope`` refused to queue (nothing written); ``reason`` is a stable machine code.
  62. ``reason`` is a stable machine code; ``str(exc)`` is the human text. 'runtime_offline' matches the
  63. #93091 item-1 failure-reason enum (plain literal here so the branches merge cleanly).
  64. """
  65. def __init__(self, reason: str, message: str):
  66. super().__init__(message)
  67. self.reason = reason
  68. # Profile names, handles and connection ids share one shape (also the local
  69. # ``message_agent`` target grammar in ``tools/bot_mode_dm.py``).
  70. _HANDLE_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$")
  71. def relay_root(root: Path | str) -> Path:
  72. return Path(root) / RELAY_DIR_NAME
  73. def _ensure_dirs(root: Path | str) -> Path:
  74. base = relay_root(root)
  75. for sub in (OUTBOX_DIR, CLAIMED_DIR, REPLIES_DIR):
  76. from hermes_constants import mkdir_under_hermes_home
  77. mkdir_under_hermes_home(base / sub)
  78. return base
  79. def _atomic_write_json(target: Path, payload: Any, *, sort_keys: bool = False) -> None:
  80. atomic_json_write(target, payload, indent=None, sort_keys=sort_keys, mode=0o600)
  81. def _bot_mode_cfg(key: str, *, loader: str) -> Any:
  82. """``bot_mode.<key>`` from config, read lazily (tools/ must not import CLI
  83. config at import time); None when absent or the config is unreadable."""
  84. try:
  85. import hermes_cli.config as cfgmod
  86. cfg = getattr(cfgmod, loader)() or {}
  87. return (cfg.get("bot_mode") or {}).get(key)
  88. except Exception:
  89. logger.debug("bot_mode.%s config read failed", key, exc_info=True)
  90. return None
  91. def _normalize_roster_row(row: Any) -> Optional[dict]:
  92. """Validated, minimal roster row or None. Rows come from the Desktop over
  93. RPC — treat as untrusted input."""
  94. if not isinstance(row, dict):
  95. return None
  96. profile = str(row.get("profile") or "").strip()
  97. handle = str(row.get("handle") or "").strip().lstrip("@") or ("hermes" if profile == "default" else profile)
  98. connection_id = str(row.get("connection_id") or "").strip()
  99. if not profile or not connection_id or not all(_HANDLE_RE.match(v) for v in (handle, profile, connection_id)):
  100. return None
  101. out = {
  102. "profile": profile, "handle": handle, "connection_id": connection_id,
  103. "connection_label": str(row.get("connection_label") or "").strip()[:80],
  104. "title": str(row.get("title") or "").strip()[:120],
  105. "description": " ".join(str(row.get("description") or "").split())[:160],
  106. }
  107. # Liveness kept only when a real bool: absent == unknown == fail-open on enqueue.
  108. if isinstance(row.get("online"), bool):
  109. out["online"] = row["online"]
  110. return out
  111. def write_remote_roster(root: Path | str, rows: Any) -> int:
  112. """Atomically persist the Desktop-pushed remote roster. Returns count."""
  113. base = _ensure_dirs(root)
  114. by_key: dict[tuple[str, str], dict] = {}
  115. for norm in filter(None, map(_normalize_roster_row, rows if isinstance(rows, list) else [])):
  116. by_key.setdefault((norm["connection_id"], norm["profile"]), norm)
  117. cleaned = [by_key[k] for k in sorted(by_key)]
  118. _atomic_write_json(base / ROSTER_FILE, {"updated_at": int(time.time()), "agents": cleaned}, sort_keys=True)
  119. return len(cleaned)
  120. def read_remote_roster(root: Path | str) -> list[dict]:
  121. """The current remote roster (possibly empty). Never raises."""
  122. try:
  123. raw = (relay_root(root) / ROSTER_FILE).read_text(encoding="utf-8-sig")
  124. data = json.loads(raw)
  125. agents = data.get("agents") if isinstance(data, dict) else None
  126. return [r for r in map(_normalize_roster_row, agents) if r] if isinstance(agents, list) else []
  127. except FileNotFoundError:
  128. return []
  129. except Exception:
  130. logger.debug("bot_relay roster read failed", exc_info=True)
  131. return []
  132. def _target_ids(row: dict) -> set[str]:
  133. """Lower-cased routing ids of a roster row: its @handle and profile folder id."""
  134. return {row["handle"].lower(), row["profile"].lower()}
  135. def _target_aliases(row: dict) -> set[str]:
  136. """Every lower-cased bare form that addresses ``row``: routing ids plus the Bot Mode title's
  137. mention slugs (``"CoS Bot"`` → ``cos-bot``/``cosbot``, what the Desktop picker inserts). A remote
  138. ``default`` is ``@hermes`` on every gateway, so its title is the only bare form that can single it out."""
  139. return _target_ids(row) | alias_forms(row.get("title") or "")
  140. def resolve_remote_target(raw_target: str, roster: list[dict]) -> Any:
  141. """Matched row for a bare handle/profile/title slug (unique across connections) or
  142. ``<handle|profile|title-slug>@<connection-id>``; ``"ambiguous"`` for a bare form on several
  143. connections; None otherwise. An exact handle/profile match beats a title slug, so a title
  144. colliding with another row's handle never steals it."""
  145. want, at, conn = (p.strip() for p in str(raw_target or "").strip().lstrip("@").partition("@"))
  146. if not want or (at and not conn):
  147. return None
  148. want = want.lower()
  149. rows = [row for row in roster if not conn or row["connection_id"].lower() == conn.lower()]
  150. matches = [row for row in rows if want in _target_ids(row)] or [row for row in rows if want in _target_aliases(row)]
  151. if not matches:
  152. return None
  153. return matches[0] if len(matches) == 1 else "ambiguous"
  154. def _title_slug(row: dict) -> str:
  155. """The Bot Mode title's slug form (``"CoS Bot"`` → ``cos-bot``, what the picker inserts); "" when the
  156. title is empty, reserved (a bot titled "Hermes") or not a valid handle."""
  157. title = str(row.get("title") or "")
  158. slug = re.sub(r"[^a-z0-9_-]+", "-", title.strip().lower()).strip("-")
  159. return slug if slug in alias_forms(title) else ""
  160. def remote_target_forms(roster: list[dict], local_taken: set[str] | frozenset[str] = frozenset()) -> list[str]:
  161. """One unambiguous target string per row, shortest first: the bare handle when no other remote
  162. row and no LOCAL profile (``local_taken``: this gateway's handles and friendly-name slugs) answers
  163. to it; else the title slug under the same test (a remote ``default`` titled "CoS Bot" is
  164. ``@cos-bot``, since bare ``@hermes`` is always this gateway's own default); else
  165. ``handle@connection``. Mirrors ``resolve_remote_target``."""
  166. taken = {form.lower() for form in local_taken}
  167. id_claims: dict[str, int] = {}
  168. alias_claims: dict[str, int] = {}
  169. for row in roster:
  170. for form in _target_ids(row):
  171. id_claims[form] = id_claims.get(form, 0) + 1
  172. for form in _target_aliases(row):
  173. alias_claims[form] = alias_claims.get(form, 0) + 1
  174. def _form(row: dict) -> str:
  175. # The handle needs only be unique among routing ids (resolution gives it precedence over a
  176. # colliding title); a title slug must be unique among every alias.
  177. for candidate, claims in ((row["handle"], id_claims), (_title_slug(row), alias_claims)):
  178. if candidate and candidate.lower() not in taken and claims.get(candidate.lower(), 0) == 1:
  179. return candidate
  180. return f"{row['handle']}@{row['connection_id']}"
  181. return [_form(row) for row in roster]
  182. _SENDER_STAMP_RE = re.compile(r"^(Message from 🤖 .+? \(@)([A-Za-z0-9_-]+)(\): )", re.DOTALL)
  183. def qualify_sender_stamp(message: str, from_handle: Any, from_connection: Any, roster: list[dict],
  184. local_taken: set[str] | frozenset[str] = frozenset()) -> str:
  185. """Rewrite a relayed DM's ``Message from 🤖 <name> (@<handle>):`` stamp so the handle is the
  186. form THIS gateway can reply to: the sender's row in the local relay roster as
  187. ``remote_target_forms`` renders it, else ``handle@connection``. A relayed ``@hermes`` is another
  188. machine's default — left bare, a reply lands on the recipient's own default (#103731)."""
  189. handle, conn = str(from_handle or "").strip().lstrip("@"), str(from_connection or "").strip()
  190. match = _SENDER_STAMP_RE.match(str(message or ""))
  191. if not match or not conn or not _HANDLE_RE.match(handle) or not _HANDLE_RE.match(conn):
  192. return message
  193. forms = dict(zip(((r["connection_id"].lower(), r["handle"].lower()) for r in roster), remote_target_forms(roster, local_taken)))
  194. form = forms.get((conn.lower(), handle.lower())) or f"{handle}@{conn}"
  195. return f"{match.group(1)}{form}{match.group(3)}{message[match.end():]}"
  196. def _envelope_ttl_seconds() -> int:
  197. """Configured drain TTL (``bot_mode.envelope_ttl_seconds``), read per-drain.
  198. ``0`` (or negative) disables expiry."""
  199. val = _bot_mode_cfg("envelope_ttl_seconds", loader="load_config_readonly")
  200. if val is None:
  201. return DEFAULT_ENVELOPE_TTL_SECONDS
  202. try:
  203. return int(val)
  204. except (TypeError, ValueError, OverflowError):
  205. logger.debug("Invalid bot_mode.envelope_ttl_seconds %r; using fallback", val)
  206. return DEFAULT_ENVELOPE_TTL_SECONDS
  207. def _target_liveness(root: Path | str, target: dict) -> Optional[bool]:
  208. """Tri-state liveness: True / False / None (unknown → callers fail open). Offline =
  209. explicit ``online: false`` or ABSENT from a *fresh* roster; a missing, unreadable,
  210. empty or stale roster proves nothing → None. Never raises."""
  211. try:
  212. try:
  213. age = time.time() - (relay_root(root) / ROSTER_FILE).stat().st_mtime
  214. except OSError:
  215. return None
  216. roster = read_remote_roster(root) if age <= ROSTER_FRESH_SECONDS else []
  217. if not roster:
  218. return None
  219. key = (str(target.get("connection_id") or ""), str(target.get("profile") or ""))
  220. row = next((r for r in roster if (r["connection_id"], r["profile"]) == key), None)
  221. if row is None:
  222. return False # fresh roster no longer lists the target — offline
  223. return row["online"] if isinstance(row.get("online"), bool) else None
  224. except Exception:
  225. logger.debug("bot_relay liveness check failed", exc_info=True)
  226. return None
  227. def enqueue_envelope(root: Path | str, *, target: dict, message: str, sender_profile: str, sender_handle: str) -> dict:
  228. """Queue a cross-connection DM for the Desktop relay; returns the envelope. Raises
  229. ``EnvelopeRefusedError`` ('runtime_offline') without writing when the target is
  230. definitively offline; unknown liveness enqueues (fail-open)."""
  231. if _target_liveness(root, target) is False:
  232. label = (f"@{target.get('handle') or target.get('profile') or '?'} on "
  233. f"{target.get('connection_label') or target.get('connection_id') or '?'}")
  234. raise EnvelopeRefusedError("runtime_offline", f"{label} is offline right now — the message was NOT queued. "
  235. "Try again once that machine reconnects to the Desktop.")
  236. base = _ensure_dirs(root)
  237. envelope = {
  238. "id": uuid.uuid4().hex, "created_at": int(time.time()), "canonical_delivery_v1": True,
  239. "from_profile": sender_profile, "from_handle": sender_handle,
  240. "target_connection": target["connection_id"], "target_profile": target["profile"],
  241. "target_handle": target["handle"], "message": message,
  242. }
  243. _atomic_write_json(base / OUTBOX_DIR / f"{envelope['id']}.json", envelope)
  244. return envelope
  245. def _expire_if_stale(root: Path | str, path: Path, ttl: float, now: float) -> bool:
  246. """True when the outbox envelope is older than ``ttl``; writes the 'queued_expired'
  247. reply so the sender's waiter resolves (best effort). Unreadable envelopes are left for the claim."""
  248. try:
  249. env = json.loads(path.read_text(encoding="utf-8-sig")) # BOM-tolerant (pm-era read fix)
  250. if not isinstance(env, dict):
  251. raise ValueError(f"expected a JSON object, got {type(env).__name__}")
  252. created = float(env.get("created_at") or path.stat().st_mtime)
  253. except (OSError, ValueError):
  254. return False
  255. if now - created <= ttl:
  256. return False
  257. with contextlib.suppress(OSError, ValueError):
  258. write_reply(root, str(env.get("id") or ""), reason="queued_expired", error=(
  259. f"queued message to @{env.get('target_handle') or '?'} on {env.get('target_connection') or '?'} "
  260. f"expired after {ttl}s waiting for the Desktop to drain it — it was NOT delivered. "
  261. "Resend once the Desktop reconnects."))
  262. return True
  263. def _queued_at(path: Path) -> tuple[float, str]:
  264. """Claim order for one outbox entry: oldest first. ``mtime`` is what ``_sweep_stale`` already
  265. treats as an envelope's age, and unlike the whole-second ``created_at`` field it separates two
  266. DMs sent in the same second. The name only breaks ties."""
  267. with contextlib.suppress(OSError):
  268. return (path.stat().st_mtime, path.name)
  269. return (0.0, path.name)
  270. def claim_pending_envelopes(root: Path | str) -> list[dict]:
  271. """Drain the outbox (rename → claimed/ so a second drain can't double-deliver).
  272. TTL-expired envelopes get a 'queued_expired' reply and are removed instead.
  273. Envelopes older than ``bot_mode.envelope_ttl_seconds`` are NOT delivered: each gets an error reply
  274. (reason ``'queued_expired'``) so the sender's waiter resolves, and its outbox file is removed (#93091
  275. item 2).
  276. """
  277. base = _ensure_dirs(root)
  278. _sweep_stale(base)
  279. ttl = _envelope_ttl_seconds()
  280. now = time.time()
  281. out: list[tuple[tuple[float, str], dict]] = []
  282. # Oldest first: the Desktop delivers each target's claimed envelopes in the order this list
  283. # gives them, so a sender's two DMs to one agent arrive in the order they were sent. Sorting
  284. # by filename ordered them by ``uuid4().hex`` — at random.
  285. for path in sorted((base / OUTBOX_DIR).glob("*.json"), key=_queued_at):
  286. if ttl > 0 and _expire_if_stale(root, path, ttl, now):
  287. with contextlib.suppress(OSError):
  288. path.unlink()
  289. continue
  290. claimed = base / CLAIMED_DIR / path.name
  291. order = _queued_at(path)
  292. with contextlib.suppress(OSError, ValueError):
  293. os.replace(path, claimed) # atomic claim; the rename keeps the enqueue mtime
  294. envelope = json.loads(claimed.read_text(encoding="utf-8-sig"))
  295. if not isinstance(envelope, dict):
  296. raise ValueError(f"expected a JSON object, got {type(envelope).__name__}")
  297. out.append((order, envelope))
  298. seen = {row["id"] for _order, row in out}
  299. # One enqueue-ordered list: a claimed-but-unanswered DM (Desktop reconnected before its reply)
  300. # is OLDER than mail queued since, so appending replays after fresh rows reversed them.
  301. out.extend(_replay_unanswered(root, base, seen, now))
  302. return [envelope for _order, envelope in sorted(out, key=lambda item: item[0])]
  303. def _replay_unanswered(root: Path | str, base: Path, seen: set, now: float) -> list[tuple[tuple[float, str], dict]]:
  304. """``claimed/`` canonical envelopes whose reply has not landed yet, replayed on EVERY drain.
  305. A drained envelope is not "delivered": its stable id is handed out again until the target's
  306. terminal reply is written under that id, so a Desktop that disconnects between ``outbox.drain``
  307. and ``bot_relay.deliver`` (#111021, #111207) or loses the drain result cannot drop the DM. The
  308. replay is safe because ``bot_relay.deliver`` forwards the SAME id to the target profile's
  309. authority, which admits one turn per id and answers a retry with the existing record — never a
  310. second turn. Once ``created_at + REPLY_WAIT_SECONDS`` passes with no reply the waiter is gone
  311. (or about to be): a ``delivery_timeout`` reply is written so it learns, and the envelope is
  312. never handed out again. Each envelope comes back with its ``_queued_at`` enqueue key so the
  313. drain can merge it with fresh outbox rows in send order.
  314. """
  315. out: list[tuple[tuple[float, str], dict]] = []
  316. for path in sorted((base / CLAIMED_DIR).glob("*.json"), key=_queued_at):
  317. if (base / REPLIES_DIR / path.name).exists():
  318. continue
  319. with contextlib.suppress(OSError, ValueError):
  320. envelope = json.loads(path.read_text(encoding="utf-8-sig"))
  321. if not isinstance(envelope, dict):
  322. raise ValueError(f"expected a JSON object, got {type(envelope).__name__}")
  323. if envelope.get("id") in seen:
  324. continue
  325. env_id = str(envelope.get("id") or "")
  326. label = f"@{envelope.get('target_handle') or '?'} on {envelope.get('target_connection') or '?'}"
  327. if envelope.get("canonical_delivery_v1") is not True:
  328. _settle_pre_upgrade_claim(root, path, env_id, label, now)
  329. continue
  330. created = float(envelope.get("created_at") or path.stat().st_mtime)
  331. if now - created > REPLY_WAIT_SECONDS:
  332. write_reply(root, env_id, reason="delivery_timeout", error=(
  333. f"no reply from {label} within {REPLY_WAIT_SECONDS}s of sending — the Desktop picked "
  334. "the message up but never reported a delivery. It will not be retried; resend if it matters."))
  335. continue
  336. out.append((_queued_at(path), envelope))
  337. return out
  338. def _settle_pre_upgrade_claim(root: Path | str, path: Path, env_id: str, label: str, now: float) -> None:
  339. """A ``claimed/`` envelope from before canonical delivery (no ``canonical_delivery_v1``) is never
  340. replayed: its old lane ran a CLI turn the authority has no record of, so re-admitting the id could
  341. run the DM twice. Left alone, though, its sender's waiter watched a reply that never came. Once the
  342. old drain's own re-offer point passes (``REOFFER_AFTER_SECONDS`` after its claim — the claim mtime
  343. — by which time a live delivery has posted the Desktop's timeout reply) the outcome is settled as
  344. ``unknown``: the waiter resolves, the envelope is never offered again, and nothing auto-resends."""
  345. from tools.bot_failure_reasons import UNKNOWN
  346. if now - path.stat().st_mtime < REOFFER_AFTER_SECONDS:
  347. return
  348. write_reply(root, env_id, reason=UNKNOWN, error=(
  349. f"the message to {label} was picked up before this gateway was upgraded and its delivery was "
  350. "never reported — it may or may not have been delivered. It will not be retried; check with "
  351. "the recipient before resending."))
  352. def write_reply(root: Path | str, envelope_id: str, *, reply: str = "", error: str = "", reason: str = "") -> Path:
  353. """Persist the relayed reply (or delivery error) for the waiter. ``reason`` (typed
  354. code, ``tools.bot_failure_reasons``) is classified from ``error`` when omitted.
  355. Idempotent by envelope id — the first settled reply stands, error or not. That is safe because
  356. a replayed envelope (``_replay_unanswered``) carries the SAME id into the target authority,
  357. which admits one turn per id, so a later write is a duplicate or a bookkeeping timeout, never
  358. a truer answer."""
  359. base = _ensure_dirs(root)
  360. safe = str(envelope_id or "").strip()
  361. if not re.match(r"^[0-9a-f]{32}$", safe):
  362. raise ValueError(f"invalid envelope id: {envelope_id!r}")
  363. path = base / REPLIES_DIR / f"{safe}.json"
  364. if path.exists():
  365. # Idempotent by envelope id: the first settled reply is the one the waiter already read (or
  366. # will). A re-offered delivery's second outcome — or a late duplicate — never displaces it.
  367. return path
  368. err, code = str(error or ""), str(reason or "")
  369. if not code and err:
  370. from tools.bot_failure_reasons import classify_agent_error
  371. code = classify_agent_error(err)
  372. _atomic_write_json(path, {"id": safe, "at": int(time.time()), "reply": str(reply or ""), "error": err, "reason": code})
  373. return path
  374. def _sweep_stale(base: Path, *, now: float | None = None) -> int:
  375. return sum(unlink_files_older_than(base / sub, "*.json", STALE_AFTER_SECONDS, now=now)
  376. for sub in (CLAIMED_DIR, REPLIES_DIR, OUTBOX_DIR))
  377. def cleanup_bot_relay_artifacts(max_age_hours: float | None = None) -> int:
  378. """Hourly sweep of stale relay artifacts (DM plaintext; ``_sweep_stale`` otherwise runs
  379. only on Desktop drains). ``max_age_hours`` is for ``cleanup_*_cache`` signature parity only."""
  380. del max_age_hours
  381. try:
  382. base = relay_root(_hermes_root(Path(_default_home())))
  383. return _sweep_stale(base) if base.is_dir() else 0
  384. except Exception:
  385. logger.debug("bot_relay artifact sweep failed", exc_info=True)
  386. return 0
  387. def waiter_command(root: Path | str, envelope: dict) -> str:
  388. """Shell command that blocks until the reply file appears, then prints it; spawned
  389. via ``terminal_tool(background=True, notify_on_complete=True)`` so its stdout arrives
  390. as the same completion notification local DMs use.
  391. A ``tools/bot_mode_dm.py --wait-reply`` entrypoint, like the local delivery runner — not
  392. ``python -c``. The approval gate flags inline interpreter code ("script execution via -e/-c
  393. flag"), and ``approvals.single_query_mode`` defaults to ``deny`` for the one-shot ``-Q`` turn a
  394. bot replies from, so the reply waiter was refused exactly when a bot answered a teammate: the
  395. message was delivered, the reply never woke the sender. Roster fields ride as argv (``shlex``
  396. quoted), never as source text, so a hostile handle or connection id stays data.
  397. """
  398. reply_path = str(relay_root(root) / REPLIES_DIR / f"{envelope['id']}.json")
  399. label = f"@{envelope.get('target_handle', '')} on {envelope.get('target_connection', '')}"
  400. runner = str(Path(__file__).resolve().with_name("bot_mode_dm.py"))
  401. argv = [sys.executable or "python3", runner, "--wait-reply", reply_path, label, str(REPLY_WAIT_SECONDS)]
  402. if sys.platform == "win32":
  403. # Same rewrite as the delivery runner: the tracked local backend uses Git Bash on native
  404. # Windows, where forward-slash drive paths run and backslash paths parse as command names.
  405. argv = [part.replace("\\", "/") for part in argv]
  406. return shlex.join(argv)
  407. def _hermes_cli() -> str:
  408. """Prefer this install's published launcher, then interpreter/PATH fallbacks.
  409. A long-lived caller can still run in an older dependency generation. Its
  410. sibling console script pins that generation, whereas the published launcher
  411. selects current dependencies at child start. Keep the historical fallbacks
  412. for external/developer installs that have no published launcher (#93590).
  413. """
  414. # Do not select batch shims: cmd.exe reinterprets otherwise literal argv
  415. # (for example an ampersand in a query-file path), even with shell=False.
  416. name = "hermes.exe" if sys.platform == "win32" else "hermes"
  417. published = Path(__file__).resolve().parents[1] / ".hermes" / "bin" / name
  418. if published.is_file():
  419. return str(published)
  420. sibling = Path(sys.executable or "").parent / name
  421. return str(sibling) if sibling.is_file() else shutil.which("hermes") or "hermes"
  422. class DeliveryAuthor:
  423. """A relayed turn's author as an in-process object. ``bot_relay.deliver`` builds it from the sender fields
  424. an admitted gateway client relays for another connection; nothing verifies the sender itself. A JSON
  425. client cannot build one, so ``prompt.submit`` accepts the object and refuses a dict."""
  426. __slots__ = ("author",)
  427. def __init__(self, author: dict) -> None:
  428. self.author = dict(author)
  429. def __eq__(self, other: object) -> bool:
  430. return isinstance(other, DeliveryAuthor) and other.author == self.author
  431. def __repr__(self) -> str:
  432. return f"DeliveryAuthor({self.author!r})"
  433. def delivery_turn_author(from_profile: Any, from_handle: Any, from_connection: Any = None) -> Optional[dict]:
  434. """The author of a relayed DM's recipient turn, built from the sender fields as the relaying client reports
  435. them. A relayed DM always comes from another gateway, so the id carries the Desktop's id for the sender's
  436. connection (``local`` included) and only the recipient's own profiles are bare ``bot:<profile>``. None when
  437. the envelope names no sender."""
  438. from agent.turn_author import bot_author_id
  439. profile = str(from_profile or "").strip()
  440. if not profile:
  441. return None
  442. return {"id": bot_author_id(profile, str(from_connection or "")), "name": str(from_handle or "").strip() or profile,
  443. "is_bot": True}
  444. def _delivery_child_session_env_names() -> tuple[str, ...]:
  445. """Session-bound env names to strip from a delivery child, from ``gateway.session_context``.
  446. Synced with the session binding surface as vars are added; deliberately NOT a
  447. ``HERMES_SESSION_*`` prefix match, which would also strip non-identity knobs
  448. (e.g. ``HERMES_SESSION_STALL_TIMEOUT``)."""
  449. from gateway.session_context import _VAR_MAP
  450. return tuple(_VAR_MAP)
  451. def relaying_principal_author(principal: str) -> dict:
  452. """The author of a relayed DM whose sender fields cannot be trusted: a logged-in client named them.
  453. Server-derived and unspoofable — the id is built from the caller's minted identity digest, never from
  454. anything the client sent — and still a BOT author, because the recipient's memory routes on that:
  455. Honcho writes a bot-authored turn into the bot's own a2a session and refuses conclusion / profile /
  456. mirror writes for it, while an unattributed turn is treated as the human's (#107598 review). The
  457. human-facing signature stays in the message text the sender composed."""
  458. from agent.turn_author import bot_author_id
  459. return {"id": bot_author_id("relay", str(principal or "").strip()), "name": "relayed teammate", "is_bot": True}
  460. def delivery_env(author: Optional[dict], profile_home: str | Path | None = None) -> dict[str, str]:
  461. """Environment for one delivery turn's ``hermes -p <profile>`` child. The dispatcher's own
  462. HERMES_TURN_AUTHOR is dropped first so a delivery without an author never inherits the author of the turn
  463. that sent it. Dispatcher session identity (the canonical ``gateway.session_context`` session env names) is
  464. dropped too: a nested recipient that ``message_agent``s onward must not stamp that grandchild
  465. notify with the grandparent's key, or the live recipient never resumes. The child runs the target
  466. profile's Bot Chat turn, so it starts from THAT profile's env (``served_profile_child_env``: launch
  467. profile ``.env`` / TERMINAL_* residue dropped, target secrets overlaid), never the multiplexer's raw
  468. ``os.environ``; ``-p`` alone only pinned HERMES_HOME. ``profile_home`` is the target's home when the
  469. caller knows it (relay RPC, roster); otherwise the active override, and under multiplex the launch
  470. home."""
  471. from agent.secret_scope import current_secret_scope, is_multiplex_active
  472. from agent.turn_author import TURN_AUTHOR_ENV, turn_author_env
  473. from hermes_constants import get_hermes_home_override, get_routing_process_hermes_home
  474. from tools.environments.local import served_profile_child_env
  475. # ``_profile_home`` answers None for the launch profile by design and a relay RPC binds no scope,
  476. # so under multiplex an empty target means the launch profile, not "unknown" (the fail-closed
  477. # raise; cf. the slash-worker spawn, #115427). An explicit target beats an override or bound scope
  478. # inside ``served_profile_child_env``, so fill it only when both are absent; a single-profile host
  479. # keeps its pass-through env.
  480. target_home = profile_home
  481. if (not target_home and is_multiplex_active() and not get_hermes_home_override()
  482. and current_secret_scope() is None):
  483. target_home = get_routing_process_hermes_home()
  484. env = served_profile_child_env(base=os.environ, target_home=target_home, inherit_credentials=True)
  485. env.pop(TURN_AUTHOR_ENV, None)
  486. for name in _delivery_child_session_env_names():
  487. env.pop(name, None)
  488. if author:
  489. env.update(turn_author_env(author))
  490. return env
  491. # Two deliveries into the SAME profile must never run Bot Chat turns concurrently.
  492. # Deliveries are separate ``hermes`` subprocesses, so the lock is a per-profile
  493. # lockfile under ``<root>/bot_relay/locks/`` held with ``fcntl.flock`` for exactly
  494. # the turn window; the kernel releases it on fd close (incl. process death), so a
  495. # crashed turn can never wedge the profile.
  496. # ── per-profile turn lock (#93091) ─────────────────────────────────────────── Two deliveries into the SAME
  497. # target profile must never run their Bot Chat turns concurrently: deliveries spawn separate ``hermes``
  498. # subprocesses, so an in-memory mutex is useless — the lock is a per-profile lockfile under
  499. # ``<root>/bot_relay/locks/`` held with ``fcntl.flock`` for exactly the turn execution window. flock is
  500. # released by the kernel when the holder's fd closes (including process death), so a crashed turn can never
  501. # wedge the profile. A queued delivery waits up to ``bot_mode.turn_wait_seconds`` and then fails with a
  502. # structured 'target_busy' refusal instead of blocking forever.
  503. class TurnBusyError(RuntimeError):
  504. """A delivery turn is already running for the target profile (``waited_seconds`` ≈ time queued).
  505. ``reason`` is 'target_busy' — extends the #93091 item-1 structured refusal enum. ``waited_seconds`` is
  506. roughly how long the caller queued behind the current turn before giving up.
  507. """
  508. reason = "target_busy"
  509. def __init__(self, profile: str, waited_seconds: float):
  510. self.profile, self.waited_seconds = profile, waited_seconds
  511. super().__init__(f"target_busy: another delivery turn is already running for profile '{profile}' — "
  512. f"queued behind it for ~{round(waited_seconds)}s without it finishing. "
  513. "The message was NOT delivered; retry shortly.")
  514. def turn_wait_seconds() -> float:
  515. """Wait budget for a queued delivery turn (config, lazily read)."""
  516. val = _bot_mode_cfg("turn_wait_seconds", loader="load_config")
  517. return float(TURN_WAIT_SECONDS_FALLBACK) if val is None else max(0.0, float(val))
  518. def turn_lock_path(root: Path | str, profile: str) -> Path:
  519. """Per-profile lockfile path (short — safe on macOS temp roots)."""
  520. safe = re.sub(r"[^a-zA-Z0-9_-]", "_", str(profile or ""))[:64] or "_"
  521. return relay_root(root) / LOCKS_DIR / f"{safe}.lock"
  522. @contextlib.contextmanager
  523. def acquire_turn_lock(root: Path | str, profile: str, timeout_seconds: float | None = None) -> Iterator[Path]:
  524. """Hold ``profile``'s cross-process turn lock for the ``with`` body: non-blocking
  525. flock probe + short-sleep retry up to the budget (``bot_mode.turn_wait_seconds``
  526. unless ``timeout_seconds``); raises :class:`TurnBusyError` when exhausted. No
  527. ordering among waiters, but every waiter is bounded. Without ``fcntl`` (Windows)
  528. the lock is a no-op — those installs never had this race path."""
  529. try:
  530. import fcntl
  531. except ImportError: # pragma: no cover — Windows
  532. logger.debug("bot turn lock disabled: fcntl unavailable on this platform")
  533. yield turn_lock_path(root, profile)
  534. return
  535. budget = turn_wait_seconds() if timeout_seconds is None else max(0.0, float(timeout_seconds))
  536. path = turn_lock_path(root, profile)
  537. path.parent.mkdir(parents=True, exist_ok=True)
  538. fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600)
  539. try:
  540. start = time.monotonic()
  541. deadline = start + budget
  542. while True:
  543. try:
  544. fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  545. break
  546. except OSError:
  547. now = time.monotonic()
  548. if now >= deadline:
  549. raise TurnBusyError(profile, now - start)
  550. time.sleep(min(0.1, max(0.005, deadline - now)))
  551. try:
  552. yield path
  553. finally:
  554. with contextlib.suppress(OSError): # kernel releases on close anyway
  555. fcntl.flock(fd, fcntl.LOCK_UN)
  556. finally:
  557. os.close(fd)