bot_live_delivery.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417
  1. """Canonical Bot Chat transport and durable legacy delivery receipts.
  2. Only gateway/session_bot.py admits execution. Old claimed/terminal records remain
  3. inspectable; the legacy UI claim consumer is retired. No authority means refusal.
  4. Receipt storage derives from FalconOrtiz's live-owner mailbox (#101564).
  5. """
  6. from __future__ import annotations
  7. import json
  8. import logging
  9. import os
  10. import re
  11. import time
  12. import uuid
  13. from contextlib import ExitStack, contextmanager
  14. from hermes_dir_acl import make_user_dir, unusable_dir_remedy
  15. from utils import atomic_json_write, fsync_directory
  16. from pathlib import Path
  17. from typing import Any, Callable
  18. from hermes_cli.active_sessions import _FileLock
  19. log = logging.getLogger(__name__)
  20. DELIVERY_DIR_NAME = "bot_live_delivery"
  21. _OWNER_KEYS = ("profile_home", "session_id", "lease_id", "live_session_id")
  22. _TERMINAL = frozenset({"settled", "failed", "cancelled", "ambiguous"})
  23. def _bot_chat_tip(home: Path) -> str | None:
  24. """Current Bot Chat tip id, or None when the profile has no store or no Bot Chat yet."""
  25. from hermes_state import SessionDB
  26. if not (home / 'state.db').is_file():
  27. return None
  28. db = SessionDB(db_path=home / 'state.db', read_only=True)
  29. try:
  30. row = db.get_session_by_title('Bot Chat')
  31. return db.get_compression_tip(row['id']) if row else None
  32. finally:
  33. db.close()
  34. def find_canonical_live_owner(profile_home: Path | str) -> dict[str, Any] | None:
  35. """Discover the profile authority and its exact Bot Chat without acquiring a lease.
  36. ``session_id`` is the current Bot Chat tip, or ``''`` when the profile has none yet: the
  37. authority's deliver door creates it on first delivery (main's ``--create-if-missing``),
  38. so a missing chat is not a refusal. None only when the profile has no store at all."""
  39. from hermes_cli.gateway_runtime import discover_gateway_endpoint
  40. home = Path(profile_home).resolve()
  41. discovery = discover_gateway_endpoint(home, timeout=5)
  42. if discovery.state != 'ready' or discovery.endpoint is None:
  43. raise ValueError('profile authority is not ready')
  44. if not (home / 'state.db').is_file():
  45. return None
  46. tip = _bot_chat_tip(home)
  47. return dict(profile_home=str(home), session_id=tip or '', canonical=True,
  48. lease_id=discovery.endpoint.instance_id, live_session_id=tip or '')
  49. class PreDialError(ValueError):
  50. """The authority call died before its RPC reached the socket: nothing was admitted, so
  51. this is a definite no-send the sender may safely retry — never the ambiguous
  52. post-admission state that orders 'do not resend' (#139303). A ``ValueError`` like the
  53. "authority is not ready" refusals it wraps, so existing ``except ValueError`` callers hold."""
  54. def authority_delivery(home, params):
  55. """Call only this home's already-running authority; never start a fallback."""
  56. return _authority_rpc(home, 'bot_relay.deliver', params)
  57. def authority_settle(home, params):
  58. """The same authority's ``bot_relay.settle``: the delivery id settled as never sent, or its
  59. existing receipt (``gateway/session_bot.py:_settle``)."""
  60. return _authority_rpc(home, 'bot_relay.settle', params)
  61. def _authority_rpc(home, method, params):
  62. import asyncio
  63. from hermes_cli.gateway_runtime import discover_gateway_endpoint
  64. from hermes_cli.gateway_client import GATEWAY_WS_PROTOCOL, GatewayClient, _session_ticket, gateway_ws_target
  65. from websockets.asyncio.client import connect
  66. home = Path(home).resolve()
  67. rpc_started = False
  68. async def request():
  69. nonlocal rpc_started
  70. discovery = await asyncio.to_thread(discover_gateway_endpoint, home, timeout=5)
  71. if discovery.state != 'ready' or discovery.endpoint is None:
  72. raise ValueError('profile authority is not ready')
  73. endpoint = discovery.endpoint
  74. ticket = await asyncio.to_thread(_session_ticket, home, endpoint)
  75. url, protocols = gateway_ws_target(endpoint, ticket)
  76. # Loopback authority dial: never through HTTP(S)_PROXY (websockets>=14 honours it by default).
  77. async with connect(url, subprotocols=protocols, open_timeout=10, proxy=None) as ws:
  78. if ws.subprotocol != GATEWAY_WS_PROTOCOL:
  79. raise ValueError('authority protocol mismatch')
  80. async with GatewayClient(ws) as client:
  81. rpc_started = True # the RPC is going out: past here a failure may have been admitted
  82. return await client.rpc(method, **params)
  83. try:
  84. try:
  85. asyncio.get_running_loop()
  86. except RuntimeError:
  87. return asyncio.run(request())
  88. # A gateway-hosted tool call runs on a thread with a live loop, where asyncio.run() raises
  89. # before anything is dialed (#139303).
  90. from model_tools import _run_async
  91. return _run_async(request())
  92. except Exception as exc:
  93. if not rpc_started:
  94. raise PreDialError(str(exc)) from exc
  95. raise
  96. def _owner(home: Path | str, owner: dict[str, Any]) -> dict[str, str]:
  97. pinned = {key: owner.get(key) for key in _OWNER_KEYS}
  98. # An empty session_id is the discovered "no Bot Chat yet" state; the door creates it.
  99. if not all(isinstance(value, str) and (value or key in ("session_id", "live_session_id"))
  100. for key, value in pinned.items()):
  101. raise ValueError("owner requires profile_home, session_id, lease_id and live_session_id")
  102. if pinned["profile_home"] != str(Path(home).resolve()):
  103. raise ValueError("owner belongs to a different profile home")
  104. return pinned
  105. def _delivery_id(value: str) -> str:
  106. if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{32,64}", value) is None:
  107. raise ValueError("delivery id must be 32 to 64 lowercase hex characters")
  108. return value
  109. def _root(home: Path | str) -> Path:
  110. return Path(home).resolve() / "runtime" / DELIVERY_DIR_NAME
  111. _warned_unhardened: set[Path] = set()
  112. # Mailbox dirs already reported unusable by this process: WARNING once per dir, DEBUG after,
  113. # because the receipt readers re-enter ``_locked`` while Bot Chat is waiting.
  114. _warned_unusable: set[Path] = set()
  115. class MailboxUnusable(PermissionError):
  116. """This token cannot list, create or lock inside the mailbox dir (#139485, #138872).
  117. On Windows an elevated install/update creates ``runtime/bot_live_delivery`` owned by
  118. Administrators with no entry for the user; the same user's non-elevated gateway then
  119. cannot even open ``.lock``. Bot Chat deliveries pause; nothing else depends on the dir.
  120. A ``PermissionError`` subclass so every existing caller keeps its handling.
  121. """
  122. def _dir_owner(path: Path) -> str:
  123. """Best-effort owner name for the log line; never raises (an ACL that hides it is the reported case)."""
  124. if os.name == "nt":
  125. try:
  126. import pywintypes
  127. import win32security
  128. except ImportError as exc:
  129. return f"unknown ({exc})"
  130. try:
  131. descriptor = win32security.GetFileSecurity(str(path), win32security.OWNER_SECURITY_INFORMATION)
  132. name, domain, _ = win32security.LookupAccountSid(None, descriptor.GetSecurityDescriptorOwner())
  133. except pywintypes.error as exc:
  134. return f"unknown ({exc})"
  135. return f"{domain}\\{name}" if domain else name
  136. import pwd # windows-footgun: ok — POSIX branch only
  137. try:
  138. uid = path.stat().st_uid
  139. except OSError as exc:
  140. return f"unknown ({exc})"
  141. try:
  142. return pwd.getpwuid(uid).pw_name
  143. except KeyError:
  144. return f"uid {uid}"
  145. def _unusable(root: Path, exc: PermissionError) -> MailboxUnusable:
  146. remedy = unusable_dir_remedy(root)
  147. level = logging.DEBUG if root in _warned_unusable else logging.WARNING
  148. _warned_unusable.add(root)
  149. log.log(level, "bot_live_delivery: mailbox directory %s (owner %s) is not usable by this process (%s); "
  150. "Bot Chat deliveries are paused until it is repaired, other clients are unaffected. To repair, %s",
  151. root, _dir_owner(root), exc, remedy)
  152. return MailboxUnusable(exc.errno, f"bot mailbox directory is not usable: {exc}", str(root))
  153. def _prepare(root: Path) -> Path:
  154. """Create/harden the mailbox dir and its ``.lock`` file; returns the lock path."""
  155. created = not root.is_dir()
  156. make_user_dir(root.parent, parents=True, exist_ok=True)
  157. make_user_dir(root, mode=0o700, exist_ok=True)
  158. try:
  159. root.chmod(0o700)
  160. except OSError as exc:
  161. # On POSIX chmod fails only when another uid owns the dir: refuse a mailbox someone else controls.
  162. if os.name != "nt" and root.stat().st_uid != os.getuid(): # windows-footgun: ok — POSIX only
  163. raise
  164. # A foreign Windows ACL can deny FILE_WRITE_ATTRIBUTES even when this process can
  165. # still use the existing mailbox. Hardening must not abort recovery. The owner's
  166. # receipt reads and writes re-enter this lock, so WARNING once per dir and DEBUG after.
  167. level = logging.DEBUG if root in _warned_unhardened else logging.WARNING
  168. _warned_unhardened.add(root)
  169. log.log(level, "bot_live_delivery: could not harden mailbox directory %s (%s)", root, exc)
  170. if created:
  171. # Only a fresh mailbox dir needs its parents durably linked; the owner's
  172. # receipt reads and writes re-enter this lock, and two directory fsyncs
  173. # per call would be disk churn for nothing.
  174. fsync_directory(root.parent)
  175. fsync_directory(root.parent.parent)
  176. lock = root / ".lock"
  177. fd = os.open(lock, os.O_CREAT | os.O_WRONLY, 0o600)
  178. os.close(fd)
  179. return lock
  180. @contextmanager
  181. def _locked(home: Path | str):
  182. """Hold the mailbox dir lock. A dir this token cannot use raises ``MailboxUnusable``
  183. (logged once per dir with its owner and the remedy); errors inside the body pass through."""
  184. root = _root(home)
  185. with ExitStack() as stack:
  186. try:
  187. stack.enter_context(_FileLock(_prepare(root)))
  188. except PermissionError as exc:
  189. raise _unusable(root, exc) from exc
  190. _warned_unusable.discard(root)
  191. yield root
  192. def _read(path: Path) -> dict[str, Any] | None:
  193. """Exact-id read: absent → None; unreadable or not a JSON object → raises (callers fail closed)."""
  194. try:
  195. record = json.loads(path.read_text(encoding="utf-8-sig"))
  196. except FileNotFoundError:
  197. return None
  198. if not isinstance(record, dict):
  199. raise ValueError(f"ticket {path.name} is not a JSON object ({type(record).__name__})")
  200. return record
  201. def _ticket_shape_error(path: Path, record: dict[str, Any]) -> str | None:
  202. """Why a parsed ticket is unusable by the scans, or None when it is well-formed.
  203. A ticket that parses as JSON but lost a field (truncated rewrite, foreign
  204. writer, hand edit) would otherwise raise KeyError/TypeError out of a
  205. directory scan and wedge delivery for the whole profile.
  206. """
  207. owner = record.get("owner")
  208. created_at, sequence = record.get("created_at"), record.get("sequence", record.get("created_at"))
  209. if record.get("delivery_id") != path.stem or record.get("id") != path.stem:
  210. return "id does not match filename"
  211. status = record.get("status")
  212. if not isinstance(status, str) or status not in ({"queued", "claimed"} | _TERMINAL):
  213. return f"unknown status {status!r}"
  214. if not all(isinstance(v, int) and not isinstance(v, bool) for v in (created_at, sequence)):
  215. return "created_at/sequence are not integers"
  216. if not isinstance(owner, dict) or not all(isinstance(owner.get(k), str) and owner[k] for k in _OWNER_KEYS):
  217. return "owner pin is incomplete"
  218. return None
  219. def _write(path: Path, record: dict[str, Any]) -> None:
  220. atomic_json_write(path, record, indent=None, sort_keys=True, fsync_dir=True, mode=0o600)
  221. def deliver_to_live_owner(
  222. profile_home: Path | str, owner: dict[str, Any], message: str,
  223. *, delivery_id: str | None = None, author: dict[str, Any] | None = None,
  224. notification_category: str = "result",
  225. ) -> dict[str, Any]:
  226. """Return durable admission immediately, without waiting for the owner.
  227. Retry with the same id AND pinned owner/message to inspect the existing
  228. state. Reusing an id with a different payload is an error, never an overwrite.
  229. """
  230. return _door(authority_delivery, profile_home, owner, message, delivery_id=delivery_id, author=author,
  231. notification_category=notification_category)
  232. def settle_held_delivery(profile_home: Path | str, owner: dict[str, Any], message: str, *, delivery_id: str,
  233. author: dict[str, Any] | None = None) -> dict[str, Any]:
  234. """Finalize a refusal marked ``admitted: false`` under the same identity: the owner's terminal
  235. not-sent receipt (``refused``), or the receipt of an attempt that admitted the id meanwhile."""
  236. return _door(authority_settle, profile_home, owner, message, delivery_id=delivery_id, author=author)
  237. def _door(call: Callable[[Path, dict], dict], profile_home, owner, message, *, delivery_id=None, author=None,
  238. notification_category="result"):
  239. pinned = _owner(profile_home, owner)
  240. if not isinstance(message, str):
  241. raise ValueError("message must be a string")
  242. if notification_category not in ('result', 'diagnostic'):
  243. raise ValueError("invalid notification category")
  244. home = Path(profile_home).resolve()
  245. category = {"notification_category": "diagnostic"} if notification_category == "diagnostic" else {}
  246. # The authority refuses a category-changing replay (admission_conflict), so the admitted
  247. # category is the producer's; echo it so callers can check the receipt's identity.
  248. return {**call(home, dict(id=_delivery_id(delivery_id if delivery_id is not None else uuid.uuid4().hex),
  249. profile=home.name if home.parent.name == "profiles" else "default",
  250. message=message, **({"session_id": pinned["session_id"]} if pinned["session_id"] else {}),
  251. **({"author": dict(author)} if author else {}), **category)), **category}
  252. def owner_holds_delivery(profile_home: Path | str, record: dict) -> bool:
  253. """Whether the profile authority still holds ``record`` (main's bounded DM wait asks this).
  254. An admission lives in the authority's durable FIFO, so it is held while the authority is
  255. reachable; an unreachable one raises (outcome unknown, never cancelled). Only an unadmitted
  256. legacy record whose profile has no store is provably ownerless.
  257. """
  258. owner = find_canonical_live_owner(profile_home) # raises while the authority is unreachable
  259. return owner is not None or bool(record.get("admission_id"))
  260. def cancel_queued_delivery(
  261. profile_home: Path | str, delivery_id: str, *, error: str, reason: str,
  262. ) -> dict[str, Any] | None:
  263. """Cancel an ownerless queued receipt without racing a consumer's claim.
  264. A claim or terminal result that won the mailbox lock is returned unchanged.
  265. """
  266. key = _delivery_id(delivery_id)
  267. with _locked(profile_home) as root:
  268. path = root / f"{key}.json"
  269. record = _read(path)
  270. # An authority admission lives in its FIFO: only the authority may settle it.
  271. if record is None or record["status"] != "queued" or record.get("admission_id"):
  272. return record
  273. record.update(status="cancelled", reply="", error=error, reason=reason,
  274. completed_at=time.time_ns())
  275. _write(path, record)
  276. return record
  277. def read_delivery_result(profile_home: Path | str, delivery_id: str) -> dict[str, Any] | None:
  278. """Read admission/claim/terminal state without waiting or deleting its receipt."""
  279. key = _delivery_id(delivery_id)
  280. if not _root(profile_home).is_dir():
  281. return None
  282. # atomic_replace can fall back to an in-place rewrite on Windows. Use the writer's
  283. # short lock so local pollers never open a partially rewritten or sharing-locked file.
  284. with _locked(profile_home) as root:
  285. record = _read(root / f'{key}.json')
  286. if record is not None and record.get('admission_id') and record.get('status') == 'canonical':
  287. # Older owners used this transitional spelling. Publication/recovery updates it;
  288. # a reader never dials the owner or replays an admission just to inspect its receipt.
  289. return {**record, 'status': 'queued'}
  290. return record
  291. _PENDING = ("queued", "claimed")
  292. _POLL_SECONDS = 0.5
  293. def _poll_receipt(profile_home: Path | str, delivery_id: str) -> dict[str, Any] | None:
  294. """A waiter's read: the receipt file the owner republishes when the admission settles (and on
  295. restart recovery). Never an authority RPC: a wait outlives owner restarts, and replaying the
  296. envelope (``read_delivery_result``) is for producers re-confirming their admission."""
  297. record = _read(_root(profile_home) / f"{_delivery_id(delivery_id)}.json")
  298. if record is not None and record.get("status") == "canonical":
  299. # Admitted with its outcome not yet published (an older owner's last write): pending.
  300. return {**record, "status": "queued"}
  301. return record
  302. def await_delivery(
  303. profile_home: Path | str, delivery_id: str, timeout: float | None,
  304. *, should_stop: Callable[[], bool] | None = None,
  305. ) -> dict[str, Any] | None:
  306. """Poll a receipt until the owner settles it, ``timeout`` lapses, or ``should_stop`` says so.
  307. Every transport that hands a turn to a live Bot Chat owner (local ``message_agent``, the
  308. Desktop relay, ``hermes peer dm`` and ``hermes peer run``) waits on the same receipt; keeping
  309. the loop here is what stops the lanes drifting (one lane returned a receipt sentence instead
  310. of the reply, two never waited at all). Returns the last record read — still pending when the
  311. budget lapsed, None when the receipt was never readable. Polls the local receipt file only, so
  312. an owner restart mid-wait is just a longer pending stretch.
  313. """
  314. deadline = None if timeout is None else time.monotonic() + timeout
  315. while True:
  316. record = _poll_receipt(profile_home, delivery_id)
  317. if record is None or record["status"] not in _PENDING:
  318. return record
  319. if should_stop is not None and should_stop():
  320. return record
  321. remaining = None if deadline is None else deadline - time.monotonic()
  322. if remaining is not None and remaining <= 0:
  323. return record
  324. time.sleep(_POLL_SECONDS if remaining is None else min(_POLL_SECONDS, remaining))
  325. async def await_delivery_async(
  326. profile_home: Path | str, delivery_id: str, timeout: float | None,
  327. *, should_stop: Callable[[], bool] | None = None,
  328. ) -> dict[str, Any] | None:
  329. """``await_delivery`` for an event loop: never blocks a worker thread for the whole budget."""
  330. import asyncio
  331. deadline = None if timeout is None else time.monotonic() + timeout
  332. while True:
  333. record = await asyncio.to_thread(_poll_receipt, profile_home, delivery_id)
  334. if record is None or record["status"] not in _PENDING:
  335. return record
  336. if should_stop is not None and should_stop():
  337. return record
  338. remaining = None if deadline is None else deadline - time.monotonic()
  339. if remaining is not None and remaining <= 0:
  340. return record
  341. await asyncio.sleep(_POLL_SECONDS if remaining is None else min(_POLL_SECONDS, remaining))