guest_handoff.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393
  1. """Guest handoff: after a Nous account replaces the free tier, move the free tier's apps into it.
  2. The account service moves a free-tier identity's connected apps (and its preview cohort) into a
  3. signed-in account with one call, ``POST /api/anonymous/claim {"token": "<anon_...>"}``, authorized by
  4. the account's own ``hermes-cli`` Bearer token. Hermes never asks first: the move just happens after
  5. the account is saved, and the result (the account email, what moved, what needs reconnecting) is
  6. shown afterwards. Two halves:
  7. - :func:`note_replacement` runs at the lowest write seams (``auth._store_provider_state`` and
  8. ``auth_nous._write_shared_nous_state``), under the caller's auth-store locks, whenever a non-guest
  9. Nous state replaces a guest. It writes a pending record and does no network. Records sit beside
  10. the shared Nous store, keyed by a hash of the ``anon_`` credential.
  11. - :func:`drain` makes one attempt per due record, outside every lock: each sign-in surface calls it
  12. after its save, and the retry worker, ``hermes auth status`` and ``hermes auth upgrade`` call it
  13. for a move a transient error left retrying.
  14. User copy never says guest / anonymous / claim.
  15. """
  16. from __future__ import annotations
  17. import hashlib
  18. import json
  19. import logging
  20. import threading
  21. import time
  22. from contextlib import contextmanager
  23. from datetime import UTC, datetime
  24. from pathlib import Path
  25. from typing import Any, Iterator, Optional
  26. logger = logging.getLogger("hermes_cli.auth")
  27. IDLE, PENDING, RETRYING, CLAIMED, GONE, FAILED = "idle", "pending", "retrying", "claimed", "gone", "failed"
  28. _ACTIVE = frozenset({PENDING, RETRYING})
  29. HANDOFF_FILENAME = "guest_handoff.json"
  30. CLAIM_TIMEOUT_SECONDS = 10.0
  31. # While an attempt is in flight its record is not due, so a second process never sends it too.
  32. _LEASE_SECONDS = CLAIM_TIMEOUT_SECONDS + 20.0
  33. # Wait before each retry of a transient failure; the attempt after the last one is final.
  34. _RETRY_DELAYS_SECONDS = (60.0, 600.0, 3600.0, 6 * 3600.0)
  35. # ``_is_expiring`` treats any token as expiring inside this skew, so the account token refreshes now.
  36. _FORCE_REFRESH_SKEW = 10**9
  37. # Outcome of one claim response.
  38. _CLAIMED, _GONE, _RETRY, _FAIL = "claimed", "gone", "retry", "fail"
  39. # A plain reason per final code; any other code shows only the generic sentence, never the code.
  40. _FAILURE_COPY = {
  41. "client_not_allowed": "This sign-in can't receive them. Sign in again with `hermes auth add nous`.",
  42. "account_locked": "This Nous account is locked.",
  43. }
  44. # The service's refusal codes; anything unlisted retries (``_classify``).
  45. _REFUSALS = {
  46. "unknown_token": _GONE, "already_retired": _GONE,
  47. "client_not_allowed": _FAIL, "account_locked": _FAIL, "invalid_request": _FAIL,
  48. }
  49. _records_lock_holder = threading.local()
  50. def _now() -> float:
  51. return time.time()
  52. def _iso(epoch: float) -> str:
  53. return datetime.fromtimestamp(epoch, tz=UTC).isoformat()
  54. def _token_key(anon_token: str) -> str:
  55. return hashlib.sha256(anon_token.encode("utf-8")).hexdigest()[:32]
  56. def _home_key() -> str:
  57. from hermes_constants import hermes_home_key
  58. return hermes_home_key()
  59. def _records_path() -> Path:
  60. from hermes_cli.auth_nous import _nous_shared_store_path
  61. return _nous_shared_store_path().with_name(HANDOFF_FILENAME)
  62. def _load(path: Path) -> dict[str, Any]:
  63. try:
  64. return json.loads(path.read_text(encoding="utf-8-sig"))
  65. except FileNotFoundError:
  66. return {"_schema": 1, "records": {}}
  67. def _records() -> dict[str, Any]:
  68. try:
  69. return _load(_records_path())["records"]
  70. except RuntimeError:
  71. return {} # the test seat belt on the shared store path
  72. def _update(path: Path, key: str, change) -> Optional[dict[str, Any]]:
  73. """Apply *change(record) -> record | None* to one record under the records lock and return what
  74. was written (None: nothing written). The innermost lock: the hook takes it under the auth-store
  75. locks, and it never holds another lock itself."""
  76. from hermes_cli.auth import _file_lock, _save_private_json
  77. with _file_lock(path.with_suffix(".lock"), _records_lock_holder, 15.0, f"Timed out waiting for {path.name}"):
  78. data = _load(path)
  79. record = change(data["records"].get(key))
  80. if record is not None:
  81. record["updated_at"] = _iso(_now())
  82. data["records"][key] = record
  83. _save_private_json(path, data, sort_keys=True)
  84. return record
  85. # --- The hook: no network, runs under the writer's locks ------------------------------------------
  86. def note_store_write(auth_store: dict[str, Any], provider_id: str, state: Any) -> None:
  87. """``auth._store_provider_state``'s hook: the Nous state the store holds now is the one replaced
  88. (the profile's own, else the root store's it falls back to)."""
  89. if provider_id == "nous":
  90. from hermes_cli.auth import _load_provider_state
  91. note_replacement(_load_provider_state(auth_store, "nous"), state)
  92. def note_replacement(previous: Any, new: Any) -> None:
  93. """Record that a non-guest Nous state replaced the guest *previous*. Idempotent per ``anon_``
  94. credential; a finished record is never reopened."""
  95. from hermes_cli.anon_auth import is_guest_state
  96. if not is_guest_state(previous) or is_guest_state(new) or not isinstance(new, dict):
  97. return
  98. anon_token = previous.get("anon_token")
  99. if not isinstance(anon_token, str) or not anon_token:
  100. return
  101. try:
  102. path = _records_path()
  103. except RuntimeError:
  104. return
  105. home = _home_key()
  106. def change(record: Optional[dict[str, Any]]) -> dict[str, Any]:
  107. if record is None:
  108. record = {"state": PENDING, "anon_token": anon_token, "attempts": 0, "profiles": [],
  109. "created_at": _iso(_now())}
  110. if home not in record["profiles"]:
  111. record["profiles"].append(home)
  112. return record
  113. record = _update(path, _token_key(anon_token), change)
  114. logger.info("Nous sign-in replaced the free tier; handoff %s (%s)", record["state"],
  115. _token_key(anon_token)[:8])
  116. # --- The runner: outside every lock ---------------------------------------------------------------
  117. def _due(record: dict[str, Any], now: float) -> bool:
  118. return record.get("state") in _ACTIVE and float(record.get("next_attempt_at") or 0) <= now
  119. def drain() -> list[dict[str, Any]]:
  120. """One attempt for each of this profile's due records; returns their public views. Call it only
  121. outside the auth-store locks: it sends a request."""
  122. home, now = _home_key(), _now()
  123. due = [key for key, record in _records().items() if home in record.get("profiles", ()) and _due(record, now)]
  124. return [view for key in due if (view := _run_one(_records_path(), key)) is not None]
  125. def _run_one(path: Path, key: str) -> Optional[dict[str, Any]]:
  126. from hermes_cli.auth import get_provider_auth_state
  127. from hermes_cli.auth_constants import AuthError
  128. try:
  129. bearer = _bearer()
  130. except (AuthError, TimeoutError) as exc: # a busy store lock defers the move like a missing token
  131. logger.info("Free-tier handoff (%s): account token unavailable (%s)", key[:8],
  132. getattr(exc, "code", None) or type(exc).__name__)
  133. return None
  134. record = _update(path, key, _reserve)
  135. if record is None:
  136. return None # another run holds it, or it settled meanwhile
  137. prior = record["attempts"] - 1
  138. outcome, payload, retry_after = _post(record["anon_token"], bearer, get_provider_auth_state("nous") or {})
  139. logger.info("Free-tier handoff attempt %d (%s): %s", prior + 1, key[:8], payload.get("error") or outcome)
  140. record = _update(path, key, lambda r: _apply(r, outcome, payload, retry_after, prior))
  141. if outcome == _CLAIMED and payload.get("cohort_carried") is True:
  142. _refresh_account_token()
  143. return public_view(record) if record else None
  144. def _reserve(record: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
  145. """Count the attempt and lease the record before the request leaves: a lost answer still counts,
  146. and no other run sends it meanwhile."""
  147. now = _now()
  148. if record is None or not _due(record, now):
  149. return None
  150. return {**record, "attempts": int(record.get("attempts") or 0) + 1, "last_attempt_at": _iso(now),
  151. "next_attempt_at": now + _LEASE_SECONDS}
  152. def _bearer(*, force: bool = False) -> str:
  153. from hermes_cli import auth
  154. # The resolver's 5 s memo can still hold the guest's token from just before the save.
  155. with auth._RESOLVE_TOKEN_CACHE_LOCK:
  156. auth._RESOLVE_TOKEN_CACHE.pop(_home_key(), None)
  157. if force:
  158. return auth.resolve_nous_access_token(timeout_seconds=CLAIM_TIMEOUT_SECONDS,
  159. refresh_skew_seconds=_FORCE_REFRESH_SKEW)
  160. return auth.resolve_nous_access_token(timeout_seconds=CLAIM_TIMEOUT_SECONDS)
  161. def _refresh_account_token() -> None:
  162. """The account's token picks up the carried cohort only on its next mint, so mint it now."""
  163. from hermes_cli.auth_constants import AuthError
  164. try:
  165. _bearer(force=True)
  166. except AuthError as exc:
  167. logger.info("Free-tier handoff: account token refresh after the move failed: %s", exc)
  168. def _post(anon_token: str, bearer: str, account: dict[str, Any]) -> tuple[str, dict[str, Any], Optional[float]]:
  169. from agent.retry_utils import parse_retry_after_seconds
  170. from hermes_cli.anon_auth import _anon_headers
  171. from hermes_cli.auth import _nous_portal_base_url, _resolve_verify
  172. from hermes_cli.auth_constants import httpx
  173. from hermes_cli.auth_nous import _nous_http_client
  174. verify = _resolve_verify(insecure=None, ca_bundle=None, auth_state=account)
  175. try:
  176. with _nous_http_client(CLAIM_TIMEOUT_SECONDS, verify) as client:
  177. response = client.post(
  178. f"{_nous_portal_base_url(account).rstrip('/')}/api/anonymous/claim",
  179. headers={**_anon_headers(), "authorization": f"Bearer {bearer}"}, json={"token": anon_token})
  180. except httpx.HTTPError as exc:
  181. return _RETRY, {"error": "unreachable", "detail": type(exc).__name__}, None
  182. try:
  183. payload = response.json()
  184. except ValueError:
  185. payload = {}
  186. payload = payload if isinstance(payload, dict) else {}
  187. return _classify(response.status_code, payload), payload, parse_retry_after_seconds(response.headers)
  188. def _classify(status: int, payload: dict[str, Any]) -> str:
  189. if status == 200:
  190. return _CLAIMED if payload.get("status") == "claimed" else _RETRY
  191. return _REFUSALS.get(str(payload.get("error") or ""), _RETRY)
  192. def _apply(record: dict[str, Any], outcome: str, payload: dict[str, Any], retry_after: Optional[float],
  193. prior_attempts: int) -> dict[str, Any]:
  194. if record["state"] not in _ACTIVE:
  195. return record # a finished move stays finished, whatever a late attempt heard
  196. record = {**record, "error": payload.get("error"), "next_attempt_at": None}
  197. if outcome == _CLAIMED:
  198. connectors = payload.get("connectors") if isinstance(payload.get("connectors"), dict) else None
  199. record.update(state=CLAIMED, error=None, result={
  200. "account_email": payload.get("account_email"),
  201. "destination_org_id": payload.get("destination_org_id"),
  202. # A service that does not report what moved (an older one) reads as unknown, never "none".
  203. "connectors_kind": (connectors or {}).get("kind") or "unknown",
  204. "moved_toolkits": list((connectors or {}).get("moved_toolkits") or []),
  205. "dropped_toolkits": list((connectors or {}).get("dropped_toolkits") or []),
  206. "cohort_carried": payload.get("cohort_carried") is True})
  207. elif outcome == _GONE:
  208. # Gone after an earlier attempt is most likely that attempt's lost success.
  209. record.update(state=GONE, probably_moved=payload.get("error") == "already_retired" or prior_attempts > 0)
  210. elif outcome == _FAIL or prior_attempts >= len(_RETRY_DELAYS_SECONDS):
  211. record.update(state=FAILED)
  212. else:
  213. record.update(state=RETRYING, next_attempt_at=_now() + (retry_after or _RETRY_DELAYS_SECONDS[prior_attempts]))
  214. if record["state"] in (CLAIMED, GONE):
  215. record.pop("anon_token", None) # retired credential: nothing left to retry with
  216. return record
  217. # --- The worker: retries a transient failure in a long-lived process ------------------------------
  218. _WORKER_FIRST_TICK_SECONDS = 15.0
  219. _WORKER_INTERVAL_SECONDS = 60.0
  220. _worker_lock = threading.Lock()
  221. _worker_stop = threading.Event()
  222. _worker_thread: Optional[threading.Thread] = None
  223. def start_worker() -> threading.Thread:
  224. """Start the process-wide retry worker (idempotent)."""
  225. global _worker_thread
  226. with _worker_lock:
  227. if _worker_thread is not None and _worker_thread.is_alive():
  228. return _worker_thread
  229. _worker_stop.clear()
  230. # health: allow HX012 -- process-wide worker for the process's own profile; it must not inherit the starter's scope
  231. _worker_thread = threading.Thread(target=_worker_loop, daemon=True, name="nous-guest-handoff")
  232. _worker_thread.start()
  233. return _worker_thread
  234. def stop_worker(timeout: float = 5.0) -> None:
  235. global _worker_thread
  236. with _worker_lock:
  237. thread = _worker_thread
  238. _worker_stop.set()
  239. if thread is not None and thread.is_alive():
  240. thread.join(timeout=timeout)
  241. with _worker_lock:
  242. if _worker_thread is thread:
  243. _worker_thread = None
  244. def _worker_loop() -> None:
  245. from tui_gateway.launch_profile_policy import launch_profile_scope_if_multiplexed
  246. wait = _WORKER_FIRST_TICK_SECONDS
  247. while not _worker_stop.wait(wait):
  248. try:
  249. with launch_profile_scope_if_multiplexed():
  250. drain()
  251. except Exception:
  252. logger.warning("Free-tier handoff retry tick failed; the records stay for the next one", exc_info=True)
  253. wait = _WORKER_INTERVAL_SECONDS
  254. # --- Read side: status for the surfaces -----------------------------------------------------------
  255. def public_view(record: dict[str, Any]) -> dict[str, Any]:
  256. """The wire shape the surfaces read; never the credential."""
  257. result = record.get("result") or {}
  258. next_at = record.get("next_attempt_at")
  259. return {
  260. "state": record.get("state", IDLE), "account_email": result.get("account_email"),
  261. "connectors_kind": result.get("connectors_kind"),
  262. "moved_toolkits": result.get("moved_toolkits", []), "dropped_toolkits": result.get("dropped_toolkits", []),
  263. "probably_moved": bool(record.get("probably_moved")), "error": record.get("error"),
  264. "retry_after": max(0, int(float(next_at) - _now())) if next_at else 0,
  265. "updated_at": record.get("updated_at")}
  266. def handoff_status() -> dict[str, Any]:
  267. """This profile's latest handoff, or ``{"state": "idle"}``. A pure read (no lock, no network)."""
  268. home = _home_key()
  269. mine = [r for r in _records().values() if home in r.get("profiles", ())]
  270. if not mine:
  271. return {"state": IDLE}
  272. return public_view(max(mine, key=lambda r: str(r.get("updated_at") or "")))
  273. def record_status(anon_token: str) -> Optional[dict[str, Any]]:
  274. """The handoff of one free-tier credential, or None when no record exists. A pure read: what a
  275. sign-in reports when its own run attempted nothing (another process held the record)."""
  276. record = _records().get(_token_key(anon_token))
  277. return public_view(record) if record else None
  278. def wire_status(status: dict[str, Any]) -> dict[str, Any]:
  279. """A status as the desktop reads it: the fields, plus the user copy for them (:func:`result_lines`)."""
  280. return {**status, "lines": result_lines(status)}
  281. def status_lines(status: dict[str, Any]) -> list[str]:
  282. """User copy for a non-idle handoff (``hermes auth status nous``)."""
  283. lines = result_lines(status)
  284. kind = status.get("connectors_kind")
  285. moved_some = kind in ("moved", "swapped") and status.get("moved_toolkits")
  286. if status.get("state") == CLAIMED and not moved_some and kind != "conflict":
  287. lines[0] = f"Signed in as {status.get('account_email') or 'your account'}. {lines[0]}"
  288. return lines
  289. def result_lines(status: dict[str, Any]) -> list[str]:
  290. """What a handoff did, for the line after "Signed in as ..." (every sign-in surface) and for
  291. :func:`status_lines`."""
  292. from tools.tool_labels import app_title
  293. state = status.get("state")
  294. where = status.get("account_email") or "your account"
  295. moved = [app_title(slug) for slug in status.get("moved_toolkits") or []]
  296. dropped = [app_title(slug) for slug in status.get("dropped_toolkits") or []]
  297. if state in _ACTIVE:
  298. return ["Moving your free-tier apps to this account; Hermes keeps trying in the background."]
  299. if state == FAILED:
  300. reason = _FAILURE_COPY.get(str(status.get("error") or ""))
  301. return ["Your free-tier apps could not be moved." + (f" {reason}" if reason else "")]
  302. if state == GONE:
  303. return ["Your free-tier apps were probably moved already." if status.get("probably_moved")
  304. else "Nothing left to move from the free tier."]
  305. if state != CLAIMED:
  306. return []
  307. kind = status.get("connectors_kind")
  308. # An empty moved list is the truth: the service refuses a move it could not read, and gives a
  309. # free tier that only read its connectors a vendor row, so kind alone says nothing.
  310. if kind in ("moved", "swapped") and moved:
  311. lines = [f"Your apps moved to {where}: {', '.join(moved)}."]
  312. elif kind == "conflict":
  313. lines = [f"{where} already had its own apps, so the free-tier apps stayed behind."]
  314. elif kind in ("none", "moved", "swapped"):
  315. lines = ["There were no free-tier apps to move."]
  316. else:
  317. lines = ["Any free-tier apps are in this account now."]
  318. if dropped:
  319. lines.append(f"Reconnect in {where}: {', '.join(dropped)}.")
  320. return lines