shared_metrics_setup.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. """hermes.provider_setup.count: where connecting a model provider starts, succeeds, fails or is walked away from.
  2. A flow is ``started`` once the user has picked a provider. Its end is recorded by the surface that
  3. ran it (``completed`` / ``failed`` + a closed failure class). A flow nobody finished cannot record
  4. itself, so ``started`` also drops a small marker under the owning profile's store dir (the
  5. process-exit pattern): the finisher claims it by rename, and a marker whose process is gone (or that
  6. has been pending longer than any real flow takes) is reported ``abandoned`` by the next setup start
  7. or Hermes start in that profile. Whoever claims the marker records, so each flow ends exactly once.
  8. Only the catalog provider name leaves (custom endpoints read ``custom``); never a key, token, base
  9. URL or error text. Rows are saved synchronously: a setup killed right after ``started`` must still
  10. have left its row, and none of this runs on a hot path.
  11. """
  12. from __future__ import annotations
  13. import contextlib
  14. import json
  15. import logging
  16. import os
  17. import threading
  18. import time
  19. from dataclasses import dataclass
  20. from pathlib import Path
  21. from typing import Any, Iterator
  22. logger = logging.getLogger(__name__)
  23. MARKER_DIRNAME = "provider_setup_markers"
  24. # Device codes expire in ~15 minutes; a flow still pending an hour later was left open and forgotten.
  25. STALE_AFTER_S = 3600
  26. # Picker rows that are not a provider (menus, removal) never start a setup flow.
  27. _NOT_A_PROVIDER = frozenset({"aux-config", "cancel", "reasoning", "remove-custom"})
  28. _AUTH_ERROR_TYPES = frozenset({"AuthError", "SignInCopyError"})
  29. _OAUTH_SESSION_KEY = "_metrics_setup_flow"
  30. _OAUTH_FAILURE_KEY = "_metrics_setup_failure"
  31. # Ends that are not failures: the user stopped it, or let the sign-in code run out.
  32. _NOT_FAILURES = frozenset({"cancelled", "expired"})
  33. # Tokens other tools read too (gh, the Hugging Face Hub): saving one is not connecting that provider.
  34. _SHARED_TOKENS = frozenset({"GH_TOKEN", "GITHUB_TOKEN", "HF_TOKEN"})
  35. # OAuth session status (+ reason) at its end -> (event, failure_class).
  36. _OAUTH_ENDINGS = {
  37. "approved": ("completed", "none"),
  38. "cancelled": ("abandoned", "none"),
  39. "denied": ("failed", "auth"),
  40. "expired": ("abandoned", "none"),
  41. }
  42. _OAUTH_REASONS = {
  43. "user_declined": ("abandoned", "none"),
  44. "superseded": ("abandoned", "none"),
  45. "timeout": ("abandoned", "none"),
  46. "account_retired": ("failed", "auth"),
  47. "anon_unreachable": ("failed", "network"),
  48. }
  49. @dataclass
  50. class SetupFlow:
  51. home: str
  52. surface: str
  53. provider: str # already the catalog metric name
  54. marker: Path
  55. def markers_dir(home: Path) -> Path:
  56. return home / "telemetry" / "shared_metrics" / MARKER_DIRNAME
  57. @contextlib.contextmanager
  58. def _bound(hermes_home: Any) -> Iterator[None]:
  59. token = None
  60. if hermes_home:
  61. from hermes_constants import set_hermes_home_override
  62. token = set_hermes_home_override(str(hermes_home))
  63. try:
  64. yield
  65. finally:
  66. if token is not None:
  67. from hermes_constants import reset_hermes_home_override
  68. reset_hermes_home_override(token)
  69. def collection_enabled(hermes_home: Any = None) -> bool:
  70. """The owning profile's shared-metrics gate (cheap; lets async callers skip the thread hop)."""
  71. try:
  72. with _bound(hermes_home):
  73. from .relay_shared_metrics import enabled
  74. return bool(enabled())
  75. except Exception:
  76. return False
  77. def provider_setup_fields(*, surface: Any, provider: Any, event: Any, failure_class: Any = "none") -> dict[str, str] | None:
  78. from . import shared_metrics_contract as contract
  79. from .shared_metrics_catalog import provider_metric_name
  80. if surface not in contract.PROVIDER_SETUP_SURFACES or event not in contract.PROVIDER_SETUP_EVENTS:
  81. return None
  82. failure = "none"
  83. if event == "failed" and failure_class in _NOT_FAILURES: # a cancel or a lapsed code is a walk-away
  84. event = "abandoned"
  85. elif event == "failed":
  86. failure = failure_class if failure_class in contract.PROVIDER_SETUP_FAILURE_CLASSES - {"none"} else "other"
  87. return {"event": event, "failure_class": failure, "provider": provider_metric_name(provider), "surface": surface}
  88. def setup_failure_class(exc: BaseException) -> str:
  89. """Closed class for an exception that ended a setup flow; inert (type checks only). ``cancelled``
  90. and ``expired`` (the sign-in code ran out before approval) are recorded ``abandoned``."""
  91. from hermes_cli.auth_error_copy import is_cancelled, is_device_code_expired, is_network_error
  92. names = {cls.__name__ for cls in type(exc).__mro__}
  93. if isinstance(getattr(exc, "setup_failure_class", None), str): # classified where it was raised
  94. return exc.setup_failure_class
  95. # Esc in the setup menus, or consent declined on the provider's page.
  96. if is_cancelled(exc) or "_SetupCancelled" in names or getattr(exc, "oauth_error_code", "") == "access_denied":
  97. return "cancelled"
  98. # Before the expiry check: a code that ran out while the service kept failing is an outage.
  99. if is_network_error(exc) or (exc.__cause__ is not None and is_network_error(exc.__cause__)):
  100. return "network"
  101. if is_device_code_expired(exc):
  102. return "expired"
  103. if names & _AUTH_ERROR_TYPES:
  104. return "auth"
  105. return "other"
  106. def _save_row(fields: dict[str, str] | None) -> bool:
  107. """Save one row and wait for the store; a row the contract cannot build is settled as is."""
  108. if fields is None:
  109. return True
  110. from .shared_metrics_contract import PROVIDER_SETUP_MARK
  111. from .shared_metrics_events import emit_saved
  112. return emit_saved([(PROVIDER_SETUP_MARK, fields)]) == 1
  113. def begin_provider_setup(surface: str, provider: Any, *, hermes_home: Any = None) -> SetupFlow | None:
  114. """Record ``started`` and drop the pending marker; None (no files, no rows) when collection is off."""
  115. try:
  116. with _bound(hermes_home):
  117. from .relay_shared_metrics import enabled
  118. if not provider or provider in _NOT_A_PROVIDER or not enabled():
  119. return None
  120. fields = provider_setup_fields(surface=surface, provider=provider, event="started")
  121. if fields is None:
  122. return None
  123. from gateway.status import get_process_start_time
  124. from hermes_constants import get_hermes_home
  125. home = get_hermes_home()
  126. report_abandoned_setups(home)
  127. directory = markers_dir(home)
  128. directory.mkdir(parents=True, exist_ok=True)
  129. pid = os.getpid()
  130. marker = directory / f"{surface}-{pid}-{time.time_ns()}.json"
  131. record = {
  132. "pid": pid, "start_time": get_process_start_time(pid), "started_at": time.time(),
  133. "surface": surface, "provider": fields["provider"],
  134. }
  135. tmp = marker.with_name(f".{marker.name}.tmp")
  136. tmp.write_text(json.dumps(record), encoding="utf-8")
  137. os.replace(tmp, marker)
  138. _save_row(fields)
  139. return SetupFlow(home=str(home), surface=surface, provider=fields["provider"], marker=marker)
  140. except Exception:
  141. logger.debug("Provider setup start not recorded", exc_info=True)
  142. return None
  143. def finish_provider_setup(flow: SetupFlow | None, event: str, failure_class: str = "none") -> None:
  144. """Record the flow's end once: only if its marker is still unclaimed (never raises)."""
  145. if flow is None:
  146. return
  147. try:
  148. from .shared_metrics_process import settle_claim
  149. claimed = flow.marker.with_name(f"{flow.marker.name}.{os.getpid()}.reporting")
  150. try:
  151. os.replace(flow.marker, claimed)
  152. except OSError: # already reported (abandoned) by another start
  153. return
  154. with _bound(flow.home):
  155. fields = provider_setup_fields(
  156. surface=flow.surface, provider=flow.provider, event=event, failure_class=failure_class,
  157. )
  158. settle_claim(claimed, flow.marker, _save_row(fields))
  159. except Exception:
  160. logger.debug("Provider setup end not recorded", exc_info=True)
  161. def _reportable(record: Any) -> bool:
  162. from gateway.status import runtime_status_pid_is_live
  163. from .shared_metrics_contract import _non_negative_number
  164. if not isinstance(record, dict):
  165. return True
  166. started = _non_negative_number(record.get("started_at"))
  167. if started is None or time.time() - started > STALE_AFTER_S:
  168. return True
  169. return not runtime_status_pid_is_live(record)
  170. def report_abandoned_setups(home: Path) -> None:
  171. """Report every pending marker whose flow can no longer finish as ``abandoned``.
  172. Callers bind ``home`` (the scan runs at a setup start or on the process-exit reporter thread).
  173. """
  174. from .shared_metrics_process import _claimer_alive, settle_claim
  175. directory = markers_dir(home)
  176. if not directory.is_dir():
  177. return
  178. for path in sorted(directory.iterdir()):
  179. if path.name.startswith(".") or ".json" not in path.name:
  180. continue
  181. if path.name.endswith(".reporting") and _claimer_alive(path):
  182. continue
  183. try:
  184. record = json.loads(path.read_text(encoding="utf-8-sig"))
  185. except (OSError, ValueError):
  186. record = None
  187. if not path.name.endswith(".reporting") and not _reportable(record):
  188. continue
  189. original = path.with_name(path.name.split(".json")[0] + ".json")
  190. claimed = original.with_name(f"{original.name}.{os.getpid()}.reporting")
  191. try:
  192. os.replace(path, claimed)
  193. except OSError: # a concurrent reporter or the finisher won
  194. continue
  195. fields = provider_setup_fields(
  196. surface=record.get("surface"), provider=record.get("provider"), event="abandoned",
  197. ) if isinstance(record, dict) else None
  198. settle_claim(claimed, original, _save_row(fields))
  199. def record_provider_setup_done(surface: str, provider: Any, *, hermes_home: Any = None, background: bool = False) -> None:
  200. """A setup that starts and lands in one action (an API key saved from a form): both rows at once.
  201. ``background`` keeps a request handler from waiting on a cold metrics runtime; the owning home is
  202. captured here, since a thread does not inherit the profile binding."""
  203. try:
  204. with _bound(hermes_home):
  205. from .relay_shared_metrics import enabled
  206. if not provider or not enabled():
  207. return
  208. from hermes_constants import get_hermes_home
  209. home = str(get_hermes_home())
  210. def run() -> None:
  211. finish_provider_setup(begin_provider_setup(surface, provider, hermes_home=home), "completed")
  212. if background:
  213. threading.Thread(target=run, name="hermes-setup-metrics", daemon=True).start()
  214. else:
  215. run()
  216. except Exception:
  217. logger.debug("Provider setup not recorded", exc_info=True)
  218. def provider_for_api_key_env(env_var: Any, *, connecting: bool = False) -> str | None:
  219. """The shipped provider whose own API key lives in ``env_var``; None for any other variable. A bare
  220. key save cannot tell connecting a provider from configuring a tool, so unless the caller says it is
  221. ``connecting`` one, ecosystem tokens and keys a tool's settings panel (TTS, STT, image...) also asks
  222. for are None too."""
  223. from hermes_cli.auth import PROVIDER_REGISTRY
  224. from hermes_cli.tools_config import TOOL_CATEGORIES
  225. if env_var == "OPENROUTER_API_KEY": # the aggregator is not a registry entry
  226. return "openrouter"
  227. if not connecting and (env_var in _SHARED_TOKENS or any(
  228. env_var == entry.get("key") for category in TOOL_CATEGORIES.values()
  229. for row in category.get("providers", ()) for entry in row.get("env_vars") or ()
  230. )):
  231. return None
  232. return next((slug for slug, pconfig in PROVIDER_REGISTRY.items()
  233. if env_var in (getattr(pconfig, "api_key_env_vars", None) or ())), None)
  234. def record_api_key_saved(env_var: Any, value: Any, previous: Any, surface: str, *, connecting: bool = False) -> None:
  235. """Count a provider API key saved from a settings form: only a new or changed non-empty value (a
  236. clear or a same-value re-save connects nothing; other variables are not setups)."""
  237. try:
  238. from .relay_shared_metrics import enabled
  239. if not isinstance(value, str) or not value.strip() or value == previous:
  240. return
  241. if enabled() and (provider := provider_for_api_key_env(env_var, connecting=connecting)):
  242. record_provider_setup_done(surface, provider, background=True)
  243. except Exception:
  244. logger.debug("API key setup not recorded", exc_info=True)
  245. # ---- CLI flows (`hermes setup`, `hermes model`, first-run chat) ------------------------------
  246. _cli = threading.local()
  247. def note_provider_setup_saved() -> None:
  248. """A CLI flow persisted the provider/model choice (called by the shared save helpers)."""
  249. hints = getattr(_cli, "hints", None)
  250. if hints is not None:
  251. hints["saved"] = True
  252. def note_provider_setup_failure(failure_class: str) -> None:
  253. """A CLI flow reported a failure it then returned from instead of raising (first one wins)."""
  254. hints = getattr(_cli, "hints", None)
  255. if hints is not None and hints.get("failure") is None:
  256. hints["failure"] = failure_class
  257. def note_sign_in_failure(exc: BaseException) -> None:
  258. with contextlib.suppress(Exception):
  259. note_provider_setup_failure(setup_failure_class(exc))
  260. def _model_route() -> tuple[Any, ...]:
  261. from hermes_cli.config import load_config
  262. model = load_config().get("model")
  263. if not isinstance(model, dict):
  264. return (model,)
  265. return tuple(model.get(key) for key in ("provider", "default", "base_url"))
  266. @contextlib.contextmanager
  267. def provider_setup_surface(surface: str) -> Iterator[None]:
  268. """Name the surface whose provider picker runs inside (``hermes model``, the setup wizard, the
  269. first-run chat prompt). The picker is shared, so only these entry points know which one it is."""
  270. previous = getattr(_cli, "surface", None)
  271. _cli.surface = surface
  272. try:
  273. yield
  274. except BaseException as exc:
  275. if not _is_go_back(exc): # a Back replays the wizard section: its flow may resume there
  276. _close_backed_out()
  277. raise
  278. else:
  279. _close_backed_out()
  280. finally:
  281. _cli.surface = previous
  282. def _is_go_back(exc: BaseException) -> bool:
  283. return any(cls.__name__ == "_SetupGoBack" for cls in type(exc).__mro__)
  284. def _close_backed_out() -> None:
  285. """The user went Back out of a provider flow and never resumed it: it ends ``abandoned``."""
  286. pending, _cli.backed_out = getattr(_cli, "backed_out", None), None
  287. if pending is not None:
  288. finish_provider_setup(pending[0], "abandoned")
  289. def _resumes(flow: SetupFlow, surface: Any, provider: Any) -> bool:
  290. try:
  291. from .shared_metrics_catalog import provider_metric_name
  292. return provider not in _NOT_A_PROVIDER and (flow.surface, flow.provider) == (surface, provider_metric_name(provider))
  293. except Exception:
  294. return False
  295. @contextlib.contextmanager
  296. def cli_provider_setup(provider: Any) -> Iterator[None]:
  297. """Track one CLI provider flow under the entry point's surface. Without one (fallback picking,
  298. tests driving the picker directly) nothing is tracked.
  299. Completed when the flow saved a choice or changed the model route; a flow that returns without
  300. either failed with the class it reported, else the user backed out (``abandoned``). Back (Left
  301. arrow) leaves the flow open: the replayed picker re-entering the same provider continues it (one
  302. ``started``); picking another provider, or leaving the entry point, ends it ``abandoned``.
  303. """
  304. surface = getattr(_cli, "surface", None)
  305. pending, _cli.backed_out = getattr(_cli, "backed_out", None), None
  306. if pending is not None and not _resumes(pending[0], surface, provider):
  307. finish_provider_setup(pending[0], "abandoned")
  308. pending = None
  309. flow, before = pending or (begin_provider_setup(surface, provider) if surface else None, None)
  310. if flow is None:
  311. yield
  312. return
  313. if pending is None:
  314. with contextlib.suppress(Exception):
  315. before = _model_route()
  316. hints: dict[str, Any] = {"saved": False, "failure": None}
  317. _cli.hints = hints
  318. try:
  319. yield
  320. except BaseException as exc:
  321. _cli.hints = None
  322. if _is_go_back(exc):
  323. _cli.backed_out = (flow, before)
  324. else:
  325. finish_provider_setup(flow, "failed", setup_failure_class(exc))
  326. raise
  327. _cli.hints = None
  328. landed = hints["saved"]
  329. if not landed:
  330. with contextlib.suppress(Exception):
  331. landed = before is not None and _model_route() != before
  332. if landed:
  333. finish_provider_setup(flow, "completed")
  334. elif hints["failure"]:
  335. finish_provider_setup(flow, "failed", hints["failure"])
  336. else:
  337. finish_provider_setup(flow, "abandoned")
  338. # ---- dashboard / Desktop OAuth sessions ------------------------------------------------------
  339. def web_setup_surface() -> str:
  340. from hermes_cli.process_identity import is_desktop_owned_backend
  341. return "desktop" if is_desktop_owned_backend() else "dashboard"
  342. def begin_oauth_setup(provider: str, hermes_home: Any) -> SetupFlow | None:
  343. try:
  344. return begin_provider_setup(web_setup_surface(), provider, hermes_home=hermes_home)
  345. except Exception:
  346. logger.debug("OAuth setup start not recorded", exc_info=True)
  347. return None
  348. def attach_oauth_setup(sess: dict[str, Any] | None, flow: SetupFlow | None) -> None:
  349. if flow is not None and isinstance(sess, dict):
  350. sess[_OAUTH_SESSION_KEY] = flow
  351. def note_oauth_failure(sess: dict[str, Any] | None, exc: BaseException) -> None:
  352. """A sign-in poller died with ``exc``: keep its closed class for the settle (the status alone is
  353. ``error`` for a lapsed code, a dropped network and a refusal alike)."""
  354. with contextlib.suppress(Exception):
  355. if isinstance(sess, dict):
  356. sess[_OAUTH_FAILURE_KEY] = setup_failure_class(exc)
  357. def settle_oauth_setup(sess: dict[str, Any] | None, *, abandoned: bool = False) -> None:
  358. """End an OAuth session's flow from its status (no-op while pending, or when already ended)."""
  359. try:
  360. if not isinstance(sess, dict) or sess.get(_OAUTH_SESSION_KEY) is None:
  361. return
  362. status = "cancelled" if sess.get("cancelled") else str(sess.get("status") or "")
  363. ending = _OAUTH_REASONS.get(str(sess.get("reason") or "")) if status in {"denied", "error"} else None
  364. ending = ending or _OAUTH_ENDINGS.get(status)
  365. if ending is None and status == "error":
  366. ending = ("failed", sess.get(_OAUTH_FAILURE_KEY) or "other")
  367. if ending is None:
  368. if not abandoned:
  369. return
  370. ending = ("abandoned", "none")
  371. flow = sess.pop(_OAUTH_SESSION_KEY, None)
  372. finish_provider_setup(flow, *ending)
  373. except Exception:
  374. logger.debug("OAuth setup end not recorded", exc_info=True)