analytics.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421
  1. """Raw-YAML config and token/cost analytics dashboard routes.
  2. Extracted from ``hermes_cli.web_server``; app state and helpers are late-bound through
  3. :mod:`hermes_cli.web_deps` (cycle-safe, monkeypatch-friendly).
  4. """
  5. import asyncio
  6. import sqlite3
  7. import time
  8. from typing import Any, Dict, List, Optional
  9. import hermes_yaml as yaml
  10. from fastapi import APIRouter, HTTPException, Query
  11. from hermes_cli.config import get_config_path, read_raw_config
  12. from hermes_cli.web_deps import late
  13. from hermes_cli.web_routers._common import corrupt_store_as_status
  14. from hermes_cli.web_server_profiles import (
  15. _approval_mode_of, _aux_task_summary, _aux_usage_rows, _broadcast_gateway_session_info, _is_other_profile, _merge_aux_into_by_model,
  16. )
  17. from hermes_cli.web_models import RawConfigUpdate
  18. router = APIRouter()
  19. # Late-bound so a test's monkeypatch on the owning module wins at call time.
  20. _open_session_db_for_profile = late("_open_session_db_for_profile", "hermes_cli.web_server_sessions")
  21. _session_db_path_for_profile = late("_session_db_path_for_profile", "hermes_cli.web_server_sessions")
  22. _profile_scope = late("_profile_scope", "hermes_cli.web_server_profiles")
  23. save_config = late("save_config", "hermes_cli.config")
  24. # ── Raw YAML config ──────────────────────────────────────────────────────────
  25. @router.get("/api/config/raw")
  26. async def get_config_raw(profile: Optional[str] = None):
  27. """Raw config.yaml text plus its resolved path.
  28. ``path`` is resolved inside ``_profile_scope`` so the Config page header
  29. shows the file the switched profile actually reads/writes — /api/status's
  30. ``config_path`` is machine-global and always reports the dashboard
  31. process's own profile, which is wrong under the global profile switcher.
  32. """
  33. def _run():
  34. from hermes_cli.config_backend import supports_file_tooling
  35. if not supports_file_tooling():
  36. # The raw editor shows the file's bytes; a non-file backend has no file.
  37. raise HTTPException(status_code=409, detail="This config backend has no raw config file")
  38. with _profile_scope(profile):
  39. path = get_config_path()
  40. if not path.exists(): # config-reader: ok — file tooling (gated above)
  41. return {"yaml": "", "path": str(path)}
  42. return {"yaml": path.read_text(encoding="utf-8-sig"), "path": str(path)} # config-reader: ok — file tooling (gated above)
  43. return await asyncio.to_thread(_run)
  44. @router.put("/api/config/raw")
  45. async def update_config_raw(body: RawConfigUpdate, profile: Optional[str] = None):
  46. def _run():
  47. parsed = yaml.safe_load(body.yaml_text)
  48. if not isinstance(parsed, dict):
  49. raise HTTPException(status_code=400, detail="YAML must be a mapping")
  50. with _profile_scope(body.profile or profile):
  51. # Full-document replacement: the editor owns the whole file; never
  52. # merge omitted sections back from disk.
  53. # See #62723.
  54. approvals_mode_changed = _approval_mode_of(parsed) != _approval_mode_of(read_raw_config())
  55. save_config(parsed, merge_existing=False)
  56. # Same indicator refresh as the schema-driven save.
  57. if approvals_mode_changed and not _is_other_profile(body.profile or profile):
  58. _broadcast_gateway_session_info()
  59. return {"ok": True}
  60. try:
  61. return await asyncio.to_thread(_run)
  62. except yaml.YAMLError as e:
  63. raise HTTPException(status_code=400, detail=f"Invalid YAML: {e}")
  64. def _rows(db, sql: str, cutoff: float) -> list[dict[str, Any]]:
  65. return [dict(r) for r in db._conn.execute(sql, (cutoff,)).fetchall()]
  66. def _get_usage_analytics(days: int = 30, profile: Optional[str] = None):
  67. from agent.insights import InsightsEngine
  68. db = _open_session_db_for_profile(profile, read_only=True)
  69. try:
  70. cutoff = time.time() - (days * 86400)
  71. # Local calendar day, per-row (DST-correct), the same day /insights uses (agent/insights.py).
  72. daily = _rows(db, """
  73. SELECT date(started_at, 'unixepoch', 'localtime') as day,
  74. SUM(input_tokens) as input_tokens,
  75. SUM(output_tokens) as output_tokens,
  76. SUM(cache_read_tokens) as cache_read_tokens,
  77. SUM(reasoning_tokens) as reasoning_tokens,
  78. COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
  79. COALESCE(SUM(actual_cost_usd), 0) as actual_cost,
  80. COUNT(*) as sessions,
  81. SUM(COALESCE(api_call_count, 0)) as api_calls
  82. FROM sessions WHERE started_at > ?
  83. GROUP BY day ORDER BY day
  84. """, cutoff)
  85. by_model = _rows(db, """
  86. SELECT model,
  87. SUM(input_tokens) as input_tokens,
  88. SUM(output_tokens) as output_tokens,
  89. COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
  90. COUNT(*) as sessions,
  91. SUM(COALESCE(api_call_count, 0)) as api_calls
  92. FROM sessions WHERE started_at > ? AND model IS NOT NULL
  93. GROUP BY model ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC
  94. """, cutoff)
  95. # Fold in auxiliary usage (vision, compression, ...) from session_model_usage.
  96. # Aux calls never touch the sessions counters, so this is add-only — no double count.
  97. # Without it the models list shows only the main agent model even when aux models are actively
  98. # burning tokens (issue #23270).
  99. aux_rows = _aux_usage_rows(db, cutoff)
  100. by_model = _merge_aux_into_by_model(by_model, aux_rows)
  101. totals = _rows(db, """
  102. SELECT SUM(input_tokens) as total_input,
  103. SUM(output_tokens) as total_output,
  104. SUM(cache_read_tokens) as total_cache_read,
  105. SUM(reasoning_tokens) as total_reasoning,
  106. COALESCE(SUM(estimated_cost_usd), 0) as total_estimated_cost,
  107. COALESCE(SUM(actual_cost_usd), 0) as total_actual_cost,
  108. COUNT(*) as total_sessions,
  109. SUM(COALESCE(api_call_count, 0)) as total_api_calls
  110. FROM sessions WHERE started_at > ?
  111. """, cutoff)[0]
  112. usage = InsightsEngine(db).get_usage_breakdown(days=days)
  113. return {
  114. "daily": daily,
  115. "by_model": by_model,
  116. "by_task": _aux_task_summary(aux_rows), # "what is compression costing me"
  117. "totals": totals,
  118. "period_days": days,
  119. "skills": usage["skills"],
  120. "tools": usage["tools"], # per-tool-name counts; desktop aggregates per toolset
  121. }
  122. finally:
  123. db.close()
  124. @router.get("/api/analytics/usage")
  125. async def get_usage_analytics(
  126. days: int = Query(30, ge=1, le=365),
  127. profile: Optional[str] = None,
  128. ):
  129. """``days`` is clamped to 1-365 (idea from #74778): huge or non-positive
  130. values would force expensive full-history SQL and InsightsEngine work, or
  131. produce empty/inverted time windows. The UI only offers 7/30/90-day
  132. presets."""
  133. with corrupt_store_as_status(_session_db_path_for_profile(profile)):
  134. return await asyncio.to_thread(_get_usage_analytics, days, profile)
  135. _USAGE_KEYS = (
  136. "input_tokens", "output_tokens", "cache_read_tokens", "reasoning_tokens",
  137. "estimated_cost", "actual_cost", "api_calls", "tool_calls",
  138. )
  139. def _has_usage(row: dict[str, Any]) -> bool:
  140. return any((row.get(key) or 0) != 0 for key in _USAGE_KEYS)
  141. def _fold_session_only_rows(raw_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
  142. """Fold model rows that carry no billing_provider and no usage into the single
  143. accounted provider row for that model.
  144. Session rows can be created before the first billable call finishes; if that early row
  145. records only the model name while a later row has real accounting, the Models page used
  146. to show a duplicate "0 tokens / — API calls" card. Only folds when ownership is
  147. unambiguous (exactly one provider row).
  148. """
  149. rows_by_model: dict[str, list[dict[str, Any]]] = {}
  150. for row in raw_rows:
  151. rows_by_model.setdefault(row.get("model") or "", []).append(row)
  152. rows: list[dict[str, Any]] = []
  153. for model_rows in rows_by_model.values():
  154. provider_rows = [r for r in model_rows if r.get("billing_provider")]
  155. if len(provider_rows) != 1:
  156. rows.extend(model_rows)
  157. continue
  158. target = provider_rows[0]
  159. for row in model_rows:
  160. if row is target or row.get("billing_provider") or _has_usage(row):
  161. continue
  162. target["sessions"] = (target.get("sessions") or 0) + (row.get("sessions") or 0)
  163. target["last_used_at"] = max(target.get("last_used_at") or 0, row.get("last_used_at") or 0)
  164. total_tokens = (target.get("input_tokens") or 0) + (target.get("output_tokens") or 0)
  165. sessions = target.get("sessions") or 0
  166. target["avg_tokens_per_session"] = total_tokens / sessions if sessions else 0
  167. rows.append(target)
  168. rows.extend(
  169. r for r in model_rows
  170. if r is not target and (r.get("billing_provider") or _has_usage(r))
  171. )
  172. return rows
  173. def _model_capabilities(provider: str, model_name: str) -> dict:
  174. """models.dev capability metadata for the card; {} when unknown or lookup fails."""
  175. try:
  176. from agent.models_dev import get_model_capabilities
  177. mc = get_model_capabilities(provider=provider, model=model_name)
  178. except Exception:
  179. return {}
  180. if mc is None:
  181. return {}
  182. return {
  183. "supports_tools": mc.supports_tools,
  184. "supports_vision": mc.supports_vision,
  185. "supports_reasoning": mc.supports_reasoning,
  186. "context_window": mc.context_window,
  187. "max_output_tokens": mc.max_output_tokens,
  188. "model_family": mc.model_family,
  189. }
  190. _AUX_SUMMED_KEYS = (
  191. "input_tokens", "output_tokens", "cache_read_tokens", "reasoning_tokens", "estimated_cost", "api_calls",
  192. )
  193. def _merge_aux_into_rows(raw_rows: list[dict[str, Any]], aux_rows: list[dict[str, Any]]) -> None:
  194. """Add auxiliary usage onto the matching (model, billing_provider) row in place.
  195. Aux calls happen inside sessions the sessions-derived row already counted, so
  196. ``sessions`` is never added onto an existing row; only an aux-only pair (no
  197. sessions-derived row) gets a new row that carries its own session count.
  198. """
  199. index: dict[tuple, dict[str, Any]] = {
  200. (row.get("model") or "", row.get("billing_provider") or ""): row
  201. for row in raw_rows
  202. }
  203. for aux in aux_rows:
  204. key = (aux.get("model") or "unknown", aux.get("billing_provider") or "")
  205. target = index.get(key)
  206. if target is None:
  207. target = {
  208. "model": key[0],
  209. "billing_provider": key[1],
  210. **{k: 0 for k in _AUX_SUMMED_KEYS},
  211. "actual_cost": 0,
  212. "sessions": aux.get("sessions") or 0,
  213. "tool_calls": 0,
  214. "last_used_at": None,
  215. "avg_tokens_per_session": 0,
  216. }
  217. index[key] = target
  218. raw_rows.append(target)
  219. for k in _AUX_SUMMED_KEYS:
  220. target[k] = (target.get(k) or 0) + (aux.get(k) or 0)
  221. if aux.get("last_used_at") is not None:
  222. target["last_used_at"] = max(target.get("last_used_at") or 0, aux["last_used_at"])
  223. sessions = target.get("sessions") or 0
  224. if sessions:
  225. target["avg_tokens_per_session"] = (
  226. (target.get("input_tokens") or 0) + (target.get("output_tokens") or 0)
  227. ) / sessions
  228. _MODEL_CARD_KEYS = (
  229. "input_tokens", "output_tokens", "cache_read_tokens", "reasoning_tokens",
  230. "estimated_cost", "actual_cost", "sessions", "api_calls", "tool_calls",
  231. "last_used_at", "avg_tokens_per_session",
  232. )
  233. def _attach_tool_calls(db, cutoff: float, raw_rows: list[dict[str, Any]]) -> None:
  234. """Fill the ``tool_calls`` card metric for per-call rows, in place.
  235. Tool calls are session-level data (``sessions.tool_call_count``), not per API
  236. call, so they cannot come from ``session_model_usage``. Attribute each
  237. session-window (model, billing_provider) tool-call total to the card of the
  238. pair its sessions row records — the session's last active route — falling
  239. back to any card of that model when the pair has no per-call row (route
  240. switched after the last call). Never zeroes the metric (#71778).
  241. """
  242. pair_rows = _rows(db, """
  243. SELECT model, billing_provider, SUM(COALESCE(tool_call_count, 0)) as tool_calls
  244. FROM sessions WHERE started_at > ? AND model IS NOT NULL AND model != ''
  245. GROUP BY model, billing_provider
  246. """, cutoff)
  247. by_pair = {
  248. (r.get("model") or "", r.get("billing_provider") or ""): r.get("tool_calls") or 0
  249. for r in pair_rows
  250. }
  251. index: dict[tuple, dict[str, Any]] = {}
  252. by_model: dict[str, list[dict[str, Any]]] = {}
  253. for row in raw_rows:
  254. row["tool_calls"] = 0
  255. index.setdefault((row["model"], row.get("billing_provider") or ""), row)
  256. by_model.setdefault(row["model"], []).append(row)
  257. for (model, provider), calls in by_pair.items():
  258. target = index.get((model, provider)) or (by_model.get(model) or [None])[0]
  259. if target is not None:
  260. target["tool_calls"] += calls
  261. def _get_models_analytics(days: int = 30, profile: Optional[str] = None):
  262. """Per-model token/cost/session breakdown plus models.dev capability metadata."""
  263. db = _open_session_db_for_profile(profile, read_only=True)
  264. try:
  265. cutoff = time.time() - (days * 86400)
  266. # Main usage from session_model_usage: every API call's delta lands there
  267. # with the model/provider active at call time, so a mid-session /model
  268. # switch (or a silent fallback rewrite) splits across the pairs that
  269. # actually ran. The sessions table keeps only the final pair, and
  270. # grouping it attributed the whole session to it (#71778). Insights
  271. # (_compute_model_breakdown) reads the same table.
  272. try:
  273. cur = db._conn.execute("""
  274. SELECT u.model,
  275. u.billing_provider,
  276. SUM(u.input_tokens) as input_tokens,
  277. SUM(u.output_tokens) as output_tokens,
  278. SUM(u.cache_read_tokens) as cache_read_tokens,
  279. SUM(u.reasoning_tokens) as reasoning_tokens,
  280. COALESCE(SUM(u.estimated_cost_usd), 0) as estimated_cost,
  281. COALESCE(SUM(u.actual_cost_usd), 0) as actual_cost,
  282. COUNT(DISTINCT u.session_id) as sessions,
  283. SUM(COALESCE(u.api_call_count, 0)) as api_calls,
  284. MAX(u.last_seen) as last_used_at,
  285. AVG(u.input_tokens + u.output_tokens) as avg_tokens_per_session
  286. FROM session_model_usage u
  287. JOIN sessions s ON s.id = u.session_id
  288. WHERE s.started_at > ? AND u.model IS NOT NULL AND u.model != ''
  289. AND u.task = ''
  290. GROUP BY u.model, u.billing_provider
  291. ORDER BY SUM(u.input_tokens) + SUM(u.output_tokens) DESC
  292. """, (cutoff,))
  293. raw_rows = [dict(r) for r in cur.fetchall()]
  294. except sqlite3.OperationalError:
  295. raw_rows = [] # pre-v17 DB without session_model_usage
  296. if not raw_rows:
  297. # No per-call rows in the window (pre-table DB): fall back to the
  298. # sessions aggregate, which for those sessions is the only source.
  299. raw_rows = _rows(db, """
  300. SELECT model,
  301. billing_provider,
  302. SUM(input_tokens) as input_tokens,
  303. SUM(output_tokens) as output_tokens,
  304. SUM(cache_read_tokens) as cache_read_tokens,
  305. SUM(reasoning_tokens) as reasoning_tokens,
  306. COALESCE(SUM(estimated_cost_usd), 0) as estimated_cost,
  307. COALESCE(SUM(actual_cost_usd), 0) as actual_cost,
  308. COUNT(*) as sessions,
  309. SUM(COALESCE(api_call_count, 0)) as api_calls,
  310. SUM(tool_call_count) as tool_calls,
  311. MAX(started_at) as last_used_at,
  312. AVG(input_tokens + output_tokens) as avg_tokens_per_session
  313. FROM sessions WHERE started_at > ? AND model IS NOT NULL AND model != ''
  314. GROUP BY model, billing_provider
  315. ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC
  316. """, cutoff)
  317. else:
  318. _attach_tool_calls(db, cutoff, raw_rows)
  319. # Aux usage (vision/compression/title/approval/...) is folded into the
  320. # (model, provider) row the sessions query already produced. #23270 made
  321. # aux-only models visible; _aux_usage_rows groups by (model, task,
  322. # provider), so appending each row emitted one card per aux task beside
  323. # the main card, and their session counts summed past
  324. # totals.total_sessions (#89631). Only a pair with no sessions-derived
  325. # row becomes a new row, carrying its own session count.
  326. _merge_aux_into_rows(raw_rows, _aux_usage_rows(db, cutoff))
  327. rows = _fold_session_only_rows(raw_rows)
  328. rows.sort(
  329. key=lambda r: (r.get("input_tokens") or 0) + (r.get("output_tokens") or 0),
  330. reverse=True,
  331. )
  332. models = [
  333. {
  334. "model": row["model"],
  335. "provider": row.get("billing_provider") or "",
  336. **{key: row[key] for key in _MODEL_CARD_KEYS},
  337. "capabilities": _model_capabilities(row.get("billing_provider") or "", row["model"]),
  338. }
  339. for row in rows
  340. ]
  341. totals = _rows(db, """
  342. SELECT SUM(input_tokens) as total_input,
  343. SUM(output_tokens) as total_output,
  344. SUM(cache_read_tokens) as total_cache_read,
  345. SUM(reasoning_tokens) as total_reasoning,
  346. COALESCE(SUM(estimated_cost_usd), 0) as total_estimated_cost,
  347. COALESCE(SUM(actual_cost_usd), 0) as total_actual_cost,
  348. COUNT(*) as total_sessions,
  349. SUM(COALESCE(api_call_count, 0)) as total_api_calls
  350. FROM sessions WHERE started_at > ? AND model IS NOT NULL AND model != ''
  351. """, cutoff)[0]
  352. # Counted over the same merged row set the cards come from, so a model
  353. # reached only through auxiliary usage is in the header as well as on
  354. # the page (#89631).
  355. totals["distinct_models"] = len({row["model"] for row in rows})
  356. return {"models": models, "totals": totals, "period_days": days}
  357. finally:
  358. db.close()
  359. @router.get("/api/analytics/models")
  360. async def get_models_analytics(
  361. days: int = Query(30, ge=1, le=365),
  362. profile: Optional[str] = None,
  363. ):
  364. """Return model analytics without blocking the serving event loop."""
  365. with corrupt_store_as_status(_session_db_path_for_profile(profile)):
  366. return await asyncio.to_thread(_get_models_analytics, days, profile)