slash_commands_status.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708
  1. """Read-only gateway introspection commands: /status, /context, /usage, /agents, /insights, /topup.
  2. Bound onto ``GatewayRunner`` through ``GatewaySlashCommandsMixin``."""
  3. from __future__ import annotations
  4. import logging
  5. import asyncio
  6. import hashlib
  7. import os
  8. import re
  9. import time
  10. from typing import Any
  11. from agent.account_usage import fetch_account_usage, render_account_usage_lines
  12. from agent.i18n import t
  13. from gateway.config import Platform
  14. from gateway.platforms.event import MessageEvent
  15. from gateway.session_transcript import TranscriptReadError
  16. from hermes_cli.status_report import build_status_fields
  17. # Log-record parity with gateway/run.py and the origin module.
  18. logger = logging.getLogger("gateway.run")
  19. _LIST_CAP = 12 # /agents shows at most this many rows per section
  20. def _clean_str(value: Any) -> str:
  21. """Strip and return a non-empty string value, or empty string."""
  22. return value.strip() if isinstance(value, str) and value.strip() else ""
  23. def _int_value(value: Any) -> int:
  24. """Safely coerce to int."""
  25. try:
  26. return int(value)
  27. except (TypeError, ValueError):
  28. return 0
  29. def _n(obj, attr: str):
  30. return getattr(obj, attr, 0) or 0
  31. def _fmt(n) -> str:
  32. return f"{n:,}"
  33. def _pct(used, total) -> float: # clamped occupancy percentage; 0 for an unknown window
  34. return min(100, used / total * 100) if total else 0
  35. def _clip(text: str, limit: int) -> str:
  36. return text[: limit - 3] + "..." if len(text) > limit else text
  37. def _transcript_estimate(history) -> tuple[int, int]:
  38. """``(approx_tokens, message_count)`` over the user/assistant messages of a transcript."""
  39. from agent.model_metadata import estimate_messages_tokens_rough
  40. msgs = [m for m in history if m.get("role") in {"user", "assistant"} and m.get("content")]
  41. return estimate_messages_tokens_rough(msgs), len(msgs)
  42. async def _quiet(call, default=None):
  43. """Await ``call()`` fail-open: any exception (sync or in the awaitable) yields *default*."""
  44. try:
  45. return await call()
  46. except Exception:
  47. return default
  48. # English source for the transcript-read-failure reply; callers render ``history_unreadable()``
  49. # (``gateway.shared.history_unreadable``) so the active language applies.
  50. HISTORY_UNREADABLE = ("⚠️ I can't read this conversation's history right now (your earlier messages "
  51. "exist but cannot be loaded). Run `hermes doctor --fix` on the host, or use /new "
  52. "to start fresh.")
  53. def history_unreadable() -> str:
  54. return t("gateway.shared.history_unreadable")
  55. def _configured_provider() -> str:
  56. """``model.provider`` from the gateway config ("" when unset)."""
  57. from gateway.run import _load_gateway_config
  58. user_config = _load_gateway_config()
  59. model_cfg = user_config.get("model", {}) if isinstance(user_config, dict) else {}
  60. return _clean_str(model_cfg.get("provider")) if isinstance(model_cfg, dict) else ""
  61. def _quiet_sync(call, default=None):
  62. """Sync twin of ``_quiet``."""
  63. try:
  64. return call()
  65. except Exception:
  66. return default
  67. def _status_model_route(
  68. status_agent, active_override: dict, persisted_route: dict, session_row: dict, session_entry
  69. ):
  70. """``(model, provider, context_used, context_total, route)`` for /status.
  71. Order: live/cached agent route -> active session override -> persisted recent route ->
  72. SessionDB row -> gateway config (only loaded when something is still missing). ``route`` carries
  73. the ``provider`` / ``base_url`` / ``api_key`` of the winning source only, so a later context-window
  74. lookup queries the endpoint that serves the displayed model (never a losing route's endpoint);
  75. a winner without a ``base_url`` leaves the lookup on the default runtime route.
  76. """
  77. from gateway.run import _AGENT_PENDING_SENTINEL, _load_gateway_config, _resolve_gateway_model
  78. context_used = context_total = 0
  79. routes: list[tuple[str, str, dict]] = []
  80. if status_agent is not None and status_agent is not _AGENT_PENDING_SENTINEL:
  81. routes.append((_clean_str(getattr(status_agent, "model", "")),
  82. _clean_str(getattr(status_agent, "provider", "")),
  83. {"base_url": _clean_str(getattr(status_agent, "base_url", "")),
  84. "api_key": _clean_str(getattr(status_agent, "api_key", ""))}))
  85. ctx = getattr(status_agent, "context_compressor", None)
  86. if ctx is not None:
  87. context_used = max(0, _int_value(getattr(ctx, "last_prompt_tokens", 0)))
  88. context_total = _int_value(getattr(ctx, "context_length", 0))
  89. routes.append((_clean_str(active_override.get("model")),
  90. _clean_str(active_override.get("provider")),
  91. {"base_url": _clean_str(active_override.get("base_url")),
  92. "api_key": _clean_str(active_override.get("api_key"))}))
  93. routes.append((_clean_str(persisted_route.get("model")),
  94. _clean_str(persisted_route.get("billing_provider")), {}))
  95. row_route = (_clean_str(session_row.get("model")), _clean_str(session_row.get("billing_provider")), {})
  96. # First fully-resolved (model AND provider) route wins; the SessionDB row is used even if partial.
  97. model_name, provider_name, route = next((r for r in routes if r[0] and r[1]), row_route)
  98. context_used = context_used or _int_value(getattr(session_entry, "last_prompt_tokens", 0))
  99. user_config: dict[str, Any] = {}
  100. if not model_name or not provider_name:
  101. user_config = _quiet_sync(_load_gateway_config, {})
  102. model_cfg = user_config.get("model", {}) if isinstance(user_config, dict) else {}
  103. model_cfg = model_cfg if isinstance(model_cfg, dict) else {}
  104. model_name = model_name or _resolve_gateway_model(user_config)
  105. provider_name = provider_name or _clean_str(model_cfg.get("provider"))
  106. # No raw ``model.context_length`` pin here: the resolver applies it only while the displayed
  107. # route still matches the configured one (a session /model switch must not inherit it).
  108. return model_name, provider_name, context_used, context_total, {"provider": provider_name, **route}
  109. def _context_compressor_lines(agent, ctx, used: int) -> list[str]:
  110. """/context full view: auto-compression threshold/headroom, compression count + last savings,
  111. and cumulative throughput (labelled as throughput, NOT context size)."""
  112. lines: list[str] = []
  113. from agent.context_breakdown import context_display_source
  114. mark = "~" if context_display_source(ctx) != "provider_usage" else ""
  115. threshold = _n(ctx, "threshold_tokens")
  116. threshold_pct = f"{_n(ctx, 'threshold_percent') * 100:.0f}"
  117. if threshold > 0:
  118. if used >= threshold:
  119. lines.append(t("gateway.context.over_threshold", threshold=_fmt(threshold),
  120. threshold_pct=threshold_pct))
  121. else:
  122. lines.append(t("gateway.context.threshold", threshold=_fmt(threshold),
  123. threshold_pct=threshold_pct, to_go=mark + _fmt(threshold - used)))
  124. compressions = _n(ctx, "compression_count")
  125. lines.append(t("gateway.context.compressions", count=compressions))
  126. savings = getattr(ctx, "_last_compression_savings_pct", None) if compressions else None
  127. if savings is not None:
  128. lines.append(t("gateway.context.last_savings", savings=f"{savings:.0f}"))
  129. lines += [
  130. "",
  131. t("gateway.context.totals_header", calls=_n(agent, "session_api_calls")),
  132. t("gateway.context.totals_line",
  133. input=_fmt(_n(agent, "session_input_tokens")),
  134. output=_fmt(_n(agent, "session_output_tokens")),
  135. reasoning=_fmt(_n(agent, "session_reasoning_tokens"))),
  136. t("gateway.context.total_billed", total=_fmt(_n(agent, "session_total_tokens"))),
  137. t("gateway.context.throughput_note"),
  138. ]
  139. return lines
  140. def _agents_delegation_lines(d: dict) -> list[str]:
  141. """/agents rows for one background delegation. Live per-child activity comes from the
  142. registry's progress sampler: api calls, current tool, seconds since last activity."""
  143. goal = _clip(" ".join(str(d.get("goal") or "").split()), 70)
  144. status = d.get("status", "?")
  145. row = f"- `{d.get('delegation_id', '?')}` · {status}"
  146. quiet = d.get("stalled_after_quiet_seconds")
  147. if status == "stalling" and quiet is not None:
  148. row += t("gateway.agents.no_progress", seconds=f"{quiet:.0f}")
  149. elif status != "stalling" and d.get("seconds_since_progress", 0) >= 60:
  150. row += t("gateway.agents.quiet_for", seconds=f"{d['seconds_since_progress']:.0f}")
  151. if goal:
  152. row += f" · {goal}"
  153. lines = [row]
  154. for i, child in enumerate(d.get("children_activity") or []):
  155. if not isinstance(child, dict):
  156. continue
  157. tool = child.get("current_tool")
  158. doing = f"`{tool}`" if tool else t("gateway.agents.between_turns")
  159. part = t("gateway.agents.child_row", index=i + 1, calls=child.get("api_calls", "?"), doing=doing)
  160. idle = child.get("seconds_since_activity")
  161. lines.append(part + (t("gateway.agents.active_ago", seconds=f"{idle:.0f}") if idle is not None else ""))
  162. return lines
  163. def _usage_agent_stats_lines(agent) -> list[str]:
  164. """/usage session block for a live agent: rate limits, token breakdown (matches the CLI),
  165. context window and compression count."""
  166. lines: list[str] = []
  167. rl_state = agent.get_rate_limit_state()
  168. if rl_state and rl_state.has_data:
  169. from agent.rate_limit_tracker import format_rate_limit_compact
  170. lines += [t("gateway.usage.rate_limits", state=format_rate_limit_compact(rl_state)), ""]
  171. lines += [
  172. t("gateway.usage.header_session"),
  173. t("gateway.usage.label_model", model=agent.model),
  174. t("gateway.usage.label_input_tokens", count=_fmt(_n(agent, "session_input_tokens"))),
  175. t("gateway.usage.label_output_tokens", count=_fmt(_n(agent, "session_output_tokens"))),
  176. t("gateway.usage.label_total", count=_fmt(agent.session_total_tokens)),
  177. t("gateway.usage.label_api_calls", count=agent.session_api_calls),
  178. ]
  179. ctx = agent.context_compressor
  180. if ctx.last_prompt_tokens > 0:
  181. pct = _pct(ctx.last_prompt_tokens, ctx.context_length)
  182. from agent.context_breakdown import context_display_source
  183. from agent.context_pin import agent_context_suffix
  184. mark = "~" if context_display_source(ctx) != "provider_usage" else ""
  185. lines.append(t("gateway.usage.label_context", used=mark + _fmt(ctx.last_prompt_tokens),
  186. total=_fmt(ctx.context_length), pct=f"{mark}{pct:.0f}") + agent_context_suffix(agent))
  187. if ctx.compression_count:
  188. lines.append(t("gateway.usage.label_compressions", count=ctx.compression_count))
  189. return lines
  190. def _capped_rows(items: list, render) -> list[str]:
  191. """Render up to ``_LIST_CAP`` items via *render* (list of lines each) plus an overflow line."""
  192. lines: list[str] = []
  193. for item in items[:_LIST_CAP]:
  194. lines.extend(render(item))
  195. if len(items) > _LIST_CAP:
  196. lines.append(t("gateway.agents.more", count=len(items) - _LIST_CAP))
  197. return lines
  198. class GatewayStatusCommandsMixin:
  199. """Read-only gateway introspection commands: /status, /context, /usage, /agents, /insights, /topup."""
  200. async def _handle_status_command(self, event: MessageEvent) -> str:
  201. """Handle /status command."""
  202. from gateway.run import _AGENT_PENDING_SENTINEL
  203. source = event.source
  204. session_entry = await self.async_session_store.get_or_create_session(source)
  205. session_key = session_entry.session_key
  206. # Keep the sentinel distinct: a starting/pending run is not a usable agent for
  207. # model/context display, but it still occupies the session slot.
  208. agent = self._running_agents.get(session_key)
  209. is_running = agent is not None and agent is not _AGENT_PENDING_SENTINEL
  210. # Pending /queue follow-ups (slot + overflow).
  211. adapter = self.adapters.get(source.platform) if source else None
  212. queue_depth = self._queue_depth(session_key, adapter=adapter)
  213. title, session_row, db_total_tokens, persisted_route = await self._status_session_db_facts(
  214. session_entry.session_id
  215. )
  216. # Prefer the live or cached agent (actual runtime route + context compressor); fall back
  217. # to an active /model override, then SessionDB metadata + last_prompt_tokens so /status
  218. # stays useful between turns. Rehydrate first so this precedence survives gateway restarts.
  219. status_agent = agent if is_running else self._cached_agent_for(session_key)
  220. self._rehydrate_session_model_override(session_key)
  221. active_override = self._session_model_override(session_key) or {}
  222. model_name, provider_name, context_used, context_total, route = _status_model_route(
  223. status_agent, active_override, persisted_route, session_row, session_entry
  224. )
  225. if not context_total and model_name:
  226. # Same resolver /context uses (off-loop: it can probe /models). A window the resolver only
  227. # invented (unknown model → DEFAULT_FALLBACK_CONTEXT) stays hidden rather than being shown
  228. # as a real limit; the occupancy-only line below is honest for that case.
  229. resolved = await self._resolve_route_context(source, model_name, route)
  230. if resolved is not None and resolved.context_source != "default":
  231. context_total = _int_value(resolved.context_length)
  232. fields = build_status_fields(
  233. session_entry.session_id, None, session_row, title=title, model=model_name, provider=provider_name,
  234. created=session_entry.created_at, last_activity=session_entry.updated_at,
  235. tokens=db_total_tokens, agent_running=is_running,
  236. )
  237. lines = [t("gateway.status.header"), "",
  238. t("gateway.status.session_id", session_id=fields["session_id"])]
  239. if fields["title"]:
  240. lines.append(t("gateway.status.title", title=fields["title"]))
  241. lines += [t("gateway.status.created", timestamp=fields["created"]),
  242. t("gateway.status.last_activity", timestamp=fields["last_activity"])]
  243. if fields["model"] and fields["provider"]:
  244. lines.append(t("gateway.status.model_provider", model=fields["model"], provider=fields["provider"]))
  245. elif fields["model"]:
  246. lines.append(t("gateway.status.model", model=fields["model"]))
  247. from agent.context_breakdown import context_display_source
  248. mark = "~" if context_display_source(getattr(status_agent, "context_compressor", None)) != "provider_usage" else ""
  249. if context_total:
  250. pct = min(100, round((context_used / context_total) * 100))
  251. from agent.context_pin import agent_context_suffix # (pinned) / (fallback) for the agent's own window
  252. compressor_total = _n(getattr(status_agent, "context_compressor", None), "context_length")
  253. suffix = agent_context_suffix(status_agent) if compressor_total and compressor_total == context_total else ""
  254. lines.append(t("gateway.status.context", used=mark + _fmt(context_used), total=_fmt(context_total),
  255. pct=f"{mark}{pct}") + suffix)
  256. elif context_used:
  257. lines.append(t("gateway.status.context_used", used=mark + _fmt(context_used)))
  258. state = t("gateway.status.state_yes") if fields["agent_running"] else t("gateway.status.state_no")
  259. lines += [t("gateway.status.tokens", tokens=fields["tokens"]),
  260. t("gateway.status.agent_running", state=state)]
  261. if queue_depth:
  262. lines.append(t("gateway.status.queued", count=queue_depth))
  263. if source.platform == Platform.MATRIX:
  264. scope = getattr(self.adapters.get(Platform.MATRIX), "_matrix_session_scope",
  265. os.getenv("MATRIX_SESSION_SCOPE", "auto"))
  266. lines += [
  267. "",
  268. t("gateway.status.matrix_scope_header"),
  269. t("gateway.status.matrix_scope_room", room=source.chat_name or source.chat_id),
  270. t("gateway.status.matrix_scope_room_id", room_id=source.chat_id),
  271. t("gateway.status.matrix_scope_thread", thread_id=source.thread_id or t("gateway.shared.none_value")),
  272. t("gateway.status.matrix_scope_mode", scope=scope),
  273. t("gateway.status.matrix_scope_key",
  274. session_key=self._redact_matrix_session_key(session_key)),
  275. ]
  276. lines += ["", t("gateway.status.platforms", platforms=', '.join(p.value for p in self.adapters))]
  277. return "\n".join(lines)
  278. async def _status_session_db_facts(self, session_id: str):
  279. """``(title, session_row, db_total_tokens, persisted_route)`` for /status; each fail-open.
  280. Token totals come from the SQLite session DB, not SessionStore: run_agent.py persists per-turn
  281. token deltas into sessions_db, never into SessionEntry (its total_tokens is always 0).
  282. """
  283. db = self._session_db
  284. if not db:
  285. return None, {}, 0, {}
  286. title = await _quiet(lambda: db.get_session_title(session_id))
  287. row = await _quiet(lambda: db.get_session(session_id))
  288. session_row = row if isinstance(row, dict) else {}
  289. db_total_tokens = sum(
  290. _int_value(session_row.get(k))
  291. for k in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens")
  292. )
  293. route = await _quiet(lambda: db.get_recent_session_model_route(session_id))
  294. return title, session_row, db_total_tokens, route if isinstance(route, dict) else {}
  295. @staticmethod
  296. def _redact_matrix_session_key(session_key: str) -> str:
  297. """Return a stable Matrix session-key fingerprint for shared room status."""
  298. digest = hashlib.sha256(str(session_key or "").encode("utf-8")).hexdigest()[:12]
  299. return f"sha256:{digest}"
  300. async def _handle_context_command(self, event: MessageEvent) -> str:
  301. """Handle /context — the deep context-window view (/status has the one-line summary).
  302. Gauge, auto-compression threshold/headroom, compression count + last savings, and cumulative
  303. throughput (labelled as throughput, NOT context size). Resolution: running agent -> cached
  304. agent -> SessionStore/SessionDB metadata -> transcript estimate. ``all`` adds listings.
  305. """
  306. source = event.source
  307. session_entry = await self.async_session_store.get_or_create_session(source)
  308. expanded = event.get_command_args().strip().lower() in {"all", "full", "details"}
  309. # Running agent first (mid-turn), then cached agent (between turns).
  310. agent = self._resident_agent_for(self._session_key_for_source(source)) or None
  311. ctx = getattr(agent, "context_compressor", None) if agent else None
  312. used, context_length, model_name, window_suffix = await self._resolve_context_figures(
  313. agent, ctx, session_entry, source
  314. )
  315. from agent.context_breakdown import context_display_source
  316. mark = "~" if context_display_source(ctx) != "provider_usage" else ""
  317. # Gauge path: preserve the provenance of the selected occupancy figure.
  318. if used > 0 and context_length > 0:
  319. pct = _pct(used, context_length)
  320. filled = round(pct / 100 * 24)
  321. lines = [
  322. t("gateway.context.header"), "",
  323. t("gateway.context.model", model=model_name or "?"),
  324. t("gateway.context.window", total=_fmt(context_length)) + window_suffix,
  325. t("gateway.context.in_use", used=mark + _fmt(used), total=_fmt(context_length), pct=f"{mark}{pct:.0f}"),
  326. t("gateway.context.bar", bar="█" * max(0, filled) + "░" * max(0, 24 - filled)),
  327. t("gateway.context.headroom", headroom=mark + _fmt(max(0, context_length - used))),
  328. "",
  329. ]
  330. # Full view — compression / throughput need the live agent.
  331. lines += _context_compressor_lines(agent, ctx, used) if ctx is not None else [
  332. t("gateway.context.detail_after_first")]
  333. # Per-category estimated breakdown (+ optional expanded listings). Same chars/4 engine
  334. # the desktop popover and /usage use; plain text (monospace isn't guaranteed on
  335. # messaging platforms). Fail-open: rendering errors never break /context.
  336. breakdown = await asyncio.to_thread(self._context_breakdown_block, agent, source, expanded) if agent else []
  337. return "\n".join(lines + ([""] + breakdown if breakdown else []))
  338. # Last resort: rough estimate from transcript
  339. try:
  340. history = await self.async_session_store.load_transcript(session_entry.session_id)
  341. except TranscriptReadError:
  342. return history_unreadable()
  343. if not history:
  344. return t("gateway.context.no_data")
  345. approx, count = _transcript_estimate(history)
  346. return "\n".join([
  347. t("gateway.context.header"), "",
  348. t("gateway.context.estimated", count=_fmt(approx), messages=count),
  349. t("gateway.context.detail_after_first"),
  350. ])
  351. async def _resolve_context_figures(self, agent, ctx, session_entry, source):
  352. """``(used, context_length, model_name, window_suffix)`` for /context: used = compressor -> SessionStore;
  353. model = agent -> SessionDB row; window = compressor -> gateway model route -> model metadata. The suffix
  354. is `` (pinned)`` / `` (fallback)`` from whichever source supplied the window."""
  355. from agent.context_pin import agent_context_suffix, context_window_suffix
  356. used = max(0, _n(ctx, "last_prompt_tokens")) or max(0, _int_value(getattr(session_entry, "last_prompt_tokens", 0)))
  357. context_length = _n(ctx, "context_length")
  358. suffix = agent_context_suffix(agent) if context_length else ""
  359. model_name = _clean_str(getattr(agent, "model", "")) if agent is not None else ""
  360. if not model_name and self._session_db:
  361. row = await _quiet(lambda: self._session_db.get_session(session_entry.session_id))
  362. model_name = _clean_str(row.get("model", "")) if isinstance(row, dict) else ""
  363. if not context_length:
  364. resolved = await self._resolve_route_context(source, model_name)
  365. if resolved is not None:
  366. model_name = model_name or resolved.model
  367. context_length = _int_value(resolved.context_length)
  368. suffix = {"config": " (pinned)", "default": " (fallback)"}.get(resolved.context_source, "")
  369. if not context_length and model_name:
  370. from agent.model_metadata import get_model_context_length
  371. context_length = _int_value(
  372. await _quiet(lambda: asyncio.to_thread(get_model_context_length, model_name))
  373. )
  374. suffix = context_window_suffix(context_length, None, model=model_name)
  375. return used, context_length, model_name, suffix
  376. async def _resolve_route_context(self, source, model_name: str, route: dict | None = None):
  377. """``_resolve_gateway_model_context`` for a non-resident session, run off the event loop inside
  378. the profile serving ``source`` (multiplex). Fail-open: None on any error."""
  379. from gateway.run import _profile_runtime_scope, _resolve_gateway_model_context
  380. def _resolve():
  381. if getattr(getattr(self, "config", None), "multiplex_profiles", False):
  382. with _profile_runtime_scope(self._resolve_profile_home_for_source(source)):
  383. return _resolve_gateway_model_context(model_name or None, route)
  384. return _resolve_gateway_model_context(model_name or None, route)
  385. return await _quiet(lambda: asyncio.to_thread(_resolve))
  386. async def _handle_agents_command(self, event: MessageEvent) -> str:
  387. """Handle /agents command - list active agents and running tasks."""
  388. from gateway.run import _AGENT_PENDING_SENTINEL
  389. from tools.process_registry import format_uptime_short, process_registry
  390. now = time.time()
  391. current_session_key = self._session_key_for_source(event.source)
  392. running_started: dict = getattr(self, "_running_agents_ts", {}) or {}
  393. agent_rows: list[dict] = []
  394. for session_key, agent in (getattr(self, "_running_agents", {}) or {}).items():
  395. pending = agent is _AGENT_PENDING_SENTINEL
  396. agent_rows.append({
  397. "session_key": session_key,
  398. "elapsed": max(0, int(now - float(running_started.get(session_key, now)))),
  399. "state": t("gateway.agents.state_starting") if pending else t("gateway.agents.state_running"),
  400. "session_id": "" if pending else str(getattr(agent, "session_id", "") or ""),
  401. "model": "" if pending else str(getattr(agent, "model", "") or ""),
  402. })
  403. agent_rows.sort(key=lambda row: row["elapsed"], reverse=True)
  404. procs = _quiet_sync(process_registry.list_sessions, [])
  405. running_processes = [p for p in procs if p.get("status") == "running"]
  406. background_tasks = [task for task in (getattr(self, "_background_tasks", set()) or set())
  407. if hasattr(task, "done") and not task.done()]
  408. # Background (async) delegations — delegate_task(background=true).
  409. # Live per-child activity comes from the registry's progress sampler (#51690): api calls, current
  410. # tool, seconds since last activity.
  411. from tools.async_delegation import list_async_delegations
  412. delegations = [d for d in _quiet_sync(list_async_delegations, [])
  413. if d.get("status") in ("running", "stalling", "finalizing")]
  414. def _agent_row(idx_row):
  415. idx, row = idx_row
  416. current = t("gateway.agents.this_chat") if row["session_key"] == current_session_key else ""
  417. sid = f" · `{row['session_id']}`" if row["session_id"] else ""
  418. model = f" · `{row['model']}`" if row["model"] else ""
  419. return [f"{idx}. `{row['session_key']}` · {row['state']} · "
  420. f"{format_uptime_short(row['elapsed'])}{sid}{model}{current}"]
  421. def _proc_row(proc):
  422. cmd = _clip(" ".join(str(proc.get("command", "")).split()), 90)
  423. return [f"- `{proc.get('session_id', '?')}` · "
  424. f"{format_uptime_short(int(proc.get('uptime_seconds', 0)))} · `{cmd}`"]
  425. lines = [t("gateway.agents.header"), "", t("gateway.agents.active_agents", count=len(agent_rows))]
  426. lines += _capped_rows(list(enumerate(agent_rows, 1)), _agent_row)
  427. lines += ["", t("gateway.agents.running_processes", count=len(running_processes))]
  428. lines += _capped_rows(running_processes, _proc_row)
  429. lines += ["", t("gateway.agents.async_jobs", count=len(background_tasks))]
  430. if delegations:
  431. lines += ["", t("gateway.agents.background_delegations", count=len(delegations))]
  432. lines += _capped_rows(delegations, _agents_delegation_lines)
  433. if not (agent_rows or running_processes or background_tasks or delegations):
  434. lines += ["", t("gateway.agents.none")]
  435. return "\n".join(lines)
  436. async def _handle_topup_command(self, event: MessageEvent) -> str:
  437. """Handle /topup -- show the Nous balance and hand off to the portal. Does NOT charge, confirm,
  438. or track payment (that happens in the browser; the next /topup shows the new balance)."""
  439. from agent.account_usage import build_credits_view
  440. view = await _quiet(lambda: asyncio.to_thread(build_credits_view, markdown=True))
  441. if view is None or not view.logged_in:
  442. return t("gateway.credits.not_logged_in")
  443. # Drop the helper's 📈 header; we print our own.
  444. lines = [t("gateway.topup.header")] + [ln for ln in view.balance_lines if not ln.lstrip().startswith("📈")]
  445. if view.identity_line:
  446. lines += ["", view.identity_line]
  447. if view.topup_url:
  448. lines += ["", t("gateway.topup.portal_link", url=view.topup_url), t("gateway.topup.hint")]
  449. return "\n".join(lines)
  450. def _context_breakdown_block(self, agent, source, expanded: bool) -> list[str]:
  451. """/context per-category block (plain text, chars/4 estimate, same engine as /usage).
  452. Runs in a thread; returns [] and never raises."""
  453. try:
  454. from agent.context_breakdown import compute_context_details, render_context_breakdown_lines
  455. try:
  456. payload = self._session_context_breakdown(agent, source)
  457. except TranscriptReadError:
  458. return [history_unreadable()] # a read failure is not an empty transcript
  459. if not (payload.get("categories") or []):
  460. return []
  461. details = _quiet_sync(lambda: compute_context_details(agent), {"skills": [], "toolsets": []}) if expanded else None
  462. from agent.context_file_sources import context_file_sources_for_agent, render_context_file_lines
  463. file_lines = _quiet_sync(lambda: render_context_file_lines(context_file_sources_for_agent(agent)), [])
  464. return render_context_breakdown_lines(payload, details=details, grid=False) + ([""] + file_lines if file_lines else [])
  465. except Exception:
  466. return []
  467. def _session_context_breakdown(self, agent, source) -> dict:
  468. """Per-category context estimate (chars/4) for *agent* over the session transcript (sync).
  469. Raises ``TranscriptReadError`` (unreadable rows must not pass as an empty transcript)."""
  470. from agent.context_breakdown import compute_session_context_breakdown
  471. store = self.session_store
  472. try:
  473. history = store.load_transcript(store.get_or_create_session(source).session_id) or []
  474. except TranscriptReadError:
  475. raise
  476. except Exception:
  477. history = []
  478. return compute_session_context_breakdown(agent, history)
  479. def _context_breakdown_lines(self, agent, source) -> list[str]:
  480. """/usage per-category context breakdown (chars/4 estimate). Returns [] and never raises."""
  481. try:
  482. try:
  483. payload = self._session_context_breakdown(agent, source)
  484. except TranscriptReadError:
  485. return [history_unreadable()]
  486. categories = payload.get("categories") or []
  487. if not categories:
  488. return []
  489. total = payload.get("estimated_total") or 0
  490. out = [t("gateway.usage.breakdown_header")]
  491. for cat in categories:
  492. tokens = int(cat.get("tokens") or 0)
  493. if tokens <= 0:
  494. continue
  495. cat_id = str(cat.get("id") or "")
  496. label = t(f"gateway.usage.breakdown_cat_{cat_id}")
  497. if label.endswith(f"breakdown_cat_{cat_id}"): # missing key: t() echoes it back
  498. label = str(cat.get("label") or cat_id)
  499. pct = round(tokens / total * 100) if total else 0
  500. out.append(t("gateway.usage.breakdown_line", label=label, count=_fmt(tokens), pct=f"~{pct}"))
  501. return out if len(out) > 1 else []
  502. except Exception:
  503. return []
  504. async def _handle_usage_command(self, event: MessageEvent) -> str:
  505. """Handle /usage -- token usage for the current session (live or cached agent) plus
  506. account/credit blocks; ``/usage reset [--force]`` redeems a banked Codex reset credit."""
  507. source = event.source
  508. session_key = self._session_key_for_source(source)
  509. raw_args = event.get_command_args().strip()
  510. args = [a.lower() for a in raw_args.split()] if raw_args else []
  511. wants_reset = bool(args) and args[0] == "reset"
  512. if args and not wants_reset:
  513. return t("gateway.usage.unknown_subcommand", args=raw_args)
  514. # Running agent first (mid-turn), then cached agent (between turns).
  515. agent = self._resident_agent_for(session_key)
  516. # Provider/base_url/api_key for the account-usage fetch: live agent first, else persisted
  517. # billing data on the SessionDB row so `/usage` still returns account info between turns.
  518. provider, base_url, api_key = (
  519. getattr(agent, k, None) if agent else None for k in ("provider", "base_url", "api_key")
  520. )
  521. if not provider and getattr(self, "_session_db", None) is not None:
  522. provider, base_url = await self._persisted_billing_route(source)
  523. if not provider:
  524. # Fresh or evicted session with no persisted route (e.g. /usage right after login):
  525. # fall back to the configured provider, as /status does, so account limits such as
  526. # Codex subscription windows still render from on-disk credentials (#15167).
  527. provider = await _quiet(lambda: asyncio.to_thread(_configured_provider)) or None
  528. if wants_reset:
  529. if str(provider or "").strip().lower() != "openai-codex":
  530. return t("gateway.usage.reset_wrong_provider")
  531. from agent.account_usage import redeem_codex_reset_credit
  532. result = await asyncio.to_thread(
  533. redeem_codex_reset_credit, base_url=base_url, api_key=api_key, force="--force" in args[1:],
  534. )
  535. return result.message
  536. # Account usage off the event loop so slow provider APIs don't block the gateway;
  537. # failures are non-fatal (account_lines stays []).
  538. account_snapshot = provider and await _quiet(
  539. lambda: asyncio.to_thread(fetch_account_usage, provider, base_url=base_url, api_key=api_key)
  540. )
  541. account_lines = (
  542. render_account_usage_lines(account_snapshot, markdown=True) if account_snapshot else []
  543. )
  544. # Nous credits + monthly-grant gauge (shared with CLI/TUI). Gates on "a Nous account is
  545. # logged in" — NOT the inference provider — so a Nous user inferring elsewhere still sees
  546. # a balance. Fail-open: never break /usage.
  547. from agent.account_usage import nous_credits_lines
  548. credits_state = getattr(agent, "_credits_state", None) if agent else None
  549. credits_lines = await _quiet(
  550. lambda: asyncio.to_thread(nous_credits_lines, markdown=True, credits_state=credits_state),
  551. [],
  552. )
  553. # Window plans: a chat's /topup has no Keep going control, so point the setting at the Portal.
  554. from agent.plan_limits import portal_wording
  555. credits_lines = [portal_wording(line) for line in credits_lines]
  556. def _with_account_blocks(lines: list[str]) -> str:
  557. # Each block is preceded by a blank divider only when something precedes it.
  558. for block in (account_lines, credits_lines):
  559. if block:
  560. if lines:
  561. lines.append("")
  562. lines.extend(block)
  563. return "\n".join(lines)
  564. if agent and hasattr(agent, "session_total_tokens") and agent.session_api_calls > 0:
  565. lines = _usage_agent_stats_lines(agent)
  566. # Per-category breakdown (chars/4 estimate, same engine as the desktop popover): prompt
  567. # / tools / skills / memory off the live agent, conversation from the transcript.
  568. breakdown_lines = await asyncio.to_thread(self._context_breakdown_lines, agent, source)
  569. if breakdown_lines:
  570. lines += [""] + breakdown_lines
  571. return _with_account_blocks(lines)
  572. # No agent at all -- rough count from session history
  573. session_entry = await self.async_session_store.get_or_create_session(source)
  574. try:
  575. history = await self.async_session_store.load_transcript(session_entry.session_id)
  576. except TranscriptReadError:
  577. return history_unreadable()
  578. if history:
  579. approx, count = _transcript_estimate(history)
  580. return _with_account_blocks([
  581. t("gateway.usage.header_session_info"),
  582. t("gateway.usage.label_messages", count=count),
  583. t("gateway.usage.label_estimated_context", count=_fmt(approx)),
  584. t("gateway.usage.detailed_after_first"),
  585. ])
  586. if account_lines or credits_lines:
  587. return _with_account_blocks([])
  588. return t("gateway.usage.no_data")
  589. async def _persisted_billing_route(self, source):
  590. """``(provider, base_url)`` from the SessionDB row / most recent route when no agent is resident."""
  591. async def _rows():
  592. entry = await self.async_session_store.get_or_create_session(source)
  593. persisted = await self._session_db.get_session(entry.session_id) or {}
  594. route = await self._session_db.get_recent_session_model_route(entry.session_id)
  595. return persisted, route if isinstance(route, dict) else {}
  596. persisted, recent = await _quiet(_rows, ({}, {}))
  597. row = recent if recent.get("billing_provider") else persisted
  598. return row.get("billing_provider"), row.get("billing_base_url")
  599. async def _handle_insights_command(self, event: MessageEvent) -> str:
  600. """Handle /insights [N | --days N] [--source S] -- usage insights and analytics."""
  601. # Normalize Unicode dashes (Telegram/iOS auto-converts -- to em/en dash)
  602. args = re.sub(r'[\u2012\u2013\u2014\u2015](days|source)', r'--\1', event.get_command_args().strip())
  603. days, source = 30, None
  604. parts = args.split()
  605. i = 0
  606. while i < len(parts):
  607. flag, value = parts[i], parts[i + 1] if i + 1 < len(parts) else None
  608. if flag == "--days" and value is not None:
  609. try:
  610. days = int(value)
  611. except ValueError:
  612. return t("gateway.insights.invalid_days", value=value)
  613. i += 2
  614. elif flag == "--source" and value is not None:
  615. source, i = value, i + 2
  616. else:
  617. days = int(flag) if flag.isdigit() else days
  618. i += 1
  619. try:
  620. from hermes_state_registry import acquire
  621. from agent.insights import InsightsEngine
  622. def _run_insights():
  623. db = acquire()
  624. try:
  625. engine = InsightsEngine(db)
  626. return engine.format_gateway(engine.generate(days=days, source=source))
  627. finally:
  628. from hermes_state_registry import release_or_close
  629. release_or_close(db)
  630. # Not a bare hop: ``SessionDB()`` resolves ``get_hermes_home()`` at call time, a
  631. # contextvar set by ``_profile_runtime_scope``; a default-executor hop starts with an
  632. # EMPTY context and would read the DEFAULT profile's state.db.
  633. return await self._run_in_executor_with_context(_run_insights)
  634. except Exception as e:
  635. logger.error("Insights command error: %s", e, exc_info=True)
  636. return t("gateway.insights.error", error=e)