models.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. """Model assignment dashboard routes: model info/options/recommended default, auxiliary + MoA slots, /api/model/set.
  2. Extracted from ``hermes_cli.web_server``; helpers/state that tests monkeypatch on
  3. ``web_server`` stay there and are resolved late at call time (cycle-safe).
  4. """
  5. import asyncio
  6. import concurrent.futures
  7. import logging
  8. from typing import Optional
  9. from fastapi import APIRouter, HTTPException
  10. from hermes_cli.web_deps import LateState, late
  11. from hermes_cli.web_server_config import (
  12. _AUX_TASK_SLOTS, _UNSET, _apply_model_assignment_sync, _dashboard_code_skew_guard,
  13. _plugin_aux_tasks, _prepare_main_assignment,
  14. )
  15. from agent.model_metadata import is_local_endpoint
  16. from starlette.concurrency import run_in_threadpool
  17. from hermes_cli.web_models import ModelAssignment, MoaConfigPayload, MoaModelSlot
  18. from hermes_cli.web_routers._common import _CONFIG_MUTATION_LOCK, config_write_scope, http_failure
  19. _log = logging.getLogger("hermes_cli.web_server")
  20. router = APIRouter()
  21. # Late-bound so a test's monkeypatch on the owning module wins at call time.
  22. _config_profile_scope = late("_config_profile_scope", "hermes_cli.web_server_profiles")
  23. _profile_scope = late("_profile_scope", "hermes_cli.web_server_profiles")
  24. load_config = late("load_config", "hermes_cli.config")
  25. save_config = late("save_config", "hermes_cli.config")
  26. require_readable_config_before_write = late("require_readable_config_before_write", "hermes_cli.config")
  27. _EMPTY_MODEL_INFO: dict = {
  28. "model": "", "provider": "", "auto_context_length": 0, "config_context_length": 0,
  29. "effective_context_length": 0, "capabilities": {},
  30. }
  31. _CAPABILITY_FIELDS = ("supports_tools", "supports_vision", "supports_reasoning", "context_window",
  32. "max_output_tokens", "model_family")
  33. def _main_model_fields(model_cfg) -> tuple[str, str]:
  34. """(model, provider) from config's ``model`` section, which may be a plain string."""
  35. if isinstance(model_cfg, dict):
  36. return model_cfg.get("default", model_cfg.get("name", "")), model_cfg.get("provider", "")
  37. return (str(model_cfg) if model_cfg else ""), ""
  38. def _load_config_scoped(profile: Optional[str]) -> dict:
  39. with _profile_scope(profile):
  40. return load_config()
  41. # Blocking budget for /api/model/info's context-length resolution. The resolver
  42. # chain (agent.model_metadata.get_model_context_length) runs several sequential
  43. # provider probes, each with its own multi-second timeout, so an unreachable or
  44. # blackholed model.base_url can hold this response for tens of seconds — and the
  45. # Desktop Model Settings page waits on it (#63214).
  46. _MODEL_INFO_PROBE_BUDGET_S = 5.0
  47. def _bounded_context_length_probe(model: str, base_url: str, provider: str) -> int:
  48. """``get_model_context_length`` with the route's blocking budget.
  49. On timeout the abandoned probe keeps running in its worker thread (bounded
  50. by its own per-request timeouts) while the response degrades to
  51. ``auto_context_length = 0`` ("auto-detected: unknown").
  52. """
  53. from agent.model_metadata import get_model_context_length
  54. pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="model-info-probe")
  55. try:
  56. return pool.submit(
  57. get_model_context_length, model=model, base_url=base_url, provider=provider,
  58. config_context_length=None
  59. ).result(timeout=_MODEL_INFO_PROBE_BUDGET_S)
  60. except concurrent.futures.TimeoutError:
  61. _log.warning(
  62. "GET /api/model/info: context-length probe for %r at %s exceeded %.1fs — returning unknown",
  63. model, base_url or "<default>", _MODEL_INFO_PROBE_BUDGET_S,
  64. )
  65. return 0
  66. finally:
  67. # wait=False: never block the response (or interpreter exit) on the
  68. # abandoned probe.
  69. pool.shutdown(wait=False)
  70. @router.get("/api/model/info")
  71. def get_model_info(profile: Optional[str] = None):
  72. """Resolved metadata for the configured model: auto-detected vs configured
  73. context length (so the UI can show "Auto-detected: 200K" beside the
  74. override) plus models.dev capabilities when available."""
  75. try:
  76. model_cfg = _load_config_scoped(profile).get("model", "")
  77. model_name, provider = _main_model_fields(model_cfg)
  78. base_url = model_cfg.get("base_url", "") if isinstance(model_cfg, dict) else ""
  79. config_ctx = model_cfg.get("context_length") if isinstance(model_cfg, dict) else None
  80. if not model_name:
  81. return dict(_EMPTY_MODEL_INFO, provider=provider)
  82. try:
  83. # config_context_length=None: ignore the override — we want the auto value.
  84. # Bounded: the resolver's provider probes can hang for tens of seconds
  85. # when model.base_url is unreachable (#63214).
  86. auto_ctx = _bounded_context_length_probe(model_name, base_url, provider)
  87. except Exception:
  88. auto_ctx = 0
  89. config_ctx_int = config_ctx if isinstance(config_ctx, int) and config_ctx > 0 else 0
  90. caps = {}
  91. try:
  92. from agent.models_dev import get_model_capabilities
  93. mc = get_model_capabilities(provider=provider, model=model_name)
  94. if mc is not None:
  95. caps = {name: getattr(mc, name) for name in _CAPABILITY_FIELDS}
  96. except Exception:
  97. pass
  98. return {
  99. "model": model_name, "provider": provider, "auto_context_length": auto_ctx,
  100. "config_context_length": config_ctx_int,
  101. "effective_context_length": config_ctx_int or auto_ctx, # what the agent actually uses
  102. "capabilities": caps,
  103. }
  104. except HTTPException:
  105. # Unknown/invalid profile must surface as 404, not degrade into a
  106. # 200 with empty model info (which would render as "no model set").
  107. raise
  108. except Exception:
  109. _log.exception("GET /api/model/info failed")
  110. return dict(_EMPTY_MODEL_INFO)
  111. @router.get("/api/model/options")
  112. async def get_model_options(
  113. profile: Optional[str] = None,
  114. refresh: bool = False,
  115. include_unconfigured: bool = False,
  116. explicit_only: bool = False,
  117. ):
  118. """Authenticated providers + curated model lists — REST twin of the ``model.options``
  119. JSON-RPC on tui_gateway, same response shape so ``ModelPickerDialog`` shares the types.
  120. ``profile`` scopes the picker context so the Models page reads the SAME profile
  121. /api/model/set writes. ``refresh`` busts the per-provider model-id disk cache
  122. (picker's explicit "Refresh Models"); normal opens stay on the 1h cache."""
  123. with http_failure("GET /api/model/options failed", 500, detail="Failed to list model options"):
  124. skew_msg = _dashboard_code_skew_guard()
  125. if skew_msg:
  126. _log.warning("GET /api/model/options refused: %s", skew_msg)
  127. raise HTTPException(status_code=503, detail=f"Restart required: {skew_msg}")
  128. from hermes_cli.inventory import build_model_options_payload, load_picker_context
  129. def _build_payload_scoped() -> dict:
  130. # Full sync picker build off the event loop under the requested profile.
  131. # _config_profile_scope (contextvar only, no skill-module lock): the build can
  132. # block 15s on a models.dev cache miss, and _profile_scope's RLock held across
  133. # that starves concurrent /api/config and freezes the server.
  134. with _config_profile_scope(profile):
  135. return build_model_options_payload(
  136. load_picker_context(), explicit_only=bool(explicit_only),
  137. include_unconfigured=bool(include_unconfigured), refresh=bool(refresh))
  138. return await run_in_threadpool(_build_payload_scoped)
  139. def _nous_recommended_default() -> dict:
  140. from hermes_cli.models import recommended_nous_default_model
  141. return recommended_nous_default_model()
  142. @router.get("/api/model/recommended-default")
  143. def get_recommended_default_model(provider: str = "", profile: Optional[str] = None):
  144. """Recommended default model for a freshly-authenticated provider, mirroring
  145. ``hermes model``'s curation so GUI onboarding lands on a sensible default.
  146. Nous honors the user's free/paid tier. Any other provider gets the preferred
  147. silent default when its curated list carries it, else the first curated model —
  148. aggregator lists lead with the priciest Anthropic flagship, which must never be
  149. the model a user lands on without explicitly picking it.
  150. Response: {"provider", "model", "free_tier": bool | None} — free_tier only for
  151. Nous; ``model`` may be empty (caller degrades gracefully)."""
  152. slug = (provider or "").strip().lower()
  153. if slug == "nous":
  154. try:
  155. # The tier, Portal URL and recommendation caches are all per profile home.
  156. with _config_profile_scope(profile):
  157. return _nous_recommended_default()
  158. except HTTPException:
  159. raise # an unknown ?profile= is the scope's 404, not an empty recommendation
  160. except Exception:
  161. _log.exception("GET /api/model/recommended-default (nous) failed")
  162. return {"provider": "nous", "model": "", "free_tier": None}
  163. try:
  164. from hermes_cli.inventory import build_models_payload, load_picker_context
  165. from hermes_cli.models import pick_silent_default_model
  166. # build_models_payload -> list_authenticated_providers -> _save_discovered_models_to_config:
  167. # this GET lazily PERSISTS discovered custom-provider models, so it needs the scope too.
  168. with _config_profile_scope(profile):
  169. payload = build_models_payload(load_picker_context())
  170. for row in payload.get("providers", []):
  171. if str(row.get("slug", "")).lower() == slug:
  172. models = [str(m) for m in (row.get("models") or [])]
  173. return {"provider": slug, "model": pick_silent_default_model(models, provider=slug), "free_tier": None}
  174. return {"provider": slug, "model": "", "free_tier": None}
  175. except HTTPException:
  176. raise # an unknown ?profile= is the scope's 404, not an empty recommendation
  177. except Exception:
  178. _log.exception("GET /api/model/recommended-default failed")
  179. return {"provider": slug, "model": "", "free_tier": None}
  180. @router.get("/api/model/auxiliary")
  181. def get_auxiliary_models(profile: Optional[str] = None):
  182. """Current auxiliary task assignments: ``{"tasks": [{task, provider, model,
  183. base_url}, ...], "main": {provider, model}}``. ``profile`` scopes the read —
  184. without it the Models page would show the dashboard profile's pins while
  185. /api/model/set wrote the selected profile's.
  186. Built-in slots come first; plugin-registered tasks follow, each carrying the
  187. ``label``/``hint``/``plugin`` the plugin declared (built-ins are labelled client-side) and
  188. ``inherit_from`` (base slot key or null). An inheriting row also carries ``effective``: the
  189. route it resolves to right now, which is the base's while the row itself is unpinned.
  190. ``provider``/``model``/``base_url`` stay the slot's own stored values on every row."""
  191. from agent.auxiliary_client import _get_auxiliary_task_config
  192. with http_failure("GET /api/model/auxiliary failed", 500, detail="Failed to read auxiliary config"):
  193. with _profile_scope(profile):
  194. cfg = load_config()
  195. # Inside the scope on purpose: plugin discovery and the resolver key on the same
  196. # context-local home, so profile B's rows come from B's plugins and B's config.
  197. plugin_tasks = _plugin_aux_tasks()
  198. effective = {entry["key"]: _get_auxiliary_task_config(entry["key"])
  199. for entry in plugin_tasks if entry.get("inherit_from")}
  200. aux_cfg = cfg.get("auxiliary", {})
  201. if not isinstance(aux_cfg, dict):
  202. aux_cfg = {}
  203. def _row(slot: str) -> dict:
  204. slot_cfg = aux_cfg.get(slot, {}) if isinstance(aux_cfg.get(slot), dict) else {}
  205. base_url = str(slot_cfg.get("base_url", "") or "")
  206. return {
  207. "task": slot, "provider": str(slot_cfg.get("provider", "auto") or "auto"),
  208. "model": str(slot_cfg.get("model", "") or ""), "base_url": base_url,
  209. "reasoning_effort": str(slot_cfg.get("reasoning_effort") or "") or None,
  210. # Lets the UI tell a free local/LAN pin from a forgotten paid-provider pin.
  211. "local_endpoint": is_local_endpoint(base_url),
  212. }
  213. tasks = [_row(slot) for slot in _AUX_TASK_SLOTS]
  214. tasks.extend({
  215. **_row(entry["key"]),
  216. "label": str(entry.get("display_name") or entry["key"]),
  217. "hint": str(entry.get("description") or ""),
  218. "plugin": str(entry.get("plugin") or ""),
  219. "inherit_from": entry.get("inherit_from") or None,
  220. **({"effective": {field: str(effective[entry["key"]].get(field) or "")
  221. for field in ("provider", "model", "base_url")}}
  222. if entry["key"] in effective else {}),
  223. } for entry in plugin_tasks)
  224. model, provider = _main_model_fields(cfg.get("model", {}))
  225. return {"tasks": tasks, "main": {"provider": str(provider or ""), "model": str(model or "")}}
  226. @router.get("/api/model/moa")
  227. def get_moa_models(profile: Optional[str] = None):
  228. """Return the configured Mixture-of-Agents provider/model slots."""
  229. with http_failure("GET /api/model/moa failed", 500, detail="Failed to read MoA config"):
  230. from hermes_cli.moa_config import normalize_moa_config
  231. with _profile_scope(profile):
  232. cfg = load_config()
  233. return normalize_moa_config(cfg.get("moa") if isinstance(cfg, dict) else {})
  234. _MOA_PRESET_FIELDS = (
  235. "reference_temperature", "aggregator_temperature", "reference_timeout",
  236. "degraded_reference_policy", "fanout", "enabled",
  237. )
  238. def _slot_dict(slot: MoaModelSlot) -> dict:
  239. # Drop unset optionals so saved slots stay minimal ({provider, model}).
  240. return {k: v for k, v in slot.dict().items() if v is not None}
  241. def _preset_dict(preset) -> dict:
  242. """Raw preset dict from a MoaPresetPayload or the flat MoaConfigPayload fields."""
  243. return {
  244. "reference_models": [_slot_dict(slot) for slot in preset.reference_models],
  245. "aggregator": _slot_dict(preset.aggregator),
  246. **{name: getattr(preset, name) for name in _MOA_PRESET_FIELDS},
  247. }
  248. @router.put("/api/model/moa")
  249. def set_moa_models(body: MoaConfigPayload, profile: Optional[str] = None):
  250. """Persist the Mixture-of-Agents provider/model slots."""
  251. with http_failure("PUT /api/model/moa failed", 500, detail="Failed to save MoA config"):
  252. from hermes_cli import config as config_mod
  253. from hermes_cli.moa_config import normalize_moa_config, validate_moa_payload
  254. # Serialize the full RMW with dashboard autosaves and with non-HTTP
  255. # save_config callers, which only hold the config module's reentrant lock.
  256. with config_write_scope(body.profile or profile), config_mod._CONFIG_LOCK:
  257. legacy_flat_update = not body.presets
  258. if not legacy_flat_update:
  259. raw = {
  260. "default_preset": body.default_preset,
  261. "active_preset": body.active_preset,
  262. "presets": {name: _preset_dict(preset) for name, preset in body.presets.items()},
  263. }
  264. else:
  265. raw = _preset_dict(body) # legacy flat payload from older clients
  266. # Reject-don't-repair: normalize_moa_config() silently swaps any preset with
  267. # incomplete slots for the hardcoded defaults — correct tolerance at READ time,
  268. # silent data loss at WRITE time (desktop autosave of a half-filled slot replaced
  269. # the user's whole preset). Refuse loudly so no client can corrupt config here.
  270. # See #64156.
  271. problems = validate_moa_payload(raw)
  272. if problems:
  273. raise HTTPException(status_code=422, detail="Invalid MoA config: " + "; ".join(problems))
  274. # Read the raw profile file because the effective config contains defaults and
  275. # ``merge_existing=True`` deep-merges dicts, which resurrects omitted preset names.
  276. # Keep unknown MoA keys and sibling sections by rebuilding only the MoA section on
  277. # the raw document, while treating the submitted named map as authoritative.
  278. # Strict read: the merge below builds a new dict and would discard the
  279. # failed-read marker, allowing a transient fallback to replace the file.
  280. raw_cfg = require_readable_config_before_write()
  281. existing_moa = raw_cfg.get("moa")
  282. if isinstance(existing_moa, dict) and "privacy_filter" in existing_moa:
  283. # ``privacy_filter`` is emitted by the normalizer but is not declared by
  284. # MoaConfigPayload; an editor round-trip must not reset it.
  285. raw["privacy_filter"] = existing_moa["privacy_filter"]
  286. normalized = normalize_moa_config(raw)
  287. if legacy_flat_update and isinstance(existing_moa, dict):
  288. existing_presets = existing_moa.get("presets")
  289. existing_default = existing_moa.get("default_preset")
  290. if isinstance(existing_presets, dict):
  291. if isinstance(existing_default, str) and existing_default in existing_presets:
  292. # Older clients update the flat default preset; keep the existing
  293. # default name and any valid active selection from the stored map.
  294. synthesized_default = normalized.get("default_preset")
  295. if (
  296. isinstance(synthesized_default, str)
  297. and synthesized_default in normalized["presets"]
  298. ):
  299. normalized["presets"][existing_default] = normalized["presets"].pop(
  300. synthesized_default
  301. )
  302. normalized["default_preset"] = existing_default
  303. existing_active = existing_moa.get("active_preset")
  304. if isinstance(existing_active, str) and existing_active in existing_presets:
  305. normalized["active_preset"] = existing_active
  306. updated = config_mod._merge_partial_save(raw_cfg, {"moa": normalized})
  307. if not legacy_flat_update:
  308. # ``_merge_partial_save`` retains undeclared metadata on surviving presets;
  309. # filter after that merge so omitted names are authoritative deletions.
  310. merged_presets = updated["moa"]["presets"]
  311. updated["moa"]["presets"] = {
  312. name: merged_presets[name] for name in normalized["presets"]
  313. }
  314. save_config(updated)
  315. return {"ok": True, **normalized}
  316. @router.post("/api/model/set")
  317. async def set_model_assignment(body: ModelAssignment, profile: Optional[str] = None):
  318. """Assign a model to the main slot or an auxiliary task slot. Writes
  319. ``~/.hermes/config.yaml`` — applies to **new** sessions only; a running chat
  320. PTY hot-swaps via the ``/model`` slash command instead."""
  321. scope, task = (body.scope or "").strip().lower(), (body.task or "").strip().lower()
  322. provider, model = (body.provider or "").strip(), (body.model or "").strip()
  323. base_url, api_key = (body.base_url or "").strip(), (body.api_key or "").strip()
  324. if scope not in {"main", "auxiliary"}:
  325. raise HTTPException(status_code=400, detail="scope must be 'main' or 'auxiliary'")
  326. with http_failure("POST /api/model/set failed", 500, detail="Failed to save model assignment"):
  327. # #99859 (R2): the options picker already refuses on code skew; the WRITE path
  328. # must too — a stale process persisting a post-update model string is the
  329. # invalid-model-serving failure the reporter hit.
  330. skew_msg = _dashboard_code_skew_guard()
  331. if skew_msg:
  332. _log.warning("POST /api/model/set refused: %s", skew_msg)
  333. raise HTTPException(status_code=503, detail=f"Restart required: {skew_msg}")
  334. # Expensive-model warning runs BEFORE the profile scope is entered: _profile_scope
  335. # must never be held across an await (the RLock is reentrant per-thread, so a second
  336. # coroutine interleaving on the event-loop thread could cross-restore module globals).
  337. if model and not body.confirm_expensive_model:
  338. try:
  339. from hermes_cli.model_selection_guards import combined_selection_warning
  340. # Pricing lookup can hit models.dev / a /models endpoint on a cache miss — off the loop.
  341. warning = await asyncio.to_thread(combined_selection_warning, model, provider=provider, base_url=base_url)
  342. except Exception:
  343. warning = None
  344. if warning is not None:
  345. return {"ok": False, "scope": scope, "provider": provider, "model": model,
  346. "confirm_required": True, "confirm_message": warning.message}
  347. reasoning_effort = body.reasoning_effort if "reasoning_effort" in body.model_fields_set else _UNSET
  348. def _apply_assignment():
  349. # Same RMW span as PUT /api/config: applyMainModel fires this while the
  350. # settings-page autosave is in flight — hold the mutation lock. switch_model's
  351. # catalog fetches / endpoint probes are network I/O, so they run BEFORE the lock;
  352. # only load→apply→save holds it.
  353. with _profile_scope(body.profile or profile):
  354. prepared = (_prepare_main_assignment(load_config(), provider, model, base_url, api_key)
  355. if scope == "main" else None)
  356. with _CONFIG_MUTATION_LOCK:
  357. return _apply_model_assignment_sync(
  358. scope, provider, model, task, base_url, api_key,
  359. reasoning_effort=reasoning_effort, prepared=prepared)
  360. return await asyncio.to_thread(_apply_assignment)