models_reasoning_caps.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307
  1. """Per-model reasoning capabilities from OpenRouter-schema ``/v1/models`` catalogs.
  2. Split out of ``hermes_cli.models``. OpenRouter and
  3. Nous Portal share one implementation parametrized by :class:`_CapsSource`; the per-source module
  4. globals (``_openrouter_reasoning_caps_cache``, ``_nous_caps_disk_checked``, ...) stay defined on
  5. ``hermes_cli.models`` — tests reset them there — and are read/written by attribute name.
  6. Tri-state contract for callers deciding whether to emit reasoning controls: a dict with
  7. ``supports_reasoning: True`` (+ ``supported_efforts``, ``mandatory``) — the route advertises
  8. reasoning controls; ``supports_reasoning: False`` — the catalog knows the model and it does NOT
  9. accept them (definitive negative); ``None`` — unknown (catalog not loaded, model not listed,
  10. malformed).
  11. """
  12. from __future__ import annotations
  13. import contextvars
  14. import json
  15. import logging
  16. import os
  17. import threading
  18. import time
  19. import urllib.request
  20. from dataclasses import dataclass
  21. from pathlib import Path
  22. from typing import Any, Callable, Optional
  23. logger = logging.getLogger("hermes_cli.models")
  24. Caps = dict[str, Optional[dict[str, Any]]]
  25. def _origin():
  26. from hermes_cli import models
  27. return models
  28. def parse_openrouter_reasoning_capabilities(item: Any) -> Optional[dict[str, Any]]:
  29. """Normalize one OpenRouter catalog entry's reasoning metadata.
  30. ``supported_parameters`` contains ``"reasoning"`` when the route accepts reasoning controls at
  31. all; a top-level ``reasoning`` object may add ``mandatory`` / ``supported_efforts``. A missing
  32. or malformed ``supported_parameters`` is "unknown" (None), like ``_openrouter_model_supports_tools``.
  33. """
  34. if not isinstance(item, dict):
  35. return None
  36. params = item.get("supported_parameters")
  37. if not isinstance(params, list):
  38. return None
  39. if "reasoning" not in params:
  40. return {"supports_reasoning": False}
  41. reasoning = item.get("reasoning")
  42. if not isinstance(reasoning, dict):
  43. reasoning = {}
  44. raw_efforts = reasoning.get("supported_efforts")
  45. efforts: Optional[list[str]] = None
  46. if isinstance(raw_efforts, list):
  47. efforts = list(dict.fromkeys(str(e).strip().lower() for e in raw_efforts if str(e).strip()))
  48. return {"supports_reasoning": True, "supported_efforts": efforts, "mandatory": reasoning.get("mandatory") is True}
  49. # ── Disk mirror ────────────────────────────────────────────────────────
  50. #
  51. # In-process caches are always cold in a short-lived process, and every consumer is on a hot path
  52. # that must never block on HTTP — so without a disk copy, `hermes -p`, a cron job, or a freshly
  53. # booted gateway answers "capability unknown" for its whole first turn. One file holds every
  54. # catalog keyed by URL: OpenRouter and the Portal list different models, and a staging Portal must
  55. # not answer for production.
  56. _REASONING_CAPS_DISK_TTL_SECONDS = 24 * 3600
  57. def _reasoning_caps_disk_path() -> Path:
  58. from hermes_constants import get_hermes_home
  59. return get_hermes_home() / "cache" / "reasoning_caps.json"
  60. def _read_reasoning_caps_disk() -> dict[str, Any]:
  61. from hermes_cli.models import _read_json_cache
  62. return _read_json_cache(_reasoning_caps_disk_path()) or {}
  63. def _load_reasoning_caps_disk(url: str) -> tuple[Optional[Caps], float]:
  64. """Return ``(caps, age_seconds)`` for *url*, or ``(None, 0.0)``."""
  65. entry = _read_reasoning_caps_disk().get(url)
  66. caps = entry.get("caps") if isinstance(entry, dict) else None
  67. if not isinstance(caps, dict) or not caps:
  68. return None, 0.0
  69. try:
  70. age = max(0.0, time.time() - float(entry.get("ts") or 0))
  71. except (TypeError, ValueError):
  72. age = float(_REASONING_CAPS_DISK_TTL_SECONDS)
  73. return {str(mid): model_caps for mid, model_caps in caps.items()}, age
  74. def _save_reasoning_caps_disk(url: str, caps: Caps) -> None:
  75. """Merge *url*'s catalog into the shared disk mirror, atomically."""
  76. from hermes_cli.models import _write_json_cache
  77. try:
  78. data = _read_reasoning_caps_disk()
  79. data[url] = {"ts": time.time(), "caps": caps}
  80. _write_json_cache(_reasoning_caps_disk_path(), data, indent=0, separators=(",", ":"))
  81. except Exception as exc:
  82. logger.debug("Failed to save reasoning-caps disk cache: %s", exc)
  83. def _warm_reasoning_caps_async(refresh) -> None:
  84. """Run *refresh* in a daemon thread (fire-and-forget) so a cold/stale cache is warm for the
  85. next call or, via the disk mirror, the next process without this turn blocking on HTTP.
  86. Callers own the once-per-process guard; the fetch keeps its own failure TTL."""
  87. if os.environ.get("PYTEST_CURRENT_TEST"):
  88. return
  89. # copy_context: the Portal URL and disk mirror are the calling profile's, not the launch home's.
  90. threading.Thread(target=contextvars.copy_context().run, args=(refresh,), name="reasoning-caps-warm", daemon=True).start()
  91. def _hydrate_reasoning_caps_from_disk(url: str, refresh) -> Optional[Caps]:
  92. """The disk copy of *url*'s catalog, queueing *refresh* when it's stale. A copy past its TTL is
  93. still returned — a stale verdict beats no verdict, and capabilities change rarely."""
  94. caps, age = _load_reasoning_caps_disk(url)
  95. if caps is not None and age >= _REASONING_CAPS_DISK_TTL_SECONDS:
  96. _warm_reasoning_caps_async(refresh)
  97. return caps
  98. def _seed_reasoning_caps(url: str, items: Any) -> Optional[Caps]:
  99. """Parse a ``/v1/models`` ``data`` array and mirror it for *url*.
  100. Takes the payload rather than fetching it, so picker and pricing fetches (same document) leave
  101. the mirror warm at no network cost. None when the array has no usable entries — callers
  102. remember that as a failure rather than caching empty.
  103. """
  104. if not isinstance(items, list):
  105. return None
  106. caps_by_id: Caps = {}
  107. for item in items:
  108. mid = str(item.get("id") or "").strip() if isinstance(item, dict) else ""
  109. if mid:
  110. caps_by_id[mid] = parse_openrouter_reasoning_capabilities(item)
  111. if not caps_by_id:
  112. return None
  113. _save_reasoning_caps_disk(url, caps_by_id)
  114. return caps_by_id
  115. def _fetch_reasoning_caps_catalog(url: str, timeout: float) -> Optional[Caps]:
  116. """Fetch one OpenRouter-shaped ``/v1/models`` catalog → per-model caps; None when unreachable or
  117. empty so callers remember the failure. Sends a User-Agent: the Portal 403s anonymous reads."""
  118. from hermes_cli.version_info import hermes_user_agent
  119. m = _origin()
  120. try:
  121. req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": hermes_user_agent()})
  122. with m._urlopen_model_catalog_request(req, timeout=timeout) as resp:
  123. payload = json.loads(resp.read().decode())
  124. except Exception:
  125. return None
  126. return _seed_reasoning_caps(url, payload.get("data"))
  127. # ── Per-source cache (OpenRouter, Nous Portal) ─────────────────────────
  128. @dataclass(frozen=True)
  129. class _CapsSource:
  130. """One catalog's cache slots on ``hermes_cli.models`` plus how to name its URL.
  131. ``cache``: model id → parsed caps from one full-catalog fetch, kept for the process lifetime.
  132. ``failed_at``: monotonic timestamp of the last FAILED fetch; suppresses re-fetch storms from
  133. per-turn callers for 60s (mirrors the LM Studio/Ollama probe caching). ``disk_checked`` /
  134. ``warm_started``: once-per-process guards for the disk hydrate and the background warm.
  135. """
  136. cache: str
  137. failed_at: str
  138. disk_checked: str
  139. warm_started: str
  140. url: Callable[[], str]
  141. # True when the URL (hence the catalog) follows the active profile's credentials/.env: under a
  142. # routed profile the slots then live per home, and the once-per-process guards must not let the
  143. # launch profile's disk hydrate or warm count as another profile's.
  144. per_profile: bool = False
  145. def get(self, slot: str):
  146. if not self.per_profile:
  147. return getattr(_origin(), getattr(self, slot))
  148. from hermes_cli.models_profile_cache import profile_slot_get
  149. return profile_slot_get(_origin(), getattr(self, slot), False if slot in ("disk_checked", "warm_started") else None)
  150. def set(self, slot: str, value) -> None:
  151. if not self.per_profile:
  152. setattr(_origin(), getattr(self, slot), value)
  153. return
  154. from hermes_cli.models_profile_cache import profile_slot_set
  155. profile_slot_set(_origin(), getattr(self, slot), value)
  156. def _fetch_caps(src: _CapsSource, timeout: float = 6.0, *, force: bool = False) -> Optional[Caps]:
  157. """Fetch + cache the source's per-model caps. None (without poisoning the cache) when
  158. unreachable, so callers retry later and fall back meanwhile."""
  159. cached = src.get("cache")
  160. if cached is not None and not force:
  161. return cached
  162. failed_at = src.get("failed_at")
  163. if failed_at is not None and (time.monotonic() - failed_at) < 60:
  164. return None
  165. caps_by_id = _fetch_reasoning_caps_catalog(src.url(), timeout)
  166. if caps_by_id is None:
  167. src.set("failed_at", time.monotonic())
  168. return None
  169. src.set("cache", caps_by_id)
  170. return caps_by_id
  171. def _caps_cached(src: _CapsSource) -> Optional[Caps]:
  172. """Cache-only caps: memory, else the disk mirror. Never HTTP.
  173. One disk attempt per process: for the Portal, naming the catalog means resolving credentials,
  174. which can itself reach the network to refresh a token — too expensive for a per-turn caller.
  175. """
  176. if src.get("cache") is None and not src.get("disk_checked"):
  177. src.set("disk_checked", True)
  178. src.set("cache", _hydrate_reasoning_caps_from_disk(src.url(), lambda: _fetch_caps(src, force=True)))
  179. return src.get("cache")
  180. def _model_caps(src: _CapsSource, model_id: Optional[str], *, timeout: float, allow_fetch: bool) -> Optional[dict[str, Any]]:
  181. model = str(model_id or "").strip()
  182. if not model:
  183. return None
  184. caps_by_id = _caps_cached(src)
  185. if caps_by_id is None and allow_fetch:
  186. caps_by_id = _fetch_caps(src, timeout=timeout)
  187. return caps_by_id.get(model) if caps_by_id is not None else None
  188. def _warm_caps_async(src: _CapsSource) -> None:
  189. if src.get("warm_started") or _caps_cached(src) is not None:
  190. return
  191. src.set("warm_started", True)
  192. _warm_reasoning_caps_async(lambda: _fetch_caps(src, force=True))
  193. def refresh_reasoning_caps_async(provider: Optional[str]) -> None:
  194. """Force a background re-fetch of *provider*'s reasoning-capability catalog.
  195. The in-memory cache is otherwise held for the process lifetime, so a route that flips to
  196. reasoning-mandatory mid-process (GLM-5.3-flash, Sep 2026) keeps being sent disables it now
  197. rejects. Called from the conversation loop's reasoning_mandatory recovery so the profile guard
  198. is right again on the next request; no-op for providers without a catalog.
  199. """
  200. src = {"nous": _NOUS_CAPS, "nous-portal": _NOUS_CAPS, "nousresearch": _NOUS_CAPS,
  201. "openrouter": _OPENROUTER_CAPS}.get(str(provider or "").strip().lower())
  202. if src is not None:
  203. _warm_reasoning_caps_async(lambda: _fetch_caps(src, force=True))
  204. _OPENROUTER_CATALOG_URL = "https://openrouter.ai/api/v1/models"
  205. _OPENROUTER_CAPS = _CapsSource(
  206. "_openrouter_reasoning_caps_cache", "_openrouter_reasoning_caps_failed_at",
  207. "_openrouter_caps_disk_checked", "_openrouter_caps_warm_started",
  208. lambda: _OPENROUTER_CATALOG_URL,
  209. )
  210. # Nous Portal serves OpenRouter's catalog schema, so the same parser and contract apply. Its own
  211. # cache because the two catalogs list different models (and different capabilities for shared ids).
  212. _NOUS_CAPS = _CapsSource(
  213. "_nous_reasoning_caps_cache", "_nous_reasoning_caps_failed_at",
  214. "_nous_caps_disk_checked", "_nous_caps_warm_started",
  215. lambda: nous_catalog_url(), per_profile=True,
  216. )
  217. def nous_catalog_url() -> str:
  218. """The Portal ``/v1/models`` URL for the endpoint we actually talk to (``NOUS_INFERENCE_BASE_URL``
  219. → resolved credential base → prod), so a staging profile reads staging's capabilities."""
  220. from hermes_cli.models_pricing import _resolve_nous_pricing_credentials
  221. return f"{_resolve_nous_pricing_credentials()[1]}/v1/models"
  222. # Live-catalog metadata first (ported from PrimeIntellect-ai/prime-agent#1258): OpenRouter's /v1/models
  223. # entries advertise reasoning support via supported_parameters + a reasoning object, which covers every
  224. # routed vendor without a hand-maintained prefix list. The static prefix allowlist repeatedly went
  225. # stale one vendor at a time (nvidia/ missing → #75386; same class as tencent/, xiaomi/ additions before
  226. # it) — metadata makes new vendors work without a code change. One catalog fetch per process, cached;
  227. # unknown (catalog unreachable / unlisted model) falls back to the static list.
  228. def openrouter_model_reasoning_capabilities(
  229. model_id: Optional[str], *, timeout: float = 6.0, allow_fetch: bool = False,
  230. ) -> Optional[dict[str, Any]]:
  231. """Live-catalog reasoning capabilities for an OpenRouter model (tri-state, see module doc).
  232. CACHE-ONLY by default — safe on per-request hot paths (never blocks on HTTP)."""
  233. return _model_caps(_OPENROUTER_CAPS, model_id, timeout=timeout, allow_fetch=allow_fetch)
  234. def nous_model_reasoning_capabilities(
  235. model_id: Optional[str], *, timeout: float = 6.0, allow_fetch: bool = False,
  236. ) -> Optional[dict[str, Any]]:
  237. """Nous Portal counterpart of :func:`openrouter_model_reasoning_capabilities`; warm the cache
  238. with :func:`warm_nous_reasoning_caps_async` from hot paths."""
  239. return _model_caps(_NOUS_CAPS, model_id, timeout=timeout, allow_fetch=allow_fetch)
  240. def warm_openrouter_reasoning_caps_async() -> None:
  241. """Warm the OpenRouter reasoning-capability cache in the background."""
  242. _warm_caps_async(_OPENROUTER_CAPS)
  243. def warm_nous_reasoning_caps_async() -> None:
  244. """Nous Portal counterpart of :func:`warm_openrouter_reasoning_caps_async`."""
  245. _warm_caps_async(_NOUS_CAPS)