models_dev.py 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032
  1. """Models.dev registry integration — primary database for providers and models.
  2. Resolution: in-memory cache (fresh, or stale served while one background daemon
  3. thread refreshes) → disk cache (~/.hermes/models_dev_cache.json, any age) →
  4. network only when no cache exists. Failed refreshes back off 5 min process-wide.
  5. Refreshes use ETag conditional GET when a servable registry is held. Hot paths
  6. pass ``allow_network=False`` and never do I/O. A corrupt/empty disk cache is
  7. quarantined, never served as ``{}``. ``models_dev.url`` in config.yaml = mirror."""
  8. import contextlib
  9. import json
  10. import logging
  11. import re
  12. import threading
  13. import time
  14. from dataclasses import dataclass
  15. from pathlib import Path
  16. from typing import Any, Dict, List, Optional, Tuple
  17. from utils import atomic_json_write, atomic_write_text
  18. from hermes_constants import openrouter_variant_base
  19. import requests
  20. logger = logging.getLogger(__name__)
  21. MODELS_DEV_URL = "https://models.dev/api.json"
  22. _MODELS_DEV_CACHE_TTL = 4 * 3600 # 4 hours — ETag conditional GET makes refresh cheap
  23. _MODELS_DEV_RETRY_DELAY = 300 # 5 minutes after a failed refresh
  24. _MODELS_DEV_FETCH_DEADLINE = 15.0 # whole download; ~600 KB gzipped fits at ~40 KB/s
  25. # In-memory cache
  26. _models_dev_cache: dict[str, Any] = {}
  27. _models_dev_cache_time: float = 0
  28. _models_dev_retry_after: float = 0
  29. _models_dev_fetch_lock = threading.Lock()
  30. _models_dev_refresh_lock = threading.Lock()
  31. _models_dev_refresh_in_flight = False
  32. _models_dev_download_slot = threading.Lock() # held by the one live download, abandoned or not
  33. @dataclass
  34. class ModelInfo:
  35. """Full metadata for a single model from models.dev."""
  36. id: str
  37. name: str
  38. family: str
  39. provider_id: str # models.dev provider ID (e.g. "anthropic")
  40. # Capabilities
  41. reasoning: bool = False
  42. tool_call: bool = False
  43. attachment: bool = False # supports image/file attachments (vision)
  44. temperature: bool = False
  45. structured_output: bool = False
  46. open_weights: bool = False
  47. # Modalities
  48. input_modalities: tuple[str, ...] = () # ("text", "image", "pdf", ...)
  49. output_modalities: tuple[str, ...] = ()
  50. # Limits
  51. context_window: int = 0
  52. max_output: int = 0
  53. max_input: Optional[int] = None
  54. # Cost (per million tokens, USD)
  55. cost_input: float = 0.0
  56. cost_output: float = 0.0
  57. cost_cache_read: Optional[float] = None
  58. cost_cache_write: Optional[float] = None
  59. # Metadata
  60. knowledge_cutoff: str = ""
  61. release_date: str = ""
  62. status: str = "" # "alpha", "beta", "deprecated", or ""
  63. interleaved: Any = False # True or {"field": "reasoning_content"}
  64. def has_cost_data(self) -> bool:
  65. return self.cost_input > 0 or self.cost_output > 0
  66. def supports_vision(self) -> bool:
  67. return self.attachment or "image" in self.input_modalities
  68. def supports_pdf(self) -> bool:
  69. return "pdf" in self.input_modalities
  70. def supports_audio_input(self) -> bool:
  71. return "audio" in self.input_modalities
  72. def format_capabilities(self) -> str:
  73. """Human-readable capabilities, e.g. 'reasoning, tools, vision, PDF'."""
  74. flags = (
  75. (self.reasoning, "reasoning"), (self.tool_call, "tools"), (self.supports_vision(), "vision"), (self.supports_pdf(), "PDF"),
  76. (self.supports_audio_input(), "audio"), (self.structured_output, "structured output"), (self.open_weights, "open weights"),
  77. )
  78. return ", ".join(label for on, label in flags if on) or "basic"
  79. @dataclass
  80. class ProviderInfo:
  81. """Full metadata for a provider from models.dev."""
  82. id: str # models.dev provider ID
  83. name: str # display name
  84. env: tuple[str, ...] # env var names for API key
  85. api: str # base URL
  86. doc: str = "" # documentation URL
  87. model_count: int = 0
  88. @dataclass
  89. class ModelCapabilities:
  90. """Structured capability metadata for a model from models.dev."""
  91. supports_tools: bool = True
  92. supports_vision: Optional[bool] = None
  93. supports_reasoning: Optional[bool] = None
  94. context_window: int = 200000
  95. max_output_tokens: Optional[int] = None
  96. model_family: str = ""
  97. # Hermes provider names → models.dev provider IDs
  98. PROVIDER_TO_MODELS_DEV: dict[str, str] = {
  99. "openrouter": "openrouter", "novita": "novita-ai", "anthropic": "anthropic",
  100. "openai": "openai", "openai-api": "openai", "openai-codex": "openai", "zai": "zai",
  101. "kimi": "kimi-for-coding", "kimi-coding": "kimi-for-coding",
  102. "moonshot": "kimi-for-coding", "stepfun": "stepfun",
  103. "kimi-coding-cn": "kimi-for-coding", "minimax": "minimax",
  104. "minimax-oauth": "minimax", "minimax-cn": "minimax-cn", "deepseek": "deepseek",
  105. "alibaba": "alibaba", "qwen-oauth": "alibaba",
  106. "ai-gateway": "vercel", "opencode-zen": "opencode",
  107. "opencode-go": "opencode-go",
  108. "kilocode": "kilo", "fireworks": "fireworks-ai",
  109. "huggingface": "huggingface", "gemini": "google", "google": "google",
  110. "xai": "xai",
  111. "xai-oauth": "xai", # OAuth is a transport path for the same xAI catalog
  112. "xiaomi": "xiaomi", "nvidia": "nvidia",
  113. # Meta Model API (Muse Spark, api.meta.ai): models.dev keys it "meta", the
  114. # Hermes provider is "meta-ai"; both aliases are needed or muse-spark-*
  115. # falls back to the generic 256K default instead of its true 1M window.
  116. "meta-ai": "meta", "meta": "meta", "groq": "groq", "mistral": "mistral",
  117. "togetherai": "togetherai", "perplexity": "perplexity", "cohere": "cohere",
  118. "ollama-cloud": "ollama-cloud",
  119. }
  120. # Reverse mapping: models.dev id → Hermes ids (built lazily; many-to-one).
  121. _MODELS_DEV_TO_PROVIDER: Optional[dict[str, list[str]]] = None
  122. def _profile_models_dev_ids() -> dict[str, str]:
  123. """``{name: models_dev_id}`` declared by registered provider profiles the static map lacks.
  124. Read at lookup time (never merged into ``PROVIDER_TO_MODELS_DEV``)."""
  125. from providers import list_providers
  126. return {p.name: p.models_dev_id for p in list_providers()
  127. if p.models_dev_id and p.name not in PROVIDER_TO_MODELS_DEV}
  128. def models_dev_provider_map() -> dict[str, str]:
  129. """Hermes provider id → models.dev id: ``PROVIDER_TO_MODELS_DEV``, then profile ``models_dev_id``s."""
  130. return {**PROVIDER_TO_MODELS_DEV, **_profile_models_dev_ids()}
  131. def models_dev_id_for(provider: str) -> Optional[str]:
  132. """models.dev id for an exact Hermes provider id (static map first, then its profile), or None."""
  133. key = (provider or "").strip()
  134. if not key or key in PROVIDER_TO_MODELS_DEV:
  135. return PROVIDER_TO_MODELS_DEV.get(key)
  136. return _profile_models_dev_ids().get(key)
  137. def _models_dev_to_hermes_ids(mdev_id: str) -> list[str]:
  138. """Return the Hermes provider ids that map to *mdev_id* (may be [])."""
  139. global _MODELS_DEV_TO_PROVIDER
  140. if _MODELS_DEV_TO_PROVIDER is None:
  141. _MODELS_DEV_TO_PROVIDER = {}
  142. for hermes_id, mapped in PROVIDER_TO_MODELS_DEV.items():
  143. _MODELS_DEV_TO_PROVIDER.setdefault(mapped, []).append(hermes_id)
  144. profile_ids = [name for name, mapped in _profile_models_dev_ids().items() if mapped == mdev_id]
  145. return [*_MODELS_DEV_TO_PROVIDER.get(mdev_id, []), *profile_ids]
  146. def _dict_or_empty(value: Any) -> dict[str, Any]:
  147. return value if isinstance(value, dict) else {}
  148. def _configured_catalog_provider(
  149. provider: str, *, config: Optional[dict[str, Any]] = None,
  150. ) -> Optional[str]:
  151. """``catalog_provider`` declared on a custom provider's ``providers.<name>`` row (or legacy
  152. ``custom_providers[]`` entry): the catalogued vendor whose models it resells. None when unset."""
  153. name = (provider or "").strip()
  154. if name.lower().startswith("custom:"):
  155. name = name[len("custom:"):]
  156. if not name or models_dev_id_for(name) is not None:
  157. return None
  158. provider_config = (
  159. _cfg_get("providers", name, default=None, config=config)
  160. if config is not None
  161. else _cfg_get("providers", name, default=None)
  162. )
  163. alias = _dict_or_empty(provider_config).get("catalog_provider")
  164. if not alias:
  165. legacy = (
  166. _cfg_get("custom_providers", default=None, config=config)
  167. if config is not None
  168. else _cfg_get("custom_providers", default=None)
  169. )
  170. alias = next((e.get("catalog_provider") for e in (legacy if isinstance(legacy, list) else [])
  171. if isinstance(e, dict) and str(e.get("name") or "").strip() == name), None)
  172. alias = str(alias or "").strip()
  173. return alias or None
  174. def _models_dev_id(
  175. provider: str, *, config: Optional[dict[str, Any]] = None,
  176. ) -> Optional[str]:
  177. """models.dev provider id for a Hermes provider id, or None. A custom provider reaches the
  178. catalog only through its configured ``catalog_provider`` alias (#112649)."""
  179. key = (provider or "").strip()
  180. mdev_id = models_dev_id_for(key)
  181. if mdev_id is None:
  182. alias = _configured_catalog_provider(key, config=config)
  183. mdev_id = (models_dev_id_for(alias) or alias) if alias else None
  184. if mdev_id is not None and mdev_id not in models_dev_provider_map().values() \
  185. and mdev_id not in fetch_models_dev(allow_network=False):
  186. # A mistyped alias must not leak into ModelInfo.provider_id; the row stays on its own slug.
  187. if (key, alias) not in _UNKNOWN_CATALOG_PROVIDER_WARNED:
  188. _UNKNOWN_CATALOG_PROVIDER_WARNED.add((key, alias))
  189. logger.warning("providers.%s: catalog_provider %r is neither a Hermes provider id nor a "
  190. "models.dev id; ignoring", key, alias)
  191. mdev_id = None
  192. return mdev_id
  193. _UNKNOWN_CATALOG_PROVIDER_WARNED: set = set() # (provider, alias) warned once per process
  194. def _cfg_get(*keys: str, default: Any, config: Optional[dict[str, Any]] = None) -> Any:
  195. """``cfg_get`` over the read-only config; *default* on any failure."""
  196. try:
  197. from hermes_cli.config import cfg_get, load_config_readonly
  198. return cfg_get(config if config is not None else load_config_readonly(), *keys, default=default)
  199. except Exception:
  200. return default
  201. def _hermes_path(name: str) -> Path:
  202. from hermes_constants import get_hermes_home
  203. return get_hermes_home() / name
  204. def _get_cache_path() -> Path:
  205. return _hermes_path("models_dev_cache.json")
  206. def _get_etag_path() -> Path:
  207. return _hermes_path("models_dev_cache.etag")
  208. def _quietly(what: str, fn, default=None):
  209. """Run *fn*; on any exception log ``"Failed to <what>: %s"`` at debug and return *default*."""
  210. try:
  211. return fn()
  212. except Exception as e:
  213. logger.debug("Failed to %s: %s", what, e)
  214. return default
  215. def _load_etag() -> str:
  216. """Last-known ETag from disk, or "" if missing."""
  217. return _quietly("load models.dev ETag", lambda: _get_etag_path().read_text(encoding="utf-8-sig").strip() if _get_etag_path().exists() else "", "")
  218. def _save_etag(etag: str) -> None:
  219. def write() -> None:
  220. etag_path = _get_etag_path()
  221. from hermes_constants import mkdir_under_hermes_home
  222. mkdir_under_hermes_home(etag_path.parent)
  223. atomic_write_text(etag_path, etag)
  224. _quietly("save models.dev ETag", write)
  225. def _clear_etag() -> None:
  226. """Delete the ETag sidecar so the next fetch is unconditional: an If-None-Match without a
  227. servable cache invites a 304 that leaves the process with no data at all."""
  228. _quietly("clear models.dev ETag", lambda: _get_etag_path().unlink(missing_ok=True))
  229. def _get_models_dev_url() -> str:
  230. """The models.dev API URL, honoring the ``models_dev.url`` config override."""
  231. url = _cfg_get("models_dev", "url", default="")
  232. # Module global (not a captured constant) so patching MODELS_DEV_URL works.
  233. return url.strip() if isinstance(url, str) and url.strip() else MODELS_DEV_URL
  234. def _validate_registry(data: Any) -> bool:
  235. """True if *data* is a non-empty dict suitable for serving."""
  236. return isinstance(data, dict) and len(data) > 0
  237. def _load_disk_cache() -> dict[str, Any]:
  238. """Load the disk cache; a corrupt/empty one is quarantined with a warning so it never
  239. masquerades as ``{}`` and breaks provider/model resolution."""
  240. try:
  241. cache_path = _get_cache_path()
  242. if cache_path.exists():
  243. with open(cache_path, encoding="utf-8-sig") as f:
  244. data = json.load(f)
  245. if _validate_registry(data):
  246. return data
  247. logger.warning("models.dev disk cache is corrupt or empty; quarantining (will refetch from network)")
  248. _quarantine_corrupt_cache(cache_path)
  249. except Exception as e:
  250. logger.warning("Failed to load models.dev disk cache; quarantining: %s", e)
  251. with contextlib.suppress(Exception):
  252. _quarantine_corrupt_cache(_get_cache_path())
  253. return {}
  254. def _quarantine_corrupt_cache(cache_path: Path) -> None:
  255. """Rename a rejected cache aside and drop its ETag sidecar. Renaming makes the rejection a
  256. one-time event — otherwise every hot-path call that finds the in-memory cache empty re-parses
  257. and re-warns until a network fetch succeeds. The sidecar vouches for a registry we no longer hold."""
  258. try:
  259. cache_path.rename(cache_path.with_suffix(".json.corrupt"))
  260. except Exception as e:
  261. logger.debug("Could not quarantine corrupt models.dev cache: %s", e)
  262. _clear_etag()
  263. def _disk_cache_age_seconds() -> Optional[float]:
  264. """Age of the disk cache file in seconds, or None if missing/unreadable. An mtime in the future
  265. (clock skew) is also None — unknown freshness — so callers fall through to the network."""
  266. def stat() -> Optional[float]:
  267. cache_path = _get_cache_path()
  268. age = time.time() - cache_path.stat().st_mtime if cache_path.exists() else -1
  269. return age if age >= 0 else None
  270. return _quietly("stat models.dev disk cache", stat)
  271. def _save_disk_cache(data: dict[str, Any], etag: str = "") -> None:
  272. """Save the registry atomically, plus the ETag sidecar when non-empty."""
  273. _quietly("save models.dev disk cache", lambda: atomic_json_write(_get_cache_path(), data, indent=None, separators=(",", ":")))
  274. if etag:
  275. _save_etag(etag)
  276. # Network refresh: all state mutation happens under _models_dev_fetch_lock.
  277. class _NotModified(Exception):
  278. """Server returned 304 Not Modified — existing cache is still valid."""
  279. def _fetch_models_dev_from_network(*, conditional: bool = False) -> tuple[dict[str, Any], str]:
  280. """Fetch the live registry; returns ``(registry, etag)`` (etag "" if none). Raises on network
  281. errors and on an empty/invalid payload. ``conditional`` sends ``If-None-Match`` with the sidecar's
  282. ETag and raises ``_NotModified`` on 304 — pass True ONLY while holding ``_models_dev_fetch_lock``
  283. AND a servable registry, or a 304 leaves the process with no data."""
  284. from agent.deadline import DeadlineExpired, run_bounded_sync
  285. headers: dict[str, str] = {}
  286. if conditional and (etag := _load_etag()):
  287. headers["If-None-Match"] = etag
  288. url = _get_models_dev_url()
  289. # A download abandoned at the deadline keeps its thread and socket until the server stops; it
  290. # owns this slot until then, so a later attempt (force_refresh skips the backoff) fails fast
  291. # instead of stacking a second transfer on the same slow link.
  292. if not _models_dev_download_slot.acquire(blocking=False):
  293. raise RuntimeError("an abandoned models.dev download is still running")
  294. started = threading.Event()
  295. def download() -> tuple[int, Any, str]:
  296. started.set()
  297. try:
  298. # (connect, read) bounds each socket op, not the download: a trickling body never trips it.
  299. response = requests.get(url, headers=headers, timeout=(5, 10))
  300. if response.status_code == 304:
  301. return 304, None, ""
  302. response.raise_for_status()
  303. return response.status_code, response.json(), response.headers.get("ETag", "")
  304. finally:
  305. _models_dev_download_slot.release()
  306. # The cold-cache fetch runs inside a caller's request (a fresh home's first session.create
  307. # reaches it through tool availability), so the whole exchange, DNS included, gets one
  308. # wall-clock cap; on expiry the caller falls back to the stale/empty cache and the backoff.
  309. try:
  310. bounded = run_bounded_sync(download, _MODELS_DEV_FETCH_DEADLINE, label="models.dev fetch")
  311. except BaseException:
  312. if not started.is_set(): # the worker never ran (thread start failed), so the slot is ours
  313. _models_dev_download_slot.release()
  314. raise
  315. if bounded.timed_out:
  316. raise DeadlineExpired("models.dev fetch", _MODELS_DEV_FETCH_DEADLINE)
  317. status, data, etag = bounded.value
  318. if status == 304:
  319. raise _NotModified()
  320. if not _validate_registry(data):
  321. raise ValueError("models.dev returned an empty or invalid registry")
  322. return data, etag
  323. def _mark_stale_cache_grace() -> None:
  324. """Give stale cache data a 5-minute in-memory grace before retrying refresh. Only ever moves the
  325. timestamp forward, so a background refresh that completed meanwhile keeps its fresh stamp."""
  326. global _models_dev_cache_time
  327. _models_dev_cache_time = max(_models_dev_cache_time, time.time() - _MODELS_DEV_CACHE_TTL + _MODELS_DEV_RETRY_DELAY)
  328. def _serve_stale(msg: str, *args: Any) -> dict[str, Any]:
  329. """Arm the grace window, kick off a background refresh, return the held cache."""
  330. _mark_stale_cache_grace()
  331. _start_background_refresh_models_dev()
  332. logger.debug(msg, *args)
  333. return _models_dev_cache
  334. def _commit_registry(data: dict[str, Any], *, etag: str = "", where: str) -> None:
  335. """Persist a fetched registry: disk + in-mem + clear backoff. Callers hold ``_models_dev_fetch_lock``
  336. so a failing refresh on one path can never stomp state a succeeding refresh just committed."""
  337. global _models_dev_cache, _models_dev_cache_time, _models_dev_retry_after
  338. _save_disk_cache(data, etag)
  339. _models_dev_cache = data
  340. _models_dev_cache_time = time.time()
  341. _models_dev_retry_after = 0
  342. logger.debug(
  343. "Refreshed models.dev registry (%s): %d providers, %d total models", where, len(data),
  344. sum(len(p.get("models", {})) for p in data.values() if isinstance(p, dict)),
  345. )
  346. def _confirm_cache_not_modified(*, where: str) -> None:
  347. """After a 304: clear backoff and re-mark the held cache fresh (disk is untouched — only the
  348. freshness marker advances). Caller holds the lock."""
  349. global _models_dev_cache_time, _models_dev_retry_after
  350. if not _models_dev_cache:
  351. # Should be unreachable (conditional GETs require a servable cache) but previously caused a
  352. # permanent empty-registry loop: drop the sidecar and arm the backoff rather than marking {} "fresh".
  353. _clear_etag()
  354. _models_dev_retry_after = time.time() + _MODELS_DEV_RETRY_DELAY
  355. logger.warning("models.dev returned 304 but no cached registry is held (%s); "
  356. "cleared ETag sidecar, will refetch unconditionally", where)
  357. return
  358. _models_dev_cache_time = time.time()
  359. _models_dev_retry_after = 0
  360. logger.debug("models.dev registry unchanged (304 Not Modified, %s); cache re-confirmed fresh", where)
  361. def _note_refresh_failure(exc: Exception, *, where: str) -> None:
  362. """Arm the process-wide 5-minute backoff. Caller holds the lock."""
  363. global _models_dev_retry_after
  364. _models_dev_retry_after = time.time() + _MODELS_DEV_RETRY_DELAY
  365. logger.debug("models.dev refresh failed (%s); retry suppressed for %ds: %s", where, _MODELS_DEV_RETRY_DELAY, exc)
  366. def _refresh_locked(where: str) -> Optional[dict[str, Any]]:
  367. """One conditional fetch + state update; caller holds ``_models_dev_fetch_lock``. Returns the
  368. registry to serve, or None when the fetch failed (backoff armed)."""
  369. try:
  370. data, etag = _fetch_models_dev_from_network(conditional=bool(_models_dev_cache))
  371. _commit_registry(data, etag=etag, where=where)
  372. return data
  373. except _NotModified:
  374. _confirm_cache_not_modified(where=where)
  375. return _models_dev_cache
  376. except Exception as e:
  377. _note_refresh_failure(e, where=where)
  378. return None
  379. def _background_refresh_models_dev() -> None:
  380. """Best-effort refresh after serving stale cache data."""
  381. global _models_dev_refresh_in_flight
  382. try:
  383. # Fetch INSIDE the lock, symmetric with the foreground path: the conditional-GET inputs (memory
  384. # cache + sidecar) can't be mutated mid-fetch by a concurrent force_refresh and the two paths
  385. # can't double-download. Hot-path callers never touch this lock.
  386. with _models_dev_fetch_lock:
  387. _refresh_locked("background")
  388. finally:
  389. with _models_dev_refresh_lock:
  390. _models_dev_refresh_in_flight = False
  391. def _start_background_refresh_models_dev() -> None:
  392. """Start one daemon refresh worker if none is running and the failure backoff has elapsed."""
  393. global _models_dev_refresh_in_flight
  394. if time.time() < _models_dev_retry_after:
  395. return
  396. with _models_dev_refresh_lock:
  397. if _models_dev_refresh_in_flight:
  398. return
  399. _models_dev_refresh_in_flight = True
  400. thread = threading.Thread(target=_background_refresh_models_dev, name="models-dev-refresh", daemon=True)
  401. try:
  402. thread.start()
  403. except Exception as e:
  404. # Thread/fd exhaustion: clear the flag so refresh isn't disabled for the rest of the process.
  405. with _models_dev_refresh_lock:
  406. _models_dev_refresh_in_flight = False
  407. logger.debug("Failed to start models.dev refresh thread: %s", e)
  408. def fetch_models_dev(force_refresh: bool = False, *, allow_network: bool = True) -> dict[str, Any]:
  409. """Fetch the models.dev registry (dict keyed by provider ID; {} on failure). Cache hierarchy:
  410. fresh in-memory → stale in-memory (served now, refreshed in one background daemon thread — stale
  411. beats a foreground timeout) → disk of any age (stale triggers the same background refresh) →
  412. singleflight foreground fetch. A failed refresh suppresses automatic refreshes for 5 minutes.
  413. ``force_refresh=True`` bypasses the cache fast paths and the backoff, falling back to cached data
  414. only if the call fails. ``allow_network=False`` returns any memory/disk cache and never makes a request."""
  415. global _models_dev_cache, _models_dev_cache_time, _models_dev_retry_after
  416. if not allow_network:
  417. if not _models_dev_cache and (disk_data := _load_disk_cache()):
  418. _models_dev_cache = disk_data
  419. disk_age = _disk_cache_age_seconds()
  420. _models_dev_cache_time = time.time() - disk_age if disk_age is not None else 0
  421. return _models_dev_cache
  422. if not force_refresh:
  423. # Stage 1: fresh in-memory cache — the hot path, no I/O.
  424. if _models_dev_cache and (time.time() - _models_dev_cache_time) < _MODELS_DEV_CACHE_TTL:
  425. return _models_dev_cache
  426. # Stage 2: stale in-memory cache beats blocking on the network.
  427. if _models_dev_cache:
  428. return _serve_stale("Using stale in-memory models.dev cache; refreshing in background")
  429. # Stage 3: disk cache (cold-start only). A stale disk cache is deliberately usable so
  430. # resolution doesn't hang when models.dev is unreachable.
  431. disk_age = _disk_cache_age_seconds()
  432. if disk_age is not None and (disk_data := _load_disk_cache()):
  433. _models_dev_cache = disk_data
  434. if disk_age >= _MODELS_DEV_CACHE_TTL:
  435. return _serve_stale("Using stale models.dev disk cache (age=%.0fs); refreshing in background", disk_age)
  436. # Anchor the in-mem TTL to the file's age so an aging cache isn't extended by another full TTL.
  437. _models_dev_cache_time = time.time() - disk_age
  438. logger.debug("Loaded models.dev from fresh disk cache (%d providers, age=%.0fs)", len(disk_data), disk_age)
  439. return _models_dev_cache
  440. # Process-wide backoff: don't make every caller retry an unreachable endpoint while no usable cache exists.
  441. if time.time() < _models_dev_retry_after:
  442. return _models_dev_cache
  443. # Stage 4: singleflight foreground fetch. Recheck state under the lock — another caller may
  444. # have refreshed or armed the backoff while we waited.
  445. with _models_dev_fetch_lock:
  446. if not force_refresh and (_models_dev_cache or time.time() < _models_dev_retry_after):
  447. return _models_dev_cache
  448. # Cold force_refresh: stages 1-3 were skipped, so hydrate memory from disk first so the
  449. # conditional GET fires and a 304 can re-confirm it.
  450. if force_refresh and not _models_dev_cache and (disk := _load_disk_cache()):
  451. _models_dev_cache = disk
  452. _models_dev_cache_time = 0 # servable but not fresh
  453. served = _refresh_locked("foreground")
  454. if served is not None:
  455. return served
  456. # Stage 5: network failed — serve any stale memory/disk cache. Freshness stays expired;
  457. # the retry-after timestamp gates the next attempt.
  458. if not _models_dev_cache:
  459. _models_dev_cache = _load_disk_cache()
  460. _models_dev_cache_time = 0
  461. if _models_dev_cache:
  462. logger.debug("Loaded stale models.dev disk cache (%d providers)", len(_models_dev_cache))
  463. return _models_dev_cache
  464. def _registry_provider(mdev_id: str, allow_network: bool) -> Optional[dict[str, Any]]:
  465. """The raw models.dev provider entry, or None."""
  466. # Keep the zero-argument call on the allow_network path: dozens of test sites monkeypatch fetch_models_dev with zero-arg lambdas.
  467. registry = fetch_models_dev() if allow_network else fetch_models_dev(allow_network=False)
  468. provider_data = registry.get(mdev_id)
  469. return provider_data if isinstance(provider_data, dict) else None
  470. def _registry_models(mdev_id: str, *, allow_network: bool) -> Optional[dict[str, Any]]:
  471. """The ``models`` dict of a models.dev provider entry, or None."""
  472. provider_data = _registry_provider(mdev_id, allow_network)
  473. models = provider_data.get("models", {}) if provider_data is not None else None
  474. return models if isinstance(models, dict) else None
  475. def _get_provider_models(
  476. provider: str, *, allow_network: bool = False, config: Optional[dict[str, Any]] = None,
  477. ) -> Optional[dict[str, Any]]:
  478. """Resolve a Hermes provider ID to its models dict, or None if unknown.
  479. ``allow_network`` defaults to False — hot-path callers must never block."""
  480. mdev_id = _models_dev_id(provider, config=config)
  481. return _registry_models(mdev_id, allow_network=allow_network) if mdev_id else None
  482. _OPENROUTER_CATALOG_PROVIDERS = frozenset({"openrouter"})
  483. def _openrouter_catalog_lookup_base(provider: str, model: str) -> Optional[str]:
  484. """Return the base id to retry a catalog lookup with, or ``None``.
  485. OpenRouter's ``:nitro`` / ``:floor`` / ``:exacto`` / ``:online`` are
  486. request-time routing modifiers: they change which endpoint serves the
  487. request, never which model runs. ``/models`` and models.dev list only the
  488. base id, so a routed id must resolve to the base model's metadata.
  489. Scoped to OpenRouter so a genuine ``model:tag`` on another provider (an
  490. Ollama tag, a ``:cloud`` catalog key) is never rewritten.
  491. Deliberately excludes ``:free``, ``:batch``, ``:extended``, and
  492. ``:thinking``: those are REAL catalog SKUs with their own entries and
  493. their own — sometimes different — context windows. Stripping them would
  494. report a window LARGER than the model actually has. Their real entries
  495. are found by the exact/case-insensitive passes above, and a genuinely
  496. absent SKU must miss so ``model_overrides`` ``_default`` fill-gap
  497. semantics still apply.
  498. """
  499. if provider not in _OPENROUTER_CATALOG_PROVIDERS:
  500. return None
  501. return openrouter_variant_base(model)
  502. def _iter_model_entries(
  503. models: dict[str, Any], model: str, *, suffix_fallback: bool = True, provider: str = ""
  504. ):
  505. """Yield ``(model_id, entry)`` candidates: exact, case-insensitive, then (optionally)
  506. ``:cloud``/``-cloud`` suffixed forms. Suffix fallback: some providers (ollama-cloud) store
  507. ``kimi-k2.6:cloud`` while the live API returns the bare name; without it context lookup falls to
  508. stale OpenRouter metadata and trips the 64k minimum-context guard. Every consumer shares this
  509. order so a suffix-keyed catalog model counts as KNOWN for ``model_overrides`` fill-gap ``_default``.
  510. ``provider`` enables the OpenRouter routing-variant fallback as a LAST
  511. resort — after exact, case-insensitive, and ``:cloud`` matching — so a
  512. real catalog SKU always wins over its base.
  513. """
  514. for name in ([model] + [model + suffix for suffix in (":cloud", "-cloud")] if suffix_fallback else [model]):
  515. entry = models.get(name)
  516. if isinstance(entry, dict):
  517. yield name, entry
  518. name_lower = name.lower()
  519. for mid, mdata in models.items():
  520. if mid.lower() == name_lower and isinstance(mdata, dict):
  521. yield mid, mdata
  522. routed_base = _openrouter_catalog_lookup_base(provider, model)
  523. if routed_base is not None:
  524. # Recursion is bounded: the base never carries a recognized variant suffix,
  525. # and provider is cleared so the retry cannot loop.
  526. yield from _iter_model_entries(
  527. models, routed_base, suffix_fallback=suffix_fallback, provider=""
  528. )
  529. def _find_model_entry(
  530. models: dict[str, Any], model: str, provider: str = ""
  531. ) -> Optional[dict[str, Any]]:
  532. """First catalog entry for *model* (exact, case-insensitive, suffix), or None."""
  533. return next((entry for _mid, entry in _iter_model_entries(models, model, provider=provider)), None)
  534. def _extract_limit(entry: Any, key: str) -> Optional[int]:
  535. """Positive int ``entry.limit[key]`` or None (audio/image models have context=0)."""
  536. value = _dict_or_empty(_dict_or_empty(entry).get("limit")).get(key)
  537. return int(value) if isinstance(value, (int, float)) and value > 0 else None
  538. def _extract_context(entry: dict[str, Any]) -> Optional[int]:
  539. """Context length from a models.dev model entry, or None if invalid/zero."""
  540. return _extract_limit(entry, "context")
  541. def lookup_models_dev_context(provider: str, model: str, *, allow_network: bool = False) -> Optional[int]:
  542. """Context window in tokens for provider+model, or None if not found. An EXPLICIT ``model_overrides``
  543. entry wins over the catalog; ``_default`` fills the gap only when the catalog has no answer (the
  544. self-unblock path for wrong/missing context in models.dev). Catalog entries with context=0 are
  545. skipped in favour of later candidates. ``allow_network`` defaults to False — runs every turn.
  546. See #84482.
  547. """
  548. override_ctx = _override_context_window(provider, model)
  549. if override_ctx is not None:
  550. return override_ctx
  551. declared_ctx = _override_int(_provider_model_capabilities(provider, model), "context_window")
  552. if declared_ctx is not None:
  553. return declared_ctx
  554. models = _get_provider_models(provider, allow_network=allow_network)
  555. catalog_ctx = next((ctx for _mid, entry in _iter_model_entries(models, model, provider=provider) if (ctx := _extract_context(entry))), None) if models is not None else None
  556. return catalog_ctx if catalog_ctx is not None else _default_override_context(provider)
  557. # Per-model overrides (config.yaml → model_overrides). Canonical schema (the ONLY key space consumers
  558. # accept): context_window, supports_tools, supports_vision, supports_reasoning,
  559. # model_family. ``<provider>.<model_id>`` is an explicit partial patch that always wins over the
  560. # catalog. ``<provider>._default`` / top-level ``_default`` are FILL-GAP defaults: they apply ONLY to
  561. # models the catalog does not know and never displace catalog data. Provider keys accept the Hermes
  562. # or models.dev id; model ids match exactly, then case-insensitively (mirroring catalog lookup).
  563. # Resolution semantics: 1. 2. See #84482, #8731.
  564. _OVERRIDE_WARNED_KEYS: set = set()
  565. # Safe defaults for models absent from the catalog (tools on, 200K context). Capability fields and the
  566. # output limit stay absent so consumers see them as unknown instead of a synthesized value.
  567. _UNKNOWN_MODEL_BASE: dict[str, Any] = {"limit": {"context": 200000}, "tool_call": True}
  568. # Account-gated models may be usable before models.dev has indexed them. Keep
  569. # their capabilities available for an explicitly selected/discovered model
  570. # without adding them to any picker catalog.
  571. _DEEPSEEK_FLASH_VISION: dict[str, Any] = {
  572. "limit": {"context": 1_000_000, "output": 384_000},
  573. "modalities": {"input": ["text", "image"], "output": ["text"]},
  574. "tool_call": True,
  575. "reasoning": True,
  576. "family": "deepseek-flash",
  577. }
  578. _BUILTIN_MODEL_METADATA: dict[tuple[str, str], dict[str, Any]] = {
  579. ("openai", "gpt-6-astra"): {
  580. "limit": {"context": 1_050_000, "output": 128_000},
  581. "modalities": {"input": ["text", "image"], "output": ["text"]},
  582. "tool_call": True,
  583. "reasoning": True,
  584. "family": "gpt-6",
  585. },
  586. # Native DeepSeek V4.1-Flash is multimodal (https://api-docs.deepseek.com/guides/vision).
  587. # models.dev lagged the 2026-09-10 rename; without this, a cold/empty cache treats
  588. # ``deepseek-flash`` as unknown → image_input_mode falls through to lossy text.
  589. # ``deepseek-v4-pro`` stays catalog-only: vendor docs still mark it text-only.
  590. ("deepseek", "deepseek-flash"): _DEEPSEEK_FLASH_VISION,
  591. ("deepseek", "deepseek-v4-flash"): _DEEPSEEK_FLASH_VISION,
  592. ("deepseek", "deepseek-v4.1-flash"): _DEEPSEEK_FLASH_VISION,
  593. ("deepseek", "deepseek-v4-flash-vision-exp"): _DEEPSEEK_FLASH_VISION,
  594. }
  595. def _load_model_overrides(*, config: Optional[dict[str, Any]] = None) -> dict[str, Any]:
  596. """The ``model_overrides`` config section ({} on any failure). Deliberately not memoized:
  597. ``load_config_readonly()`` is already (mtime, size)-cached upstream, and an ``id(cfg)``-keyed
  598. layer can serve stale overrides after a reload when CPython reuses the dict address."""
  599. overrides = (
  600. _cfg_get("model_overrides", default={}, config=config)
  601. if config is not None
  602. else _cfg_get("model_overrides", default={})
  603. )
  604. return _dict_or_empty(overrides)
  605. def _provider_override_section(provider: str, *, config: Optional[dict[str, Any]] = None) -> Optional[dict[str, Any]]:
  606. """Override section for *provider* (keyed by Hermes OR models.dev id), or None."""
  607. overrides = (
  608. _load_model_overrides(config=config)
  609. if config is not None
  610. else _load_model_overrides()
  611. )
  612. provider_key = (provider or "").strip()
  613. if not overrides or not provider_key:
  614. return None
  615. # Forward (Hermes → models.dev id) and reverse (caller passed a models.dev id, config keyed by Hermes id) aliases.
  616. candidates = [provider_key, models_dev_id_for(provider_key), *_models_dev_to_hermes_ids(provider_key)]
  617. return next((section for section in (overrides.get(key) if key else None for key in candidates) if isinstance(section, dict)), None)
  618. def _explicit_model_override(provider: str, model: str, *, config: Optional[dict[str, Any]] = None) -> Optional[dict[str, Any]]:
  619. """Explicit per-provider+model override dict (exact, then case-insensitive skipping the ``_default`` sentinel), or None."""
  620. model_key = (model or "").strip()
  621. section = _provider_override_section(provider, config=config) if model_key else None
  622. if section is None:
  623. return None
  624. entry = section.get(model_key)
  625. if isinstance(entry, dict):
  626. return entry
  627. model_lower = model_key.lower()
  628. return next((mdata for mid, mdata in section.items() if mid != "_default" and mid.lower() == model_lower and isinstance(mdata, dict)), None)
  629. def _default_model_override(provider: str, *, config: Optional[dict[str, Any]] = None) -> Optional[dict[str, Any]]:
  630. """Fill-gap ``_default`` override: per-provider first, then global; or None."""
  631. section = _provider_override_section(provider, config=config)
  632. if section is not None and isinstance(section.get("_default"), dict):
  633. return section["_default"]
  634. overrides = (
  635. _load_model_overrides(config=config)
  636. if config is not None
  637. else _load_model_overrides()
  638. )
  639. global_default = overrides.get("_default")
  640. return global_default if isinstance(global_default, dict) else None
  641. def _override_for(
  642. provider: str, model: str, *, catalog_hit: bool, config: Optional[dict[str, Any]] = None,
  643. ) -> Optional[dict[str, Any]]:
  644. """Explicit override if any; else the ``_default`` only on a catalog miss."""
  645. explicit = _explicit_model_override(provider, model, config=config)
  646. return explicit if explicit is not None or catalog_hit else _default_model_override(provider, config=config)
  647. def _override_int(override: dict[str, Any], key: str) -> Optional[int]:
  648. """Coerce an override field to a positive int, warning once on garbage."""
  649. raw = override.get(key)
  650. if raw is None:
  651. return None
  652. with contextlib.suppress(TypeError, ValueError):
  653. value = int(raw)
  654. if value > 0:
  655. return value
  656. warn_key = (key, repr(raw))
  657. if warn_key not in _OVERRIDE_WARNED_KEYS:
  658. _OVERRIDE_WARNED_KEYS.add(warn_key)
  659. logger.warning("model_overrides: ignoring invalid %s value %r (expected a positive integer)", key, raw)
  660. return None
  661. def _override_context_window(provider: str, model: str) -> Optional[int]:
  662. """EXPLICITLY overridden context_window, or None. Explicit-only on purpose: this runs early in the
  663. resolution chain (agent/model_metadata.py, before custom_providers and live probes) where a
  664. ``_default`` must not preempt more specific sources; fill-gap defaults apply in ``lookup_models_dev_context``."""
  665. ov = _explicit_model_override(provider, model)
  666. return _override_int(ov, "context_window") if ov is not None else None
  667. # Catalog miss — a _default override may fill the gap (#84482).
  668. def _default_override_context(provider: str) -> Optional[int]:
  669. """Fill-gap context from a ``_default`` override, for catalog misses."""
  670. default = _default_model_override(provider)
  671. return _override_int(default, "context_window") if default is not None else None
  672. def _override_to_catalog_shape(override: dict[str, Any]) -> tuple[dict[str, Any], Optional[bool]]:
  673. """Translate canonical override keys into a models.dev-shaped patch. Returns ``(patch, vision)`` —
  674. vision is out-of-band because it maps onto the ``modalities.input`` list rather than a scalar field."""
  675. patch: dict[str, Any] = {}
  676. limit = {
  677. catalog_key: value for catalog_key, override_key in (("context", "context_window"),)
  678. if (value := _override_int(override, override_key)) is not None
  679. }
  680. if limit:
  681. patch["limit"] = limit
  682. for override_key, catalog_key in (("supports_tools", "tool_call"), ("supports_reasoning", "reasoning")):
  683. if override_key in override:
  684. patch[catalog_key] = bool(override[override_key])
  685. vision: Optional[bool] = None
  686. if "supports_vision" in override:
  687. vision = patch["attachment"] = bool(override["supports_vision"])
  688. if "model_family" in override:
  689. patch["family"] = str(override["model_family"] or "")
  690. return patch, vision
  691. def _merge_catalog_entry_with_override(raw: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
  692. """Patch a catalog entry with a canonical-schema override. Sub-dicts (``limit``, ``modalities``) are
  693. merged, not clobbered — setting only ``context_window`` must not wipe the catalog's ``limit.output``."""
  694. shaped, vision_override = _override_to_catalog_shape(override)
  695. merged = dict(raw)
  696. limit_patch = shaped.pop("limit", None)
  697. if limit_patch:
  698. merged["limit"] = {**_dict_or_empty(raw.get("limit")), **limit_patch}
  699. if vision_override is not None:
  700. base_mods = dict(_dict_or_empty(raw.get("modalities")))
  701. input_mods = base_mods.get("input")
  702. input_mods = list(input_mods) if isinstance(input_mods, list) else []
  703. if vision_override and "image" not in input_mods:
  704. input_mods.append("image")
  705. elif not vision_override and "image" in input_mods:
  706. input_mods.remove("image")
  707. base_mods["input"] = input_mods
  708. merged["modalities"] = base_mods
  709. merged.update(shaped)
  710. return merged
  711. def _builtin_model_metadata(
  712. provider: str, model: str, *, config: Optional[dict[str, Any]] = None,
  713. ) -> Optional[dict[str, Any]]:
  714. """Built-in metadata for a provider/model pair, if Hermes has a vendor-specific entry."""
  715. provider_key = _models_dev_id(provider, config=config) or (provider or "").strip()
  716. return _BUILTIN_MODEL_METADATA.get((provider_key, (model or "").strip().lower()))
  717. def _relay_vision_marker_metadata(provider: str, model: str) -> Optional[dict[str, Any]]:
  718. """Fill-gap base for an OpenCode Zen/Go ``*-vision*`` model id the catalog does not know. The relays
  719. resell vendor previews (``deepseek-v4-flash-vision-exp``) before models.dev indexes them, and the id's
  720. ``-vision`` token is the vendor's own capability marker; without it ``image_input_mode: auto`` treats
  721. the model as text-only and detours images through the lossy describe path (#96066). Every other field
  722. keeps the unknown-model defaults, so only vision is claimed."""
  723. from hermes_cli.models import opencode_provider_family
  724. if "-vision" not in (model or "").strip().lower() or opencode_provider_family(provider) is None:
  725. return None
  726. return {**_UNKNOWN_MODEL_BASE, "modalities": {"input": ["text", "image"], "output": ["text"]}}
  727. def _provider_model_capabilities(provider: str, model: str) -> dict[str, Any]:
  728. """Exact-model declaration from the registered ``ProviderProfile.model_capabilities`` (canonical
  729. ``model_overrides`` schema). The ONE plugin seam: every consumer that reads models.dev through this
  730. module (picker badges, image routing, ``/api/model/info``, context lookup) sees it (#102115)."""
  731. from providers import get_provider_profile
  732. profile = get_provider_profile(provider)
  733. return profile.model_capabilities.get(model, {}) if profile is not None else {}
  734. def _apply_overrides(
  735. provider: str, model: str, entry: Optional[dict[str, Any]], *, config: Optional[dict[str, Any]] = None,
  736. ) -> Optional[dict[str, Any]]:
  737. """Catalog/builtin metadata, patched by the plugin's declaration, then by the explicit user override.
  738. ``_UNKNOWN_MODEL_BASE`` is the base on a catalog miss; a fill-gap ``_default`` applies only when no
  739. source knows the model. None when nothing knows it."""
  740. builtin = _builtin_model_metadata(provider, model, config=config)
  741. base = entry if entry is not None else builtin
  742. declared = _provider_model_capabilities(provider, model)
  743. if declared:
  744. base = _merge_catalog_entry_with_override(base if base is not None else _UNKNOWN_MODEL_BASE, declared)
  745. override = _override_for(provider, model, catalog_hit=base is not None, config=config)
  746. if base is None:
  747. base = _relay_vision_marker_metadata(provider, model)
  748. return base if override is None else _merge_catalog_entry_with_override(base if base is not None else _UNKNOWN_MODEL_BASE, override)
  749. def _entry_supports_vision(entry: dict[str, Any]) -> bool:
  750. """Prefer explicit ``modalities.input`` (the older ``attachment`` flag can be stale or too broad
  751. for image routing); fall back to it only when the input modalities are absent/invalid."""
  752. input_mods = _dict_or_empty(entry.get("modalities", {})).get("input")
  753. return "image" in input_mods if isinstance(input_mods, list) else bool(entry.get("attachment", False))
  754. def get_model_capabilities(
  755. provider: str, model: str, *, allow_network: bool = False, config: Optional[dict[str, Any]] = None,
  756. ) -> Optional[ModelCapabilities]:
  757. """Capability metadata from the models.dev cache, or None if unresolvable. EXPLICIT ``model_overrides``
  758. patch catalog fields; ``_default`` fills the gap only for models the catalog does not know. Unspecified
  759. fields fall through to the catalog, or to safe defaults. ``allow_network`` defaults to False (hot path).
  760. EXPLICIT ``model_overrides`` entries (per-provider+model) win over catalog values for the fields they
  761. set. ``_default`` entries fill the gap only for models the catalog does not know — the supported
  762. self-unblock path for custom/local models (#8731) and for models with wrong metadata in models.dev
  763. (#84482).
  764. """
  765. models = _get_provider_models(provider, allow_network=allow_network, config=config)
  766. entry = _find_model_entry(models, model, provider) if models is not None else None
  767. unknown_base = (
  768. entry is None
  769. and _builtin_model_metadata(provider, model, config=config) is None
  770. )
  771. raw = _apply_overrides(provider, model, entry, config=config)
  772. if raw is None:
  773. return None
  774. return ModelCapabilities(
  775. supports_tools=bool(raw.get("tool_call", False)),
  776. supports_vision=(
  777. None
  778. if unknown_base and "attachment" not in raw and "modalities" not in raw
  779. else _entry_supports_vision(raw)
  780. ),
  781. supports_reasoning=None if unknown_base and "reasoning" not in raw else bool(raw.get("reasoning", False)),
  782. context_window=_extract_limit(raw, "context") or 200000,
  783. max_output_tokens=_extract_limit(raw, "output"),
  784. model_family=raw.get("family", "") or "",
  785. )
  786. def list_provider_models(provider: str, *, allow_network: bool = True) -> list[str]:
  787. """All model IDs for a provider ([] if unknown). ``allow_network`` defaults to True: the model
  788. picker is interactive and a fresh catalog is worth a short wait."""
  789. from hermes_cli.models import normalize_provider
  790. provider = normalize_provider(provider) or provider
  791. models = _get_provider_models(provider, allow_network=allow_network)
  792. return [mid for mid in models if not _should_hide_from_provider_catalog(provider, mid)] if models is not None else []
  793. # Non-agentic or noise models (TTS, embedding, dated preview snapshots, live/streaming-only, image-only).
  794. _NOISE_PATTERNS: re.Pattern = re.compile(
  795. r"-tts\b|embedding|live-|-(preview|exp)-\d{2,4}[-_]|" r"-image\b|-image-preview\b|-customtools\b", re.IGNORECASE)
  796. # Hidden from the Gemini catalogs surfaced in setup/model selection (capability metadata stays available for direct use).
  797. _GOOGLE_HIDDEN_MODELS = frozenset({
  798. # Low-TPM Gemma models that trip Google input-token quota walls under agent-style traffic.
  799. "gemma-4-31b-it", "gemma-4-26b-it", "gemma-4-26b-a4b-it",
  800. "gemma-3-1b", "gemma-3-1b-it", "gemma-3-2b", "gemma-3-2b-it",
  801. "gemma-3-4b", "gemma-3-4b-it", "gemma-3-12b", "gemma-3-12b-it",
  802. "gemma-3-27b", "gemma-3-27b-it",
  803. # Stale/retired Google slugs that 404 on the current endpoints.
  804. "gemini-1.5-flash", "gemini-1.5-pro", "gemini-1.5-flash-8b",
  805. "gemini-2.0-flash", "gemini-2.0-flash-lite",
  806. })
  807. def _should_hide_from_provider_catalog(provider: str, model_id: str) -> bool:
  808. return (provider or "").strip().lower() in {"gemini", "google"} and (model_id or "").strip().lower() in _GOOGLE_HIDDEN_MODELS
  809. def list_agentic_models(provider: str, *, allow_network: bool = True) -> list[str]:
  810. """Model IDs suitable for agentic use: tool_call=True, minus hidden and noise models. [] on any
  811. failure. ``allow_network`` defaults to True (called from interactive model selection)."""
  812. models = _get_provider_models(provider, allow_network=allow_network)
  813. return [
  814. mid for mid, entry in models.items()
  815. if isinstance(entry, dict) and not _should_hide_from_provider_catalog(provider, mid) and entry.get("tool_call", False) and not _NOISE_PATTERNS.search(mid)
  816. ] if models is not None else []
  817. def _parse_model_info(model_id: str, raw: dict[str, Any], provider_id: str) -> ModelInfo:
  818. """Convert a raw models.dev model entry dict into a ModelInfo dataclass."""
  819. cost = _dict_or_empty(raw.get("cost"))
  820. modalities = _dict_or_empty(raw.get("modalities"))
  821. def _mods(key: str) -> tuple[str, ...]:
  822. mods = modalities.get(key) or []
  823. return tuple(mods) if isinstance(mods, list) else ()
  824. def _cost(key: str) -> Optional[float]:
  825. return float(cost[key]) if cost.get(key) is not None else None
  826. return ModelInfo(
  827. id=model_id, name=raw.get("name", "") or model_id, family=raw.get("family", "") or "", provider_id=provider_id,
  828. **{k: bool(raw.get(k, False)) for k in ("reasoning", "tool_call", "attachment", "temperature", "structured_output", "open_weights")},
  829. input_modalities=_mods("input"), output_modalities=_mods("output"),
  830. context_window=_extract_limit(raw, "context") or 0, max_output=_extract_limit(raw, "output") or 0, max_input=_extract_limit(raw, "input"),
  831. cost_input=float(cost.get("input", 0) or 0), cost_output=float(cost.get("output", 0) or 0),
  832. cost_cache_read=_cost("cache_read"), cost_cache_write=_cost("cache_write"),
  833. knowledge_cutoff=raw.get("knowledge", "") or "", release_date=raw.get("release_date", "") or "",
  834. status=raw.get("status", "") or "", interleaved=raw.get("interleaved", False),
  835. )
  836. def _parse_provider_info(provider_id: str, raw: dict[str, Any]) -> ProviderInfo:
  837. """Convert a raw models.dev provider entry dict into a ProviderInfo."""
  838. env = raw.get("env") or []
  839. models = raw.get("models") or {}
  840. return ProviderInfo(
  841. id=provider_id, name=raw.get("name", "") or provider_id, env=tuple(env) if isinstance(env, list) else (),
  842. api=raw.get("api", "") or "", doc=raw.get("doc", "") or "", model_count=len(models) if isinstance(models, dict) else 0,
  843. )
  844. def get_provider_info(
  845. provider_id: str, *, allow_network: bool = True, config: Optional[dict[str, Any]] = None,
  846. ) -> Optional[ProviderInfo]:
  847. """Provider metadata by Hermes or models.dev ID, or None if not cataloged. ``allow_network`` defaults to True (interactive setup)."""
  848. mdev_id = _models_dev_id(provider_id, config=config) or provider_id
  849. raw = _registry_provider(mdev_id, allow_network)
  850. return _parse_provider_info(mdev_id, raw) if raw is not None else None
  851. def get_model_info(
  852. provider_id: str, model_id: str, *, allow_network: bool = False, config: Optional[dict[str, Any]] = None,
  853. ) -> Optional[ModelInfo]:
  854. """Full model metadata by Hermes or models.dev provider ID (exact match, then case-insensitive), or
  855. None if not found. EXPLICIT ``model_overrides`` patch known catalog models; ``_default`` fills the gap
  856. only for unknown ones. ``allow_network`` defaults to False — cost guard and inventory are hot paths.
  857. ``model_overrides`` entries use the SAME canonical schema as every other consumer (``context_window``,
  858. ``supports_*``, ``model_family``) — they are translated into the catalog shape at
  859. this boundary, and sub-dicts (``limit``, ``modalities``) are merged rather than clobbered. See #84482,
  860. #8731.
  861. """
  862. mdev_id = _models_dev_id(provider_id, config=config) or provider_id
  863. models = _registry_models(mdev_id, allow_network=allow_network)
  864. mid, entry = next(_iter_model_entries(models, model_id, suffix_fallback=False, provider=provider_id), (model_id, None)) if models is not None else (model_id, None)
  865. # Not in catalog — an override (explicit or _default) may still provide it.
  866. raw = _apply_overrides(provider_id, model_id, entry, config=config)
  867. return _parse_model_info(mid, raw, mdev_id) if raw is not None else None