models_validate.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811
  1. """Validate a requested ``/model`` value against the active provider's catalog.
  2. Split out of ``hermes_cli.models``. Catalog fetchers defined in ``hermes_cli.models`` are looked up
  3. there at call time (``_m.<name>``) so ``patch("hermes_cli.models.<name>")`` mocks keep intercepting;
  4. local-server probes are looked up on ``hermes_cli.models_local`` (``_ml.<name>``) the same way.
  5. Every provider branch returns a verdict dict (see :func:`_verdict`) or ``None`` for "not decided
  6. here — keep walking the ladder". The ladder ORDER is behavior (see ``_LADDER``).
  7. """
  8. from __future__ import annotations
  9. import re
  10. from dataclasses import dataclass
  11. from difflib import get_close_matches
  12. from typing import Any, Callable, Optional
  13. from utils import base_url_host_matches
  14. from hermes_constants import openrouter_variant_base
  15. # ── Verdicts ─────────────────────────────────────────────────────────────
  16. def _verdict(accepted: bool, persist: bool, recognized: bool, message: Optional[str]) -> dict[str, Any]:
  17. return {"accepted": accepted, "persist": persist, "recognized": recognized, "message": message}
  18. def _accept() -> dict[str, Any]:
  19. return _verdict(True, True, True, None)
  20. def _accept_with_note(message: str) -> dict[str, Any]:
  21. return _verdict(True, True, True, message)
  22. def _reject(message: str) -> dict[str, Any]:
  23. return _verdict(False, False, False, message)
  24. def _soft_accept(message: Optional[str]) -> dict[str, Any]:
  25. """Accept + persist an unrecognized name, with a warning."""
  26. return _verdict(True, True, False, message)
  27. # ── Catalog matching ─────────────────────────────────────────────────────
  28. @dataclass
  29. class _Match:
  30. exact: bool = False
  31. suggestion_text: str = ""
  32. def verdict(self, req: _Request) -> Optional[dict[str, Any]]:
  33. """Accept on exact membership, else None so the branch composes its own message."""
  34. return _accept() if self.exact else None
  35. def _match_in_catalog(
  36. query: str,
  37. candidates,
  38. *,
  39. case_insensitive: bool = False,
  40. suggest_query: Optional[str] = None,
  41. suggest_cutoff: float = 0.5,
  42. suggest_label: str = "Similar models",
  43. ) -> _Match:
  44. """Shared ladder: exact membership → suggestion text. Never rewrites the id: a requested model
  45. that is merely CLOSE to a catalog entry is the user's selection (a newer release the listing
  46. lacks, a dated snapshot, a qualifier) and goes to the wire verbatim — fuzzy "auto-correction"
  47. swapped `deepseek-v4.1-flash` for `deepseek-v4-flash`, `gemini-3.8-flash` for `gemini-3.6-flash`
  48. and `model:nitro` for `model` under the user's own label. The vendor's 400 names the valid ids.
  49. ``case_insensitive`` matches lower-cased ids and maps results back to the catalog's spelling
  50. (MiniMax ships mixed-case ids). ``suggest_query`` overrides the string the suggestion search
  51. uses (some branches search on the raw request, not the lookup form)."""
  52. pool = list(candidates)
  53. display = None
  54. if suggest_query is None:
  55. suggest_query = query
  56. if case_insensitive:
  57. display = {c.lower(): c for c in pool}
  58. pool = list(display)
  59. query, suggest_query = query.lower(), suggest_query.lower()
  60. def _show(cid: str) -> str:
  61. return display[cid] if display is not None else cid
  62. if query in set(pool):
  63. return _Match(exact=True)
  64. suggestions = get_close_matches(suggest_query, pool, n=3, cutoff=suggest_cutoff)
  65. if not suggestions:
  66. return _Match()
  67. return _Match(suggestion_text=f"\n {suggest_label}: " + ", ".join(f"`{_show(s)}`" for s in suggestions))
  68. # ── Request context ──────────────────────────────────────────────────────
  69. @dataclass
  70. class _Request:
  71. requested: str
  72. lookup: str # id used for catalog membership (profile-normalized / preset base)
  73. provider: Optional[str] # raw caller value (Ollama checks look at this, not ``normalized``)
  74. normalized: str
  75. api_key: Optional[str]
  76. base_url: Optional[str]
  77. api_mode: Optional[str]
  78. headers: Optional[dict[str, str]]
  79. # ── Provider branches (None = not decided here) ─────────────────────────
  80. def _validate_moa(req: _Request) -> dict[str, Any]:
  81. try:
  82. from hermes_cli.config import load_config
  83. from hermes_cli.moa_config import normalize_moa_config
  84. cfg = normalize_moa_config(load_config().get("moa") or {})
  85. if req.requested in cfg["presets"]:
  86. return _accept()
  87. return _reject(f"MoA preset `{req.requested}` was not found. Run `hermes moa list`.")
  88. except Exception as exc:
  89. return _reject(f"Could not read MoA presets: {exc}")
  90. def _reject_whitespace(req: _Request) -> Optional[dict[str, Any]]:
  91. # Cloud catalogs never contain spaces. Self-hosted servers and a user-configured
  92. # base_url do (VLLM "My Custom Model", a local router's "Go reasoning"). The
  93. # step stays here — later branches must not see a cloud id the catalog cannot serve.
  94. if any(ch.isspace() for ch in req.requested) and not _whitespace_allowed(req):
  95. return _reject("Model names cannot contain spaces.")
  96. return None
  97. _SELF_HOSTED_PROVIDERS = frozenset({
  98. "lmstudio", "ollama", "local", "vllm", "llamacpp", "llama.cpp", "llama-cpp",
  99. })
  100. def _provider_token(provider: Optional[str]) -> str:
  101. return str(provider or "").strip().lower()
  102. def _is_self_hosted_provider(provider: Optional[str]) -> bool:
  103. """Local servers and the custom endpoint bucket, including aliases that normalize to it."""
  104. raw = _provider_token(provider)
  105. if not raw:
  106. return False
  107. if raw == "custom" or raw.startswith("custom:"):
  108. return True
  109. from hermes_cli import models as _m
  110. normalized = _m.normalize_provider(raw)
  111. if normalized == "custom" or normalized.startswith("custom:"):
  112. return True
  113. return raw in _SELF_HOSTED_PROVIDERS or normalized in _SELF_HOSTED_PROVIDERS
  114. def _non_public_host(host: str) -> bool:
  115. """Loopback, LAN, and single-label hosts are a user's endpoint, never a vendor catalog."""
  116. host = (host or "").lower().rstrip(".")
  117. if not host or host in {"localhost", "127.0.0.1", "::1", "0.0.0.0"} or host.endswith(".localhost"):
  118. return bool(host)
  119. if host.endswith((".local", ".lan", ".internal", ".home", ".localdomain")):
  120. return True
  121. if "." not in host:
  122. return True
  123. parts = host.split(".")
  124. if len(parts) == 4 and all(part.isdigit() for part in parts):
  125. octets = [int(part) for part in parts]
  126. if octets[0] in {10, 127} or (octets[0] == 192 and octets[1] == 168):
  127. return True
  128. if octets[0] == 172 and 16 <= octets[1] <= 31:
  129. return True
  130. return False
  131. def _stock_host(provider: str) -> str:
  132. """Vendor host for *provider*, offline. Empty when this install cannot name one.
  133. ``get_provider`` can answer with a different id (``openai`` → the OpenRouter
  134. overlay when models.dev is cold). A foreign row is not this provider's stock
  135. endpoint — using it would exempt the real cloud host.
  136. """
  137. from hermes_cli.providers import get_provider
  138. from utils import base_url_hostname
  139. token = _provider_token(provider)
  140. if not token:
  141. return ""
  142. try:
  143. pdef = get_provider(token, allow_network=False)
  144. except Exception:
  145. return ""
  146. if pdef is None or _provider_token(pdef.id) != token:
  147. return ""
  148. return base_url_hostname(pdef.base_url or "")
  149. def provider_allows_model_whitespace(provider: Optional[str], base_url: Optional[str] = None) -> bool:
  150. """True when a spaced id is a real selection, not a cloud-catalog typo.
  151. Self-hosted providers always qualify. A base_url qualifies when the user
  152. configured it: a non-public host, or a public host that is not this
  153. provider's own stock endpoint. A public host with no stock to compare
  154. against stays rejected, so a cloud URL cannot slip through a cold catalog.
  155. """
  156. if _is_self_hosted_provider(provider):
  157. return True
  158. url = str(base_url or "").strip()
  159. if not url:
  160. return False
  161. from utils import base_url_hostname
  162. host = base_url_hostname(url)
  163. if not host:
  164. return False
  165. if _non_public_host(host):
  166. return True
  167. raw = _provider_token(provider)
  168. from hermes_cli import models as _m
  169. normalized = _m.normalize_provider(raw) if raw else ""
  170. stock = _stock_host(normalized or raw)
  171. return bool(stock) and host != stock
  172. def _whitespace_allowed(req: _Request) -> bool:
  173. # The openrouter→custom rewrite lives on ``normalized``; aliases live on the raw value.
  174. if _is_self_hosted_provider(req.normalized) or _is_self_hosted_provider(req.provider):
  175. return True
  176. return provider_allows_model_whitespace(req.provider or req.normalized, req.base_url)
  177. def offered_model_ids(models, provider: Optional[str], base_url: Optional[str] = None) -> list:
  178. """Ids a picker may show. Drops whitespace the validator will refuse; keeps the rest."""
  179. ids = list(models or [])
  180. if provider_allows_model_whitespace(provider, base_url):
  181. return ids
  182. return [model_id for model_id in ids if not (isinstance(model_id, str) and any(ch.isspace() for ch in model_id))]
  183. def drop_unofferable_model_ids(rows: list) -> None:
  184. """In-place: picker rows must not offer an id ``validate_requested_model`` will refuse for whitespace."""
  185. for row in rows:
  186. if not isinstance(row, dict):
  187. continue
  188. provider = row.get("slug")
  189. base_url = row.get("api_url") or row.get("base_url")
  190. models = row.get("models")
  191. if isinstance(models, list):
  192. filtered = offered_model_ids(models, provider, base_url)
  193. removed = len(models) - len(filtered)
  194. if removed:
  195. row["models"] = filtered
  196. total = row.get("total_models")
  197. if isinstance(total, int):
  198. row["total_models"] = max(0, total - removed)
  199. featured = row.get("featured_models")
  200. if isinstance(featured, list):
  201. row["featured_models"] = offered_model_ids(featured, provider, base_url)
  202. def _parse_openrouter_preset(req: _Request) -> Optional[dict[str, Any]]:
  203. """OpenRouter presets are account-scoped, so ``@preset/<slug>`` never appears in the public
  204. /v1/models listing. A bare preset is accepted unverified; ``<model>@preset/<slug>`` validates
  205. the base model; the full id (suffix included) goes to the wire. OpenRouter validates the slug
  206. at request time."""
  207. marker = "@preset/"
  208. if marker not in req.requested:
  209. return None
  210. if req.requested.count(marker) != 1:
  211. preset_slug, preset_base = "", req.requested
  212. else:
  213. preset_base, preset_slug = req.requested.split(marker, 1)
  214. if re.fullmatch(r"[A-Za-z0-9._~-]+", preset_slug) is None:
  215. return _reject("OpenRouter preset slugs must be non-empty URL-safe identifiers using only "
  216. "letters, digits, '.', '_', '~', or '-'.")
  217. if not preset_base:
  218. return _soft_accept(None)
  219. req.lookup = preset_base
  220. return None
  221. def _validate_lmstudio(req: _Request) -> dict[str, Any]:
  222. from hermes_cli import models_local as _ml
  223. from hermes_cli.auth import AuthError
  224. # probe_lmstudio_models distinguishes None (unreachable / malformed) from [] (reachable,
  225. # nothing chat-capable loaded); fetch_lmstudio_models collapses both to [].
  226. try:
  227. models = _ml.probe_lmstudio_models(api_key=req.api_key, base_url=req.base_url)
  228. except AuthError as exc:
  229. return _reject(f"{exc} Set `LM_API_KEY` (or update it) to match the server's bearer token.")
  230. if models is None:
  231. return _reject(f"Could not reach LM Studio's `/api/v1/models` to validate `{req.requested}`.")
  232. if not models:
  233. return _reject("LM Studio is reachable but no chat-capable models are loaded. "
  234. f"Load `{req.requested}` in LM Studio (Developer tab → Load Model) and try again.")
  235. if req.lookup in set(models):
  236. return _accept()
  237. return _reject(f"Model `{req.requested}` was not found in LM Studio's model listing.")
  238. def _ollama_probe_headers(req: _Request) -> dict[str, str]:
  239. """Headers for the Ollama native probe. Configured ``providers.ollama.extra_headers`` apply only
  240. when the probed endpoint is the configured one (never leak them to a different host). Caller
  241. headers win; a caller ``api_key`` becomes the Authorization header unless the caller sent one."""
  242. from hermes_cli import models as _m
  243. from hermes_cli import models_local as _ml
  244. from hermes_cli.models_local import _configured_ollama_base_url, _drop_authorization
  245. configured_base = _configured_ollama_base_url()
  246. configured_allowed = not configured_base or _ml._same_ollama_native_root(req.base_url or "", configured_base)
  247. configured = _m._get_ollama_native_headers(req.base_url, api_key=req.api_key) if configured_allowed else {}
  248. if req.headers is None:
  249. return configured
  250. out = dict(configured)
  251. _drop_authorization(out)
  252. out.update(req.headers)
  253. if req.api_key and not any(key.lower() == "authorization" for key in req.headers):
  254. _drop_authorization(out)
  255. out["Authorization"] = f"Bearer {req.api_key}"
  256. return out
  257. def _validate_ollama_native(req: _Request) -> Optional[dict[str, Any]]:
  258. """Runs for EVERY provider: the native ``/api/tags`` catalog is used whenever the endpoint
  259. looks like a local Ollama server. Also resolves ``base_url`` for the raw ``ollama`` provider,
  260. which later branches (custom) rely on."""
  261. from hermes_cli import models as _m
  262. from hermes_cli import models_local as _ml
  263. if str(req.provider or "").strip().lower() == "ollama" and not req.base_url:
  264. req.base_url = _m._get_ollama_base_url()
  265. headers = _ollama_probe_headers(req)
  266. if not _ml.should_use_ollama_native_catalog(req.provider, req.base_url, headers=headers):
  267. return None
  268. models = _ml.probe_ollama_local_models(req.base_url, headers=headers)
  269. if models is None:
  270. # A failed native probe is not authoritative; fall back to the OpenAI-compatible catalog.
  271. models = _m.probe_api_models(
  272. req.api_key, _ml._normalize_openai_base_url(req.base_url), request_headers=headers,
  273. ).get("models")
  274. if models is None:
  275. return _soft_accept(
  276. f"Note: could not reach this Ollama endpoint's `/api/tags` model listing to validate `{req.requested}`. "
  277. "Hermes will save the model name, but local Ollama model discovery could not verify it."
  278. )
  279. match = _match_in_catalog(req.lookup, models, suggest_label="Similar local Ollama models")
  280. if match.exact:
  281. return _accept()
  282. empty_hint = " No models are currently listed by `/api/tags`." if not models else ""
  283. return _soft_accept(
  284. f"Note: `{req.requested}` was not found in this Ollama endpoint's `/api/tags` model listing."
  285. f"{empty_hint} It may still work if the server supports hidden or aliased models."
  286. f"{match.suggestion_text}"
  287. )
  288. def _validate_custom(req: _Request) -> dict[str, Any]:
  289. from hermes_cli import models as _m
  290. # Probe with the auth shape the api_mode expects.
  291. anthropic_style = req.api_mode == "anthropic_messages"
  292. probe_kwargs = {"api_mode": req.api_mode} if anthropic_style else {}
  293. probe = _m.probe_api_models(req.api_key, req.base_url, request_headers=req.headers, **probe_kwargs)
  294. api_models = probe.get("models")
  295. if api_models is not None:
  296. match = _match_in_catalog(req.lookup, api_models, suggest_query=req.requested)
  297. verdict = match.verdict(req)
  298. if verdict is not None:
  299. return verdict
  300. message = (
  301. f"Note: `{req.requested}` was not found in this custom endpoint's model listing "
  302. f"({probe.get('probed_url')}). It may still work if the server supports hidden or aliased models."
  303. f"{match.suggestion_text}"
  304. )
  305. if probe.get("used_fallback"):
  306. message += (f"\n Endpoint verification succeeded after trying `{probe.get('resolved_base_url')}`. "
  307. "Consider saving that as your base URL.")
  308. return _soft_accept(message)
  309. # Many OpenAI-compatible and Anthropic-compatible proxies (DashScope coding plan, Cline,
  310. # MiniMax) never implement GET /models; /chat/completions works fine. Rejecting the switch
  311. # here bricked `/model` for them (#12220), so both chat modes persist the name unverified.
  312. accepted = req.api_mode in ("chat_completions", "anthropic_messages")
  313. message = f"Note: could not reach this custom endpoint's model listing at `{probe.get('probed_url')}`. "
  314. if accepted:
  315. message += (f"`{req.requested}` was accepted without verification — if this endpoint does not "
  316. "serve it, inference will fail; check the provider's model catalog or the model name.")
  317. else:
  318. message += f"`{req.requested}` was not saved; the endpoint should expose `/models` for verification."
  319. if probe.get("suggested_base_url"):
  320. message += f"\n If this server expects `/v1`, try base URL: `{probe.get('suggested_base_url')}`"
  321. return _verdict(accepted, True, False, message)
  322. def _static_catalog(normalized: str) -> list[str]:
  323. from hermes_cli import models as _m
  324. try:
  325. return _m.provider_model_ids(normalized)
  326. except Exception:
  327. return []
  328. _STATIC_FAMILY_PREFIXES = {
  329. # Plausibility gate (#45006): the soft-accept (#16172 / #19729) exists for entitlement-gated *hidden*
  330. # slugs the curated listing hasn't caught up with — but those are always the provider's own family
  331. # (openai-codex -> gpt-*; xai-oauth -> grok-*). Accepting an unrelated typed name (e.g. `qwen3.5-4b`,
  332. # `llama-3.1-8b`) here turns what should be an actionable "did you mean --provider <x>?" error into a
  333. # confusing success that 400s on the next turn. Only soft- accept names that share the provider's family
  334. # prefix; reject the rest with guidance to pin the right provider.
  335. "openai-codex": ("gpt-", "codex-", "o1", "o3", "o4"),
  336. "xai-oauth": ("grok-",),
  337. }
  338. _STATIC_LABELS = {"openai-codex": "OpenAI Codex", "xai-oauth": "xAI Grok OAuth (SuperGrok / Premium+)"}
  339. def _family_head(model_id: str) -> str:
  340. """Vendor family token of a model id: ``gpt-5.5`` → ``gpt``, ``claude-opus-5`` → ``claude``."""
  341. return re.split(r"[-./:]", model_id.strip().lower(), maxsplit=1)[0]
  342. def static_model_provider_conflict(model_name: str, provider: Optional[str], *, limit: int = 5) -> Optional[dict[str, Any]]:
  343. """Offline model×provider coherence from the curated catalogs only (no network: this runs on
  344. ``session.create``). ``None`` = coherent or undecidable — custom / aggregator / catalog-less
  345. providers, names in the provider's own family (a newer ``gpt-*`` the curated list lacks) and
  346. names no vendor lists (hidden or preview slugs) stay permissive. A conflict is a name outside
  347. the provider's family that another native vendor's catalog lists — or any foreign-family name
  348. on the OAuth catalogs with a strict family gate (``_STATIC_FAMILY_PREFIXES``) (#96817)."""
  349. from hermes_cli import models as _m
  350. requested = (model_name or "").strip()
  351. normalized = _m.normalize_provider(provider)
  352. catalog = list(_m._PROVIDER_MODELS.get(normalized, ()))
  353. if not requested or not catalog or normalized == "moa" or normalized in _m._AGGREGATOR_PROVIDERS:
  354. return None
  355. if _m._model_in_provider_catalog(requested.lower(), _m._provider_keys(normalized)):
  356. return None
  357. if _family_head(requested) in {_family_head(m) for m in catalog}:
  358. return None
  359. strict = normalized in _STATIC_FAMILY_PREFIXES
  360. if not strict and next(_m._static_catalog_matches(requested, normalized), None) is None:
  361. return None
  362. suggestions = get_close_matches(requested, catalog, n=limit, cutoff=0.4) or catalog[:limit]
  363. label = _m._PROVIDER_LABELS.get(normalized, normalized)
  364. return {
  365. "model": requested, "provider": normalized, "suggestions": suggestions,
  366. "message": (f"Model `{requested}` is not served by provider `{normalized}` ({label}). "
  367. f"Closest {label} models: " + ", ".join(f"`{s}`" for s in suggestions) + "."),
  368. }
  369. def _validate_static_catalog(req: _Request) -> Optional[dict[str, Any]]:
  370. """openai-codex / xai-oauth: no /v1/models probing — validate against the curated catalog.
  371. Returns None (fall through) when the catalog is empty."""
  372. catalog = _static_catalog(req.normalized)
  373. if req.normalized == "openai-codex":
  374. from agent.model_metadata import CODEX_CONTEXT_VARIANT_SUFFIX, is_codex_context_variant
  375. # Ineligible ``-900k`` aliases must be rejected BEFORE the hidden-slug soft-accept:
  376. # the suffix is a Hermes picker convention, so an unknown `*-900k` can never be a real
  377. # hidden provider slug — soft-accepting one silently runs at 272K on a different model.
  378. if req.lookup.strip().lower().endswith(CODEX_CONTEXT_VARIANT_SUFFIX) and req.lookup not in set(catalog):
  379. if is_codex_context_variant(req.lookup):
  380. # Valid variant a stale catalog hasn't synthesized yet.
  381. return _accept()
  382. base_guess = req.lookup[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)]
  383. return _reject(
  384. f"`{req.requested}` is not a valid large-context variant — `{base_guess}` enforces the "
  385. "standard 272K window on Codex, so no `-900k` option exists for it. Pick the base model, "
  386. "or a verified variant from the `/model` picker (e.g. `gpt-5.6-sol-900k`)."
  387. )
  388. if not catalog:
  389. return None
  390. match = _match_in_catalog(req.lookup, catalog)
  391. verdict = match.verdict(req)
  392. if verdict is not None:
  393. return verdict
  394. label = _STATIC_LABELS[req.normalized]
  395. # Plausibility gate: the soft-accept exists for entitlement-gated *hidden* slugs the curated
  396. # listing hasn't caught up with — always the provider's own family (gpt-* / grok-*). An
  397. # unrelated name (`qwen3.5-4b`) would turn an actionable "did you mean --provider <x>?" into
  398. # a confusing success that 400s on the next turn, so reject it with guidance instead.
  399. prefixes = _STATIC_FAMILY_PREFIXES.get(req.normalized, ())
  400. lower = req.lookup.strip().lower()
  401. if prefixes and not any(lower.startswith(p) for p in prefixes):
  402. return _reject(
  403. f"`{req.requested}` doesn't look like a {label} model and isn't in its listing, so it was not "
  404. "accepted. If it belongs to another configured provider, switch with `--provider <slug>` "
  405. f"(or select it from the `/model` picker).{match.suggestion_text}"
  406. )
  407. return _soft_accept(
  408. f"Note: `{req.requested}` was not found in the {label} model listing. "
  409. "It may still work if your account has access to a newer or hidden model ID."
  410. f"{match.suggestion_text}"
  411. )
  412. def _validate_minimax(req: _Request) -> Optional[dict[str, Any]]:
  413. """MiniMax has no /models endpoint — static catalog, case-insensitive (ids like MiniMax-M2.7).
  414. Returns None when the catalog is empty."""
  415. catalog = _static_catalog(req.normalized)
  416. if not catalog:
  417. return None
  418. match = _match_in_catalog(req.lookup, catalog, case_insensitive=True)
  419. return match.verdict(req) or _soft_accept(
  420. f"Note: `{req.requested}` was not found in the MiniMax catalog."
  421. f"{match.suggestion_text}"
  422. "\n MiniMax does not expose a /models endpoint, so Hermes cannot verify the model name."
  423. "\n The model may still work if it exists on the server."
  424. )
  425. def _validate_anthropic(req: _Request) -> Optional[dict[str, Any]]:
  426. """Native Anthropic: /v1/models needs x-api-key (or OAuth Bearer) + anthropic-version, so the
  427. generic Bearer probe 401s — use the native fetcher. None (fall through) when no token is
  428. resolvable or the network failed."""
  429. from hermes_cli import models as _m
  430. models = _m._fetch_anthropic_models(base_url=req.base_url or None, api_key=req.api_key or None)
  431. if models is None:
  432. return None
  433. match = _match_in_catalog(req.lookup, models, suggest_query=req.requested)
  434. # Accept anyway — Anthropic gates newer/preview models (snapshot IDs, early access) behind
  435. # accounts even though they aren't listed on /v1/models.
  436. return match.verdict(req) or _soft_accept(
  437. f"Note: `{req.requested}` was not found in Anthropic's /v1/models listing. "
  438. f"It may still work if you have early-access or snapshot IDs."
  439. f"{match.suggestion_text}"
  440. )
  441. def _validate_anthropic_messages(req: _Request) -> dict[str, Any]:
  442. """Anthropic Messages transport: probe /v1/models and soft-accept either way, but say which
  443. happened — a proxy that never implemented the listing is a different situation from a reachable
  444. listing that simply doesn't name the slug (vendors alias ids: ``kimi-k3`` is served as ``k3``)."""
  445. from hermes_cli import models as _m
  446. models = _m.fetch_api_models(req.api_key, req.base_url, api_mode=req.api_mode)
  447. if models is None:
  448. return _soft_accept(
  449. f"Note: could not verify `{req.requested}` against this endpoint's model listing. Many "
  450. "Anthropic-compatible proxies do not implement GET /v1/models. The model name has been accepted "
  451. "without verification."
  452. )
  453. # Vendor alias pairs sit below the default 0.5 similarity cutoff (kimi-k3 vs k3 ≈ 0.44).
  454. match = _match_in_catalog(req.lookup, models, case_insensitive=True, suggest_query=req.requested,
  455. suggest_cutoff=0.4)
  456. return match.verdict(req) or _soft_accept(
  457. f"Note: `{req.requested}` is not named in this endpoint's model listing (it may still serve it "
  458. f"under an alias).{match.suggestion_text}"
  459. "\n The model name has been accepted without verification."
  460. )
  461. def _nous_portal_recommended_names() -> set[str]:
  462. """Lower-cased ids from the Portal's live recommended-models feed (empty on any failure)."""
  463. from hermes_cli import models as _m
  464. try:
  465. payload = _m.fetch_nous_recommended_models(_m._resolve_nous_portal_url())
  466. return {
  467. name.lower()
  468. for tier in ("freeRecommendedModels", "paidRecommendedModels")
  469. for entry in (payload.get(tier) or [])
  470. if (name := _m._extract_model_name(entry))
  471. }
  472. except Exception:
  473. return set()
  474. def _validate_managed_local(req: _Request) -> Optional[dict[str, Any]]:
  475. """The managed llama.cpp runtime: the staged library on disk is the source of truth, not the
  476. live listing. The router's model list is spawn-only (a GGUF landed after its start is
  477. invisible to GET /models until a bounce), so validating a freshly downloaded model against
  478. the live listing rejects the very file the user just staged — the Local Models "Use" flow
  479. and the composer picker could never succeed for a non-catalog model. A staged id accepts
  480. (case-insensitive: typing matches the file name, the router registers the preset id);
  481. anything else falls through to the live listing, which stays authoritative for ids that
  482. were never downloaded here."""
  483. from hermes_cli.local_runtime.bootstrap import staged_model_ids
  484. staged = {sid.lower() for sid in staged_model_ids()}
  485. if req.lookup.strip().lower() in staged:
  486. return _accept_with_note(
  487. f"Note: `{req.requested}` was not found in the live /v1/models listing "
  488. "but is downloaded in the managed local-models library — accepted."
  489. )
  490. return None
  491. def _profile_catalog(normalized: str, base_url: Optional[str] = None) -> tuple[list[str], bool]:
  492. """``(catalog, authoritative)`` for a profile whose catalog is not the generic
  493. ``{base_url}/models`` listing — it overrides ``fetch_models`` or points ``models_url``
  494. elsewhere — so that listing is not authoritative for it (a relay may 200 with a different
  495. product catalog, #101705). The catalog is *authoritative* (a miss is a reject, never an
  496. acceptance by the generic listing, #116667) when the profile serves it from an endpoint of
  497. its own and it is available; a bare ``fetch_models`` override may just re-shape the generic
  498. listing, and an unavailable/empty catalog keeps the generic listing as the validator.
  499. ``([], False)`` for profiles without a catalog of their own."""
  500. from providers import get_provider_profile
  501. from providers.base import ProviderProfile
  502. profile = get_provider_profile(normalized)
  503. if profile is None:
  504. return [], False
  505. # A non-default runtime base_url is a user-configured relay/proxy; its own /models listing
  506. # decides validation, not the provider profile's canonical catalog endpoint.
  507. if base_url and (base_url.rstrip("/") != (profile.base_url or "").rstrip("/")):
  508. return [], False
  509. generic = (profile.base_url or "").rstrip("/") + "/models"
  510. own_endpoint = bool(profile.models_url) and profile.models_url.rstrip("/") != generic
  511. if not own_endpoint and type(profile).fetch_models is ProviderProfile.fetch_models:
  512. return [], False
  513. catalog = _static_catalog(normalized)
  514. return catalog, own_endpoint and bool(catalog)
  515. def _validate_live_listing(req: _Request) -> Optional[dict[str, Any]]:
  516. """Generic live /v1/models probe. Returns None when the API was unreachable (the caller then
  517. tries Bedrock discovery / the curated catalog). A profile that owns its catalog is validated
  518. against that catalog (``provider_model_ids`` — the picker's list) before the generic listing."""
  519. from hermes_cli import models as _m
  520. catalog, authoritative = _profile_catalog(req.normalized, req.base_url)
  521. if catalog:
  522. match = _match_in_catalog(req.lookup, catalog, suggest_query=req.requested)
  523. if match.exact:
  524. return _accept()
  525. if authoritative:
  526. # The catalog endpoint the profile declares decides: a miss there is a reject, never
  527. # an acceptance by the generic listing, which for such relays lists a different
  528. # product line.
  529. return _reject(
  530. f"Model `{req.requested}` was not found in this provider's catalog.{match.suggestion_text}")
  531. api_models = _m.fetch_api_models(req.api_key, req.base_url)
  532. if api_models is None:
  533. return None
  534. if req.normalized == "gemini":
  535. # Gemini's OpenAI-compat listing prefixes ids with "models/"; curated list and user input
  536. # use the bare id, so strip before comparing.
  537. api_models = [m[len("models/"):] if isinstance(m, str) and m.startswith("models/") else m for m in api_models]
  538. match = _match_in_catalog(req.lookup, api_models)
  539. if match.exact:
  540. return _accept()
  541. # OpenRouter routing variants (":nitro", ":floor", ...) are request-time modifiers, not
  542. # catalog entries — validate the BASE but keep the suffixed id.
  543. variant_base = openrouter_variant_base(req.lookup) if req.normalized == "openrouter" else None
  544. if variant_base is not None and variant_base in set(api_models):
  545. return _accept()
  546. # Listed but not found: the account may reach models absent from the public listing
  547. # (e.g. Z.AI Pro/Max plans use glm-5 on coding endpoints) — warn but allow where plausible.
  548. # Curated-catalog soft-accept: providers omit valid models from live listings (stale cache,
  549. # partial rollout, gated previews). EXCEPTION: official OpenAI hosts (canonical + data-
  550. # residency regional) — their listing is access-scoped and authoritative, so an absent model
  551. # is one this key CANNOT serve; a soft-accept would 400 at first use. Custom OpenAI-compatible
  552. # proxies keep the fallback.
  553. listing_authoritative = False
  554. if req.normalized in ("openai", "openai-api"):
  555. from hermes_cli.providers import is_official_openai_host
  556. listing_authoritative = is_official_openai_host(req.base_url)
  557. if not listing_authoritative and _m._model_in_provider_catalog(
  558. (variant_base or req.lookup).lower(), _m._provider_keys(req.normalized)
  559. ):
  560. return _accept_with_note(f"Note: `{req.requested}` was not found in the live /v1/models listing "
  561. "but exists in the curated catalog — accepted.")
  562. # Nous: the Portal's recommended-models feed can list a model before the curated list or the
  563. # docs-hosted manifest catches up; `hermes chat` already accepts those at model-list build
  564. # time, so mirror that source of truth for per-message /model validation.
  565. if req.normalized == "nous" and req.lookup.lower() in _nous_portal_recommended_names():
  566. return _accept_with_note(f"Note: `{req.requested}` was not found in the live /v1/models listing "
  567. "but is a current Nous Portal recommendation — accepted.")
  568. return _reject(f"Model `{req.requested}` was not found in this provider's model listing.{match.suggestion_text}")
  569. def _validate_bedrock(req: _Request) -> Optional[dict[str, Any]]:
  570. """Bedrock's runtime URL has no /models; discovery goes through the AWS control plane
  571. (ListFoundationModels + ListInferenceProfiles). Any failure falls through (None)."""
  572. try:
  573. from agent.bedrock_adapter import discover_bedrock_models, resolve_bedrock_runtime_region
  574. region = resolve_bedrock_runtime_region()
  575. discovered_ids = {m["id"] for m in discover_bedrock_models(region)}
  576. match = _match_in_catalog(req.requested, list(discovered_ids), suggest_cutoff=0.4)
  577. if match.exact:
  578. return _accept()
  579. # Still accept (custom inference profiles / cross-account access), but warn.
  580. return _soft_accept(
  581. f"Note: `{req.requested}` was not found in Bedrock model discovery for {region}. "
  582. f"It may still work with custom inference profiles or cross-account access."
  583. f"{match.suggestion_text}"
  584. )
  585. except Exception:
  586. return None
  587. def _validate_external_process(req: _Request) -> Optional[dict[str, Any]]:
  588. """Process providers have no HTTP listing: the picker's list (``provider_model_ids`` — the
  589. CLI's live catalog merged with the declared one) plus the profile's short aliases is the whole
  590. truth, so a listed id is accepted outright and an unlisted one gets the catalog verdict without
  591. the misleading "endpoint was unreachable" note."""
  592. from providers import get_provider_profile
  593. profile = get_provider_profile(req.normalized)
  594. if profile is None or profile.auth_type != "external_process":
  595. return None
  596. if req.lookup.lower() in {k.lower() for k in profile.model_aliases}:
  597. return _accept()
  598. catalog = _static_catalog(req.normalized) or list(profile.fallback_models)
  599. match = _match_in_catalog(req.lookup, catalog, case_insensitive=True)
  600. if match.exact:
  601. return _accept()
  602. return match.verdict(req) or _soft_accept(
  603. f"Note: `{req.requested}` is not declared by {profile.display_name or profile.name}."
  604. f"{match.suggestion_text}\n The model may still work if the local client accepts it.")
  605. def _validate_catalog_fallback(req: _Request) -> dict[str, Any]:
  606. """/models unreachable: validate against the curated ``provider_model_ids()`` list so gateway
  607. /model switches keep working while a provider's endpoint is down (otherwise switch_model() would
  608. fail and the gateway never writes the session override). No catalog → accept with a warning."""
  609. from hermes_cli import models as _m
  610. label = _m._PROVIDER_LABELS.get(req.normalized, req.normalized)
  611. catalog = _static_catalog(req.normalized)
  612. if not catalog:
  613. return _soft_accept(f"Note: could not reach the {label} API to validate `{req.requested}`. "
  614. "If the service isn't down, this model may not be valid.")
  615. match = _match_in_catalog(req.lookup, catalog, case_insensitive=True)
  616. if match.exact:
  617. return _accept()
  618. # Same OpenRouter routing-variant rule as the live-listing path.
  619. if req.normalized == "openrouter":
  620. variant_base = openrouter_variant_base(req.lookup)
  621. if variant_base is not None and variant_base.lower() in {m.lower() for m in catalog}:
  622. return _accept()
  623. return _soft_accept(
  624. f"Note: `{req.requested}` was not found in the {label} curated catalog "
  625. f"and the /models endpoint was unreachable.{match.suggestion_text}"
  626. f"\n The model may still work if it exists on the provider."
  627. )
  628. # ── Orchestrator ─────────────────────────────────────────────────────────
  629. def _is_custom(req: _Request) -> bool:
  630. return req.normalized == "custom" or req.normalized.startswith("custom:")
  631. def _for(*providers: str) -> Callable[[_Request], bool]:
  632. return lambda req: req.normalized in providers
  633. # (gate, branch): the branch runs when the gate passes; the first non-None verdict wins. ORDER IS
  634. # BEHAVIOR: moa → whitespace (skipped only for self-hosted providers and a user-configured
  635. # base_url) → OpenRouter preset parse → LM Studio → Ollama native → custom →
  636. # codex/xai static → MiniMax → managed local (staged library) → Anthropic native →
  637. # Anthropic Messages → external process → live listing → Bedrock → curated-catalog fallback (always decides).
  638. _LADDER: tuple[tuple[Callable[[_Request], bool], Callable[[_Request], Optional[dict[str, Any]]]], ...] = (
  639. (_for("moa"), _validate_moa),
  640. (lambda req: True, _reject_whitespace),
  641. (_for("openrouter"), _parse_openrouter_preset),
  642. (_for("lmstudio"), _validate_lmstudio),
  643. (lambda req: True, _validate_ollama_native),
  644. (_is_custom, _validate_custom),
  645. (_for("openai-codex", "xai-oauth"), _validate_static_catalog),
  646. (_for("minimax", "minimax-cn"), _validate_minimax),
  647. (_for("llamacpp", "llama.cpp", "llama-cpp"), _validate_managed_local),
  648. (_for("anthropic"), _validate_anthropic),
  649. (lambda req: req.api_mode == "anthropic_messages", _validate_anthropic_messages),
  650. (lambda req: True, _validate_external_process),
  651. (lambda req: True, _validate_live_listing),
  652. # API unreachable — accept and persist, but warn so typos don't silently break things.
  653. (_for("bedrock"), _validate_bedrock),
  654. (lambda req: True, _validate_catalog_fallback),
  655. )
  656. def validate_requested_model(
  657. model_name: str,
  658. provider: Optional[str],
  659. *,
  660. api_key: Optional[str] = None,
  661. base_url: Optional[str] = None,
  662. api_mode: Optional[str] = None,
  663. headers: Optional[dict[str, str]] = None,
  664. ) -> dict[str, Any]:
  665. """Validate a ``/model`` value for the active provider → dict with ``accepted`` (switch now),
  666. ``persist`` (safe to save to config), ``recognized`` (matched a known provider catalog),
  667. ``message`` (optional warning / guidance). The requested id is never rewritten: what the user
  668. selected is what the wire sees."""
  669. from hermes_cli import models as _m
  670. requested = (model_name or "").strip()
  671. normalized = _m.normalize_provider(provider)
  672. if normalized == "openrouter" and base_url and not base_url_host_matches(base_url, "openrouter.ai"):
  673. normalized = "custom"
  674. from providers import get_provider_profile
  675. profile = get_provider_profile(normalized) if normalized else None
  676. lookup = (profile.normalize_model_id(requested, api_key=api_key or "") if profile is not None else None) or requested
  677. if not requested:
  678. return _reject("Model name cannot be empty.")
  679. req = _Request(requested, lookup, provider, normalized, api_key, base_url, api_mode, headers)
  680. for gate, branch in _LADDER:
  681. if gate(req):
  682. verdict = branch(req)
  683. if verdict is not None:
  684. return verdict
  685. raise AssertionError("unreachable: _validate_catalog_fallback always decides")