models_local.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701
  1. """Local / self-hosted model servers.
  2. Ollama (native ``/api/tags`` probe, request headers, base-url resolution), LM Studio
  3. (``/api/v1/models``, load-on-demand) and Ollama Cloud (live + models.dev merged catalog with a
  4. disk cache). Split out of ``hermes_cli.models``; helpers still defined there are looked up on
  5. ``hermes_cli.models`` at call time so ``patch("hermes_cli.models.<name>")`` mocks keep intercepting.
  6. """
  7. from __future__ import annotations
  8. import hashlib
  9. import http.client
  10. import json
  11. import logging
  12. import os
  13. import time
  14. import urllib.error
  15. import urllib.parse
  16. import urllib.request
  17. from pathlib import Path
  18. from typing import Any, NamedTuple, Optional
  19. from agent.secret_scope import get_secret_str
  20. from hermes_cli.urllib_security import url_origin
  21. # Log-record parity with the origin module.
  22. logger = logging.getLogger("hermes_cli.models")
  23. _OLLAMA_DEFAULT_PORT = 11434
  24. def _strip_suffixes(root: str, suffixes: tuple[str, ...]) -> str:
  25. """Drop the first matching path suffix (and any trailing slash) from *root*."""
  26. for suffix in suffixes:
  27. if root.endswith(suffix):
  28. return root[: -len(suffix)].rstrip("/")
  29. return root
  30. def _normalize_openai_base_url(base_url: Optional[str]) -> str:
  31. """Add a usable HTTP scheme without changing an OpenAI API path."""
  32. value = str(base_url or "").strip()
  33. if value.startswith(":"):
  34. return "http://127.0.0.1" + value
  35. if value and "://" not in value:
  36. return "http://" + value
  37. return value
  38. def _root_for_ollama_native_api(base_url: str) -> str:
  39. """Convert an OpenAI-style Ollama base URL to the native API root."""
  40. root = str(base_url or "").strip().rstrip("/")
  41. if root.startswith(":"):
  42. root = "http://127.0.0.1" + root
  43. elif root and "://" not in root:
  44. root = "http://" + root
  45. return _strip_suffixes(root, ("/api/tags", "/v1/models", "/api", "/v1"))
  46. def _configured_ollama_base_url() -> str:
  47. """``providers.ollama.base_url`` (legacy keys ``api`` / ``url``), or ``""``."""
  48. from hermes_cli.models import _get_provider_config_dict
  49. cfg = _get_provider_config_dict("ollama")
  50. return str(cfg.get("base_url") or cfg.get("api") or cfg.get("url") or "").strip()
  51. def _ollama_host_from_env(env_host: str) -> str:
  52. """Apply Ollama's own ``OLLAMA_HOST`` defaulting rules (port 11434, IPv6 bracketing)."""
  53. port = _OLLAMA_DEFAULT_PORT
  54. if env_host.startswith(":") and not env_host.startswith("::"):
  55. return "127.0.0.1" + env_host
  56. if env_host.startswith("[") and env_host.endswith("]"):
  57. return f"{env_host}:{port}"
  58. if "://" in env_host:
  59. try:
  60. parsed = urllib.parse.urlsplit(env_host)
  61. if parsed.hostname and parsed.port is None:
  62. hostname = parsed.hostname
  63. if ":" in hostname and not hostname.startswith("["):
  64. hostname = f"[{hostname}]"
  65. userinfo = parsed.netloc.rsplit("@", 1)[0] + "@" if "@" in parsed.netloc else ""
  66. return parsed._replace(netloc=f"{userinfo}{hostname}:{port}").geturl()
  67. except ValueError:
  68. pass
  69. return env_host
  70. if env_host.count(":") > 1 and not env_host.startswith("["):
  71. return f"[{env_host}]:{port}"
  72. if ":" not in env_host:
  73. return f"{env_host}:{port}"
  74. return env_host
  75. def _get_ollama_base_url() -> str:
  76. """Resolve the local Ollama-compatible endpoint URL: explicit ``providers.ollama.base_url``
  77. (wires local endpoints without changing the active provider) → active ``model.base_url`` when
  78. the active provider is ollama, or custom AND the endpoint actually serves ``/api/tags``
  79. (otherwise the picker would probe an unrelated endpoint and hide the local catalog) →
  80. ``OLLAMA_HOST`` → Ollama's local default."""
  81. from hermes_cli.models import _get_model_config_dict
  82. configured = _configured_ollama_base_url()
  83. if configured:
  84. return configured
  85. model_cfg = _get_model_config_dict()
  86. model_provider = str(model_cfg.get("provider", "") or "").strip().lower()
  87. model_base = str(model_cfg.get("base_url", "") or "").strip()
  88. if model_provider == "ollama" and model_base:
  89. return model_base
  90. if model_provider == "custom" and model_base:
  91. try:
  92. if should_use_ollama_native_catalog("custom", model_base):
  93. return model_base
  94. except (OSError, RuntimeError, TypeError, ValueError):
  95. pass
  96. env_host = os.getenv("OLLAMA_HOST", "").strip()
  97. return _ollama_host_from_env(env_host) if env_host else "http://localhost:11434"
  98. def _api_key_from_provider_config(entry: dict, *env_keys: str) -> str:
  99. """``api_key`` from a provider config block, else the env var named by the first set *env_keys*.
  100. The variable is read through the profile secret scope (fresh ``.env``, never another
  101. profile's process env under multiplexing), like every other credential read (#67935).
  102. """
  103. api_key = str(entry.get("api_key") or "").strip()
  104. if api_key:
  105. return api_key
  106. key_env = str(next((entry.get(k) for k in env_keys if entry.get(k)), "") or "").strip()
  107. return get_secret_str(key_env, "").strip() if key_env else ""
  108. def _drop_authorization(headers: dict[str, str]) -> None:
  109. for key in tuple(headers):
  110. if key.lower() == "authorization":
  111. del headers[key]
  112. def _get_ollama_request_headers() -> dict[str, str]:
  113. """Return configured headers and credentials for native Ollama requests."""
  114. from hermes_cli.models import _get_provider_config_dict
  115. entry = _get_provider_config_dict("ollama")
  116. try:
  117. from hermes_cli.config import normalize_extra_headers
  118. result = normalize_extra_headers(entry.get("extra_headers"))
  119. except (ImportError, OSError, RuntimeError, TypeError, ValueError):
  120. result = {}
  121. api_key = _api_key_from_provider_config(entry, "key_env", "api_key_env")
  122. if api_key and not any(key.lower() == "authorization" for key in result):
  123. result["Authorization"] = f"Bearer {api_key}"
  124. return result
  125. def _get_ollama_native_headers(base_url: Optional[str], *, api_key: Optional[str] = None) -> dict[str, str]:
  126. """Ollama credentials and headers for one endpoint origin. Configured headers apply only when
  127. *base_url* shares the configured Ollama root; an explicit *api_key* replaces any configured
  128. Authorization variant rather than inheriting it."""
  129. configured_base = _configured_ollama_base_url()
  130. from agent.command_token_source import materialize_probe_api_key
  131. explicit_key = materialize_probe_api_key(api_key)
  132. configured_matches = bool(configured_base and base_url and _same_ollama_native_root(base_url, configured_base))
  133. if not configured_matches and not explicit_key:
  134. return {}
  135. headers = _get_ollama_request_headers() if configured_matches else {}
  136. if explicit_key or callable(api_key):
  137. _drop_authorization(headers)
  138. if explicit_key:
  139. headers["Authorization"] = f"Bearer {explicit_key}"
  140. return headers
  141. # Native /api/tags probe caches, keyed by root (+ header fingerprint): successful catalogs,
  142. # failure timestamps (short negative TTL), and whether the root answered the native probe.
  143. _OLLAMA_LOCAL_MODELS_CACHE_TTL: int = 300 # seconds
  144. _OLLAMA_LOCAL_MODELS_CACHE: dict[str, tuple[tuple[str, ...], float]] = {}
  145. _OLLAMA_LOCAL_PROBE_FAILURE_CACHE: dict[str, float] = {}
  146. _OLLAMA_LOCAL_PROBE_REACHABLE: dict[str, bool] = {}
  147. _OLLAMA_LOCAL_PROBE_FAILURE_TTL: int = 30
  148. _OLLAMA_LOCAL_CACHE_MAX_ENTRIES: int = 256
  149. def _evict_related_ollama_cache_entries(key: str) -> None:
  150. _OLLAMA_LOCAL_MODELS_CACHE.pop(key, None)
  151. _OLLAMA_LOCAL_PROBE_REACHABLE.pop(key, None)
  152. for failure_key in list(_OLLAMA_LOCAL_PROBE_FAILURE_CACHE):
  153. if failure_key == key or failure_key.startswith(f"{key}|timeout:"):
  154. _OLLAMA_LOCAL_PROBE_FAILURE_CACHE.pop(failure_key, None)
  155. def _remember_ollama_cache(cache: dict[str, Any], key: str, value: Any) -> None:
  156. if key not in cache and len(cache) >= _OLLAMA_LOCAL_CACHE_MAX_ENTRIES:
  157. _evict_related_ollama_cache_entries(next(iter(cache)).split("|timeout:", 1)[0])
  158. cache[key] = value
  159. def _ollama_probe_cache_key(root: str, headers: Optional[dict[str, str]]) -> str:
  160. if not headers:
  161. return root
  162. normalized_headers = sorted((str(key).lower(), str(value)) for key, value in headers.items())
  163. header_blob = json.dumps(normalized_headers, ensure_ascii=False, separators=(",", ":")).encode("utf-8", errors="replace")
  164. return f"{root}|headers:{hashlib.blake2b(header_blob, digest_size=8).hexdigest()}"
  165. def _parse_ollama_tags(payload: Any) -> Optional[list[str]]:
  166. """Model ids from an ``/api/tags`` payload; None when the shape is not Ollama's."""
  167. raw_models = payload.get("models") if isinstance(payload, dict) else None
  168. if not isinstance(raw_models, list):
  169. return None
  170. models: list[str] = []
  171. for item in raw_models:
  172. if not isinstance(item, dict):
  173. return None
  174. model_id = str(item.get("model") or item.get("name") or "").strip()
  175. if model_id and model_id not in models:
  176. models.append(model_id)
  177. if raw_models and not models:
  178. return None
  179. return models
  180. def probe_ollama_local_models(
  181. base_url: Optional[str] = None,
  182. timeout: float = 2.0,
  183. headers: Optional[dict[str, str]] = None,
  184. ) -> Optional[list[str]]:
  185. """Probe local Ollama-compatible models from native ``/api/tags`` (Ollama's authoritative local
  186. catalog; ``/v1/models`` is not required for local servers). ``None`` when the endpoint cannot be
  187. reached or returns malformed data; a list (possibly empty) when it was reachable."""
  188. from hermes_cli.models import _get_ollama_base_url, _urlopen_model_catalog_request
  189. from hermes_cli.version_info import hermes_user_agent
  190. root = _root_for_ollama_native_api(base_url or _get_ollama_base_url())
  191. if not root:
  192. return None
  193. cache_key = _ollama_probe_cache_key(root, headers)
  194. failure_key = f"{cache_key}|timeout:{float(timeout):.3f}"
  195. cached = _OLLAMA_LOCAL_MODELS_CACHE.get(cache_key)
  196. if cached is not None and time.monotonic() - cached[1] < _OLLAMA_LOCAL_MODELS_CACHE_TTL:
  197. return list(cached[0])
  198. failed_at = _OLLAMA_LOCAL_PROBE_FAILURE_CACHE.get(failure_key)
  199. if failed_at is not None:
  200. if time.monotonic() - failed_at < _OLLAMA_LOCAL_PROBE_FAILURE_TTL:
  201. return None
  202. _OLLAMA_LOCAL_PROBE_FAILURE_CACHE.pop(failure_key, None)
  203. try:
  204. request_headers = {"User-Agent": hermes_user_agent(), **(headers or {})}
  205. req = urllib.request.Request(root.rstrip("/") + "/api/tags", headers=request_headers)
  206. with _urlopen_model_catalog_request(req, timeout=timeout) as resp:
  207. models = _parse_ollama_tags(json.loads(resp.read().decode()))
  208. except (ValueError, OSError, TimeoutError, http.client.HTTPException, urllib.error.URLError,
  209. json.JSONDecodeError, UnicodeDecodeError):
  210. models = None
  211. if models is None:
  212. _remember_ollama_cache(_OLLAMA_LOCAL_PROBE_REACHABLE, cache_key, False)
  213. _remember_ollama_cache(_OLLAMA_LOCAL_PROBE_FAILURE_CACHE, failure_key, time.monotonic())
  214. return None
  215. _remember_ollama_cache(_OLLAMA_LOCAL_PROBE_REACHABLE, cache_key, True)
  216. _OLLAMA_LOCAL_PROBE_FAILURE_CACHE.pop(failure_key, None)
  217. _remember_ollama_cache(_OLLAMA_LOCAL_MODELS_CACHE, cache_key, (tuple(models), time.monotonic()))
  218. return models
  219. def fetch_ollama_local_models(
  220. base_url: Optional[str] = None,
  221. timeout: float = 2.0,
  222. headers: Optional[dict[str, str]] = None,
  223. ) -> Optional[list[str]]:
  224. """Fetch local Ollama-compatible models, preserving probe failure as ``None``."""
  225. return probe_ollama_local_models(base_url, timeout, headers=headers)
  226. def _same_ollama_native_root(left: str, right: str) -> bool:
  227. """Return True when two Ollama/OpenAI-style base URLs share an API root."""
  228. left_root = _root_for_ollama_native_api(left).rstrip("/")
  229. right_root = _root_for_ollama_native_api(right).rstrip("/")
  230. if not left_root or not right_root:
  231. return False
  232. try:
  233. left_parts = urllib.parse.urlsplit(left_root)
  234. right_parts = urllib.parse.urlsplit(right_root)
  235. return (
  236. url_origin(left_root) == url_origin(right_root)
  237. and left_parts.path.rstrip("/") == right_parts.path.rstrip("/")
  238. )
  239. except (AttributeError, ValueError):
  240. return False
  241. _NEVER_OLLAMA_PROVIDERS = frozenset({"openrouter", "nous", "anthropic", "openai", "openai-codex", "gemini", "ollama-cloud"})
  242. _LOCAL_LIKE_PROVIDERS = frozenset({"", "custom", "local", "llamacpp", "llama.cpp", "llama-cpp", "vllm"})
  243. def should_use_ollama_native_catalog(
  244. provider: Optional[str],
  245. base_url: Optional[str],
  246. headers: Optional[dict[str, str]] = None,
  247. ) -> bool:
  248. """True when model discovery should use local Ollama ``/api/tags``: the caller asked for Ollama
  249. explicitly, the base URL matches ``providers.ollama.base_url``, or an ambiguous custom URL on
  250. Ollama's default port actually serves ``/api/tags``. (Bare ``ollama`` is normalized to
  251. ``custom`` elsewhere so runtime paths share the OpenAI client, but ``/api/tags`` is the
  252. authoritative local list; other custom endpoints keep the ``/models`` probe.)"""
  253. requested = str(provider or "").strip().lower()
  254. root = _root_for_ollama_native_api(base_url or "")
  255. if root:
  256. try:
  257. host = (urllib.parse.urlparse(root).hostname or "").lower()
  258. if host == "ollama.com" or host.endswith(".ollama.com"):
  259. return False
  260. except ValueError:
  261. pass
  262. if requested in _NEVER_OLLAMA_PROVIDERS:
  263. return False
  264. configured_base = _configured_ollama_base_url()
  265. if requested == "ollama":
  266. if not root:
  267. return False
  268. if configured_base and not _same_ollama_native_root(root, configured_base):
  269. return probe_ollama_local_models(root, timeout=0.5, headers=headers) is not None
  270. return True
  271. if configured_base and _same_ollama_native_root(root, configured_base):
  272. return True
  273. if not root:
  274. return False
  275. if requested not in _LOCAL_LIKE_PROVIDERS and not requested.startswith("custom:"):
  276. return False
  277. if requested == "custom:ollama" or requested.endswith("-ollama"):
  278. return True
  279. try:
  280. if urllib.parse.urlparse(root).port != _OLLAMA_DEFAULT_PORT:
  281. return False
  282. except ValueError:
  283. return False
  284. return probe_ollama_local_models(root, timeout=0.5, headers=headers) is not None
  285. def _ollama_local_catalog(force_refresh: bool) -> list[str]:
  286. """Catalog for the raw ``ollama`` provider: native ``/api/tags`` when the endpoint is a real
  287. Ollama server, else the OpenAI-style ``/v1/models`` of the configured gateway (incl. Ollama
  288. Cloud)."""
  289. from hermes_cli.models import _get_provider_config_dict, fetch_api_models
  290. if force_refresh:
  291. _OLLAMA_LOCAL_MODELS_CACHE.clear()
  292. _OLLAMA_LOCAL_PROBE_FAILURE_CACHE.clear()
  293. _OLLAMA_LOCAL_PROBE_REACHABLE.clear()
  294. base_url = _get_ollama_base_url()
  295. headers = _get_ollama_native_headers(base_url)
  296. if should_use_ollama_native_catalog("ollama", base_url, headers=headers):
  297. native_models = fetch_ollama_local_models(base_url, headers=headers) if headers else fetch_ollama_local_models(base_url)
  298. native_key = _ollama_probe_cache_key(_root_for_ollama_native_api(base_url), headers or None)
  299. if native_models or _OLLAMA_LOCAL_PROBE_REACHABLE.get(native_key) is True:
  300. return native_models or []
  301. config = _get_provider_config_dict("ollama")
  302. fallback_key = _api_key_from_provider_config(config, "key_env")
  303. fallback_base = _normalize_openai_base_url(config.get("base_url") or base_url)
  304. fallback_headers = _get_ollama_native_headers(fallback_base, api_key=fallback_key)
  305. return fetch_api_models(fallback_key, fallback_base, headers=fallback_headers or None) or []
  306. def _lmstudio_server_root(base_url: Optional[str]) -> Optional[str]:
  307. """LM Studio server root: users paste the OpenAI runtime URL (``.../v1``) or the native prefix
  308. (``.../api``, ``.../api/v1``); native probes append ``/api/v1/...`` themselves."""
  309. return _strip_suffixes((base_url or "").strip().rstrip("/"), ("/api/v1", "/api", "/v1")) or None
  310. def _lmstudio_request_headers(api_key: Optional[str] = None) -> dict:
  311. """HTTP headers for LM Studio native API requests."""
  312. from hermes_cli.version_info import hermes_user_agent
  313. from agent.command_token_source import materialize_probe_api_key
  314. token = materialize_probe_api_key(api_key)
  315. return {"User-Agent": hermes_user_agent(), **({"Authorization": f"Bearer {token}"} if token else {})}
  316. def _lmstudio_fetch_raw_models(
  317. api_key: Optional[str] = None,
  318. base_url: Optional[str] = None,
  319. timeout: float = 5.0,
  320. ) -> Optional[list[dict]]:
  321. """Raw model list from LM Studio's ``/api/v1/models``; None on network errors / malformed
  322. payloads; raises ``AuthError`` on HTTP 401/403."""
  323. from hermes_cli.models import _urlopen_model_catalog_request
  324. server_root = _lmstudio_server_root(base_url)
  325. if not server_root:
  326. return None
  327. request = urllib.request.Request(server_root + "/api/v1/models", headers=_lmstudio_request_headers(api_key))
  328. try:
  329. with _urlopen_model_catalog_request(request, timeout=timeout) as resp:
  330. payload = json.loads(resp.read().decode())
  331. except urllib.error.HTTPError as exc:
  332. if exc.code in {401, 403}:
  333. from hermes_cli.auth import AuthError
  334. raise AuthError(
  335. f"LM Studio rejected the request with HTTP {exc.code}.",
  336. provider="lmstudio",
  337. code="auth_rejected",
  338. ) from exc
  339. logger.debug("LM Studio probe at %s failed with HTTP %s", server_root, exc.code)
  340. return None
  341. except Exception as exc:
  342. logger.debug("LM Studio probe at %s failed: %s", server_root, exc)
  343. return None
  344. raw_models = payload.get("models") if isinstance(payload, dict) else None
  345. if not isinstance(raw_models, list):
  346. logger.debug("LM Studio probe at %s returned malformed payload (no `models` list)", server_root)
  347. return None
  348. return raw_models
  349. def _lmstudio_raw_models_or_none(api_key, base_url, timeout) -> Optional[list[dict]]:
  350. """``_lmstudio_fetch_raw_models`` with every failure (incl. AuthError) collapsed to None."""
  351. try:
  352. return _lmstudio_fetch_raw_models(api_key=api_key, base_url=base_url, timeout=timeout)
  353. except Exception:
  354. return None
  355. def _lmstudio_entry_for(raw_models: list, model: str) -> Optional[dict]:
  356. for raw in raw_models:
  357. if isinstance(raw, dict) and (raw.get("key") == model or raw.get("id") == model):
  358. return raw
  359. return None
  360. def probe_lmstudio_models(
  361. api_key: Optional[str] = None,
  362. base_url: Optional[str] = None,
  363. timeout: float = 5.0,
  364. ) -> Optional[list[str]]:
  365. """Chat-capable LM Studio model keys — a valid empty list when the server is reachable but has
  366. no non-embedding models; ``None`` on network errors, malformed responses, or bad base URLs.
  367. Raises ``AuthError`` on HTTP 401/403 so token issues surface separately from reachability."""
  368. raw_models = _lmstudio_fetch_raw_models(api_key=api_key, base_url=base_url, timeout=timeout)
  369. if raw_models is None:
  370. return None
  371. keys: list[str] = []
  372. for raw in raw_models:
  373. if not isinstance(raw, dict) or str(raw.get("type") or "").strip().lower() == "embedding":
  374. continue
  375. key = str(raw.get("key") or raw.get("id") or "").strip()
  376. if key and key not in keys:
  377. keys.append(key)
  378. return keys
  379. def fetch_lmstudio_models(
  380. api_key: Optional[str] = None,
  381. base_url: Optional[str] = None,
  382. timeout: float = 5.0,
  383. ) -> list[str]:
  384. """LM Studio chat-capable model keys; ``[]`` when unreachable/malformed. Raises ``AuthError`` on
  385. HTTP 401/403 so callers can tell a wrong ``LM_API_KEY`` from an unreachable server."""
  386. return probe_lmstudio_models(api_key=api_key, base_url=base_url, timeout=timeout) or []
  387. class LMStudioLoadResult(NamedTuple):
  388. """Verified LM Studio runtime plus load-attempt provenance."""
  389. context_length: Optional[int]
  390. load_attempted: bool = False
  391. rejected: bool = False
  392. def _positive_int(value: Any) -> Optional[int]:
  393. if isinstance(value, int) and not isinstance(value, bool) and value > 0:
  394. return value
  395. return None
  396. def _lmstudio_loaded_context(entry: Optional[dict]) -> Optional[int]:
  397. """First positive ``loaded_instances[*].config.context_length`` of a model entry."""
  398. instances = entry.get("loaded_instances") if entry is not None else None
  399. if not isinstance(instances, list):
  400. return None
  401. for instance in instances:
  402. config = instance.get("config") if isinstance(instance, dict) else None
  403. parsed = _positive_int(config.get("context_length") if isinstance(config, dict) else None)
  404. if parsed is not None:
  405. return parsed
  406. return None
  407. def ensure_lmstudio_model_loaded(
  408. model: str,
  409. base_url: Optional[str],
  410. api_key: Optional[str],
  411. target_context_length: Optional[int],
  412. timeout: float = 120.0,
  413. *,
  414. return_load_result: bool = False,
  415. ) -> Optional[int] | LMStudioLoadResult:
  416. """Ensure ``model`` is loaded and return verified runtime context.
  417. Existing loaded-instance context is authoritative. Cold loads omit ``context_length`` unless the
  418. caller supplied an explicit override; the returned context comes from LM Studio's echoed or
  419. refreshed state."""
  420. from hermes_cli.models import _urlopen_model_catalog_request
  421. def _result(context_length: Optional[int], *, load_attempted: bool = False, rejected: bool = False):
  422. result = LMStudioLoadResult(context_length, load_attempted, rejected)
  423. return result if return_load_result else context_length
  424. server_root = _lmstudio_server_root(base_url)
  425. if not server_root:
  426. return _result(None)
  427. explicit_context = _positive_int(target_context_length)
  428. if target_context_length is not None and explicit_context is None:
  429. return _result(None)
  430. target_entry = _lmstudio_entry_for(_lmstudio_raw_models_or_none(api_key, base_url, 10) or [], model)
  431. if target_entry is None:
  432. return _result(None)
  433. max_ctx = _positive_int(target_entry.get("max_context_length"))
  434. if explicit_context is not None and max_ctx is not None and explicit_context > max_ctx:
  435. return _result(None, rejected=True)
  436. current_context = _lmstudio_loaded_context(target_entry)
  437. if current_context is not None:
  438. return _result(current_context)
  439. loaded_instances = target_entry.get("loaded_instances")
  440. if not isinstance(loaded_instances, list) or loaded_instances:
  441. return _result(None)
  442. load_payload: dict[str, Any] = {"model": model, "echo_load_config": True}
  443. if explicit_context is not None:
  444. load_payload["context_length"] = explicit_context
  445. try:
  446. load_request = urllib.request.Request(
  447. server_root + "/api/v1/models/load",
  448. data=json.dumps(load_payload).encode(),
  449. headers={**_lmstudio_request_headers(api_key), "Content-Type": "application/json"},
  450. method="POST",
  451. )
  452. with _urlopen_model_catalog_request(load_request, timeout=timeout) as resp:
  453. response_body = resp.read()
  454. except Exception:
  455. return _result(None, load_attempted=True)
  456. try:
  457. response_payload = json.loads(response_body.decode())
  458. except Exception:
  459. response_payload = None
  460. load_config = response_payload.get("load_config") if isinstance(response_payload, dict) else None
  461. applied_context = _positive_int(load_config.get("context_length")) if isinstance(load_config, dict) else None
  462. if applied_context is not None:
  463. return _result(applied_context, load_attempted=True)
  464. refreshed_models = _lmstudio_raw_models_or_none(api_key, base_url, 10)
  465. if refreshed_models is None:
  466. return _result(None, load_attempted=True)
  467. return _result(_lmstudio_loaded_context(_lmstudio_entry_for(refreshed_models, model)), load_attempted=True)
  468. def lmstudio_model_reasoning_options(
  469. model: str,
  470. base_url: Optional[str],
  471. api_key: Optional[str] = None,
  472. timeout: float = 5.0,
  473. ) -> list[str]:
  474. """Reasoning ``allowed_options`` LM Studio publishes for ``model`` under
  475. ``capabilities.reasoning`` in ``/api/v1/models``; ``[]`` when unknown, unreachable, or absent."""
  476. raw = _lmstudio_entry_for(_lmstudio_raw_models_or_none(api_key, base_url, timeout) or [], model)
  477. if raw is None:
  478. return []
  479. caps = raw.get("capabilities")
  480. reasoning = caps.get("reasoning") if isinstance(caps, dict) else None
  481. opts = reasoning.get("allowed_options") if isinstance(reasoning, dict) else None
  482. if isinstance(opts, list):
  483. return [str(o).strip().lower() for o in opts if isinstance(o, str)]
  484. return []
  485. def ollama_model_supports_thinking(
  486. model: str,
  487. base_url: Optional[str],
  488. api_key: Optional[str] = None,
  489. timeout: float = 5.0,
  490. ) -> Optional[bool]:
  491. """Tri-state: True if an Ollama (Cloud or local) model advertises ``thinking`` in native
  492. ``/api/show`` ``capabilities`` (authoritative; OpenAI-compat ``/v1/models`` omits it), False
  493. when the probe succeeded without it, None when it failed (caller treats as "don't emit")."""
  494. import httpx
  495. server_url = (base_url or "").strip().rstrip("/")
  496. server_url = server_url.removesuffix("/v1")
  497. bare_model = _strip_ollama_cloud_suffix((model or "").strip())
  498. if not server_url or not bare_model:
  499. return None
  500. from agent.command_token_source import materialize_probe_api_key
  501. token = materialize_probe_api_key(api_key)
  502. try:
  503. with httpx.Client(timeout=timeout, headers={"Authorization": f"Bearer {token}"} if token else {}) as client:
  504. resp = client.post(f"{server_url}/api/show", json={"name": bare_model})
  505. if resp.status_code != 200:
  506. return None
  507. caps = resp.json().get("capabilities")
  508. if isinstance(caps, list):
  509. return "thinking" in caps
  510. except Exception:
  511. return None
  512. return None
  513. _OLLAMA_CLOUD_CACHE_TTL = 3600 # 1 hour
  514. def _strip_ollama_cloud_suffix(model_id: str) -> str:
  515. """Strip the ``:cloud`` / ``-cloud`` suffix models.dev appends to Ollama Cloud IDs (the live
  516. API uses bare ids), so the dedup merge does not produce duplicates."""
  517. for suffix in (":cloud", "-cloud"):
  518. if model_id.endswith(suffix):
  519. return model_id[: -len(suffix)]
  520. return model_id
  521. def _ollama_cloud_cache_path() -> Path:
  522. from hermes_constants import get_hermes_home
  523. return get_hermes_home() / "ollama_cloud_models_cache.json"
  524. def _load_ollama_cloud_cache(*, ignore_ttl: bool = False) -> Optional[dict]:
  525. """Load cached Ollama Cloud models from disk (None when missing, empty, or stale)."""
  526. from hermes_cli.models import _read_json_cache
  527. try:
  528. data = _read_json_cache(_ollama_cloud_cache_path())
  529. models = data.get("models") if data is not None else None
  530. if not (isinstance(models, list) and models):
  531. return None
  532. if not ignore_ttl and (time.time() - data.get("cached_at", 0)) > _OLLAMA_CLOUD_CACHE_TTL:
  533. return None # stale
  534. return data
  535. except Exception:
  536. return None
  537. def _save_ollama_cloud_cache(models: list[str]) -> None:
  538. """Persist the merged Ollama Cloud model list to disk. Best-effort."""
  539. from hermes_cli.models import _write_json_cache
  540. try:
  541. _write_json_cache(_ollama_cloud_cache_path(), {"models": models, "cached_at": time.time()}, indent=None)
  542. except Exception:
  543. pass
  544. def fetch_ollama_cloud_models(
  545. api_key: Optional[str] = None,
  546. base_url: Optional[str] = None,
  547. *,
  548. force_refresh: bool = False,
  549. cache_only: bool = False,
  550. ) -> list[str]:
  551. """Ollama Cloud models: fresh disk cache (< 1h, unless force_refresh) → live ``/v1/models``
  552. (freshest) merged with models.dev additions (deduped, live first) → stale cache → models.dev
  553. only → ``[]``. ``cache_only`` (GUI read path) never runs the 8s network probe and never writes
  554. the disk cache. Never None."""
  555. from hermes_cli.models import fetch_api_models
  556. if not force_refresh:
  557. cached = _load_ollama_cloud_cache()
  558. if cached is not None:
  559. return cached["models"]
  560. api_key = api_key or os.getenv("OLLAMA_API_KEY", "")
  561. base_url = base_url or os.getenv("OLLAMA_BASE_URL", "") or "https://ollama.com/v1"
  562. # cache_only (GUI read path): skip only the network probe. The models.dev additions are a local
  563. # cache read, so the row still populates with what is known; the live catalog lands next open.
  564. live_models = [] if cache_only else ((fetch_api_models(api_key, base_url, timeout=8.0) or []) if api_key else [])
  565. mdev_models: list[str] = []
  566. try:
  567. from agent.models_dev import list_agentic_models
  568. mdev_models = list_agentic_models("ollama-cloud")
  569. except Exception:
  570. pass
  571. merged: list[str] = []
  572. for m in [*live_models, *(_strip_ollama_cloud_suffix(m) for m in mdev_models)]:
  573. if m and m not in merged:
  574. merged.append(m)
  575. if live_models:
  576. # Persist only a result that included the live catalog: writing the models.dev-only list here
  577. # (cache_only, or a failed probe) would stamp it fresh, drop the live-only ids, and make the
  578. # next non-cache_only call serve that trimmed list for an hour instead of probing.
  579. _save_ollama_cloud_cache(merged)
  580. return merged
  581. stale = _load_ollama_cloud_cache(ignore_ttl=True)
  582. return stale["models"] if stale is not None else merged