models_pricing.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754
  1. """Live model pricing.
  2. OpenRouter-compatible ``/v1/models`` pricing fetch with a per-endpoint/per-credential cache,
  3. Nous Portal sale chrome and org-policy filtering, and the Vercel AI Gateway / Novita / Fireworks /
  4. DeepInfra pricing adapters. Split out of ``hermes_cli.models``; helpers still defined there are
  5. looked up on ``hermes_cli.models`` at call time so ``patch("hermes_cli.models.<name>")`` mocks keep
  6. intercepting.
  7. """
  8. from __future__ import annotations
  9. import json
  10. import os
  11. import threading
  12. import time
  13. import urllib.request
  14. from typing import Any, Optional
  15. from hermes_cli.models_reasoning_caps import _seed_reasoning_caps
  16. # Cache: maps model_id → {"prompt": str, "completion": str} per endpoint
  17. _pricing_cache: dict[str, dict[str, dict[str, str]]] = {}
  18. # (profile key, provider) → endpoint cache key last fetched, so cached_only reads find the right entry.
  19. _pricing_provider_cache_keys: dict[tuple[str, str], str] = {}
  20. # A failed fetch caches its empty result too, so an unreachable endpoint isn't re-dialed on every
  21. # call — but only until this deadline. Cached forever, one blip at startup would mean no live model
  22. # discovery for the life of a process that runs for weeks (gateway, desktop backend), silently:
  23. # every caller falls back to a curated list meanwhile.
  24. _FAILED_CATALOG_TTL_SECONDS = 120.0
  25. _pricing_cache_retry_after: dict[str, float] = {}
  26. # Guards a catalog and its deadline as one unit: expiring one key (reader or sweep) pops two tables, and a
  27. # refresh landing between the pops would otherwise keep its catalog without its deadline, forever.
  28. _pricing_cache_lock = threading.Lock()
  29. def _cached_catalog(cache_key: str) -> Optional[dict[str, dict[str, Any]]]:
  30. """The cached catalog for *cache_key*, or None to go fetch it."""
  31. with _pricing_cache_lock:
  32. cached = _pricing_cache.get(cache_key)
  33. if cached is None:
  34. return None
  35. retry_after = _pricing_cache_retry_after.get(cache_key)
  36. if retry_after is not None and time.monotonic() >= retry_after:
  37. _pricing_cache.pop(cache_key, None)
  38. _pricing_cache_retry_after.pop(cache_key, None)
  39. return None
  40. return cached
  41. def _cache_catalog(
  42. cache_key: str,
  43. result: dict[str, dict[str, Any]],
  44. ttl_seconds: Optional[float] = None,
  45. ) -> dict[str, dict[str, Any]]:
  46. """Cache a catalog result, giving an empty one an expiry. *ttl_seconds* expires a non-empty
  47. result too — only for a catalog whose contents depend on server-side state the client cannot
  48. observe (an org's model policy can change while a long-lived process holds the entry).
  49. Entries past their deadline are dropped here as the reader would: a rotated credential's
  50. catalog is otherwise never read again and stays resident for the life of the process."""
  51. with _pricing_cache_lock:
  52. now = time.monotonic()
  53. for key, deadline in list(_pricing_cache_retry_after.items()):
  54. if now >= deadline:
  55. _pricing_cache.pop(key, None)
  56. _pricing_cache_retry_after.pop(key, None)
  57. _pricing_cache[cache_key] = result
  58. if not result:
  59. _pricing_cache_retry_after[cache_key] = now + _FAILED_CATALOG_TTL_SECONDS
  60. elif ttl_seconds:
  61. _pricing_cache_retry_after[cache_key] = now + ttl_seconds
  62. else:
  63. _pricing_cache_retry_after.pop(cache_key, None)
  64. return result
  65. # NUL cannot appear in a URL, so this cannot collide with a real base URL.
  66. _PRICING_AUTH_KEY_PREFIX = "\x00auth:"
  67. def _pricing_auth_fingerprint(api_key: str | None) -> str:
  68. """Cache-key suffix identifying the credential a catalog was read with: a governed endpoint
  69. answers each token with the catalog its org may reach, so two credentials cannot share an
  70. entry. blake2b for fingerprinting only (same rationale as ``_custom_endpoint_fingerprint``)."""
  71. if not api_key:
  72. return ""
  73. import hashlib
  74. digest = hashlib.blake2b(api_key.encode("utf-8", errors="replace"), digest_size=8)
  75. return _PRICING_AUTH_KEY_PREFIX + digest.hexdigest()
  76. def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]:
  77. """Pricing already cached for *base_url* (with or without ``/v1``), or ``{}``; never fetches.
  78. Prefers an authenticated catalog, scanning newest first (callers hold no credential) and
  79. skipping expired entries so a rotated credential does not answer from its predecessor's."""
  80. root = _strip_v1((base_url or "").rstrip("/"))
  81. authed_prefix = root + _PRICING_AUTH_KEY_PREFIX
  82. for key in reversed(list(_pricing_cache)):
  83. if key.startswith(authed_prefix):
  84. cached = _cached_catalog(key)
  85. if cached:
  86. return cached
  87. return _cached_catalog(root) or {}
  88. def pricing_fetch_suppressed(base_url: str) -> bool:
  89. """A fetch for *base_url* failed recently and its empty result is still cached, so re-fetching now
  90. returns that ``{}`` without dialing (see ``_cache_catalog``). Lets a caller that would otherwise
  91. start a background refresh on a cold peek skip it for the rest of the failure window."""
  92. root = _strip_v1((base_url or "").rstrip("/"))
  93. now = time.monotonic()
  94. return any(
  95. _pricing_cache.get(key) == {} and _pricing_cache_retry_after.get(key, 0.0) > now
  96. for key in list(_pricing_cache)
  97. if key == root or key.startswith(root + _PRICING_AUTH_KEY_PREFIX)
  98. )
  99. def _strip_v1(url: str) -> str:
  100. return url[:-3].rstrip("/") if url.endswith("/v1") else url
  101. def _format_price_per_mtok(per_token_str: str) -> str:
  102. """Per-token price string → $/Mtok string. Always 2 decimals so right-justified prices align;
  103. sub-cent prices (deep-discount cache-hit promos) widen precision until the value shows, keep
  104. one extra digit and trim trailing zeros instead of collapsing to "$0.00"."""
  105. try:
  106. val = float(per_token_str)
  107. except (TypeError, ValueError):
  108. return "?"
  109. if val == 0:
  110. return "free"
  111. per_m = val * 1_000_000
  112. text = f"{per_m:.2f}"
  113. if per_m < 0.01:
  114. prec = 3
  115. while prec < 12 and round(per_m, prec) == 0:
  116. prec += 1
  117. text = f"{per_m:.{min(prec + 1, 12)}f}".rstrip("0").rstrip(".")
  118. return f"${text}"
  119. def _price_float(raw: Any, *, positive: bool) -> float | None:
  120. """*raw* as a finite float (> 0, or >= 0 when not *positive*); None when unset/invalid/NaN."""
  121. if raw in (None, ""):
  122. return None
  123. try:
  124. n = float(raw)
  125. except (TypeError, ValueError):
  126. return None
  127. if n != n or (n <= 0 if positive else n < 0):
  128. return None
  129. return n
  130. def _sale_pct(current: Any, original: Any) -> int | None:
  131. """Percent discount when *current* is strictly below *original* (both positive finite)."""
  132. cur, orig = _price_float(current, positive=True), _price_float(original, positive=True)
  133. if cur is None or orig is None or cur >= orig:
  134. return None
  135. return round((1.0 - (cur / orig)) * 100)
  136. def compute_sale_discount(prompt: str, completion: str, original: Any) -> tuple[int, str, str] | None:
  137. """Sale chrome from gateway ``pricing.original`` (Nous Portal only; callers gate on the provider
  138. and opted in via ``include_sale_original=True``): ``(discount_percent, was_prompt_raw,
  139. was_completion_raw)`` when ``original`` is a dict and the current prompt (fallback: completion)
  140. rate is strictly below the original. Free / $0 models get a flat 100% off, with "was" prices
  141. only when the gateway served an original (a natively-free stealth model gets bare "-100%")."""
  142. orig_dict = original if isinstance(original, dict) else {}
  143. was_prompt = orig_dict.get("prompt")
  144. was_completion = orig_dict.get("completion")
  145. was_prompt_str = str(was_prompt) if was_prompt not in (None, "") else ""
  146. was_completion_str = str(was_completion) if was_completion not in (None, "") else ""
  147. if _price_float(prompt, positive=False) == 0 and _price_float(completion, positive=False) in (0, None):
  148. return (100, was_prompt_str, was_completion_str)
  149. if not isinstance(original, dict) or (not was_prompt_str and not was_completion_str):
  150. return None
  151. pct = _sale_pct(prompt, was_prompt)
  152. if pct is not None:
  153. return (pct, was_prompt_str, was_completion_str) if pct >= 1 else None
  154. pct = _sale_pct(completion, was_completion)
  155. if pct is not None:
  156. return (pct, was_prompt_str, was_completion_str) if pct >= 1 else None
  157. return None
  158. def _get_json(url: str, headers: dict[str, str], timeout: float, opener=None) -> Optional[dict]:
  159. """GET *url* as JSON via the origin's catalog opener (or *opener*); None on any failure."""
  160. from hermes_cli.models import _urlopen_model_catalog_request
  161. try:
  162. req = urllib.request.Request(url, headers=headers)
  163. with (opener or _urlopen_model_catalog_request)(req, timeout=timeout) as resp:
  164. return json.loads(resp.read().decode())
  165. except Exception:
  166. return None
  167. def _pricing_entry(pricing: dict, prompt_key: str = "prompt", completion_key: str = "completion") -> dict[str, Any]:
  168. """Picker-shape ``{prompt, completion[, input_cache_read, input_cache_write]}`` from a catalog
  169. ``pricing`` block whose cache fields already use the hermes names."""
  170. entry: dict[str, Any] = {
  171. "prompt": str(pricing.get(prompt_key, "")),
  172. "completion": str(pricing.get(completion_key, "")),
  173. }
  174. for key in ("input_cache_read", "input_cache_write"):
  175. if pricing.get(key):
  176. entry[key] = str(pricing[key])
  177. return entry
  178. def _per_token(per_mtok: Any) -> str:
  179. """$/MTok → the per-token price string the picker expects."""
  180. return str(float(per_mtok) / 1_000_000)
  181. def _catalog_items(payload: dict) -> list[dict]:
  182. return [item for item in payload.get("data", []) if isinstance(item, dict)]
  183. def fetch_models_with_pricing(
  184. api_key: str | None = None,
  185. base_url: str = "https://openrouter.ai/api",
  186. timeout: float = 8.0,
  187. *,
  188. force_refresh: bool = False,
  189. include_sale_original: bool = False,
  190. cache_ttl_seconds: Optional[float] = None,
  191. ) -> dict[str, dict[str, Any]]:
  192. """Fetch ``/v1/models`` (any OpenRouter-compatible endpoint) → ``{model_id: {prompt, completion,
  193. ...}}``, cached per *base_url* and per credential so one caller's catalog never answers
  194. another's read. *include_sale_original* (Nous Portal only) copies the gateway's pre-discount
  195. ``pricing.original`` rates through as a nested ``original`` dict for sale chrome."""
  196. from hermes_cli.chat_catalog import catalog_item_is_generation
  197. from hermes_cli.models import _openrouter_model_supports_tools
  198. from hermes_cli.version_info import hermes_user_agent
  199. url_root = (base_url or "").rstrip("/")
  200. cache_key = url_root + _pricing_auth_fingerprint(api_key)
  201. if not force_refresh:
  202. cached = _cached_catalog(cache_key)
  203. if cached is not None:
  204. return cached
  205. url = url_root + "/v1/models"
  206. headers = {"Accept": "application/json", "User-Agent": hermes_user_agent()}
  207. if api_key:
  208. headers["Authorization"] = f"Bearer {api_key}"
  209. payload = _get_json(url, headers, timeout)
  210. if payload is None:
  211. return _cache_catalog(cache_key, {})
  212. # Same document the reasoning-capability fetch would pull — mirror it so a later hot-path
  213. # lookup (and the next process) has an answer without its own round-trip.
  214. _seed_reasoning_caps(url, payload.get("data"))
  215. result: dict[str, dict[str, Any]] = {}
  216. for item in payload.get("data", []):
  217. mid, pricing = item.get("id"), item.get("pricing")
  218. if mid and isinstance(pricing, dict):
  219. entry = _pricing_entry(pricing)
  220. # Sale chrome is Nous Portal-only; never copy pricing.original for other catalogs.
  221. original = pricing.get("original") if include_sale_original else None
  222. if isinstance(original, dict):
  223. orig_entry = {key: str(original[key]) for key in ("prompt", "completion", "input_cache_read", "input_cache_write")
  224. if original.get(key) not in (None, "")}
  225. if orig_entry.get("prompt") or orig_entry.get("completion"):
  226. entry["original"] = orig_entry
  227. # Nous Portal-only: the gateway bills this row to a subscription the account holds, not to credits.
  228. if include_sale_original and item.get("billing_mode") == "subscription":
  229. entry["billing_mode"] = "subscription"
  230. # Usage-window plans only (only those callers ever see the mark): the gateway marks the
  231. # models that count against the plan's monthly large-model limit.
  232. if include_sale_original and item.get("top_tier") is True:
  233. entry["top_tier"] = True
  234. # Same callers: the gateway's per-plan picker label (free / low / medium / top_tier).
  235. if include_sale_original and isinstance(item.get("usage_level"), str):
  236. entry["usage_level"] = item["usage_level"]
  237. # Nous only: the catalog's display name, so a limit line can name a model the way the
  238. # picker does ("Model A", not the routed id).
  239. if include_sale_original and isinstance(item.get("name"), str) and item["name"].strip() \
  240. and item["name"].strip() != mid:
  241. entry["name"] = item["name"].strip()
  242. # Nous Portal-only: on-sale rows join the picker, which must not offer a tool-less model.
  243. if include_sale_original and not _openrouter_model_supports_tools(item):
  244. entry["tools"] = False
  245. # ...nor an image/video generation model (the chat-catalog rule, applied to the row itself).
  246. if include_sale_original and catalog_item_is_generation(item):
  247. entry["generation"] = True
  248. result[mid] = entry
  249. return _cache_catalog(cache_key, result, cache_ttl_seconds)
  250. def fetch_ai_gateway_pricing(timeout: float = 8.0, *, force_refresh: bool = False) -> dict[str, dict[str, str]]:
  251. """Vercel AI Gateway /v1/models pricing, translating its ``input`` / ``output`` field names to
  252. the picker's ``prompt`` / ``completion`` (cache read/write names already match)."""
  253. from hermes_constants import AI_GATEWAY_BASE_URL
  254. cache_key = AI_GATEWAY_BASE_URL.rstrip("/")
  255. if not force_refresh:
  256. cached = _cached_catalog(cache_key)
  257. if cached is not None:
  258. return cached
  259. payload = _get_json(f"{cache_key}/models", {"Accept": "application/json"}, timeout, opener=urllib.request.urlopen)
  260. if payload is None:
  261. return _cache_catalog(cache_key, {})
  262. result: dict[str, dict[str, str]] = {}
  263. for item in _catalog_items(payload):
  264. mid, pricing = item.get("id"), item.get("pricing")
  265. if mid and isinstance(pricing, dict):
  266. result[mid] = _pricing_entry(pricing, "input", "output")
  267. return _cache_catalog(cache_key, result)
  268. def _resolve_openrouter_api_key() -> str:
  269. """Best-effort OpenRouter API key for pricing fetch."""
  270. return os.getenv("OPENROUTER_API_KEY", "").strip()
  271. _DEFAULT_NOUS_INFERENCE_BASE = "https://inference-api.nousresearch.com"
  272. def _resolve_nous_pricing_credentials() -> tuple[str, str]:
  273. """``(api_key, base_url)`` for Nous Portal pricing; base_url is the bare origin (no ``/v1``).
  274. Precedence mirrors runtime credential resolution: ``NOUS_INFERENCE_BASE_URL`` (staging /
  275. preview) → resolved credential ``base_url`` → production default. Without the override a
  276. staging profile's sale ``pricing.original`` would never reach the pickers."""
  277. try:
  278. from hermes_cli.auth import _nous_inference_env_override
  279. env_base = _nous_inference_env_override()
  280. except Exception:
  281. env_base = None
  282. api_key = creds_base = ""
  283. try:
  284. from hermes_cli.anon_challenge import background_caller
  285. from hermes_cli.auth import resolve_nous_runtime_credentials
  286. with background_caller(): # a pricing read never opens a free-tier challenge
  287. creds = resolve_nous_runtime_credentials()
  288. if creds:
  289. api_key = creds.get("api_key", "") or ""
  290. creds_base = (creds.get("base_url", "") or "").strip()
  291. except Exception:
  292. pass
  293. base_url = (env_base or creds_base or _DEFAULT_NOUS_INFERENCE_BASE).rstrip("/")
  294. base_url = base_url.removesuffix("/v1")
  295. return (api_key, base_url)
  296. # How long a Nous catalog stays trusted. Its contents depend on the org's policy, which an admin
  297. # can change at any time and the client cannot observe, so a long-lived process must re-ask.
  298. # Other providers' catalogs carry no such state and keep the default no-expiry caching.
  299. _NOUS_CATALOG_TTL_SECONDS = 300.0
  300. def _fetch_nous_pricing(api_key: str, base_url: str, *, force_refresh: bool) -> dict[str, dict[str, Any]]:
  301. """Shared by pricing and policy lookups so both read one cache entry."""
  302. return fetch_models_with_pricing(
  303. api_key=api_key,
  304. base_url=base_url,
  305. force_refresh=force_refresh,
  306. include_sale_original=True, # Sale chrome (pricing.original) is Nous Portal-only.
  307. cache_ttl_seconds=_NOUS_CATALOG_TTL_SECONDS,
  308. )
  309. def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str]]:
  310. """The Nous model ids the caller's org may reach (keys of an authenticated ``GET /v1/models``,
  311. which omits policy-blocked rows), or ``None`` to not filter: no policy (or a token too old to
  312. say), an anonymous read (unfiltered catalog), or an empty read (a fetch failure, not an org
  313. that may reach nothing)."""
  314. try:
  315. from hermes_cli.nous_account import nous_policy_present
  316. if nous_policy_present() is not True:
  317. return None
  318. except Exception:
  319. return None
  320. api_key, base_url = _resolve_nous_pricing_credentials()
  321. if not api_key or not base_url:
  322. return None
  323. return set(_fetch_nous_pricing(api_key, base_url, force_refresh=force_refresh)) or None
  324. # Past this size an allowed set reads as a whole catalog rather than an allowlist, and is not
  325. # worth showing in place of an empty picker.
  326. _NOUS_POLICY_APPEND_MAX = 64
  327. def restrict_to_nous_policy(
  328. model_ids: list[str],
  329. allowed: Optional[set[str]],
  330. *,
  331. rescue_empty: bool = False,
  332. ) -> list[str]:
  333. """*model_ids* narrowed to *allowed*, preserving order. A ``:free`` sibling is kept when its
  334. base model is reachable (the gateway admits a row when any requestable id passes); over-listing
  335. costs a 403 from the authoritative gate, hiding a servable row is unrecoverable client-side.
  336. *rescue_empty*: an allowlist naming only models the curated manifest lacks would leave an empty
  337. picker — worse than no filter — so return the allowlist itself. Opt-in per list: an already-
  338. empty list (a paid tier's gated models) means "nothing to gate", not "nothing survived"."""
  339. if not allowed:
  340. return list(model_ids)
  341. kept = [mid for mid in model_ids if mid in allowed or mid.split(":", 1)[0] in allowed]
  342. if rescue_empty and not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX:
  343. return sorted(allowed)
  344. return kept
  345. def _remember_provider_cache_key(provider: str, cache_key: str) -> None:
  346. from hermes_cli.models import _pricing_profile_key
  347. _pricing_provider_cache_keys[(_pricing_profile_key(), provider)] = cache_key
  348. def _fetch_openrouter_pricing(*, force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  349. _remember_provider_cache_key("openrouter", _OPENROUTER_PRICING_BASE)
  350. return fetch_models_with_pricing(
  351. api_key=_resolve_openrouter_api_key(),
  352. base_url=_OPENROUTER_PRICING_BASE,
  353. force_refresh=force_refresh,
  354. )
  355. def _fetch_ai_gateway_pricing_for_provider(*, force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  356. _remember_provider_cache_key("ai-gateway", _ai_gateway_pricing_scope())
  357. return fetch_ai_gateway_pricing(force_refresh=force_refresh)
  358. def _fetch_novita_pricing_for_provider(*, force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  359. _remember_provider_cache_key("novita", _novita_pricing_scope())
  360. return _fetch_novita_pricing(force_refresh=force_refresh)
  361. def _kilo_pricing_scope() -> str:
  362. """The kilocode provider profile's gateway base (``KILOCODE_BASE_URL`` overrides it) —
  363. the endpoint the doctor connectivity probe and chat requests already use, so pricing reads
  364. the same server the user is actually wired to."""
  365. from providers import get_provider_profile
  366. base_url = (
  367. os.getenv("KILOCODE_BASE_URL", "").strip()
  368. or getattr(get_provider_profile("kilocode"), "base_url", "")
  369. or ""
  370. )
  371. return base_url.rstrip("/").removesuffix("/v1")
  372. def _fetch_kilocode_pricing(timeout: float = 8.0, *, force_refresh: bool = False) -> dict[str, dict[str, str]]:
  373. """Kilo Gateway ``{base}/models`` pricing (public OpenRouter-shaped catalog — per-token
  374. ``prompt``/``completion``/``input_cache_read`` strings, no key required, attached best-effort
  375. so an authenticated read also carries the key), cached on the catalog URL.
  376. ``isFree`` is the explicit free flag the picker badge needs — authoritative over the
  377. rates, because a free router (``kilo-auto/free``) can carry non-zero underlying pricing.
  378. The gateway's ``-1`` rate sentinel means "varies" (auto-router models); no price is shown
  379. rather than a fabricated one, so such rows are skipped.
  380. """
  381. from hermes_cli.version_info import hermes_user_agent
  382. cache_key = _kilo_pricing_scope()
  383. if not cache_key:
  384. return {}
  385. if not force_refresh:
  386. cached = _cached_catalog(cache_key)
  387. if cached is not None:
  388. return cached
  389. headers = {"Accept": "application/json", "User-Agent": hermes_user_agent()}
  390. api_key = os.getenv("KILOCODE_API_KEY", "").strip()
  391. if api_key:
  392. headers["Authorization"] = f"Bearer {api_key}"
  393. payload = _get_json(cache_key + "/models", headers, timeout)
  394. if payload is None:
  395. return _cache_catalog(cache_key, {})
  396. result: dict[str, dict[str, str]] = {}
  397. for item in _catalog_items(payload):
  398. mid, pricing = item.get("id"), item.get("pricing")
  399. if not mid or not isinstance(pricing, dict):
  400. continue
  401. entry = _pricing_entry(pricing)
  402. if item.get("isFree"):
  403. entry["prompt"] = entry["completion"] = "0"
  404. entry.pop("input_cache_read", None)
  405. # The gateway's ``-1`` sentinel means "rate varies" (auto-router models); no price is
  406. # shown rather than a fabricated one. _price_float also drops any other unusable value
  407. # (blank / non-numeric), so a malformed row cannot poison the rest of the catalog.
  408. if _price_float(entry["prompt"], positive=False) is None or _price_float(entry["completion"], positive=False) is None:
  409. continue
  410. if entry:
  411. result[str(mid)] = entry
  412. return _cache_catalog(cache_key, result)
  413. def _fetch_fireworks_pricing_for_provider(*, force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  414. _remember_provider_cache_key("fireworks", _FIREWORKS_PRICING_KEY)
  415. return _fireworks_pricing_from_models_dev(force_refresh=force_refresh)
  416. def _fetch_kilocode_pricing_for_provider(*, force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  417. _remember_provider_cache_key("kilocode", _kilo_pricing_scope())
  418. return _fetch_kilocode_pricing(force_refresh=force_refresh)
  419. def _fetch_nous_pricing_for_provider(*, force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  420. api_key, base_url = _resolve_nous_pricing_credentials()
  421. if not base_url:
  422. return {}
  423. # Register the SAME key fetch_models_with_pricing writes (base + credential fingerprint):
  424. # _cached_only_pricing resolves cache entries through this map, and a logged-in account's
  425. # catalog lands under the fingerprinted key — a bare base here meant the picker's
  426. # cached_only read never saw the rows the prewarm filled (review on #132015).
  427. # pricing_cache_scope's last-resort fallback reads this value too; there it serves only as
  428. # an identity token for the prewarm single-flight, where including the credential is
  429. # equally correct (a key rotation means a new catalog and deserves a fresh worker).
  430. _remember_provider_cache_key("nous", base_url.rstrip("/") + _pricing_auth_fingerprint(api_key))
  431. return _fetch_nous_pricing(api_key, base_url, force_refresh=force_refresh)
  432. _OPENROUTER_PRICING_BASE = "https://openrouter.ai/api"
  433. _FIREWORKS_PRICING_KEY = "models.dev/fireworks"
  434. def _ai_gateway_pricing_scope() -> str:
  435. from hermes_constants import AI_GATEWAY_BASE_URL
  436. return AI_GATEWAY_BASE_URL.rstrip("/")
  437. def _novita_pricing_scope() -> str:
  438. return (os.getenv("NOVITA_BASE_URL", "").strip() or "https://api.novita.ai/openai/v1").rstrip("/")
  439. def get_cached_nous_inference_base_url() -> str:
  440. """The profile's persisted Nous endpoint (bare origin, no ``/v1``) without refreshing auth."""
  441. try:
  442. from hermes_cli.auth import (
  443. _load_auth_store, _load_provider_state, _optional_base_url, _validate_nous_inference_url_from_network,
  444. )
  445. state = _load_provider_state(_load_auth_store(), "nous") or {}
  446. url = _validate_nous_inference_url_from_network(_optional_base_url(state.get("inference_base_url"))) or ""
  447. return url.rstrip("/").removesuffix("/v1")
  448. except Exception:
  449. return ""
  450. # Static endpoint identity per provider; dynamic ones (deepinfra, nous) are resolved in pricing_cache_scope.
  451. _STATIC_PRICING_SCOPES = {
  452. "openrouter": lambda: _OPENROUTER_PRICING_BASE,
  453. "ai-gateway": _ai_gateway_pricing_scope,
  454. "novita": _novita_pricing_scope,
  455. "kilocode": _kilo_pricing_scope,
  456. "fireworks": lambda: _FIREWORKS_PRICING_KEY,
  457. }
  458. def resolve_pricing_provider(provider: str, *, base_url: str = "") -> str:
  459. """Return the canonical pricing source for a provider, or ``""`` when unproven.
  460. A ``custom:<name>`` inventory slug identifies a user configuration entry, not its upstream.
  461. It can deliberately shadow a canonical name while pointing at a proxy, so custom entries may
  462. use pricing only when their concrete endpoint is recognized as that canonical upstream.
  463. """
  464. from hermes_cli.models import normalize_provider
  465. normalized = normalize_provider(provider)
  466. if not normalized.startswith("custom:"):
  467. return normalized
  468. if not base_url:
  469. return ""
  470. try:
  471. from agent.model_metadata import _URL_TO_PROVIDER, _infer_provider_from_url
  472. from providers import get_provider_profile
  473. from utils import base_url_host_matches, base_url_hostname
  474. inferred = _infer_provider_from_url(base_url)
  475. profile = get_provider_profile(inferred) if inferred else None
  476. except Exception:
  477. return ""
  478. # ``_infer_provider_from_url`` is deliberately broad for user-facing hints. Pricing is a
  479. # trust boundary: require an exact official hostname (or one of its explicit subdomains),
  480. # never a mere substring such as ``openrouter.ai.attacker.invalid``. A profile's claimed
  481. # hosts count only when named exactly.
  482. claimed = (*profile.hostnames, *profile.metadata_hostnames) if profile is not None else ()
  483. if not inferred or not (base_url_hostname(base_url) in claimed or any(
  484. provider == inferred and not host.startswith(".") and base_url_host_matches(base_url, host)
  485. for host, provider in _URL_TO_PROVIDER.items()
  486. )):
  487. return ""
  488. return normalize_provider(inferred)
  489. def pricing_cache_scope(
  490. provider: str, *, base_url: str = "", current_provider: str = "", current_base_url: str = ""
  491. ) -> str:
  492. """The current endpoint identity a provider's pricing cache is keyed on. Resolves local configuration
  493. only, never fetches: picker prewarm single-flight uses it so an endpoint rotation can start a new
  494. worker while the previous endpoint is still slow or unreachable."""
  495. from hermes_cli.models import _deepinfra_catalog_url, _pricing_profile_key, normalize_provider
  496. normalized = resolve_pricing_provider(provider, base_url=base_url)
  497. static = _STATIC_PRICING_SCOPES.get(normalized)
  498. if static:
  499. return static()
  500. if normalized == "deepinfra":
  501. return _deepinfra_catalog_url()[0]
  502. if normalized == "nous":
  503. try:
  504. from hermes_cli.auth import _nous_inference_env_override
  505. env_base = _nous_inference_env_override()
  506. except Exception:
  507. env_base = None
  508. if env_base:
  509. return env_base.rstrip("/").removesuffix("/v1")
  510. if normalize_provider(current_provider) == "nous" and current_base_url:
  511. return current_base_url.rstrip("/").removesuffix("/v1")
  512. persisted_base = get_cached_nous_inference_base_url()
  513. if persisted_base:
  514. return persisted_base
  515. return _pricing_provider_cache_keys.get((_pricing_profile_key(), normalized), _DEFAULT_NOUS_INFERENCE_BASE)
  516. return ""
  517. def _cached_only_pricing(normalized: str) -> dict[str, dict[str, str]]:
  518. """Process-resident pricing for *normalized* without any provider I/O."""
  519. from hermes_cli.models import _deepinfra_catalog_cache, _deepinfra_catalog_url, _pricing_profile_key
  520. if normalized == "deepinfra":
  521. cache_key, _url = _deepinfra_catalog_url()
  522. return _fetch_deepinfra_pricing() if cache_key in _deepinfra_catalog_cache else {}
  523. cache_key = _pricing_provider_cache_keys.get((_pricing_profile_key(), normalized))
  524. if cache_key is None and normalized in ("openrouter", "ai-gateway", "fireworks"):
  525. cache_key = _STATIC_PRICING_SCOPES[normalized]()
  526. return (_cached_catalog(cache_key) or {}) if cache_key else {}
  527. def get_pricing_for_provider(
  528. provider: str, *, base_url: str = "", force_refresh: bool = False, cached_only: bool = False
  529. ) -> dict[str, dict[str, str]]:
  530. """Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita,
  531. deepinfra, fireworks, kilocode); ``{}`` for everything else. ``cached_only`` never starts
  532. provider I/O: normal picker opens use it so cold endpoints cannot hold the response path,
  533. while a background prewarm fills the same caches for later opens."""
  534. normalized = resolve_pricing_provider(provider, base_url=base_url)
  535. if cached_only:
  536. return _cached_only_pricing(normalized)
  537. fetcher = _PRICING_FETCHERS.get(normalized)
  538. return fetcher(force_refresh=force_refresh) if fetcher else {}
  539. def _fireworks_pricing_from_models_dev(*, force_refresh: bool = False) -> dict[str, dict[str, str]]:
  540. """Fireworks picker pricing from the models.dev registry cache (``fetch_models_dev()`` keeps a
  541. shared in-memory + disk cache, 1h TTL) — a pure dict transform, no per-render network call."""
  542. cache_key = "models.dev/fireworks"
  543. if not force_refresh:
  544. cached = _cached_catalog(cache_key)
  545. if cached is not None:
  546. return cached
  547. result: dict[str, dict[str, str]] = {}
  548. try:
  549. from agent.models_dev import _get_provider_models
  550. for mid, entry in (_get_provider_models("fireworks") or {}).items():
  551. cost = entry.get("cost") if isinstance(entry, dict) else None
  552. if not isinstance(cost, dict):
  553. continue
  554. inp, out = cost.get("input"), cost.get("output")
  555. if inp is None and out is None:
  556. continue
  557. row = {"prompt": _per_token(inp or 0), "completion": _per_token(out or 0)}
  558. if cost.get("cache_read"):
  559. row["input_cache_read"] = _per_token(cost["cache_read"])
  560. result[str(mid)] = row
  561. except Exception:
  562. result = {}
  563. return _cache_catalog(cache_key, result)
  564. def _fetch_novita_pricing(timeout: float = 8.0, *, force_refresh: bool = False) -> dict[str, dict[str, str]]:
  565. """NovitaAI /v1/models pricing (per-million prices in units of 0.0001 USD → per-token strings),
  566. cached on the resolved base URL so menu renders don't re-hit the network."""
  567. from hermes_cli.version_info import hermes_user_agent
  568. api_key = os.getenv("NOVITA_API_KEY", "").strip()
  569. if not api_key:
  570. return {}
  571. cache_key = (os.getenv("NOVITA_BASE_URL", "").strip() or "https://api.novita.ai/openai/v1").rstrip("/")
  572. if not force_refresh:
  573. cached = _cached_catalog(cache_key)
  574. if cached is not None:
  575. return cached
  576. headers = {"Authorization": f"Bearer {api_key}", "Accept": "application/json", "User-Agent": hermes_user_agent()}
  577. payload = _get_json(cache_key + "/models", headers, timeout)
  578. if payload is None:
  579. return _cache_catalog(cache_key, {})
  580. result: dict[str, dict[str, str]] = {}
  581. for item in _catalog_items(payload):
  582. mid = item.get("id")
  583. inp, out = item.get("input_token_price_per_m"), item.get("output_token_price_per_m")
  584. if not mid or (inp is None and out is None):
  585. continue
  586. result[str(mid)] = {
  587. "prompt": str(float(inp or 0) / 10_000 / 1_000_000),
  588. "completion": str(float(out or 0) / 10_000 / 1_000_000),
  589. }
  590. return _cache_catalog(cache_key, result)
  591. def _fetch_deepinfra_pricing(timeout: float = 5.0, *, force_refresh: bool = False) -> dict[str, dict[str, str]]:
  592. """DeepInfra chat-model pricing: ``input_tokens`` / ``output_tokens`` / ``cache_read_tokens`` in
  593. $/MTok → per-token ``prompt`` / ``completion`` / ``input_cache_read`` (cached by the by-tag
  594. helper)."""
  595. from hermes_cli.models import _fetch_deepinfra_models_by_tag
  596. items = _fetch_deepinfra_models_by_tag("chat", timeout=timeout, force_refresh=force_refresh)
  597. result: dict[str, dict[str, str]] = {}
  598. for item in items or []:
  599. metadata = item.get("metadata") or {}
  600. pricing = metadata.get("pricing") if isinstance(metadata, dict) else None
  601. if not isinstance(pricing, dict):
  602. continue
  603. entry = {
  604. ours: _per_token(pricing[theirs])
  605. for theirs, ours in (("input_tokens", "prompt"), ("output_tokens", "completion"), ("cache_read_tokens", "input_cache_read"))
  606. if pricing.get(theirs) is not None
  607. }
  608. if entry:
  609. result[item["id"]] = entry
  610. return result
  611. _PRICING_FETCHERS = {
  612. "openrouter": _fetch_openrouter_pricing,
  613. "ai-gateway": _fetch_ai_gateway_pricing_for_provider,
  614. "novita": _fetch_novita_pricing_for_provider,
  615. "deepinfra": _fetch_deepinfra_pricing,
  616. "fireworks": _fetch_fireworks_pricing_for_provider,
  617. "kilocode": _fetch_kilocode_pricing_for_provider,
  618. "nous": _fetch_nous_pricing_for_provider,
  619. }