model_metadata.py 144 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627
  1. """Model metadata, context lengths, and token estimation utilities.
  2. Pure utility functions with no AIAgent dependency. Used by ContextCompressor
  3. and run_agent.py for pre-flight context checks.
  4. """
  5. import contextlib
  6. import hashlib
  7. import ipaddress
  8. import json
  9. import logging
  10. import re
  11. import time
  12. from pathlib import Path
  13. from typing import Any, Callable, Dict, List, Optional, Tuple
  14. from urllib.parse import urlparse
  15. import hermes_yaml as yaml
  16. from agent import model_metadata_http
  17. from agent.model_metadata_cache import _write_context_cache, locked_context_cache
  18. from agent.context_default_record import mark_context_defaulted, records_context_defaults
  19. from agent.model_metadata_llamacpp import llamacpp_context
  20. from agent.model_metadata_memo_bounds import MEMO_LOCK, bound_codex_caches, remember_endpoint_models as _remember_endpoint_models
  21. from utils import atomic_json_write, base_url_host_matches, base_url_hostname
  22. from hermes_constants import OPENROUTER_MODELS_URL, openrouter_variant_base
  23. from agent.message_metadata import PERSISTENCE_ONLY_MESSAGE_FIELDS, unpriced_replay_fields, without_persistence_fields
  24. from agent.model_metadata_token_memo import ( # _MSG_TOKENS_CACHE: tests and benches read it here
  25. _MSG_TOKENS_CACHE, MemoPins, lookup_message_tokens, remember_message_tokens,
  26. )
  27. logger = logging.getLogger(__name__)
  28. # Snapshot for callers inspecting this constant; prefix routing queries the registry live.
  29. try:
  30. from providers import list_providers as _list_providers
  31. except Exception:
  32. def _list_providers():
  33. return []
  34. _PROVIDER_PREFIXES: frozenset[str] = frozenset(
  35. value.lower()
  36. for profile in _list_providers()
  37. for value in (profile.name, *profile.aliases)
  38. )
  39. _OLLAMA_TAG_PATTERN = re.compile(r"^(\d+\.?\d*b|latest|stable|q\d|fp?\d|instruct|chat|coder|vision|text)", re.IGNORECASE)
  40. # Tailscale CGNAT (RFC 6598): `ipaddress.is_private` excludes it, yet Ollama
  41. # reached over Tailscale must count as local (timeout auto-bumps).
  42. _TAILSCALE_CGNAT = ipaddress.IPv4Network("100.64.0.0/10")
  43. def _strip_provider_prefix(model: str) -> str:
  44. """Strip a registry-known provider prefix: ``"local:m"`` -> ``"m"``. Ollama ``model:tag``
  45. ids are preserved even when the model half is a provider name (``qwen:0.5b``)."""
  46. if ":" not in model or model.startswith("http"):
  47. return model
  48. prefix, suffix = model.split(":", 1)
  49. try:
  50. from providers import get_provider_profile
  51. is_provider = get_provider_profile(prefix.strip().lower()) is not None
  52. except Exception:
  53. is_provider = False
  54. if is_provider and not _OLLAMA_TAG_PATTERN.match(suffix.strip()):
  55. return suffix
  56. return model
  57. _model_metadata_cache: dict[str, dict[str, Any]] = {}
  58. _model_metadata_cache_time: float = 0
  59. _MODEL_CACHE_TTL = 3600
  60. # In-memory memo keyed by (base_url, api-key fingerprint): per-key gateways return a per-key catalog, and
  61. # in a multiplexed process two profiles may share a URL with different keys. The disk memo stays per URL.
  62. _endpoint_model_metadata_cache: dict[tuple[str, str], dict[str, dict[str, Any]]] = {}
  63. _endpoint_model_metadata_cache_time: dict[tuple[str, str], float] = {}
  64. _ENDPOINT_MODEL_CACHE_TTL = 300
  65. # Server-type verdicts (server_type, monotonic_ts): positive ones live an hour so a
  66. # server swap on the same port is re-detected; None gets the short TTL so a
  67. # transient failure recovers in minutes without re-running the waterfall each turn.
  68. _ENDPOINT_PROBE_TTL_SECONDS = 3600.0
  69. # A failed probe verdict (server_type is None — no known endpoint answered) is cached for a much shorter
  70. # window: the in-memory entry exists only to keep one image-bearing turn from re-running the 5-request
  71. # waterfall on every subsequent turn (#89863 — a keyed remote endpoint answered 401 to each leg and the None
  72. # verdict was never cached, so every turn re-probed). Short TTL keeps a transient failure (server starting
  73. # up, key being fixed) recoverable within minutes instead of pinning "undetected" for an hour.
  74. _ENDPOINT_PROBE_FAILURE_TTL_SECONDS = 300.0
  75. _endpoint_probe_path_cache: dict[str, tuple] = {}
  76. # Routable-but-dead endpoints (corp LAN off-VPN) blackhole TCP: once ANY probe paid
  77. # a full connect timeout, later probes short-circuit for a while.
  78. _ENDPOINT_BLACKHOLE_TTL_SECONDS = 30.0
  79. _endpoint_blackhole_cache: dict[str, float] = {} # host:port -> monotonic ts
  80. def _parse_base_url(base_url: str, scheme: str = "http"):
  81. """``urlparse`` of the normalized URL (``scheme`` prepended when absent); None when empty."""
  82. normalized = _normalize_base_url(base_url)
  83. if not normalized:
  84. return None
  85. return urlparse(normalized if "://" in normalized else f"{scheme}://{normalized}")
  86. def _endpoint_host_key(base_url: str) -> Optional[str]:
  87. """``host:port`` key (None without a host) so every probe path for one server shares an entry."""
  88. try:
  89. parsed = _parse_base_url(base_url)
  90. if parsed is None or not parsed.hostname:
  91. return None
  92. return f"{parsed.hostname}:{parsed.port or (443 if parsed.scheme == 'https' else 80)}"
  93. except Exception:
  94. return None
  95. def _note_endpoint_blackholed(base_url: str) -> None:
  96. """Record that a probe to ``base_url`` timed out during TCP connect."""
  97. key = _endpoint_host_key(base_url)
  98. if key is not None:
  99. _endpoint_blackhole_cache[key] = time.monotonic()
  100. logger.debug("Endpoint %s timed out connecting — skipping further probes for %.0fs", key, _ENDPOINT_BLACKHOLE_TTL_SECONDS)
  101. def _endpoint_blackholed(base_url: str) -> bool:
  102. """True if a recent probe to ``base_url`` timed out during TCP connect (cache lookup only)."""
  103. if _ENDPOINT_BLACKHOLE_TTL_SECONDS <= 0:
  104. return False
  105. key = _endpoint_host_key(base_url)
  106. seen = _endpoint_blackhole_cache.get(key) if key is not None else None
  107. if seen is None:
  108. return False
  109. if (time.monotonic() - seen) >= _ENDPOINT_BLACKHOLE_TTL_SECONDS:
  110. del _endpoint_blackhole_cache[key]
  111. return False
  112. return True
  113. def _note_if_connect_timeout(exc: BaseException, base_url: str) -> None:
  114. """Blackhole ``base_url`` when ``exc`` is a connect-phase timeout."""
  115. if _is_connect_timeout(exc):
  116. _note_endpoint_blackholed(base_url)
  117. def _is_connect_timeout(exc: BaseException) -> bool:
  118. """Read timeouts prove the server accepted a connection, not a blackhole."""
  119. import httpx
  120. return isinstance(exc, httpx.ConnectTimeout)
  121. # Disk L2 for local-endpoint probes so back-to-back CLI cold starts skip the waterfall.
  122. # Only SUCCESSFUL probes persist (a down server must not pin a negative verdict).
  123. _LOCAL_PROBE_DISK_TTL_SECONDS = 300.0
  124. def _cache_file(name: str) -> Path:
  125. from hermes_constants import get_hermes_home
  126. return get_hermes_home() / "cache" / name
  127. def _load_json_dict(path: Path) -> dict[str, Any]:
  128. """JSON object at ``path``, or {} when missing/invalid."""
  129. try:
  130. with path.open("r", encoding="utf-8-sig") as f:
  131. data = json.load(f)
  132. return data if isinstance(data, dict) else {}
  133. except Exception:
  134. return {}
  135. def _ttl_memo_get(path: Path, key: str, ttl: float, *, ts_key: str, value_key: str) -> Optional[Any]:
  136. """Fresh (< ``ttl`` seconds) ``value_key`` of the ``key`` entry in a TTL memo file, else None."""
  137. entry = _load_json_dict(path).get(key)
  138. if not isinstance(entry, dict):
  139. return None
  140. try:
  141. if (time.time() - float(entry[ts_key])) >= ttl:
  142. return None
  143. return entry[value_key]
  144. except Exception:
  145. return None
  146. def _ttl_memo_put(path: Path, key: str, value: Any, ttl: float, *, ts_key: str, value_key: str, what: str, ts_first: bool = False) -> None:
  147. """Write ``key`` into a TTL memo file, dropping expired siblings. Best-effort."""
  148. try:
  149. now = time.time()
  150. data = {k: v for k, v in _load_json_dict(path).items() if isinstance(v, dict) and (now - float(v.get(ts_key, 0))) < ttl}
  151. data[key] = {ts_key: now, value_key: value} if ts_first else {value_key: value, ts_key: now}
  152. atomic_json_write(path, data, indent=0, separators=(",", ":"))
  153. except Exception as e:
  154. logger.debug("Failed to save %s: %s", what, e)
  155. def _local_probe_disk_cache_path() -> Path:
  156. return _cache_file("local_endpoint_probes.json")
  157. def _local_probe_disk_get(kind: str, key: str) -> Optional[Any]:
  158. """Return a fresh cached value for ``kind:key``, else None."""
  159. return _ttl_memo_get(_local_probe_disk_cache_path(), f"{kind}:{key}", _LOCAL_PROBE_DISK_TTL_SECONDS, ts_key="ts", value_key="value")
  160. def _local_probe_disk_put(kind: str, key: str, value: Any) -> None:
  161. """Persist a successful probe result. Best-effort; prunes stale entries."""
  162. _ttl_memo_put(_local_probe_disk_cache_path(), f"{kind}:{key}", value, _LOCAL_PROBE_DISK_TTL_SECONDS, ts_key="ts", value_key="value", what="local probe disk cache")
  163. def _get_model_metadata_cache_path() -> Path:
  164. """Path to the OpenRouter model metadata disk cache."""
  165. return _cache_file("openrouter_model_metadata.json")
  166. def _model_metadata_disk_cache_age_seconds() -> Optional[float]:
  167. """Disk-cache age in seconds, or None if freshness is unknown."""
  168. try:
  169. age = time.time() - _get_model_metadata_cache_path().stat().st_mtime
  170. return age if age >= 0 else None
  171. except Exception:
  172. return None
  173. def _load_model_metadata_disk_cache() -> dict[str, dict[str, Any]]:
  174. """Processed OpenRouter metadata cache from disk ({} on any failure)."""
  175. try:
  176. with _get_model_metadata_cache_path().open("r", encoding="utf-8-sig") as f:
  177. data = json.load(f)
  178. return {str(key): value for key, value in data.items() if isinstance(value, dict)} if isinstance(data, dict) else {}
  179. except Exception as e:
  180. logger.debug("Failed to load OpenRouter model metadata disk cache: %s", e)
  181. return {}
  182. def _save_model_metadata_disk_cache(data: dict[str, dict[str, Any]]) -> None:
  183. try:
  184. atomic_json_write(_get_model_metadata_cache_path(), data, indent=0, separators=(",", ":"))
  185. except Exception as e:
  186. logger.debug("Failed to save OpenRouter model metadata disk cache: %s", e)
  187. def _get_endpoint_metadata_cache_path() -> Path:
  188. """On-disk memo of remote ``/models`` probes (see ``_endpoint_disk_cache_get``)."""
  189. return _cache_file("endpoint_model_metadata.json")
  190. def _endpoint_disk_cache_get(normalized: str) -> Optional[dict[str, dict[str, Any]]]:
  191. """Fresh cross-process memo of a remote ``/models`` probe (same TTL as in-memory): one-shot
  192. runs (``hermes -q``, cron) start cold and Nous bypasses the persistent context cache, so
  193. without this every launch paid the live probe. Local endpoints are never memoized."""
  194. models = _ttl_memo_get(_get_endpoint_metadata_cache_path(), normalized, _ENDPOINT_MODEL_CACHE_TTL, ts_key="at", value_key="models")
  195. return models if isinstance(models, dict) else None
  196. def _endpoint_disk_cache_put(normalized: str, cache: dict[str, dict[str, Any]]) -> None:
  197. """Memoize a successful remote ``/models`` probe; expired siblings are dropped."""
  198. _ttl_memo_put(
  199. _get_endpoint_metadata_cache_path(), normalized, cache, _ENDPOINT_MODEL_CACHE_TTL,
  200. ts_key="at", value_key="models", what="endpoint model metadata disk cache", ts_first=True)
  201. # Descending probe tiers for unknown models; tier[0] is also the default fallback.
  202. CONTEXT_PROBE_TIERS = [256_000, 128_000, 64_000, 32_000, 16_000, 8_000]
  203. DEFAULT_FALLBACK_CONTEXT = CONTEXT_PROBE_TIERS[0]
  204. _FALLBACK_WARNED: set = set() # the fallback is never cached, so dedupe its warning per (model, base_url)
  205. def _warn_context_length_fallback(model: str, base_url: str) -> None:
  206. """Warn once per model+endpoint that detection failed and the hard default is used,
  207. so small-context models (8K, 32K) don't silently get 256K and fail at the API."""
  208. mark_context_defaulted() # the "fallback" label reads this, not a re-derivation (agent.context_default_record)
  209. key = (model, base_url or "")
  210. if key not in _FALLBACK_WARNED:
  211. _FALLBACK_WARNED.add(key)
  212. logger.warning("Could not determine context length for model %r (base_url=%s) — falling back to %s tokens. "
  213. "Set model.context_length in config.yaml to override.", model, base_url or "default", f"{DEFAULT_FALLBACK_CONTEXT:,}")
  214. # Sessions, model switches and cron jobs reject models below this (too little working memory).
  215. MINIMUM_CONTEXT_LENGTH = 64_000
  216. # In-process (model, base_url) -> (result, monotonic_ts) memo for local probes: one
  217. # startup resolves the same model several times (banner, /model, compressor). Never persisted.
  218. _LOCAL_CTX_PROBE_TTL_SECONDS = 30.0
  219. _LOCAL_CTX_PROBE_CACHE: dict[tuple, tuple] = {}
  220. # In-process (model, region) -> monotonic_ts memo of a FAILED Bedrock context probe. That probe pads
  221. # prompts of 1.3M/2.2M tokens and attempts up to two converse calls (agent/bedrock_adapter.py
  222. # _BEDROCK_PROBE_TIERS; the second tier is sent only when the first yields no parseable limit, i.e. on
  223. # the failure path), and its failures are deliberately never persisted, so without a negative memo a
  224. # model whose probe keeps failing (un-enabled model, opaque InternalServerException, unparseable length
  225. # error) would re-send them on every resolution. Negative only, in memory only, and bounded — same
  226. # reasoning as _ENDPOINT_PROBE_FAILURE_TTL_SECONDS: a failure is usually transient (expired SSO
  227. # session, offline box), so it must expire rather than stick.
  228. _BEDROCK_PROBE_FAILURE_TTL_SECONDS = 300.0
  229. _BEDROCK_PROBE_FAILURE_CACHE: dict[tuple, float] = {}
  230. # Family-pattern fallbacks, used only when provider-aware sources all miss.
  231. # Lookups are longest-key-first substring matches, so dict order is cosmetic
  232. # and a specific key must be STRICTLY longer than its catch-all.
  233. DEFAULT_CONTEXT_LENGTHS = {
  234. # Anthropic — bare ids only (prefixed ids resolve via OpenRouter/models.dev
  235. # and would collide: "anthropic/claude-sonnet-4" ⊂ "anthropic/claude-sonnet-4.6").
  236. "claude-fable-5": 1000000, "claude-fable": 1000000, "claude-opus-5": 1000000, "claude-sonnet-5": 1000000,
  237. "claude-opus-4-8": 1000000, "claude-opus-4.8": 1000000, "claude-opus-4-7": 1000000, "claude-opus-4.7": 1000000,
  238. "claude-opus-4-6": 1000000, "claude-sonnet-4-6": 1000000, "claude-opus-4.6": 1000000, "claude-sonnet-4.6": 1000000,
  239. "claude": 200000, # catch-all for older Claude models
  240. # OpenAI — direct-API windows (Codex OAuth caps gpt-5.4+/5.5/5.6 at 272K, resolved by
  241. # its own branch). 5.4-nano/-mini are 400k, not 1.05M; gpt-5.3-codex-spark is
  242. # Codex-OAuth-only and listed so "gpt-5" (400k) doesn't win.
  243. "gpt-6-astra": 1050000, # also matches -pro (verified live on OpenRouter)
  244. "gpt-6.1-sol": 1050000, "gpt-6.1-terra": 1050000, # Sol -pro too; both live on OpenRouter (Terra also Nous Portal, 2026-10-26)
  245. "gpt-6-sol": 1050000, "gpt-6-luna": 1050000, # -pro too (OpenRouter live 2026-09-22)
  246. "gpt-5.6-luna": 1050000, "gpt-5.6-terra": 1050000, "gpt-5.6-sol": 1050000, "gpt-5.5": 1050000,
  247. "gpt-5.4-nano": 400000, "gpt-5.4-mini": 400000, "gpt-5.4": 1050000,
  248. "gpt-5.3-codex-spark": 128000, "gpt-5.1-chat": 128000, "gpt-5": 400000,
  249. "gpt-4.1": 1047576, "gpt-4": 128000,
  250. # Google / Gemma ("gemma4" is Ollama-style naming, e.g. gemma4:31b-cloud)
  251. "gemini": 1048576,
  252. "gemma-4": 256000, "gemma4": 256000, "gemma-4-31b": 256000, "gemma-3": 131072, "gemma": 8192,
  253. # DeepSeek — V4 family is 1M; deepseek-chat/-reasoner alias v4-flash modes. ``deepseek-flash``
  254. # (version-less canonical id, 2026-09 Flash refresh) needs a discrete entry or the
  255. # longest-key-first scan falls through to the 128K ``deepseek`` catch-all below.
  256. # https://api-docs.deepseek.com/zh-cn/quick_start/pricing
  257. "deepseek-v4-pro": 1_000_000, "deepseek-v4.1-flash": 1_000_000, "deepseek-v4-flash": 1_000_000, "deepseek-chat": 1_000_000,
  258. "deepseek-reasoner": 1_000_000, "deepseek-flash": 1_000_000, "deepseek": 128000,
  259. # Meta; Muse Spark family (1.1/1.2/1.3, -contributor(-free), meta/ prefixed) is 1M per OpenRouter,
  260. # models.dev and api.commandcode.ai /models — keep the "muse-spark" prefix (bare "muse" would match
  261. # muse-image/muse-voice). Thinking Machines inkling (covers inkling-small and :free/:batch variants)
  262. "llama": 131072, "muse-spark-1.3": 1_048_576, "muse-spark": 1_048_576, "inkling": 1_048_576,
  263. # Qwen — https://help.aliyun.com/zh/model-studio/developer-reference/ (3.8-max/flash
  264. # 1M verified on OpenRouter & Nous portal 2026-08; qwen3-max = 256K Coding Plan snapshot)
  265. "qwen3.8-max": 1_000_000, "qwen3.8-flash": 1_000_000, "qwen3.6-plus": 1048576, "qwen3.7-plus": 1048576,
  266. "qwen3-coder-plus": 1000000, "qwen3-coder": 262144, "qwen3-max": 262144, "qwen": 131072,
  267. # MiniMax — M3 is 1M; M2.x is 204,800. https://platform.minimax.io/docs/api-reference/text-chat-openai
  268. "minimax-m3": 1000000, "minimax": 204800,
  269. # GLM — Nous + OpenRouter /v1/models (2026-09-09): 5.3 / 5.3-flash 1,310,720 (:batch/:US 1,048,576);
  270. # 5.3-flashx 1,048,576 (2026-09-20; its own key, else the shorter 5.3-flash entry wins by substring);
  271. # 5.2 1,048,576; 5 / 5.1 / 4.7 / 4.6 204,800; *-turbo / 4.7-flash 202,752 (the catch-all).
  272. # The OpenRouter :free variant is capped; the longer key wins.
  273. "glm-5.3": 1_310_720, "glm-5.3-flash": 1_310_720, "glm-5.3:batch": 1_048_576, "glm-5.3:us": 1_048_576,
  274. "glm-5.3-flash:batch": 1_048_576, "glm-5.3-flash:us": 1_048_576, "glm-5.3-flashx": 1_048_576,
  275. "glm-5.2": 1_048_576, "glm-5.2:free": 256_000,
  276. "glm-5.1": 204_800, "glm-5-turbo": 202752, "glm-5v-turbo": 202752, "glm-5": 204_800,
  277. "glm-4.7-flash": 202752, "glm-4.7": 204_800, "glm-4.6v": 131072, "glm-4.6": 204_800, "glm": 202752,
  278. # xAI — /v1/models returns no context_length, so these prevent probe-down on api.x.ai
  279. # custom providers (docs.x.ai). grok-composer(-2.5-fast, Grok Build CLI) is OAuth-only:
  280. # 200k usable (the /v1/responses ~262144 input+output budget is a separate limit).
  281. # grok-build-latest aliases grok-4.5; grok-4-fast / grok-4.20 also match their
  282. # -(non-)reasoning and -multi-agent variants; "grok" is the catch-all.
  283. "grok-composer": 200000, "grok-build-latest": 500000, "grok-build": 256000, "grok-code-fast": 256000,
  284. "grok-2-vision": 8192, "grok-4-fast": 2000000, "grok-4.20": 2000000,
  285. "grok-4.6": 500000, "grok-4.5": 500000, "grok-4.3": 1000000, "grok-4": 256000,
  286. "grok-3": 131072, "grok-2": 131072, "grok": 131072,
  287. # Kimi — K3 is 1 Mi (matches the endpoint-scoped override); older Kimi 256K.
  288. "kimi-k3": 1_048_576, "kimi": 262144,
  289. # Upstage Solar — /v1/models returns no context_length. Later generations and new lineups
  290. # default to 512K (Upstage /v1/solar/models max_model_len, 2026-09).
  291. "solar-open2": 262144, "solar-pro3": 131072, "solar-pro2": 65536, "solar-mini": 32768, "solar-": 524288,
  292. # Tencent Hunyuan (262144 = 256 × 1024, aligned with OpenRouter live metadata)
  293. "hy4-preview": 1_048_576, "hy3-preview": 262144, "hy3": 262144,
  294. # "Ox Alpha" stealth model (OpenCode Zen / OpenRouter slugs); "Union Alpha" stealth model
  295. # (OpenRouter ``stealth/union-alpha``, 262144 per /api/v1/models); NVIDIA Nemotron (128K
  296. # except 3.5 Lightning and 3.5 Super); Poolside Laguna 2.1 (:free / -free slugs); Arcee; OpenRouter.
  297. "x-preview-f": 1_048_576, "ox-alpha": 1_048_576, "union-alpha": 262144,
  298. "nemotron-3.5-lightning": 1_000_000, "nemotron-3.5-super-vl": 1_000_000, "nemotron": 131072,
  299. "laguna-s-2.1": 262144, "laguna-xs-2.1": 262144, "trinity": 262144, "elephant": 262144,
  300. # Hugging Face Inference Providers — model IDs use org/name format
  301. "Qwen/Qwen3.5-397B-A17B": 131072, "Qwen/Qwen3.5-35B-A3B": 131072, "deepseek-ai/DeepSeek-V3.2": 65536,
  302. "moonshotai/Kimi-K2.5": 262144, "moonshotai/Kimi-K2.6": 262144, "moonshotai/Kimi-K2-Thinking": 262144,
  303. "MiniMaxAI/MiniMax-M2.5": 204800, "XiaomiMiMo/MiMo-V2-Flash": 262144,
  304. "mimo-v2.6-pro-ultraspeed": 1048576, "mimo-v2.6-pro": 1048576, "mimo-v2.6-flash": 1048576,
  305. "mimo-v2-pro": 1048576, "mimo-v2.5-pro": 1048576, "mimo-v2.5": 1048576, "mimo-v2-omni": 262144, "mimo-v2-flash": 262144,
  306. "zai-org/GLM-5": 202752,
  307. }
  308. # xAI Grok models that ACCEPT `reasoning.effort` (verified live against
  309. # /v1/responses). Unlisted Grok models still reason natively but 400 on the
  310. # parameter, so callers must send no `reasoning` key rather than a default `medium`.
  311. # grok-4.5 accepts low/medium/high; 4.6/4.7 also accept xhigh. They REJECT "none".
  312. _GROK_EFFORT_CAPABLE_PREFIXES = ("grok-3-mini", "grok-4.20-multi-agent", "grok-4.3", "grok-4.5", "grok-4.6", "grok-4.7")
  313. def grok_supports_reasoning_effort(model: str) -> bool:
  314. """Allowlist check (aggregator prefixes like ``x-ai/`` stripped); unknown Grok models get no effort dial."""
  315. name = (model or "").strip().lower().rsplit("/", 1)[-1]
  316. return bool(name) and any(name == prefix or name.startswith(prefix + "-") for prefix in _GROK_EFFORT_CAPABLE_PREFIXES)
  317. # OpenAI chat-era families served on api.openai.com that 400 on ANY ``reasoning`` field
  318. # ("Unsupported parameter: 'reasoning.effort' is not supported with this model"): gpt-3.5,
  319. # gpt-4 / gpt-4-turbo / gpt-4o / gpt-4.1 / gpt-4.5 and the chatgpt-* snapshots. A denylist so
  320. # an unknown future OpenAI model keeps its effort dial (fail-open); ``ft:`` fine-tune ids are
  321. # ``ft:<base>:<org>::<id>`` and inherit the base model's contract.
  322. _OPENAI_NON_REASONING_RE = re.compile(r"^(?:ft:)?(?:gpt-3\.5|gpt-4(?![0-9])|chatgpt-)")
  323. def openai_model_rejects_reasoning(model: str) -> bool:
  324. """True for an OpenAI model id (aggregator ``openai/`` prefix stripped) that rejects the
  325. Responses ``reasoning`` parameter outright, so callers send no ``reasoning`` key at all."""
  326. name = (model or "").strip().lower().rsplit("/", 1)[-1]
  327. return bool(_OPENAI_NON_REASONING_RE.match(name))
  328. def is_grok_46_family(model: str) -> bool:
  329. """Whether *model* is a Grok 4.6 family identifier."""
  330. name = (model or "").strip().lower().replace("_", "-").rsplit("/", 1)[-1]
  331. return name == "grok-4.6" or name.startswith("grok-4.6-")
  332. # Claude models that accept ``speed: "fast"`` (https://platform.claude.com/docs/en/build-with-claude/fast-mode).
  333. # Exact ids, not a family prefix: Opus 4.7 answers the parameter with an error, Opus 4.6 silently
  334. # runs and bills at standard speed, and a future Opus is unsupported until the docs list it.
  335. _ANTHROPIC_FAST_MODE_MODELS = frozenset({"claude-opus-4-8", "claude-opus-5", "claude-opus-5-5"})
  336. def is_anthropic_fast_mode_model(model: Optional[str]) -> bool:
  337. """Whether *model* accepts Anthropic fast mode. Accepts vendor-prefixed, dotted, variant and
  338. dated spellings (``anthropic/claude-opus-5.5``, ``claude-opus-4-8-20260601``). Dedicated
  339. ``...-fast`` ids select fast inference through the model field and are not in the list."""
  340. name = str(model or "").strip().lower().split(":", 1)[0].rsplit("/", 1)[-1]
  341. name = re.sub(r"(\d)\.(\d)", r"\1-\2", name)
  342. return re.sub(r"-\d{8}$", "", name) in _ANTHROPIC_FAST_MODE_MODELS
  343. _CONTEXT_LENGTH_KEYS = (
  344. "context_length", "context_window", "context_size", "max_context_length", "max_position_embeddings",
  345. "max_model_len", "max_input_tokens", "max_sequence_length", "max_seq_len", "n_ctx_train", "n_ctx", "ctx_size",
  346. )
  347. _MAX_COMPLETION_KEYS = ("max_completion_tokens", "max_output_tokens", "max_tokens")
  348. _LOCAL_HOSTS = ("localhost", "127.0.0.1", "::1", "0.0.0.0")
  349. # Docker / Podman / Lima DNS names that resolve to the host machine
  350. _CONTAINER_LOCAL_SUFFIXES = (".docker.internal", ".containers.internal", ".lima.internal")
  351. def _normalize_base_url(base_url: str) -> str:
  352. return (base_url or "").strip().rstrip("/")
  353. def _auth_headers(api_key: object = "") -> dict[str, str]:
  354. from agent.command_token_source import materialize_probe_api_key
  355. token = materialize_probe_api_key(api_key)
  356. return {"Authorization": f"Bearer {token}"} if token else {}
  357. def _is_custom_endpoint(base_url: str) -> bool:
  358. normalized = _normalize_base_url(base_url)
  359. return bool(normalized) and not base_url_host_matches(normalized, "openrouter.ai")
  360. # Host substring -> provider. Profiles claim further hosts through ``hostnames`` /
  361. # ``metadata_hostnames`` (``providers.profile_for_host``), resolved per home at lookup time.
  362. _URL_TO_PROVIDER: dict[str, str] = {
  363. "api.openai.com": "openai", "chatgpt.com": "openai", "api.anthropic.com": "anthropic",
  364. "api.z.ai": "zai", "open.bigmodel.cn": "zai",
  365. "api.moonshot.ai": "kimi-coding", "api.moonshot.cn": "kimi-coding-cn", "api.kimi.com": "kimi-coding",
  366. "api.stepfun.ai": "stepfun", "api.stepfun.com": "stepfun", "api.arcee.ai": "arcee", "api.minimax": "minimax",
  367. "dashscope.aliyuncs.com": "alibaba", "dashscope-intl.aliyuncs.com": "alibaba", "portal.qwen.ai": "qwen-oauth",
  368. "openrouter.ai": "openrouter", "generativelanguage.googleapis.com": "gemini",
  369. "inference-api.nousresearch.com": "nous", "api.deepseek.com": "deepseek",
  370. "api.fireworks.ai": "fireworks", "opencode.ai": "opencode-go", "api.x.ai": "xai",
  371. "integrate.api.nvidia.com": "nvidia", "api.xiaomimimo.com": "xiaomi", "xiaomimimo.com": "xiaomi",
  372. "api.gmi-serving.com": "gmi", "api.novita.ai": "novita",
  373. "tokenhub.tencentmaas.com": "tencent-tokenhub", "api.lkeap.cloud.tencent.com": "tencent-tokenplan",
  374. "ollama.com": "ollama-cloud",
  375. }
  376. # Auto-extend with provider-profile hostnames not already mapped.
  377. try:
  378. for _pp in _list_providers():
  379. _host = _pp.get_hostname()
  380. if _host and _host not in _URL_TO_PROVIDER:
  381. _URL_TO_PROVIDER[_host] = _pp.name
  382. except Exception:
  383. pass
  384. def _infer_provider_from_url(base_url: str) -> Optional[str]:
  385. """models.dev provider name for a base URL (custom endpoints need no explicit provider)."""
  386. parsed = _parse_base_url(base_url, "https")
  387. if parsed is None:
  388. return None
  389. host = parsed.netloc.lower() or parsed.path.lower()
  390. from providers import profile_for_host
  391. if (profile := profile_for_host(host, metadata=True)) is not None: # exact claims beat substring rows
  392. return profile.name
  393. for url_part, provider in _URL_TO_PROVIDER.items():
  394. if url_part in host:
  395. return provider
  396. return None
  397. def _strip_openrouter_routing_variant(
  398. model: str, base_url: str = "", provider: str = ""
  399. ) -> str:
  400. """Strip an OpenRouter routing-variant suffix for catalog lookup.
  401. ``:nitro`` / ``:floor`` / ``:exacto`` / ``:online`` are request-time
  402. routing modifiers, NOT catalog entries — OpenRouter's ``/models`` lists
  403. only the base id, and a variant shares the base model's context window.
  404. Without this, every lookup below misses and the resolver falls through to
  405. a generic family default (``x-ai/grok-4.6:nitro`` → the 131K ``grok``
  406. catch-all instead of its real 2M window).
  407. Only the id used for LOOKUP is rewritten. The suffixed id the caller holds
  408. stays on the wire, so the routing opt-in is preserved — the same rule
  409. :func:`hermes_cli.models.validate_requested_model` applies. Sharing the
  410. base's cache key is intentional: the window is identical, so a variant and
  411. its base must never disagree.
  412. Narrow by design: only applied when the request actually routes through
  413. OpenRouter, so a local ``model:tag`` that happens to end in one of these
  414. words is untouched.
  415. """
  416. if not model:
  417. return model
  418. is_openrouter = (provider or "").strip().lower() == "openrouter" or (
  419. bool(base_url) and _infer_provider_from_url(base_url) == "openrouter"
  420. )
  421. if not is_openrouter:
  422. return model
  423. base = openrouter_variant_base(model)
  424. if base is None:
  425. return model
  426. logger.debug(
  427. "Resolving context length for OpenRouter routing variant %r via base id %r",
  428. model, base,
  429. )
  430. return base
  431. def _is_known_provider_base_url(base_url: str) -> bool:
  432. return _infer_provider_from_url(base_url) is not None
  433. def _lmstudio_server_root(base_url: str) -> str:
  434. """LM Studio server root for native ``/api/v1`` endpoints."""
  435. root = _normalize_base_url(base_url)
  436. for suffix in ("/api/v1", "/api", "/v1"):
  437. if root.endswith(suffix):
  438. return root[: -len(suffix)].rstrip("/")
  439. return root
  440. def _server_root(base_url: str) -> str:
  441. """Probe root for a local server: IPv4-resolved, ``/v1`` suffix stripped."""
  442. server_url = _localhost_to_ipv4(base_url.rstrip("/"))
  443. return server_url.removesuffix("/v1")
  444. # Families whose generation digit is part of the name (``solar-mini`` vs ``solar-mini4``): their keys
  445. # match only on an id boundary, after folding aggregator slugs (``solar-pro-3``) into the native form.
  446. _BOUNDARY_MATCHED_KEY_PREFIXES = ("solar-",)
  447. _HYPHENATED_GENERATION_RE = re.compile(
  448. rf"((?:{'|'.join(map(re.escape, _BOUNDARY_MATCHED_KEY_PREFIXES))})[a-z]+)-(\d{{1,2}})(?=[-:.@]|$)")
  449. def _catalog_key_matches(key: str, model_lower: str) -> bool:
  450. """Substring match with version separators normalised on both sides, so a relay slug like
  451. ``z-ai-glm-5-3`` still hits the ``glm-5.3`` entry instead of the ``glm`` catch-all (#97398).
  452. Boundary-matched families: a key must be followed by ``-:.@`` or the end, and a key ending in
  453. ``-`` is the family default for bare names (``org/`` allowed) continuing with a lineup letter."""
  454. if key.startswith(_BOUNDARY_MATCHED_KEY_PREFIXES):
  455. model_lower = _HYPHENATED_GENERATION_RE.sub(r"\1\2", model_lower)
  456. if key.endswith("-"):
  457. return re.match(re.escape(key) + "[a-z]", model_lower.rsplit("/", 1)[-1]) is not None
  458. return re.search(re.escape(key) + r"(?:[-:.@]|$)", model_lower) is not None
  459. return key in model_lower or _normalize_model_version(key) in _normalize_model_version(model_lower)
  460. def _longest_key_match(table: dict[str, int], model_lower: str) -> Optional[tuple[str, int]]:
  461. """First ``(key, value)`` whose key is a substring of ``model_lower``, longest key first so
  462. specific entries (``gpt-5.4-mini``) beat their family catch-all (``gpt-5``); ties keep table order."""
  463. for key, value in sorted(table.items(), key=lambda x: len(x[0]), reverse=True):
  464. if _catalog_key_matches(key, model_lower):
  465. return key, value
  466. return None
  467. def _ollama_show_context(data: dict[str, Any], *, gguf_first: bool, minimum: Optional[int] = None) -> Optional[int]:
  468. """Context length from an Ollama ``/api/show`` payload. ``parameters.num_ctx`` is the RUNTIME
  469. window (Modelfile override), ``model_info.*.context_length`` the GGUF training max. Local users
  470. control num_ctx (prefer it); hosted operators may cap it arbitrarily (``gguf_first``)."""
  471. def _num_ctx():
  472. for line in data.get("parameters", "").split("\n"):
  473. parts = line.strip().split()
  474. if "num_ctx" in line and len(parts) >= 2:
  475. try:
  476. yield int(parts[-1])
  477. except ValueError:
  478. continue
  479. def _gguf():
  480. for key, value in data.get("model_info", {}).items():
  481. if "context_length" in key and isinstance(value, (int, float)):
  482. yield int(value)
  483. for reader in ((_gguf, _num_ctx) if gguf_first else (_num_ctx, _gguf)):
  484. for ctx in reader():
  485. if minimum is None or ctx >= minimum:
  486. return ctx
  487. return None
  488. # (host, canonical paths, model ids, context) — see _endpoint_scoped_context_length.
  489. _ENDPOINT_SCOPED_CONTEXT = (
  490. ("api.kimi.com", {"/coding", "/coding/v1"}, {"k3", "kimi-k3", "kimi-k3-cot"}, 1_048_576),
  491. ("integrate.api.nvidia.com", {"/v1"}, {"deepseek-ai/deepseek-v4-pro"}, 262_144),
  492. )
  493. def _endpoint_scoped_context_length(model: str, base_url: str) -> Optional[int]:
  494. """Context confirmed for one provider endpoint only (see _ENDPOINT_SCOPED_CONTEXT): Kimi Coding
  495. serves K3 at 1 Mi only on the canonical ``api.kimi.com/coding`` host (legacy Moonshot keys do
  496. not); NVIDIA NIM serves deepseek-v4-pro at 262,144 while DeepSeek's native endpoint is 1M."""
  497. try:
  498. parsed = urlparse(_normalize_base_url(base_url))
  499. port = parsed.port
  500. except ValueError:
  501. return None
  502. # Only canonical https://host[:443]/path with no credentials/query/fragment.
  503. if parsed.scheme.lower() != "https" or port not in (None, 443) or parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment:
  504. return None
  505. host, path, model_key = (parsed.hostname or "").lower(), parsed.path.rstrip("/"), model.strip().lower()
  506. return next((ctx for scoped_host, paths, models, ctx in _ENDPOINT_SCOPED_CONTEXT if host == scoped_host and path in paths and model_key in models), None)
  507. def _skip_persistent_context_cache(base_url: str, provider: str) -> bool:
  508. """Providers whose disk context cache must not short-circuit probing: LM Studio (loaded
  509. context is transient), Codex OAuth (entitlement-specific window; a persisted fallback would suppress revalidation)."""
  510. return (provider or "").strip().lower() in {"lmstudio", "openai-codex"}
  511. def _is_codex_route(provider: str, base_url: str, custom_providers: list | None) -> bool:
  512. """True when the request travels the Codex Responses wire regardless of host: the native
  513. ``openai-codex`` provider (also behind a ``HERMES_CODEX_BASE_URL`` / ``model.base_url`` proxy)
  514. or a custom entry declaring ``api_mode: codex_responses``. The transport, not the hostname,
  515. decides which window the model actually gets (#116191)."""
  516. if (provider or "").strip().lower() == "openai-codex":
  517. return True
  518. if not base_url:
  519. return False
  520. with contextlib.suppress(Exception): # config unreadable → not a known Codex route
  521. from hermes_cli.config import get_custom_provider_api_mode
  522. return get_custom_provider_api_mode(base_url, custom_providers) == "codex_responses"
  523. return False
  524. def _save_unless_skipped(model: str, base_url: str, ctx: int, provider: str) -> None:
  525. """Persist ``ctx`` unless the provider opts out of the disk cache."""
  526. if not _skip_persistent_context_cache(base_url, provider):
  527. save_context_length(model, base_url, ctx)
  528. def _maybe_cache_local_context_length(model: str, base_url: str, length: int) -> None:
  529. """Persist a probed local window only at/above MINIMUM_CONTEXT_LENGTH: sub-minimum windows are
  530. still returned so agent_init can reject them, but must not be blessed into the disk cache."""
  531. if length >= MINIMUM_CONTEXT_LENGTH:
  532. save_context_length(model, base_url, length)
  533. def _probe_local_context_length(model: str, base_url: str, api_key: str, provider: str) -> Optional[int]:
  534. """Live local probe; persists a positive result unless the provider opts out of the disk cache."""
  535. local_ctx = _query_local_context_length(model, base_url, api_key=api_key)
  536. if not (local_ctx and local_ctx > 0):
  537. return None
  538. if not _skip_persistent_context_cache(base_url, provider):
  539. _maybe_cache_local_context_length(model, base_url, local_ctx)
  540. return local_ctx
  541. def _reconcile_local_cached_context_length(model: str, base_url: str, cached: int, api_key: str = "") -> int:
  542. """*cached* unless a live local probe reports a different limit (operators restart
  543. vLLM/Ollama with a new --max-model-len / num_ctx under the same id). A failed
  544. probe keeps the disk entry; sub-minimum live windows invalidate but are not persisted."""
  545. live_ctx = _query_local_context_length(model, base_url, api_key=api_key)
  546. if not (live_ctx and live_ctx > 0 and live_ctx != cached):
  547. return cached
  548. if live_ctx < MINIMUM_CONTEXT_LENGTH:
  549. logger.info("Live local probe for %s@%s reports %s (< minimum %s); invalidating stale cache — agent init should reject", model, base_url, f"{live_ctx:,}", f"{MINIMUM_CONTEXT_LENGTH:,}")
  550. else:
  551. logger.info("Reconciling stale local cache entry %s@%s: %s -> %s (live probe)", model, base_url, f"{cached:,}", f"{live_ctx:,}")
  552. _invalidate_cached_context_length(model, base_url)
  553. if live_ctx >= MINIMUM_CONTEXT_LENGTH:
  554. _maybe_cache_local_context_length(model, base_url, live_ctx)
  555. return live_ctx
  556. def is_local_endpoint(base_url: str) -> bool:
  557. """True for loopback, container-internal DNS, unqualified hosts, RFC-1918,
  558. link-local and Tailscale CGNAT (so a trusted Ollama box over Tailscale gets
  559. the same timeout auto-bumps as localhost)."""
  560. try:
  561. parsed = _parse_base_url(base_url)
  562. host = parsed.hostname or "" if parsed is not None else None
  563. except Exception:
  564. return False
  565. if host is None:
  566. return False
  567. # Unqualified hostnames (no dots) are local by definition — Docker Compose service names, /etc/hosts
  568. # entries, mDNS — as is `*.local` (RFC 6762 mDNS, LAN-only). IPv6 literals have no dots either, so
  569. # they are excluded here and classified by scope below (a global address is not local).
  570. if host in _LOCAL_HOSTS or host.endswith(_CONTAINER_LOCAL_SUFFIXES) or host.endswith(".local") or (host and "." not in host and ":" not in host):
  571. return True
  572. try:
  573. addr = ipaddress.ip_address(host)
  574. if addr.is_private or addr.is_loopback or addr.is_link_local or (isinstance(addr, ipaddress.IPv4Address) and addr in _TAILSCALE_CGNAT):
  575. return True
  576. except ValueError:
  577. pass
  578. # Dotted quad that ipaddress rejected but still looks like a private range
  579. # (e.g. 172.26.x.x for WSL) or Tailscale CGNAT (100.64.x.x–100.127.x.x).
  580. parts = host.split(".")
  581. if len(parts) != 4:
  582. return False
  583. try:
  584. first, second = int(parts[0]), int(parts[1])
  585. except ValueError:
  586. return False
  587. return first == 10 or (first == 172 and 16 <= second <= 31) or (first == 192 and second == 168) or (first == 100 and 64 <= second <= 127)
  588. def _localhost_to_ipv4(url: str) -> str:
  589. """``localhost`` HOST -> ``127.0.0.1`` (Windows dual-stack resolves ::1 first and pays a ~2s IPv6
  590. connect timeout on IPv4-only servers). Scheme-anchored so ``?upstream=http://localhost`` is untouched."""
  591. if not url or not isinstance(url, str):
  592. return url # non-string values (test doubles, lazy config) pass through
  593. return re.sub(r"^(https?://)localhost(?=[:/]|$)", r"\g<1>127.0.0.1", url, count=1)
  594. def detect_local_server_type(base_url: str, api_key: str = "") -> Optional[str]:
  595. """Probe known endpoints: "ollama", "lm-studio", "vllm", "llamacpp", or None (TTL-cached)."""
  596. import httpx
  597. # IPv4-resolve BEFORE deriving server/LM Studio URLs and the cache lookup, so localhost and 127.0.0.1 share a cache entry.
  598. normalized = _localhost_to_ipv4(_normalize_base_url(base_url))
  599. # A hosted provider (api.openai.com, api.anthropic.com, ...) never runs Ollama/LM Studio/llama.cpp/vLLM:
  600. # skip the waterfall so egress logs do not fill with 404s for /api/tags, /v1/props, /version (#61421).
  601. # Local addresses are never in that table, and ollama.com is the one hosted host that does speak
  602. # Ollama's /api/tags, so it keeps the probe.
  603. if _infer_provider_from_url(normalized) not in (None, "ollama-cloud"):
  604. return None
  605. server_url = _server_root(normalized)
  606. lmstudio_url = _lmstudio_server_root(normalized)
  607. cached = _endpoint_probe_path_cache.get(server_url)
  608. if cached is not None and (time.monotonic() - cached[1]) < (_ENDPOINT_PROBE_TTL_SECONDS if cached[0] is not None else _ENDPOINT_PROBE_FAILURE_TTL_SECONDS):
  609. return cached[0]
  610. # Blackholed host: skip the waterfall. Deliberately NOT written to the
  611. # hour-long verdict cache, which would pin "undetected" after it comes back.
  612. if _endpoint_blackholed(server_url):
  613. return None
  614. disk_hit = _local_probe_disk_get("server_type", server_url)
  615. if isinstance(disk_hit, str):
  616. _endpoint_probe_path_cache[server_url] = (disk_hit, time.monotonic())
  617. return disk_hit
  618. # Most specific first: (name, paths tried until one answers 200, body check). LM Studio answers /api/tags
  619. # with {"error": ...} and 200, so Ollama's body must carry "models"; older llama.cpp builds have no /v1 prefix.
  620. waterfall = (
  621. ("lm-studio", (f"{lmstudio_url}/api/v1/models",), lambda r: True),
  622. ("ollama", (f"{server_url}/api/tags",), lambda r: "models" in r.json()),
  623. ("llamacpp", (f"{server_url}/v1/props", f"{server_url}/props"), lambda r: "default_generation_settings" in r.text),
  624. ("vllm", (f"{server_url}/version",), lambda r: "version" in r.json()),
  625. )
  626. result: Optional[str] = None
  627. try:
  628. with httpx.Client(timeout=2.0, headers=_auth_headers(api_key), verify=model_metadata_http.resolve_verify(base_url)) as client:
  629. for name, urls, check in waterfall:
  630. try:
  631. for url in urls:
  632. r = client.get(url)
  633. if r.status_code == 200:
  634. break
  635. if r.status_code == 200 and check(r):
  636. result = name
  637. break
  638. except Exception as exc:
  639. # A connect timeout condemns the host: skip the remaining legs.
  640. if _is_connect_timeout(exc):
  641. _note_endpoint_blackholed(server_url)
  642. raise
  643. except Exception:
  644. pass
  645. # Negative verdict in memory only (never on disk — failures are often transient).
  646. _endpoint_probe_path_cache[server_url] = (result, time.monotonic())
  647. if result is not None:
  648. _local_probe_disk_put("server_type", server_url, result)
  649. return result
  650. # Cache the negative verdict in memory only (never on disk — a failure is often transient: server starting,
  651. # key being fixed) so the very next turn does not re-run the whole waterfall against an endpoint that just
  652. # answered nothing (#89863).
  653. def _iter_nested_dicts(value: Any):
  654. if isinstance(value, dict):
  655. yield value
  656. for nested in value.values():
  657. yield from _iter_nested_dicts(nested)
  658. elif isinstance(value, list):
  659. for item in value:
  660. yield from _iter_nested_dicts(item)
  661. def _coerce_reasonable_int(value: Any, minimum: int = 1024, maximum: int = 10_000_000) -> Optional[int]:
  662. if isinstance(value, bool):
  663. return None
  664. try:
  665. result = int(value.strip().replace(",", "") if isinstance(value, str) else value)
  666. except (TypeError, ValueError):
  667. return None
  668. return result if minimum <= result <= maximum else None
  669. def _extract_first_int(payload: dict[str, Any], keys: tuple[str, ...]) -> Optional[int]:
  670. keyset = {key.lower() for key in keys}
  671. for mapping in _iter_nested_dicts(payload):
  672. for key, value in mapping.items():
  673. if str(key).lower() in keyset:
  674. coerced = _coerce_reasonable_int(value)
  675. if coerced is not None:
  676. return coerced
  677. return None
  678. def _extract_flat_context_length(payload: dict[str, Any]) -> Optional[int]:
  679. """Top-level-only context WINDOW read (no nested walk, so an unrelated nested section can't leak
  680. a same-named key). ``max_tokens`` is NOT a window key: on OpenAI-compatible passthroughs it is the max OUTPUT."""
  681. return next((c for c in map(_coerce_reasonable_int, (payload.get(k) for k in _CONTEXT_LENGTH_KEYS)) if c is not None), None)
  682. def _context_length_from_model_payload(payload: dict[str, Any]) -> Optional[int]:
  683. """Context window from a ``/v1/models`` object: window keys first, ``max_tokens`` last (Anthropic
  684. payloads carry ``max_input_tokens`` = 1M window AND ``max_tokens`` = 128k OUTPUT cap)."""
  685. if not isinstance(payload, dict):
  686. return None
  687. ctx = _extract_flat_context_length(payload)
  688. if ctx is not None:
  689. return ctx
  690. raw = payload.get("max_tokens")
  691. return int(raw) if isinstance(raw, (int, float)) and int(raw) > 0 else None
  692. # Generic ``/models`` pricing: an explicit ``unit`` beside the rates wins; without one, a token rate
  693. # at or above $0.001/token ($1,000/MTok — no real model charges that) can only be a per-million quote.
  694. _PRICING_UNIT_DIVISORS = {
  695. "per_token": 1, "per_1k_tokens": 1_000, "per_thousand_tokens": 1_000,
  696. "per_1m_tokens": 1_000_000, "per_million_tokens": 1_000_000,
  697. }
  698. _PER_MILLION_QUOTE_MIN = 0.001
  699. _TOKEN_RATE_FIELDS = ("prompt", "completion", "cache_read", "cache_write")
  700. def _normalize_token_rates(pricing: dict[str, Any], unit: Any) -> dict[str, Any]:
  701. """Rescale the generic path's token rates to per-token strings (the contract usage_pricing
  702. multiplies by 1e6), the way the Novita/DeepInfra branches already do for their known units."""
  703. rates: dict[str, float] = {}
  704. for key in _TOKEN_RATE_FIELDS:
  705. try:
  706. rates[key] = float(pricing[key])
  707. except (KeyError, TypeError, ValueError):
  708. continue
  709. divisor = _PRICING_UNIT_DIVISORS.get(str(unit or "").strip().lower())
  710. if divisor is None:
  711. divisor = 1_000_000 if any(v >= _PER_MILLION_QUOTE_MIN for v in rates.values()) else 1
  712. if divisor != 1:
  713. pricing.update({key: str(value / divisor) for key, value in rates.items()})
  714. return pricing
  715. def _extract_pricing(payload: dict[str, Any]) -> dict[str, Any]:
  716. def _per_token(source: dict[str, Any], fields: dict[str, str], scale) -> dict[str, Any]:
  717. # Provider $/MTok (or Novita's 1/10_000-$ per M) -> per-token strings, the same path usage_pricing uses for OpenRouter.
  718. return {target: str(scale(float(source[key]))) for target, key in fields.items() if source.get(key) is not None}
  719. novita_fields = {"prompt": "input_token_price_per_m", "completion": "output_token_price_per_m"}
  720. if any(payload.get(k) is not None for k in novita_fields.values()):
  721. return _per_token(payload, novita_fields, lambda v: v / 10_000 / 1_000_000)
  722. # DeepInfra ships pricing under ``metadata.pricing`` in $/MTok.
  723. metadata = payload.get("metadata")
  724. deepinfra_pricing = metadata.get("pricing") if isinstance(metadata, dict) else None
  725. deepinfra_fields = {"prompt": "input_tokens", "completion": "output_tokens", "cache_read": "cache_read_tokens"}
  726. if isinstance(deepinfra_pricing, dict) and any(k in deepinfra_pricing for k in deepinfra_fields.values()):
  727. return _per_token(deepinfra_pricing, deepinfra_fields, lambda v: v / 1_000_000)
  728. alias_map = {
  729. "prompt": ("prompt", "input", "input_cost_per_token", "prompt_token_cost"),
  730. "completion": ("completion", "output", "output_cost_per_token", "completion_token_cost"),
  731. "request": ("request", "request_cost"),
  732. "cache_read": ("cache_read", "cached_prompt", "input_cache_read", "cache_read_cost_per_token"),
  733. "cache_write": ("cache_write", "cache_creation", "input_cache_write", "cache_write_cost_per_token"),
  734. }
  735. for mapping in _iter_nested_dicts(payload):
  736. normalized = {str(key).lower(): value for key, value in mapping.items()}
  737. pricing: dict[str, Any] = {}
  738. for target, aliases in alias_map.items():
  739. for alias in aliases:
  740. if alias in normalized and normalized[alias] not in {None, ""}:
  741. pricing[target] = normalized[alias]
  742. break
  743. if pricing:
  744. return _normalize_token_rates(pricing, normalized.get("unit"))
  745. return {}
  746. def _add_model_aliases(cache: dict[str, dict[str, Any]], model_id: str, entry: dict[str, Any]) -> None:
  747. cache[model_id] = entry
  748. if "/" in model_id:
  749. cache.setdefault(model_id.split("/", 1)[1], entry)
  750. def fetch_model_metadata(force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  751. """Fetch model metadata from OpenRouter (cached for 1 hour)."""
  752. global _model_metadata_cache, _model_metadata_cache_time
  753. if not force_refresh:
  754. if _model_metadata_cache and (time.time() - _model_metadata_cache_time) < _MODEL_CACHE_TTL:
  755. return _model_metadata_cache
  756. disk_age = _model_metadata_disk_cache_age_seconds()
  757. if disk_age is not None and disk_age < _MODEL_CACHE_TTL:
  758. disk_cache = _load_model_metadata_disk_cache()
  759. if disk_cache:
  760. _model_metadata_cache = disk_cache
  761. _model_metadata_cache_time = time.time() - disk_age
  762. return _model_metadata_cache
  763. try:
  764. # (connect, read) tuple: a flat timeout lets urllib3 block per retry stage through proxies that 403 CONNECT.
  765. # See #46620.
  766. response = model_metadata_http.get(OPENROUTER_MODELS_URL, timeout=(5, 10), verify=model_metadata_http.resolve_verify())
  767. response.raise_for_status()
  768. cache = {}
  769. for model in response.json().get("data", []):
  770. model_id = model.get("id", "")
  771. entry = {
  772. "context_length": model.get("context_length", 128000),
  773. "max_completion_tokens": model.get("top_provider", {}).get("max_completion_tokens", 4096),
  774. "name": model.get("name", model_id), "pricing": model.get("pricing", {}),
  775. }
  776. canonical = model.get("canonical_slug", "")
  777. for alias in ((model_id, canonical) if canonical and canonical != model_id else (model_id,)):
  778. _add_model_aliases(cache, alias, entry)
  779. _model_metadata_cache = cache
  780. _model_metadata_cache_time = time.time()
  781. _save_model_metadata_disk_cache(cache)
  782. logger.debug("Fetched metadata for %s models from OpenRouter", len(cache))
  783. return cache
  784. except Exception as e:
  785. logger.warning("Failed to fetch model metadata from OpenRouter: %s", e)
  786. if _model_metadata_cache:
  787. return _model_metadata_cache
  788. disk_cache = _load_model_metadata_disk_cache()
  789. if disk_cache:
  790. _model_metadata_cache = disk_cache
  791. disk_age = _model_metadata_disk_cache_age_seconds()
  792. stale_by = min(disk_age, _MODEL_CACHE_TTL) if disk_age is not None else _MODEL_CACHE_TTL - 1
  793. _model_metadata_cache_time = time.time() - stale_by
  794. return _model_metadata_cache
  795. return {}
  796. def _endpoint_model_entry(model: dict[str, Any], model_id: str, context_length: Optional[int]) -> dict[str, Any]:
  797. """Cache entry for one ``/models`` item; optional keys are set only when known."""
  798. optional = (("context_length", context_length), ("max_completion_tokens", _extract_first_int(model, _MAX_COMPLETION_KEYS)), ("pricing", _extract_pricing(model) or None))
  799. return {"name": model.get("name", model_id), **{k: v for k, v in optional if v is not None}}
  800. def _lmstudio_loaded_context(model: dict[str, Any]) -> Optional[int]:
  801. """Context of the first loaded LM Studio instance (the runtime value), else None."""
  802. for inst in model.get("loaded_instances", []) or []:
  803. cfg = inst.get("config", {}) if isinstance(inst, dict) else None
  804. ctx = cfg.get("context_length") if isinstance(cfg, dict) else None
  805. if isinstance(ctx, int) and ctx > 0:
  806. return ctx
  807. return None
  808. def _lmstudio_native_models(normalized: str, headers: dict[str, str]) -> dict[str, dict[str, Any]]:
  809. """LM Studio ``/api/v1/models`` → cache; context comes from the first loaded instance."""
  810. response = model_metadata_http.get(_lmstudio_server_root(normalized).rstrip("/") + "/api/v1/models", headers=headers, timeout=(5, 10), verify=model_metadata_http.resolve_verify(normalized))
  811. response.raise_for_status()
  812. cache: dict[str, dict[str, Any]] = {}
  813. for model in response.json().get("models", []):
  814. model_id = (model.get("key") or model.get("id")) if isinstance(model, dict) else None
  815. if not model_id:
  816. continue
  817. entry = _endpoint_model_entry(model, model_id, _lmstudio_loaded_context(model))
  818. _add_model_aliases(cache, model_id, entry)
  819. alt_id = model.get("id")
  820. if isinstance(alt_id, str) and alt_id and alt_id != model_id:
  821. _add_model_aliases(cache, alt_id, entry)
  822. return cache
  823. def _apply_llamacpp_props(cache: dict[str, dict[str, Any]], request_candidate: str, headers: dict[str, str], verify) -> None:
  824. """Overwrite ``context_length`` with llama.cpp's allocated ``n_ctx`` from /props (``/v1/props``, then
  825. ``/props`` for older builds). In router mode the bare endpoint 400s, so each LOADED child is read
  826. via ``/props?model=``; unloaded children are skipped — probing could autoload them."""
  827. base = request_candidate.rstrip("/").replace("/v1", "")
  828. def _props(params=None):
  829. resp = model_metadata_http.get(base + "/v1/props", params=params, headers=headers, timeout=5, verify=verify)
  830. if resp.is_error:
  831. resp = model_metadata_http.get(base + "/props", params=params, headers=headers, timeout=5, verify=verify)
  832. return resp
  833. def _n_ctx(props: dict[str, Any]) -> Any:
  834. return (props.get("default_generation_settings") or {}).get("n_ctx")
  835. props_resp = _props()
  836. if not props_resp.is_error:
  837. props = props_resp.json()
  838. n_ctx, model_alias = _n_ctx(props), props.get("model_alias", "")
  839. if n_ctx and model_alias and model_alias in cache:
  840. cache[model_alias]["context_length"] = n_ctx
  841. return
  842. native = model_metadata_http.get(base + "/models", headers=headers, timeout=5, verify=verify)
  843. if native.is_error:
  844. return
  845. for child in (native.json() or {}).get("data", [])[:16]:
  846. child_id = child.get("id") if isinstance(child, dict) else None
  847. if not child_id or child_id not in cache or (child.get("status") or {}).get("value") not in ("loaded", "ready"):
  848. continue
  849. pr = _props({"model": child_id, "autoload": "false"}) # it may have unloaded since /models
  850. child_ctx = _n_ctx(pr.json()) if not pr.is_error else None
  851. if child_ctx:
  852. cache[child_id]["context_length"] = child_ctx
  853. def _endpoint_memo_key(normalized: str, api_key: object) -> tuple[str, str]:
  854. from agent.credential_persistence import fingerprint_secret_value
  855. # Callable (minted) keys are not fingerprinted here: doing so would mint on every cache hit.
  856. return normalized, (fingerprint_secret_value(api_key) or "") if isinstance(api_key, str) else ""
  857. def _parse_models_payload(payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
  858. cache: dict[str, dict[str, Any]] = {}
  859. for model in payload.get("data", []):
  860. model_id = model.get("id") if isinstance(model, dict) else None
  861. if model_id:
  862. _add_model_aliases(cache, model_id, _endpoint_model_entry(model, model_id, _extract_first_int(model, _CONTEXT_LENGTH_KEYS)))
  863. return cache
  864. def fetch_endpoint_model_metadata(base_url: str, api_key: str = "", force_refresh: bool = False) -> dict[str, dict[str, Any]]:
  865. """Model metadata from an OpenAI-compatible ``/models`` endpoint (cached per base URL)."""
  866. normalized = _normalize_base_url(base_url)
  867. if not normalized or base_url_host_matches(normalized, "openrouter.ai"):
  868. return {}
  869. local = is_local_endpoint(normalized)
  870. memo_key = _endpoint_memo_key(normalized, api_key)
  871. if not force_refresh:
  872. cached = _endpoint_model_metadata_cache.get(memo_key)
  873. if cached is not None and (time.time() - _endpoint_model_metadata_cache_time.get(memo_key, 0)) < _ENDPOINT_MODEL_CACHE_TTL:
  874. return cached
  875. memo = _endpoint_disk_cache_get(normalized) if not local else None
  876. if memo is not None:
  877. return _remember_endpoint_models(memo_key, memo)
  878. # Blackholed: return empty WITHOUT caching so it is retried once the entry expires.
  879. if _endpoint_blackholed(normalized):
  880. return {}
  881. alternate = normalized[:-3].rstrip("/") if normalized.endswith("/v1") else normalized + "/v1"
  882. candidates = [normalized] + ([alternate] if alternate != normalized else [])
  883. headers = _auth_headers(api_key)
  884. verify = model_metadata_http.resolve_verify(normalized)
  885. last_error: Optional[Exception] = None
  886. if local:
  887. try:
  888. if detect_local_server_type(normalized, api_key=api_key) == "lm-studio":
  889. return _remember_endpoint_models(memo_key, _lmstudio_native_models(normalized, headers))
  890. except Exception as exc:
  891. last_error = exc
  892. _note_if_connect_timeout(exc, normalized)
  893. for candidate in candidates:
  894. # A connect timeout condemns the host, not the path.
  895. if _endpoint_blackholed(normalized):
  896. break
  897. # Cache keys stay unrewritten; only the outbound target is IPv4-resolved.
  898. request_candidate = _localhost_to_ipv4(candidate)
  899. url = request_candidate.rstrip("/") + "/models"
  900. try:
  901. with model_metadata_http.stream(url, headers=headers, timeout=(5, 10), verify=verify) as response:
  902. if response.status_code in (401, 403):
  903. logger.debug("Model metadata probe received HTTP %s from %s; stopping candidate probing", response.status_code, url)
  904. break
  905. response.raise_for_status()
  906. response.read()
  907. payload = response.json()
  908. cache = _parse_models_payload(payload)
  909. if any(m.get("owned_by") == "llamacpp" for m in payload.get("data", []) if isinstance(m, dict)):
  910. with contextlib.suppress(Exception):
  911. _apply_llamacpp_props(cache, request_candidate, headers, verify)
  912. if cache and not local:
  913. _endpoint_disk_cache_put(normalized, cache)
  914. return _remember_endpoint_models(memo_key, cache)
  915. except Exception as exc:
  916. last_error = exc
  917. _note_if_connect_timeout(exc, normalized)
  918. if last_error:
  919. logger.debug("Failed to fetch model metadata from %s/models: %s", normalized, last_error)
  920. return _remember_endpoint_models(memo_key, {})
  921. def _resolve_endpoint_context_length(model: str, base_url: str, api_key: str = "") -> Optional[int]:
  922. """Resolve context length from an endpoint's live ``/models`` metadata."""
  923. endpoint_metadata = fetch_endpoint_model_metadata(base_url, api_key=api_key)
  924. matched = endpoint_metadata.get(model)
  925. if not matched and len(endpoint_metadata) == 1:
  926. matched = next(iter(endpoint_metadata.values()))
  927. elif not matched and model: # substring match; "" would match EVERY key and poison the window
  928. matched = next((entry for key, entry in endpoint_metadata.items() if model in key or key in model), None)
  929. context_length = matched.get("context_length") if matched else None
  930. return context_length if isinstance(context_length, int) else None
  931. def _get_context_cache_path() -> Path:
  932. """Path to the persistent context length cache file."""
  933. from hermes_constants import get_hermes_home
  934. return get_hermes_home() / "context_length_cache.yaml"
  935. def _load_context_cache_document() -> dict:
  936. """Read scalar lengths and their provenance from the same atomic document."""
  937. path = _get_context_cache_path()
  938. if not path.exists():
  939. return {}
  940. try:
  941. with open(path, encoding="utf-8-sig") as f:
  942. data = yaml.safe_load(f)
  943. if not isinstance(data, dict):
  944. return {}
  945. for section in ("context_lengths", "bedrock_confirmed_v1"):
  946. if not isinstance(data.get(section), dict):
  947. data[section] = {}
  948. return data
  949. return data.get("context_lengths") or {}
  950. except Exception as e:
  951. logger.debug("Failed to load context length cache: %s", e)
  952. return {}
  953. def _load_context_cache() -> dict[str, int]:
  954. """Load scalar lengths, preserving the legacy reader contract."""
  955. return _load_context_cache_document().get("context_lengths") or {}
  956. def _context_cache_key(model: str, base_url: str) -> str:
  957. """Canonical ``model@base_url`` key; trailing slashes stripped so ``/v1`` and ``/v1/`` share one entry."""
  958. return f"{model}@{(base_url or '').rstrip('/')}"
  959. @locked_context_cache
  960. def save_context_length(model: str, base_url: str, length: int, *, source: str = "") -> None:
  961. """Persist a discovered context length under ``model@base_url`` (same model, different providers, different limits)."""
  962. # 0/negative is always a bug and would make get_model_context_length() return 0 (`0 is not None`).
  963. if length <= 0:
  964. logger.warning("Refusing to cache non-positive context length %s -> %s tokens", f"{model}@{base_url}", length)
  965. return
  966. key = _context_cache_key(model, base_url)
  967. document = _load_context_cache_document()
  968. cache = document.get("context_lengths") or {}
  969. confirmed = document.get("bedrock_confirmed_v1") or {}
  970. confirmed = confirmed if isinstance(confirmed, dict) else {}
  971. old_confirmed = confirmed.copy()
  972. # Generic writes revoke provenance even for the same number. The marker
  973. # binds to the value, so an older writer cannot leave mismatched evidence.
  974. for alias in (key, f"{model}@{base_url}", f"{key}/"):
  975. confirmed.pop(alias, None)
  976. if source == "bedrock-confirmed-v1":
  977. confirmed[key] = length
  978. if cache.get(key) == length and confirmed == old_confirmed:
  979. return # already stored
  980. cache[key] = length
  981. try:
  982. _write_context_cache(cache, confirmed)
  983. logger.info("Cached context length %s -> %s tokens", key, f"{length:,}")
  984. except Exception as e:
  985. logger.debug("Failed to save context length cache: %s", e)
  986. def save_provider_context_length(model: str, base_url: str, length: int, provider: str = "") -> None:
  987. """Persist a provider-confirmed window, distinguishing it from legacy Bedrock fallbacks."""
  988. if _is_bedrock_context(base_url, provider):
  989. save_context_length(model, base_url or "bedrock://", length, source="bedrock-confirmed-v1")
  990. else:
  991. save_context_length(model, base_url, length)
  992. def get_cached_context_length(model: str, base_url: str, *, bedrock_confirmed: bool = False) -> Optional[int]:
  993. """Look up a previously discovered context length for model+provider."""
  994. key = _context_cache_key(model, base_url)
  995. document = _load_context_cache_document()
  996. cache = document.get("context_lengths") or {}
  997. if not isinstance(cache, dict):
  998. return None
  999. # Legacy rows may carry a trailing slash, so probe the canonical key, the literal form and the slashed canonical form.
  1000. matched_key = next((k for k in (key, f"{model}@{base_url}", f"{key}/") if cache.get(k) is not None), None)
  1001. length = cache.get(matched_key)
  1002. if type(length) is not int:
  1003. return None
  1004. if bedrock_confirmed:
  1005. confirmed = document.get("bedrock_confirmed_v1")
  1006. marker = confirmed.get(matched_key) if isinstance(confirmed, dict) else None
  1007. if type(marker) is not int or marker != length:
  1008. return None
  1009. return length
  1010. def _invalidate_cached_context_length(model: str, base_url: str) -> None:
  1011. """Drop a stale entry and its probe cooldown using the persisted cache URL.
  1012. Bedrock callers without a runtime endpoint must pass ``bedrock://``, as
  1013. the resolver does; a blank URL addresses the distinct generic cache key.
  1014. In-process caches are dropped unconditionally; only the file update takes the cache lock.
  1015. """
  1016. from agent.model_metadata_cache import drop_context_cache_rows
  1017. key = _context_cache_key(model, base_url)
  1018. # Also drop the in-memory TTL probe entries, or the next resolution inside the TTL window reuses the stale value.
  1019. bare, stripped = _strip_provider_prefix(model), (base_url or "").rstrip("/")
  1020. _LOCAL_CTX_PROBE_CACHE.pop((bare, stripped), None)
  1021. _LOCAL_CTX_PROBE_CACHE.pop(("ollama_show", bare, stripped), None)
  1022. # Same for a memoised Bedrock probe failure (keyed by region, which the caller does not know):
  1023. # the entry being dropped is the reason to ask the probe again, not to wait out its TTL.
  1024. from hermes_constants import hermes_home_key
  1025. for memo_key in list(_BEDROCK_PROBE_FAILURE_CACHE): # snapshot: another thread may be memoising
  1026. if memo_key[:2] == (hermes_home_key(), stripped) and memo_key[2] in (model, bare):
  1027. _BEDROCK_PROBE_FAILURE_CACHE.pop(memo_key, None)
  1028. # Every key shape get_cached_context_length consults.
  1029. drop_context_cache_rows({key, f"{model}@{base_url}", f"{key}/"}, key)
  1030. def get_next_probe_tier(current_length: int) -> Optional[int]:
  1031. """Next lower probe tier, or None if already at minimum."""
  1032. return next((tier for tier in CONTEXT_PROBE_TIERS if tier < current_length), None)
  1033. def parse_context_limit_from_error(error_msg: str) -> Optional[int]:
  1034. """Context limit quoted in a provider error ("maximum context length is 32768 tokens"), if any.
  1035. A message about only an OUTPUT cap ("... model output limit of 16384") never says "context";
  1036. bail out so the generic "limit ... of N" pattern can't cache the output cap as the window."""
  1037. error_lower = error_msg.lower()
  1038. if ("output limit" in error_lower or "output tokens" in error_lower or "output token" in error_lower) and "context" not in error_lower:
  1039. return None
  1040. patterns = (
  1041. r'max_model_len\s*(?:is\s*)?[:=(]?\s*(\d{4,})', # vLLM: "max_model_len 32768", "=32768", ": 32768", "(32768)", "is 32768"
  1042. r'maximum model length\s*(?:is\s*)?[:=(]?\s*(\d{4,})', # vLLM alt: "maximum model length 131072", "... is 131072"
  1043. r'(?:max(?:imum)?|limit)\s*(?:context\s*)?(?:length|size|window)?\s*(?:is|of|:)?\s*(\d{4,})',
  1044. r'context\s*(?:length|size|window)\s*(?:is|of|:)?\s*(\d{4,})',
  1045. r'(\d{4,})\s*(?:token)?\s*(?:context|limit)',
  1046. r'>\s*(\d{4,})\s*(?:max|limit|token)', # "250000 tokens > 200000 maximum"
  1047. r'(\d{4,})\s*(?:max(?:imum)?)\b', # "200000 maximum"
  1048. # Gemini: "input token count is 32825 but model only supports up to
  1049. # 32768" — anchor on the phrase so the input count isn't captured.
  1050. r'supports?\s+(?:only\s+)?up\s+to\s+(\d{4,})',
  1051. )
  1052. for match in filter(None, (re.search(pattern, error_lower) for pattern in patterns)):
  1053. limit = int(match.group(1))
  1054. if 1024 <= limit <= 10_000_000: # sanity: must be a plausible window
  1055. return limit
  1056. return None
  1057. def get_context_length_from_provider_error(error_msg: str, current_context_length: int) -> Optional[int]:
  1058. """Provider-reported limit LOWER than the current window, else None. Overflow recovery must
  1059. not invent a window: when the provider only says the input is too long, callers keep the
  1060. configured length and compress rather than stepping down guessed probe tiers."""
  1061. parsed_limit = parse_context_limit_from_error(error_msg)
  1062. return parsed_limit if parsed_limit is not None and parsed_limit < current_context_length else None
  1063. # OpenAI's original overflow wording, copied by vLLM / llama-cpp-python: "(36865 in the messages,
  1064. # 65536 in the completion)"; legacy completions: "(771 in your prompt; 4000 for the completion)".
  1065. # The first figure is the prompt the server MEASURED, the second the requested max_tokens.
  1066. _COMPLETION_SPLIT_RE = re.compile(
  1067. r'\((\d+)\s+(?:tokens\s+)?in (?:the messages|your prompt|the prompt)\s*[;,]\s*'
  1068. r'(\d+)\s+(?:tokens\s+)?(?:in|for) the completion\)'
  1069. )
  1070. def _completion_split_budget(error_lower: str) -> Optional[int]:
  1071. """window - measured prompt from the OpenAI-style parenthetical split, or None when the wording
  1072. is absent or the prompt alone fills the window (a genuine input overflow -> compress)."""
  1073. split = _COMPLETION_SPLIT_RE.search(error_lower)
  1074. ctx = re.search(r'maximum context length is (\d+)', error_lower)
  1075. if not split or not ctx:
  1076. return None
  1077. available = int(ctx.group(1)) - int(split.group(1))
  1078. return available if available >= 1 else None
  1079. def parse_available_output_tokens_from_error(error_msg: str) -> Optional[int]:
  1080. """Available OUTPUT tokens from a "max_tokens too large" error, or None. Distinct from "prompt
  1081. too long" (-> compress): here input + requested_output > window, so the fix is a smaller
  1082. max_tokens for this call and context_length must NOT be touched."""
  1083. error_lower = error_msg.lower()
  1084. if not _any_phrase_group(error_lower, _PARSEABLE_OUTPUT_CAP_SIGNALS):
  1085. return None
  1086. # Direct cap figures, most specific first: "exceeds model's maximum output tokens (65536)", "Range of
  1087. # max_tokens should be [1, 65536]" (upper bound is the cap), Anthropic "max_tokens: 100000 > 64000, which
  1088. # is the maximum allowed number of output tokens" (the ceiling is the right-hand side), Anthropic
  1089. # "= available_tokens: 10000", last "= N".
  1090. for pattern in (
  1091. r'exceeds model(?:\'s)? maximum output tokens\s*\(?\s*(\d+)\s*\)?',
  1092. r'max_tokens\s*:\s*\d+\s*>\s*(\d+)\s*,?\s*which is the maximum allowed number of output tokens',
  1093. r'range of max_tokens should be\s*\[\s*\d+\s*,\s*(\d+)\s*\]',
  1094. r'available_tokens[:\s]+(\d+)',
  1095. r'available\s+tokens[:\s]+(\d+)',
  1096. # Switchyard: "max_tokens cannot exceed the configured model output limit of 16384".
  1097. r'output limit (?:of|is)\s*(\d+)',
  1098. # Azure OpenAI: "max_tokens is too large: 65536. This model supports at most 32768 completion tokens."
  1099. r'supports at most\s+(\d+)\s*(?:completion\s+)?tokens',
  1100. # Scaleway: "max_completion_tokens is limited to 16384 for glm-5.2".
  1101. r'(?:max_tokens|max_completion_tokens) is limited to\s*(\d+)',
  1102. r'=\s*(\d+)\s*$',
  1103. ):
  1104. match = re.search(pattern, error_lower)
  1105. if match and int(match.group(1)) >= 1:
  1106. return int(match.group(1))
  1107. # OpenRouter/Nous: "maximum context length is N … (A of text input, B of tool input, C in the output)" -> ctx - A - B.
  1108. _m_ctx = re.search(r'maximum context length is (\d+)', error_lower)
  1109. _m_parts = re.search(r'\((\d+)\s+of text input,\s*(\d+)\s+of tool input,\s*(\d+)\s+in the output\)', error_lower)
  1110. if _m_ctx and _m_parts:
  1111. _available = int(_m_ctx.group(1)) - int(_m_parts.group(1)) - int(_m_parts.group(2))
  1112. if _available >= 1:
  1113. return _available
  1114. _split_available = _completion_split_budget(error_lower)
  1115. if _split_available is not None:
  1116. return _split_available
  1117. # LM Studio / llama.cpp: window in tokens, prompt in CHARACTERS; ~3 chars/token over-reserves the input.
  1118. _m_ctx_tok = re.search(r'maximum context length is (\d+)\s*token', error_lower)
  1119. _m_chars = re.search(r'prompt contains (\d+)\s*character', error_lower)
  1120. if _m_ctx_tok and _m_chars:
  1121. _available = int(_m_ctx_tok.group(1)) - (int(_m_chars.group(1)) + 2) // 3
  1122. if _available >= 1:
  1123. return _available
  1124. # SGLang: "maximum context length of 131072 tokens. You requested a total of 132528 tokens: 66992 tokens
  1125. # from the input messages and 65536 tokens for the completion" -> window - input (None when the input
  1126. # alone overflows -> compress).
  1127. _m_sglang = _sglang_window_and_input(error_lower)
  1128. if _m_sglang and _m_sglang[0] - _m_sglang[1] >= 1:
  1129. return _m_sglang[0] - _m_sglang[1]
  1130. # vLLM: window and prompt both in TOKENS; available = window - input (None when the input alone
  1131. # overflows -> compress). When max_tokens is the BINDING constraint vLLM reports "at least N input
  1132. # tokens" with N == window + 1 - requested_output, so window - N == requested_output - 1 and each
  1133. # retry walks the cap down by the safety margin without ever fitting: halve the cap instead.
  1134. _m_vllm_input = re.search(r'prompt contains (?:at least )?(\d+)\s*input tokens', error_lower)
  1135. if _m_ctx_tok and _m_vllm_input:
  1136. _available = int(_m_ctx_tok.group(1)) - int(_m_vllm_input.group(1))
  1137. _m_requested_out = re.search(r'requested (\d+)\s*output tokens', error_lower)
  1138. if 'at least' in error_lower and _m_requested_out:
  1139. _requested_out = int(_m_requested_out.group(1))
  1140. if _available >= _requested_out - 1:
  1141. # The budget is derived from the constraint, not measured.
  1142. return max(1, _requested_out // 2)
  1143. if _available >= 1:
  1144. return _available
  1145. return None
  1146. # Each entry is a phrase group; the group matches when ALL phrases are present.
  1147. # DashScope, Anthropic (available_tokens / "maximum allowed number of output tokens"), OpenRouter/Nous,
  1148. # LM Studio/llama.cpp, generic "should be <= N", OpenAI-compat relays.
  1149. _OUTPUT_CAP_SIGNALS = (
  1150. ("range of max_tokens should be",), ("available_tokens",), ("available tokens",),
  1151. ("in the output", "maximum context length"), ("requested", "output tokens"),
  1152. ("should be",), ("less than or equal",), ("must be",), ("exceeds model", "maximum output tokens"),
  1153. ("output limit",), ("maximum allowed number of output tokens",),
  1154. ("max_tokens is too large", "supports at most"), ("tokens from the input messages", "tokens for the completion"),
  1155. ("limited to",), # Scaleway: "max_completion_tokens is limited to 16384 for <model>" (#67453)
  1156. )
  1157. _INPUT_OVERFLOW_SIGNALS = (
  1158. "prompt is too long", "prompt too long", "input is too long", "input token",
  1159. "prompt length", "prompt contains", "reduce the length",
  1160. )
  1161. # Narrower than _OUTPUT_CAP_SIGNALS: only phrasings we can extract a number from.
  1162. # "requested N output tokens" means the OUTPUT cap is the problem (the input fits) —
  1163. # reduce max_tokens, don't compress. DashScope's bounded range upper bound IS the
  1164. # real max-output cap ("Range of max_tokens should be [1, 65536]").
  1165. _PARSEABLE_OUTPUT_CAP_SIGNALS = (
  1166. ("max_tokens", "available_tokens"), ("max_tokens", "available tokens"),
  1167. ("in the output", "maximum context length"),
  1168. ("maximum context length", "requested", "output tokens"),
  1169. ("maximum context length", "in the completion"), ("maximum context length", "for the completion"),
  1170. ("range of max_tokens should be",), ("exceeds model", "maximum output tokens"),
  1171. ("output limit",), ("max_tokens", "maximum allowed number of output tokens"),
  1172. ("max_tokens is too large", "supports at most"), ("tokens from the input messages", "tokens for the completion"),
  1173. ("limited to",),
  1174. )
  1175. def _sglang_window_and_input(error_lower: str) -> Optional[tuple[int, int]]:
  1176. """``(window, input_tokens)`` from SGLang's wording, else None; both figures are explicit there."""
  1177. _m_ctx = re.search(r'maximum context length of (\d+)\s*token', error_lower)
  1178. _m_in = re.search(r'(\d+)\s*tokens from the input messages', error_lower)
  1179. return (int(_m_ctx.group(1)), int(_m_in.group(1))) if _m_ctx and _m_in else None
  1180. def _any_phrase_group(text: str, groups: tuple) -> bool:
  1181. return any(all(p in text for p in group) for group in groups)
  1182. def is_output_cap_error(error_msg: str) -> bool:
  1183. """Yes/no sibling of :func:`parse_available_output_tokens_from_error` for unparseable wordings. An
  1184. output-cap 400 misclassified as context overflow death-loops the compressor (same max_tokens, same
  1185. rejection). Signal: talks about max_tokens as a cap/range/limit and NOT about an oversized input."""
  1186. error_lower = error_msg.lower()
  1187. # The OpenAI-style split names neither max_tokens nor "output tokens" and ends with "reduce the
  1188. # length", so it fails both gates below; the measured prompt decides instead (#90607).
  1189. if _completion_split_budget(error_lower) is not None:
  1190. return True
  1191. # An error that ALSO describes an oversized INPUT is a genuine overflow — compression can fix it.
  1192. # SGLang states both figures: input >= window is that same genuine overflow.
  1193. _m_sglang = _sglang_window_and_input(error_lower)
  1194. return (
  1195. any(p in error_lower for p in ("max_tokens", "max_output_tokens", "max_completion_tokens", "tokens for the completion"))
  1196. and _any_phrase_group(error_lower, _OUTPUT_CAP_SIGNALS)
  1197. and not any(p in error_lower for p in _INPUT_OVERFLOW_SIGNALS)
  1198. and not (_m_sglang and _m_sglang[1] >= _m_sglang[0])
  1199. )
  1200. def _model_id_matches(candidate_id: str, lookup_model: str) -> bool:
  1201. """Exact match, or ``publisher/slug`` (LM Studio native ids) whose slug equals the configured name."""
  1202. return candidate_id == lookup_model or ("/" in candidate_id and candidate_id.rsplit("/", 1)[1] == lookup_model)
  1203. def _ollama_show(server_url: str, api_key: str, bare_model: str, timeout: float = 3.0, *, note_blackhole: bool = False) -> Optional[dict[str, Any]]:
  1204. """Ollama ``/api/show`` JSON for ``bare_model``, or None on any failure (``note_blackhole``: connect timeouts condemn the host)."""
  1205. import httpx
  1206. try:
  1207. with httpx.Client(timeout=timeout, headers=_auth_headers(api_key), verify=model_metadata_http.resolve_verify(server_url)) as client:
  1208. resp = client.post(f"{server_url}/api/show", json={"name": bare_model})
  1209. return resp.json() if resp.status_code == 200 else None
  1210. except Exception as exc:
  1211. if note_blackhole:
  1212. _note_if_connect_timeout(exc, server_url)
  1213. return None
  1214. def _is_ollama_server(base_url: str, api_key: str) -> bool:
  1215. try:
  1216. # Forward the API key: a remote API-keyed endpoint answers the probe waterfall with 401s without it,
  1217. # and an unauthorized probe can never produce a positive verdict (#89863).
  1218. return detect_local_server_type(base_url, api_key=api_key) == "ollama"
  1219. except Exception:
  1220. return False
  1221. def query_ollama_num_ctx(model: str, base_url: str, api_key: str = "") -> Optional[int]:
  1222. """Ollama ``/api/show`` context (Modelfile num_ctx, else GGUF max); the value to send as ``num_ctx``."""
  1223. bare_model, server_url = _strip_provider_prefix(model), _server_root(base_url)
  1224. if not _is_ollama_server(base_url, api_key):
  1225. return None
  1226. _disk_key = f"{server_url}|{bare_model}"
  1227. disk_hit = _local_probe_disk_get("ollama_num_ctx", _disk_key)
  1228. if isinstance(disk_hit, int) and disk_hit > 0:
  1229. return disk_hit
  1230. data = _ollama_show(server_url, api_key, bare_model)
  1231. ctx = _ollama_show_context(data, gguf_first=False) if data is not None else None
  1232. if ctx is not None:
  1233. _local_probe_disk_put("ollama_num_ctx", _disk_key, ctx)
  1234. return ctx
  1235. def query_ollama_supports_vision(model: str, base_url: str, api_key: str = "") -> Optional[bool]:
  1236. """True/False when Ollama ``/api/show`` reports vision support (``capabilities`` on 0.6.0+, else
  1237. ``model_info.*.vision.block_count``); None when unreachable, not Ollama, or model unknown."""
  1238. bare_model = _strip_provider_prefix(model)
  1239. if not bare_model or not base_url or not _is_ollama_server(base_url, api_key):
  1240. return None
  1241. data = _ollama_show(_server_root(base_url), api_key, bare_model)
  1242. if data is None:
  1243. return None
  1244. caps = data.get("capabilities")
  1245. if isinstance(caps, list) and caps:
  1246. return any(str(cap).lower() == "vision" for cap in caps)
  1247. model_info = data.get("model_info")
  1248. if isinstance(model_info, dict) and any("vision.block_count" in str(key).lower() for key in model_info):
  1249. return True
  1250. return None
  1251. def _memo_local_probe(cache_key: tuple, probe: Callable[[], Optional[int]]) -> Optional[int]:
  1252. """Short-TTL, positive-only memo of a local probe (see _LOCAL_CTX_PROBE_CACHE): a failure during
  1253. a startup race must not suppress the retry once the server is up, so only truthy results memoize."""
  1254. now = time.monotonic()
  1255. cached = _LOCAL_CTX_PROBE_CACHE.get(cache_key)
  1256. if cached is not None and (now - cached[1]) < _LOCAL_CTX_PROBE_TTL_SECONDS:
  1257. return cached[0]
  1258. result = probe()
  1259. if result:
  1260. _LOCAL_CTX_PROBE_CACHE[cache_key] = (result, now)
  1261. return result
  1262. def _query_ollama_api_show(model: str, base_url: str, api_key: str = "") -> Optional[int]:
  1263. """Provider-agnostic Ollama ``/api/show`` probe (any hostname; non-Ollama servers 404 fast).
  1264. GGUF-first (hosted users can't set num_ctx) — the reverse of query_ollama_num_ctx(), hence the namespaced memo key."""
  1265. return _memo_local_probe(("ollama_show", _strip_provider_prefix(model), base_url.rstrip("/")), lambda: _query_ollama_api_show_uncached(model, base_url, api_key=api_key))
  1266. def _query_ollama_api_show_uncached(model: str, base_url: str, api_key: str = "") -> Optional[int]:
  1267. """Uncached body of ``_query_ollama_api_show`` — one POST to ``/api/show``."""
  1268. server_url = _server_root(base_url)
  1269. if _endpoint_blackholed(server_url):
  1270. return None
  1271. data = _ollama_show(server_url, api_key, model, timeout=5.0, note_blackhole=True)
  1272. # Hosted Ollama: the GGUF max is authoritative (the operator may have capped num_ctx arbitrarily).
  1273. return _ollama_show_context(data, gguf_first=True, minimum=1024) if data is not None else None
  1274. def _model_name_suggests_kimi(model: str) -> bool:
  1275. """Kimi family (``kimi-*``, ``moonshotai/*``) — guard against stale 32K underreports."""
  1276. lower = model.lower()
  1277. return lower.startswith("kimi") or "moonshot" in lower
  1278. def _model_name_suggests_minimax_m3(model: str) -> bool:
  1279. """MiniMax M3 on any surface — models.dev underreport guard and agent_runtime_helpers cache-control gating."""
  1280. return "minimax-m3" in model.lower()
  1281. # Catalog keys added AFTER the model was reachable via a shorter catch-all (or the
  1282. # 256K fallback): older builds persisted that smaller value and the step-1 cache hit
  1283. # would pin it forever. Only list keys whose catalog value is STRICTLY ABOVE every
  1284. # shorter matching key and the 256K fallback — the threshold is inferred from them.
  1285. _PRE_CATALOG_STALE_KEYS = frozenset({
  1286. "minimax-m3", # 1M; "minimax" catch-all persisted 204,800
  1287. "muse-spark-1.3", "muse-spark", # 1M; pre-entry builds fell through to the 256K fallback
  1288. "grok-4.3", "grok-4.6", # 1M / 500K; "grok-4" catch-all persisted 256,000
  1289. "grok-4-fast", "grok-4.20", # 2M; fell through to the 256K fallback
  1290. "qwen3.6-plus", # 1M; "qwen" catch-all persisted 131,072
  1291. # V4 / V4.1 Flash: 1M. Pre-entry builds matched the family catch-all and persisted 128K.
  1292. "deepseek-flash", "deepseek-v4.1-flash", "deepseek-v4-flash", "deepseek-v4-pro",
  1293. "deepseek-chat", "deepseek-reasoner",
  1294. })
  1295. def _stale_pre_catalog_cache_entry(model: str, cached: int) -> bool:
  1296. """True when a persisted window is a pre-catalog leftover: the model resolves (longest-key-first) to a
  1297. _PRE_CATALOG_STALE_KEYS key and the cached value is <= the largest shorter matching catch-all (or 256K)."""
  1298. model_lower = model.lower()
  1299. matches = [(key, value) for key, value in DEFAULT_CONTEXT_LENGTHS.items() if _catalog_key_matches(key, model_lower)]
  1300. if not matches:
  1301. return False
  1302. specific_key, specific_value = max(matches, key=lambda kv: len(kv[0]))
  1303. if specific_key not in _PRE_CATALOG_STALE_KEYS or cached >= specific_value:
  1304. return False
  1305. shorter_values = [v for k, v in matches if len(k) < len(specific_key)]
  1306. return cached <= max(shorter_values, default=DEFAULT_FALLBACK_CONTEXT)
  1307. def _model_name_suggests_minimax(model: str) -> bool:
  1308. """MiniMax family (``minimax*``, ``minimaxai/*``) — guard against stale 32K underreports (real: 204.8K)."""
  1309. lower = model.lower()
  1310. return lower.startswith("minimax") or "minimaxai/" in lower
  1311. def _model_name_suggests_stale_32k_underreport(model: str) -> bool:
  1312. """Model families known to be wrongly underreported as 32K."""
  1313. return _model_name_suggests_kimi(model) or _model_name_suggests_minimax(model)
  1314. def _query_local_context_length(model: str, base_url: str, api_key: str = "") -> Optional[int]:
  1315. """Local-server context probe, short-TTL cached (see _LOCAL_CTX_PROBE_CACHE)."""
  1316. return _memo_local_probe((_strip_provider_prefix(model), base_url.rstrip("/")), lambda: _query_local_context_length_uncached(model, base_url, api_key=api_key))
  1317. def _positive_int(value: Any) -> Optional[int]:
  1318. return int(value) if isinstance(value, (int, float)) and value else None
  1319. def _lmstudio_context(client, lmstudio_url: str, model: str) -> Optional[int]:
  1320. """LM Studio native /api/v1/models (the OpenAI-compat list omits context);
  1321. loaded-instance config is the runtime value."""
  1322. resp = client.get(f"{lmstudio_url}/api/v1/models")
  1323. if resp.status_code != 200:
  1324. return None
  1325. for m in resp.json().get("models", []):
  1326. if _model_id_matches(m.get("key", ""), model) or _model_id_matches(m.get("id", ""), model):
  1327. return next((ctx for ctx in (_positive_int(inst.get("config", {}).get("context_length")) for inst in m.get("loaded_instances", [])) if ctx is not None), None)
  1328. return None
  1329. def _openai_models_list_context(client, server_url: str, model: str) -> Optional[int]:
  1330. """/v1/models list: match by id, else the sole model on single-model servers (llama.cpp reports a GGUF path as id)."""
  1331. resp = client.get(f"{server_url}/v1/models")
  1332. if resp.status_code != 200:
  1333. return None
  1334. models_list = resp.json().get("data", [])
  1335. matched = next((m for m in models_list if isinstance(m, dict) and _model_id_matches(m.get("id", ""), model)), None)
  1336. if matched is None and len(models_list) == 1:
  1337. matched = models_list[0]
  1338. if matched is None:
  1339. return None
  1340. # Runtime n_ctx (llama.cpp nests it under meta) beats n_ctx_train, which can exceed what the server allocates.
  1341. sources = [s for s in (matched, matched.get("meta") or {}) if isinstance(s, dict)]
  1342. for reader in (lambda src: _positive_int(src.get("n_ctx")), _context_length_from_model_payload):
  1343. for source in sources:
  1344. ctx = reader(source)
  1345. if ctx is not None:
  1346. return ctx
  1347. return None
  1348. def _query_local_context_length_uncached(model: str, base_url: str, api_key: str = "") -> Optional[int]:
  1349. """Query a local server for the model's context length."""
  1350. import httpx
  1351. model = _strip_provider_prefix(model)
  1352. server_url = _server_root(base_url)
  1353. lmstudio_url = _localhost_to_ipv4(_lmstudio_server_root(base_url))
  1354. if _endpoint_blackholed(server_url):
  1355. return None
  1356. try:
  1357. server_type = detect_local_server_type(base_url, api_key=api_key)
  1358. except Exception:
  1359. server_type = None
  1360. def _ollama_ctx(client) -> Optional[int]:
  1361. # Ollama: num_ctx (runtime window) before the GGUF training max, or conversations
  1362. # grow past what Ollama allocated and silently truncate. Matches query_ollama_num_ctx().
  1363. resp = client.post(f"{server_url}/api/show", json={"name": model})
  1364. return _ollama_show_context(resp.json(), gguf_first=False) if resp.status_code == 200 else None
  1365. def _model_detail_ctx(client) -> Optional[int]:
  1366. # LM Studio / vLLM / llama.cpp / Anthropic-compat proxies: /v1/models/{model}.
  1367. # Skipped for unrecognised servers (server_type None, e.g. LiteLLM proxies): they commonly
  1368. # gate this endpoint behind admin auth (#25848) and log an ERROR per probe even though the
  1369. # 401 falls through to the /v1/models list below, which works universally.
  1370. if server_type is None:
  1371. return None
  1372. resp = client.get(f"{server_url}/v1/models/{model}")
  1373. return _context_length_from_model_payload(resp.json()) if resp.status_code == 200 else None
  1374. typed = {
  1375. "ollama": _ollama_ctx,
  1376. "lm-studio": lambda client: _lmstudio_context(client, lmstudio_url, model),
  1377. "llamacpp": lambda client: llamacpp_context(client, server_url, model),
  1378. }.get(server_type)
  1379. probes = ([typed] if typed else []) + [_model_detail_ctx, lambda client: _openai_models_list_context(client, server_url, model)]
  1380. try:
  1381. with httpx.Client(timeout=3.0, headers=_auth_headers(api_key), verify=model_metadata_http.resolve_verify(base_url)) as client:
  1382. return next((ctx for ctx in (probe(client) for probe in probes) if ctx is not None), None)
  1383. except Exception as exc:
  1384. _note_if_connect_timeout(exc, server_url)
  1385. return None
  1386. def _normalize_model_version(model: str) -> str:
  1387. """Dots -> dashes so Nous ids (claude-opus-4-6) compare with OpenRouter's (claude-opus-4.6)."""
  1388. return model.replace(".", "-")
  1389. def _query_anthropic_context_length(model: str, base_url: str, api_key: Any) -> Optional[int]:
  1390. """Anthropic /v1/models max_input_tokens; OAuth tokens (sk-ant-oat*) 401 and are skipped.
  1391. ``api_key`` may be a ``key_cmd`` callable token source; the metadata probe never mints — a
  1392. callable is not a Console key, so the lookup is skipped like an OAuth token (#114967).
  1393. """
  1394. if not api_key or not isinstance(api_key, str) or api_key.startswith("sk-ant-oat"):
  1395. return None
  1396. try:
  1397. base = base_url.rstrip("/").removesuffix("/v1")
  1398. headers = {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
  1399. resp = model_metadata_http.get(f"{base}/v1/models?limit=1000", headers=headers, timeout=(5, 10), verify=model_metadata_http.resolve_verify(base_url))
  1400. if resp.status_code != 200:
  1401. return None
  1402. for m in resp.json().get("data", []):
  1403. ctx = m.get("max_input_tokens") if m.get("id") == model else None
  1404. if isinstance(ctx, int) and ctx > 0:
  1405. return ctx
  1406. except Exception as e:
  1407. logger.debug("Anthropic /v1/models query failed: %s", e)
  1408. return None
  1409. # Codex OAuth `context_window` values (what Codex enforces — lower than the direct API for the same
  1410. # slugs). Fallback when the live probe fails; longest-key-first. gpt-5.3-codex-spark is listed so "gpt-5.3-codex" doesn't win.
  1411. _CODEX_OAUTH_CONTEXT_FALLBACK: dict[str, int] = {
  1412. "gpt-6-astra": 272_000, "gpt-6.1-sol": 272_000, "gpt-6-sol": 272_000, "gpt-6-luna": 272_000,
  1413. "gpt-5.1-codex-max": 272_000, "gpt-5.1-codex-mini": 272_000, "gpt-5.3-codex": 272_000,
  1414. "gpt-5.3-codex-spark": 128_000, "gpt-5.2-codex": 272_000, "gpt-5.4-mini": 272_000,
  1415. "gpt-5.6-sol": 272_000, "gpt-5.6-terra": 272_000, "gpt-5.6-luna": 272_000, "gpt-daybreak-blue-latest": 272_000,
  1416. "gpt-5.5": 272_000, "gpt-5.4": 272_000, "gpt-5.2": 272_000, "gpt-5": 272_000,
  1417. }
  1418. # Codex OAuth advertises 272K for these families but ACCEPTS ~900K+ (verified live; gpt-5.5 and
  1419. # gpt-5.4-mini genuinely reject >272K). 900K keeps ≥11K margin. OPT-IN ONLY via explicit ``-900k``
  1420. # picker variants (a 900K default burned subscription usage); the suffix is stripped before the wire.
  1421. # The bump fires ONLY when the resolved value is exactly the stale 272,000. ``gpt-5.6`` and the gpt-6
  1422. # tiers are FAMILY PREFIXES so dated snapshots (``gpt-6-sol-2026-09-22``) inherit the bump (``-pro`` slugs
  1423. # aren't routable on Codex); ``gpt-5.4`` is EXACT because gpt-5.4-mini enforces 272K. The gpt-6 tiers
  1424. # carry the 5.6 verdict forward: they replace Sol/Terra/Luna on the same Codex route, and the live
  1425. # catalog's ``max_context_window`` still caps the bump (#105443) if it publishes a lower ceiling.
  1426. _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES: dict[str, int] = {
  1427. "gpt-5.6": 900_000, "gpt-6-sol": 900_000, "gpt-6-luna": 900_000,
  1428. }
  1429. _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT: dict[str, int] = {
  1430. "gpt-5.4": 900_000, "gpt-daybreak-blue-latest": 900_000,
  1431. "gpt-6-astra": 900_000, # advertised 272K; 920,043 input OK, 1,000,043 rejected (live 2026-09-04)
  1432. # advertised 272K; 918,137 input OK, ~931K rejected (live 2026-09-29), consistent with the 922K left by
  1433. # the model page's 1.05M context minus 128K max output (derived, not a published input cap).
  1434. # EXACT, not a prefix: the dotted slug is its own line and ``-pro`` is not routable.
  1435. "gpt-6.1-sol": 900_000,
  1436. }
  1437. _CODEX_OAUTH_STALE_ADVERTISED_CTX = 272_000 # the only advertised value the bump may override
  1438. CODEX_CONTEXT_VARIANT_SUFFIX = "-900k" # picker-only opt-in suffix; never sent on the wire
  1439. # The ONLY bases eligible for ``-900k``: routable, live-verified. No family prefixing (it would synthesize
  1440. # dead ``-pro`` variants); dated snapshots of the 5.6 / gpt-6 tier bases are allowed. gpt-daybreak-blue-latest is a verified Sol alias.
  1441. _CODEX_900K_SNAPSHOT_BASES = ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-6-sol", "gpt-6-luna")
  1442. _CODEX_900K_ELIGIBLE_BASES = frozenset({*_CODEX_900K_SNAPSHOT_BASES, "gpt-5.4", "gpt-daybreak-blue-latest", "gpt-6-astra", "gpt-6.1-sol"})
  1443. _CODEX_900K_SNAPSHOT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
  1444. def _bare_codex_slug(model: Optional[str]) -> str:
  1445. """Lowercased slug without ``vendor/`` (display/auxiliary callers pass ``openai/gpt-5.6-sol-900k``).
  1446. Display/auxiliary callers pass ids like ``openai/gpt-5.6-sol-900k``; the main-agent path normalizes the
  1447. namespace away earlier, but this resolver must accept both shapes (#92797 review).
  1448. """
  1449. return (model or "").strip().lower().rsplit("/", 1)[-1]
  1450. def is_codex_900k_base(model: Optional[str]) -> bool:
  1451. """Single source of truth for ``-900k`` eligibility (picker, resolution, /model validation, wire stripping)."""
  1452. slug = _bare_codex_slug(model)
  1453. if not slug or slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX):
  1454. return False
  1455. # Dated snapshots of the routable tier bases (gpt-5.6-sol-2026-07-09, gpt-6-sol-2026-09-22) also qualify.
  1456. return slug in _CODEX_900K_ELIGIBLE_BASES or any(
  1457. slug.startswith(base + "-") and _CODEX_900K_SNAPSHOT_RE.match(slug[len(base) + 1:]) for base in _CODEX_900K_SNAPSHOT_BASES
  1458. )
  1459. def _codex_variant_base(model: Optional[str]) -> Optional[str]:
  1460. """Lowercased eligible base of a VALID ``-900k`` variant, else None (``gpt-5.5-900k`` is an invalid alias)."""
  1461. slug = _bare_codex_slug(model)
  1462. base = slug[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)] if slug.endswith(CODEX_CONTEXT_VARIANT_SUFFIX) else None
  1463. return base if base is not None and is_codex_900k_base(base) else None
  1464. def is_codex_context_variant(model: Optional[str]) -> bool:
  1465. """Suffix AND eligible base — ``gpt-5.5-900k`` is an invalid alias, not a variant."""
  1466. return _codex_variant_base(model) is not None
  1467. def strip_codex_context_variant_suffix(model: Optional[str]) -> str:
  1468. """Wire-safe slug with a VALID ``-900k`` suffix removed (vendor prefix kept); an ineligible alias is
  1469. returned unchanged so it fails honestly at the API instead of running as another model."""
  1470. raw = (model or "").strip()
  1471. return raw[: -len(CODEX_CONTEXT_VARIANT_SUFFIX)] if _codex_variant_base(raw) is not None else raw
  1472. def has_codex_context_variant(model_bare: str) -> bool:
  1473. """Picker-side alias of :func:`is_codex_900k_base`."""
  1474. return is_codex_900k_base(model_bare)
  1475. def _verified_codex_ctx_for_slug(model_bare: str) -> Optional[int]:
  1476. """Live-verified cap for a VALID ``-900k`` variant only; base slugs and ineligible aliases -> None."""
  1477. base = _codex_variant_base(model_bare)
  1478. if base is None:
  1479. return None
  1480. exact = _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT.get(base)
  1481. if exact is not None:
  1482. return exact
  1483. return next((ctx for key, ctx in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES.items() if base == key or base.startswith((key + "-", key + "."))), None)
  1484. _codex_oauth_context_cache: dict[str, tuple[dict[str, int], float]] = {}
  1485. # ``{slug: max_context_window}`` from the same fetch, keyed by the same token fingerprint. Only the
  1486. # opted-in ``-900k`` bump reads it (#105443); a catalog without the field leaves the entry empty.
  1487. _codex_oauth_max_context_cache: dict[str, dict[str, int]] = {}
  1488. _CODEX_OAUTH_CONTEXT_CACHE_TTL = 3600 # 1 hour
  1489. _CODEX_OAUTH_CONTEXT_NEGATIVE_TTL = 300 # a probe that found no catalog is retried after 5 minutes; must stay < TTL
  1490. # The Codex models endpoint reads ``client_version`` as a Codex CLI compatibility version and
  1491. # hides models whose ``minimal_client_version`` is newer. "0.0.0" used to be the ungated sentinel
  1492. # returning the whole account catalog, but since the GPT-6 Sol/Luna rollout it returns a FROZEN
  1493. # legacy list (astra + the 5.6 trio) while any version at or above the newest
  1494. # ``minimal_client_version`` (0.155.0, 1.0.0, 99.0.0 alike, live 2026-09-22) returns everything the
  1495. # account is entitled to (#119412). So ask as the newest possible client first and keep "0.0.0" as
  1496. # the fallback for the day the backend rejects out-of-sequence versions again (it used to return an
  1497. # empty catalog for them; omitting the parameter is HTTP 400).
  1498. CODEX_MODELS_CATALOG_ENDPOINT = "https://chatgpt.com/backend-api/codex/models"
  1499. CODEX_NEWEST_CLIENT_VERSION = "99.0.0"
  1500. CODEX_UNGATED_CLIENT_VERSION = "0.0.0"
  1501. def _codex_catalog_urls(base_url: str = "") -> tuple[str, ...]:
  1502. """Catalog URLs against ``base_url`` when it names a Codex-compatible gateway, else the
  1503. hard-coded endpoint. A custom base's credential belongs to that service — sending it to
  1504. chatgpt.com (or fetching the direct catalog behind a gateway's back) is wrong (#121486)."""
  1505. base = (base_url or "").strip().rstrip("/")
  1506. endpoint = f"{base}/models" if base else CODEX_MODELS_CATALOG_ENDPOINT
  1507. return tuple(
  1508. f"{endpoint}?client_version={v}"
  1509. for v in (CODEX_NEWEST_CLIENT_VERSION, CODEX_UNGATED_CLIENT_VERSION)
  1510. )
  1511. CODEX_MODELS_CATALOG_URLS = _codex_catalog_urls()
  1512. def _codex_catalog_probe_allowed(access_token: str, base_url: str = "") -> bool:
  1513. """Whether a catalog probe may carry ``access_token`` to ``base_url``'s ``/models``.
  1514. The caller binds ``base_url`` to the credential's own route, so a custom gateway is asked with
  1515. its own key (opaque or JWT). chatgpt.com only accepts ChatGPT OAuth access tokens — JWTs — so a
  1516. non-JWT credential aimed there is a gateway key composed with the wrong host: refuse it
  1517. (defense in depth, mirroring ``_probe_codex_quota_restored``'s gate; #121486).
  1518. """
  1519. if not access_token:
  1520. return False
  1521. base = (base_url or "").strip() or CODEX_MODELS_CATALOG_ENDPOINT
  1522. if not base_url_host_matches(base, "chatgpt.com"):
  1523. return True
  1524. from hermes_cli.auth_constants import _decode_jwt_claims
  1525. return bool(_decode_jwt_claims(access_token))
  1526. def fetch_codex_catalog_entries(get: Callable[[str], Any], base_url: str = "") -> tuple[list[Any], Optional[int]]:
  1527. """``(models, last_status)`` from the first catalog URL that answers HTTP 200 with a non-empty
  1528. ``models`` list; ``get(url)`` is any client returning an object with ``status_code``/``json()``.
  1529. An empty or non-200 answer on the newest-client URL falls through to the ``0.0.0`` sentinel."""
  1530. status: Optional[int] = None
  1531. for url in _codex_catalog_urls(base_url):
  1532. resp = get(url)
  1533. status = resp.status_code
  1534. if status != 200:
  1535. continue
  1536. data = resp.json()
  1537. entries = data.get("models") if isinstance(data, dict) else None
  1538. if isinstance(entries, list) and entries:
  1539. return entries, status
  1540. return [], status
  1541. def _codex_oauth_token_fingerprint(access_token: str, base_url: str = "") -> str:
  1542. """Non-secret cache key for a Codex OAuth access token (plus the base it was probed against —
  1543. a gateway's catalog can differ from chatgpt.com's for the same forwarded token)."""
  1544. return hashlib.sha256(f"{access_token}\n{(base_url or '').strip().rstrip('/')}".encode()).hexdigest()[:16]
  1545. def _remember_no_codex_catalog(cache_key: str) -> None:
  1546. """A probe that found no catalog (down, non-200, not Codex-shaped) is remembered briefly: context
  1547. resolution runs on init, /status, aux and @-reference turns, and each miss can block ~30s. The entry is
  1548. stamped as if cached ``TTL - NEGATIVE_TTL`` ago, so the shared freshness check expires it after
  1549. ``_CODEX_OAUTH_CONTEXT_NEGATIVE_TTL``. A fresh catalog a concurrent probe just stored is kept, and the
  1550. max cache is left alone so an observed cap survives."""
  1551. with MEMO_LOCK:
  1552. now = time.time()
  1553. current = _codex_oauth_context_cache.get(cache_key)
  1554. if current is not None and current[0] and now - current[1] < _CODEX_OAUTH_CONTEXT_CACHE_TTL:
  1555. return
  1556. bound_codex_caches(cache_key)
  1557. _codex_oauth_context_cache[cache_key] = ({}, now - _CODEX_OAUTH_CONTEXT_CACHE_TTL + _CODEX_OAUTH_CONTEXT_NEGATIVE_TTL)
  1558. def _fetch_codex_oauth_context_lengths_with_source(access_token: str, base_url: str = "") -> tuple[dict[str, int], bool]:
  1559. """Account-scoped catalogue context lengths and HTTP freshness; also retain max-context and
  1560. speed metadata. In-process hits report False and must not drive persistent writes."""
  1561. if not _codex_catalog_probe_allowed(access_token, base_url):
  1562. return {}, False
  1563. now = time.time()
  1564. cache_key = _codex_oauth_token_fingerprint(access_token, base_url)
  1565. cached = _codex_oauth_context_cache.get(cache_key)
  1566. if cached is not None and now - cached[1] < _CODEX_OAUTH_CONTEXT_CACHE_TTL:
  1567. return cached[0], False
  1568. # Without ChatGPT-Account-ID /backend-api/codex/models returns ``{"models":[]}`` (HTTP 200) and
  1569. # the probe silently falls back; residency-enforced workspaces 401 without the residency header.
  1570. from agent.codex_headers import codex_account_headers
  1571. headers = {"Authorization": f"Bearer {access_token}", **codex_account_headers(access_token)}
  1572. try:
  1573. entries, status = fetch_codex_catalog_entries(
  1574. lambda url: model_metadata_http.get(url, headers=headers, timeout=(5, 10), verify=model_metadata_http.resolve_verify()),
  1575. base_url=base_url,
  1576. )
  1577. if status != 200:
  1578. logger.debug("Codex /models probe returned HTTP %s; falling back to hardcoded defaults", status)
  1579. _remember_no_codex_catalog(cache_key)
  1580. return {}, False
  1581. except Exception as exc:
  1582. logger.debug("Codex /models probe failed: %s", exc)
  1583. _remember_no_codex_catalog(cache_key)
  1584. return {}, False
  1585. from agent.codex_service_tiers import remember_catalog
  1586. remember_catalog(cache_key, entries)
  1587. result: dict[str, int] = {}
  1588. max_result: dict[str, int] = {}
  1589. for item in entries:
  1590. slug, ctx, max_ctx = (item.get("slug"), item.get("context_window"), item.get("max_context_window")) if isinstance(item, dict) else (None, None, None)
  1591. if isinstance(slug, str) and isinstance(ctx, int) and ctx > 0:
  1592. result[slug.strip()] = ctx
  1593. if isinstance(max_ctx, int) and max_ctx > 0:
  1594. max_result[slug.strip()] = max_ctx
  1595. if not result:
  1596. _remember_no_codex_catalog(cache_key)
  1597. return {}, False
  1598. # Max first: a reader that sees the fresh context entry must also see its cap.
  1599. with MEMO_LOCK:
  1600. bound_codex_caches(cache_key)
  1601. _codex_oauth_max_context_cache[cache_key] = max_result
  1602. _codex_oauth_context_cache[cache_key] = (result, now)
  1603. return result, True
  1604. def _codex_catalog_key(keys, slug: str) -> Optional[str]:
  1605. """The catalog key for ``slug``: exact, then case-insensitive in case casing drifts."""
  1606. return slug if slug in keys else next((k for k in keys if k.lower() == slug.lower()), None)
  1607. def _cached_codex_catalog_max(access_token: str, base_url: str, slug: str) -> Optional[int]:
  1608. """``max_context_window`` the catalog behind ``(access_token, base_url)`` last published for ``slug``."""
  1609. published = _codex_oauth_max_context_cache.get(_codex_oauth_token_fingerprint(access_token, base_url), {})
  1610. return published.get(_codex_catalog_key(published, slug))
  1611. def _resolve_codex_oauth_context_length_with_source(model: str, access_token: str = "", base_url: str = "") -> tuple[Optional[int], str]:
  1612. """``(context_length, source)`` for a Codex OAuth slug. source: "live" (fresh authenticated probe —
  1613. the only one eligible for persistent writes), "memory" (same-token in-process hit), "fallback"
  1614. (static table), or "" when unresolved."""
  1615. model_bare = _strip_provider_prefix(model).strip()
  1616. if not model_bare:
  1617. return None, ""
  1618. def _apply_verified_bump(ctx: int, source: str, catalog_max: Optional[int] = None) -> tuple[int, str]:
  1619. """Lift an EXACT stale 272K advertisement to the verified cap for opted-in ``-900k`` variants only,
  1620. never above the catalog's own ``max_context_window`` when it publishes one (#105443)."""
  1621. bumped = _verified_codex_ctx_for_slug(model_bare)
  1622. if bumped is not None and ctx == _CODEX_OAUTH_STALE_ADVERTISED_CTX:
  1623. bumped = min(bumped, catalog_max) if catalog_max else bumped
  1624. logger.debug("Codex OAuth context for %s: advertised %d raised to live-verified %d", model_bare, ctx, bumped)
  1625. return bumped, source
  1626. return ctx, source
  1627. # The Codex catalog only knows the base slug (no -900k, no vendor/).
  1628. # ``-900k`` variants are Hermes picker aliases — the Codex catalog only knows the base slug, so resolve
  1629. # against the stripped id. Also drop any ``vendor/`` namespace (``openai/gpt-5.6-sol-900k``): the
  1630. # main-agent path normalizes it away before reaching here, but display/auxiliary callers pass it through
  1631. # (#92797 review).
  1632. lookup_bare = _bare_codex_slug(strip_codex_context_variant_suffix(model_bare))
  1633. catalog_max = None
  1634. if access_token:
  1635. live, fresh_probe = _fetch_codex_oauth_context_lengths_with_source(access_token, base_url=base_url)
  1636. # Read after the fetch; a failed refresh past the TTL keeps the cap already observed.
  1637. catalog_max = _cached_codex_catalog_max(access_token, base_url, lookup_bare)
  1638. slug = _codex_catalog_key(live, lookup_bare)
  1639. if slug is not None:
  1640. return _apply_verified_bump(live[slug], "live" if fresh_probe else "memory", catalog_max)
  1641. hit = _longest_key_match(_CODEX_OAUTH_CONTEXT_FALLBACK, lookup_bare.lower())
  1642. return _apply_verified_bump(hit[1], "fallback", catalog_max) if hit else (None, "")
  1643. def _resolve_nous_context_length(model: str, base_url: str = "", api_key: str = "") -> tuple[Optional[int], str]:
  1644. """``(context_length, source)`` for a Nous Portal model: portal /v1/models is authoritative
  1645. ("portal"). Fallback matches OR's prefixed ids against the bare Nous id with dot/dash
  1646. normalisation ("openrouter" — callers must NOT persist it, or a portal blip freezes the wrong value)."""
  1647. if base_url:
  1648. portal_ctx = _resolve_endpoint_context_length(model, base_url, api_key=api_key)
  1649. if portal_ctx is not None:
  1650. return portal_ctx, "portal"
  1651. metadata = fetch_model_metadata()
  1652. def _safe_ctx(or_id: str, entry: dict) -> Optional[int]:
  1653. """Context length minus the known stale 32K underreports (same guard as step 6)."""
  1654. ctx = entry.get("context_length")
  1655. if ctx is not None and ctx <= 32768 and _model_name_suggests_stale_32k_underreport(or_id):
  1656. logger.info("Rejecting OpenRouter metadata context=%s for %r (known 32K underreport, Nous path); falling through to hardcoded defaults", ctx, or_id)
  1657. return None
  1658. return ctx
  1659. model_lower, normalized = model.lower(), _normalize_model_version(model).lower()
  1660. def _pairs(or_id: str):
  1661. bare = or_id.split("/", 1)[1] if "/" in or_id else or_id
  1662. return ((bare.lower(), model_lower), (_normalize_model_version(bare).lower(), normalized))
  1663. def _exact(or_id: str) -> bool:
  1664. return any(candidate == query for candidate, query in _pairs(or_id))
  1665. def _prefix(or_id: str) -> bool:
  1666. return any(candidate.startswith(query) and (len(candidate) == len(query) or candidate[len(query)] in "-:.") for candidate, query in _pairs(or_id))
  1667. # Direct id, then exact bare-id match (with dot/dash normalisation), then prefix match on a
  1668. # separator boundary — separate passes so any exact hit beats every prefix hit.
  1669. for matcher in (lambda or_id: or_id == model, _exact, _prefix):
  1670. for or_id, entry in metadata.items():
  1671. ctx = _safe_ctx(or_id, entry) if matcher(or_id) else None
  1672. if ctx is not None:
  1673. return ctx, "openrouter"
  1674. return None, ""
  1675. def _validate_cached_context_length(model: str, base_url: str, cached: int, *, api_key: str = "") -> Optional[int]:
  1676. """Step 1 of get_model_context_length: accept, repair, or drop a persisted entry. Returns the
  1677. value to use, or None to fall through to live resolution. Order matters: a value must be
  1678. rejected as bogus before any provider-specific handling."""
  1679. # Drop rules: (predicate, log level, message, shown value). 0/negative is always a bug (`0 is not
  1680. # None` would hand the compressor a zero window); Kimi/MiniMax are underreported as 32K by stale
  1681. # third-party metadata; pre-catalog leftovers persisted a shorter catch-all (see _PRE_CATALOG_STALE_KEYS).
  1682. drop_rules = (
  1683. (cached <= 0, logger.warning, "Dropping non-positive cache entry %s@%s -> %s; re-resolving", cached),
  1684. (cached <= 32768 and _model_name_suggests_stale_32k_underreport(model), logger.info,
  1685. "Dropping stale cached context entry %s@%s -> %s (known 32K underreport); re-resolving via hardcoded defaults", f"{cached:,}"),
  1686. (_stale_pre_catalog_cache_entry(model, cached), logger.info,
  1687. "Dropping stale pre-catalog cache entry %s@%s -> %s; re-resolving via hardcoded defaults", f"{cached:,}"),
  1688. )
  1689. for hit, log, msg, shown in drop_rules:
  1690. if hit:
  1691. log(msg, model, base_url, shown)
  1692. _invalidate_cached_context_length(model, base_url)
  1693. return None
  1694. # Nous Portal: /v1/models is authoritative. Bypass (don't drop) the cache so step
  1695. # 5b reconciles OR-seeded entries without touching disk when the portal is down.
  1696. if _infer_provider_from_url(base_url) == "nous":
  1697. logger.debug("Bypassing persistent cache for %s@%s (Nous portal authoritative)", model, base_url)
  1698. return None
  1699. # For local endpoints, run the probe that respects configured Modelfile context values first.
  1700. # _query_local_context_length prefers num_ctx from Modelfile, while _query_ollama_api_show returns the
  1701. # GGUF training max first which can be larger and would create a false-safe window for compression
  1702. # (#63122). Non-local endpoints preserve the existing GGUF-first behavior.
  1703. if is_local_endpoint(base_url):
  1704. return _reconcile_local_cached_context_length(model, base_url, cached, api_key=api_key)
  1705. return cached
  1706. def _bedrock_probe_failed_recently(key: tuple) -> bool:
  1707. """Prune expired failures and check this home/endpoint/model/region's cooldown."""
  1708. now = time.monotonic()
  1709. for memo_key, failed_at in list(_BEDROCK_PROBE_FAILURE_CACHE.items()):
  1710. if now - failed_at >= _BEDROCK_PROBE_FAILURE_TTL_SECONDS:
  1711. _BEDROCK_PROBE_FAILURE_CACHE.pop(memo_key, None)
  1712. return key in _BEDROCK_PROBE_FAILURE_CACHE
  1713. def _is_bedrock_context(base_url: str, provider: str = "") -> bool:
  1714. return provider == "bedrock" or bool(
  1715. base_url and base_url_hostname(base_url).startswith("bedrock-runtime.")
  1716. and base_url_host_matches(base_url, "amazonaws.com")
  1717. )
  1718. def _resolve_bedrock_context_length(model: str, base_url: str) -> Optional[int]:
  1719. """Step 1b: Bedrock static table + one cached live probe (Bedrock exposes no context window via
  1720. metadata APIs); None when boto3 is absent. Only provider-confirmed windows from a probe
  1721. or runtime error are reused. The table answers a call, never the cache. Keys use base_url,
  1722. or synthetic bedrock:// when absent, consistently with provider-error writers."""
  1723. try:
  1724. from agent.bedrock_adapter import get_bedrock_context_length, probe_bedrock_context_length, resolve_bedrock_region
  1725. except ImportError:
  1726. return None # boto3 not installed — fall through to generic resolution
  1727. cache_key_url = base_url or "bedrock://"
  1728. cached = get_cached_context_length(model, cache_key_url, bedrock_confirmed=True)
  1729. if cached is not None and cached > 0:
  1730. return cached
  1731. # Legacy scalars have no trustworthy source. Ignore them until a successful
  1732. # probe replaces them; deleting here could erase a concurrent confirmed write
  1733. # or reset the failure cooldown repeatedly when the cache is read-only.
  1734. # Region from the base_url host first, then the standard AWS chain. An empty region disables probing (table only).
  1735. _m = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url) if base_url else None
  1736. region = _m.group(1) if _m else ""
  1737. if not region:
  1738. with contextlib.suppress(Exception):
  1739. region = resolve_bedrock_region()
  1740. from hermes_constants import hermes_home_key
  1741. memo_key = (hermes_home_key(), cache_key_url.rstrip('/'), model, region)
  1742. if region and not _bedrock_probe_failed_recently(memo_key):
  1743. probed = probe_bedrock_context_length(model, region)
  1744. if probed:
  1745. # The probe is the only authoritative source, so it is the only thing worth persisting:
  1746. # a table fallback written here would be served forever (this branch runs before it),
  1747. # and the probe would never be consulted for the model again.
  1748. save_provider_context_length(model, cache_key_url, probed, provider="bedrock")
  1749. _BEDROCK_PROBE_FAILURE_CACHE.pop(memo_key, None) # success ends the failure window
  1750. return probed
  1751. _BEDROCK_PROBE_FAILURE_CACHE[memo_key] = time.monotonic()
  1752. return get_bedrock_context_length(model, probe=False) # static table / default: answers this call only
  1753. def _resolve_custom_endpoint_context_length(model: str, base_url: str, api_key: str, provider: str) -> int:
  1754. """Steps 2-3 for a truly custom endpoint: /models, local probes, Ollama /api/show, catalog, default."""
  1755. context_length = _resolve_endpoint_context_length(model, base_url, api_key=api_key)
  1756. if context_length is not None:
  1757. return context_length
  1758. # Local endpoints: the num_ctx-aware probe first — _query_ollama_api_show is GGUF-first, which
  1759. # can be larger and create a false-safe compression window.
  1760. local_ctx = _probe_local_context_length(model, base_url, api_key, provider) if is_local_endpoint(base_url) else None
  1761. if local_ctx:
  1762. return local_ctx
  1763. # 2b. Ollama native /api/show (GGUF-first for non-local). Non-Ollama servers 404/405 quickly.
  1764. ctx = _query_ollama_api_show(model, base_url, api_key=api_key)
  1765. if ctx is not None:
  1766. _save_unless_skipped(model, base_url, ctx, provider)
  1767. return ctx
  1768. # 3. Probe-down fallback after endpoint-specific detection failed
  1769. logger.info(
  1770. "Could not detect context length for model %r at %s — defaulting to %s tokens (probe-down). "
  1771. "Set model.context_length in config.yaml to override.",
  1772. model, base_url, f"{DEFAULT_FALLBACK_CONTEXT:,}",
  1773. )
  1774. # 3b. Hardcoded catalog as a last resort: a proxied Anthropic gateway fails the probes above
  1775. # but its model name still matches DEFAULT_CONTEXT_LENGTHS.
  1776. hit = _longest_key_match(DEFAULT_CONTEXT_LENGTHS, model.lower())
  1777. if hit:
  1778. logger.info("Using hardcoded context length %s for model %r (custom endpoint, catalog match on %r)", f"{hit[1]:,}", model, hit[0])
  1779. return hit[1]
  1780. # Same silent-256K bug class as the step-9 fallback — warn here too.
  1781. _warn_context_length_fallback(model, base_url)
  1782. return DEFAULT_FALLBACK_CONTEXT
  1783. def _resolve_custom_codex_route_context_length(model: str, base_url: str, api_key: str, provider: str) -> int:
  1784. """Step 2 for a custom ``api_mode: codex_responses`` route — a Codex proxy on a generic URL.
  1785. The Codex OAuth table (with the opted-in ``-900k`` bump) answers first: the proxy's own /models
  1786. and the direct-API catalog both advertise the 1.05M direct window that Codex does not honour.
  1787. The proxy's context_window is never trusted; only an opted-in ``-900k`` variant asks the route's own
  1788. catalog (with the route's own key), and only for its max_context_window cap. Slugs the table does
  1789. not know take the ordinary endpoint probes."""
  1790. ctx, _source = _resolve_codex_oauth_context_length_with_source(model)
  1791. if ctx and api_key and is_codex_context_variant(model):
  1792. # The -900k bump is an account-level assumption; a gateway whose own catalog publishes a
  1793. # Codex-shaped max_context_window caps it, as the native provider's live catalog does.
  1794. _fetch_codex_oauth_context_lengths_with_source(api_key, base_url)
  1795. cap = _cached_codex_catalog_max(api_key, base_url, _bare_codex_slug(strip_codex_context_variant_suffix(model)))
  1796. if cap and cap < ctx:
  1797. logger.info("Route catalog at %s caps %r at max_context_window %s", base_url, model, f"{cap:,}")
  1798. ctx = cap
  1799. if ctx:
  1800. logger.info("Using Codex OAuth context length %s for model %r (codex_responses route at %s)", f"{ctx:,}", model, base_url)
  1801. return ctx
  1802. return _resolve_custom_endpoint_context_length(model, base_url, api_key, provider)
  1803. def _resolve_moa_context_length(model: str, custom_providers: list | None) -> Optional[int]:
  1804. """Step 0a: MoA virtual provider — ``model`` is a preset name, so every probe would miss. Resolve
  1805. the aggregator's real provider+model (references are advisory). None on any failure."""
  1806. try:
  1807. from hermes_cli.config import get_compatible_custom_providers, load_config
  1808. from hermes_cli.moa_config import resolve_moa_preset
  1809. from hermes_cli.runtime_provider import resolve_runtime_provider
  1810. config = load_config()
  1811. if custom_providers is None:
  1812. custom_providers = get_compatible_custom_providers(config)
  1813. agg = resolve_moa_preset(config.get("moa") or {}, model).get("aggregator") or {}
  1814. agg_provider = str(agg.get("provider") or "").strip()
  1815. agg_model = str(agg.get("model") or "").strip()
  1816. if agg_model and agg_provider and agg_provider.lower() != "moa":
  1817. rt = resolve_runtime_provider(requested=agg_provider, target_model=agg_model)
  1818. return get_model_context_length(
  1819. agg_model, base_url=rt.get("base_url", "") or "", api_key=rt.get("api_key", "") or "",
  1820. provider=rt.get("provider") or agg_provider, custom_providers=custom_providers,
  1821. )
  1822. except Exception:
  1823. logger.debug("MoA aggregator context-length resolution failed", exc_info=True)
  1824. return None
  1825. def _config_override_context_length(model: str, base_url: str, provider: str, custom_providers: list | None) -> Optional[int]:
  1826. """Steps 0b-0c: config-only overrides (never touch the network). 0b: EXPLICIT model_overrides
  1827. only — fill-gap _default entries apply inside lookup_models_dev_context once the catalog has
  1828. missed, so a _default can never preempt custom_providers or live probes. 0c: custom_providers."""
  1829. # This is the supported self-unblock path for models with wrong context in models.dev (#84482) and for
  1830. # custom/local models (#8731).
  1831. if provider and model:
  1832. with contextlib.suppress(Exception): # fall through to other resolution paths
  1833. from agent.models_dev import _override_context_window
  1834. mo_ctx = _override_context_window(provider, model)
  1835. if mo_ctx is not None and mo_ctx > 0:
  1836. return mo_ctx
  1837. # 0c. custom_providers per-model override — check before any probe. This closes the gap where /model
  1838. # switch and display paths used to fall back to 128K despite the user having a per-model context_length
  1839. # set. See #15779.
  1840. # Not gated on custom_providers: callers that never load the route list pass None and the
  1841. # helper self-resolves it from config (#69807).
  1842. if base_url and model:
  1843. with contextlib.suppress(Exception): # fall through to probing
  1844. from hermes_cli.config import get_custom_provider_context_length
  1845. cp_ctx = get_custom_provider_context_length(model=model, base_url=base_url, custom_providers=custom_providers)
  1846. if cp_ctx:
  1847. return cp_ctx
  1848. return None
  1849. def _resolve_provider_aware_context_length(model: str, base_url: str, api_key: str, provider: str, effective_provider: str) -> Optional[int]:
  1850. """Step 5: provider-specific sources, tried in order; None when all miss."""
  1851. # 5a. The profile's live, credentialed catalog: account-specific models absent from models.dev.
  1852. from providers import get_provider_profile
  1853. profile = get_provider_profile(effective_provider) if effective_provider else None
  1854. with contextlib.suppress(Exception): # fall through to models.dev
  1855. if profile is not None and (ctx := profile.live_context_length(model, api_key=api_key or "", base_url=base_url or "")):
  1856. return ctx
  1857. # 5b/5c. Nous portal and Codex OAuth (lower limits than the direct API for the same slug; its
  1858. # own /models is authoritative). Persist ONLY the authoritative source ("portal" / "live"): an
  1859. # OR-fallback or static-table value cached on a blip would be frozen in by step 1 forever.
  1860. sourced = {
  1861. "nous": lambda: _resolve_nous_context_length(model, base_url=base_url or "", api_key=api_key or "") + ("portal",),
  1862. "openai-codex": lambda: _resolve_codex_oauth_context_length_with_source(model, access_token=api_key or "", base_url=base_url or "") + ("live",),
  1863. }.get(effective_provider)
  1864. if sourced is not None:
  1865. ctx, source, persist_on = sourced()
  1866. if ctx:
  1867. if base_url and source == persist_on:
  1868. save_context_length(model, base_url, ctx)
  1869. return ctx
  1870. if effective_provider in {"gmi", "commandcode", "commandcode-anthropic"} and base_url:
  1871. # GMI and CommandCode expose authoritative context_length via /models (e.g. muse-spark 1M) but are
  1872. # not in models.dev, and as known providers they skip step 2's probe — else they fell to 256K.
  1873. ctx = _resolve_endpoint_context_length(model, base_url, api_key=api_key)
  1874. if ctx is not None:
  1875. return ctx
  1876. # 5e. Ollama native /api/show for any base_url that is not a known non-Ollama provider (there
  1877. # the POST always 404s and cost ~300ms on the first turn).
  1878. if base_url:
  1879. inferred = _infer_provider_from_url(base_url)
  1880. ctx = _query_ollama_api_show(model, base_url, api_key=api_key) if inferred is None or "ollama" in inferred else None
  1881. if ctx is not None:
  1882. _save_unless_skipped(model, base_url, ctx, provider)
  1883. return ctx
  1884. # 5f. OpenRouter live /models — authoritative for OR-routed models, so it must win over models.dev
  1885. # and the family catch-all (a brand-new slug would otherwise fall to the generic "claude": 200K).
  1886. if effective_provider == "openrouter":
  1887. or_ctx = (fetch_model_metadata().get(model) or {}).get("context_length")
  1888. # Guard against the known OpenRouter Kimi-family 32k underreport.
  1889. if isinstance(or_ctx, int) and or_ctx > 0 and not (or_ctx == 32768 and _model_name_suggests_kimi(model)):
  1890. return or_ctx
  1891. if effective_provider:
  1892. from agent.models_dev import lookup_models_dev_context
  1893. ctx = lookup_models_dev_context(effective_provider, model)
  1894. if ctx:
  1895. # MiniMax M3: models.dev reports 512K but actual context is 1M — prefer the hardcoded catalog.
  1896. catalog = DEFAULT_CONTEXT_LENGTHS.get("minimax-m3") if _model_name_suggests_minimax_m3(model) else None
  1897. if catalog and ctx < catalog:
  1898. logger.info("Rejecting models.dev context=%s for %r (MiniMax-M3 underreport); using hardcoded default %s", ctx, model, f"{catalog:,}")
  1899. ctx = catalog
  1900. return ctx
  1901. return None
  1902. @records_context_defaults
  1903. def get_model_context_length(
  1904. model: str, base_url: str = "", api_key: str = "", config_context_length: int | None = None,
  1905. provider: str = "", custom_providers: list | None = None,
  1906. ) -> int:
  1907. """Context length for a model. Resolution order: 0 config override / MoA aggregator /
  1908. model_overrides / custom_providers / endpoint-scoped; 1 persistent cache (Nous, LM
  1909. Studio, Codex OAuth bypass it) and Bedrock; 2-3 custom endpoints (/models, local
  1910. probe, Ollama); 4 Anthropic /v1/models (API keys only); 5 provider-aware (profile live
  1911. catalog, Nous, Codex OAuth, GMI, Ollama, OpenRouter live, models.dev); 6 OpenRouter for
  1912. unknown providers; 7 local server; 8 hardcoded defaults; 9 256K fallback."""
  1913. # 0. Explicit config override — user knows best
  1914. if isinstance(config_context_length, int) and config_context_length > 0:
  1915. return config_context_length
  1916. if (provider or "").strip().lower() == "moa":
  1917. ctx = _resolve_moa_context_length(model, custom_providers)
  1918. if ctx is not None:
  1919. return ctx
  1920. ctx = _config_override_context_length(model, base_url, provider, custom_providers)
  1921. if ctx is not None:
  1922. return ctx
  1923. # Malformed URLs (unmatched IPv6 bracket) make urllib.parse raise; treat them as unknown so
  1924. # the inference layer reports the configuration error itself.
  1925. if base_url:
  1926. try:
  1927. _ = urlparse(_normalize_base_url(base_url)).port
  1928. except ValueError:
  1929. base_url = ""
  1930. # A blank model id would fuzzy-match an arbitrary catalog entry (`"" in key` is vacuously
  1931. # true) and persist it under a junk "@<base_url>" cache key.
  1932. if not str(model or "").strip():
  1933. logger.info("No model id provided for context length resolution — defaulting to %s tokens.", f"{DEFAULT_FALLBACK_CONTEXT:,}")
  1934. return DEFAULT_FALLBACK_CONTEXT
  1935. model = _strip_provider_prefix(model) # "local:x" -> "x"; Ollama "model:tag" colons preserved
  1936. # OpenRouter routing variants (":nitro", ":floor", ...) are request-time
  1937. # modifiers, not catalog entries — resolve the window from the BASE id.
  1938. # Deliberately placed AFTER the explicit config overrides above (0b/0c) so
  1939. # a user who pinned the fully-suffixed id keeps winning, and BEFORE every
  1940. # cache/catalog lookup below so the base's real window is found instead of
  1941. # a generic family default. Mirrors the validation path's base/suffix split
  1942. # in hermes_cli.models.validate_requested_model.
  1943. model = _strip_openrouter_routing_variant(model, base_url=base_url, provider=provider)
  1944. # Endpoint-scoped metadata goes AHEAD of the persistent cache so a value learned on a
  1945. # multiplexed provider's other endpoint cannot override it.
  1946. endpoint_context = _endpoint_scoped_context_length(model, base_url)
  1947. if endpoint_context is not None:
  1948. return endpoint_context
  1949. # A profile that qualifies its own bound (external processes have no /models probe) wins
  1950. # over the generic caches below; explicit user/endpoint overrides above still take precedence.
  1951. from providers import get_provider_profile
  1952. profile = get_provider_profile(provider)
  1953. context = profile.get_model_context_length(model) if profile else None
  1954. if type(context) is int and context > 0:
  1955. return context
  1956. is_bedrock_context = _is_bedrock_context(base_url, provider)
  1957. # A Codex Responses route is keyed on its transport, not its host: behind a proxy
  1958. # (HERMES_CODEX_BASE_URL, model.base_url, custom api_mode: codex_responses) the URL looks
  1959. # generic while the window is still the Codex OAuth one (#116191).
  1960. codex_route = _is_codex_route(provider, base_url, custom_providers)
  1961. # 1. Persistent cache (LM Studio / Codex routes excluded — see _skip_persistent_context_cache).
  1962. cached = get_cached_context_length(model, base_url) if base_url and not is_bedrock_context and not codex_route and not _skip_persistent_context_cache(base_url, provider) else None
  1963. validated = _validate_cached_context_length(model, base_url, cached, api_key=api_key) if cached is not None else None
  1964. if validated is not None:
  1965. return validated
  1966. # 1b. AWS Bedrock. Must run BEFORE the custom-endpoint step: bedrock-runtime.* is not in
  1967. # _URL_TO_PROVIDER and would fail the /models probe into the default.
  1968. ctx = _resolve_bedrock_context_length(model, base_url) if is_bedrock_context else None
  1969. if ctx is not None:
  1970. return ctx
  1971. if provider == "novita" or (base_url and base_url_host_matches(base_url, "api.novita.ai")):
  1972. ctx = _resolve_endpoint_context_length(model, base_url or "https://api.novita.ai/openai/v1", api_key=api_key)
  1973. if ctx is not None:
  1974. if base_url:
  1975. save_context_length(model, base_url, ctx)
  1976. return ctx
  1977. # 2. Live /models for truly custom endpoints. Known providers skip this: their /models may
  1978. # report a provider-imposed limit (e.g. 128k) rather than the window. The native
  1979. # openai-codex provider skips it too even on a proxy URL — step 5 runs its live catalog probe.
  1980. if _is_custom_endpoint(base_url) and not _is_known_provider_base_url(base_url) and (provider or "").strip().lower() != "openai-codex":
  1981. resolve = _resolve_custom_codex_route_context_length if codex_route else _resolve_custom_endpoint_context_length
  1982. return resolve(model, base_url, api_key, provider)
  1983. # 4. Anthropic /v1/models API (only for regular API keys, not OAuth)
  1984. if provider == "anthropic" or (base_url and base_url_hostname(base_url) == "api.anthropic.com"):
  1985. ctx = _query_anthropic_context_length(model, base_url or "https://api.anthropic.com", api_key)
  1986. if ctx:
  1987. return ctx
  1988. # 5. Provider-aware lookups — before the generic OR cache, since the same model has
  1989. # different limits per provider. Generic providers are inferred from the URL.
  1990. effective_provider = provider
  1991. if base_url and (not effective_provider or effective_provider in {"openrouter", "custom"}):
  1992. effective_provider = _infer_provider_from_url(base_url) or effective_provider
  1993. ctx = _resolve_provider_aware_context_length(model, base_url, api_key, provider, effective_provider)
  1994. if ctx is not None:
  1995. return ctx
  1996. # 6. OpenRouter metadata, provider-unaware fallback — only when the provider is unknown (OR
  1997. # data is community-maintained); 32K underreport guard.
  1998. if not effective_provider:
  1999. metadata = fetch_model_metadata()
  2000. if model in metadata:
  2001. or_ctx = metadata[model].get("context_length", DEFAULT_FALLBACK_CONTEXT)
  2002. if or_ctx == 32768 and _model_name_suggests_stale_32k_underreport(model):
  2003. logger.info("Rejecting OpenRouter metadata context=%s for %r (known 32K underreport); falling through to hardcoded defaults", or_ctx, model)
  2004. else:
  2005. return or_ctx
  2006. # 7. Local server before hardcoded defaults — ``Hermes-3-Llama-3.1-70B`` matches ``llama``
  2007. # (131072) even when vLLM runs at a lower ``--max-model-len``.
  2008. local_ctx = _probe_local_context_length(model, base_url, api_key, provider) if base_url and is_local_endpoint(base_url) else None
  2009. if local_ctx:
  2010. return local_ctx
  2011. # 8. Hardcoded defaults: `key in model` only — the reverse would let "claude-sonnet-4" match "claude-sonnet-4-6" and return 1M.
  2012. hit = _longest_key_match(DEFAULT_CONTEXT_LENGTHS, model.lower())
  2013. if hit:
  2014. return hit[1]
  2015. # 9. Default fallback — warn (deduped) so small-context models don't silently get 256K.
  2016. _warn_context_length_fallback(model, base_url)
  2017. return DEFAULT_FALLBACK_CONTEXT
  2018. async def get_model_context_length_async(model: str, base_url: str = "", api_key: str = "", config_context_length: int | None = None, provider: str = "", custom_providers: list | None = None) -> int:
  2019. """get_model_context_length on a worker thread (its blocking HTTP would stall the event loop)."""
  2020. import asyncio
  2021. return await asyncio.to_thread(
  2022. get_model_context_length, model, base_url=base_url, api_key=api_key,
  2023. config_context_length=config_context_length, provider=provider, custom_providers=custom_providers)
  2024. # CJK/Hangul/Kana codepoints (~1 token each), counted in one C-level regex pass: Hangul
  2025. # Jamo (+Ext-A), CJK radicals/ideographs (+compat), Hangul syllables, fullwidth/halfwidth.
  2026. # Rough chars-per-token ratio for ASCII text; the single source for every "N tokens ≈ N*4 chars"
  2027. # budget conversion (context files, tool-output budgets, whisper prompt cap, compressor metadata).
  2028. CHARS_PER_TOKEN = 4
  2029. _CJK_DENSE_RE = re.compile("[\u1100-\u11ff\u2e80-\u9fff\ua960-\ua97f\uac00-\ud7af\uf900-\ufaff\uff00-\uffef]")
  2030. def _is_cjk_token_dense_char(ch: str) -> bool:
  2031. return _CJK_DENSE_RE.fullmatch(ch) is not None
  2032. def estimate_tokens_rough(text: str) -> int:
  2033. """Rough token estimate: CJK/Hangul/Kana codepoints ~1 token each; everything else ceil(UTF-8 bytes/4).
  2034. Ceiling keeps short texts from estimating 0. Runs on every preflight walk, so all-ASCII stays O(1).
  2035. Byte-counting (not chars) is the corrective for non-CJK, non-ASCII text: Cyrillic/Greek/Arabic are 2
  2036. bytes/char so count ~chars/2, matching real BPE cost (~2-3 chars/token) where chars/4 under-counted
  2037. ~2x and let sessions ride the provider ceiling below the compaction threshold. Calibrated vs
  2038. cl100k/o200k/Qwen2.5 (estimate/real): Russian 0.67->1.24, Arabic 0.53->0.96, Hindi 0.34->0.90,
  2039. Greek 0.37->0.68; accented Latin barely moves (French 1.02->1.03). errors="replace": lone surrogates
  2040. (routine in tool output; see message_sanitization) must not turn an estimate into a raise."""
  2041. if not text:
  2042. return 0
  2043. text = str(text)
  2044. if text.isascii(): # flag check on CPython; ASCII cannot contain token-dense CJK
  2045. return (len(text) + 3) // CHARS_PER_TOKEN
  2046. stripped = _CJK_DENSE_RE.sub("", text)
  2047. dense = len(text) - len(stripped)
  2048. return dense + ((len(stripped.encode("utf-8", "replace")) + 3) // CHARS_PER_TOKEN)
  2049. def estimate_messages_tokens_rough(messages: list[dict[str, Any]], *, charge_stale_thinking: bool = True) -> int:
  2050. """Rough token estimate for a message list (pre-flight only). Images cost the per-image price
  2051. learned from provider usage (``agent.image_token_cost``; flat default before calibration)
  2052. rather than their base64 length. ``charge_stale_thinking=False`` mirrors the tail-budget
  2053. walk (``context_compressor._estimate_msg_budget_tokens``): on non-echo routes stale reasoning
  2054. rides the wire only for the NEWEST assistant turn, so excluding it keeps the compaction TRIGGER
  2055. in the same size class as the walk — otherwise reasoning-heavy sessions fire preflight forever."""
  2056. from agent.image_token_cost import current_image_token_cost
  2057. image_cost = current_image_token_cost()
  2058. if not charge_stale_thinking:
  2059. messages = _strip_stale_thinking_for_estimate(messages)
  2060. return sum(_estimate_message_tokens_cached(msg, image_cost) for msg in messages)
  2061. def estimate_native_anthropic_messages_tokens_rough(messages: list[dict[str, Any]]) -> int:
  2062. """Estimate native Anthropic messages with replayed readable thinking charged exactly once."""
  2063. from agent.message_sanitization import native_anthropic_accounting_projection
  2064. projected, replayed_thinking = native_anthropic_accounting_projection(messages)
  2065. return estimate_messages_tokens_rough(projected) + sum(
  2066. estimate_tokens_rough(text) for text in replayed_thinking
  2067. )
  2068. def estimate_native_anthropic_request_tokens_rough(
  2069. messages: list[dict[str, Any]], *, system_prompt: str = "",
  2070. tools: Optional[list[dict[str, Any]]] = None,
  2071. ) -> int:
  2072. """Request estimate for native Anthropic; opaque replay bytes never enter text accounting."""
  2073. from agent.message_sanitization import native_anthropic_accounting_projection
  2074. projected, replayed_thinking = native_anthropic_accounting_projection(messages)
  2075. return estimate_request_tokens_rough(
  2076. projected, system_prompt=system_prompt, tools=tools
  2077. ) + sum(estimate_tokens_rough(text) for text in replayed_thinking)
  2078. # Thinking-text keys replayed for at most the newest assistant turn on non-echo routes — must stay
  2079. # in lockstep with ``context_compressor._NEWEST_TURN_ONLY_BUDGET_KEYS``.
  2080. _STALE_THINKING_ESTIMATE_KEYS = ("reasoning", "reasoning_content")
  2081. def _strip_stale_thinking_for_estimate(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
  2082. """Copy of ``messages`` with stale thinking keys removed (newest kept). Shallow stripped copies
  2083. share the original value objects, so the per-message memo still hits for the stripped shape."""
  2084. def _is_assistant(m: Any) -> bool:
  2085. return isinstance(m, dict) and m.get("role") == "assistant"
  2086. newest = next((i for i in range(len(messages) - 1, -1, -1) if _is_assistant(messages[i])), -1)
  2087. return [
  2088. {k: v for k, v in m.items() if k not in _STALE_THINKING_ESTIMATE_KEYS}
  2089. if i != newest and _is_assistant(m) and any(m.get(k) for k in _STALE_THINKING_ESTIMATE_KEYS) else m
  2090. for i, m in enumerate(messages)
  2091. ]
  2092. # Per-message token-estimate memo keyed by an exact value fingerprint: strings by ``id()`` AND
  2093. # pinned (strong ref in the entry, so the id can't be reused and immutability makes id-equality
  2094. # value-equality); numbers/bools/None by value; dicts/lists structurally in key order (``str(shadow)``
  2095. # depends on it); any other type aborts the memo. api_messages shallow-copies dicts but shares the strings.
  2096. # ``estimate_messages_tokens_rough`` is called on the full history every loop iteration (conversation_loop
  2097. # preflight), repeatedly during compaction telemetry, and inside an O(n^2) shrink loop in moa_loop. The
  2098. # per-message helpers are pure functions of the message's value, so a memo keyed on a fingerprint that
  2099. # uniquely determines the value is exactly equivalent. Fingerprint design (soundness argument): While the
  2100. # entry lives, that id cannot be reused by another object, so id-equality implies object-equality — strings
  2101. # are immutable, so value-equality too (no #50372-style aliasing). Equal fingerprints therefore imply
  2102. # deep-equal messages built from identical immutable leaves ⇒ identical ``str(shadow)`` bytes ⇒ identical
  2103. # estimate. Because the api_messages build shallow-copies history dicts each iteration, the copies share the
  2104. # same content strings — so unchanged history messages hit the memo even though the outer dicts are fresh
  2105. # objects every turn. Storage, LRU + retained-bytes bounds and release live in ``model_metadata_token_memo``.
  2106. def _msg_fingerprint(value: Any, pins: list) -> Any:
  2107. if value is None or value is True or value is False:
  2108. return value
  2109. t = type(value)
  2110. if t is str:
  2111. pins.append(value)
  2112. return ("s", id(value))
  2113. if t is int or t is float:
  2114. return ("n", t.__name__, value)
  2115. if t is dict:
  2116. return ("d", tuple((_msg_fingerprint(k, pins), _msg_fingerprint(v, pins)) for k, v in value.items()))
  2117. if t is list or t is tuple:
  2118. return ("l" if t is list else "t", tuple(_msg_fingerprint(v, pins) for v in value))
  2119. raise ValueError("unfingerprintable message value")
  2120. def _estimate_message_tokens_cached(msg: Any, image_cost: int) -> int:
  2121. """Text tokens + images x ``image_cost``; the memo holds text and image COUNT so a recalibrated
  2122. per-image price re-prices cached rows without invalidating them."""
  2123. def _compute() -> tuple[int, int]:
  2124. return _estimate_message_tokens_without_images(msg), _count_image_tokens(msg, 1)
  2125. try:
  2126. pins = MemoPins()
  2127. # Persistence-only fields (identity, timestamps, display metadata) never reach the estimate: keep them
  2128. # out of the key so stamping them neither costs a walk nor misses the memo.
  2129. key = _msg_fingerprint(
  2130. without_persistence_fields(msg) if type(msg) is dict else msg, pins)
  2131. cached = lookup_message_tokens(key)
  2132. except Exception:
  2133. text, images = _compute()
  2134. return text + images * image_cost
  2135. if cached is not None:
  2136. return cached[1] + cached[2] * image_cost
  2137. text, images = _compute()
  2138. remember_message_tokens(key, (pins, text, images))
  2139. return text + images * image_cost
  2140. def _count_parts(parts: Any, types: set) -> int:
  2141. return sum(1 for part in parts if isinstance(part, dict) and part.get("type") in types) if isinstance(parts, list) else 0
  2142. _IMAGE_PART_TYPES = frozenset({"image", "image_url", "input_image"})
  2143. def _count_image_tokens(msg: dict[str, Any], cost_per_image: int) -> int:
  2144. """Count image-like content parts in a message; return their token cost."""
  2145. if not isinstance(msg, dict):
  2146. return 0
  2147. content = msg.get("content")
  2148. count = _count_parts(content, _IMAGE_PART_TYPES)
  2149. count += _count_parts(msg.get("_anthropic_content_blocks"), {"image"})
  2150. # Multimodal tool results that haven't been converted yet.
  2151. if isinstance(content, dict) and content.get("_multimodal"):
  2152. count += _count_parts(content.get("content"), {"image", "image_url"})
  2153. # Responses ``function_call_output`` items carry converted tool-result
  2154. # parts under ``output`` (the converter moves chat ``content`` there).
  2155. count += _count_parts(msg.get("output"), _IMAGE_PART_TYPES)
  2156. return count * cost_per_image
  2157. def strip_opaque_replay_items(items: Any) -> Any:
  2158. """``codex_reasoning_items`` with ``encrypted_content`` blanked for local token estimation.
  2159. The ciphertext is priced by the provider's own count, never by its bytes (a compaction
  2160. checkpoint alone can be 5M chars, #100611); only real usage prices it."""
  2161. if not isinstance(items, list):
  2162. return items
  2163. return [
  2164. {k: ("" if k == "encrypted_content" else v) for k, v in item.items()} if isinstance(item, dict) else item
  2165. for item in items
  2166. ]
  2167. def _wire_message_shadow(msg: dict[str, Any]) -> dict[str, Any]:
  2168. """Shadow of a message holding only what the provider actually receives.
  2169. * ``api_content`` SUBSTITUTES ``content`` (mirrors ``turn_context.substitute_api_content`` exactly):
  2170. only a non-empty STRING sidecar on a user/assistant row displaces content; substituting any
  2171. other shape would UNDERcount — the dangerous direction.
  2172. * Base64 images become a placeholder; ``_count_image_tokens`` charges them flat.
  2173. * ``reasoning`` never ships as-is (request builds pop it after optionally promoting it into
  2174. ``reasoning_content``); counting both inflated estimates up to +53%.
  2175. * Opaque provider blobs (``encrypted_content`` on codex reasoning / compaction items) are
  2176. ciphertext the provider prices by its OWN token count, never by bytes; a native compaction
  2177. checkpoint alone can be 5M chars (#100611). They contribute 0 here: only real usage ever
  2178. prices them, and the usage anchor carries that price forward."""
  2179. sidecar = msg.get("api_content")
  2180. sidecar_wins = isinstance(sidecar, str) and bool(sidecar) and msg.get("role") in ("user", "assistant")
  2181. drop_reasoning_dup = isinstance(_rc := msg.get("reasoning_content"), str) and bool(_rc.strip())
  2182. unpriced = unpriced_replay_fields()
  2183. shadow: dict[str, Any] = {}
  2184. for k, v in msg.items():
  2185. if k in unpriced or k in PERSISTENCE_ONLY_MESSAGE_FIELDS or (k == "reasoning" and drop_reasoning_dup):
  2186. continue
  2187. if k == "api_content":
  2188. if sidecar_wins:
  2189. shadow["content"] = v
  2190. elif k == "content" and sidecar_wins:
  2191. continue
  2192. elif k == "content" and isinstance(v, list):
  2193. shadow[k] = [
  2194. {"type": part.get("type"), "image": "[stripped]"}
  2195. if isinstance(part, dict) and part.get("type") in _IMAGE_PART_TYPES
  2196. else part
  2197. for part in v
  2198. ]
  2199. elif k == "content" and isinstance(v, dict) and v.get("_multimodal"):
  2200. shadow[k] = v.get("text_summary", "")
  2201. elif k == "output" and isinstance(v, list):
  2202. # Responses ``function_call_output`` output parts: strip the image
  2203. # payload like the ``content`` branch above so encoded bytes are
  2204. # priced by the flat per-image model, never as text.
  2205. shadow[k] = [
  2206. {"type": part.get("type"), "image": "[stripped]"}
  2207. if isinstance(part, dict) and part.get("type") in _IMAGE_PART_TYPES
  2208. else part
  2209. for part in v
  2210. ]
  2211. elif k == "codex_reasoning_items":
  2212. shadow[k] = strip_opaque_replay_items(v)
  2213. elif k == "encrypted_content": # a Responses reasoning/compaction item passed as a row
  2214. shadow[k] = ""
  2215. else:
  2216. shadow[k] = v
  2217. return shadow
  2218. def _estimate_message_tokens_without_images(msg: dict[str, Any]) -> int:
  2219. """Token estimate for a message shadow with image payloads stripped."""
  2220. return estimate_tokens_rough(str(_wire_message_shadow(msg) if isinstance(msg, dict) else msg))
  2221. def estimate_request_tokens_rough(
  2222. messages: list[dict[str, Any]], *, system_prompt: str = "", tools: Optional[list[dict[str, Any]]] = None, charge_stale_thinking: bool = True,
  2223. ) -> int:
  2224. """Rough token estimate for a full request: system prompt + messages + tool schemas (50+ tools
  2225. add 20-30K on their own). ``charge_stale_thinking`` is forwarded — pass False when the route
  2226. provably strips stale thinking (``message_sanitization.stale_thinking_reaches_wire``)."""
  2227. total = estimate_tokens_rough(system_prompt) if system_prompt else 0
  2228. if messages:
  2229. # Positional call: test seams and plugin engines monkeypatch estimate_messages_tokens_rough with (messages)-only signatures.
  2230. total += estimate_messages_tokens_rough(messages) if charge_stale_thinking else estimate_messages_tokens_rough(messages, charge_stale_thinking=False)
  2231. if tools:
  2232. total += _estimate_tools_tokens_rough(tools)
  2233. return total
  2234. # Keyed by ``id(tools)``; bounded, oldest-first eviction. Repeated ``str(tools)`` on
  2235. # large schemas stalls GUI event loops under GIL pressure.
  2236. _TOOLS_TOKENS_CACHE: dict[int, tuple[int, str, str, int]] = {}
  2237. _TOOLS_TOKENS_CACHE_MAX = 256
  2238. def _tool_name_for_cache(tool: Any) -> str:
  2239. if not isinstance(tool, dict):
  2240. return ""
  2241. fn = tool.get("function")
  2242. name = fn.get("name") if isinstance(fn, dict) else None
  2243. name = name if isinstance(name, str) else tool.get("name")
  2244. return name if isinstance(name, str) else ""
  2245. def _estimate_tools_tokens_rough(tools: list[dict[str, Any]]) -> int:
  2246. if not tools:
  2247. return 0
  2248. key = id(tools)
  2249. signature = (len(tools), _tool_name_for_cache(tools[0]), _tool_name_for_cache(tools[-1]))
  2250. cached = _TOOLS_TOKENS_CACHE.get(key)
  2251. if cached is not None and cached[:3] == signature:
  2252. return cached[3]
  2253. # Sum the major schema fields (descriptions + parameters dominate).
  2254. total_chars = 0
  2255. for tool in tools:
  2256. if not isinstance(tool, dict):
  2257. continue
  2258. fn = tool.get("function")
  2259. src = fn if isinstance(fn, dict) else tool
  2260. params = src.get("parameters") or {}
  2261. total_chars += sum(len(v) for v in (src.get("name") or "", src.get("description") or "") if isinstance(v, str))
  2262. try: # JSON is closer to wire size than repr()
  2263. total_chars += len(json.dumps(params, ensure_ascii=False, separators=(",", ":")))
  2264. except Exception:
  2265. total_chars += len(str(params))
  2266. tokens = (total_chars + 3) // 4
  2267. if len(_TOOLS_TOKENS_CACHE) >= _TOOLS_TOKENS_CACHE_MAX:
  2268. _TOOLS_TOKENS_CACHE.pop(next(iter(_TOOLS_TOKENS_CACHE)), None)
  2269. _TOOLS_TOKENS_CACHE[key] = (*signature, tokens)
  2270. return tokens