model_setup_flows_custom.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436
  1. """Custom OpenAI-compatible endpoint wizards: the ad-hoc ``custom`` flow and the
  2. ``custom_providers`` / ``providers.<key>`` named-endpoint flow.
  3. Imports of hermes_cli.main / auth / config / models stay lazy (main.py import cycle;
  4. tests patch them at call time). Prompt strings and config write order are behavior.
  5. """
  6. from __future__ import annotations
  7. import contextlib
  8. import os
  9. import urllib.parse
  10. from hermes_cli.cli_output import line_input
  11. from hermes_cli.providers import custom_provider_slug
  12. from hermes_cli.model_setup_flows_common import (
  13. _HTTP, _ask, _commit_model_config, _load_config_model_section,
  14. _prune_replaced_custom_model_config_credentials, _radiolist, _say)
  15. def _parse_context_length(text: str):
  16. """``128k`` / ``128,000`` -> int; None when blank, non-positive, or unparsable (warns)."""
  17. if not text:
  18. return None
  19. try:
  20. value = int(text.replace(",", "").replace("k", "000").replace("K", "000"))
  21. except ValueError:
  22. print(f"Invalid context length: {text} — will auto-detect.")
  23. return None
  24. return value if value > 0 else None
  25. def _report_context_length_detection(model_name: str, base_url: str, api_key: str) -> None:
  26. """Tell the user what the auto-detect resolver found for *model_name* at *base_url* (#2513).
  27. The runtime resolver (``get_model_context_length``) probes /models, local servers and the
  28. catalogs, then falls back to ``DEFAULT_FALLBACK_CONTEXT``; without this line a blank
  29. context-length prompt gave no hint whether the saved endpoint runs on a detected value or
  30. the silent default that shapes compression and cache windows. Feedback only — the value
  31. is NOT written to config, which would freeze a probe result into a permanent override.
  32. """
  33. try:
  34. from agent.model_metadata import DEFAULT_FALLBACK_CONTEXT, get_model_context_length
  35. from hermes_cli.banner import _format_context_length
  36. detected = get_model_context_length(model_name, base_url=base_url, api_key=api_key or "")
  37. except Exception: # a failing probe must never block the save
  38. return
  39. if detected and detected != DEFAULT_FALLBACK_CONTEXT:
  40. print(f" Context length auto-detected: {_format_context_length(detected)} tokens")
  41. else:
  42. print(f" Context length: not detected — using the default {_format_context_length(DEFAULT_FALLBACK_CONTEXT)} tokens "
  43. f"(set model.context_length in config.yaml to override)")
  44. def _probe_custom_endpoint(effective_key: str, effective_url: str) -> tuple[dict, str]:
  45. """Verify a custom endpoint via ``probe_api_models`` and report; returns
  46. ``(probe, effective_url)`` where the URL may be the working fallback base."""
  47. from hermes_cli.models import probe_api_models
  48. probe = probe_api_models(effective_key, effective_url)
  49. if probe.get("used_fallback") and probe.get("resolved_base_url"):
  50. print(f"Warning: endpoint verification worked at {probe['resolved_base_url']}/models, "
  51. f"not the exact URL you entered. Saving the working base URL instead.")
  52. effective_url = probe["resolved_base_url"]
  53. elif probe.get("models") is not None:
  54. print(f"Verified endpoint via {probe.get('probed_url')} ({len(probe.get('models') or [])} model(s) visible)")
  55. else:
  56. print(f"Warning: could not verify this endpoint via {probe.get('probed_url')}. Hermes will still save it.")
  57. suggested = probe.get("suggested_base_url")
  58. if suggested and suggested.endswith("/v1"):
  59. print(f" If this server expects /v1 in the path, try base URL: {suggested}")
  60. elif suggested:
  61. print(f" If /v1 should not be in the base URL, try: {suggested}")
  62. return probe, effective_url
  63. def _pick_detected_model(detected_models: list) -> str:
  64. """Model-name step of the custom flow: confirm a single detection, number-pick from
  65. several, or type one. Raises KeyboardInterrupt/EOFError like the prompts it wraps."""
  66. manual = "Model name (e.g. gpt-4, llama-3-70b): "
  67. if len(detected_models) == 1:
  68. print(f" Detected model: {detected_models[0]}")
  69. if input(" Use this model? [Y/n]: ").strip().lower() in {"", "y", "yes"}:
  70. return detected_models[0]
  71. return line_input(manual).strip()
  72. if len(detected_models) > 1:
  73. print(" Available models:")
  74. for i, m in enumerate(detected_models, 1):
  75. print(f" {i}. {m}")
  76. pick = input(f" Select model [1-{len(detected_models)}] or type name: ").strip()
  77. if pick.isdigit() and 1 <= int(pick) <= len(detected_models):
  78. return detected_models[int(pick) - 1]
  79. return pick
  80. return line_input(manual).strip()
  81. def _model_flow_custom(config):
  82. """Custom endpoint: collect URL, API key, and model name; also saved to ``custom_providers`` so
  83. it appears in the provider menu on subsequent runs."""
  84. from hermes_cli.main_provider_setup import _auto_provider_name, _prompt_custom_api_mode_selection, _save_custom_provider
  85. from hermes_cli.auth import _save_model_choice, deactivate_provider
  86. from hermes_cli.config import custom_endpoint_key_env, get_env_value, save_env_value
  87. from hermes_cli.secret_prompt import masked_secret_prompt
  88. current_url = get_env_value("OPENAI_BASE_URL") or ""
  89. current_key = get_env_value("OPENAI_API_KEY") or ""
  90. print("Custom OpenAI-compatible endpoint configuration:")
  91. if current_url:
  92. print(f" Current URL: {current_url}")
  93. if current_key:
  94. print(f" Current key: {current_key[:8]}...")
  95. print()
  96. try:
  97. base_url = line_input(f"API base URL [{current_url or 'e.g. https://api.example.com/v1'}]: ").strip()
  98. api_key = masked_secret_prompt(f"API key [{current_key[:8] + '...' if current_key else 'optional'}]: ").strip()
  99. except (KeyboardInterrupt, EOFError):
  100. print("\nCancelled.")
  101. return
  102. if not base_url and not current_url:
  103. print("No URL provided. Cancelled.")
  104. return
  105. effective_url = base_url or current_url
  106. if not effective_url.startswith(_HTTP):
  107. print(f"Invalid URL: {effective_url} (must start with http:// or https://)")
  108. return
  109. effective_key = api_key or current_key
  110. # Most local servers (Ollama, vLLM, llama.cpp) need /v1 for OpenAI-compatible
  111. # chat completions — offer to append it when the URL looks local without it.
  112. _url_lower = effective_url.rstrip("/").lower()
  113. _looks_local = any(h in _url_lower for h in ("localhost", "127.0.0.1", "0.0.0.0", ":11434", ":8080", ":5000"))
  114. if _looks_local and not _url_lower.endswith("/v1"):
  115. _say("", " Hint: Did you mean to add /v1 at the end?",
  116. " Most local model servers (Ollama, vLLM, llama.cpp) require it.", f" e.g. {effective_url.rstrip('/')}/v1")
  117. if _ask(" Add /v1? [Y/n]: ", raw=True, cancel_msg=None, on_cancel="n").lower() in {"", "y", "yes"}:
  118. effective_url = effective_url.rstrip("/") + "/v1"
  119. print(f" Updated URL: {effective_url}")
  120. print()
  121. probe, effective_url = _probe_custom_endpoint(effective_key, effective_url)
  122. # Ask for the API mode explicitly so codex-compatible custom providers don't
  123. # silently fall back to chat_completions.
  124. current_model_cfg = config.get("model")
  125. current_api_mode = str(current_model_cfg.get("api_mode") or "").strip() if isinstance(current_model_cfg, dict) else ""
  126. api_mode = _prompt_custom_api_mode_selection(effective_url, current_api_mode=current_api_mode)
  127. print(f" API mode: {api_mode}" if api_mode else " API mode: auto-detect")
  128. # Select model — use probe results when available, fall back to manual input
  129. try:
  130. model_name = _pick_detected_model(probe.get("models") or [])
  131. context_length_str = line_input("Context length in tokens [leave blank for auto-detect]: ").strip()
  132. # Display name — shown in the provider menu on future runs
  133. default_name = _auto_provider_name(effective_url)
  134. display_name = line_input(f"Display name [{default_name}]: ").strip() or default_name
  135. except (KeyboardInterrupt, EOFError):
  136. print("\nCancelled.")
  137. return
  138. context_length = _parse_context_length(context_length_str)
  139. if context_length is None and model_name:
  140. _report_context_length_detection(model_name, effective_url, effective_key)
  141. # The key goes to .env and config.yaml only references it. Keyed on host:port
  142. # so two servers on one machine keep separate credentials.
  143. # See #69449.
  144. custom_key_env = ""
  145. if effective_key:
  146. _parsed = urllib.parse.urlparse(effective_url)
  147. _identity = _parsed.hostname or ""
  148. if _parsed.port:
  149. _identity = f"{_identity}_{_parsed.port}"
  150. custom_key_env = custom_endpoint_key_env(_identity)
  151. save_env_value(custom_key_env, effective_key)
  152. print(f" API key saved to .env as {custom_key_env}")
  153. def _apply_endpoint(model: dict) -> None:
  154. model["provider"] = "custom"
  155. model["base_url"] = effective_url
  156. # A previous endpoint's key_env pointer would outrank the credential written below.
  157. model.pop("key_env", None)
  158. model.pop("api_key_env", None)
  159. if custom_key_env:
  160. model["api_key"] = f"${{{custom_key_env}}}"
  161. if api_mode:
  162. model["api_mode"] = api_mode
  163. else:
  164. model.pop("api_mode", None)
  165. if model_name:
  166. _save_model_choice(model_name)
  167. cfg, model = _load_config_model_section()
  168. _apply_endpoint(model)
  169. _commit_model_config(cfg)
  170. # Sync the caller's config dict so the setup wizard's final save_config(config)
  171. # doesn't overwrite model.provider/base_url with its stale values.
  172. config["model"] = dict(model)
  173. print(f"Default model set to: {model_name} (via {effective_url})")
  174. else:
  175. if base_url or api_key:
  176. deactivate_provider()
  177. # Even without a model name, persist the endpoint on the caller's config dict.
  178. _caller_model = config.get("model")
  179. if not isinstance(_caller_model, dict):
  180. _caller_model = {"default": _caller_model} if _caller_model else {}
  181. _apply_endpoint(_caller_model)
  182. config["model"] = _caller_model
  183. print("Endpoint saved. Use `/model` in chat or `hermes model` to set a model.")
  184. # Auto-save to custom_providers so it appears in the menu next time
  185. _save_custom_provider(effective_url, effective_key, model_name or "", context_length=context_length,
  186. name=display_name, api_mode=api_mode, key_env=custom_key_env)
  187. _prune_replaced_custom_model_config_credentials(effective_url, provider_name=display_name)
  188. def _configured_model_ids(cfg_models) -> list[str]:
  189. """Model ids from a ``custom_providers[].models`` mapping or list (marker keys skipped)."""
  190. if isinstance(cfg_models, dict):
  191. markers = {"__explicit_model_allowlist__", "__discovered_model_catalog__"}
  192. return [str(m) for m in cfg_models if m not in markers and str(m).strip()]
  193. out: list[str] = []
  194. if isinstance(cfg_models, list):
  195. for entry in cfg_models:
  196. if isinstance(entry, dict):
  197. model_id = str(entry.get("id") or entry.get("model") or "").strip()
  198. else:
  199. model_id = str(entry).strip() if isinstance(entry, str) else ""
  200. if model_id:
  201. out.append(model_id)
  202. return out
  203. def _discover_named_custom_models(provider_info: dict, api_key: str, configured_models: list, explicit_catalog: bool):
  204. """Live catalog probe for a named custom endpoint (native ``/api/tags`` for Ollama).
  205. Returns ``(models, native_catalog_empty)``; persists the live catalog as a side effect."""
  206. from hermes_cli.config import normalize_extra_headers
  207. from hermes_cli.models import fetch_api_models, _get_ollama_native_headers
  208. from hermes_cli.models_local import (
  209. fetch_ollama_local_models,
  210. _normalize_openai_base_url,
  211. should_use_ollama_native_catalog,
  212. )
  213. from agent.command_token_source import build_command_token_provider, materialize_probe_api_key
  214. source = build_command_token_provider(provider_info.get("key_cmd", ""), provider_info["name"])
  215. api_key = materialize_probe_api_key(source if source is not None else api_key)
  216. name, base_url = provider_info["name"], provider_info["base_url"]
  217. api_mode = provider_info.get("api_mode", "")
  218. provider_key = (provider_info.get("provider_key") or "").strip()
  219. print("Fetching available models...")
  220. fetch_kwargs = {"timeout": 8.0}
  221. if api_mode:
  222. fetch_kwargs["api_mode"] = api_mode
  223. native_catalog_provider = "ollama" if provider_key.lower() == "ollama" or name.strip().lower() == "ollama" else "custom"
  224. extra_headers = normalize_extra_headers(provider_info.get("extra_headers")) or {}
  225. candidate_headers = _get_ollama_native_headers(base_url, api_key=api_key)
  226. for key in tuple(candidate_headers):
  227. if any(key.lower() == existing.lower() for existing in extra_headers):
  228. del candidate_headers[key]
  229. candidate_headers.update(extra_headers)
  230. caller_has_authorization = any(key.lower() == "authorization" for key in extra_headers)
  231. if api_key and not caller_has_authorization:
  232. for key in tuple(candidate_headers):
  233. if key.lower() == "authorization":
  234. del candidate_headers[key]
  235. candidate_headers["Authorization"] = f"Bearer {api_key}"
  236. use_native = should_use_ollama_native_catalog(native_catalog_provider, base_url, headers=candidate_headers or None)
  237. native_headers_arg = candidate_headers or None if use_native else (extra_headers or None)
  238. native_catalog_empty = False
  239. if use_native:
  240. if explicit_catalog and configured_models:
  241. live_models = configured_models
  242. else:
  243. live_models = fetch_ollama_local_models(base_url, timeout=8.0, headers=native_headers_arg)
  244. native_catalog_empty = live_models == []
  245. if live_models is None:
  246. live_models = fetch_api_models(api_key, _normalize_openai_base_url(base_url), headers=native_headers_arg, **fetch_kwargs)
  247. native_catalog_empty = False
  248. else:
  249. live_models = fetch_api_models(api_key, base_url, headers=native_headers_arg, **fetch_kwargs)
  250. models = configured_models if explicit_catalog else [] if native_catalog_empty else (live_models or configured_models)
  251. # Persist the live catalog to the custom_providers entry so no-probe surfaces
  252. # (dashboard, desktop, ACP) show the full list; mirrors model_switch.py's
  253. # _save_discovered_models_to_config. A failed save is non-fatal.
  254. if live_models:
  255. with contextlib.suppress(Exception):
  256. from hermes_cli.model_switch_providers import _entry_credentials, _save_discovered_models_to_config
  257. _save_discovered_models_to_config(
  258. base_url, live_models, api_mode=api_mode, headers=extra_headers or None,
  259. credential_identity=_entry_credentials(provider_info, "key_env", "api_key_env")[2])
  260. return models, native_catalog_empty
  261. def _pick_named_custom_model(name: str, models: list, saved_model: str):
  262. """Searchable radiolist over *models* (numbered prompt without curses); None = cancelled."""
  263. default_idx = models.index(saved_model) if saved_model and saved_model in models else 0
  264. print(f"Found {len(models)} model(s):\n")
  265. menu_items = [f"{m} (current)" if m == saved_model else m for m in models] + ["Cancel"]
  266. idx = _radiolist(f"Select model from {name}:", menu_items, default_idx, searchable=True)
  267. if idx is not None:
  268. print()
  269. else:
  270. for i, m in enumerate(models, 1):
  271. print(f" {i}. {m}{' (current)' if m == saved_model else ''}")
  272. _say(f" {len(models) + 1}. Cancel", "")
  273. try:
  274. val = input(f"Choice [1-{len(models) + 1}]: ").strip()
  275. if not val:
  276. print("Cancelled.")
  277. return None
  278. idx = int(val) - 1
  279. except (ValueError, KeyboardInterrupt, EOFError):
  280. print("\nCancelled.")
  281. return None
  282. if idx < 0 or idx >= len(models):
  283. print("Cancelled.")
  284. return None
  285. return models[idx]
  286. def _model_flow_named_custom(config, provider_info):
  287. """Named custom provider from ``custom_providers`` / ``providers.<key>``: probes the model
  288. catalog (native ``/api/tags`` for endpoints conservatively identified as Ollama); a previously
  289. saved model is pre-selected and is the fallback when probing fails."""
  290. from hermes_cli.main_provider_setup import _custom_provider_api_key_config_value, _custom_provider_base_url_config_value, _save_custom_provider
  291. from hermes_cli.auth import _save_model_choice
  292. from hermes_cli.config import load_config, save_config
  293. from hermes_cli.model_switch import _entry_models_discovered, _models_config_is_allowlist
  294. name = provider_info["name"]
  295. base_url = provider_info["base_url"]
  296. api_mode = provider_info.get("api_mode", "")
  297. api_key = provider_info.get("api_key", "")
  298. key_env = provider_info.get("key_env", "")
  299. saved_model = provider_info.get("model", "")
  300. provider_key = (provider_info.get("provider_key") or "").strip()
  301. # Resolve key_env through the profile secret scope (fresh .env; never another
  302. # profile's process env under multiplexing), like the runtime does (#67935).
  303. if not api_key and key_env:
  304. from agent.secret_scope import get_secret_str
  305. api_key = get_secret_str(key_env, "")
  306. # Only configured credentials may be persisted, never a short-lived probe token.
  307. config_api_key = _custom_provider_api_key_config_value(provider_info, api_key)
  308. # ``discover_models: false`` (default True) uses the configured ``models:`` list
  309. # verbatim and skips the live probe, so operators can restrict the picker to the
  310. # subset their plan serves. Same semantics as the slash-command picker.
  311. # This lets operators restrict the picker to the subset their plan actually serves instead of the
  312. # endpoint's full catalog (#18726: Baidu Qianfan returns 100+ models for a 2-3 model plan).
  313. discover = provider_info.get("discover_models", True)
  314. if isinstance(discover, str):
  315. discover = discover.lower() not in {"false", "no", "0"}
  316. cfg_models = provider_info.get("models", {})
  317. explicit_catalog = _models_config_is_allowlist(cfg_models, _entry_models_discovered(provider_info))
  318. configured_models = _configured_model_ids(cfg_models)
  319. _say(f" Provider: {name}", f" URL: {base_url}")
  320. if saved_model:
  321. print(f" Current: {saved_model}")
  322. print()
  323. native_catalog_empty = False
  324. if not discover:
  325. # Never probe. The active model is a usable sole choice, not a catalog.
  326. models = configured_models or ([saved_model] if saved_model else [])
  327. print(f"Using configured models (discover_models: false): {len(models)}")
  328. else:
  329. models, native_catalog_empty = _discover_named_custom_models(provider_info, api_key, configured_models, explicit_catalog)
  330. if models:
  331. model_name = _pick_named_custom_model(name, models, saved_model)
  332. if model_name is None:
  333. return
  334. elif saved_model and not native_catalog_empty:
  335. print("Could not fetch models from endpoint.")
  336. model_name = _ask(f"Model name [{saved_model}]: ")
  337. if model_name is None:
  338. return
  339. model_name = model_name or saved_model
  340. else:
  341. print("Could not fetch models from endpoint. Enter model name manually.")
  342. model_name = _ask("Model name: ")
  343. if model_name is None:
  344. return
  345. if not model_name:
  346. print("No model specified. Cancelled.")
  347. return
  348. # Activate and save the model to the custom_providers entry
  349. _save_model_choice(model_name)
  350. cfg, model = _load_config_model_section()
  351. # The endpoint being activated owns the credential: drop the previous endpoint's pointer
  352. # (key_env outranks the provider entry's own key at resolution time).
  353. model.pop("key_env", None)
  354. model.pop("api_key_env", None)
  355. if provider_key:
  356. model["provider"] = custom_provider_slug(name, provider_key)
  357. model.pop("base_url", None)
  358. model.pop("api_key", None)
  359. else:
  360. model["provider"] = "custom"
  361. model["base_url"] = _custom_provider_base_url_config_value(provider_info, base_url)
  362. if config_api_key:
  363. model["api_key"] = config_api_key
  364. # Apply api_mode from custom_providers entry, or clear stale value
  365. if api_mode:
  366. model["api_mode"] = api_mode
  367. else:
  368. model.pop("api_mode", None) # let runtime auto-detect from URL
  369. _commit_model_config(cfg)
  370. # Persist the selected model back to whichever schema owns this endpoint.
  371. if provider_key:
  372. cfg = load_config()
  373. providers_cfg = cfg.get("providers")
  374. provider_entry = providers_cfg.get(provider_key) if isinstance(providers_cfg, dict) else None
  375. if isinstance(provider_entry, dict):
  376. provider_entry["default_model"] = model_name
  377. # Only persist an inline api_key when the user originally had one
  378. # (literal or ``${VAR}``). Entries relying on ``key_env`` must not get
  379. # a synthesized api_key — the runtime resolves key_env directly and
  380. # writing it would downgrade credential hygiene.
  381. had_inline_api_key = bool(
  382. str(provider_info.get("api_key_ref", "") or "").strip() or str(provider_info.get("api_key", "") or "").strip()
  383. )
  384. if had_inline_api_key and config_api_key and not str(provider_entry.get("api_key", "") or "").strip():
  385. provider_entry["api_key"] = config_api_key
  386. if key_env and not str(provider_entry.get("key_env", "") or "").strip():
  387. provider_entry["key_env"] = key_env
  388. cfg["providers"] = providers_cfg
  389. save_config(cfg)
  390. else:
  391. # Save model name to the custom_providers entry for next time
  392. _save_custom_provider(base_url, config_api_key, model_name, api_mode=api_mode)
  393. _say(f"\n✅ Model set to: {model_name}", f" Provider: {name} ({base_url})")