agent_init_ollama.py 3.7 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667
  1. """Init-time local-server (Ollama) num_ctx detection and the compressor clamp to it."""
  2. from __future__ import annotations
  3. from agent.agent_runtime_helpers import _ra
  4. from agent.model_metadata import is_local_endpoint, query_ollama_num_ctx
  5. def _configure_ollama_num_ctx(agent, _model_cfg, _config_context_length):
  6. # Ollama defaults num_ctx to 2048, so detect the max window and send num_ctx per request.
  7. # model.ollama_num_ctx overrides; model.context_length caps the detected value (VRAM).
  8. agent._ollama_num_ctx: int | None = None
  9. _override = _model_cfg.get("ollama_num_ctx") if isinstance(_model_cfg, dict) else None
  10. if _override is not None:
  11. try:
  12. agent._ollama_num_ctx = int(_override)
  13. except (TypeError, ValueError):
  14. _ra().logger.debug("Invalid ollama_num_ctx config value: %r", _override)
  15. if agent._ollama_num_ctx is None and agent.base_url and is_local_endpoint(agent.base_url):
  16. try:
  17. # api_key may be a callable (Entra token provider); detection needs a string.
  18. _key = agent.api_key if isinstance(agent.api_key, str) else ""
  19. _detected = query_ollama_num_ctx(agent.model, agent.base_url, api_key=_key or "")
  20. if _detected and _detected > 0:
  21. agent._ollama_num_ctx = _detected
  22. except Exception as exc:
  23. _ra().logger.debug("Local server num_ctx detection failed: %s", exc)
  24. # Cap auto-detected num_ctx to the explicit context_length (GGUF metadata can advertise
  25. # 256K+ and Ollama would allocate that much VRAM); never override an explicit num_ctx.
  26. if (
  27. agent._ollama_num_ctx
  28. and _config_context_length
  29. and _override is None
  30. and agent._ollama_num_ctx > _config_context_length
  31. ):
  32. _ra().logger.info(
  33. "Ollama num_ctx capped: %d -> %d (model.context_length override)",
  34. agent._ollama_num_ctx, _config_context_length,
  35. )
  36. agent._ollama_num_ctx = _config_context_length
  37. if agent._ollama_num_ctx and not agent.quiet_mode:
  38. # Name the real source: a config override is honoured on any local server, /api/show is Ollama-only.
  39. _ra().logger.info(
  40. "Local server num_ctx: will request %d tokens (%s)",
  41. agent._ollama_num_ctx,
  42. "model.ollama_num_ctx" if _override is not None else "model max from Ollama /api/show",
  43. )
  44. def _clamp_compressor_to_ollama_num_ctx(agent):
  45. # Recalibrate the compressor to the served window: every request runs at num_ctx, so a
  46. # trigger derived from the probed model window could sit above it and never fire.
  47. # A config that sets only model.ollama_num_ctx (without model.context_length) previously left the
  48. # compressor targeting the probed window while the server truncated/rejected at num_ctx — the compaction
  49. # trigger could sit several times ABOVE the real served window and never fire. Clamp the compressor's
  50. # window to the effective num_ctx so threshold math operates on the context the server actually serves.
  51. # (Overlaps #60103's silent-clamp dead zone; this is the init-order half.)
  52. _cc_window = getattr(agent.context_compressor, "context_length", 0) or 0
  53. if agent._ollama_num_ctx and agent._ollama_num_ctx > 0 and _cc_window and agent._ollama_num_ctx < _cc_window:
  54. _ra().logger.info(
  55. "Compressor window clamped to Ollama num_ctx: %d -> %d",
  56. _cc_window, agent._ollama_num_ctx,
  57. )
  58. agent.context_compressor.update_model(
  59. model=agent.model, context_length=agent._ollama_num_ctx, base_url=agent.base_url,
  60. api_key=getattr(agent, "api_key", ""), provider=agent.provider, api_mode=agent.api_mode,
  61. )