context_pin.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124
  1. """``model.context_length`` is an explicit pin: it wins over provider metadata (#66168).
  2. Two helpers make that visible without changing the numeric value used: a label for every
  3. surface that renders the context window, and a one-time startup warning when the pin
  4. disagrees with what the provider is known to advertise for the model. The same label marks a
  5. window the resolver only defaulted to (``fallback``), so it is not read as a detected limit.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. from typing import Optional
  10. logger = logging.getLogger(__name__)
  11. _warned_pins: set = set()
  12. _MARKER_SUFFIX = {"pinned": " (pinned)", "fallback": " (fallback)"}
  13. def is_context_pinned(context_length, config_context_length) -> bool:
  14. """True when the displayed ``context_length`` is the ``model.context_length`` pin."""
  15. return (
  16. isinstance(config_context_length, int)
  17. and not isinstance(config_context_length, bool)
  18. and config_context_length > 0
  19. and context_length == config_context_length
  20. )
  21. def context_pin_suffix(context_length, config_context_length) -> str:
  22. """`` (pinned)`` when the shown value comes from ``model.context_length``, else ``""``."""
  23. return " (pinned)" if is_context_pinned(context_length, config_context_length) else ""
  24. def is_fallback_context_length(context_length, *, model: str, provider: str = "", base_url: str = "") -> bool:
  25. """True when ``context_length`` is a default the resolver fell back to for ``model`` on this route (Bedrock's
  26. static 128K, or the generic 256K), not a known or detected limit. The resolver records that itself
  27. (agent.context_default_record), so this agrees with every source it consults; ``provider`` is accepted for
  28. call-site symmetry. Fail-open: False on any error."""
  29. if type(context_length) is not int or context_length <= 0 or not isinstance(model, str) or not model:
  30. return False
  31. try:
  32. from agent.context_default_record import was_context_defaulted
  33. return was_context_defaulted(context_length, model, base_url)
  34. except Exception:
  35. logger.debug("context fallback check failed for %s", model, exc_info=True)
  36. return False
  37. def context_window_marker(context_length, config_context_length, *, model: str, provider: str = "",
  38. base_url: str = "") -> str:
  39. """``"pinned"`` (the ``model.context_length`` pin), ``"fallback"`` (a default, not a known limit) or ``""``."""
  40. if is_context_pinned(context_length, config_context_length):
  41. return "pinned"
  42. fallback = is_fallback_context_length(context_length, model=model, provider=provider, base_url=base_url)
  43. return "fallback" if fallback else ""
  44. def context_window_suffix(context_length, config_context_length, **route) -> str:
  45. """`` (pinned)`` / `` (fallback)`` / ``""`` for a rendered window; ``route`` is model, provider, base_url."""
  46. return _MARKER_SUFFIX.get(context_window_marker(context_length, config_context_length, **route), "")
  47. def agent_context_marker(agent) -> str:
  48. """:func:`context_window_marker` for an agent's live compressor window."""
  49. compressor = getattr(agent, "context_compressor", None)
  50. pin = getattr(compressor, "_config_context_length", getattr(agent, "_config_context_length", None))
  51. # The compressor's route is what resolved the window; the agent's is the fallback for doubles without one.
  52. route = {name: getattr(compressor, name, None) or getattr(agent, name, "") or "" for name in ("model", "provider", "base_url")}
  53. window = getattr(compressor, "context_length", None) # first: reading it resolves the window
  54. own = getattr(compressor, "_context_default", None) # (model, base_url, defaulted window | None) from that lookup
  55. if isinstance(own, tuple) and len(own) == 3 and own[:2] == (getattr(compressor, "model", None), getattr(compressor, "base_url", None)):
  56. if is_context_pinned(window, pin):
  57. return "pinned"
  58. return "fallback" if type(own[2]) is int and own[2] == window else ""
  59. return context_window_marker(window, pin, **route)
  60. def agent_context_suffix(agent) -> str:
  61. """`` (pinned)`` / `` (fallback)`` / ``""`` for an agent's live compressor window."""
  62. return _MARKER_SUFFIX.get(agent_context_marker(agent), "")
  63. def advertised_context_length(model: str, base_url: str = "") -> Optional[int]:
  64. """Provider-advertised window from LOCAL sources only (persistent cache learned on this
  65. endpoint, models.dev disk cache, hardcoded catalog). Never a network probe: users pin
  66. precisely when the endpoint cannot report its window, so the check must not add startup
  67. latency or a failing request."""
  68. from agent.model_metadata import (
  69. DEFAULT_CONTEXT_LENGTHS, _load_model_metadata_disk_cache, _longest_key_match,
  70. _strip_provider_prefix, get_cached_context_length,
  71. )
  72. model = _strip_provider_prefix(str(model or ""))
  73. if not model:
  74. return None
  75. if base_url:
  76. cached = get_cached_context_length(model, base_url)
  77. if cached:
  78. return int(cached)
  79. entry = _load_model_metadata_disk_cache().get(model) or {}
  80. ctx = entry.get("context_length") if isinstance(entry, dict) else None
  81. if isinstance(ctx, int) and ctx > 0:
  82. return ctx
  83. hit = _longest_key_match(DEFAULT_CONTEXT_LENGTHS, model.lower())
  84. return hit[1] if hit else None
  85. def warn_once_on_pin_disagreement(model: str, base_url: str, config_context_length) -> bool:
  86. """Log ONE warning per (model, pin) when ``model.context_length`` disagrees with the
  87. advertised window. Returns True when the warning fired (the pin still wins)."""
  88. if not is_context_pinned(config_context_length, config_context_length):
  89. return False
  90. advertised = advertised_context_length(model, base_url)
  91. if not advertised or advertised == config_context_length:
  92. return False
  93. key = (str(model), int(config_context_length))
  94. if key in _warned_pins:
  95. return False
  96. _warned_pins.add(key)
  97. logger.warning(
  98. "model.context_length pins %s at %s tokens but the provider advertises %s; the pin wins. "
  99. "Remove model.context_length from config.yaml to use the advertised window.",
  100. model, f"{config_context_length:,}", f"{advertised:,}",
  101. )
  102. return True