context_switch_guard.py 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164
  1. """Warn when an in-session model switch will trigger preflight compression on the next turn."""
  2. from __future__ import annotations
  3. from typing import Any, Callable, List, Optional
  4. from agent.model_metadata import MINIMUM_CONTEXT_LENGTH
  5. from hermes_cli.model_switch import ModelSwitchResult, resolve_display_context_length
  6. def _append_warning(result: ModelSwitchResult, text: str) -> None:
  7. if result.warning_message:
  8. result.warning_message = f"{result.warning_message} | {text}"
  9. else:
  10. result.warning_message = text
  11. def _threshold_tokens(compressor: Any, model: str, context_length: int, provider: str = "") -> int:
  12. """The trigger the compressor WILL use after the switch (cap, model_thresholds and small-window
  13. floor included), so the warning quotes the real number; duck-typed engines keep the plain ratio."""
  14. preview = getattr(compressor, "preview_threshold_tokens", None)
  15. if callable(preview):
  16. return int(preview(model, context_length, provider))
  17. return max(int(context_length * float(getattr(compressor, "threshold_percent", 0.5))), MINIMUM_CONTEXT_LENGTH)
  18. def _estimate_tokens(agent: Any, messages: Optional[list[dict]]) -> Optional[int]:
  19. cc = getattr(agent, "context_compressor", None)
  20. if cc is None:
  21. return None
  22. if messages is not None:
  23. protect = (
  24. int(getattr(cc, "protect_first_n", 3)) + int(getattr(cc, "protect_last_n", 20)) + 1)
  25. if len(messages) <= protect:
  26. return None
  27. try:
  28. from agent.model_metadata import estimate_request_tokens_rough
  29. system_prompt = getattr(agent, "_cached_system_prompt", None) or ""
  30. tools = getattr(agent, "tools", None)
  31. return int(
  32. estimate_request_tokens_rough(
  33. messages, system_prompt=system_prompt, tools=tools or None))
  34. except Exception:
  35. pass
  36. # session_prompt_tokens is a lifetime sum, not occupancy (#126343): never a fallback here.
  37. last = int(getattr(cc, "last_prompt_tokens", 0) or 0)
  38. return last if last > 0 else None
  39. def merge_preflight_compression_warning(
  40. result: ModelSwitchResult,
  41. *,
  42. agent: Any = None,
  43. messages: Optional[list[dict]] = None,
  44. custom_providers: list | None = None,
  45. config_context_length: int | None = None,
  46. configured_model: str | None = None,
  47. configured_provider: str | None = None,
  48. configured_base_url: str | None = None) -> None:
  49. """If the next user message will likely preflight-compress, append a warning."""
  50. if not result.success or agent is None:
  51. return
  52. if not getattr(agent, "compression_enabled", True):
  53. return
  54. cc = getattr(agent, "context_compressor", None)
  55. if cc is None:
  56. return
  57. # Fall back to the agent's custom providers: without them the shrink warning used the
  58. # hardcoded catalog (e.g. "qwen" → 131072) even when the provider declared 1M.
  59. if custom_providers is None:
  60. custom_providers = getattr(agent, "_custom_providers", None)
  61. def _or_agent(value, attr):
  62. return value if value is not None else getattr(agent, attr, None)
  63. old_ctx = int(getattr(cc, "context_length", 0) or 0)
  64. new_ctx = resolve_display_context_length(
  65. result.new_model,
  66. result.target_provider,
  67. base_url=result.base_url or getattr(agent, "base_url", "") or "",
  68. api_key=result.api_key or getattr(agent, "api_key", "") or "",
  69. model_info=result.model_info,
  70. custom_providers=custom_providers,
  71. config_context_length=config_context_length,
  72. configured_model=_or_agent(configured_model, "model"),
  73. configured_provider=_or_agent(configured_provider, "provider"),
  74. configured_base_url=_or_agent(configured_base_url, "base_url"))
  75. if not new_ctx:
  76. return
  77. estimate = _estimate_tokens(agent, messages)
  78. if estimate is None:
  79. return
  80. new_threshold = _threshold_tokens(cc, result.new_model, new_ctx, result.target_provider)
  81. if estimate < new_threshold:
  82. return
  83. if int(getattr(cc, "_ineffective_compression_count", 0) or 0) >= 2:
  84. return
  85. parts: list[str] = []
  86. if old_ctx and new_ctx < old_ctx:
  87. parts.append(f"Context window shrinks ({old_ctx:,} → {new_ctx:,}). ")
  88. parts.append(
  89. f"Session is ~{estimate:,} tokens; "
  90. f"{result.new_model} allows {new_ctx:,} "
  91. f"(auto-compress at ~{new_threshold:,}). "
  92. f"Your next message will run preflight compression before the model replies.")
  93. _append_warning(result, "".join(parts))
  94. def enrich_model_switch_warnings_for_gateway(
  95. result: ModelSwitchResult,
  96. runner: Any,
  97. *,
  98. session_key: str,
  99. source: Any,
  100. custom_providers: list | None = None,
  101. load_gateway_config: Callable[[], dict] | None = None) -> None:
  102. """Gateway helper: cached agent + session DB messages."""
  103. lock = getattr(runner, "_agent_cache_lock", None)
  104. cache = getattr(runner, "_agent_cache", None)
  105. agent = None
  106. if lock is not None and cache is not None:
  107. with lock:
  108. entry = cache.get(session_key)
  109. if entry and entry[0] is not None:
  110. agent = entry[0]
  111. if agent is None:
  112. return
  113. configured: dict = dict.fromkeys(
  114. ("config_context_length", "configured_model", "configured_provider", "configured_base_url"))
  115. if load_gateway_config is not None:
  116. try:
  117. cfg = load_gateway_config()
  118. model_cfg = cfg.get("model", {}) if isinstance(cfg, dict) else {}
  119. if isinstance(model_cfg, dict) and model_cfg.get("context_length") is not None:
  120. configured.update(
  121. config_context_length=int(model_cfg["context_length"]),
  122. configured_model=model_cfg.get("default") or model_cfg.get("model"),
  123. configured_provider=model_cfg.get("provider"),
  124. configured_base_url=model_cfg.get("base_url"))
  125. except Exception:
  126. pass
  127. messages = None
  128. db = getattr(runner, "_session_db", None)
  129. store = getattr(runner, "session_store", None)
  130. if db is not None and store is not None:
  131. try:
  132. entry = store.get_or_create_session(source)
  133. messages = db.get_messages_as_conversation(entry.session_id)
  134. except Exception:
  135. pass
  136. merge_preflight_compression_warning(
  137. result, agent=agent, messages=messages, custom_providers=custom_providers, **configured)