route_binding.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151
  1. """Bind a provider route onto a live agent in place, and put the primary route back.
  2. ``bind_route_entry`` is the fallback chain's swap (client, pool, compressor, reasoning,
  3. extra_body) shared with per-turn routes such as the voice-chat model; ``reinstall_primary_runtime``
  4. is the inverse that ``restore_primary_runtime`` runs once its gates pass.
  5. """
  6. from __future__ import annotations
  7. import logging
  8. from typing import Any, Dict, Optional, Tuple
  9. from utils import base_url_host_matches
  10. logger = logging.getLogger(__name__)
  11. def bind_route_entry(agent: Any, entry: dict[str, Any], provider: str, model: str) -> Optional[tuple[str, str]]:
  12. """Swap ``agent`` onto ``entry`` (a fallback-chain-shaped dict). Returns ``(old_model,
  13. old_provider)``, or None when the provider has no usable client. Sets ``_fallback_activated``
  14. so the next turn's ``restore_primary_runtime`` reverts it; raises on a failed swap."""
  15. from agent.auxiliary_client import resolve_provider_client
  16. from agent.chat_completion_helpers import (
  17. _fallback_api_mode_hint, _fallback_api_mode_resolved, _rebind_fallback_credential_pool,
  18. _reresolve_fallback_reasoning_config, _rescope_fallback_extra_body, _update_fallback_context_compressor,
  19. )
  20. from hermes_cli.fallback_config import resolve_entry_api_key
  21. # Pass the entry's base_url/api_key so custom endpoints (Ollama Cloud) resolve instead
  22. # of falling through to OpenRouter defaults.
  23. base_url_hint = (entry.get("base_url") or "").strip() or None
  24. api_key_hint = resolve_entry_api_key(entry)
  25. api_mode_explicit, api_mode = _fallback_api_mode_hint(entry, provider, base_url_hint)
  26. # Ollama Cloud: OLLAMA_API_KEY from env when the entry has no key. Host match, not
  27. # substring — GHSA-76xc-57q6-vm5m.
  28. if base_url_hint and base_url_host_matches(base_url_hint, "ollama.com") and not api_key_hint:
  29. from agent.secret_scope import get_secret
  30. api_key_hint = get_secret("OLLAMA_API_KEY") or None
  31. # raw_codex=True: the main agent needs direct responses.stream() access for Codex providers.
  32. client, _resolved_model = resolve_provider_client(
  33. provider, model=model, raw_codex=True, explicit_base_url=base_url_hint, explicit_api_key=api_key_hint,
  34. api_mode=api_mode)
  35. if client is None:
  36. return None
  37. if provider == "moa":
  38. # A MoA entry means the preset itself, exactly like ``provider: moa`` in config. The client
  39. # is the preset's aggregator and only proves the preset resolves; bind the facade with the
  40. # same pins every other MoA build site uses (#112525, #112623).
  41. base_url, api_mode = "moa://local", "chat_completions"
  42. else:
  43. try:
  44. from hermes_cli.model_normalize import normalize_model_for_provider
  45. model = normalize_model_for_provider(model, provider)
  46. except Exception as norm_err: # health: allow BLE001 -- moved verbatim from try_activate_fallback; a bad catalog entry keeps the raw id
  47. logger.warning("Could not normalize fallback model %r for provider %r: %s", model, provider, norm_err)
  48. base_url = str(client.base_url)
  49. from hermes_cli.providers import is_actual_route
  50. if is_actual_route(provider, base_url):
  51. api_mode = "chat_completions"
  52. elif not api_mode_explicit and api_mode == "chat_completions":
  53. api_mode = _fallback_api_mode_resolved(agent, provider, model, base_url)
  54. old_model, old_provider, old_base_url = agent.model, agent.provider, agent.base_url
  55. # Clear the per-config context_length override so the new model's own context window is
  56. # resolved instead of the previous model's stale value (#22387).
  57. agent._config_context_length = None
  58. agent.model, agent.provider, agent.requested_provider = model, provider, provider
  59. agent.base_url, agent.api_mode = base_url, api_mode
  60. # reasoning_content echo opt-in travels with the active provider; restore_primary_runtime reverts it.
  61. agent._reasoning_echo_flag = bool(entry.get("reasoning_echo", False))
  62. if hasattr(agent, "_transport_cache"):
  63. agent._transport_cache.clear()
  64. from agent.turn_recovery_codex import reset_codex_reasoning_replay
  65. reset_codex_reasoning_replay(agent)
  66. agent._fallback_activated = True
  67. _rebind_fallback_credential_pool(agent, provider, model)
  68. if provider == "moa":
  69. from agent.moa_loop import bind_moa_runtime
  70. bind_moa_runtime(agent, model)
  71. else:
  72. from agent.client_lifecycle import _swap_fallback_clients
  73. _swap_fallback_clients(agent, client, provider, model, base_url, api_mode)
  74. from agent.agent_runtime_helpers import sync_credential_pool_entry_id
  75. sync_credential_pool_entry_id(agent)
  76. agent._use_prompt_caching, agent._use_native_cache_layout = agent._anthropic_prompt_cache_policy(
  77. provider=provider, base_url=base_url, api_mode=api_mode, model=model)
  78. agent._ensure_lmstudio_runtime_loaded() # LM Studio: preload before probing context length
  79. _update_fallback_context_compressor(agent)
  80. _reresolve_fallback_reasoning_config(agent)
  81. _rescope_fallback_extra_body(agent, old_model, old_provider, old_base_url)
  82. return old_model, old_provider
  83. def reinstall_runtime_snapshot(agent: Any, rt: dict[str, Any]) -> None:
  84. """Put ``agent`` on the runtime recorded in ``rt`` (a ``_primary_runtime``-shaped snapshot):
  85. identity, client, caching flags, compressor, reasoning and prompt identity. Credential pool
  86. and fallback bookkeeping are the caller's. Raises on failure."""
  87. from agent.agent_runtime_helpers import (
  88. _apply_primary_runtime_fields, _rebuild_primary_client, _restore_runtime_capabilities,
  89. )
  90. _apply_primary_runtime_fields(agent, rt)
  91. from agent.turn_recovery_codex import reset_codex_reasoning_replay
  92. reset_codex_reasoning_replay(agent)
  93. _restore_runtime_capabilities(agent, rt)
  94. agent._use_prompt_caching = rt["use_prompt_caching"]
  95. # Default to native layout for snapshots predating the native-vs-proxy split.
  96. agent._use_native_cache_layout = rt.get(
  97. "use_native_cache_layout",
  98. agent.api_mode == "anthropic_messages" and agent.provider == "anthropic",
  99. )
  100. # An operator cache disable (_cache_disabled) must survive snapshot restoration.
  101. if getattr(agent, "_cache_disabled", False):
  102. agent._use_prompt_caching = False
  103. agent._use_native_cache_layout = False
  104. _rebuild_primary_client(agent, rt, reason="restore_primary")
  105. agent.context_compressor.update_model(
  106. model=rt["compressor_model"], context_length=rt["compressor_context_length"],
  107. base_url=rt["compressor_base_url"], api_key=rt["compressor_api_key"],
  108. provider=rt["compressor_provider"], api_mode=rt.get("compressor_api_mode", ""),
  109. )
  110. # Same rule as fallback activation: refresh an existing verdict only; never-probed sessions stay lazy.
  111. if getattr(agent, "_compression_feasibility_checked", False) is True:
  112. from agent.conversation_compression_feasibility import revalidate_compression_feasibility
  113. revalidate_compression_feasibility(agent)
  114. # Older snapshots have no reasoning_config; keep the current value.
  115. saved_reasoning = rt.get("reasoning_config")
  116. if saved_reasoning is not None:
  117. agent.reasoning_config = dict(saved_reasoning)
  118. # Reset the stale-call circuit breaker: its streak measured the route being left.
  119. from agent.chat_completion_helpers import _reset_stale_streak, rewrite_prompt_model_identity
  120. _reset_stale_streak(agent)
  121. # Undo the identity rewrite so the prompt is byte-identical to the stored copy again
  122. # (prefix cache match).
  123. rewrite_prompt_model_identity(agent, rt["model"], rt["provider"])
  124. def reinstall_primary_runtime(
  125. agent: Any, rt: dict[str, Any], primary_provider: str, primary_model: str, matches_primary,
  126. load_primary_pool, prefetched_pool=None, prefetched: bool = False,
  127. ) -> None:
  128. """Put ``agent`` back on its ``_primary_runtime`` snapshot ``rt`` and clear the fallback state."""
  129. from agent.agent_runtime_helpers import _rebind_primary_credential_pool
  130. reinstall_runtime_snapshot(agent, rt)
  131. _rebind_primary_credential_pool(
  132. agent, primary_provider, primary_model, matches_primary, load_primary_pool, prefetched_pool, prefetched
  133. )
  134. agent._fallback_activated = False
  135. agent._fallback_index = 0
  136. agent._rate_limit_backoff_count = 0