served_model.py 3.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768
  1. """Per-response *served model* capture for routing proxies (#54864).
  2. A LiteLLM-style proxy answers with the configured alias in the body's ``model`` field and puts
  3. the deployment it really routed to in a response header (``x-litellm-model-id``, else the
  4. upstream ``x-litellm-model-api-base``). The OpenAI SDK's parsed objects drop headers, so the
  5. capture rides an ``httpx`` response hook on the client Hermes builds for the agent; it stores
  6. the header onto ``agent.last_served_model`` (``None`` when the response carried none, so a
  7. value never outlives the request that produced it). Consumers: ``agent/turn_finalizer.py``
  8. (result ``served_model`` / ``requested_model``) and the opt-in gateway footer field
  9. ``served_model`` (``gateway/runtime_footer.py``). Fail-open everywhere.
  10. """
  11. from __future__ import annotations
  12. import logging
  13. from typing import Any, Optional
  14. logger = logging.getLogger(__name__)
  15. SERVED_MODEL_HEADERS: tuple[str, ...] = ("x-litellm-model-id", "x-litellm-model-api-base")
  16. _HOOK_MARK = "_hermes_served_model_hook"
  17. def served_model_from_headers(headers: Any) -> Optional[str]:
  18. """First non-empty served-model header, or ``None``."""
  19. if headers is None or not hasattr(headers, "get"):
  20. return None
  21. for name in SERVED_MODEL_HEADERS:
  22. value = headers.get(name)
  23. if value not in (None, ""):
  24. return str(value).strip()
  25. return None
  26. def install_served_model_capture(agent: Any, client: Any) -> None:
  27. """Register the response hook on *client*'s ``httpx`` transport (idempotent per client)."""
  28. http_client = getattr(client, "_client", None)
  29. hooks = getattr(http_client, "event_hooks", None)
  30. if not isinstance(hooks, dict):
  31. return
  32. if any(getattr(h, _HOOK_MARK, False) for h in hooks.get("response", ())):
  33. return
  34. def _on_response(response: Any) -> None:
  35. try:
  36. if 200 <= int(getattr(response, "status_code", 0)) < 300:
  37. agent.last_served_model = served_model_from_headers(getattr(response, "headers", None))
  38. except Exception:
  39. logger.debug("served-model header capture skipped", exc_info=True)
  40. setattr(_on_response, _HOOK_MARK, True)
  41. try:
  42. # httpx copies on assignment; rebuild the mapping instead of mutating the live list.
  43. http_client.event_hooks = {**hooks, "response": [*hooks.get("response", ()), _on_response]}
  44. except Exception:
  45. logger.debug("served-model hook install skipped", exc_info=True)
  46. def result_model_fields(agent: Any) -> dict[str, Optional[str]]:
  47. """``requested_model`` / ``served_model`` for the turn result: the proxy header when the
  48. last response carried one, else Hermes' own fallback route (primary → active model)."""
  49. served = getattr(agent, "last_served_model", None)
  50. requested = agent.model
  51. if not served and getattr(agent, "_fallback_activated", False):
  52. primary = str((getattr(agent, "_primary_runtime", None) or {}).get("model") or "").strip()
  53. if primary and primary != agent.model:
  54. requested, served = primary, agent.model
  55. return {"requested_model": requested, "served_model": served or None}