fast_mode.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120
  1. """Bounded fast-mode windows (``/fast auto`` and ``/fast cold``).
  2. ``agent.service_tier``: ``None`` (normal), ``"priority"`` / ``"ultrafast"`` (static tiers,
  3. pinned into ``agent.request_overrides`` at build time), ``"auto"`` (every user turn opens a
  4. window of ``agent.fast_auto_seconds``) or ``"cold"`` (only a session's first turn,
  5. no prior history, opens it). The provider's fast override is layered onto request
  6. kwargs only while the window is open; only per-request params (``service_tier`` /
  7. ``speed``) vary, so the request body stays byte-identical. Anthropic keeps a separate
  8. prompt cache per speed, so each Anthropic window boundary re-writes the prefix at the
  9. new speed.
  10. """
  11. from __future__ import annotations
  12. import time
  13. from typing import Any
  14. BOUNDED_MODES = frozenset({"auto", "cold"})
  15. DEFAULT_WINDOW_SECONDS = 60
  16. # Documented fast-mode rate-limit headers; a limit of 0 means the organization has no fast
  17. # capacity for the model (https://platform.claude.com/docs/en/build-with-claude/fast-mode).
  18. _FAST_LIMIT_HEADERS = ("anthropic-fast-input-tokens-limit", "anthropic-fast-output-tokens-limit")
  19. #: Tiers sent on every request of the session (OpenAI ``service_tier`` values; ``priority`` also
  20. #: selects Anthropic/xAI fast mode). Ultrafast is OpenAI-only and gated per model.
  21. STATIC_TIERS = frozenset({"priority", "ultrafast"})
  22. # Codex app-server names for wire tiers it accepts (turn/start.serviceTier); a tier missing here is not sent.
  23. CODEX_TIER_WORDS: dict[str, str] = {"priority": "fast"}
  24. NORMAL_TIER_WORDS = frozenset({"", "normal", "default", "standard", "off", "none"})
  25. # User/config word -> agent.service_tier. The single table every surface (config loaders, /fast
  26. # on CLI / gateway / TUI) parses through, so a new tier is one edit.
  27. SERVICE_TIER_WORDS: dict[str, str] = {
  28. "fast": "priority", "priority": "priority", "on": "priority",
  29. "ultrafast": "ultrafast", "ultra": "ultrafast", "auto": "auto", "cold": "cold",
  30. }
  31. # Typed /fast shorthands -> canonical /fast word (pickers always send the canonical value).
  32. FAST_ARG_ALIASES: dict[str, str] = {"ultra": "ultrafast"}
  33. def parse_service_tier(raw: Any) -> str | None:
  34. """``agent.service_tier`` for a user/config word; None for normal and for unknown words."""
  35. value = str(raw or "").strip().lower()
  36. return None if value in NORMAL_TIER_WORDS else SERVICE_TIER_WORDS.get(value)
  37. def parse_exact_service_tier(raw: Any) -> str:
  38. """Strict :func:`parse_service_tier` for an explicit client pick: ``""`` pins normal, an unknown
  39. word raises ``ValueError`` instead of silently reading as normal."""
  40. value = str(raw or "").strip().lower()
  41. if value not in NORMAL_TIER_WORDS and value not in SERVICE_TIER_WORDS:
  42. raise ValueError(f"unknown service tier: {value}")
  43. return parse_service_tier(value) or ""
  44. def service_tier_word(tier: Any) -> str:
  45. """The user-facing word for a stored tier (``priority`` -> ``fast``, None/"" -> ``normal``)."""
  46. return {"priority": "fast", None: "normal", "": "normal"}.get(tier, tier)
  47. def begin_turn(agent: Any, conversation_history: Any) -> None:
  48. """Open (or refuse) the fast window at a user-turn boundary."""
  49. mode = getattr(agent, "service_tier", None)
  50. agent._fast_until = 0.0
  51. if mode not in BOUNDED_MODES:
  52. return
  53. if mode == "cold" and any(
  54. isinstance(m, dict) and m.get("role") in ("user", "assistant", "tool")
  55. for m in (conversation_history or ())
  56. ):
  57. return
  58. try:
  59. window = float(getattr(agent, "fast_auto_seconds", DEFAULT_WINDOW_SECONDS))
  60. except (TypeError, ValueError):
  61. window = DEFAULT_WINDOW_SECONDS
  62. agent._fast_until = time.monotonic() + max(window, 0.0)
  63. def effective_request_overrides(agent: Any) -> dict[str, Any]:
  64. """``agent.request_overrides`` plus the fast override while the window is open, minus
  65. ``speed`` for a model this session learned has no fast capacity."""
  66. overrides = dict(getattr(agent, "request_overrides", None) or {})
  67. if getattr(agent, "service_tier", None) in BOUNDED_MODES and time.monotonic() < getattr(agent, "_fast_until", 0.0):
  68. from hermes_cli.models import resolve_fast_mode_overrides
  69. base_url = getattr(agent, "base_url", None)
  70. if getattr(agent, "api_mode", None) == "anthropic_messages":
  71. base_url = getattr(agent, "_anthropic_base_url", None) or base_url
  72. overrides.update(
  73. resolve_fast_mode_overrides(getattr(agent, "model", None), provider=getattr(agent, "provider", None), base_url=base_url) or {}
  74. )
  75. if "speed" in overrides and getattr(agent, "model", None) in (getattr(agent, "_fast_mode_unavailable_models", None) or ()):
  76. overrides.pop("speed", None)
  77. from agent.codex_service_tiers import apply_daybreak, negotiate_request_tier
  78. negotiate_request_tier(agent, overrides)
  79. apply_daybreak(agent, overrides)
  80. return overrides
  81. def fast_mode_unprovisioned(api_error: Any, api_kwargs: Any) -> bool:
  82. """True for a 429 on a ``speed: "fast"`` request whose fast-mode limit header is 0. The
  83. organization has no fast capacity for the model, so waiting or rotating keys cannot help."""
  84. if getattr(api_error, "status_code", None) != 429 or not isinstance(api_kwargs, dict):
  85. return False
  86. if (api_kwargs.get("extra_body") or {}).get("speed") != "fast":
  87. return False
  88. headers = getattr(getattr(api_error, "response", None), "headers", None)
  89. if headers is None:
  90. return False
  91. return any(str(headers.get(name, "")).strip() == "0" for name in _FAST_LIMIT_HEADERS)
  92. def mark_fast_mode_unavailable(agent: Any) -> bool:
  93. """Stop sending ``speed`` for the current model for the rest of the session. False when the
  94. model was already marked, so the caller retries at most once per model."""
  95. model = getattr(agent, "model", None)
  96. unavailable = getattr(agent, "_fast_mode_unavailable_models", None)
  97. if not isinstance(unavailable, set):
  98. unavailable = agent._fast_mode_unavailable_models = set()
  99. if not model or model in unavailable:
  100. return False
  101. unavailable.add(model)
  102. return True