gemini_long_decode.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102
  1. """Gemini long decode: one response spread over several generateContent hops.
  2. Gemini 4 caps each request (32,768 tokens unary, 65,536 streamed) but lets ``maxOutputTokens`` reach
  3. 1,048,576 minus the input. A hop that hits the per-request cap with budget left ends with
  4. ``finishReason: CONTINUATION`` plus a ``continuationToken``; the client resends the SAME request with
  5. that token (never the partial output) until any other finish reason. Each hop re-reads the prompt plus
  6. everything generated so far, so per-hop ``promptTokenCount`` grows: usage is summed for billing (what
  7. the google-genai SDK reports and Google bills) while the first hop's prompt is kept as the context size,
  8. because compaction fed the summed prompt would fire far too early. A dropped stream resumes from its
  9. last checkpoint token rather than restarting (Google bills a dropped stream up to that checkpoint).
  10. Docs: https://ai.google.dev/gemini-api/docs/generate-content/long-decode
  11. """
  12. from __future__ import annotations
  13. from typing import Any, Optional
  14. # Effort above ``high`` (Gemini 4's top thinkingLevel) buys a larger total thinking+output budget.
  15. # 262,144 is Google's starting point for hard reasoning; the API caps 1,048,576 at the window minus input.
  16. _LONG_DECODE_BUDGETS = {"xhigh": 262_144, "max": 1_048_576, "ultra": 1_048_576}
  17. # Smallest per-request cap (unary; streamed hops get 65,536): hop counts derived from it are upper bounds.
  18. _HOP_TOKENS = 32_768
  19. _USAGE_COUNT_KEYS = (
  20. "promptTokenCount", "cachedContentTokenCount", "candidatesTokenCount", "thoughtsTokenCount",
  21. "toolUsePromptTokenCount", "totalTokenCount",
  22. )
  23. def long_decode_max_output_tokens(model: str, reasoning_config: Any) -> Optional[int]:
  24. """Total output budget a Gemini 4 request gets for its reasoning effort, or None (per-request cap)."""
  25. name = (model or "").strip().lower().removeprefix("google/")
  26. if not name.startswith("gemini-4") or not isinstance(reasoning_config, dict) or reasoning_config.get("enabled") is False:
  27. return None
  28. return _LONG_DECODE_BUDGETS.get(str(reasoning_config.get("effort") or "").strip().lower())
  29. def long_decode_hops(max_output_tokens: Any) -> int:
  30. """Most hops a request with this total budget can take; 1 when it fits one request. The stale
  31. watchdog scales by it (one timer spans every unary hop) and the hop loops stop one past it, so a
  32. server that keeps answering CONTINUATION can't bill forever."""
  33. budget = max_output_tokens if isinstance(max_output_tokens, int) and not isinstance(max_output_tokens, bool) else 0
  34. return max(1, -(-budget // _HOP_TOKENS))
  35. def long_decode_stale_multiplier(api_payload: Any) -> int:
  36. """Non-streaming stale budgets are one timer around every unary hop of a long decode: scale it by
  37. the hop count so a healthy multi-hop answer isn't killed and re-billed from hop 1. The run-budget
  38. cap still applies afterwards; streaming re-arms on every chunk and needs no scaling."""
  39. extra = api_payload.get("extra_body") if isinstance(api_payload, dict) else None
  40. return long_decode_hops(extra.get("max_output_tokens")) if isinstance(extra, dict) else 1
  41. def _candidate(payload: dict[str, Any]) -> dict[str, Any]:
  42. candidates = payload.get("candidates") if isinstance(payload, dict) else None
  43. first = candidates[0] if isinstance(candidates, list) and candidates else None
  44. return first if isinstance(first, dict) else {}
  45. def continuation_token(payload: dict[str, Any]) -> Optional[str]:
  46. """The token to resend with when this hop ended on CONTINUATION; None on any other finish."""
  47. cand = _candidate(payload)
  48. token = cand.get("continuationToken")
  49. return token if str(cand.get("finishReason") or "").upper() == "CONTINUATION" and isinstance(token, str) and token else None
  50. def merge_hop_usage(hops: list[dict[str, Any]]) -> dict[str, Any]:
  51. """Sum every count across hops; ``initialPromptTokenCount`` (the google-genai name) is hop 1's prompt."""
  52. merged = {key: sum(int(h.get(key) or 0) for h in hops) for key in _USAGE_COUNT_KEYS if any(key in h for h in hops)}
  53. first = hops[0] if hops else {}
  54. merged["initialPromptTokenCount"] = int(first.get("promptTokenCount") or 0)
  55. merged["initialCachedContentTokenCount"] = int(first.get("cachedContentTokenCount") or 0)
  56. return merged
  57. def merge_hop_responses(hops: list[dict[str, Any]]) -> dict[str, Any]:
  58. """One generateContent response whose model turn joins every hop's parts, ending as the last hop did."""
  59. parts = [part for hop in hops for part in ((_candidate(hop).get("content") or {}).get("parts") or [])]
  60. last = _candidate(hops[-1])
  61. candidate = {**last, "content": {**(last.get("content") or {}), "role": "model", "parts": parts}}
  62. usage = merge_hop_usage([hop.get("usageMetadata") or {} for hop in hops])
  63. return {**hops[-1], "candidates": [candidate], "usageMetadata": usage}
  64. def split_stream_continuation(event: dict[str, Any], hop_usage: list[dict[str, Any]]) -> tuple[dict[str, Any], Optional[str]]:
  65. """``(event to translate, token to resend with if the stream ends here)`` for one SSE event.
  66. A CONTINUATION finish is not the end: its finish reason and (per-stream cumulative) usage are
  67. held back in ``hop_usage`` and the token returned for the next hop. The real final event gets
  68. the usage summed over every hop. A token on an event with no finish is a mid-stream checkpoint
  69. (every ~5K tokens past 32K): it commits that event and everything before it, so the caller resumes
  70. a dropped stream by resending the request with it; a drop with no clean checkpoint restarts.
  71. """
  72. token = continuation_token(event)
  73. if token:
  74. hop_usage.append(event.get("usageMetadata") or {})
  75. cand = {k: v for k, v in _candidate(event).items() if k not in ("finishReason", "continuationToken")}
  76. return {k: v for k, v in event.items() if k != "usageMetadata"} | {"candidates": [cand]}, token
  77. cand = _candidate(event)
  78. if hop_usage and cand.get("finishReason"):
  79. return {**event, "usageMetadata": merge_hop_usage([*hop_usage, event.get("usageMetadata") or {}])}, None
  80. checkpoint = None if cand.get("finishReason") else cand.get("continuationToken")
  81. return event, checkpoint if isinstance(checkpoint, str) and checkpoint else None