reasoning_carriers.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. """Provider-native assistant reasoning carriers outside the standard reasoning fields.
  2. * Gemini text-turn ``thought_signature`` (message-level ``extra_content.google`` on the OpenAI-compat
  3. and Vertex endpoints; the non-functionCall ``thoughtSignature`` part on the native API). Replaying it
  4. makes the server rehydrate the turn's thoughts (live: 31 -> 237/250 prompt tokens).
  5. * Top-level assistant keys a provider profile declares (``ProviderProfile.message_carrier_opaque`` /
  6. ``message_carrier_text``: an opaque reasoning blob and its readable text), plus the tool-call
  7. ``function.thought_signature`` some gateways sign Gemini 3 calls with.
  8. Capture stores them top-level on the live assistant dict AND as ONE private ``reasoning_details`` record,
  9. so they persist in the existing column and survive gateway/resume replay. Its type ends in ``.native_assistant``, which no profile declares, so
  10. the chat-completions transport never forwards the record itself; ``shape_wire_carriers`` lifts each
  11. field back onto the wire copy only for the route that reads it (the endpoint profile's
  12. ``replay_message_carriers``) and strips it everywhere else, including keys of records whose declaring
  13. plugin is no longer installed.
  14. """
  15. from __future__ import annotations
  16. from typing import Any, NamedTuple, Optional
  17. CARRIER_TYPE = "hermes.native_assistant"
  18. _GEMINI_KEY = "extra_content"
  19. def declared_carrier_keys() -> tuple[tuple[str, ...], tuple[str, ...]]:
  20. """``(opaque keys, text keys)`` declared by the registered profiles, in registration order. Call once per
  21. message/stream/request, never per delta."""
  22. from providers import list_providers
  23. profiles = list_providers()
  24. opaque = (p.message_carrier_opaque for p in profiles if p.message_carrier_opaque)
  25. text = (p.message_carrier_text for p in profiles if p.message_carrier_text)
  26. return tuple(dict.fromkeys(opaque)), tuple(dict.fromkeys(text))
  27. def _message_keys() -> tuple[str, ...]:
  28. opaque, text = declared_carrier_keys()
  29. return (_GEMINI_KEY, *dict.fromkeys(opaque + text))
  30. def field(obj: Any, name: str) -> Any:
  31. """``obj.<name>``, else the same key from pydantic ``model_extra`` (unknown SDK fields park there)."""
  32. value = getattr(obj, name, None)
  33. for side in ("model_extra", "provider_data"): # pydantic unknown fields / normalized responses
  34. if value is None and isinstance(getattr(obj, side, None), dict):
  35. value = getattr(obj, side).get(name)
  36. dump = getattr(value, "model_dump", None)
  37. return dump() if callable(dump) else value
  38. def thought_signature(extra: Any) -> Optional[str]:
  39. """Gemini signature inside an ``extra_content`` dict (``google.thought_signature`` or flat)."""
  40. if not isinstance(extra, dict):
  41. return None
  42. google = extra.get("google")
  43. sig = google.get("thought_signature") if isinstance(google, dict) else extra.get("thought_signature")
  44. return sig if isinstance(sig, str) and sig.strip() else None
  45. def carrier_record(assistant_message: Any) -> Optional[dict]:
  46. """The private ``reasoning_details`` record for one response message, or None when it has no carrier."""
  47. record: dict[str, Any] = {}
  48. if sig := thought_signature(field(assistant_message, _GEMINI_KEY)):
  49. record[_GEMINI_KEY] = {"google": {"thought_signature": sig}}
  50. for key in _message_keys()[1:]:
  51. value = field(assistant_message, key)
  52. if isinstance(value, str) and value:
  53. record[key] = value
  54. return {"type": CARRIER_TYPE, **record} if record else None
  55. class StreamCarriers:
  56. """Accumulates the carriers from streamed deltas: the latest signature / opaque blob per declared key (a
  57. provider may send a new opaque before each tool call and expect only the latest back) and the concatenated
  58. text of each declared text key. Keys are resolved once per stream."""
  59. def __init__(self) -> None:
  60. self.opaque_keys, self.text_keys = declared_carrier_keys()
  61. self.signature: Optional[str] = None
  62. self.opaque: dict[str, str] = {}
  63. self.text_parts: dict[str, list[str]] = {}
  64. def feed(self, delta: Any, reasoning: Optional[str]) -> Optional[str]:
  65. """Absorb one delta; returns ``reasoning``, else its first declared text-carrier fragment (readable reasoning)."""
  66. self.signature = thought_signature(field(delta, _GEMINI_KEY)) or self.signature
  67. for key in self.opaque_keys:
  68. opaque = field(delta, key)
  69. if isinstance(opaque, str) and opaque:
  70. self.opaque[key] = opaque
  71. for key in self.text_keys:
  72. text = field(delta, key)
  73. if isinstance(text, str) and text:
  74. self.text_parts.setdefault(key, []).append(text)
  75. reasoning = text if reasoning is None else reasoning
  76. return reasoning
  77. def apply(self, response: Any) -> None:
  78. choices = getattr(response, "choices", None)
  79. message = getattr(choices[0], "message", None) if choices else None
  80. if message is None:
  81. return
  82. if self.signature:
  83. message.extra_content = {"google": {"thought_signature": self.signature}}
  84. for key, value in self.opaque.items():
  85. setattr(message, key, value)
  86. for key, parts in self.text_parts.items():
  87. setattr(message, key, "".join(parts))
  88. class _WireRoute(NamedTuple):
  89. model: Any
  90. base_url: str
  91. gemini: bool # the route consumes the Gemini text-turn signature
  92. profile: Any # profile serving the host (``providers.profile_for_host``) or None
  93. keep_fn_sig: bool # replayed tool calls keep ``function.thought_signature``
  94. keys: tuple[str, ...] # every message-level carrier key core knows about
  95. scope: Any # who remembers a failed hook (``provider_route_hooks.ask_profile``): the agent, or None
  96. def _wire_route(model: Any, base_url: Any, scope: Any = None) -> _WireRoute:
  97. from providers import profile_for_host
  98. from agent.provider_route_hooks import ask_profile
  99. from agent.transports.chat_completions import _model_consumes_thought_signature
  100. url = str(base_url or "")
  101. profile = profile_for_host(url)
  102. keep_fn_sig = profile is not None and bool(ask_profile(
  103. profile, "replays_function_thought_signature", False, model=model, base_url=url, scope=scope))
  104. # Same gate as tool-call extra_content, so the two signature carriers can never disagree.
  105. return _WireRoute(model, url, _model_consumes_thought_signature(model), profile, keep_fn_sig, _message_keys(),
  106. scope)
  107. # ``replay_message_carriers`` answers by which carrier keys are present (providers/base.py), so its answer
  108. # for one key set is kept as a key mapping and applied to every message with that key set: one hook call
  109. # per (route, key set) instead of one per carrier-bearing message per request.
  110. _REPLAY_PLANS: dict[tuple, Any] = {}
  111. _REPLAY_PLANS_MAX = 512
  112. _REWRITES = object() # the hook rewrites values: ask it per message
  113. _FAILED = object()
  114. def _replay_plan(answer: Any, values: dict) -> Any:
  115. """``((out key, source key), ...)`` reproducing *answer* from *values*; None when ambiguous."""
  116. if not isinstance(answer, dict):
  117. return _REWRITES
  118. plan = []
  119. for out_key, out_value in answer.items():
  120. sources = [k for k, v in values.items() if v == out_value]
  121. if not sources:
  122. return _REWRITES
  123. if len(sources) > 1:
  124. return None
  125. plan.append((out_key, sources[0]))
  126. return tuple(plan)
  127. def _replay_carriers(route: _WireRoute, values: dict) -> dict:
  128. from providers import registry_generation
  129. from agent.provider_route_hooks import ask_profile, hook_failed
  130. if hook_failed(route.profile, "replay_message_carriers", route.scope):
  131. return {}
  132. key = (registry_generation(), id(route.profile), type(route.profile), route.model, route.base_url,
  133. tuple(sorted(values)))
  134. plan = _REPLAY_PLANS.get(key)
  135. if plan is not None and plan is not _REWRITES:
  136. return {out: values[src] for out, src in plan}
  137. answer = ask_profile(route.profile, "replay_message_carriers", _FAILED, values, model=route.model,
  138. base_url=route.base_url, scope=route.scope)
  139. if answer is _FAILED:
  140. return {}
  141. if plan is None and (plan := _replay_plan(answer, values)) is not None:
  142. if len(_REPLAY_PLANS) >= _REPLAY_PLANS_MAX:
  143. _REPLAY_PLANS.clear()
  144. _REPLAY_PLANS[key] = plan
  145. return dict(answer or {})
  146. def _strip_function_signature(tool_calls: list) -> list:
  147. return [
  148. {**tc, "function": {k: v for k, v in tc["function"].items() if k != "thought_signature"}}
  149. if isinstance(tc, dict) and isinstance(tc.get("function"), dict) and "thought_signature" in tc["function"]
  150. else tc
  151. for tc in tool_calls
  152. ]
  153. def _shape(msg: Any, route: _WireRoute) -> Any:
  154. if not isinstance(msg, dict) or msg.get("role") != "assistant":
  155. return msg
  156. details = msg.get("reasoning_details")
  157. details = details if isinstance(details, list) else []
  158. record = next((d for d in details if isinstance(d, dict) and d.get("type") == CARRIER_TYPE), None)
  159. tool_calls = msg.get("tool_calls")
  160. tool_calls = tool_calls if isinstance(tool_calls, list) else []
  161. fn_sig = not route.keep_fn_sig and any(
  162. isinstance(tc, dict) and isinstance(tc.get("function"), dict) and "thought_signature" in tc["function"]
  163. for tc in tool_calls)
  164. # A persisted record's own keys are stripped too, so rows from an uninstalled provider plugin never leak.
  165. keys = route.keys if record is None else (*route.keys, *(k for k in record if k != "type" and k not in route.keys))
  166. if record is None and not fn_sig and not any(k in msg for k in keys):
  167. return msg
  168. out = {k: v for k, v in msg.items() if k not in keys}
  169. # Live history carries the keys top-level; a resumed row only has the persisted record.
  170. values = {**(record or {}), **{k: msg[k] for k in keys if k in msg}}
  171. if record is not None:
  172. rest = [d for d in details if d is not record]
  173. if rest:
  174. out["reasoning_details"] = rest
  175. else:
  176. out.pop("reasoning_details", None)
  177. if route.gemini and values.get(_GEMINI_KEY):
  178. out[_GEMINI_KEY] = values[_GEMINI_KEY]
  179. if route.profile is not None:
  180. out.update(_replay_carriers(route, values))
  181. if fn_sig:
  182. out["tool_calls"] = _strip_function_signature(tool_calls)
  183. return out
  184. def shape_wire_carriers(messages: list, *, model: Any, base_url: Any, scope: Any = None) -> list:
  185. """Wire copy of ``messages`` with each carrier on the route that reads it and nowhere else.
  186. Never mutates the input; unchanged messages are returned by identity. *scope* is the agent (failed
  187. profile hooks stay failed for it; see ``provider_route_hooks.ask_profile``)."""
  188. from agent.provider_route_hooks import hook_failed
  189. route = _wire_route(model, base_url, scope)
  190. failed_before = route.profile is None or hook_failed(route.profile, "replay_message_carriers", scope)
  191. shaped = [_shape(m, route) for m in messages]
  192. if not failed_before and hook_failed(route.profile, "replay_message_carriers", scope):
  193. # Failed mid-request: reshape so every message takes the fallback, as every later request will.
  194. shaped = [_shape(m, route) for m in messages]
  195. return messages if all(a is b for a, b in zip(shaped, messages)) else shaped