chat_completion_helpers_relay.py 3.9 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182
  1. """Relay-side accumulator for the chat_completions streaming wire.
  2. Relay invokes its collector for every post-intercept chunk and then its finalizer as soon
  3. as the provider stream ends — concurrently with Hermes' consumer thread, which may not have
  4. read the last chunk yet. The finalizer therefore builds Relay's recorded response from
  5. collector-observed state only, never from the consumer loop's closures. Sibling of
  6. ``relay_llm.AnthropicStreamAccumulator``; Bedrock and Codex follow the same contract.
  7. """
  8. from __future__ import annotations
  9. from types import SimpleNamespace
  10. from typing import Any
  11. from agent.chat_completion_helpers_tool_calls import _ToolCallAccumulator
  12. from agent.message_content import flatten_message_text
  13. from agent.reasoning_summaries import separate_glued_reasoning_blocks
  14. def _tool_call_delta_view(tc_delta: Any) -> Any:
  15. """Attribute view of a JSON tool-call delta for ``_ToolCallAccumulator.feed`` (written
  16. against SDK objects). Only ``function`` is wrapped: ``feed`` passes ``extra_content``
  17. (a dict) straight through ``_dump_if_model``, so a recursive view would corrupt it."""
  18. if not isinstance(tc_delta, dict):
  19. return tc_delta
  20. function = tc_delta.get("function")
  21. return SimpleNamespace(**{**tc_delta,
  22. "function": SimpleNamespace(**function) if isinstance(function, dict) else function})
  23. class RelayChatAccumulator:
  24. """Rebuild a chat.completion from Relay's post-intercept chunk dicts."""
  25. def __init__(self) -> None:
  26. self._content: list[str] = []
  27. self._reasoning: list[str] = []
  28. self._refusal: list[str] = [] # OpenAI ``delta.refusal`` — a refusal is content, not an empty stream
  29. self._reasoning_details: list[Any] = [] # opaque provider records (signed blocks); order preserved verbatim
  30. self._tool_calls = _ToolCallAccumulator()
  31. self._model = self._usage = self._finish_reason = None
  32. self._role = "assistant"
  33. def observe(self, chunk: Any) -> None:
  34. if not isinstance(chunk, dict):
  35. return
  36. self._model = chunk.get("model") or self._model
  37. if chunk.get("usage"):
  38. self._usage = chunk["usage"]
  39. choices = chunk.get("choices") or []
  40. choice = choices[0] if choices else None # Hermes never requests n>1
  41. if not isinstance(choice, dict):
  42. return
  43. self._finish_reason = choice.get("finish_reason") or self._finish_reason
  44. delta = choice.get("delta")
  45. if not isinstance(delta, dict):
  46. return
  47. if delta.get("role"):
  48. self._role = delta["role"]
  49. text = flatten_message_text(delta.get("content"), sep="")
  50. if text:
  51. self._content.append(text)
  52. self._reasoning_details.extend(delta.get("reasoning_details") or [])
  53. reasoning = delta.get("reasoning_content") or delta.get("reasoning")
  54. if reasoning:
  55. self._reasoning.append(separate_glued_reasoning_blocks(
  56. self._reasoning[-1] if self._reasoning else "", reasoning))
  57. refusal = delta.get("refusal")
  58. if isinstance(refusal, str) and refusal:
  59. self._refusal.append(refusal)
  60. for tc_delta in delta.get("tool_calls") or []:
  61. self._tool_calls.feed(_tool_call_delta_view(tc_delta))
  62. def finalize(self) -> dict[str, Any]:
  63. acc = self._tool_calls.materialize()
  64. message = {"role": self._role, "content": "".join(self._content) or None,
  65. "reasoning_content": "".join(self._reasoning) or None,
  66. "refusal": "".join(self._refusal) or None,
  67. "reasoning_details": self._reasoning_details or None,
  68. "tool_calls": [acc[i] for i in sorted(acc)] or None}
  69. # "stop" also covers Nous Portal ``lastOne`` usage frames, which carry no finish_reason.
  70. return {"model": self._model, "usage": self._usage,
  71. "choices": [{"message": message, "finish_reason": self._finish_reason or "stop"}]}