relay_llm.py 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964
  1. """Core NeMo Relay adapters for physical Hermes provider attempts."""
  2. from __future__ import annotations
  3. import asyncio
  4. import contextlib
  5. import contextvars
  6. import inspect
  7. import json
  8. import logging
  9. from collections.abc import Callable, Iterator
  10. from functools import partial
  11. from types import SimpleNamespace
  12. from typing import Any
  13. from agent import relay_runtime
  14. logger = logging.getLogger(__name__)
  15. _PROVIDER_MESSAGE_EXTENSION_KEYS = frozenset({"reasoning_content", "reasoning_details"})
  16. _RELAY_INTERNAL_PROVIDER_HEADERS = frozenset({"x-dynamo-parent-session-id", "x-dynamo-session-id"})
  17. _LogicalCall = tuple[relay_runtime.RelayTurnContext, Any, str]
  18. # Bound for awaiting a Relay stream's aclose() on the private loop: a wedged
  19. # close must not hang the worker thread or hold the runtime lease forever.
  20. _ACLOSE_TIMEOUT = 10.0
  21. # api_mode -> (Relay operation name, codec class name on ``relay.codecs``)
  22. _RELAY_PROTOCOL_BY_API_MODE = {
  23. "chat_completions": ("openai.chat_completions", "OpenAIChatCodec"),
  24. "codex_responses": ("openai.responses", "OpenAIResponsesCodec"),
  25. "anthropic_messages": ("anthropic.messages", "AnthropicMessagesCodec"),
  26. }
  27. def _api_mode(metadata: dict[str, Any] | None) -> str:
  28. return str((metadata or {}).get("api_mode") or "")
  29. def _relay_operation_name(provider_name: str, metadata: dict[str, Any] | None) -> str:
  30. """Return Relay's canonical operation name when Hermes knows the API mode."""
  31. protocol = _RELAY_PROTOCOL_BY_API_MODE.get(_api_mode(metadata))
  32. return protocol[0] if protocol is not None else provider_name
  33. def _relay_metadata(provider_name: str, metadata: dict[str, Any] | None) -> dict[str, Any]:
  34. """Preserve the physical provider when the operation name is canonicalized."""
  35. relay_metadata = _jsonable_dict(metadata or {})
  36. relay_metadata.setdefault("hermes.provider", provider_name)
  37. return relay_metadata
  38. class _ManagedAttempt:
  39. """Relay request state shared by the sync, async, and streaming adapters."""
  40. @classmethod
  41. def resolve(
  42. cls, session_id: str | None, request: dict[str, Any], metadata: dict[str, Any] | None,
  43. *, name: str, model_name: str,
  44. ) -> _ManagedAttempt | None:
  45. """Return the managed attempt for ``session_id`` (None: the inherited turn's), or None to run unmanaged."""
  46. if session_id is None:
  47. session_id = _current_session_id()
  48. if not session_id:
  49. return None
  50. runtime, session, parent = relay_runtime.resolve_execution_context(session_id)
  51. if runtime is None or session is None or not runtime.managed_execution_enabled():
  52. return None
  53. return cls(runtime, session, parent, request, metadata, name=name, model_name=model_name)
  54. def __init__(
  55. self, runtime: relay_runtime.RelayRuntime, session: Any, parent: Any,
  56. request: dict[str, Any], metadata: dict[str, Any] | None, *, name: str, model_name: str,
  57. ) -> None:
  58. self.runtime, self.session, self.request, self.metadata = runtime, session, request, metadata
  59. self.logical = _logical_parent(runtime, session, parent, metadata)
  60. self.parent = self.logical[1] if self.logical is not None else parent
  61. self.body = _relay_request_body(request, metadata)
  62. self.relay_request = runtime.relay.LLMRequest({}, self.body)
  63. self.codec_baseline = _codec_round_trip_request_body(
  64. runtime.relay, self.relay_request, relay_request_body=self.body, metadata=metadata
  65. )
  66. self.operation = _relay_operation_name(name, metadata)
  67. self.relay_kwargs = {
  68. "handle": self.parent, "metadata": _relay_metadata(name, metadata), "model_name": model_name,
  69. "codec": _codec(runtime.relay, metadata), "response_codec": _codec(runtime.relay, metadata),
  70. }
  71. # Provider callback bookkeeping: "value"/"json" once it returned, "error" if it raised.
  72. self.raw_response: dict[str, Any] = {}
  73. self.context = contextvars.copy_context()
  74. def provider_request(self, next_request: Any) -> dict[str, Any]:
  75. return _provider_request(
  76. self.request, next_request, relay_request_body=self.body,
  77. codec_baseline_body=self.codec_baseline, metadata=self.metadata,
  78. )
  79. def run_callback(self, callback: Callable[..., Any], *args: Any) -> Any:
  80. """Run a Hermes callback in a fresh copy of the captured context.
  81. Relay can invoke callbacks while another still owns the captured Context (hence the
  82. copy); nested relay calls run unmanaged — see relay_runtime.managed_callback_guard."""
  83. def guarded() -> Any:
  84. # See #77244.
  85. # See #77244.
  86. # Hermes-side callbacks run while the native pipeline drives this stream; nested relay calls
  87. # they make must bypass managed execution (#77244).
  88. with relay_runtime.managed_callback_guard():
  89. return callback(*args)
  90. return self.context.copy().run(guarded)
  91. def _record(self, raw: Any) -> Any:
  92. self.raw_response["value"] = raw
  93. self.raw_response["json"] = _jsonable(raw)
  94. return self.raw_response["json"]
  95. @contextlib.contextmanager
  96. def _recording_errors(self) -> Iterator[None]:
  97. try:
  98. yield
  99. except BaseException as exc:
  100. self.raw_response["error"] = exc
  101. raise
  102. def invoke(self, callback: Callable[..., Any], next_request: Any) -> Any:
  103. """Provider callback handed to Relay: run ``callback`` on Relay's (possibly rewritten) request."""
  104. with self._recording_errors():
  105. raw = self.run_callback(callback, self.provider_request(next_request))
  106. return self._record(raw)
  107. async def invoke_async(self, callback: Callable[..., Any], next_request: Any) -> Any:
  108. async def call_provider() -> Any:
  109. with relay_runtime.managed_callback_guard(): # nested relay calls run unmanaged
  110. return await callback(final_request)
  111. with self._recording_errors():
  112. final_request = self.provider_request(next_request)
  113. raw = await self.context.copy().run(asyncio.create_task, call_provider())
  114. return self._record(raw)
  115. def run_managed(self, relay_call: Callable[..., Any], *callbacks: Any) -> Any:
  116. """Return the awaitable running ``relay_call`` inside the session context."""
  117. return self.runtime.run_in_session_async(
  118. self.session, relay_call, self.operation, self.relay_request, *callbacks, **self.relay_kwargs,
  119. )
  120. def resolve_failure(self, exc: BaseException, defer_logical_completion: bool) -> Any:
  121. """Re-raise the provider's own error, or recover a completed provider result.
  122. Must be called from the ``except`` handling ``exc`` (bare ``raise``)."""
  123. callback_error = self.raw_response.get("error")
  124. if callback_error is not None and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error):
  125. raise callback_error
  126. if not isinstance(exc, Exception) or callback_error is not None or "value" not in self.raw_response:
  127. raise
  128. logger.warning(
  129. "NeMo Relay LLM post-processing failed after provider success; returning the provider response",
  130. exc_info=True,
  131. )
  132. self._complete(defer_logical_completion)
  133. return self.raw_response["value"]
  134. def result(self, managed: Any, defer_logical_completion: bool) -> Any:
  135. self._complete(defer_logical_completion)
  136. if "value" in self.raw_response and _json_equal(managed, self.raw_response["json"]):
  137. return self.raw_response["value"]
  138. return _namespace(managed)
  139. def _complete(self, defer_logical_completion: bool) -> None:
  140. if not defer_logical_completion:
  141. _complete_logical(self.logical, outcome="success")
  142. def _current_session_id() -> str | None:
  143. """Return the inherited Hermes turn's session id, or None outside a live turn."""
  144. turn = relay_runtime.active_turn()
  145. return None if turn is None else turn.lease.session_id
  146. def execute(
  147. request: dict[str, Any], callback: Callable[[dict[str, Any]], Any], *, name: str, model_name: str,
  148. session_id: str | None = None, metadata: dict[str, Any] | None = None, defer_logical_completion: bool = False,
  149. ) -> Any:
  150. """Run one non-streaming physical provider attempt through Relay.
  151. ``session_id`` defaults to the inherited Hermes turn's session (unmanaged when there is none)."""
  152. attempt = _ManagedAttempt.resolve(session_id, request, metadata, name=name, model_name=model_name)
  153. if attempt is None:
  154. return callback(request)
  155. try:
  156. managed = _run_awaitable(attempt.run_managed(
  157. attempt.runtime.relay.llm.execute, partial(attempt.invoke, callback)
  158. ))
  159. except BaseException as exc:
  160. return attempt.resolve_failure(exc, defer_logical_completion)
  161. return attempt.result(managed, defer_logical_completion)
  162. async def execute_async(
  163. request: dict[str, Any], callback: Callable[[dict[str, Any]], Any], *, name: str, model_name: str,
  164. session_id: str | None = None, metadata: dict[str, Any] | None = None, defer_logical_completion: bool = False,
  165. ) -> Any:
  166. """Async ``execute``."""
  167. attempt = _ManagedAttempt.resolve(session_id, request, metadata, name=name, model_name=model_name)
  168. if attempt is None:
  169. return await callback(request)
  170. try:
  171. managed = await attempt.run_managed(attempt.runtime.relay.llm.execute, partial(attempt.invoke_async, callback))
  172. except BaseException as exc:
  173. return attempt.resolve_failure(exc, defer_logical_completion)
  174. return attempt.result(managed, defer_logical_completion)
  175. # Run under the inherited Hermes turn when present (callers that do not know a session id).
  176. execute_current = execute
  177. execute_current_async = execute_async
  178. def _has_running_event_loop() -> bool:
  179. with contextlib.suppress(RuntimeError):
  180. return asyncio.get_running_loop() is not None
  181. return False
  182. def stream_current(
  183. request: dict[str, Any], stream_factory: Callable[[dict[str, Any]], Any], *, name: str, model_name: str,
  184. finalizer: Callable[[], Any], metadata: dict[str, Any] | None = None,
  185. defer_logical_completion: bool = False, completed_response_predicate: Callable[[Any], bool] | None = None,
  186. ) -> Any:
  187. """Run a provider stream under the inherited Hermes turn when present.
  188. With ``completed_response_predicate`` set, a factory that ignores ``stream=True`` and returns a
  189. complete response is unwrapped and returned directly (pre-Relay behavior). Detecting that primes
  190. the lazy pipeline: a genuine first chunk is buffered, but provider latency and pre-first-yield
  191. errors may surface before this returns.
  192. AnthropicAuxiliaryClient and other shims that ignore ``stream=True``), unwrap and return the completed
  193. response directly. This mirrors the pre-Relay behavior where ``call_llm(stream=True)`` returned the raw
  194. response and the consumer's own ``hasattr(stream, "choices")`` check handled it (#11732, #55933) —
  195. without the unwrap the response stays trapped as ``final_response`` on the inner ManagedLlmStream and
  196. the outer consumer sees an empty stream.
  197. """
  198. session_id = _current_session_id()
  199. # Inside a managed callback (on the Relay session's loop) a nested ManagedLlmStream would be
  200. # iterated synchronously on that loop, which asyncio forbids; the outer stream tracks this attempt.
  201. if session_id is None or _has_running_event_loop():
  202. return stream_factory(request)
  203. managed = stream(
  204. request, stream_factory, session_id=session_id, name=name, model_name=model_name,
  205. finalizer=finalizer, metadata=metadata, defer_logical_completion=defer_logical_completion,
  206. completed_response_predicate=completed_response_predicate,
  207. )
  208. if completed_response_predicate is not None:
  209. # Relay may defer the provider callback until the first pull; prime once (a real first chunk is buffered).
  210. managed._prime_completed_response()
  211. if managed.final_response is not None:
  212. # The completed response replaces the stream, so finish it deterministically rather
  213. # than leaving the loop, Relay stream, and lease for __del__/GC. The provider call
  214. # succeeded, so the logical outcome is "success" as on normal exhaustion.
  215. managed._finish_logical("success")
  216. managed._close(logical_outcome="cancelled")
  217. return managed.final_response
  218. return managed
  219. def _aclose_on_loop(loop: asyncio.AbstractEventLoop, stream: Any) -> bool:
  220. """Await ``stream.aclose()`` on ``loop`` when the stream exposes one, bounded by
  221. ``_ACLOSE_TIMEOUT``. Returns False when the attempt is abandoned: the daemon
  222. thread still owns the running loop, so the caller must leave it open rather
  223. than ``loop.close()`` under it."""
  224. close = getattr(stream, "aclose", None)
  225. if not callable(close):
  226. return True
  227. async def close_stream() -> None: # create the coroutine on ``loop``, not the caller's thread
  228. await close()
  229. try:
  230. relay_runtime._run_on_daemon_thread(
  231. lambda: loop.run_until_complete(close_stream()),
  232. name="relay-llm-stream-aclose", timeout=_ACLOSE_TIMEOUT,
  233. timeout_message="Relay stream aclose did not finish; abandoning the close attempt",
  234. )
  235. except TimeoutError:
  236. logger.warning(
  237. "Relay stream aclose exceeded %ss; abandoning the close attempt and its private loop",
  238. _ACLOSE_TIMEOUT,
  239. )
  240. return False
  241. return True
  242. def _next_provider_chunk(callback: Callable[..., Any], raw_iterator: Any) -> tuple[Any, bool]:
  243. """Read one synchronous provider chunk without leaking StopIteration through a Future."""
  244. try:
  245. return callback(next, raw_iterator), False
  246. except StopIteration:
  247. return None, True
  248. class ManagedLlmStream(Iterator[Any]):
  249. """Synchronous view of one Relay-managed provider stream, driven from the worker thread."""
  250. final_response: Any = None
  251. output_modified = _closed = _provider_completed = _delivered_unmatched = False
  252. _loop: asyncio.AbstractEventLoop | None = None
  253. _stream = _raw_stream_resource = None
  254. _runtime_lease: relay_runtime.RelayOperationLease | None = None
  255. _close_error = _callback_error = None # BaseException | None
  256. _logical: _LogicalCall | None = None
  257. _logical_response_model_name: str | None = None
  258. def __init__(
  259. self, request: dict[str, Any], stream_factory: Callable[[dict[str, Any]], Any], *, session_id: str,
  260. name: str, model_name: str, finalizer: Callable[[], Any],
  261. on_stream_created: Callable[[Any], None] | None = None, on_chunk: Callable[[Any], None] | None = None,
  262. chunk_adapter: Callable[[Any], Any] | None = None, accept_chunk: Callable[[Any], bool] | None = None,
  263. completed_response_predicate: Callable[[Any], bool] | None = None,
  264. metadata: dict[str, Any] | None = None, defer_logical_completion: bool = False,
  265. ) -> None:
  266. self._defer_logical_completion = defer_logical_completion
  267. # Only auxiliary calls report model/provider on their logical scope.
  268. auxiliary = str((metadata or {}).get("call_role") or "").startswith("auxiliary:")
  269. self._logical_model_name, self._logical_provider_name = (model_name, name) if auxiliary else (None, None)
  270. self._on_chunk, self._chunk_adapter, self._accept_chunk = on_chunk, chunk_adapter or _namespace, accept_chunk
  271. self._stream_factory, self._on_stream_created, self._finalizer = stream_factory, on_stream_created, finalizer
  272. self._completed_response_predicate = completed_response_predicate
  273. self._raw_chunks: list[tuple[Any, Any]] = []
  274. self._prefetched_chunks: list[Any] = []
  275. attempt = _ManagedAttempt.resolve(session_id, request, metadata, name=name, model_name=model_name)
  276. if attempt is None:
  277. self._start_unmanaged(request)
  278. return
  279. self._logical = attempt.logical
  280. self._start_managed(attempt)
  281. def _start_unmanaged(self, request: dict[str, Any]) -> None:
  282. raw_stream = self._stream_factory(request)
  283. predicate = self._completed_response_predicate
  284. if predicate is not None and predicate(raw_stream):
  285. self.final_response = raw_stream
  286. self._stream = iter(())
  287. return
  288. self._raw_stream_resource = raw_stream
  289. if self._on_stream_created is not None:
  290. self._on_stream_created(raw_stream)
  291. self._stream = iter(raw_stream)
  292. async def _provider_stream(self, attempt: _ManagedAttempt, next_request: Any):
  293. """Relay's provider callback: run the factory and yield JSON-encoded chunks."""
  294. run_callback = attempt.run_callback
  295. raw_stream = None
  296. try:
  297. raw_stream = run_callback(self._stream_factory, attempt.provider_request(next_request))
  298. predicate = self._completed_response_predicate
  299. if predicate is not None and run_callback(predicate, raw_stream):
  300. self.final_response = raw_stream
  301. self._provider_completed = True
  302. return
  303. if self._on_stream_created is not None:
  304. run_callback(self._on_stream_created, raw_stream)
  305. raw_iterator = run_callback(iter, raw_stream)
  306. while True:
  307. # Off the loop: Relay pulls the next provider chunk before it hands over the
  308. # current one, so a blocking read here withholds each chunk until the provider
  309. # sends the next. Text vanishes for every provider pause and a steer aborts it.
  310. chunk, exhausted = await asyncio.to_thread(_next_provider_chunk, run_callback, raw_iterator)
  311. if exhausted:
  312. break
  313. if self._accept_chunk is not None and not run_callback(self._accept_chunk, chunk):
  314. break
  315. encoded_chunk = _jsonable(chunk)
  316. self._raw_chunks.append((encoded_chunk, chunk))
  317. yield encoded_chunk
  318. self._provider_completed = True
  319. except BaseException as exc:
  320. self._callback_error = exc
  321. raise
  322. finally:
  323. close = getattr(raw_stream, "close", None)
  324. if callable(close):
  325. try:
  326. run_callback(close)
  327. except BaseException as exc:
  328. self._close_error = exc
  329. raise
  330. def _relay_finalizer(self, attempt: _ManagedAttempt) -> Any:
  331. # Relay may call this while unwinding a provider-stream failure; keep the original
  332. # error instead of a secondary "missing terminal response".
  333. if self._callback_error is not None:
  334. return None
  335. try:
  336. response = self.final_response
  337. if response is None:
  338. response = attempt.run_callback(self._finalizer)
  339. if self._logical_model_name is not None:
  340. self._logical_response_model_name = _response_model_name(response)
  341. return _jsonable(response)
  342. except BaseException as exc:
  343. self._callback_error = exc
  344. raise
  345. def _start_managed(self, attempt: _ManagedAttempt) -> None:
  346. """Open Relay's stream on a private event loop owned by this iterator."""
  347. def observe_chunk(chunk: Any) -> None:
  348. if self._on_chunk is not None:
  349. attempt.run_callback(self._on_chunk, _jsonable(chunk))
  350. self._runtime_lease = attempt.runtime.acquire_operation_lease()
  351. try:
  352. self._loop = loop = asyncio.new_event_loop()
  353. self._stream = loop.run_until_complete(
  354. attempt.run_managed(
  355. attempt.runtime.relay.llm.stream_execute, partial(self._provider_stream, attempt),
  356. observe_chunk, partial(self._relay_finalizer, attempt),
  357. )
  358. )
  359. except BaseException as exc:
  360. if self._loop is not None and self._recoverable_relay_failure(exc):
  361. self._preserve_pending_provider_chunks()
  362. return
  363. try:
  364. if self._loop is not None:
  365. self._finish_logical("cancelled" if _is_cancellation(exc) else "failed", error=exc)
  366. self._loop.close()
  367. finally:
  368. self._loop = None
  369. self._release_runtime_lease()
  370. raise
  371. def __iter__(self) -> ManagedLlmStream:
  372. return self
  373. def _prime_completed_response(self) -> None:
  374. """Advance once while preserving a genuine first chunk."""
  375. if not self._closed and not self._prefetched_chunks:
  376. with contextlib.suppress(StopIteration):
  377. self._prefetched_chunks.append(next(self))
  378. def _recoverable_relay_failure(self, exc: BaseException) -> bool:
  379. """Relay post-processing failed after the provider already succeeded.
  380. Not recoverable once Relay delivered output with no provider-source match:
  381. that chunk's source stays in ``_raw_chunks``, so a raw replay would emit
  382. already-represented content a second time (and unredacted, if the Relay
  383. transformation was the point)."""
  384. recoverable = isinstance(exc, Exception) and self._provider_completed and self._callback_error is None
  385. if recoverable and self._delivered_unmatched:
  386. logger.warning(
  387. "NeMo Relay stream post-processing failed after transformed output; "
  388. "propagating rather than replaying provider chunks",
  389. exc_info=True,
  390. )
  391. return False
  392. if recoverable:
  393. logger.warning(
  394. "NeMo Relay stream post-processing failed after provider success; preserving the provider result",
  395. exc_info=True,
  396. )
  397. return recoverable
  398. def _finish_logical(self, outcome: str, error: BaseException | None = None) -> None:
  399. """Complete the logical LLM scope unless the caller deferred it. An auxiliary stream
  400. that ended in ``error`` reports it like a deferred auxiliary call does."""
  401. if self._defer_logical_completion:
  402. return
  403. error_class = None
  404. if error is not None and self._logical_model_name is not None:
  405. from agent import auxiliary_call_outcome
  406. if auxiliary_call_outcome.is_cancellation(error):
  407. outcome = "cancelled"
  408. else:
  409. error_class = auxiliary_call_outcome.error_class(
  410. error, provider=self._logical_provider_name or "", model=self._logical_model_name)
  411. _complete_logical(
  412. self._logical, outcome=outcome, model_name=self._logical_model_name,
  413. provider_name=self._logical_provider_name, response_model_name=self._logical_response_model_name,
  414. operation_lease=self._runtime_lease, error_class=error_class,
  415. )
  416. self._logical = None
  417. def __next__(self) -> Any:
  418. if self._closed:
  419. raise StopIteration
  420. if self._prefetched_chunks:
  421. return self._prefetched_chunks.pop()
  422. if self._loop is None:
  423. chunk = next(self._stream, self) # self: exhausted sentinel
  424. if chunk is self or (self._accept_chunk is not None and not self._accept_chunk(chunk)):
  425. self._close(logical_outcome="cancelled")
  426. raise StopIteration
  427. return chunk
  428. async def next_chunk() -> Any:
  429. return await anext(self._stream)
  430. try:
  431. chunk = self._loop.run_until_complete(next_chunk())
  432. except StopAsyncIteration:
  433. if self._raw_chunks:
  434. self.output_modified = True
  435. self._finish_logical("success")
  436. self._close(logical_outcome="cancelled")
  437. raise StopIteration from None
  438. except BaseException as exc:
  439. callback_error = self._callback_error
  440. if callback_error is not None and relay_runtime._is_relay_wrapped_callback_error(exc, callback_error):
  441. self._close(logical_outcome="failed", error=callback_error)
  442. raise callback_error
  443. if self._recoverable_relay_failure(exc):
  444. self._preserve_pending_provider_chunks()
  445. return next(self)
  446. self._close(logical_outcome="cancelled" if _is_cancellation(exc) else "failed", error=exc)
  447. raise
  448. for index, (encoded, raw) in enumerate(self._raw_chunks):
  449. if _json_equal(chunk, encoded):
  450. if index > 0:
  451. self.output_modified = True
  452. del self._raw_chunks[: index + 1]
  453. return raw
  454. self.output_modified = True
  455. self._delivered_unmatched = True
  456. return self._chunk_adapter(chunk)
  457. def close(self) -> None:
  458. """Close an explicitly abandoned stream and cancel its logical call."""
  459. self._close(logical_outcome="cancelled")
  460. close_error, self._close_error = self._close_error, None
  461. if close_error is not None:
  462. raise close_error
  463. def _preserve_pending_provider_chunks(self) -> None:
  464. """Switch a failed Relay stream to its undelivered provider chunks."""
  465. pending = [raw for _encoded, raw in self._raw_chunks]
  466. self._raw_chunks.clear()
  467. loop, relay_stream = self._loop, self._stream
  468. self._loop, self._stream, self._raw_stream_resource, self._accept_chunk = None, iter(pending), None, None
  469. try:
  470. if loop is not None:
  471. try:
  472. completed = _aclose_on_loop(loop, relay_stream)
  473. except Exception:
  474. logger.debug("Relay stream cleanup failed during provider fallback", exc_info=True)
  475. completed = True
  476. if completed:
  477. loop.close()
  478. self._finish_logical("success")
  479. finally:
  480. self._release_runtime_lease()
  481. def _keep_first_close_error(self, exc: BaseException) -> None:
  482. if self._close_error is None:
  483. self._close_error = exc
  484. def _close_provider_resources(self) -> None:
  485. """Close the unmanaged provider stream/resource once each (they may be the same object)."""
  486. resources = {id(r): r for r in (self._stream, self._raw_stream_resource) if r is not None}
  487. self._stream = None
  488. self._raw_stream_resource = None
  489. for resource in resources.values():
  490. close = getattr(resource, "close", None)
  491. try:
  492. if callable(close):
  493. close()
  494. except Exception as exc:
  495. self._keep_first_close_error(exc)
  496. logger.debug("Provider stream cleanup failed", exc_info=True)
  497. def _close(self, *, logical_outcome: str, error: BaseException | None = None) -> None:
  498. if self._closed:
  499. return
  500. self._closed = True
  501. self._prefetched_chunks.clear()
  502. try:
  503. loop, self._loop = self._loop, None
  504. close_loop = loop is not None
  505. if loop is None:
  506. self._close_provider_resources()
  507. else:
  508. try:
  509. close_loop = _aclose_on_loop(loop, self._stream)
  510. except Exception as exc:
  511. self._keep_first_close_error(exc)
  512. self._finish_logical(logical_outcome, error)
  513. if close_loop:
  514. loop.close()
  515. finally:
  516. self._release_runtime_lease()
  517. def _release_runtime_lease(self) -> None:
  518. lease, self._runtime_lease = self._runtime_lease, None
  519. if lease is not None:
  520. lease.release()
  521. def __del__(self) -> None:
  522. self._close(logical_outcome="cancelled")
  523. stream = ManagedLlmStream
  524. _ANTHROPIC_APPEND_DELTAS = {"text_delta": "text", "thinking_delta": "thinking", "signature_delta": "signature"}
  525. class AnthropicStreamAccumulator:
  526. """Rebuild an Anthropic Message from post-intercept SSE events."""
  527. def __init__(self) -> None:
  528. self._message: dict[str, Any] = {}
  529. self._blocks: dict[int, dict[str, Any]] = {}
  530. def observe(self, event: Any) -> None:
  531. payload = _jsonable(event)
  532. if isinstance(payload, dict):
  533. handler = self._EVENT_HANDLERS.get(payload.get("type"))
  534. if handler is not None:
  535. handler(self, payload)
  536. def _on_message_start(self, payload: dict[str, Any]) -> None:
  537. message = payload.get("message")
  538. if isinstance(message, dict):
  539. self._message.update({k: message[k] for k in ("id", "type", "role", "model", "usage") if k in message})
  540. def _on_content_block_start(self, payload: dict[str, Any]) -> None:
  541. index, block = payload.get("index"), payload.get("content_block")
  542. if isinstance(index, int) and isinstance(block, dict):
  543. self._blocks[index] = dict(block)
  544. def _on_content_block_delta(self, payload: dict[str, Any]) -> None:
  545. index, delta = payload.get("index"), payload.get("delta")
  546. if not isinstance(index, int) or not isinstance(delta, dict):
  547. return
  548. block = self._blocks.setdefault(index, {})
  549. delta_type = delta.get("type")
  550. field = _ANTHROPIC_APPEND_DELTAS.get(delta_type)
  551. if field is not None:
  552. block[field] = str(block.get(field) or "") + str(delta.get(field) or "")
  553. elif delta_type == "input_json_delta":
  554. block["_partial_json"] = str(block.pop("_partial_json", "")) + str(delta.get("partial_json") or "")
  555. elif delta_type == "citations_delta" and "citation" in delta:
  556. block.setdefault("citations", []).append(delta["citation"])
  557. def _on_message_delta(self, payload: dict[str, Any]) -> None:
  558. delta = payload.get("delta")
  559. if isinstance(delta, dict):
  560. self._message.update({k: delta[k] for k in ("stop_reason", "stop_sequence", "stop_details") if k in delta})
  561. if "usage" in payload:
  562. usage, current_usage = payload["usage"], self._message.get("usage")
  563. if isinstance(current_usage, dict) and isinstance(usage, dict):
  564. # The SDK's MessageDeltaUsage serializes unset fields as null; they must not
  565. # erase message_start's input/cache counts.
  566. usage = {**current_usage, **{k: v for k, v in usage.items() if v is not None}}
  567. self._message["usage"] = usage
  568. _EVENT_HANDLERS = {
  569. "message_start": _on_message_start, "content_block_start": _on_content_block_start,
  570. "content_block_delta": _on_content_block_delta, "message_delta": _on_message_delta,
  571. }
  572. def finalize(self) -> dict[str, Any]:
  573. blocks = [dict(self._blocks[index]) for index in sorted(self._blocks)]
  574. for block in blocks:
  575. partial = block.pop("_partial_json", None)
  576. if partial is not None:
  577. with contextlib.suppress(TypeError, ValueError):
  578. partial = json.loads(partial)
  579. block["input"] = partial
  580. return {**self._message, "content": blocks}
  581. def response(self, base: Any = None) -> Any:
  582. """Return the attribute-shaped response consumed by Hermes."""
  583. assembled = self.finalize()
  584. content = assembled.pop("content", [])
  585. merged = {**_jsonable_dict(base), **assembled}
  586. if content or "content" not in merged:
  587. merged["content"] = content
  588. return _namespace(merged)
  589. def _logical_parent(
  590. runtime: relay_runtime.RelayRuntime, session: Any, parent: Any, metadata: dict[str, Any] | None
  591. ) -> _LogicalCall | None:
  592. """Return (turn, handle, request_id) for the turn's logical LLM scope, pushing it once."""
  593. turn = relay_runtime.active_turn(session.session_id)
  594. request_id = str((metadata or {}).get("api_request_id") or "")
  595. if turn is None or not request_id or turn.lease.host is not runtime:
  596. return None
  597. with turn.finalize_lock:
  598. if turn.closed:
  599. return None
  600. with turn.logical_llm_lock:
  601. handle = turn.logical_llm_calls.get(request_id)
  602. if handle is None:
  603. call_role = str((metadata or {}).get("call_role") or "primary")
  604. handle = turn.logical_llm_calls[request_id] = runtime.run_in_session(
  605. session, runtime.relay.scope.push, relay_runtime.LOGICAL_LLM_SCOPE,
  606. runtime.relay.ScopeType.Function, handle=parent, input={},
  607. metadata=relay_runtime.runtime_metadata(runtime.runtime_id, **{"hermes.call_role": call_role}),
  608. )
  609. return turn, handle, request_id
  610. def _complete_logical(
  611. logical: _LogicalCall | None, *, outcome: str, model_name: str | None = None, provider_name: str | None = None,
  612. response_model_name: str | None = None, operation_lease: relay_runtime.RelayOperationLease | None = None,
  613. error_class: str | None = None,
  614. ) -> None:
  615. if logical is None:
  616. return
  617. turn, handle, request_id = logical
  618. lease = turn.lease
  619. if not isinstance(lease.host, relay_runtime.RelayRuntime):
  620. return
  621. output = {"outcome": outcome}
  622. if model_name is not None and provider_name is not None:
  623. output.update({"model": model_name, "provider": provider_name})
  624. if response_model_name is not None:
  625. output["response_model"] = response_model_name
  626. if error_class is not None:
  627. output["error_class"] = error_class
  628. with turn.finalize_lock:
  629. with turn.logical_llm_lock:
  630. if turn.logical_llm_calls.get(request_id) is not handle:
  631. return
  632. if lease.session is None:
  633. return
  634. try:
  635. # Close through the top-guard: a sibling turn of the same session may hold a live
  636. # scope above this one, and popping through it would close the sibling's scope.
  637. # Letting the binding raise instead logs a traceback per overlap (#115471). The
  638. # skipped scope is reclaimed by the session-close drain (``_close_scope_handle``),
  639. # so the handle can still leave ``logical_llm_calls`` either way.
  640. popped = (operation_lease or lease.host).run_in_session(
  641. lease.session, relay_runtime.pop_relay_scope_if_top, lease.host.relay, handle,
  642. output=output, metadata=relay_runtime.runtime_metadata(lease.host.runtime_id),
  643. )
  644. except Exception:
  645. # Provider result is authoritative; retain the handle so turn finalization can retry.
  646. logger.warning("Hermes Relay logical LLM finalization failed", exc_info=True)
  647. return
  648. if popped is False:
  649. # A call running beside the turn (title generation) finishes under the turn's own
  650. # live scopes; keep its result so the drain reports it instead of an orphan.
  651. lease.host.defer_scope_output(handle, output)
  652. logger.debug(
  653. "Left logical LLM scope %s under a concurrent scope; the drain closes it with its result",
  654. request_id,
  655. )
  656. with turn.logical_llm_lock:
  657. if turn.logical_llm_calls.get(request_id) is handle:
  658. del turn.logical_llm_calls[request_id]
  659. def _is_cancellation(error: BaseException) -> bool:
  660. return isinstance(error, (asyncio.CancelledError, InterruptedError, KeyboardInterrupt))
  661. def complete_logical_call(
  662. api_request_id: str, *, outcome: str, model_name: str | None = None,
  663. provider_name: str | None = None, response_model_name: str | None = None,
  664. error_class: str | None = None,
  665. ) -> None:
  666. """Complete the active turn's logical LLM call after caller validation. ``error_class``
  667. (a classifier reason) rides on a call that reports its route: the error that ended a
  668. failed call, or the last one a successful call recovered from."""
  669. turn = relay_runtime.active_turn()
  670. if turn is None or not api_request_id:
  671. return
  672. with turn.logical_llm_lock:
  673. handle = turn.logical_llm_calls.get(api_request_id)
  674. if handle is not None:
  675. _complete_logical(
  676. (turn, handle, api_request_id), outcome=outcome, model_name=model_name,
  677. provider_name=provider_name, response_model_name=response_model_name, error_class=error_class,
  678. )
  679. def _response_model_name(response: Any) -> str | None:
  680. """Return a provider-reported model name when one is available."""
  681. value = response.get("model") if isinstance(response, dict) else getattr(response, "model", None)
  682. return value if isinstance(value, str) and value.strip() else None
  683. def _provider_request(
  684. original: dict[str, Any], request: Any, *, relay_request_body: dict[str, Any],
  685. codec_baseline_body: dict[str, Any] | None, metadata: dict[str, Any] | None,
  686. ) -> dict[str, Any]:
  687. content = getattr(request, "content", request)
  688. if not isinstance(content, dict):
  689. content = relay_request_body
  690. final = dict(original)
  691. if codec_baseline_body is not None and not _json_equal(content, relay_request_body):
  692. baseline = codec_baseline_body
  693. intercepted = _provider_request_body(content, metadata)
  694. # Typed codecs may not represent provider-specific fields: overlay only values that changed
  695. # from the codec-facing baseline so unrelated intercepts cannot delete/normalize unknown arguments.
  696. for key in baseline.keys() | intercepted.keys():
  697. if key not in intercepted:
  698. final.pop(key, None)
  699. elif key not in baseline or not _json_equal(intercepted[key], baseline[key]):
  700. final[key] = intercepted[key]
  701. _restore_provider_message_extensions(original, final, baseline=baseline, intercepted=intercepted)
  702. headers = getattr(request, "headers", None)
  703. if isinstance(headers, dict):
  704. headers = {k: v for k, v in headers.items() if str(k).lower() not in _RELAY_INTERNAL_PROVIDER_HEADERS}
  705. # Relay's managed-call trace header maps to ``extra_headers`` for known SDK adapters and custom
  706. # requests that already use that container; other native transports take protocol kwargs directly
  707. # and may reject an SDK-only argument. Non-trace middleware headers are preserved as before.
  708. supports_extra_headers = _RELAY_PROTOCOL_BY_API_MODE.get(_api_mode(metadata)) is not None or "extra_headers" in original
  709. if headers and not supports_extra_headers:
  710. headers = {k: v for k, v in headers.items() if str(k).lower() != "traceparent"}
  711. if headers:
  712. final["extra_headers"] = {**dict(final.get("extra_headers") or {}), **headers}
  713. return final
  714. def _rewrite_tools(body: dict[str, Any], match: Callable[[dict], bool], rewrite: Callable[[dict], dict]) -> None:
  715. """Rewrite each dict tool that ``match``es (in place on ``body["tools"]`` when it is a list)."""
  716. tools = body.get("tools")
  717. if isinstance(tools, list):
  718. body["tools"] = [rewrite(t) if isinstance(t, dict) and match(t) else t for t in tools]
  719. def _codex_codec_tools(body: dict[str, Any]) -> None:
  720. # The Responses SDK accepts ``tools=None`` as "no tools" while Relay's typed codec
  721. # wants an array or an absent field; only the codec-facing copy is normalized.
  722. if body.get("tools") is None:
  723. body.pop("tools", None)
  724. _rewrite_tools(
  725. body, lambda t: t.get("type") == "function" and "function" not in t,
  726. lambda t: {"type": "function", "function": {k: v for k, v in t.items() if k != "type"}},
  727. )
  728. def _chat_codec_tools(body: dict[str, Any]) -> None:
  729. _rewrite_tools(body, lambda t: "function" in t and "type" not in t, lambda t: {"type": "function", **t})
  730. # api_mode -> in-place normalizer producing the codec-facing ``tools`` shape.
  731. _CODEC_TOOL_NORMALIZERS = {"codex_responses": _codex_codec_tools, "chat_completions": _chat_codec_tools}
  732. def _relay_request_body(request: dict[str, Any], metadata: dict[str, Any] | None) -> dict[str, Any]:
  733. body = _jsonable_dict(request)
  734. # ``timeout`` configures the SDK client, not the wire: never expose it to intercepts.
  735. body.pop("timeout", None)
  736. normalize = _CODEC_TOOL_NORMALIZERS.get(_api_mode(metadata))
  737. if normalize is not None:
  738. normalize(body)
  739. return body
  740. def _restore_provider_message_extensions(
  741. original: dict[str, Any], final: dict[str, Any], *, baseline: dict[str, Any], intercepted: dict[str, Any],
  742. ) -> None:
  743. """Restore provider wire fields that Relay's typed codec cannot represent."""
  744. message_lists = tuple(body.get("messages") for body in (original, final, baseline, intercepted))
  745. if not all(isinstance(m, list) for m in message_lists) or len({len(m) for m in message_lists}) != 1:
  746. return
  747. for messages in zip(*message_lists, strict=True):
  748. if not all(isinstance(message, dict) for message in messages):
  749. continue
  750. original_message, final_message, baseline_message, intercepted_message = messages
  751. for key in _PROVIDER_MESSAGE_EXTENSION_KEYS:
  752. if key in original_message and not any(
  753. key in m for m in (baseline_message, intercepted_message, final_message)
  754. ):
  755. final_message[key] = original_message[key]
  756. def _codec_round_trip_request_body(
  757. relay: Any, relay_request: Any, *, relay_request_body: dict[str, Any], metadata: dict[str, Any] | None,
  758. ) -> dict[str, Any] | None:
  759. """Return the codec-only request shape used to identify real rewrites."""
  760. codec = _codec(relay, metadata)
  761. if codec is None:
  762. return _provider_request_body(relay_request_body, metadata)
  763. try:
  764. encoded = codec.encode(codec.decode(relay_request), relay_request)
  765. content = getattr(encoded, "content", encoded)
  766. except Exception:
  767. logger.warning("NeMo Relay request codec baseline failed; ignoring request rewrites", exc_info=True)
  768. return None
  769. if isinstance(content, dict):
  770. return _provider_request_body(content, metadata)
  771. logger.warning("NeMo Relay request codec returned an unsupported baseline; ignoring request rewrites")
  772. return None
  773. def _provider_request_body(content: dict[str, Any], metadata: dict[str, Any] | None) -> dict[str, Any]:
  774. body = dict(content)
  775. if _api_mode(metadata) == "codex_responses":
  776. _rewrite_tools(
  777. body, lambda t: t.get("type") == "function" and isinstance(t.get("function"), dict),
  778. lambda t: {"type": "function", **dict(t["function"])},
  779. )
  780. return body
  781. def _codec(relay: Any, metadata: dict[str, Any] | None) -> Any:
  782. protocol = _RELAY_PROTOCOL_BY_API_MODE.get(_api_mode(metadata))
  783. codec = getattr(getattr(relay, "codecs", None), protocol[1], None) if protocol is not None else None
  784. return codec() if callable(codec) else None
  785. def _jsonable(value: Any) -> Any:
  786. if value is None or isinstance(value, (str, int, float, bool)):
  787. return value
  788. if isinstance(value, dict):
  789. return {str(key): _jsonable(item) for key, item in value.items()}
  790. if isinstance(value, (list, tuple, set)):
  791. return [_jsonable(item) for item in value]
  792. model_dump = getattr(type(value), "model_dump", None)
  793. if callable(model_dump):
  794. try:
  795. # warnings=False: pydantic's generic-union warning would leak to the terminal
  796. # mid-response; TypeError = duck-typed model_dump without pydantic's signature.
  797. try:
  798. return _jsonable(value.model_dump(mode="json", warnings=False))
  799. except TypeError:
  800. return _jsonable(value.model_dump())
  801. except Exception:
  802. pass
  803. try:
  804. attributes = {str(key): item for key, item in vars(value).items() if not str(key).startswith("_")}
  805. except (TypeError, AttributeError):
  806. return str(value)
  807. return _jsonable(attributes) if attributes else str(value)
  808. def _jsonable_dict(value: Any) -> dict[str, Any]:
  809. """``_jsonable`` for values that must be a JSON object; anything else becomes ``{}``."""
  810. payload = _jsonable(value)
  811. return payload if isinstance(payload, dict) else {}
  812. def _namespace(value: Any) -> Any:
  813. if isinstance(value, dict):
  814. return SimpleNamespace(**{str(key): _namespace(item) for key, item in value.items()})
  815. if isinstance(value, list):
  816. return [_namespace(item) for item in value]
  817. return value
  818. def _canonical_json(value: Any, encode: Callable[[Any], Any] = _jsonable) -> str:
  819. return json.dumps(encode(value), sort_keys=True, separators=(",", ":"))
  820. def _json_equal(left: Any, right: Any) -> bool:
  821. try:
  822. return _canonical_json(left) == _canonical_json(right)
  823. except (TypeError, ValueError):
  824. return False
  825. def _run_awaitable(
  826. value: Any, *, loop_error: str = "Synchronous Relay LLM execution cannot run on an event-loop thread",
  827. ) -> Any:
  828. if not inspect.isawaitable(value):
  829. return value
  830. if _has_running_event_loop():
  831. raise RuntimeError(loop_error)
  832. return asyncio.run(value)