chat_completion_helpers.py 219 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001
  1. """API-call helpers extracted from :class:`AIAgent`: non-streaming and streaming
  2. request drivers, request kwargs builder, assistant-message materializer,
  3. provider-fallback activator, max-iterations handler, per-turn resource cleanup.
  4. Each function takes the parent ``AIAgent`` as ``agent``; AIAgent keeps thin
  5. forwarders. Symbols tests patch on ``run_agent`` (``cleanup_vm`` /
  6. ``cleanup_browser``) are resolved through :func:`_ra` at call time.
  7. """
  8. from __future__ import annotations
  9. import contextlib
  10. import contextvars
  11. import json
  12. import logging
  13. import math
  14. import os
  15. import re
  16. import sys
  17. import threading
  18. import time
  19. import uuid
  20. from dataclasses import dataclass
  21. from types import SimpleNamespace
  22. from typing import Any, Dict, Optional
  23. from hermes_cli.timeouts import get_provider_request_timeout, get_provider_stale_timeout
  24. from hermes_constants import PARTIAL_STREAM_STUB_ID, FINISH_REASON_LENGTH
  25. from agent.error_classifier import (
  26. FailoverReason, PROVIDER_STREAM_EMPTY_FRAME_ERROR_CODE, PROVIDER_STREAM_NON_JSON_ERROR_CODE,
  27. _extract_status_code)
  28. from agent.sdk_transform_bypass import bypass_chat_sdk_request_transform
  29. from agent.errors import EmptyStreamError
  30. from agent.chat_completion_local import _streams_llamacpp, stale_giveup
  31. from agent.chat_completion_headers import _chat_completion_capturing_headers
  32. from agent.chat_completion_stream_monitor import StreamingWaitMonitor, StreamLifetimeExceeded
  33. from agent.transports.chat_completions import is_router_timeout_shim, router_timeout_shim_may_follow
  34. from agent.fast_mode import effective_request_overrides
  35. from agent.turn_context import substitute_api_content
  36. from agent.gemini_native_adapter import is_native_gemini_base_url
  37. # Remote endpoints must never be fingerprinted: the probe waterfall is only valid for local/LM-Studio/Ollama
  38. # boxes. Non-Ollama remotes (sglang, vLLM, OpenAI-compat) expose Ollama-compat endpoints that can
  39. # misidentify and, without an api_key, return 401 on every leg (issue #89863).
  40. from agent.model_metadata import is_local_endpoint
  41. from agent.message_content import flatten_message_text
  42. from agent.message_metadata import PERSISTENCE_ONLY_MESSAGE_FIELDS, append_message, data_use_requirement, stamp_message_timestamp
  43. from agent.message_sanitization import (
  44. _sanitize_messages_surrogates, _sanitize_surrogates, _repair_tool_call_arguments,
  45. normalize_finish_reason as _normalize_finish_reason, sanitize_outbound_kwargs, strip_images_for_rejecting_model,
  46. )
  47. from agent.reasoning_summaries import (
  48. append_streamed_reasoning_detail, separate_glued_reasoning_blocks, streamed_reasoning_detail_text)
  49. from agent.local_prompt_progress import prompt_progress, with_prompt_progress
  50. from agent.repetition_guard import RunawayStreamWatch, is_repetition_dominated
  51. from agent.stream_single_writer import claim_stream_writer, stream_writer_is_current
  52. from tools.terminal_tool_lifecycle import is_persistent_env
  53. from utils import base_url_host_matches, base_url_hostname, env_float, env_int
  54. logger = logging.getLogger(__name__)
  55. _OPENROUTER_PROVIDER_SORT_VALUES = {"throughput", "latency", "price"}
  56. _PROVIDER_STREAM_ERROR_FINISH_REASONS = {"error", "error_finish"}
  57. _PROVIDER_STREAM_SSE_FIELDS = {"event", "data", "id", "retry"}
  58. _PROVIDER_STREAM_ERROR_TEXT_LIMIT = 4096
  59. # Fallback chain exhausted on a non-rate-limit failure (#24996): arm a short
  60. # cooldown so the NEXT turn's restore_primary_runtime stays gated instead of
  61. # resetting _fallback_index=0 and re-marshaling the whole context across every
  62. # provider again (memory/swap exhaustion on constrained hosts). Rate-limit /
  63. # billing reasons keep their own longer cooldown.
  64. _FALLBACK_EXHAUSTED_COOLDOWN_S = 5.0
  65. # Streaming 5xx unmask probe: one non-streaming re-issue per this window. Covers the
  66. # outer retry loop (up to ~3 attempts x backoff, well under 60s) so an outage doesn't
  67. # double traffic every attempt, while later turns re-arm automatically.
  68. _STREAM_5XX_PROBE_WINDOW_S = 60.0
  69. def _context_thread_target(callback):
  70. """Bind a no-argument thread target to the caller's ContextVars."""
  71. context = contextvars.copy_context()
  72. return lambda: context.run(callback)
  73. def _join_worker_for_relay_teardown(worker, *, label: str) -> None:
  74. """Bounded worker join before raising InterruptedError (#81521).
  75. Raising immediately lets turn teardown race a still-open Relay LLM scope and
  76. corrupt the LIFO stack (CLI EIO / redraw storm). Only joins when Relay managed
  77. execution is live — otherwise the join would just delay interrupt detection.
  78. """
  79. try:
  80. from agent import relay_runtime
  81. runtime = relay_runtime.get_runtime(create=False)
  82. if runtime is None or not runtime.managed_execution_enabled():
  83. return
  84. except Exception:
  85. return
  86. worker.join(timeout=2.0)
  87. if worker.is_alive():
  88. logger.warning("%s worker still alive after interrupt abort (2.0s join "
  89. "timeout); Relay teardown will best-effort drain orphaned scopes (#81521).", label)
  90. def _ra():
  91. """Lazy ``run_agent`` reference so ``patch("run_agent.cleanup_vm")`` etc. intercept."""
  92. import run_agent
  93. return run_agent
  94. class ProviderStreamError(Exception):
  95. """Provider encoded an API error as streaming content instead of an SDK error."""
  96. def __init__(self, *, status_code: Optional[int], body: dict, raw_text: str, headers: Any = None):
  97. self.status_code = status_code
  98. self.body = body
  99. self.raw_text = raw_text
  100. self.response = SimpleNamespace(headers=headers or {})
  101. super().__init__(self._format_message())
  102. def _format_message(self) -> str:
  103. error_obj = self.body.get("error", {}) if isinstance(self.body, dict) else {}
  104. if not isinstance(error_obj, dict):
  105. error_obj = {}
  106. parts = ["Provider stream returned an error event"]
  107. if self.status_code:
  108. parts.append(f"HTTP {self.status_code}")
  109. if error_obj.get("code"):
  110. parts.append(str(error_obj["code"]))
  111. text = " - ".join(parts)
  112. if error_obj.get("message"):
  113. text += f": {error_obj['message']}"
  114. return text
  115. def _status_code_from_value(value: Any) -> Optional[int]:
  116. if isinstance(value, int) and 100 <= value < 600:
  117. return value
  118. if not isinstance(value, str):
  119. return None
  120. match = re.search(r"(?:HTTP_STATUS/)?\b([1-5]\d\d)\b", value, re.IGNORECASE)
  121. return int(match.group(1)) if match else None
  122. def _status_code_from_payload(payload: Any) -> Optional[int]:
  123. if not isinstance(payload, dict):
  124. return None
  125. candidates = [payload.get(k) for k in ("status_code", "status", "http_status")]
  126. error_obj = payload.get("error")
  127. if isinstance(error_obj, dict):
  128. candidates.extend(error_obj.get(k) for k in ("status_code", "status", "http_status", "code"))
  129. candidates.append(payload.get("code"))
  130. for candidate in candidates:
  131. status_code = _status_code_from_value(candidate)
  132. if status_code is not None:
  133. return status_code
  134. return None
  135. def _json_object_from_text(text: str) -> Optional[dict]:
  136. stripped = (text or "").strip()
  137. with contextlib.suppress(json.JSONDecodeError, TypeError):
  138. if stripped.startswith("{"):
  139. decoded = json.loads(stripped)
  140. return decoded if isinstance(decoded, dict) else None
  141. return None
  142. def _parse_provider_sse_events(text: str) -> list[dict]:
  143. """Parse provider text that looks like Server-Sent Events."""
  144. events: list[dict] = []
  145. current = {"event": None, "data": [], "comments": [], "fields": {}}
  146. def _flush_current():
  147. nonlocal current
  148. if any(current.values()):
  149. status_candidates = list(current["comments"]) + [
  150. current["fields"][key]
  151. for key in ("status", "status_code", "http_status")
  152. if key in current["fields"]
  153. ]
  154. events.append({
  155. "event": current["event"],
  156. "data": "\n".join(current["data"]),
  157. "comments": list(current["comments"]),
  158. "fields": dict(current["fields"]),
  159. "status_code": next(
  160. (s for s in map(_status_code_from_value, status_candidates) if s is not None), None),
  161. })
  162. current = {"event": None, "data": [], "comments": [], "fields": {}}
  163. for raw_line in (text or "").splitlines():
  164. line = raw_line.rstrip("\r")
  165. if line == "":
  166. _flush_current()
  167. continue
  168. if line.startswith(":"):
  169. current["comments"].append(line[1:].strip())
  170. continue
  171. field, sep, value = line.partition(":")
  172. if not sep:
  173. current["fields"][field.strip().lower()] = ""
  174. continue
  175. field = field.strip().lower()
  176. value = value.removeprefix(" ")
  177. if field == "event":
  178. current["event"] = value.strip()
  179. elif field == "data":
  180. current["data"].append(value)
  181. else:
  182. current["fields"][field] = value
  183. _flush_current()
  184. return events
  185. def _provider_error_body(payload: dict, status_code: Optional[int]) -> dict:
  186. """Normalize common provider error payloads to OpenAI-style body.error."""
  187. if not isinstance(payload, dict):
  188. payload = {}
  189. elif isinstance(payload.get("error"), dict):
  190. return payload
  191. code = (payload.get("code") or payload.get("error_code") or payload.get("type")
  192. or (f"HTTP_{status_code}" if status_code else "provider_stream_error"))
  193. message = (payload.get("message") or payload.get("error_description") or payload.get("error")
  194. or "Provider stream returned an error event.")
  195. normalized_error = {"message": str(message)}
  196. if code:
  197. normalized_error["code"] = str(code)
  198. for key in ("request_id", "param", "type"):
  199. if payload.get(key):
  200. normalized_error[key] = payload[key]
  201. return {"error": normalized_error}
  202. def _provider_stream_error_from_json_decode_error(error: json.JSONDecodeError, *,
  203. response: Any = None) -> ProviderStreamError:
  204. """Preserve plain-text SSE data rejected inside the OpenAI SDK: on a non-JSON
  205. ``event: error`` the SDK raises from ``sse.json()`` before yielding a chunk,
  206. but ``JSONDecodeError.doc`` still carries the provider's original message.
  207. An EMPTY ``doc`` is the other case: the frame carried no payload at all
  208. (``data:`` / ``event: ping`` / ``id:`` alone — legal SSE keepalives and no-ops),
  209. which the SDK's ``json.loads`` rejects the same way. A gateway that is degrading
  210. answers EVERY streaming request with such frames, so this is not the provider's
  211. malformed payload and must not be reported as one: it gets its own code and
  212. the stream helper recovers by retrying without streaming."""
  213. from agent.redact import redact_sensitive_text
  214. raw_text = str(getattr(error, "doc", "") or "").strip()
  215. headers = getattr(response, "headers", None) if response is not None else None
  216. if not raw_text:
  217. return ProviderStreamError(
  218. status_code=None,
  219. body=_provider_error_body(
  220. {"code": PROVIDER_STREAM_EMPTY_FRAME_ERROR_CODE,
  221. "message": "Provider stream returned an empty SSE data frame (keepalive with no payload)."},
  222. None,
  223. ),
  224. raw_text="",
  225. headers=headers,
  226. )
  227. safe_text = redact_sensitive_text(_sanitize_surrogates(raw_text), force=True)
  228. safe_text = safe_text[:_PROVIDER_STREAM_ERROR_TEXT_LIMIT]
  229. return ProviderStreamError(
  230. status_code=None,
  231. body=_provider_error_body(
  232. {"code": PROVIDER_STREAM_NON_JSON_ERROR_CODE,
  233. "message": safe_text or "Provider stream returned non-JSON SSE data."},
  234. None,
  235. ),
  236. raw_text=safe_text,
  237. headers=headers,
  238. )
  239. def _is_provider_stream_empty_frame_error(exc: BaseException) -> bool:
  240. """True for the translated contentless-SSE-frame error. Re-streaming cannot help
  241. (a degraded gateway answers every stream that way), so the caller must change channel."""
  242. body = getattr(exc, "body", None)
  243. error_obj = body.get("error") if isinstance(body, dict) else None
  244. return isinstance(error_obj, dict) and error_obj.get("code") == PROVIDER_STREAM_EMPTY_FRAME_ERROR_CODE
  245. def _iter_provider_stream_chunks(stream, *, response: Any = None):
  246. """Yield SDK chunks while translating SDK-level SSE decode failures."""
  247. try:
  248. yield from stream
  249. except json.JSONDecodeError as error:
  250. stream_response = response() if callable(response) else response
  251. if stream_response is None:
  252. stream_response = getattr(stream, "response", None)
  253. raise _provider_stream_error_from_json_decode_error(error, response=stream_response) from error
  254. def _payload_has_error_shape(payload: Any) -> bool:
  255. if not isinstance(payload, dict):
  256. return False
  257. if isinstance(payload.get("error"), (dict, str)):
  258. return True
  259. return bool(payload.get("message")) and bool(
  260. payload.get("code") or payload.get("error_code") or _status_code_from_payload(payload) is not None)
  261. def _provider_stream_text_may_be_sse(text: str) -> bool:
  262. """Return True while pending text still looks like an SSE control block."""
  263. stripped = (text or "").lstrip()
  264. if not stripped:
  265. return False
  266. lines = stripped.splitlines()
  267. trailing_newline = stripped.endswith(("\n", "\r"))
  268. saw_sse_field = False
  269. for index, raw_line in enumerate(lines):
  270. line = raw_line.rstrip("\r")
  271. if line == "":
  272. continue
  273. if line.startswith(":"):
  274. saw_sse_field = True
  275. continue
  276. field, sep, _value = line.partition(":")
  277. field_name = field.strip().lower()
  278. if sep and field_name in _PROVIDER_STREAM_SSE_FIELDS:
  279. saw_sse_field = True
  280. continue
  281. is_last_incomplete = index == len(lines) - 1 and not trailing_newline
  282. if is_last_incomplete and any(
  283. sse_field.startswith(field_name) for sse_field in _PROVIDER_STREAM_SSE_FIELDS):
  284. return True
  285. return False
  286. return saw_sse_field
  287. def _provider_stream_error_from_text(text: str, finish_reason: Optional[str], *,
  288. response: Any = None) -> Optional[ProviderStreamError]:
  289. """Convert provider-streamed error text into an exception for retry logic."""
  290. if not text:
  291. return None
  292. if str(finish_reason or "").lower() not in _PROVIDER_STREAM_ERROR_FINISH_REASONS:
  293. return None
  294. headers = getattr(response, "headers", None) if response is not None else None
  295. def _error(payload: dict, status_code: Optional[int]) -> ProviderStreamError:
  296. return ProviderStreamError(status_code=status_code, body=_provider_error_body(payload, status_code),
  297. raw_text=text, headers=headers)
  298. for event in _parse_provider_sse_events(text):
  299. is_error_event = str(event.get("event") or "").strip().lower() == "error"
  300. payload = _json_object_from_text(event.get("data") or "") or {}
  301. status_code = event.get("status_code") or _status_code_from_payload(payload)
  302. # The finish_reason is an error here, so an error event always qualifies;
  303. # a non-error event needs an error-shaped payload or an HTTP error code.
  304. if (status_code is not None and status_code >= 400) or is_error_event or _payload_has_error_shape(payload):
  305. return _error(payload, status_code)
  306. payload = _json_object_from_text(text)
  307. if payload is not None:
  308. return _error(payload, _status_code_from_payload(payload))
  309. if text.strip():
  310. return _error({}, None)
  311. return None
  312. _IMAGE_PART_TYPES = frozenset({"image_url", "input_image", "image"})
  313. def _image_part_chars(part: dict[str, Any], image_cost: int) -> int:
  314. """Char-equivalent of one image content part: the per-image cost learned from provider usage
  315. (x4 chars/token), never the base64 payload length. A single native screenshot priced as text
  316. read as ~100K+ tokens and selected the giant-conversation watchdog tiers (#63871, #76411)."""
  317. text = part.get("text")
  318. return image_cost * 4 + (len(text) if isinstance(text, str) else 0)
  319. def _payload_chars(value: Any, image_cost: int) -> int:
  320. """``len(str(value))`` with image content parts priced at ``image_cost`` tokens each."""
  321. if value is None:
  322. return 0
  323. if isinstance(value, dict):
  324. part_type = value.get("type")
  325. # JSON-Schema nodes may hold a sub-schema (``properties.type``) or a multi-type list
  326. # under the "type" key; only scalar content-part types can ever match (#104793).
  327. if isinstance(part_type, str) and part_type in _IMAGE_PART_TYPES and any(k in value for k in ("image_url", "image", "source", "file_id")):
  328. return _image_part_chars(value, image_cost)
  329. return sum(len(str(k)) + 6 + _payload_chars(v, image_cost) for k, v in value.items())
  330. if isinstance(value, list):
  331. return sum(_payload_chars(item, image_cost) for item in value) + 2 * len(value)
  332. return len(str(value))
  333. def estimate_request_context_tokens(api_payload: Any) -> int:
  334. """Cheap char/4 context estimate for the stale-call detectors. Handles both
  335. wire shapes so Codex turns don't report ~0 tokens: list -> Chat ``messages``;
  336. dict with ``messages`` (+``tools``); dict with ``input`` (Responses API,
  337. +``instructions``/``tools``); any other dict -> sum of its values. Image parts
  338. cost the learned per-image price, not their base64 length."""
  339. from agent.image_token_cost import current_image_token_cost
  340. image_cost = current_image_token_cost()
  341. def _chars(value: Any) -> int:
  342. return _payload_chars(value, image_cost)
  343. if isinstance(api_payload, list):
  344. return sum(_chars(item) for item in api_payload) // 4
  345. if not isinstance(api_payload, dict):
  346. return _chars(api_payload) // 4
  347. messages = api_payload.get("messages")
  348. if isinstance(messages, list):
  349. total_chars = sum(_chars(item) for item in messages)
  350. if "tools" in api_payload:
  351. total_chars += _chars(api_payload.get("tools"))
  352. return total_chars // 4
  353. if "input" in api_payload:
  354. return sum(_chars(api_payload.get(k)) for k in ("input", "instructions", "tools")) // 4
  355. return sum(_chars(value) for value in api_payload.values()) // 4
  356. def _is_openai_codex_backend(agent) -> bool:
  357. from agent.codex_responses_adapter import classify_responses_route
  358. return classify_responses_route(agent).is_codex_backend
  359. def openai_codex_stale_timeout_floor(est_tokens: int) -> float:
  360. """Minimum wall-clock stale timeout for openai-codex by estimated context:
  361. subscription-backed Codex can spend minutes in admission/prefill on
  362. gateway-scale payloads, so the generic default would abort healthy calls.
  363. The floor engages above 10k estimated tokens."""
  364. for threshold, floor in ((100_000, 1200.0), (50_000, 900.0), (10_000, 600.0)):
  365. if est_tokens > threshold:
  366. return floor
  367. return 0.0
  368. def _bound_openai_codex_stale_timeout(stale_timeout: float, est_tokens: int) -> float:
  369. """Apply the openai-codex stale bounds: raise to ``openai_codex_stale_timeout_floor``
  370. so healthy gateway-scale requests aren't aborted mid-prefill, then clamp to the flat
  371. HERMES_CODEX_HARD_TIMEOUT_SECONDS ceiling (#64507, default 1500s — above the max
  372. floor, a backstop for a request that emits SOME events then wedges; 0 disables).
  373. Shared by the worker watchdogs and the inline cron path (#69734)."""
  374. floor = openai_codex_stale_timeout_floor(est_tokens)
  375. if floor:
  376. stale_timeout = max(stale_timeout, floor)
  377. hard_timeout = env_float("HERMES_CODEX_HARD_TIMEOUT_SECONDS", 1500.0)
  378. return min(stale_timeout, hard_timeout) if hard_timeout > 0 else stale_timeout
  379. def _validated_openrouter_provider_sort(raw_sort: Any) -> Optional[str]:
  380. """Return a normalized OpenRouter provider.sort value or None."""
  381. if not isinstance(raw_sort, str):
  382. return None
  383. sort_value = raw_sort.strip().lower()
  384. if not sort_value:
  385. return None
  386. if sort_value in _OPENROUTER_PROVIDER_SORT_VALUES:
  387. return sort_value
  388. logger.warning("Ignoring invalid OpenRouter provider.sort value %r (allowed: %s)", raw_sort,
  389. ", ".join(sorted(_OPENROUTER_PROVIDER_SORT_VALUES)))
  390. return None
  391. def _provider_preferences_for_agent(agent) -> dict[str, Any]:
  392. """Build the validated provider-routing object shared by request paths.
  393. ``provider_routing.models.<id>`` overlays the flat constructor values for the CURRENT
  394. ``agent.model`` (so ``/model`` switches, fallbacks, and delegated children on another
  395. model each get their own pins without any surface re-plumbing the kwargs)."""
  396. flat = {"only": agent.providers_allowed, "ignore": agent.providers_ignored, "order": agent.providers_order,
  397. "sort": agent.provider_sort, "require_parameters": agent.provider_require_parameters,
  398. "data_collection": agent.provider_data_collection}
  399. per_model = {}
  400. with contextlib.suppress(Exception):
  401. from hermes_cli.config import load_config_readonly
  402. from hermes_constants import resolve_per_model_provider_routing
  403. _pr = load_config_readonly().get("provider_routing")
  404. per_model = resolve_per_model_provider_routing(agent.model, (_pr or {}).get("models") if isinstance(_pr, dict) else None)
  405. merged = {**flat, **{k: v for k, v in per_model.items() if k in flat}}
  406. merged["sort"] = _validated_openrouter_provider_sort(merged["sort"])
  407. merged["require_parameters"] = True if merged["require_parameters"] else None
  408. return {key: value for key, value in merged.items() if value}
  409. def _prompt_cache_scope_for_agent(agent) -> str | None:
  410. """Rotation-stable logical cache scope for *agent*, or None (transports then
  411. fall back to the physical session_id, so a failure never blocks the build)."""
  412. try:
  413. from agent.prompt_cache_scope import resolve_prompt_cache_scope_safe
  414. return resolve_prompt_cache_scope_safe(agent)
  415. except Exception:
  416. logger.debug("prompt-cache scope resolution failed", exc_info=True)
  417. return None
  418. def _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs: dict, api_messages: list) -> dict:
  419. """The Nous profile body on the Messages wire, which the profile hook never reaches; never
  420. ``provider_preferences`` (OpenAI-wire routing the Portal 400s on)."""
  421. if getattr(agent, "provider", None) not in {"nous", "nous-portal", "nousresearch"}:
  422. return anthropic_kwargs
  423. try:
  424. from providers import get_provider_profile
  425. nous_profile = get_provider_profile("nous")
  426. if nous_profile is not None:
  427. anthropic_kwargs.setdefault("extra_body", {}).update(nous_profile.build_extra_body(
  428. session_id=getattr(agent, "session_id", None), data_use=data_use_requirement(api_messages)))
  429. except Exception as exc:
  430. logger.debug("Nous Portal extra_body merge failed: %s", exc)
  431. return anthropic_kwargs
  432. def _estimate_chunk_bytes(chunk: Any) -> int:
  433. """Cheap per-chunk size estimate for the stream diagnostic counters: delta
  434. string lengths plus a framing floor (~3x cheaper than ``len(repr(chunk))``
  435. in the agent's hottest loop). Unknown shapes just keep the floor."""
  436. size = 40 # SSE/JSON framing floor per chunk
  437. def _add(obj, *attrs):
  438. nonlocal size
  439. for attr in attrs:
  440. v = getattr(obj, attr, None)
  441. if isinstance(v, str):
  442. size += len(v)
  443. with contextlib.suppress(Exception):
  444. choices = getattr(chunk, "choices", None)
  445. if choices:
  446. delta = getattr(choices[0], "delta", None)
  447. if delta is not None:
  448. _add(delta, "content", "reasoning_content", "reasoning")
  449. for tc in getattr(delta, "tool_calls", None) or ():
  450. fn = getattr(tc, "function", None)
  451. if fn is not None:
  452. _add(fn, "arguments", "name")
  453. else:
  454. _add(getattr(chunk, "delta", None), "text", "partial_json")
  455. return size
  456. # ── Cross-turn stale-call circuit breaker (#58962) ─────────────────────
  457. # A session wedged against an unresponsive provider would otherwise hit the
  458. # stale detector on every call forever. ``agent._consecutive_stale_streams``
  459. # is bumped on every stale kill and reset only when a call completes or the
  460. # provider is swapped (switch_model / try_activate_fallback /
  461. # restore_primary_runtime — the streak measured the OLD provider). Past the
  462. # give-up threshold, calls abort immediately with an actionable error.
  463. def _stale_streak(agent) -> int:
  464. try:
  465. return int(getattr(agent, "_consecutive_stale_streams", 0) or 0)
  466. except Exception:
  467. return 0
  468. def _bump_stale_streak(agent) -> None:
  469. with contextlib.suppress(Exception):
  470. agent._consecutive_stale_streams = _stale_streak(agent) + 1
  471. def _reset_stale_streak(agent) -> None:
  472. with contextlib.suppress(Exception):
  473. agent._consecutive_stale_streams = 0
  474. _INTERRUPTED_WAIT_STALE_SECONDS = 30.0
  475. def _record_interrupted_provider_wait(agent, elapsed: float, *, response_started: bool) -> bool:
  476. """Count a user-aborted pre-response stall toward the stale breaker: past the
  477. wait-notice interval an interrupt is evidence of an unresponsive attempt.
  478. Mid-response and early interrupts stay neutral."""
  479. if response_started or elapsed < _INTERRUPTED_WAIT_STALE_SECONDS:
  480. return False
  481. _bump_stale_streak(agent)
  482. logger.warning("Interrupted provider wait counted as stale after %.0fs with no output; "
  483. "consecutive stale attempts=%d.", elapsed, _stale_streak(agent))
  484. return True
  485. def _report_stale_nonstream_kill(agent, api_kwargs: dict, elapsed: float, stale_timeout: float, *,
  486. inline: bool = False, hint: Optional[str] = None) -> None:
  487. """Log + status message for a stale non-streaming kill, shared by the worker
  488. poll loop and the inline ``direct_api_call`` watchdog (their kill/state
  489. sequences differ deliberately: different locking models)."""
  490. model = api_kwargs.get("model", "unknown")
  491. logger.warning("%son-streaming API call stale for %.0fs (threshold %.0fs). "
  492. "model=%s context=~%s tokens. Killing connection.", "Inline n" if inline else "N", elapsed,
  493. stale_timeout, model, f"{estimate_request_context_tokens(api_kwargs):,}")
  494. try:
  495. agent._buffer_diagnostic_status(
  496. f"⚠️ No response from provider for {int(elapsed)}s (non-streaming, model: {model}). {hint or 'Aborting call.'}")
  497. except Exception:
  498. logger.debug("stale status buffering failed", exc_info=True)
  499. def _touch_stale_kill_activity(agent, elapsed: float) -> None:
  500. try:
  501. agent._touch_activity(f"stale non-streaming call killed after {int(elapsed)}s")
  502. except Exception:
  503. logger.debug("stale activity touch failed", exc_info=True)
  504. def _check_stale_giveup(agent) -> None:
  505. """Raise immediately when the consecutive-stale streak is past the
  506. give-up threshold — no network attempt, no stale-timeout wait."""
  507. _giveup = stale_giveup()
  508. _streak = _stale_streak(agent)
  509. if _giveup > 0 and _streak >= _giveup:
  510. raise RuntimeError(
  511. "Provider has been unresponsive (no response received) for "
  512. f"{_streak} consecutive stale attempts — aborting this call to "
  513. "avoid an indefinite stall. Switch models or start a new session, then retry."
  514. )
  515. def _stream_env_stale_base() -> tuple[float, bool]:
  516. """(HERMES_STREAM_STALE_TIMEOUT or the implicit 180s, explicit) — like
  517. ``AIAgent._resolved_api_call_stale_timeout_base``; an explicit env value is the
  518. user's deadline, so it is never capped to the run budget."""
  519. return env_float("HERMES_STREAM_STALE_TIMEOUT", 180.0), "HERMES_STREAM_STALE_TIMEOUT" in os.environ
  520. def _configured_stale_base(agent) -> float:
  521. """Per-provider ``stale_timeout_seconds`` config, else HERMES_STREAM_STALE_TIMEOUT (180s)."""
  522. cfg = get_provider_stale_timeout(agent.provider, agent.model)
  523. return cfg if cfg is not None else _stream_env_stale_base()[0]
  524. def _local_stream_stale_timeout_default() -> float:
  525. """Local-provider stale ceiling: ``agent.local_stream_stale_timeout`` (900s) or
  526. HERMES_LOCAL_STREAM_STALE_TIMEOUT. Shared by the stream stale detector and the
  527. Responses first-event watchdog so both give a local server the same prefill grace."""
  528. local_default = 900.0
  529. with contextlib.suppress(Exception):
  530. from hermes_cli.config import load_config_readonly
  531. cfg = load_config_readonly() # read-only consumer — no deepcopy
  532. agent_cfg = cfg.get("agent") if isinstance(cfg, dict) else None
  533. value = agent_cfg.get("local_stream_stale_timeout") if isinstance(agent_cfg, dict) else None
  534. if isinstance(value, (int, float)):
  535. local_default = float(value)
  536. return env_float("HERMES_LOCAL_STREAM_STALE_TIMEOUT", local_default)
  537. def _scale_stale_timeout_for_context(base: float, est_tokens: int) -> float:
  538. """Large contexts: slow models think for minutes before the first token;
  539. scale the threshold or the detector kills healthy streams."""
  540. if est_tokens > 100_000:
  541. return max(base, 300.0)
  542. if est_tokens > 50_000:
  543. return max(base, 240.0)
  544. return base
  545. def _cloud_stale_timeout(base: float, api_kwargs: dict) -> float:
  546. """Cloud stale-stream patience: ``base`` scaled for context size, then floored for
  547. known reasoning models. ``model`` (OpenAI/Anthropic) wins over ``modelId`` (Bedrock);
  548. Bedrock's dotted, region-prefixed profile id can't match the floor's slug regex
  549. directly, so it is normalized as a fallback."""
  550. from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
  551. timeout = _scale_stale_timeout_for_context(base, estimate_request_context_tokens(api_kwargs))
  552. floor = get_reasoning_stale_timeout_floor(api_kwargs.get("model") or api_kwargs.get("modelId") or "")
  553. if floor is None and api_kwargs.get("modelId"):
  554. floor = _bedrock_reasoning_stale_floor(api_kwargs["modelId"])
  555. return timeout if floor is None else max(timeout, floor)
  556. def _derive_stream_stale_timeout(agent, api_kwargs: dict) -> float:
  557. """Stale-stream patience for a provider that is never a local endpoint (Bedrock):
  558. the OpenAI/Anthropic stale detector's budget minus its local branch."""
  559. return _cloud_stale_timeout_for(agent, api_kwargs)
  560. def cap_to_run_budget(agent, timeout: float) -> float:
  561. """Cap an IMPLICIT stale timeout at half the remaining --run-budget (>= 60s), so one hung
  562. call can't outlive the run and the wrap-up notice stays reachable (#97968). Shared by the
  563. streaming and non-streaming resolvers; callers skip it for explicit user settings."""
  564. run_budget = getattr(agent, "run_budget_seconds", None)
  565. started = getattr(agent, "_run_budget_started_at", None)
  566. if not run_budget or not started:
  567. return timeout
  568. remaining = float(run_budget) - (time.time() - float(started))
  569. return min(timeout, max(60.0, remaining * 0.5))
  570. def _cloud_stale_timeout_for(agent, api_kwargs: dict) -> float:
  571. """An explicit ``providers.<id>.stale_timeout_seconds`` is the operator's deadline and
  572. wins over every implicit floor — the context-size tier as well as the reasoning-model
  573. floor — so it can SHORTEN patience for a hung stream (#115024). Only the 180s default
  574. is scaled and floored."""
  575. explicit = get_provider_stale_timeout(agent.provider, agent.model)
  576. if explicit is not None:
  577. return explicit
  578. base, explicit_env = _stream_env_stale_base()
  579. timeout = _cloud_stale_timeout(base, api_kwargs)
  580. return timeout if explicit_env else cap_to_run_budget(agent, timeout)
  581. def _bedrock_reasoning_stale_floor(model_id: object) -> float | None:
  582. """Map a Bedrock inference-profile id to its reasoning stale-timeout floor.
  583. ``us.anthropic.claude-opus-4-6-v1:0`` -> strip the region prefix, then try the
  584. segment after the provider namespace (``claude-opus-4-6-v1:0``) and the id with
  585. the provider dot dashed (``deepseek-r1-v1:0``). The floor table mixes dashed
  586. and dotted versions while Bedrock always dashes, so each candidate is also
  587. tried with digit-dash-digit <-> digit-dot-digit swapped (version separators
  588. only). First non-None wins; None for unknown models.
  589. """
  590. from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
  591. if not model_id or not isinstance(model_id, str):
  592. return None
  593. name = model_id.strip().lower()
  594. for prefix in ("global.", "us.", "eu.", "apac.", "ap.", "au.", "jp.", "ca.", "sa.", "me.", "af."):
  595. if name.startswith(prefix):
  596. name = name[len(prefix):]
  597. break
  598. base_candidates = [name]
  599. if "." in name:
  600. base_candidates.append(name.rsplit(".", 1)[1]) # claude-opus-4-6-v1:0
  601. base_candidates.append(name.replace(".", "-", 1)) # deepseek-r1-v1:0
  602. candidates = dict.fromkeys(
  603. form for cand in base_candidates
  604. for form in (cand, re.sub(r"(?<=\d)-(?=\d)", ".", cand), re.sub(r"(?<=\d)\.(?=\d)", "-", cand)))
  605. return next((f for f in map(get_reasoning_stale_timeout_floor, candidates) if f is not None), None)
  606. def _bedrock_converse_call(api_kwargs: dict, *, stream: bool, on_stream_denied=None):
  607. """Pop the Hermes routing keys and call ``converse`` / ``converse_stream`` (boto3 directly) with the
  608. adapter's shared recovery: a cachePoint rejection (Nova: toolConfig.tools, #97281) or a replayed sealed
  609. reasoning rejection resends once without the offending blocks inside the same attempt; a streaming IAM
  610. denial hands off to ``on_stream_denied(client, kwargs, exc)``; a stale connection evicts the cached
  611. client so the outer retry builds a fresh pool. Streaming returns the event stream; else a SimpleNamespace."""
  612. from agent.bedrock_adapter import (_get_bedrock_runtime_client, invalidate_runtime_client,
  613. is_stale_connection_error, is_streaming_access_denied_error, normalize_converse_response,
  614. recover_from_cache_point_rejection, recover_from_redacted_reasoning_rejection)
  615. region = api_kwargs.pop("__bedrock_region__", "us-east-1")
  616. api_kwargs.pop("__bedrock_converse__", None)
  617. client = _get_bedrock_runtime_client(region, api_kwargs.get("modelId")) # the active model's timeout
  618. method = client.converse_stream if stream else client.converse
  619. finish = (lambda raw: raw.get("stream", [])) if stream else normalize_converse_response
  620. try:
  621. raw_response = method(**api_kwargs)
  622. except Exception as exc:
  623. retry_kwargs = (recover_from_cache_point_rejection(exc, api_kwargs)
  624. or recover_from_redacted_reasoning_rejection(exc, api_kwargs))
  625. if retry_kwargs is not None:
  626. return finish(method(**retry_kwargs))
  627. if on_stream_denied is not None and is_streaming_access_denied_error(exc):
  628. return on_stream_denied(client, api_kwargs, exc)
  629. if is_stale_connection_error(exc):
  630. invalidate_runtime_client(region)
  631. raise
  632. return finish(raw_response)
  633. def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client):
  634. """Run one non-streaming LLM request for the active api_mode and return it.
  635. Shared by ``interruptible_api_call`` and ``direct_api_call``. ``make_client(reason,
  636. kind=...)`` builds the per-request client (``"openai"`` / ``"anthropic_messages"``)
  637. so callers can register it with their abort/close machinery; bedrock / MoA
  638. manage their own clients. Interrupt/abort/close semantics stay in callers.
  639. """
  640. if agent.api_mode == "codex_responses":
  641. return agent._run_codex_stream(api_kwargs, client=make_client("codex_stream_request"),
  642. on_first_delta=getattr(agent, "_codex_on_first_delta", None))
  643. if agent.api_mode == "anthropic_messages":
  644. # Request-local client so the stale/interrupt watchdog aborts sockets
  645. # from the stranger thread while the worker owns the SDK close (#67142).
  646. request_client = make_client("anthropic_messages_request", kind="anthropic_messages")
  647. return agent._anthropic_messages_create(api_kwargs, client=request_client)
  648. if agent.api_mode == "bedrock_converse":
  649. return _bedrock_converse_call(api_kwargs, stream=False)
  650. if agent.provider == "moa":
  651. # MoA is a virtual provider backed by the in-process MoAClient facade — never
  652. # rebuild a request-local client from the virtual metadata. After a client
  653. # replacement agent.client may be a native OpenAI client while provider stays
  654. # "moa": pop the MoA-internal key ONLY then (the facade consumes it; stripping
  655. # it there forces a duplicate fan-out). Only the facade exposes ``prepare()`` (#78382).
  656. _completions = getattr(getattr(agent.client, "chat", None), "completions", None)
  657. if not callable(getattr(_completions, "prepare", None)):
  658. api_kwargs.pop("_moa_prepared_request", None)
  659. return agent.client.chat.completions.create(**api_kwargs)
  660. request_client = make_client("chat_completion_request")
  661. # #93650: keep the bulk wire-format payload out of the SDK's GIL-holding
  662. # request transform. No-op unless this really is the OpenAI SDK, so the
  663. # MoA facade above and the suite's stand-in clients are unaffected.
  664. api_kwargs = bypass_chat_sdk_request_transform(api_kwargs, request_client)
  665. return _chat_completion_capturing_headers(agent, request_client, api_kwargs)
  666. def should_use_direct_api_call(agent) -> bool:
  667. """Whether an OpenAI-wire request should skip the interrupt worker.
  668. Gateway cron turns (#62151) and delegated children (#60203) run inside nested
  669. thread pools that wedge before the socket opens when the request is pushed onto
  670. yet another daemon worker. Running inline drops the deepest layer; interrupts
  671. still work because the inline path registers ``agent._active_request_abort``,
  672. which ``interrupt()`` invokes cross-thread (#72227). Cron also inlines Codex
  673. Responses (#69734): both Codex paths (non-stream, and streaming via
  674. ``_stream_codex_passthrough`` -> ``_interruptible_api_call``) reach
  675. ``direct_api_call``, whose client comes from ``make_client`` so the inline stale
  676. watchdog can abort it; the stale budget keeps the openai-codex floor/hard cap.
  677. Trade-off: the worker-only Codex TTFB/progress/idle watchdogs don't run inline, so
  678. a Codex call that never sends a first byte waits the full wall-clock stale budget
  679. (600-1200s on large contexts) instead of the ~120s TTFB cutoff. Delegated children
  680. and Native/Bedrock/MoA keep their workers: cancellation and client ownership differ.
  681. """
  682. api_mode = getattr(agent, "api_mode", None)
  683. if getattr(agent, "provider", None) == "moa":
  684. return False
  685. if getattr(agent, "platform", None) == "cron":
  686. return api_mode in {"chat_completions", "codex_responses"}
  687. if api_mode != "chat_completions":
  688. return False
  689. # Delegated child — via the execution ContextVar set by _run_single_child,
  690. # with the agent's platform stamp as a fallback for callers that bypass it.
  691. with contextlib.suppress(Exception):
  692. from agent.delegation_context import is_delegated_child_context
  693. if is_delegated_child_context():
  694. return True
  695. return getattr(agent, "platform", None) == "subagent"
  696. # How often an in-flight direct_api_call refreshes last_activity_ts. Must stay well
  697. # under the async-delegation idle stall threshold (450s) and below the 30s monitor sweep.
  698. _DIRECT_API_ACTIVITY_HEARTBEAT_SECONDS = 15.0
  699. def _resolve_direct_stale_timeout(agent, api_kwargs: dict) -> float:
  700. """Stale budget for the inline call via ``agent._compute_non_stream_stale_timeout``,
  701. plus the same openai-codex floor/hard cap the worker path applies (inline cron Codex,
  702. #69734). A non-numeric result (stub agent) leaves the watchdog disarmed; a resolver
  703. that *raises* propagates — swallowing into ``inf`` would reinstate the hang."""
  704. resolver = getattr(agent, "_compute_non_stream_stale_timeout", None)
  705. value = resolver(api_kwargs) if callable(resolver) else None
  706. if isinstance(value, bool) or not isinstance(value, (int, float)):
  707. return float("inf")
  708. base_url = getattr(agent, "base_url", None)
  709. if getattr(agent, "api_mode", None) == "codex_responses" and not (base_url and is_local_endpoint(base_url)):
  710. return _bound_openai_codex_stale_timeout(float(value), estimate_request_context_tokens(api_kwargs))
  711. return float(value)
  712. def _inline_nonstream_hard_timeout(stale_timeout: float):
  713. """Socket-level backstop for inline non-streaming calls (#85252): the keepalive
  714. client uses ``read=None`` and the stranger-thread abort must not ``close()`` the
  715. FD (#29507), so a hung provider otherwise waits until TCP dies. Returns an
  716. ``httpx.Timeout`` with read == stale budget, a float if httpx is unavailable,
  717. or ``None`` when the watchdog is disarmed (non-finite budget)."""
  718. if not math.isfinite(stale_timeout) or stale_timeout <= 0:
  719. return None
  720. conn_cap = min(stale_timeout, 60.0)
  721. try:
  722. import httpx as _httpx
  723. return _httpx.Timeout(connect=conn_cap, read=stale_timeout, write=conn_cap, pool=conn_cap)
  724. except Exception:
  725. return stale_timeout
  726. class _InlineRequest:
  727. """Lifecycle state for one inline non-streaming request (#75301). Every transition
  728. happens under ``lock``: ``done`` stops a late timer bumping the stale streak after
  729. unwind; ``cancelled`` lets an interrupt own the outcome so a racing timer can't
  730. misclassify the kill as staleness; ``stale`` is the one-shot transition."""
  731. def __init__(self, agent, api_kwargs: dict, stale_timeout: float, call_start: float):
  732. self.agent = agent
  733. self.api_kwargs = api_kwargs
  734. self.stale_timeout = stale_timeout
  735. self.call_start = call_start
  736. self.client = None
  737. self.done = False
  738. self.stale = False
  739. self.cancelled = False
  740. self.lock = threading.Lock()
  741. self.abort_hook = self.abort # single bound object: identity-checked on cleanup
  742. self._hb_stop = threading.Event()
  743. self._hb = threading.Thread(target=self._activity_heartbeat, name="direct-api-activity-hb", daemon=True)
  744. self._watchdog = None
  745. def _activity_heartbeat(self) -> None:
  746. # Never put the API call itself on another worker thread — that is the nested-pool
  747. # deadlock this path exists to avoid (#60203). This ticker only refreshes the clock.
  748. while not self._hb_stop.wait(_DIRECT_API_ACTIVITY_HEARTBEAT_SECONDS):
  749. with contextlib.suppress(Exception):
  750. self.agent._touch_activity("waiting for non-streaming API response")
  751. def _on_stale(self) -> None:
  752. # Timer thread: aborts sockets only, never issues a request (keeps the no-worker
  753. # property). False = request finished or an interrupt owns the outcome; stay silent.
  754. if not self.abort("stale_call_kill"):
  755. return
  756. elapsed = time.time() - self.call_start
  757. _report_stale_nonstream_kill(self.agent, self.api_kwargs, elapsed, self.stale_timeout, inline=True)
  758. _touch_stale_kill_activity(self.agent, elapsed)
  759. def start_watchdogs(self) -> None:
  760. """Start the activity heartbeat and (for a finite budget) the stale timer."""
  761. self._hb.start()
  762. if math.isfinite(self.stale_timeout) and self.stale_timeout > 0:
  763. self._watchdog = threading.Timer(self.stale_timeout, self._on_stale)
  764. self._watchdog.name = "direct-api-stale-watchdog"
  765. self._watchdog.daemon = True
  766. self._watchdog.start()
  767. def stop_watchdogs(self) -> None:
  768. if self._watchdog is not None:
  769. self._watchdog.cancel()
  770. self.mark_done()
  771. self._hb_stop.set()
  772. self._hb.join(timeout=2.0)
  773. def _abort_client(self, client, reason: str, log_msg: str) -> None:
  774. try:
  775. self.agent._abort_request_openai_client(client, reason=reason)
  776. except Exception:
  777. logger.debug(log_msg, exc_info=True)
  778. def abort(self, reason: str) -> bool:
  779. """Abort the inline request from a watchdog/interrupt thread. Returns True
  780. when this call owned the stale transition (the timer reports/bumps once,
  781. never after an interrupt or a completed request). Aborts under the lock
  782. (same contract as _RequestClientRegistry): once released the finally may
  783. cache the client and the NEXT call check it out."""
  784. with self.lock:
  785. if self.done:
  786. return False
  787. if reason == "stale_call_kill":
  788. if self.cancelled:
  789. return False
  790. newly_stale = not self.stale
  791. if newly_stale:
  792. self.stale = True
  793. # Bump BEFORE releasing: a fast retry's reset must not be
  794. # overtaken by this older timer restoring the streak.
  795. _bump_stale_streak(self.agent)
  796. else:
  797. # Interrupt wins the lock -> owns the outcome; a later timer
  798. # must not count it as staleness.
  799. self.cancelled = True
  800. newly_stale = False
  801. if self.client is not None:
  802. self._abort_client(self.client, reason, f"Inline request abort failed ({reason})")
  803. return newly_stale
  804. def make_client(self, reason: str, kind: str = "openai"):
  805. # Only OpenAI-wire / Codex requests reach direct_api_call; ``kind`` exists
  806. # for signature parity with the dispatch helper.
  807. client = self.agent._create_request_openai_client(reason=reason, api_kwargs=self.api_kwargs)
  808. with self.lock:
  809. self.client = client
  810. stale_before_dispatch = self.stale
  811. if stale_before_dispatch:
  812. # Timer fired during client construction: the abort found no
  813. # socket, so dispatching now would open one AFTER the only
  814. # watchdog fired. Fail here instead. (Residual ms-scale window
  815. # before httpx opens its socket is accepted.)
  816. self._abort_client(client, "stale_call_kill", "Inline abort after late client registration failed")
  817. if stale_before_dispatch:
  818. raise TimeoutError(
  819. f"Non-streaming API call timed out before request dispatch (threshold: {int(self.stale_timeout)}s)")
  820. self.agent._active_request_abort = self.abort_hook
  821. return client
  822. def mark_done(self) -> None:
  823. with self.lock:
  824. self.done = True
  825. def pop_client(self):
  826. with self.lock:
  827. client, self.client = self.client, None
  828. return client
  829. def direct_api_call(agent, api_kwargs: dict):
  830. """Run a non-streaming LLM call inline on the conversation thread (cron turns,
  831. delegated children — see ``should_use_direct_api_call``): no interrupt worker,
  832. so the nested-pool deadlock cannot occur. An activity heartbeat keeps
  833. ``last_activity_ts`` advancing (else the stall monitor interrupts a healthy
  834. wait at ~450s). A stale-call watchdog bounds the request (#80759): the timer
  835. aborts in-flight sockets via the registered hook, and a per-call ``timeout``
  836. equal to the stale budget is the backstop when the abort finds nothing (#85252).
  837. Both surface a retryable ``TimeoutError`` for the outer retry loop."""
  838. _check_stale_giveup(agent)
  839. agent._touch_activity("waiting for non-streaming API response")
  840. # Resolve the budget BEFORE the heartbeat starts: the resolver may raise
  841. # (fail-closed), and a leaked heartbeat thread would mask real stalls forever.
  842. call_start = time.time()
  843. stale_timeout = _resolve_direct_stale_timeout(agent, api_kwargs)
  844. # Never override an explicit per-call timeout; otherwise pin read=stale_timeout so a
  845. # no-op abort can't leave the read=None socket hanging until TCP dies (#85252).
  846. hard_timeout = _inline_nonstream_hard_timeout(stale_timeout)
  847. if hard_timeout is not None and "timeout" not in api_kwargs:
  848. api_kwargs = {**api_kwargs, "timeout": hard_timeout}
  849. request = _InlineRequest(agent, api_kwargs, stale_timeout, call_start)
  850. request.start_watchdogs()
  851. # Only a clean return reports the reuse reason; errors/interrupts really
  852. # close the client so the retry builds a fresh pool.
  853. succeeded = False
  854. try:
  855. response = _dispatch_nonstreaming_api_request(agent, api_kwargs, make_client=request.make_client)
  856. except Exception:
  857. if getattr(agent, "_interrupt_requested", False):
  858. raise InterruptedError("Agent interrupted during API call") from None
  859. with request.lock:
  860. was_stale = request.stale
  861. if was_stale:
  862. # Our own abort caused the transport error: raise a retryable
  863. # TimeoutError, never InterruptedError ("the user wants to stop").
  864. raise TimeoutError(
  865. f"Non-streaming API call timed out after {int(time.time() - call_start)}s with no response "
  866. f"(threshold: {int(stale_timeout)}s)") from None
  867. raise
  868. else:
  869. if getattr(agent, "_interrupt_requested", False):
  870. raise InterruptedError("Agent interrupted during API call")
  871. # Mark ``done`` under the lock so a timer firing between response
  872. # arrival and unwind is a no-op and cannot overwrite the reset below.
  873. # If a timer already won, the request still completed: return it (the
  874. # reset undoes the bump; the finally discards the poisoned client).
  875. request.mark_done()
  876. _reset_stale_streak(agent)
  877. succeeded = True
  878. return response
  879. finally:
  880. request.stop_watchdogs()
  881. if getattr(agent, "_active_request_abort", None) is request.abort_hook:
  882. agent._active_request_abort = None
  883. request_client = request.pop_client()
  884. if request_client is not None:
  885. agent._close_request_openai_client(request_client,
  886. reason="request_complete" if succeeded else "request_error_cleanup")
  887. class _RequestClientRegistry:
  888. """Per-request client / stream-handle registry shared by the request worker
  889. and the stranger threads (interrupt loop, stale detector) that may abort it.
  890. ``kind`` (``"openai"`` / ``"anthropic_messages"`` / ``"stream"``) routes
  891. :meth:`close_once` (#67142). ``"stream"`` registers a stream handle: under the
  892. MoA facade the singleton client has no per-request sockets, so interrupts
  893. must close the stream object itself (#57354).
  894. Thread-ownership rule (#29507): the owning worker pops + fully closes on its
  895. way out. A *stranger* thread only aborts the sockets — never ``client.close()``
  896. — avoiding the FD-recycling race where a just-closed TLS FD was reassigned to
  897. ``kanban.db`` and the live SSL BIO wrote into the SQLite header. The abort
  898. happens under the lock: once released the worker may cache the client and the
  899. NEXT call check it out. Stream handles are safe to close from any thread.
  900. """
  901. def __init__(self, agent):
  902. self.agent = agent
  903. self.client = None
  904. self.kind = "openai"
  905. self.owner_tid = None
  906. self.diag = None # per-attempt stream diagnostics (streaming path)
  907. self.lock = threading.Lock()
  908. def set_client(self, client, *, kind: str = "openai"):
  909. with self.lock:
  910. self.client, self.kind, self.owner_tid = client, kind, threading.get_ident()
  911. return client
  912. @staticmethod
  913. def _stream_close_callable(stream):
  914. for owner in (stream, getattr(stream, "response", None)):
  915. close = getattr(owner, "close", None)
  916. if callable(close):
  917. return close
  918. return None
  919. def set_stream_handle(self, stream):
  920. return stream if self._stream_close_callable(stream) is None else self.set_client(stream, kind="stream")
  921. def _close_stream_handle(self, stream, reason: str) -> None:
  922. close = self._stream_close_callable(stream)
  923. if close is None:
  924. return
  925. try:
  926. close()
  927. logger.info("Streaming response handle closed (%s)", reason)
  928. except Exception as exc:
  929. logger.debug("Streaming response handle close failed (%s): %s", reason, exc)
  930. def close_once(self, reason: str) -> None:
  931. with self.lock:
  932. request_client, request_kind, owner_tid = self.client, self.kind, self.owner_tid
  933. stranger_thread = (
  934. request_kind != "stream"
  935. and request_client is not None
  936. and owner_tid is not None
  937. and owner_tid != threading.get_ident()
  938. )
  939. if stranger_thread:
  940. abort = (self.agent._abort_request_anthropic_client if request_kind == "anthropic_messages"
  941. else self.agent._abort_request_openai_client)
  942. abort(request_client, reason=reason)
  943. return
  944. self.client = None
  945. self.owner_tid = None
  946. if request_client is None:
  947. return
  948. if request_kind == "stream":
  949. self._close_stream_handle(request_client, reason)
  950. elif request_kind == "anthropic_messages":
  951. self.agent._close_request_anthropic_client(request_client, reason=reason)
  952. else:
  953. self.agent._close_request_openai_client(request_client, reason=reason)
  954. # Silence budget for high-or-above reasoning effort on a Codex request. GPT-5-family models at
  955. # high effort think server-side for 100-170s before the first substantive SSE event even on a
  956. # ~6KB prompt (#112909), while the token-sized tiers below hand such a prompt 12s/120s/90s; the
  957. # watchdog killed healthy requests three times in a row and blamed the provider. Applies as a
  958. # floor to the IMPLICIT defaults only -- explicit env/config values keep winning, and the stale
  959. # timeout's run-budget cap is applied AFTER this floor (AIAgent._compute_non_stream_stale_timeout).
  960. HIGH_EFFORT_SILENCE_FLOOR_SECONDS = 300.0
  961. # First-progress budget for a lifecycle-only stream on an official-Codex large request: the
  962. # stream opened but no substantive model event has arrived. Measured from the physical-attempt
  963. # start (a reconnect restarts it; lifecycle frames do not), and applied regardless of reasoning
  964. # effort. Equal to the high-effort floor today, but a separate knob so tuning one cannot silently
  965. # retune the other.
  966. CODEX_FIRST_PROGRESS_TIMEOUT_SECONDS = 300.0
  967. def _high_effort_silence_floor(agent) -> float:
  968. """``HIGH_EFFORT_SILENCE_FLOOR_SECONDS`` when the wire reasoning config is enabled at ``high`` or any
  969. stronger :data:`~agent.reasoning_effort.EFFORT_LADDER` level (xhigh/max/ultra), else 0."""
  970. from agent.reasoning_effort import EFFORT_LADDER
  971. cfg = getattr(agent, "reasoning_config", None)
  972. if not isinstance(cfg, dict) or cfg.get("enabled") is False:
  973. return 0.0
  974. effort = str(cfg.get("effort") or "").strip().lower()
  975. if effort not in EFFORT_LADDER or EFFORT_LADDER.index(effort) < EFFORT_LADDER.index("high"):
  976. return 0.0
  977. return HIGH_EFFORT_SILENCE_FLOOR_SECONDS
  978. @dataclass
  979. class _NonStreamWatchdogs:
  980. """Poll-loop thresholds for one non-streaming request."""
  981. stale_timeout: float
  982. codex: bool # api_mode == codex_responses (codex watchdogs armed)
  983. est_tokens: int
  984. ttfb_enabled: bool
  985. ttfb_timeout: float
  986. idle_enabled: bool
  987. idle_timeout: float
  988. idle_requires_progress: bool
  989. progress_timeout: float = 0.0
  990. def _resolve_nonstream_watchdogs(agent, api_kwargs: dict) -> _NonStreamWatchdogs:
  991. """Stale-call timeout plus the Codex Responses stream watchdogs.
  992. The stale detector kills a hung provider early so the retry loop can rotate
  993. credentials / fall back. Codex adds two failure modes: accepting the connection
  994. but never emitting an event (no-event TTFB cutoff; a reconnect succeeds in ~2s)
  995. and stalling after substantive model progress begins (event-idle gap; any parsed SSE
  996. event remains transport activity). Only the implicit official OpenAI Codex policy
  997. for large contexts defers arming until progress; small requests, compatible backends,
  998. and explicit overrides retain the legacy first-event semantics. Tunables:
  999. HERMES_CODEX_TTFB_TIMEOUT_SECONDS,
  1000. HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS (0 disables each),
  1001. HERMES_CODEX_TTFB_DISABLE_ABOVE_TOKENS / HERMES_CODEX_TTFB_STRICT,
  1002. HERMES_CODEX_TTFB_MAX_SECONDS (opt-in ceiling, default 0 = none), HERMES_CODEX_HARD_TIMEOUT_SECONDS.
  1003. """
  1004. # The effort floor on the STALE timeout lives inside _compute_non_stream_stale_timeout so the
  1005. # run-budget cap still bounds it; here the floor only raises the TTFB/idle implicit defaults.
  1006. stale_timeout = agent._compute_non_stream_stale_timeout(api_kwargs)
  1007. codex = agent.api_mode == "codex_responses"
  1008. openai_codex_backend = _is_openai_codex_backend(agent)
  1009. est_tokens = estimate_request_context_tokens(api_kwargs)
  1010. effort_floor = _high_effort_silence_floor(agent) if codex else 0.0
  1011. codex_floor = 0.0
  1012. # Local Responses servers keep their configured local stale/TTFB grace: the hosted
  1013. # large-context floor, hard ceiling and TTFB scale-up/cap below must not tighten it.
  1014. base_url = getattr(agent, "base_url", None)
  1015. local = bool(base_url) and is_local_endpoint(base_url)
  1016. if codex and not local:
  1017. codex_floor = openai_codex_stale_timeout_floor(est_tokens)
  1018. stale_timeout = _bound_openai_codex_stale_timeout(stale_timeout, est_tokens)
  1019. idle_default = max(effort_floor, next(
  1020. (default for threshold, default in ((100_000, 180.0), (50_000, 120.0), (10_000, 60.0)) if est_tokens > threshold),
  1021. 12.0))
  1022. # No-event TTFB cutoff. Default 120s: the SDK's own read timeout is 600s,
  1023. # and a tight 12s killed subscription-backed requests mid-prefill.
  1024. ttfb_enabled = codex
  1025. ttfb_explicit = env_float("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", -1.0) != -1.0
  1026. ttfb_timeout = env_float("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", 120.0)
  1027. if ttfb_timeout <= 0:
  1028. ttfb_enabled = False
  1029. elif codex and not local:
  1030. # Large requests legitimately spend tens of seconds in admission/prefill before the
  1031. # first SSE event: scale the cutoff up to the idle default unless TTFB_STRICT is set.
  1032. disable_above = env_float("HERMES_CODEX_TTFB_DISABLE_ABOVE_TOKENS", 10_000.0)
  1033. strict = os.environ.get("HERMES_CODEX_TTFB_STRICT", "").strip().lower() in {"1", "true", "yes", "on"}
  1034. if not strict and disable_above > 0 and est_tokens >= disable_above and ttfb_timeout < idle_default:
  1035. logger.info("Scaling codex-responses no-event TTFB watchdog from %.0fs to %.0fs "
  1036. "for large request (context=~%s tokens >= %.0f). "
  1037. "Set HERMES_CODEX_TTFB_STRICT=1 to keep the smaller cutoff.", ttfb_timeout, idle_default,
  1038. f"{est_tokens:,}", disable_above)
  1039. ttfb_timeout = idle_default
  1040. # Opt-in ceiling (0 = off): a 120s default here silently undid the scale-up above (#91621).
  1041. ttfb_cap = env_float("HERMES_CODEX_TTFB_MAX_SECONDS", 0.0)
  1042. if ttfb_cap > 0 and ttfb_timeout > ttfb_cap:
  1043. logger.info("Capping codex-responses no-event TTFB timeout from %.0fs to %.0fs "
  1044. "(context=~%s tokens) per HERMES_CODEX_TTFB_MAX_SECONDS.", ttfb_timeout, ttfb_cap,
  1045. f"{est_tokens:,}")
  1046. ttfb_timeout = ttfb_cap
  1047. elif not ttfb_explicit and local:
  1048. # A local server prefills for minutes before its first event; the chat-completions
  1049. # siblings already grant local endpoints the local stale ceiling, so the Responses
  1050. # transport gets the same grace instead of the 120s hosted cutoff (#92302).
  1051. local_ceiling = _local_stream_stale_timeout_default()
  1052. if local_ceiling > ttfb_timeout:
  1053. logger.info("Local provider detected (%s) — no-event TTFB watchdog raised from %.0fs to %.0fs "
  1054. "(agent.local_stream_stale_timeout); set HERMES_CODEX_TTFB_TIMEOUT_SECONDS for an explicit cutoff.",
  1055. base_url, ttfb_timeout, local_ceiling)
  1056. ttfb_timeout = local_ceiling
  1057. if ttfb_enabled and not ttfb_explicit:
  1058. # High-effort thinking precedes the first event; the floor outranks the cap.
  1059. ttfb_timeout = max(ttfb_timeout, effort_floor)
  1060. # An operator-set idle timeout keeps first-event semantics; only the implicit
  1061. # default defers arming until model progress. Sentinel: env_float returns the
  1062. # default for unset AND unparseable values, so both count as implicit.
  1063. idle_explicit = env_float("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", -1.0) != -1.0
  1064. idle_timeout = env_float("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", idle_default)
  1065. progress_gated = codex and openai_codex_backend and codex_floor > 0 and not idle_explicit
  1066. return _NonStreamWatchdogs(stale_timeout=stale_timeout, codex=codex, est_tokens=est_tokens,
  1067. ttfb_enabled=ttfb_enabled, ttfb_timeout=ttfb_timeout, idle_enabled=codex and idle_timeout > 0,
  1068. idle_timeout=idle_timeout, idle_requires_progress=progress_gated,
  1069. # A lifecycle frame proves transport liveness, not model progress. Bound that phase
  1070. # from the physical-attempt start; events cannot restart the grace period.
  1071. progress_timeout=CODEX_FIRST_PROGRESS_TIMEOUT_SECONDS if progress_gated else 0.0)
  1072. def _codex_silent_hang_hint(agent, api_kwargs: dict) -> Optional[str]:
  1073. hint_fn = getattr(agent, "_codex_silent_hang_hint", None)
  1074. with contextlib.suppress(Exception):
  1075. if callable(hint_fn):
  1076. return hint_fn(model=api_kwargs.get("model"))
  1077. return None
  1078. def interruptible_api_call(agent, api_kwargs: dict):
  1079. """Run the API call on a worker thread so the caller can detect interrupts
  1080. without waiting for the full HTTP round-trip. Each worker gets its own
  1081. per-request client (interrupts close only that one); a stale-call detector
  1082. kills the connection and raises so the main retry loop can back off / rotate
  1083. credentials / fall back."""
  1084. # Nested-pool contexts (cron, delegated children) wedge on a worker thread
  1085. # (#62151): run inline. See should_use_direct_api_call.
  1086. if should_use_direct_api_call(agent):
  1087. return direct_api_call(agent, api_kwargs)
  1088. _check_stale_giveup(agent) # cross-turn stale breaker (#58962), non-streaming sibling
  1089. from agent.chat_completion_nonstream import _NonStreamRequest
  1090. return _NonStreamRequest(agent, api_kwargs).run()
  1091. def _consume_ephemeral_reasoning_off(agent) -> bool:
  1092. """Consume the one-shot "answer without thinking" continuation flag.
  1093. Set by the length-continuation path when a request returned reasoning but NO
  1094. visible content (thinking ate the output cap); continuation turns never replay
  1095. prior reasoning, so thinking ON would re-burn the budget. When True the caller
  1096. overrides the wire reasoning_config with ``{"enabled": False, "effort": "none"}``
  1097. for exactly the next call. Prompt-cache cost is bounded to ONE cold prefix write
  1098. on config-sensitive providers (Anthropic, OpenAI) — far cheaper than four futile
  1099. full-budget continuations.
  1100. """
  1101. consumed = bool(getattr(agent, "_ephemeral_reasoning_off", False))
  1102. if consumed:
  1103. agent._ephemeral_reasoning_off = False
  1104. return consumed
  1105. def _reasoning_config_for_wire(agent):
  1106. """``agent.reasoning_config`` with the one-shot reasoning-off override applied.
  1107. Once the route has answered a disable with "reasoning is mandatory"
  1108. (``agent._reasoning_disable_rejected``), every disable — configured or
  1109. the one-shot continuation override — is dropped for the rest of the
  1110. session: the request goes out without a reasoning config and the route
  1111. applies its own default.
  1112. """
  1113. cfg = agent.reasoning_config
  1114. ephemeral_off = _consume_ephemeral_reasoning_off(agent)
  1115. if getattr(agent, "_reasoning_effort_rejected", False):
  1116. # The route rejected the configured reasoning LEVEL itself (#100536: ``reasoning.effort:
  1117. # max`` on an enabled config). Omit the reasoning fields for the rest of the session —
  1118. # the route default — as the auxiliary ladder does; resending would 400 identically.
  1119. agent._wire_reasoning_config = None
  1120. return None
  1121. if cfg is None:
  1122. # Unset effort: the profile's default (custom/OpenAI-compatible: medium) rather than the
  1123. # route's own, recorded below as what went out so a rejection of it lands in the branch above.
  1124. from agent.reasoning_params import unset_reasoning_default
  1125. cfg = unset_reasoning_default(agent)
  1126. if getattr(agent, "_reasoning_disable_rejected", False):
  1127. # The route rejects disables. Resend exactly what the session has
  1128. # been sending — the user's own config — so the retry lands on the
  1129. # same provider cache key as every prior request. Only a config that
  1130. # is itself a disable changes, and that session has never sent
  1131. # anything else, so nothing warm is lost: a route that said the
  1132. # disable is *mandatory-on* gets the floor effort (closest to what
  1133. # the user asked for); a relay that does not know the field gets
  1134. # nothing (route default).
  1135. if isinstance(cfg, dict) and (
  1136. cfg.get("enabled") is False or cfg.get("effort") == "none"
  1137. ):
  1138. if getattr(agent, "_reasoning_floor_required", False):
  1139. from agent.auxiliary_reasoning_floor import REASONING_FLOOR_EFFORT
  1140. floored = {**cfg, "enabled": True, "effort": REASONING_FLOOR_EFFORT}
  1141. agent._wire_reasoning_config = floored
  1142. return floored
  1143. agent._wire_reasoning_config = None
  1144. return None
  1145. agent._wire_reasoning_config = cfg
  1146. return cfg
  1147. if ephemeral_off:
  1148. cfg = {**(cfg or {}), "enabled": False, "effort": "none"}
  1149. # What actually went out: the reasoning-rejection rung reads it to tell a rejected
  1150. # disable (drop the disable) from a rejected level (drop the reasoning fields).
  1151. agent._wire_reasoning_config = cfg
  1152. return cfg
  1153. def _alias_tool_search_bridge_for_xai(agent, transport, tools_for_api):
  1154. """xAI chat-completions reserves ``tool_search`` and 400s when the bridge declares
  1155. it (#95003): rename the wire declaration; ``normalize_response`` maps calls back
  1156. via the transport's ``_last_wire_aliases`` (reset here so a stale map can't
  1157. reverse-map a name this request never aliased). Deep-copy first (#27907)."""
  1158. if transport is not None and hasattr(transport, "_last_wire_aliases"):
  1159. transport._last_wire_aliases = {}
  1160. is_xai_chat = agent.provider in {"xai", "xai-oauth"} or agent._base_url_hostname == "api.x.ai"
  1161. if not (is_xai_chat and tools_for_api):
  1162. return tools_for_api
  1163. try:
  1164. import copy as _copy_xai
  1165. from agent.transports.chat_completions import _rename_tool_search_bridge_for_xai
  1166. has_bridge = any(
  1167. (t.get("function") or {}).get("name") == "tool_search" for t in tools_for_api if isinstance(t, dict)
  1168. )
  1169. if has_bridge:
  1170. tools_for_api = _copy_xai.deepcopy(tools_for_api)
  1171. tools_for_api, alias_map = _rename_tool_search_bridge_for_xai(tools_for_api)
  1172. if transport is not None:
  1173. transport._last_wire_aliases = alias_map
  1174. except Exception as exc:
  1175. logger.warning("%s⚠️ Failed to alias tool_search bridge for xAI: %s", getattr(agent, "log_prefix", ""), exc)
  1176. return tools_for_api
  1177. def _consume_ephemeral_max_output(agent):
  1178. """Pop the one-shot ephemeral output cap; whichever path builds the request consumes it."""
  1179. ephemeral_out = getattr(agent, "_ephemeral_max_output_tokens", None)
  1180. if ephemeral_out is not None:
  1181. agent._ephemeral_max_output_tokens = None
  1182. return ephemeral_out
  1183. def _build_anthropic_kwargs(agent, api_messages, tools_for_api, reasoning_config, request_overrides):
  1184. ctx_len = getattr(agent, "context_compressor", None)
  1185. ephemeral_out = _consume_ephemeral_max_output(agent)
  1186. anthropic_kwargs = agent._get_transport().build_kwargs(model=agent.model,
  1187. messages=agent._prepare_anthropic_messages_for_api(api_messages), tools=tools_for_api,
  1188. max_tokens=ephemeral_out if ephemeral_out is not None else agent.max_tokens,
  1189. reasoning_config=reasoning_config, is_oauth=agent._is_anthropic_oauth,
  1190. preserve_dots=agent._anthropic_preserve_dots(),
  1191. context_length=ctx_len.context_length if ctx_len else None,
  1192. base_url=getattr(agent, "_anthropic_base_url", None),
  1193. fast_mode=request_overrides.get("speed") == "fast",
  1194. drop_context_1m_beta=bool(getattr(agent, "_oauth_1m_beta_disabled", False)))
  1195. # Portal reads ``tags`` / ``session_id`` on its Messages route too, but the profile hook
  1196. # is only consulted by the OpenAI-wire transport — merge here to keep sticky routing.
  1197. return _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs, api_messages)
  1198. def _build_bedrock_kwargs(agent, api_messages, tools_for_api, reasoning_config):
  1199. # Bedrock Converse — the adapter converts messages/tools and calls boto3 directly.
  1200. return agent._get_transport().build_kwargs(model=agent.model, messages=api_messages, tools=tools_for_api,
  1201. max_tokens=agent.max_tokens, region=getattr(agent, "_bedrock_region", None) or "us-east-1",
  1202. guardrail_config=getattr(agent, "_bedrock_guardrail_config", None), reasoning_config=reasoning_config)
  1203. def _build_codex_kwargs(agent, api_messages, tools_for_api, reasoning_config, request_overrides, cache_scope_id):
  1204. from agent.codex_responses_adapter import classify_responses_route
  1205. from agent.native_compaction import native_compaction_context_management
  1206. is_codex_backend, is_xai_responses, endpoint = classify_responses_route(agent)
  1207. # Native server-side compaction (gpt-5.6 on direct OpenAI / ChatGPT Codex routes
  1208. # only) — None on every other route/model, leaving the request unchanged.
  1209. context_management = native_compaction_context_management(agent, is_codex_backend=is_codex_backend,
  1210. is_xai_responses=is_xai_responses, endpoint=endpoint)
  1211. # xAI's /responses endpoint 400s on ``pattern``/``format`` schema keywords and on
  1212. # ``enum`` values containing ``/`` — strip them (#27197). Deep-copy first: the
  1213. # sanitizers mutate in place and tools_for_api aliases agent.tools (#27907).
  1214. if is_xai_responses:
  1215. try:
  1216. import copy as _copy
  1217. from tools.schema_sanitizer import strip_pattern_and_format, strip_slash_enum
  1218. tools_for_api = _copy.deepcopy(tools_for_api)
  1219. tools_for_api, _ = strip_pattern_and_format(tools_for_api)
  1220. tools_for_api, _ = strip_slash_enum(tools_for_api)
  1221. except Exception as exc:
  1222. logger.warning("%s⚠️ Failed to sanitize tool schemas for xAI: %s", getattr(agent, "log_prefix", ""), exc)
  1223. ephemeral_out = _consume_ephemeral_max_output(agent)
  1224. return agent._get_transport().build_kwargs(model=agent.model,
  1225. messages=agent._prepare_messages_for_non_vision_model(api_messages), tools=tools_for_api,
  1226. reasoning_config=reasoning_config, session_id=getattr(agent, "session_id", None),
  1227. cache_scope_id=cache_scope_id, base_url=agent.base_url,
  1228. max_tokens=ephemeral_out if ephemeral_out is not None else agent.max_tokens,
  1229. timeout=agent._resolved_api_call_timeout(), request_overrides=request_overrides,
  1230. provider=getattr(agent, "provider", None),
  1231. is_codex_backend=is_codex_backend, is_xai_responses=is_xai_responses,
  1232. profile_reasoning=agent._endpoint_reasoning_payload() if endpoint.profile_reasoning else None,
  1233. replay_encrypted_reasoning=bool(getattr(agent, "_codex_reasoning_replay_enabled", True)),
  1234. context_management=context_management, text_verbosity=getattr(agent, "text_verbosity", None))
  1235. def _build_chat_completions_kwargs(agent, api_messages, tools_for_api, reasoning_config, request_overrides, cache_scope_id):
  1236. transport = agent._get_transport()
  1237. tools_for_api = _alias_tool_search_bridge_for_xai(agent, transport, tools_for_api)
  1238. _is_qwen = agent._is_qwen_portal()
  1239. _is_or = agent._is_openrouter_url()
  1240. _host = agent._base_url_lower
  1241. _is_lmstudio = (agent.provider or "").strip().lower() == "lmstudio"
  1242. _profile_endpoint = agent._reasoning_endpoint_profile() is not None
  1243. # _fixed_temperature_for_model may return the OMIT_TEMPERATURE sentinel
  1244. # (temperature omitted entirely), a numeric override, or None.
  1245. _omit_temp, _fixed_temp = False, None
  1246. with contextlib.suppress(Exception):
  1247. from agent.auxiliary_client import _fixed_temperature_for_model, OMIT_TEMPERATURE
  1248. _ft = _fixed_temperature_for_model(agent.model, agent.base_url)
  1249. _omit_temp = _ft is OMIT_TEMPERATURE
  1250. _fixed_temp = None if _omit_temp else _ft
  1251. _prefs = _provider_preferences_for_agent(agent)
  1252. _qwen_meta = {"sessionId": agent.session_id or "hermes", "promptId": str(uuid.uuid4())} if _is_qwen else None
  1253. _profile = None
  1254. with contextlib.suppress(Exception):
  1255. from providers import get_provider_profile
  1256. _profile = get_provider_profile(agent.provider)
  1257. _ephemeral_out = _consume_ephemeral_max_output(agent)
  1258. from agent.reasoning_carriers import shape_wire_carriers
  1259. api_messages = shape_wire_carriers(api_messages, model=agent.model, base_url=agent.base_url, scope=agent)
  1260. # Strip image parts for non-vision models on BOTH paths (registered
  1261. # providers with profiles used to bypass it).
  1262. _common = dict(model=agent.model, messages=agent._prepare_messages_for_non_vision_model(api_messages),
  1263. tools=tools_for_api, base_url=agent.base_url, timeout=agent._resolved_api_call_timeout(),
  1264. max_tokens=agent.max_tokens, ephemeral_max_output_tokens=_ephemeral_out,
  1265. max_tokens_param_fn=agent._max_tokens_param, reasoning_config=reasoning_config,
  1266. request_overrides=request_overrides, session_id=getattr(agent, "session_id", None),
  1267. cache_scope_id=cache_scope_id, ollama_num_ctx=agent._ollama_num_ctx,
  1268. provider_preferences=_prefs or None, openrouter_min_coding_score=agent.openrouter_min_coding_score,
  1269. supports_reasoning=agent._supports_reasoning_extra_body(),
  1270. lmstudio_reasoning_options=agent._lmstudio_reasoning_options_cached() if _is_lmstudio else None,
  1271. qwen_session_metadata=_qwen_meta)
  1272. if _profile:
  1273. # Profiles handle per-provider quirks via hooks fed the context above.
  1274. return transport.build_kwargs(provider_profile=_profile, **_common)
  1275. # Legacy flag path: only for a provider absent from the providers/ registry.
  1276. return transport.build_kwargs(
  1277. **_common,
  1278. model_lower=(agent.model or "").lower(),
  1279. is_openrouter=_is_or,
  1280. is_nous=base_url_host_matches(_host, "nousresearch.com"),
  1281. is_qwen_portal=_is_qwen,
  1282. is_nvidia_nim=base_url_host_matches(_host, "integrate.api.nvidia.com"),
  1283. is_kimi=any(base_url_host_matches(agent.base_url, h) for h in ("api.kimi.com", "moonshot.ai", "moonshot.cn")),
  1284. is_tokenhub=base_url_host_matches(_host, "tokenhub.tencentmaas.com"),
  1285. is_custom_provider=agent.provider == "custom",
  1286. qwen_prepare_fn=agent._qwen_prepare_chat_messages if _is_qwen else None,
  1287. qwen_prepare_inplace_fn=agent._qwen_prepare_chat_messages_inplace if _is_qwen else None,
  1288. fixed_temperature=_fixed_temp,
  1289. omit_temperature=_omit_temp,
  1290. profile_endpoint=_profile_endpoint, endpoint_reasoning=agent._endpoint_reasoning_payload() if _profile_endpoint else None,
  1291. provider_name=agent.provider,
  1292. )
  1293. def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = None) -> dict:
  1294. """Build the keyword arguments dict for the active API mode.
  1295. Wraps the per-api_mode builder so the conversation-affinity headers (OpenCode's
  1296. ``x-opencode-session``, a custom provider's opt-in ``session_affinity_header``) ride on
  1297. every request regardless of transport (chat_completions / codex_responses /
  1298. anthropic_messages). No-op for every other provider.
  1299. """
  1300. from agent.opencode_affinity import merge_session_affinity_headers
  1301. kwargs = _build_api_kwargs_for_mode(agent, api_messages, tools_for_api)
  1302. return merge_session_affinity_headers(
  1303. kwargs,
  1304. getattr(agent, "provider", None),
  1305. getattr(agent, "base_url", None),
  1306. getattr(agent, "session_id", None),
  1307. )
  1308. def _build_api_kwargs_for_mode(agent, api_messages: list, tools_for_api: list | None = None) -> dict:
  1309. # One-shot continuation override — consumed exactly once, on the FIRST
  1310. # request this call builds (only one api_mode branch runs per invocation).
  1311. reasoning_config = _reasoning_config_for_wire(agent)
  1312. if tools_for_api is None:
  1313. tools_for_api = agent.tools
  1314. # The one place request_overrides are consumed: static /fast values are already pinned
  1315. # in agent.request_overrides; auto/cold windows layer the fast override per request.
  1316. request_overrides = effective_request_overrides(agent)
  1317. if agent.api_mode == "anthropic_messages":
  1318. return _build_anthropic_kwargs(agent, api_messages, tools_for_api, reasoning_config, request_overrides)
  1319. if agent.api_mode == "bedrock_converse":
  1320. return _build_bedrock_kwargs(agent, api_messages, tools_for_api, reasoning_config)
  1321. # Rotation-stable logical cache scope shared by every OpenAI-wire branch
  1322. # (memoized on the agent); anthropic/bedrock above don't use it.
  1323. cache_scope_id = _prompt_cache_scope_for_agent(agent)
  1324. builder = _build_codex_kwargs if agent.api_mode == "codex_responses" else _build_chat_completions_kwargs
  1325. return builder(agent, api_messages, tools_for_api, reasoning_config, request_overrides, cache_scope_id)
  1326. def _model_dump_safe(obj):
  1327. """``model_dump(warnings=False)`` (avoids pydantic serializer UserWarnings on
  1328. generic-union SDK models), falling back for shims that reject the kwarg."""
  1329. try:
  1330. return obj.model_dump(warnings=False)
  1331. except TypeError:
  1332. return obj.model_dump()
  1333. def _dump_if_model(value):
  1334. return _model_dump_safe(value) if hasattr(value, "model_dump") else value
  1335. def _assistant_reasoning_text(agent, assistant_message) -> Optional[str]:
  1336. """Structured reasoning, else inline ``<think>`` blocks embedded in content."""
  1337. reasoning_text = agent._extract_reasoning(assistant_message)
  1338. if not reasoning_text:
  1339. content = flatten_message_text(getattr(assistant_message, "content", None))
  1340. think_blocks = re.findall(r'<think>(.*?)</think>', content, flags=re.DOTALL)
  1341. if think_blocks:
  1342. reasoning_text = "\n\n".join(b.strip() for b in think_blocks if b.strip()) or None
  1343. if reasoning_text and agent.verbose_logging:
  1344. logging.debug(f"Captured reasoning ({len(reasoning_text)} chars): {reasoning_text}")
  1345. # When streaming is active the reasoning was already displayed during the
  1346. # stream (structured deltas or <think> tag extraction); fire only for
  1347. # non-streaming modes (gateway, batch, quiet). Anything not shown during
  1348. # streaming is caught by the CLI post-response fallback.
  1349. if reasoning_text and agent.reasoning_callback and not agent.stream_delta_callback and not agent._stream_callback:
  1350. with contextlib.suppress(Exception):
  1351. agent.reasoning_callback(reasoning_text)
  1352. return _sanitize_surrogates(reasoning_text) if reasoning_text else reasoning_text
  1353. def _assistant_content_for_storage(agent, assistant_message):
  1354. # Sanitize surrogates (Kimi/GLM via Ollama emit code points that crash json.dumps),
  1355. # strip inline <think> tags at the storage boundary (they leaked to platforms and
  1356. # polluted titles), then redact inlined credentials before the message enters
  1357. # history / state.db / gateway delivery (no-op with HERMES_REDACT_SECRETS off).
  1358. content = _sanitize_surrogates(flatten_message_text(getattr(assistant_message, "content", None)))
  1359. if isinstance(content, str) and content:
  1360. content = agent._strip_think_blocks(content).strip()
  1361. if content:
  1362. from agent.redact import redact_sensitive_text
  1363. content = redact_sensitive_text(content)
  1364. return content
  1365. def _assistant_tool_call_dict(agent, tool_call, index: int) -> dict:
  1366. raw_id = getattr(tool_call, "id", None)
  1367. call_id = getattr(tool_call, "call_id", None)
  1368. if not isinstance(call_id, str) or not call_id.strip():
  1369. call_id, _ = agent._split_responses_tool_id(raw_id)
  1370. if not isinstance(call_id, str) or not call_id.strip():
  1371. if isinstance(raw_id, str) and raw_id.strip():
  1372. call_id = raw_id.strip()
  1373. else:
  1374. _fn = getattr(tool_call, "function", None)
  1375. call_id = agent._deterministic_call_id(getattr(_fn, "name", "") if _fn else "",
  1376. getattr(_fn, "arguments", "{}") if _fn else "{}", index)
  1377. call_id = call_id.strip()
  1378. response_item_id = getattr(tool_call, "response_item_id", None)
  1379. if not isinstance(response_item_id, str) or not response_item_id.strip():
  1380. _, response_item_id = agent._split_responses_tool_id(raw_id)
  1381. response_item_id = agent._derive_responses_function_call_id(call_id,
  1382. response_item_id if isinstance(response_item_id, str) else None)
  1383. # Arguments are deliberately NOT redacted: this dict is replayed to the model every
  1384. # turn, so a ``***`` mask would break credential-dependent commands (#43083).
  1385. tc_dict = {"id": call_id, "call_id": call_id, "response_item_id": response_item_id,
  1386. "type": tool_call.type,
  1387. "function": {"name": tool_call.function.name, "arguments": tool_call.function.arguments}}
  1388. # Preserve extra_content (Gemini thought_signature) or Gemini 3 thinking
  1389. # models 400 on the next request.
  1390. # Tool-call arguments are intentionally NOT redacted here. This dict enters the in-memory conversation
  1391. # history that is replayed to the model on every subsequent turn AND persisted to state.db, which is
  1392. # itself replayed verbatim on session resume (get_messages_as_conversation). Masking a credential to
  1393. # `***` here poisons that replay: the model reads back its own `PGPASSWORD='***' psql ...` call and
  1394. # copies the placeholder into the next tool call, breaking every credential-dependent command on the
  1395. # second turn (#43083). The masking also provided no real protection — the same secret still leaks
  1396. # verbatim through tool OUTPUT (file contents, command output, diffs, the compaction block), none of
  1397. # which this pass ever touched. Keeping secrets out of the replayable store is a separate
  1398. # tokenization/vault concern, not something arg-redaction can deliver without breaking replay.
  1399. # Storage-time redaction remains governed by the `security.redact_secrets` toggle. (#19798 introduced
  1400. # this; #43083 removed it.) Preserve extra_content (e.g. Gemini thought_signature) so it is sent back on
  1401. # subsequent API calls. Without this, Gemini 3 thinking models reject the request with a 400 error.
  1402. extra = getattr(tool_call, "extra_content", None)
  1403. if extra is not None:
  1404. tc_dict["extra_content"] = _dump_if_model(extra)
  1405. # Some gateways sign Gemini 3 calls inside ``function`` (replayed only where the endpoint profile asks).
  1406. from agent.reasoning_carriers import field
  1407. fn_sig = field(tool_call, "thought_signature") or getattr(tool_call.function, "thought_signature", None)
  1408. if isinstance(fn_sig, str) and fn_sig:
  1409. tc_dict["function"]["thought_signature"] = fn_sig
  1410. return tc_dict
  1411. def build_assistant_message(agent, assistant_message, finish_reason: str) -> dict:
  1412. """Build a normalized assistant message dict (reasoning, reasoning_details,
  1413. optional tool_calls) shared by the tool-call and final-response paths.
  1414. Textless turns are NOT padded here: ``repair_empty_non_final_messages`` is the
  1415. single owner — write-time padding broke codex commentary turns and cannot
  1416. survive ``_rows_to_conversation``."""
  1417. assistant_tool_calls = getattr(assistant_message, "tool_calls", None)
  1418. reasoning_text = _assistant_reasoning_text(agent, assistant_message)
  1419. msg = stamp_message_timestamp({"role": "assistant",
  1420. "content": _assistant_content_for_storage(agent, assistant_message), "reasoning": reasoning_text,
  1421. "finish_reason": finish_reason})
  1422. raw_reasoning_content = getattr(assistant_message, "reasoning_content", None)
  1423. if raw_reasoning_content is None:
  1424. model_extra = getattr(assistant_message, "model_extra", None) or {}
  1425. if isinstance(model_extra, dict) and "reasoning_content" in model_extra:
  1426. raw_reasoning_content = model_extra["reasoning_content"]
  1427. if raw_reasoning_content is not None:
  1428. msg["reasoning_content"] = _sanitize_surrogates(raw_reasoning_content)
  1429. elif assistant_tool_calls and agent._needs_thinking_reasoning_pad():
  1430. # DeepSeek v4 / Kimi thinking modes 400 on a replayed tool-call message without
  1431. # reasoning_content; pad with a single space (empty string is rejected too).
  1432. # Without it, replaying the persisted message causes HTTP 400 ("The reasoning_content in the
  1433. # thinking mode must be passed back to the API"). Include streamed reasoning text when captured;
  1434. # otherwise pad with a single space — DeepSeek V4 Pro tightened validation and rejects empty string
  1435. # ("The reasoning content in the thinking mode must be passed back to the API"). A space satisfies
  1436. # non-empty checks everywhere without leaking fabricated reasoning. Refs #15250, #17400, #17341.
  1437. msg["reasoning_content"] = reasoning_text or " "
  1438. elif reasoning_text:
  1439. # Streaming-only providers accumulate reasoning via deltas and never set
  1440. # it on the message; replaying through a thinking model then 400s.
  1441. # Promote ONLY when nothing set the field: SDK reasoning_content and the
  1442. # tool-call pad win, and reasoning-less turns leave the field absent so
  1443. # the replay-time leak guard and promotion tiers still apply.
  1444. # Additive fallback (refs #16844, #16884). Streaming-only providers (glm, MiniMax, gpt-5.x via aigw,
  1445. # Anthropic via openai-compat shims) accumulate reasoning through ``delta.reasoning_content`` chunks
  1446. # but never land it on the message object as a top-level attribute, so neither branch above fires
  1447. # and the chain-of-thought is stored only under the internal ``reasoning`` key. When the user later
  1448. # replays that history through a DeepSeek-v4 / Kimi thinking model, the missing
  1449. # ``reasoning_content`` causes HTTP 400 ("The reasoning_content in the thinking mode must be passed
  1450. # back to the API."). Promote the already-sanitized streamed ``reasoning_text`` to
  1451. # ``reasoning_content`` at write time, but ONLY when no prior branch already set it AND we actually
  1452. # captured reasoning text. This preserves every existing behavior: - SDK-exposed
  1453. # ``reasoning_content`` (OpenAI/Moonshot/DeepSeek SDK) still wins.
  1454. msg["reasoning_content"] = reasoning_text
  1455. if getattr(assistant_message, "reasoning_details", None):
  1456. # Preserve reasoning_details exactly (opaque signature /
  1457. # encrypted_content fields) for cross-turn reasoning continuity.
  1458. preserved = []
  1459. for d in assistant_message.reasoning_details:
  1460. if isinstance(d, dict):
  1461. preserved.append(d)
  1462. elif hasattr(d, "__dict__"):
  1463. preserved.append(d.__dict__)
  1464. elif hasattr(d, "model_dump"):
  1465. preserved.append(_model_dump_safe(d))
  1466. if preserved:
  1467. msg["reasoning_details"] = preserved
  1468. # Provider-native carriers replayed verbatim on later turns:
  1469. # anthropic_content_blocks keeps interleaved thinking + tool_use order
  1470. # (reconstruction reorders signed blocks -> HTTP 400); codex_* items are
  1471. # the encrypted reasoning / exact message items Responses prefix caching
  1472. # needs.
  1473. for attr in ("anthropic_content_blocks", "bedrock_content_blocks", "codex_reasoning_items", "codex_message_items"):
  1474. value = getattr(assistant_message, attr, None)
  1475. if value:
  1476. msg[attr] = value
  1477. if attr == "codex_reasoning_items":
  1478. from agent.codex_responses_adapter import (
  1479. has_replayable_native_compaction_checkpoint,
  1480. )
  1481. if has_replayable_native_compaction_checkpoint(agent, [msg]):
  1482. note_checkpoint = getattr(
  1483. agent.context_compressor, "note_native_compaction_checkpoint", None
  1484. )
  1485. if callable(note_checkpoint):
  1486. note_checkpoint()
  1487. # The response priced the pre-checkpoint input, not the next
  1488. # compacted request. A matching durable prefix is now stale.
  1489. from agent.usage_anchor import set_usage_anchor
  1490. set_usage_anchor(agent, None)
  1491. # The next request drops every item before this checkpoint, so a repeat
  1492. # read must serve content again, not an "unchanged" stub (#32106).
  1493. # Without a task id the reset would clear every task's caches.
  1494. if task_id := getattr(agent, "_current_task_id", None):
  1495. from agent.conversation_compression import _reset_read_dedup_caches
  1496. _reset_read_dedup_caches(task_id, session_id=getattr(agent, "session_id", None) or "")
  1497. from agent.reasoning_carriers import carrier_record
  1498. if record := carrier_record(assistant_message):
  1499. # Gemini text-turn signature / profile-declared reasoning carriers: top-level for the live
  1500. # history, a private reasoning_details record for persistence (reasoning_carriers.py).
  1501. msg.update({k: v for k, v in record.items() if k != "type"})
  1502. msg["reasoning_details"] = [*msg.get("reasoning_details", ()), record]
  1503. if assistant_tool_calls:
  1504. msg["tool_calls"] = [_assistant_tool_call_dict(agent, tc, i) for i, tc in enumerate(assistant_tool_calls)]
  1505. return msg
  1506. def rewrite_prompt_model_identity(agent, model: str, provider: str) -> None:
  1507. """Rewrite the cached prompt's ``Model:``/``Provider:`` lines after a provider switch.
  1508. Not persisted: the stored row keeps the primary's labels so a restored primary replays a
  1509. byte-identical prompt (prefix cache intact). Only the LAST occurrence of each line is touched —
  1510. earlier matches may be user content (memory snapshots, context files)."""
  1511. sp = getattr(agent, "_cached_system_prompt", None)
  1512. if not isinstance(sp, str) or not sp:
  1513. return
  1514. for label, value in (("Model", model), ("Provider", provider)):
  1515. if not value:
  1516. continue
  1517. matches = list(re.finditer(rf"(?m)^{label}: .*$", sp))
  1518. if matches:
  1519. last = matches[-1]
  1520. sp = f"{sp[:last.start()]}{label}: {value}{sp[last.end():]}"
  1521. agent._cached_system_prompt = sp
  1522. def _fallback_entry_key(fb: dict) -> tuple[str, str, str]:
  1523. return (str(fb.get("provider") or "").strip().lower(), str(fb.get("model") or "").strip(),
  1524. str(fb.get("base_url") or "").strip().rstrip("/"))
  1525. def _fallback_entry_unavailable_without_network(agent, fb: dict) -> Optional[str]:
  1526. """Return a skip reason for fallback entries known to be unusable locally."""
  1527. if (fb.get("provider") or "").strip().lower() != "nous":
  1528. return None
  1529. try:
  1530. from hermes_cli.auth import get_provider_auth_state
  1531. state = get_provider_auth_state("nous") or {}
  1532. except Exception as exc:
  1533. return f"nous_auth_unreadable:{type(exc).__name__}"
  1534. has_token = any(isinstance(t, str) and t.strip() for t in (state.get("access_token"), state.get("refresh_token")))
  1535. return None if has_token else "nous_token_missing"
  1536. _FALLBACK_REASON_LABELS = {
  1537. FailoverReason.auth: "authentication failed",
  1538. FailoverReason.auth_permanent: "authentication permanently failed",
  1539. FailoverReason.billing: "billing or quota exhausted",
  1540. FailoverReason.rate_limit: "rate limit",
  1541. FailoverReason.upstream_rate_limit: "upstream model rate limit",
  1542. FailoverReason.overloaded: "provider overloaded",
  1543. FailoverReason.server_error: "provider server error",
  1544. FailoverReason.timeout: "request timeout",
  1545. FailoverReason.ssl_cert_verification: "TLS certificate verification failed",
  1546. FailoverReason.context_overflow: "context window exceeded",
  1547. FailoverReason.payload_too_large: "request payload too large",
  1548. FailoverReason.image_too_large: "image payload too large",
  1549. FailoverReason.model_not_found: "model not found",
  1550. FailoverReason.provider_policy_blocked: "provider policy blocked the request",
  1551. FailoverReason.content_policy_blocked: "content policy blocked the request",
  1552. FailoverReason.format_error: "request format rejected",
  1553. FailoverReason.role_alternation: "adjacent same-role messages rejected",
  1554. FailoverReason.invalid_encrypted_content: "encrypted reasoning state rejected",
  1555. FailoverReason.multimodal_tool_content_unsupported: "multimodal tool content unsupported",
  1556. FailoverReason.thinking_signature: "thinking signature rejected",
  1557. FailoverReason.long_context_tier: "long-context tier unavailable",
  1558. FailoverReason.oauth_long_context_beta_forbidden: "OAuth long-context beta unavailable",
  1559. FailoverReason.llama_cpp_grammar_pattern: "grammar pattern rejected",
  1560. FailoverReason.unknown: "provider failure",
  1561. }
  1562. def _fallback_reason_text(reason: FailoverReason | None) -> str:
  1563. """Return a concise operator-facing explanation for a fallback switch."""
  1564. label = _FALLBACK_REASON_LABELS.get(reason)
  1565. return label or str(getattr(reason, "value", None) or reason or "provider failure").replace("_", " ")
  1566. def _is_anthropic_wire_url(url: str) -> bool:
  1567. """Same Messages-only host match as determine_api_mode() / _detect_api_mode_for_url(): api.anthropic.com,
  1568. a /anthropic suffix, or Kimi Code's api.kimi.com/coding (its /chat/completions 404s — #77256)."""
  1569. from hermes_cli.providers import host_mandated_api_mode
  1570. return host_mandated_api_mode(url) == "anthropic_messages"
  1571. def _fallback_api_mode_hint(fb: dict, fb_provider: str, fb_base_url_hint: Optional[str]) -> tuple[bool, str]:
  1572. """(explicit, api_mode) for a fallback entry from its ORIGINAL base_url: resolve_provider_client()
  1573. rewrites a dual-surface /anthropic base to /v1, losing the Anthropic wire signal. An explicit
  1574. ``api_mode`` always wins (even "chat_completions") and suppresses later re-detection;
  1575. ``provider: anthropic`` without a base_url still resolves to anthropic_messages."""
  1576. from hermes_cli.runtime_provider import _get_named_custom_provider, _parse_api_mode
  1577. # Entries accept the same ``api_mode`` / ``transport`` spellings as ``providers.<name>``.
  1578. explicit = _parse_api_mode(fb.get("api_mode") or fb.get("transport"))
  1579. if explicit:
  1580. return True, explicit
  1581. # A named ``providers.<name>`` block declares its wire once (``api_mode``/``transport``); a
  1582. # fallback entry naming that provider inherits it instead of being re-detected from the host
  1583. # (#33062, #81932: an Anthropic-Messages or Responses-only relay on a plain host was downgraded
  1584. # to chat_completions while resolve_provider_client had already built the declared client).
  1585. if fb_provider and fb_provider not in {"custom", "moa"}:
  1586. declared = (_get_named_custom_provider(fb_provider) or {}).get("api_mode")
  1587. if declared:
  1588. return True, declared
  1589. if fb_provider == "anthropic" or (fb_base_url_hint and _is_anthropic_wire_url(fb_base_url_hint)):
  1590. return False, "anthropic_messages"
  1591. return False, "chat_completions"
  1592. def _fallback_api_mode_resolved(agent, fb_provider: str, fb_model: str, fb_base_url: str) -> str:
  1593. """Re-detect api_mode from provider / resolved base URL / model when the hint pass
  1594. landed on the chat_completions default (never called for an explicit api_mode)."""
  1595. if fb_provider == "openai-codex":
  1596. return "codex_responses"
  1597. from hermes_cli.models import opencode_model_api_mode
  1598. from hermes_cli.runtime_provider_custom import _opencode_family_for_custom
  1599. opencode_family = _opencode_family_for_custom(fb_provider, fb_base_url)
  1600. if opencode_family is not None:
  1601. # OpenCode Zen/Go/free serve Responses-only (muse-spark, gpt-*, grok-*), anthropic_messages
  1602. # (minimax, qwen) and chat_completions models behind one provider; the primary /model path
  1603. # already re-derives per model — the fallback wire must agree (#102148).
  1604. return opencode_model_api_mode(opencode_family, fb_model)
  1605. if fb_provider in {"nous", "nous-portal", "nousresearch"}:
  1606. # Portal is dual-wire: anthropic/* must land on /v1/messages (the swap rebuilds the native client).
  1607. from hermes_cli.providers import nous_api_mode
  1608. return nous_api_mode(fb_model)
  1609. if _is_anthropic_wire_url(fb_base_url):
  1610. # Named custom providers (cron-anthropic) resolve base_url from config; the hint pass never saw it.
  1611. return "anthropic_messages"
  1612. if agent._is_azure_openai_url(fb_base_url):
  1613. return "chat_completions" # Azure serves gpt-5.x on /chat/completions — no Responses API.
  1614. # Per-model provider exceptions stay inside the requires-responses predicate.
  1615. if agent._is_direct_openai_url(fb_base_url) or agent._provider_model_requires_responses_api(fb_model, provider=fb_provider):
  1616. return "codex_responses"
  1617. host = base_url_hostname(fb_base_url)
  1618. if fb_provider == "bedrock" or (host.startswith("bedrock-runtime.") and base_url_host_matches(fb_base_url, "amazonaws.com")):
  1619. return "bedrock_converse"
  1620. return "chat_completions"
  1621. def _rebind_fallback_credential_pool(agent, fb_provider: str, fb_model: str) -> None:
  1622. """Rebind the credential pool when the provider changes (else rate_limit/billing/auth recovery
  1623. mutates the wrong credentials and overwrites the fallback's base_url). Same-provider pool: kept."""
  1624. existing_pool = getattr(agent, "_credential_pool", None)
  1625. if existing_pool is not None:
  1626. pool_provider = (getattr(existing_pool, "provider", "") or "").strip().lower()
  1627. if pool_provider and pool_provider != fb_provider:
  1628. logger.info(
  1629. "Fallback to %s/%s: clearing primary credential pool (pool_provider=%s) to prevent cross-provider contamination",
  1630. fb_provider, fb_model, pool_provider)
  1631. agent._credential_pool = agent._credential_pool_entry_id = None
  1632. if getattr(agent, "_credential_pool", None) is None:
  1633. try:
  1634. from agent.credential_pool import load_pool
  1635. fallback_pool = load_pool(fb_provider)
  1636. if fallback_pool and fallback_pool.has_credentials():
  1637. agent._credential_pool = fallback_pool
  1638. logger.info("Fallback to %s/%s: attached fallback credential pool", fb_provider, fb_model)
  1639. except Exception as exc:
  1640. logger.debug("Fallback to %s/%s: could not attach credential pool: %s", fb_provider, fb_model, exc)
  1641. def _log_fallback_activated(agent, reason, old_model, old_provider, fb_model, fb_provider) -> None:
  1642. """A billing switch is a WARNING naming the profile, both models and the remedy: the gateway
  1643. persists the turn as a transient failure otherwise, and nothing in the log says the paid
  1644. model was refused for credits or how to fix it (#115702). Other reasons stay INFO."""
  1645. if reason != FailoverReason.billing:
  1646. logger.info("Fallback activated: %s → %s (%s)", old_model, fb_model, fb_provider)
  1647. return
  1648. from hermes_constants import get_hermes_home, profile_name_for_home
  1649. profile = profile_name_for_home(get_hermes_home()) or "default"
  1650. remedy = "hermes model" if profile == "default" else f"hermes -p {profile} model"
  1651. logger.warning(
  1652. "Profile %s: %s via %s refused for billing/credits — using fallback %s via %s. "
  1653. "Top up credits, or run `%s` to pick a model this account can use.",
  1654. profile, old_model, old_provider, fb_model, fb_provider, remedy,
  1655. )
  1656. def _fallback_chain_exhausted(agent, reason: FailoverReason | None) -> bool:
  1657. """Chain exhausted (always False). A non-empty chain walked on a non-rate-limit failure arms a
  1658. short cooldown so next turn's restore_primary_runtime stays gated instead of replaying the whole
  1659. context across every provider again."""
  1660. from agent.fallback_cooldown import _RATE_LIMIT_FAILOVER_REASONS
  1661. if agent._fallback_chain and reason not in _RATE_LIMIT_FAILOVER_REASONS:
  1662. agent._rate_limited_until = max(
  1663. getattr(agent, "_rate_limited_until", 0) or 0, time.monotonic() + _FALLBACK_EXHAUSTED_COOLDOWN_S)
  1664. return False
  1665. def _candidate_pool_exhausted(agent, fb_provider: str, fb_model: str) -> bool:
  1666. """True when every credential the candidate would use sits in an exhaustion cooldown longer
  1667. than the retry loop's longest wait (the 600s Retry-After cap): switching to it only fails the
  1668. turn the same way the primary just did (#89401). A short throttle still gets its chance."""
  1669. pool = getattr(agent, "_credential_pool", None)
  1670. if pool is None or (getattr(pool, "provider", "") or "").strip().lower() != fb_provider:
  1671. try:
  1672. from agent.credential_pool import load_pool
  1673. pool = load_pool(fb_provider)
  1674. except Exception:
  1675. return False
  1676. if pool is None or not pool.has_credentials() or pool.has_available(model=fb_model):
  1677. return False
  1678. until = pool.next_available_at(model=fb_model)
  1679. return until is None or until - time.time() > 600
  1680. def _should_skip_fallback_candidate(agent, fb: dict, fb_key: tuple, fb_provider: str, fb_model: str, unavailable: set) -> bool:
  1681. """True when the entry is already unavailable, malformed, locally unusable, or resolves
  1682. to the backend that just failed (falling back to it would loop the failure)."""
  1683. if fb_key in unavailable:
  1684. logger.debug("Fallback skip: %s previously marked unavailable", fb_key)
  1685. return True
  1686. if not fb_provider or not fb_model:
  1687. return True
  1688. from agent.fallback_cooldown import _is_entitlement_rejected
  1689. if _is_entitlement_rejected(agent, fb_provider, fb_model):
  1690. logger.info("Fallback skip: %s/%s was rejected as unentitled for this account", fb_provider, fb_model)
  1691. return True
  1692. if _candidate_pool_exhausted(agent, fb_provider, fb_model):
  1693. logger.warning("Fallback skip: %s/%s credential pool is exhausted (every entry in cooldown)", fb_provider, fb_model)
  1694. return True
  1695. local_skip_reason = _fallback_entry_unavailable_without_network(agent, fb)
  1696. if local_skip_reason:
  1697. unavailable.add(fb_key)
  1698. logger.warning("Fallback skip: %s/%s is not locally usable (%s); suppressing for this session", fb_provider, fb_model, local_skip_reason)
  1699. return True
  1700. # Identity semantics (axes, shim aliases, credential surfaces, multi-endpoint pools)
  1701. # are owned by agent.backend_identity — do not re-implement comparisons here.
  1702. # Skip entries that resolve to the same backend that just failed — falling back to it loops the failure.
  1703. # See #22548, #62984, #70893.
  1704. from agent.backend_identity import BackendIdentity, should_skip_candidate
  1705. current_ident = BackendIdentity.build(provider=getattr(agent, "provider", ""),
  1706. model=getattr(agent, "model", ""), base_url=str(getattr(agent, "base_url", "") or ""))
  1707. fb_ident = BackendIdentity.build(provider=fb_provider, model=fb_model, base_url=(fb.get("base_url") or ""))
  1708. if should_skip_candidate(fb_ident, current_ident):
  1709. logger.warning(
  1710. "Fallback skip: chain entry %s/%s resolves to the same backend as the current one (%s)",
  1711. fb_provider, fb_model, current_ident.base_url or current_ident.provider)
  1712. return True
  1713. return False
  1714. def _update_fallback_context_compressor(agent) -> None:
  1715. """Point compression limits at the fallback model's context window (not the primary's),
  1716. respecting the explicit model.context_length config override."""
  1717. compressor = getattr(agent, "context_compressor", None)
  1718. if not compressor:
  1719. return
  1720. from agent.model_metadata import get_model_context_length
  1721. fb_context_length = get_model_context_length(
  1722. agent.model, base_url=agent.base_url,
  1723. api_key=agent.api_key if isinstance(agent.api_key, str) else "", # callable (Entra ID) → probes need str
  1724. provider=agent.provider,
  1725. config_context_length=getattr(agent, "_config_context_length", None),
  1726. custom_providers=getattr(agent, "_custom_providers", None),
  1727. )
  1728. compressor.update_model( # callable api_key preserved → call_llm
  1729. model=agent.model, context_length=fb_context_length, base_url=agent.base_url,
  1730. api_key=getattr(agent, "api_key", ""), provider=agent.provider, api_mode=agent.api_mode,
  1731. )
  1732. # Fallback activation is an error path: refresh an EXISTING verdict eagerly (the ceiling was voided by
  1733. # update_model()), but a session that never probed keeps its lazy compaction-time probe rather than
  1734. # resolving an auxiliary client while the primary route is failing (#114707).
  1735. if getattr(agent, "_compression_feasibility_checked", False) is True:
  1736. from agent.conversation_compression_feasibility import revalidate_compression_feasibility
  1737. revalidate_compression_feasibility(agent)
  1738. def _reresolve_fallback_reasoning_config(agent) -> None:
  1739. """Per-model override > global reasoning_effort (YAML False = disabled); a config load
  1740. failure keeps the current reasoning_config rather than killing the swap."""
  1741. try:
  1742. # Re-resolve reasoning_config for the new fallback model (Closes #21256). Wrapped in try/except
  1743. # because a config load failure must not kill the swap.
  1744. from hermes_cli.config import load_config
  1745. from hermes_constants import resolve_reasoning_config
  1746. agent.reasoning_config = resolve_reasoning_config(load_config() or {}, agent.model)
  1747. logger.info("Fallback %s: reasoning_config resolved: %s", agent.model, agent.reasoning_config)
  1748. except Exception as _reasoning_err:
  1749. logger.debug("Failed to resolve reasoning_config for fallback %s; keeping current: %s", agent.model, _reasoning_err)
  1750. def _rescope_fallback_extra_body(agent, old_model: str, old_provider: str, old_base_url: str) -> None:
  1751. """Drop the OLD provider's custom_providers-contributed extra_body keys, then merge the fallback
  1752. provider's own. KEY-SCOPED: a key is dropped only if its value still equals what the old provider's
  1753. config injected — a caller override of the same key won at init and differs, so it survives;
  1754. keys the new provider redefines are re-added by the merge."""
  1755. try:
  1756. from agent.agent_init import _custom_provider_extra_body_for_agent, _merge_custom_provider_extra_body
  1757. custom_providers = getattr(agent, "_custom_providers", None) or []
  1758. old_provider_eb = _custom_provider_extra_body_for_agent(provider=old_provider, model=old_model, base_url=old_base_url, custom_providers=custom_providers) or {}
  1759. overrides = dict(getattr(agent, "request_overrides", {}) or {})
  1760. existing_eb = overrides.get("extra_body")
  1761. if isinstance(existing_eb, dict) and old_provider_eb:
  1762. scrubbed = {k: v for k, v in existing_eb.items() if not (k in old_provider_eb and v == old_provider_eb[k])}
  1763. if scrubbed:
  1764. overrides["extra_body"] = scrubbed
  1765. else:
  1766. overrides.pop("extra_body", None)
  1767. agent.request_overrides = overrides
  1768. _merge_custom_provider_extra_body(agent, custom_providers)
  1769. logger.info("Fallback %s: extra_body resolved: %s", agent.model, (getattr(agent, "request_overrides", {}) or {}).get("extra_body"))
  1770. except Exception as _eb_err:
  1771. logger.debug("Failed to resolve extra_body for fallback %s; keeping current: %s", agent.model, _eb_err)
  1772. def _buffer_fallback_notice(agent, notice: str) -> None:
  1773. """Buffer the switch notice for terminal failure AND retain it as a durable one-shot for
  1774. _emit_pending_fallback_notice (a successful fallback clears retry chatter)."""
  1775. agent._buffer_diagnostic_status(notice)
  1776. pending = getattr(agent, "_pending_fallback_notice", None)
  1777. if isinstance(pending, list):
  1778. pending.append(notice)
  1779. else:
  1780. agent._pending_fallback_notice = [str(pending), notice] if pending else [notice]
  1781. def try_activate_fallback(agent, reason: FailoverReason | None = None, reset_at=None) -> bool:
  1782. """Switch to the next fallback model/provider in the chain; False when exhausted. Swaps client,
  1783. model slug and provider in place so the retry loop continues on the new backend; client
  1784. construction goes through resolve_provider_client (no duplicated provider→key mappings)."""
  1785. from agent.fallback_cooldown import _arm_rate_limit_cooldown, switch_deferred_by_reset
  1786. if switch_deferred_by_reset(agent, reason, reset_at):
  1787. return False
  1788. cooldown_seconds = _arm_rate_limit_cooldown(agent, reason, reset_at=reset_at)
  1789. while True:
  1790. if agent._fallback_index >= len(agent._fallback_chain):
  1791. return _fallback_chain_exhausted(agent, reason)
  1792. fb = agent._fallback_chain[agent._fallback_index]
  1793. agent._fallback_index += 1
  1794. fb_key = _fallback_entry_key(fb)
  1795. if getattr(agent, "_unavailable_fallback_keys", None) is None:
  1796. agent._unavailable_fallback_keys = set()
  1797. unavailable = agent._unavailable_fallback_keys
  1798. fb_provider = (fb.get("provider") or "").strip().lower()
  1799. fb_model = (fb.get("model") or "").strip()
  1800. if _should_skip_fallback_candidate(agent, fb, fb_key, fb_provider, fb_model, unavailable):
  1801. continue
  1802. try:
  1803. from agent.route_binding import bind_route_entry
  1804. bound = bind_route_entry(agent, fb, fb_provider, fb_model)
  1805. if bound is None:
  1806. logger.warning("Fallback to %s failed: provider not configured", fb_provider)
  1807. unavailable.add(fb_key)
  1808. continue
  1809. old_model, old_provider = bound
  1810. fb_model = agent.model # normalized by the binder
  1811. rewrite_prompt_model_identity(agent, fb_model, fb_provider)
  1812. notice = (
  1813. f"⚠️ Model fallback: {old_model} via {old_provider} unavailable "
  1814. f"({_fallback_reason_text(reason)}); using {fb_model} via {fb_provider}.")
  1815. if cooldown_seconds is not None:
  1816. remaining = max(0, math.ceil(agent._rate_limited_until - time.monotonic()))
  1817. notice += f" Primary retry eligible in ~{remaining} s; recovery is not guaranteed."
  1818. _buffer_fallback_notice(agent, notice)
  1819. # ``_fallback_activated`` is also reused by `/model --once` restoration; separate
  1820. # provenance so the restore path only emits a recovery notice after a real fallback.
  1821. agent._provider_fallback_active = True
  1822. agent._provider_fallback_route = (str(fb_model), str(fb_provider))
  1823. _log_fallback_activated(agent, reason, old_model, old_provider, fb_model, fb_provider)
  1824. from hermes_cli.observability.shared_metrics_events import record_fallback
  1825. record_fallback(from_provider=old_provider, to_provider=fb_provider, reason=reason)
  1826. # The stale-call streak measured the OLD provider; carrying it over would
  1827. # short-circuit the fresh fallback before its first stream attempt.
  1828. _reset_stale_streak(agent)
  1829. from agent.native_compaction import resolve_native_compaction_capabilities
  1830. agent.runtime_capabilities = resolve_native_compaction_capabilities(
  1831. model=agent.model, base_url=agent.base_url, provider=fb_provider, is_codex_backend=fb_provider == "openai-codex")
  1832. return True
  1833. except Exception as e:
  1834. if fb_provider == "nous":
  1835. unavailable.add(fb_key)
  1836. logger.error("Failed to activate fallback %s: %s", fb_model, e)
  1837. continue # try next in chain
  1838. # Keys outside the Chat Completions schema that strict gateways (Fireworks-backed OpenCode
  1839. # Go, Mistral, Moonshot/Kimi) reject with 422. The transport's convert_messages() drops them
  1840. # in the main loop; the summary path calls chat.completions.create() directly, so mirror it.
  1841. _SUMMARY_FOREIGN_MESSAGE_KEYS = PERSISTENCE_ONLY_MESSAGE_FIELDS | {"reasoning", "finish_reason", "tool_name",
  1842. "codex_reasoning_items", "codex_message_items", "platform_message_id"}
  1843. _EMPTY_SUMMARY_RESPONSE = "I reached the iteration limit and couldn't generate a summary."
  1844. def _iteration_summary_api_messages(agent, messages: list) -> list:
  1845. """Wire-ready messages for the summary call, mirroring the main loop's api_messages build
  1846. (sidecar substitution, tool-call repair, thinking-only drop, underscore-key sweep).
  1847. ``reasoning_details`` is kept: the anthropic_messages converter rebuilds signed thinking
  1848. blocks from it, and the chat-completions transport already drops it on the wire for routes
  1849. that do not replay it (``_chat_summary_attempt`` -> ``_build_api_kwargs``)."""
  1850. needs_sanitize = agent._should_sanitize_tool_calls()
  1851. sanitize_model = agent.model
  1852. if needs_sanitize and agent.provider == "moa":
  1853. # MoA: agent.model is the virtual preset; use the real aggregator so Gemini keeps thought_signature.
  1854. agg_slot = getattr(getattr(agent, "client", None), "last_aggregator_slot", None)
  1855. sanitize_model = (agg_slot or {}).get("model") or sanitize_model
  1856. api_messages = []
  1857. for msg in messages:
  1858. api_msg = msg.copy()
  1859. agent._copy_reasoning_content_for_api(msg, api_msg)
  1860. for key in _SUMMARY_FOREIGN_MESSAGE_KEYS:
  1861. api_msg.pop(key, None)
  1862. # Mirror of the transport's role-qualified strip: ``name`` is
  1863. # schema-foreign on tool results only (strict providers reject with
  1864. # "contains item with unknown key name"); it stays on user/assistant.
  1865. if api_msg.get("role") == "tool":
  1866. api_msg.pop("name", None)
  1867. # api_content holds the exact bytes the main loop sent; substituting (not popping)
  1868. # keeps the summary's prefix identical instead of re-prefilling the largest context.
  1869. # Strict OpenAI-compatible gateways (Fireworks-backed OpenCode Go, Mistral, Moonshot/Kimi) reject
  1870. # any message key outside the Chat Completions schema. The main loop drops these via
  1871. # ChatCompletionsTransport.convert_messages(), but the summary path hand-builds messages and calls
  1872. # chat.completions.create() directly, bypassing the transport — so mirror that sanitization here:
  1873. # tool_name (SQLite FTS bookkeeping), the codex_* reasoning carriers, timestamp (preserved on
  1874. # gateway user replay entries for the stale-confirmation expiry check — #47868 rejection class), and
  1875. # every Hermes-internal underscore-prefixed scaffolding key.
  1876. substitute_api_content(api_msg)
  1877. if needs_sanitize:
  1878. agent._sanitize_tool_calls_for_strict_api(api_msg, model=sanitize_model)
  1879. api_messages.append(api_msg)
  1880. effective_system = agent._cached_system_prompt or ""
  1881. if agent.ephemeral_system_prompt:
  1882. effective_system = (effective_system + "\n\n" + agent.ephemeral_system_prompt).strip()
  1883. if effective_system:
  1884. api_messages = [{"role": "system", "content": effective_system}] + api_messages
  1885. for idx, pfm in enumerate(agent.prefill_messages or ()):
  1886. api_messages.insert((1 if effective_system else 0) + idx, pfm.copy())
  1887. # Compression/resume can orphan a tool result whose parent tool_call was summarized away.
  1888. api_messages = agent._sanitize_api_messages(api_messages)
  1889. # Same send-path vision eviction as the main loop (#89296).
  1890. from agent.context_compressor import evict_stale_outbound_tool_images, outbound_retired_image_ids
  1891. evict_stale_outbound_tool_images(api_messages, outbound_retired_image_ids(agent))
  1892. # Same per-model image strip as turn_api_request.build_api_request: this path builds
  1893. # api_messages by hand and calls _build_api_kwargs directly, so a model recorded in
  1894. # agent._image_rejecting_models would otherwise get images here → 4xx → no summary.
  1895. # Safe on the shallow row copies: the strip rebinds the row's ``content``, never the
  1896. # nested list shared with history.
  1897. strip_images_for_rejecting_model(agent, api_messages)
  1898. # Thinking-only assistant turns 400 on Anthropic-family providers; _thinking_prefill must
  1899. # survive until here so the drop pass recognizes stubs after reasoning is stripped.
  1900. api_messages = agent._drop_thinking_only_and_merge_users(api_messages)
  1901. for api_msg in api_messages: # underscore scaffolding: the transport's sweeper is bypassed here
  1902. if isinstance(api_msg, dict):
  1903. for internal_key in [k for k in api_msg if isinstance(k, str) and k.startswith("_")]:
  1904. del api_msg[internal_key]
  1905. # Same closing normalization as assemble_api_request so the summary's prefix stays
  1906. # bit-identical to the main loop's (a diverging early row defeats prefix caching).
  1907. for api_msg in api_messages:
  1908. if isinstance(api_msg.get("content"), str):
  1909. api_msg["content"] = api_msg["content"].strip()
  1910. from agent.conversation_loop import _canonicalize_api_tool_calls, _clone_message_for_send
  1911. _canonicalize_api_tool_calls(api_messages)
  1912. # Third closing pass of the main path: lone surrogates -> U+FFFD (else the SDK's utf-8
  1913. # wire encode raises and burns the summary retries). The sanitizer is in-place and these
  1914. # rows still share nested dicts with history, so clone first like the main path does.
  1915. api_messages = [_clone_message_for_send(m) for m in api_messages]
  1916. _sanitize_messages_surrogates(api_messages)
  1917. return api_messages
  1918. def _managed_summary_call(agent, api_request_id: str, request, callback, *, retry_count: int):
  1919. from agent import relay_llm
  1920. return relay_llm.execute_current(
  1921. request, callback,
  1922. name=str(getattr(agent, "provider", "") or "provider"), model_name=str(getattr(agent, "model", "") or ""),
  1923. metadata={"api_mode": str(getattr(agent, "api_mode", "") or "chat_completions"),
  1924. "api_request_id": api_request_id, "call_role": "iteration_summary", "retry_count": retry_count},
  1925. defer_logical_completion=True,
  1926. )
  1927. def _summary_text(agent, response, **normalize_kwargs) -> str:
  1928. if is_router_timeout_shim(response):
  1929. # Router failure in a 200 envelope (#68396): an empty summary takes the retry slot.
  1930. logger.warning("Iteration summary returned a router timeout shim; retrying")
  1931. return ""
  1932. normalized = agent._get_transport().normalize_response(response, **normalize_kwargs)
  1933. if normalized.tool_calls:
  1934. # No summary path executes tool calls; log so a tool-only response that falls into the
  1935. # empty-summary retry is diagnosable.
  1936. logger.warning("Iteration summary emitted tool calls; discarding them")
  1937. return (normalized.content or "").strip()
  1938. def _codex_summary_attempt(agent, api_messages: list, api_request_id: str):
  1939. def _attempt(retry_count: int) -> str:
  1940. codex_kwargs = agent._build_api_kwargs(api_messages)
  1941. # The transport emits these three as one block (transports/codex.py build_kwargs);
  1942. # strict Responses backends 400 on tool_choice/parallel_tool_calls without tools.
  1943. codex_kwargs.pop("tools", None)
  1944. codex_kwargs.pop("tool_choice", None)
  1945. codex_kwargs.pop("parallel_tool_calls", None)
  1946. # Route through the same seam as normal Codex turns: a direct _run_codex_stream
  1947. # bypasses the stale/TTFB watchdogs, interrupt handling and client cleanup, so an
  1948. # unattended cron summary could wedge forever (#70943).
  1949. return _summary_text(agent, agent._interruptible_api_call(codex_kwargs))
  1950. return _attempt
  1951. def _anthropic_summary_attempt(agent, api_messages: list, api_request_id: str):
  1952. def _attempt(retry_count: int) -> str:
  1953. ant_kw = agent._get_transport().build_kwargs(
  1954. model=agent.model, messages=api_messages, tools=None, max_tokens=agent.max_tokens,
  1955. reasoning_config=agent.reasoning_config, is_oauth=agent._is_anthropic_oauth,
  1956. preserve_dots=agent._anthropic_preserve_dots(), base_url=getattr(agent, "_anthropic_base_url", None))
  1957. ant_kw = _merge_nous_portal_messages_extra_body(agent, ant_kw, api_messages)
  1958. response = _managed_summary_call(
  1959. agent, api_request_id, ant_kw, agent._interruptible_api_call, retry_count=retry_count)
  1960. return _summary_text(agent, response, strip_tool_prefix=agent._is_anthropic_oauth)
  1961. return _attempt
  1962. def _chat_summary_attempt(agent, api_messages: list, api_request_id: str):
  1963. # Same kwargs builder as the main loop so the summary keeps the cached prefix (tools,
  1964. # prompt_cache_key, xAI alias, Moonshot sanitization). Do not omit tools or force
  1965. # tool_choice="none" here: SGLang renders the prompt with tools=None in that mode and the KV
  1966. # prefix diverges. (cache_control breakpoint decoration is not re-applied on this path.)
  1967. summary_kwargs = agent._build_api_kwargs(api_messages)
  1968. # The summary now carries ``tools``; on cache-planned routes the main loop scrubbed a deep
  1969. # copy, so ``agent.tools`` may still hold bytes the provider 400s on.
  1970. sanitize_outbound_kwargs(agent, summary_kwargs)
  1971. from agent.conversation_loop import _moa_client_consumes_prepared_request
  1972. if agent.provider == "moa" and _moa_client_consumes_prepared_request(agent.client): # transport strips data_use
  1973. summary_kwargs["_moa_prepared_request"] = agent.client.chat.completions.prepare(api_messages)
  1974. def _attempt(retry_count: int) -> str:
  1975. # Use the ordinary request-local lifecycle: a summary can be interrupted
  1976. # during a long prefill without closing the shared primary client.
  1977. response = _managed_summary_call(
  1978. agent, api_request_id, summary_kwargs, agent._interruptible_api_call,
  1979. retry_count=retry_count)
  1980. return _summary_text(agent, response)
  1981. return _attempt
  1982. _SUMMARY_ATTEMPT_BUILDERS = {"codex_responses": _codex_summary_attempt, "anthropic_messages": _anthropic_summary_attempt}
  1983. def handle_max_iterations(agent, messages: list, api_call_count: int) -> str:
  1984. """Request a summary when max iterations are reached. Returns the final response text."""
  1985. warning = f"⚠️ Reached maximum iterations ({agent.max_iterations}). Requesting summary..."
  1986. if getattr(agent, "suppress_status_output", False):
  1987. # Strict machine-readable mode (-Q, oneshot): keep diagnostics off stdout. quiet_mode is
  1988. # NOT the gate — the interactive CLI runs quiet_mode=True by default and must see this (#93220 class).
  1989. logger.warning(warning)
  1990. else:
  1991. agent._safe_print(warning, diagnostic=True)
  1992. summary_api_request_id = f"iteration-summary:{uuid.uuid4()}"
  1993. summary_call_outcome = "failed"
  1994. # Shared constant so compaction recognizers can identify this runtime nudge by its stable
  1995. # content after SessionDB projection strips metadata flags.
  1996. from agent.context_compressor import MAX_ITERATIONS_SUMMARY_REQUEST
  1997. nudge = append_message(messages, {"role": "user", "content": MAX_ITERATIONS_SUMMARY_REQUEST})
  1998. try:
  1999. api_messages = _iteration_summary_api_messages(agent, messages)
  2000. build_attempt = _SUMMARY_ATTEMPT_BUILDERS.get(agent.api_mode, _chat_summary_attempt)
  2001. attempt = build_attempt(agent, api_messages, summary_api_request_id)
  2002. # One retry on an empty summary; a summary empty once its <think> block is stripped is NOT retried.
  2003. final_response = _EMPTY_SUMMARY_RESPONSE
  2004. for retry_count in (0, 1):
  2005. text = attempt(retry_count)
  2006. if not text:
  2007. continue
  2008. if "<think>" in text:
  2009. text = re.sub(r'<think>.*?</think>\s*', '', text, flags=re.DOTALL).strip()
  2010. if text:
  2011. summary_call_outcome = "success"
  2012. append_message(messages, {"role": "assistant", "content": text})
  2013. final_response = text
  2014. break
  2015. except InterruptedError:
  2016. # Cancellation is not a summary failure: drop the unanswered nudge and let the
  2017. # finalizer end the turn as interrupted so the pending message is requeued.
  2018. summary_call_outcome = "cancelled"
  2019. if messages and messages[-1] is nudge:
  2020. messages.pop()
  2021. raise
  2022. except Exception as e:
  2023. logger.warning("Failed to get summary response: %s", e)
  2024. from agent.turn_failure_copy import site_copy
  2025. final_response = site_copy("max_iterations_no_summary", limit=agent.max_iterations)
  2026. finally:
  2027. from agent import relay_llm
  2028. relay_llm.complete_logical_call(summary_api_request_id, outcome=summary_call_outcome)
  2029. return final_response
  2030. def cleanup_task_resources(agent, task_id: str) -> None:
  2031. """Per-turn VM + browser cleanup for a task. Skips ``cleanup_vm`` for persistent
  2032. terminal envs (``_cleanup_inactive_envs`` reaps them after ``terminal.lifetime_seconds``)
  2033. and ``cleanup_browser`` in headed mode (the inactivity reaper handles idle sessions)."""
  2034. def _headed() -> bool:
  2035. from tools.browser_session_cloud import is_headed_mode
  2036. return is_headed_mode()
  2037. for label, skip, skip_what, cleanup in (
  2038. ("VM", is_persistent_env, "cleanup_vm for persistent env", lambda: _ra().cleanup_vm(task_id)),
  2039. ("browser", lambda _tid: _headed(), "cleanup_browser for headed session", lambda: _ra().cleanup_browser(task_id)),
  2040. ):
  2041. try:
  2042. if skip(task_id):
  2043. if agent.verbose_logging:
  2044. logging.debug(f"Skipping per-turn {skip_what} {task_id}; idle reaper will handle it.")
  2045. else:
  2046. cleanup()
  2047. except Exception as e:
  2048. if agent.verbose_logging:
  2049. logger.warning("Failed to cleanup %s for task %s: %s", label, task_id, e)
  2050. def _build_partial_stream_stub(role, full_content, full_reasoning, model_name, usage_obj, *,
  2051. dropped_tool_names=None, overflow_terminal=False, api_mode=None, clean_eof=False):
  2052. """Stub for an SSE stream that ended without ``finish_reason`` after
  2053. delivering content. Tagged ``PARTIAL_STREAM_STUB_ID`` + ``FINISH_REASON_LENGTH``
  2054. so the loop enters its continuation/retry path instead of accepting
  2055. truncated output as a complete turn (#32086).
  2056. ``overflow_terminal`` (``full_content=None``): the stream died on a
  2057. context-overflow error. Seeding the recovered text as a continuation stub
  2058. would grow every later request into the same overflow (#106260); the loop
  2059. treats the marker as terminal and ends the turn via the recovery contract.
  2060. ``api_mode="anthropic_messages"`` returns a Messages-shaped stub (``content``
  2061. block list + ``stop_reason="max_tokens"``) so AnthropicTransport validates it
  2062. and the loop continues instead of entering the invalid-response retry ladder
  2063. (#45908). Empty content keeps one empty text block: validate_response rejects
  2064. an empty list for ``max_tokens``.
  2065. ``clean_eof``: the stream ended with no transport exception and no
  2066. ``finish_reason`` (server/intermediary closed cleanly). Only the two
  2067. clean-EOF sites in ``_finish_chat_stream`` pass True; the stub built after a
  2068. real transport exception keeps False so the loop can word the two failure
  2069. modes differently (#102766).
  2070. """
  2071. if api_mode == "anthropic_messages":
  2072. return SimpleNamespace(
  2073. id=PARTIAL_STREAM_STUB_ID,
  2074. type="message",
  2075. role=role,
  2076. model=model_name,
  2077. content=[SimpleNamespace(type="text", text=full_content or "")],
  2078. stop_reason="max_tokens",
  2079. stop_sequence=None,
  2080. usage=usage_obj,
  2081. _dropped_tool_names=dropped_tool_names or None,
  2082. _overflow_terminal=overflow_terminal,
  2083. _clean_eof=clean_eof,
  2084. )
  2085. return SimpleNamespace(
  2086. id=PARTIAL_STREAM_STUB_ID,
  2087. model=model_name,
  2088. choices=[SimpleNamespace(
  2089. index=0,
  2090. message=SimpleNamespace(role=role, content=full_content, tool_calls=None,
  2091. reasoning_content=full_reasoning),
  2092. finish_reason=FINISH_REASON_LENGTH,
  2093. )],
  2094. usage=usage_obj,
  2095. _dropped_tool_names=dropped_tool_names or None,
  2096. _overflow_terminal=overflow_terminal,
  2097. _clean_eof=clean_eof,
  2098. )
  2099. # SSE error events from proxies (OpenRouter's {"error":{"message":"Network
  2100. # connection lost."}}) surface as SDK APIError without a status_code (unlike
  2101. # APIStatusError). They mean the upstream stream died: retry with a fresh
  2102. # connection like an httpx drop.
  2103. _SSE_CONN_PHRASES = ("connection lost", "connection reset", "connection closed", "connection terminated",
  2104. "network error", "network connection", "terminated", "peer closed", "broken pipe",
  2105. "upstream connect error")
  2106. def _rejects_stream_options(exc: BaseException) -> bool:
  2107. """A 400/422 whose body names ``stream_options`` as an unknown/extra field: strict
  2108. OpenAI-compatible endpoints (Azure AI Foundry MaaS, Pydantic ``extra_forbidden``) reject
  2109. the usage extension outright (#9705). Distinct from "stream not supported", which flips
  2110. the whole session to non-streaming."""
  2111. if getattr(exc, "status_code", None) not in (400, 422):
  2112. return False
  2113. body = f"{getattr(exc, 'body', '') or ''} {exc}".lower()
  2114. return "stream_options" in body and any(
  2115. k in body for k in ("extra", "not supported", "unrecognized", "unexpected", "unknown"))
  2116. def _wait_stream_retry_backoff(agent, delay: float) -> None:
  2117. """Sleep ``delay`` seconds in 0.1s steps, returning early as soon as the agent
  2118. is interrupted (so /stop is never held hostage by a backoff; the retry loop's
  2119. own interrupt check then ends the call)."""
  2120. deadline = time.monotonic() + max(0.0, delay)
  2121. while not getattr(agent, "_interrupt_requested", False):
  2122. remaining = deadline - time.monotonic()
  2123. if remaining <= 0:
  2124. return
  2125. time.sleep(min(0.1, remaining))
  2126. def _anthropic_connection_error_types() -> tuple:
  2127. # An Anthropic error instance implies the SDK is already imported; never import
  2128. # (or lazy-install) it from inside an error handler.
  2129. anthropic = sys.modules.get("anthropic")
  2130. return (anthropic.APIConnectionError,) if anthropic is not None else ()
  2131. def _is_sse_connection_error(exc: BaseException) -> bool:
  2132. from openai import APIError as _APIError
  2133. if not isinstance(exc, _APIError) or getattr(exc, "status_code", None):
  2134. return False
  2135. err_lower = str(exc).lower()
  2136. return any(phrase in err_lower for phrase in _SSE_CONN_PHRASES)
  2137. def _relay_stream_identity(agent, name_default: str) -> dict:
  2138. """``session_id``/``name``/``model_name`` kwargs for ``relay_llm.stream``."""
  2139. return {"session_id": str(getattr(agent, "session_id", "") or ""),
  2140. "name": str(getattr(agent, "provider", "") or name_default),
  2141. "model_name": str(getattr(agent, "model", "") or "")}
  2142. def _relay_stream_metadata(agent, api_mode: str) -> dict:
  2143. call_role = ("delegated" if getattr(agent, "is_subagent", False)
  2144. else "fallback" if int(getattr(agent, "_fallback_index", 0) or 0) > 0 else "primary")
  2145. return {"api_mode": api_mode, "api_request_id": getattr(agent, "_current_api_request_id", None),
  2146. "call_role": call_role}
  2147. def _stream_final_text(response) -> str:
  2148. with contextlib.suppress(Exception):
  2149. choices = getattr(response, "choices", None)
  2150. first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None
  2151. content = getattr(getattr(first_choice, "message", None), "content", None)
  2152. if isinstance(content, str):
  2153. return content
  2154. with contextlib.suppress(Exception):
  2155. content = getattr(response, "content", None)
  2156. if isinstance(content, str):
  2157. return content
  2158. if isinstance(content, list):
  2159. return "".join(t for t in (getattr(part, "text", None) for part in content) if isinstance(t, str))
  2160. return ""
  2161. def _with_stream_emitters(agent, run):
  2162. """Bracket ``run()`` with the agent's ``_emit_stream_start`` / ``_emit_stream_end``
  2163. hooks when present (end carries the final text on success, the error string on
  2164. failure) and re-raise."""
  2165. start = getattr(agent, "_emit_stream_start", None)
  2166. if start is not None:
  2167. start()
  2168. try:
  2169. response = run()
  2170. except Exception as exc:
  2171. end = getattr(agent, "_emit_stream_end", None)
  2172. if end is not None:
  2173. end(final_text="", finished=False, error=str(exc))
  2174. raise
  2175. end = getattr(agent, "_emit_stream_end", None)
  2176. if end is not None:
  2177. end(final_text=_stream_final_text(response), finished=True, error=None)
  2178. return response
  2179. def _stream_codex_passthrough(agent, api_kwargs: dict, on_first_delta):
  2180. """Codex streams internally via _run_codex_stream (reached through
  2181. _interruptible_api_call); park ``on_first_delta`` on the agent so it can pick
  2182. it up, and bracket the call with the stream start/end emitters."""
  2183. agent._codex_on_first_delta = on_first_delta
  2184. try:
  2185. return _with_stream_emitters(agent, lambda: agent._interruptible_api_call(api_kwargs))
  2186. finally:
  2187. agent._codex_on_first_delta = None
  2188. def _finalize_bedrock_relay_events(events):
  2189. """Relay finalizer for Bedrock: a stream without messageStop has no complete
  2190. response to record, so return None and let the live consumer raise (#109988)."""
  2191. from agent.bedrock_adapter import stream_converse_with_callbacks
  2192. try:
  2193. return stream_converse_with_callbacks({"stream": list(events)})
  2194. except EmptyStreamError:
  2195. return None
  2196. class _BedrockStream:
  2197. """Bedrock Converse streaming: boto3 ``converse_stream()`` on a worker thread
  2198. with real-time delta callbacks, polled by an interrupt / stale-event watchdog
  2199. (same UX as the Anthropic and chat_completions streams)."""
  2200. def __init__(self, agent, api_kwargs: dict, on_first_delta):
  2201. self.agent = agent
  2202. self.api_kwargs = api_kwargs
  2203. self.on_first_delta = on_first_delta
  2204. self.result = {"response": None, "error": None}
  2205. self.first_delta_fired = False
  2206. self.response_started = False
  2207. # Liveness for the boto3 worker: ``for event in event_stream`` has NO read timeout,
  2208. # so on_event stamps every event and the poll loop trips a watchdog on a long gap.
  2209. self.started_at = time.time()
  2210. self.last_event = self.started_at
  2211. # Read (not popped): the worker's own pop inside _open_stream must
  2212. # still resolve the same region.
  2213. self.region = api_kwargs.get("__bedrock_region__", "us-east-1")
  2214. # Same patience budget as the OpenAI/Anthropic stale detector.
  2215. self.stale_timeout = _derive_stream_stale_timeout(agent, api_kwargs)
  2216. def _model(self) -> str:
  2217. return self.api_kwargs.get("modelId", "unknown")
  2218. def _fire_first(self):
  2219. self.response_started = True
  2220. if not self.first_delta_fired and self.on_first_delta:
  2221. self.first_delta_fired = True
  2222. with contextlib.suppress(Exception):
  2223. self.on_first_delta()
  2224. def _after_first(self, fire):
  2225. """Wrap a delta callback so the first delivered event also fires ``on_first_delta``."""
  2226. def _on(value):
  2227. self._fire_first()
  2228. fire(value)
  2229. return _on
  2230. def _open_stream(self, next_api_kwargs: dict[str, Any]):
  2231. return _bedrock_converse_call(dict(next_api_kwargs), stream=True, on_stream_denied=self._fall_back_to_converse)
  2232. def _fall_back_to_converse(self, client, final_kwargs: dict, exc: Exception):
  2233. # InvokeModel-only IAM policies cannot stream; fall back inside the same Relay
  2234. # attempt (one lifecycle boundary).
  2235. from agent.bedrock_adapter import normalize_converse_response
  2236. self.agent._disable_streaming = True
  2237. self.agent._safe_print("\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream — "
  2238. "falling back to non-streaming InvokeModel.\n"
  2239. " Grant that action to restore streaming output.\n", diagnostic=True)
  2240. logger.info("bedrock: converse_stream denied by IAM (%s) — "
  2241. "using non-streaming converse() for this session.", type(exc).__name__)
  2242. return normalize_converse_response(client.converse(**final_kwargs))
  2243. def _worker(self):
  2244. agent = self.agent
  2245. stream = None
  2246. try:
  2247. from agent import relay_llm
  2248. from agent.bedrock_adapter import stream_converse_with_callbacks
  2249. intercepted_events = []
  2250. writer_token = {"value": None}
  2251. def _stream_created(_stream: Any) -> None:
  2252. writer_token["value"] = claim_stream_writer(agent)
  2253. def _accept_event(_event: Any) -> bool:
  2254. token = writer_token["value"]
  2255. return token is None or stream_writer_is_current(agent, token)
  2256. def _stamp_event() -> None:
  2257. self.last_event = time.time()
  2258. try:
  2259. from agent.plugin_stream_hooks import has_reasoning_stream_observer_hooks
  2260. plugin_reasoning_observer = has_reasoning_stream_observer_hooks()
  2261. except Exception:
  2262. logger.debug("plugin reasoning stream observer check failed", exc_info=True)
  2263. plugin_reasoning_observer = False
  2264. stream = relay_llm.stream(dict(self.api_kwargs), self._open_stream,
  2265. **_relay_stream_identity(agent, "bedrock"),
  2266. finalizer=lambda: _finalize_bedrock_relay_events(intercepted_events),
  2267. on_stream_created=_stream_created, on_chunk=intercepted_events.append,
  2268. chunk_adapter=lambda chunk: chunk, accept_chunk=_accept_event,
  2269. completed_response_predicate=lambda response: bool(getattr(response, "choices", None)),
  2270. metadata=_relay_stream_metadata(agent, "custom"), defer_logical_completion=True)
  2271. wants_reasoning = agent.reasoning_callback or agent.stream_delta_callback or plugin_reasoning_observer
  2272. try:
  2273. streamed_response = stream_converse_with_callbacks({"stream": stream},
  2274. on_text_delta=self._after_first(agent._fire_stream_delta) if agent._has_stream_consumers() else None,
  2275. on_tool_start=self._after_first(agent._fire_tool_gen_started),
  2276. on_reasoning_delta=self._after_first(agent._fire_reasoning_delta) if wants_reasoning else None,
  2277. on_interrupt_check=lambda: agent._interrupt_requested, on_event=_stamp_event)
  2278. except EmptyStreamError:
  2279. # IAM-denied fallback: no stream events, but converse() already completed.
  2280. if stream.final_response is None:
  2281. raise
  2282. streamed_response = None
  2283. self.result["response"] = stream.final_response or streamed_response
  2284. except Exception as e:
  2285. self.result["error"] = e
  2286. finally:
  2287. if stream is not None:
  2288. stream.close()
  2289. def _raise_if_interrupted(self, message: str, worker=None) -> None:
  2290. if not self.agent._interrupt_requested:
  2291. return
  2292. _record_interrupted_provider_wait(
  2293. self.agent, time.time() - self.started_at, response_started=self.response_started)
  2294. if worker is not None:
  2295. # Let the worker unwind Relay scopes before raising (#81521).
  2296. _join_worker_for_relay_teardown(worker, label="Bedrock streaming")
  2297. raise InterruptedError(message)
  2298. def _on_stale(self, stale_elapsed: float) -> None:
  2299. """No event past the stale timeout = wedged stream (the worker would
  2300. block in the event loop forever)."""
  2301. agent = self.agent
  2302. logger.warning("Bedrock stream stale for %.0fs (threshold %.0fs) — no events "
  2303. "received. region=%s model=%s. Aborting call.", stale_elapsed, self.stale_timeout, self.region,
  2304. self._model())
  2305. agent._buffer_diagnostic_status(f"⚠️ No events from Bedrock for {int(stale_elapsed)}s (model: {self._model()}). Aborting...")
  2306. _bump_stale_streak(agent)
  2307. # Evict the region's cached client so the NEXT call gets a fresh pool.
  2308. # This does NOT abort the in-flight botocore EventStream (no external
  2309. # cancellation exists); the daemon worker keeps reading until its
  2310. # socket errors, so THIS call ends via the TimeoutError below.
  2311. try:
  2312. from agent.bedrock_adapter import invalidate_runtime_client
  2313. invalidate_runtime_client(self.region)
  2314. except Exception as _inval_exc:
  2315. logger.debug("bedrock: stale client eviction failed: %s", _inval_exc)
  2316. self.last_event = time.time()
  2317. # Raises RuntimeError past HERMES_STREAM_STALE_GIVEUP; otherwise end
  2318. # THIS call with a TimeoutError and let the streak carry forward.
  2319. _check_stale_giveup(agent)
  2320. self.result["error"] = TimeoutError(
  2321. f"Bedrock stream produced no events for {int(stale_elapsed)}s (threshold {int(self.stale_timeout)}s) "
  2322. f"— aborting stalled stream so the retry/fallback path can recover.")
  2323. def _poll(self):
  2324. t = threading.Thread(target=_context_thread_target(self._worker), daemon=True)
  2325. t.start()
  2326. while t.is_alive():
  2327. t.join(timeout=0.3)
  2328. self._raise_if_interrupted("Agent interrupted during Bedrock API call", worker=t)
  2329. stale_elapsed = time.time() - self.last_event
  2330. if stale_elapsed > self.stale_timeout:
  2331. self._on_stale(stale_elapsed)
  2332. break
  2333. # The Bedrock callback returns a PARTIAL response on interrupt without raising
  2334. # (on_interrupt_check), so the in-loop raise may never fire. Re-check (#59999 area).
  2335. self._raise_if_interrupted("Agent interrupted during Bedrock API call (post-worker)")
  2336. if self.result["error"] is not None:
  2337. raise self.result["error"]
  2338. # Success clears the cross-turn breaker (#58962).
  2339. if self.result["response"] is not None:
  2340. _reset_stale_streak(self.agent)
  2341. return self.result["response"]
  2342. def run(self):
  2343. # Cross-turn stale-stream circuit breaker (#58962), as on the OpenAI/
  2344. # Anthropic path.
  2345. _check_stale_giveup(self.agent)
  2346. return _with_stream_emitters(self.agent, self._poll)
  2347. class _StreamingCall(StreamingWaitMonitor):
  2348. """One streaming request on the chat_completions / anthropic_messages wire.
  2349. State shared between the request worker and the poll-loop monitor (heartbeat,
  2350. stale kill, interrupt abort) lives on the instance, mutated from both threads."""
  2351. def __init__(self, agent, api_kwargs: dict, on_first_delta):
  2352. self.agent = agent
  2353. self.api_kwargs = api_kwargs
  2354. self.on_first_delta = on_first_delta
  2355. self.worker = None # request thread; None in inline mode
  2356. self.result = {"response": None, "error": None, "partial_tool_names": []}
  2357. self.clients = _RequestClientRegistry(agent)
  2358. # Request-local cancel flag: the worker recognizes its own interrupt
  2359. # force-close (RemoteProtocolError) and exits instead of retrying (#6600).
  2360. self._request_cancelled = {"value": False}
  2361. self.first_delta_fired = {"done": False}
  2362. self.deltas_were_sent = {"yes": False} # for the partial-delivery fallback
  2363. self.provider_tool_in_flight = {"yes": False}
  2364. # Last REAL chunk; the monitor detects SSE-ping-only connections with it.
  2365. self.last_chunk_time = {"t": time.time()}
  2366. self.prompt_progress = {"progress": None, "t": 0.0} # latest llama-server prompt progress
  2367. # Shared by the socket read timeout (``_stream_timeouts``) and the stale
  2368. # detector (``_resolve_stale_timeout``); None until resolved.
  2369. self._stream_stale_timeout = None
  2370. self.stream_attempt_lock = threading.Lock()
  2371. self.stream_attempt_state = {"current": 0, "cancelled": set(), "discarded_chunks": 0, "discarded_bytes": 0}
  2372. self._stale_counted_attempts: set[int] = set() # breaker counts each attempt once
  2373. self.managed_stream_holder = {"stream": None}
  2374. # Per-attempt: single-writer token, request-local client, raw HTTP response (chat wire).
  2375. self._writer_token = self._attempt_request_client = self._attempt_stream_response = None
  2376. # The route ``api_kwargs`` was assembled for; a retry must not replay it on another one.
  2377. self._request_route = self._live_route()
  2378. # ── shared small helpers ────────────────────────────────────────────
  2379. def _live_route(self) -> tuple:
  2380. agent = self.agent
  2381. return tuple(str(getattr(agent, attr, "") or "") for attr in ("model", "provider", "base_url", "api_mode"))
  2382. def _route_switched_under_request(self) -> bool:
  2383. """True once ``/model`` (``switch_model``) re-pointed the agent while this request was in
  2384. flight. The captured payload names the OLD model and is shaped for the OLD provider, but
  2385. every (re)open builds its client from the LIVE agent, so a retry would send a foreign model
  2386. slug to the new base_url (404 + a rate-limit hold, #112121). The turn loop rebuilds the
  2387. request for the current route on its own next attempt, so hand the error back to it.
  2388. """
  2389. live = self._live_route()
  2390. if live == self._request_route:
  2391. return False
  2392. logger.warning(
  2393. "Stream retry skipped: model/provider switched mid-request (%s via %s -> %s via %s); "
  2394. "handing back to the turn loop to rebuild the request for the current route.",
  2395. self._request_route[0], self._request_route[1] or self._request_route[2], live[0], live[1] or live[2],
  2396. )
  2397. return True
  2398. @staticmethod
  2399. def _quiet(fn, *args) -> None:
  2400. """Best-effort callback: never let a display hook break the stream."""
  2401. with contextlib.suppress(Exception):
  2402. fn(*args)
  2403. def _set_managed_stream(self, stream: Any) -> Any:
  2404. self.managed_stream_holder["stream"] = stream
  2405. return stream
  2406. def _close_managed_stream(self) -> None:
  2407. close = getattr(self.managed_stream_holder.pop("stream", None), "close", None)
  2408. if callable(close):
  2409. try:
  2410. close()
  2411. except Exception:
  2412. logger.debug("Managed provider stream cleanup failed", exc_info=True)
  2413. def _start_stream_attempt(self) -> int:
  2414. with self.stream_attempt_lock:
  2415. self.stream_attempt_state["current"] += 1
  2416. attempt_id = int(self.stream_attempt_state["current"])
  2417. # Attempt-local: a dead attempt's prompt progress must not mask the next load notice, and
  2418. # its tool name (died before any text) must not label a later partial stub or retry.
  2419. self.provider_tool_in_flight["yes"], self.prompt_progress["progress"] = False, None
  2420. self.result["partial_tool_names"] = []
  2421. return attempt_id
  2422. def _cancel_current_stream_attempt(self, reason: str) -> None:
  2423. with self.stream_attempt_lock:
  2424. current = int(self.stream_attempt_state["current"])
  2425. if current:
  2426. self.stream_attempt_state["cancelled"].add(current)
  2427. if current:
  2428. logger.debug("Marked stream attempt %s cancelled: %s", current, reason)
  2429. def _stream_attempt_is_active(self, stream_attempt_id: int) -> bool:
  2430. with self.stream_attempt_lock:
  2431. state = self.stream_attempt_state
  2432. return stream_attempt_id == int(state["current"]) and stream_attempt_id not in state["cancelled"]
  2433. def _stream_attempt_was_cancelled(self, stream_attempt_id: int) -> bool:
  2434. with self.stream_attempt_lock:
  2435. return stream_attempt_id in self.stream_attempt_state["cancelled"]
  2436. def _discard_stale_stream_chunk(self, stream_attempt_id: int, chunk) -> None:
  2437. try:
  2438. chunk_bytes = len(repr(chunk))
  2439. except Exception:
  2440. chunk_bytes = 0
  2441. with self.stream_attempt_lock:
  2442. state = self.stream_attempt_state
  2443. state["discarded_chunks"] += 1
  2444. state["discarded_bytes"] += chunk_bytes
  2445. discarded_chunks, discarded_bytes = state["discarded_chunks"], state["discarded_bytes"]
  2446. first = discarded_chunks == 1
  2447. (logger.warning if first else logger.debug)(
  2448. ("Discarding chunk from superseded stream attempt %s " if first else "Discarded stale stream chunk from attempt %s ")
  2449. + "(discarded_chunks=%s discarded_bytes=%s)",
  2450. stream_attempt_id, discarded_chunks, discarded_bytes,
  2451. )
  2452. def _fire_first_delta(self):
  2453. if not self.first_delta_fired["done"] and self.on_first_delta:
  2454. self.first_delta_fired["done"] = True
  2455. self._quiet(self.on_first_delta)
  2456. def _emit_text(self, text: str) -> None:
  2457. self._fire_first_delta()
  2458. self.agent._fire_stream_delta(text)
  2459. self.deltas_were_sent["yes"] = True
  2460. def _visible_text_delivered(self) -> bool:
  2461. """True when visible assistant text actually reached a stream consumer this attempt
  2462. (``_fire_stream_delta`` records only scrubbed, delivered text; ``deltas_were_sent``
  2463. flips on any content delta, including whitespace/think-only ones nobody saw)."""
  2464. return bool((getattr(self.agent, "_current_streamed_assistant_text", "") or "").strip())
  2465. def _emit_reasoning(self, text: str) -> None:
  2466. self._fire_first_delta()
  2467. self.agent._fire_reasoning_delta(text)
  2468. def _emit_tool_started(self, name: str) -> None:
  2469. self._fire_first_delta()
  2470. self.agent._fire_tool_gen_started(name)
  2471. def _route_suppressed_text(self, text: str) -> None:
  2472. """Tool-call turns suppress content streaming (no chatty preamble), but
  2473. reasoning tags inside it must still reach the display: route through
  2474. the delta callback for tag extraction (the CLI drops non-reasoning text
  2475. once the stream box is closed)."""
  2476. if self.agent.stream_delta_callback and not getattr(self.agent, "_hold_stream", False):
  2477. self._quiet(lambda: (self.agent.stream_delta_callback(text), self.agent._record_streamed_assistant_text(text)))
  2478. def _new_diag(self) -> dict:
  2479. diag = self.agent._stream_diag_init()
  2480. self.clients.diag = diag
  2481. return diag
  2482. def _count_chunk(self, diag, chunk) -> None:
  2483. """Stamp liveness for a real chunk; diagnostics are best-effort."""
  2484. self.last_chunk_time["t"] = time.time()
  2485. self.agent._touch_activity("receiving stream response")
  2486. with contextlib.suppress(Exception):
  2487. diag["chunks"] = int(diag.get("chunks", 0)) + 1
  2488. if diag.get("first_chunk_at") is None:
  2489. diag["first_chunk_at"] = self.last_chunk_time["t"]
  2490. # Delta-length estimate: ~3x cheaper than repr() per chunk.
  2491. diag["bytes"] = int(diag.get("bytes", 0)) + _estimate_chunk_bytes(chunk)
  2492. def _take_chunk(self, diag, chunk, llamacpp: bool) -> bool:
  2493. """Count a real chunk; a llama-server prompt-progress chunk (True) is liveness, no first token."""
  2494. progress = prompt_progress(chunk) if llamacpp else None
  2495. if progress is None:
  2496. self._count_chunk(diag, chunk)
  2497. return False
  2498. self.last_chunk_time["t"] = now = time.time()
  2499. self.prompt_progress["progress"], self.prompt_progress["t"] = progress, now
  2500. self.agent._touch_activity("model reading the prompt")
  2501. return True
  2502. @staticmethod
  2503. def _chunk_identity(diag, chunk, model_name, response_id, upstream_provider) -> tuple:
  2504. """Model, response id and serving provider, as the stream reports them."""
  2505. if hasattr(chunk, "model") and chunk.model:
  2506. model_name = chunk.model
  2507. if response_id is None and isinstance(getattr(chunk, "id", None), str) and chunk.id:
  2508. response_id = chunk.id
  2509. if upstream_provider is None and isinstance(getattr(chunk, "provider", None), str) and chunk.provider:
  2510. upstream_provider = chunk.provider # OpenRouter stamps who served
  2511. diag["serving_provider"] = upstream_provider.strip()[:64] # attribute a mid-stream drop (#90216)
  2512. return model_name, response_id, upstream_provider
  2513. @staticmethod
  2514. def _mark_finish_seen(diag, finish_reason) -> None:
  2515. """Record that this attempt saw a terminal finish/stop reason (#102766)."""
  2516. if finish_reason and isinstance(diag, dict) and not diag.get("finish_reason_seen"):
  2517. diag["finish_reason_seen"] = True
  2518. # ── chat_completions wire ───────────────────────────────────────────
  2519. def _stream_timeouts(self) -> tuple[float, float, float]:
  2520. """``(write, read, connect/pool)`` socket timeouts. Per-provider
  2521. ``request_timeout_seconds`` wins over HERMES_API_TIMEOUT (1800s) and
  2522. HERMES_STREAM_READ_TIMEOUT (120s); connect/pool cover the handshake, not
  2523. inference: 30s, or capped at 60s when configured."""
  2524. cfg = get_provider_request_timeout(self.agent.provider, self.agent.model)
  2525. base = cfg if cfg is not None else env_float("HERMES_API_TIMEOUT", 1800.0)
  2526. if cfg is not None:
  2527. return base, cfg, min(base, 60.0)
  2528. read = env_float("HERMES_STREAM_READ_TIMEOUT", 120.0)
  2529. stale = self._stream_stale_timeout
  2530. if read == 120.0 and self.agent.base_url and is_local_endpoint(self.agent.base_url):
  2531. read = base # local providers prefill for minutes
  2532. logger.debug("Local provider detected (%s) — stream read timeout raised to %.0fs", self.agent.base_url, read)
  2533. elif read == 120.0 and stale is not None and stale != float("inf") and stale > read:
  2534. # Reasoning models pause mid-stream for minutes; the stale detector
  2535. # tolerates that, so the raw read timeout must not fire first.
  2536. read = stale
  2537. logger.debug("Cloud reasoning stream — read timeout raised to %.0fs to match stale-stream detector", read)
  2538. return base, read, 30.0
  2539. @staticmethod
  2540. def _choiceless_chunk(chunk, finish_reason):
  2541. """Chunk with empty ``choices`` -> ``(usage, finish_reason)``. Raises
  2542. ProviderStreamError for providers (DeepInfra) that send validation errors
  2543. as in-stream chunks (choices=None + error_type/error_message), which
  2544. would otherwise surface as a misleading EmptyStreamError plus retries."""
  2545. usage = chunk.usage if hasattr(chunk, "usage") and chunk.usage else None # final usage chunk
  2546. # Without this check the error is silently dropped and the stream ends empty → EmptyStreamError →
  2547. # misleading "empty stream" message and pointless retries on the same bad request. (#65631)
  2548. _err_type = getattr(chunk, "error_type", None)
  2549. _err_msg = getattr(chunk, "error_message", None)
  2550. if _err_type or _err_msg:
  2551. _status = _status_code_from_payload({"code": _err_type, "message": _err_msg}) or _status_code_from_value(_err_type)
  2552. body = _provider_error_body(
  2553. {"code": _err_type or "provider_in_stream_error", "message": str(_err_msg or chunk)}, _status)
  2554. raise ProviderStreamError(status_code=_status, body=body, raw_text=f"{_err_type}: {_err_msg}")
  2555. # Nous Portal usage frames (choices=[] + lastOne=true, no [DONE]) are a
  2556. # clean terminal, not a drop; relabelled upstreams send 1 / "true".
  2557. # See #90848.
  2558. last_one = getattr(chunk, "lastOne", None)
  2559. if last_one is None and isinstance(getattr(chunk, "model_extra", None), dict):
  2560. last_one = chunk.model_extra.get("lastOne")
  2561. if last_one in (True, 1, "true") and finish_reason is None:
  2562. finish_reason = "stop"
  2563. return usage, finish_reason
  2564. def _open_chat_stream(self, stream_kwargs: dict[str, Any]):
  2565. # Native Gemini rejects OpenAI's usage-streaming extension; so do strict endpoints that
  2566. # already 4xx'd on it this session (``_stream_options_unsupported``, see #9705).
  2567. if not is_native_gemini_base_url(self.agent.base_url) and not getattr(self.agent, "_stream_options_unsupported", False):
  2568. stream_kwargs["stream_options"] = {"include_usage": True}
  2569. request_client = self._attempt_request_client = self.clients.set_client(
  2570. self.agent._create_request_openai_client(reason="chat_completion_stream_request", api_kwargs=stream_kwargs))
  2571. self.last_chunk_time["t"] = time.time()
  2572. self.agent._touch_activity("waiting for provider response (streaming)")
  2573. # #93650: as above — the streaming path carries the same bulk
  2574. # messages/tools payload and pays the same client-side walk.
  2575. stream_kwargs = bypass_chat_sdk_request_transform(stream_kwargs, request_client)
  2576. return request_client.chat.completions.create(**stream_kwargs)
  2577. def _chat_stream_created(self, raw_stream: Any) -> None:
  2578. response = self._attempt_stream_response = getattr(raw_stream, "response", None)
  2579. self.agent._capture_rate_limits(response)
  2580. self.agent._capture_credits(response)
  2581. self.agent._capture_nous_model_switch(response)
  2582. self.agent._stream_diag_capture_response(self.clients.diag, response)
  2583. self.agent._check_openrouter_cache_status(response)
  2584. self._writer_token = claim_stream_writer(self.agent)
  2585. self._reabort_if_cancelled(response)
  2586. def _reabort_if_cancelled(self, response: Any) -> None:
  2587. """Interrupt/stale abort that raced ``create()``: the one-shot pool sweep ran while
  2588. the connect/TLS window held no socket yet (``tcp_force_closed=0``), so nothing stopped
  2589. the request once it came up and the serve kept generating into a dropped consumer
  2590. (#98974). Response headers prove the socket exists now — shut it down (shutdown-only,
  2591. never a cross-thread close) so the worker unwinds as after a stale kill."""
  2592. with self.stream_attempt_lock:
  2593. current = int(self.stream_attempt_state["current"])
  2594. cancelled = self._request_cancelled["value"] or current in self.stream_attempt_state["cancelled"]
  2595. if not cancelled:
  2596. return
  2597. self._shutdown_stale_attempt_socket(response)
  2598. if self._attempt_request_client is not None:
  2599. # Kind-aware: the anthropic_messages wire (incl. anthropic-compatible custom endpoints)
  2600. # runs on a request-local Anthropic client with its own slot sweep.
  2601. abort = (self.agent._abort_request_anthropic_client if self.agent.api_mode == "anthropic_messages"
  2602. else self.agent._abort_request_openai_client)
  2603. abort(self._attempt_request_client, reason="cancelled_attempt_late_connect")
  2604. def _accept_chat_chunk(self, stream_attempt_id: int, chunk: Any) -> bool:
  2605. with contextlib.suppress(Exception):
  2606. choices = getattr(chunk, "choices", None)
  2607. choice = choices[0] if choices else None
  2608. delta = getattr(choice, "delta", None)
  2609. # A stale-attempt fence can win while Relay hands back a tool-call chunk: record
  2610. # the in-flight tool call (retry policy must not see a partial text response).
  2611. if getattr(delta, "tool_calls", None):
  2612. self.provider_tool_in_flight["yes"] = True
  2613. # Marker-only finish chunk (no writable delta) always passes: the fence only stops
  2614. # MORE text; fending the completion signal would mislabel a clean end as a drop.
  2615. if getattr(choice, "finish_reason", None) and not any(
  2616. getattr(delta, attr, None) for attr in ("content", "tool_calls", "reasoning_content", "reasoning")):
  2617. return True
  2618. if not self._stream_attempt_is_active(stream_attempt_id):
  2619. return False
  2620. if not self._writer_still_current("Streaming"):
  2621. return False
  2622. # Stamp BEFORE Relay processes the chunk so the watchdog can't cancel
  2623. # a live stream mid-interceptor.
  2624. self.last_chunk_time["t"] = time.time()
  2625. return True
  2626. def _writer_still_current(self, label: str) -> bool:
  2627. """Single-writer fence: False (with a warning) once a newer stream claimed the writer slot."""
  2628. token = self._writer_token
  2629. if token is None or stream_writer_is_current(self.agent, token):
  2630. return True
  2631. logger.warning(
  2632. "%s attempt superseded by a newer stream; stopping consumption to preserve the "
  2633. "single-writer invariant (model=%s).", label, self.api_kwargs.get("model", "unknown"))
  2634. return False
  2635. def _call_chat_completions(self, stream_attempt_id: int):
  2636. """Stream a chat completions response."""
  2637. import httpx as _httpx
  2638. base_timeout, read_timeout, conn_cap = self._stream_timeouts()
  2639. content_parts: list = []
  2640. reasoning_parts: list = []
  2641. # Live-display accumulator for detail-derived reasoning text: de-gluing must
  2642. # compare against what the display actually received, not ``reasoning_parts``
  2643. # (a provider that mirrors the same text in both fields would otherwise read
  2644. # as already-glued on the first chunk and get a spurious break).
  2645. detail_display_parts: list[str] = []
  2646. # OpenAI structured refusal (``delta.refusal``): the explanation streams here and
  2647. # ``delta.content`` stays empty, so an un-accumulated refusal looks like an empty
  2648. # stream and burns the empty-response retries (the non-streaming fix is #46013).
  2649. refusal_parts: list[str] = []
  2650. reasoning_details: list = [] # OpenRouter replay data (signatures, encrypted blocks)
  2651. pending_text_parts: list[str] = []
  2652. from agent.chat_completion_helpers_tool_calls import _ToolCallAccumulator
  2653. tool_calls = _ToolCallAccumulator()
  2654. tool_calls_acc = tool_calls.acc
  2655. from agent.reasoning_carriers import StreamCarriers
  2656. carriers = StreamCarriers()
  2657. finish_reason = model_name = usage_obj = None
  2658. response_id = upstream_provider = None # the provider's own id / serving upstream, from the chunks
  2659. role = "assistant"
  2660. _diag = self._new_diag()
  2661. self._writer_token = self._attempt_request_client = self._attempt_stream_response = None
  2662. from agent.chat_completion_helpers_relay import RelayChatAccumulator
  2663. relay_response = RelayChatAccumulator()
  2664. # The raw channels, not the callback-delivered text: cron, subagents and callback-less
  2665. # gateway turns register no stream callback, so nothing else sees a loop mid-stream.
  2666. content_watch, reasoning_watch = RunawayStreamWatch(), RunawayStreamWatch()
  2667. # Own watch: the display takes one reasoning representation per chunk, but the plain
  2668. # field is persisted either way and need not mirror the details.
  2669. detail_watch = RunawayStreamWatch()
  2670. runaway = None
  2671. llamacpp = _streams_llamacpp(self.agent)
  2672. def _open_stream(next_api_kwargs: dict[str, Any]):
  2673. timeout = _httpx.Timeout(connect=conn_cap, read=read_timeout, write=base_timeout, pool=conn_cap)
  2674. if llamacpp:
  2675. next_api_kwargs = with_prompt_progress(next_api_kwargs)
  2676. return self._open_chat_stream({**next_api_kwargs, "stream": True, "timeout": timeout})
  2677. def _flush_pending_stream_text():
  2678. pending_parts = list(pending_text_parts)
  2679. pending_text_parts.clear()
  2680. for text in pending_parts:
  2681. (self._route_suppressed_text if tool_calls_acc else self._emit_text)(text)
  2682. from agent import relay_llm
  2683. stream = self._set_managed_stream(relay_llm.stream(self.api_kwargs, _open_stream,
  2684. **_relay_stream_identity(self.agent, "provider"), finalizer=relay_response.finalize,
  2685. on_stream_created=self._chat_stream_created, on_chunk=relay_response.observe,
  2686. accept_chunk=lambda chunk: self._accept_chat_chunk(stream_attempt_id, chunk),
  2687. completed_response_predicate=lambda value: hasattr(value, "choices"),
  2688. metadata=_relay_stream_metadata(self.agent, "chat_completions"), defer_logical_completion=True))
  2689. if self.agent.provider == "moa":
  2690. # Hermes interrupts the managed stream; Relay alone closes the provider stream.
  2691. self.clients.set_stream_handle(stream)
  2692. def _close_half_read_stream(reason: str) -> None:
  2693. # A half-read SSE response stays checked out of the httpx pool and the finally
  2694. # would cache the client WITH the leaked connection: close on the owner first.
  2695. try:
  2696. stream.close()
  2697. except Exception:
  2698. # Still checked out: poison the slot so the finally really closes the pool.
  2699. if self._attempt_request_client is not None:
  2700. self.agent._abort_request_openai_client(self._attempt_request_client, reason=reason)
  2701. for chunk in _iter_provider_stream_chunks(stream, response=lambda: self._attempt_stream_response):
  2702. is_progress = self._take_chunk(_diag, chunk, llamacpp)
  2703. if self.agent._interrupt_requested:
  2704. _close_half_read_stream("interrupt_stream_close_failed")
  2705. break
  2706. if not self._stream_attempt_is_active(stream_attempt_id):
  2707. self._discard_stale_stream_chunk(stream_attempt_id, chunk)
  2708. continue
  2709. if is_progress:
  2710. continue
  2711. model_name, response_id, upstream_provider = self._chunk_identity(
  2712. _diag, chunk, model_name, response_id, upstream_provider)
  2713. if not chunk.choices:
  2714. usage, finish_reason = self._choiceless_chunk(chunk, finish_reason)
  2715. usage_obj = usage or usage_obj
  2716. self._mark_finish_seen(_diag, finish_reason)
  2717. continue
  2718. choice = chunk.choices[0]
  2719. delta = choice.delta
  2720. # Read finish_reason/usage BEFORE any content-shape `continue`: the SSE-echo
  2721. # guard can swallow a merged finish chunk (vLLM standalone ':' tokens).
  2722. finish_reason = _normalize_finish_reason(getattr(choice, "finish_reason", None)) or finish_reason
  2723. self._mark_finish_seen(_diag, finish_reason)
  2724. if hasattr(chunk, "usage") and chunk.usage:
  2725. usage_obj = chunk.usage
  2726. reasoning_text = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)
  2727. # Same ``model_extra`` fallback as the non-streaming path: a reasoning-only stream
  2728. # whose deltas carry only this field otherwise trips the empty-stream guard (#56516).
  2729. if reasoning_text is None and isinstance(getattr(delta, "model_extra", None), dict):
  2730. reasoning_text = delta.model_extra.get("reasoning_content") or delta.model_extra.get("reasoning")
  2731. # A profile-declared text carrier doubles as the readable reasoning.
  2732. reasoning_text = carriers.feed(delta, reasoning_text)
  2733. if reasoning_text:
  2734. # Summary-part models omit the separator between markdown blocks; re-insert it.
  2735. reasoning_text = separate_glued_reasoning_blocks(
  2736. reasoning_parts[-1] if reasoning_parts else "", reasoning_text)
  2737. reasoning_parts.append(reasoning_text)
  2738. # Structured reasoning_details deltas carry the provider's replay data; the
  2739. # non-streaming path already keeps them, so dropping them here lost
  2740. # reasoning continuity on nearly every turn. Pydantic parks unknown fields
  2741. # in ``model_extra``.
  2742. rd_delta = getattr(delta, "reasoning_details", None)
  2743. if rd_delta is None and isinstance(getattr(delta, "model_extra", None), dict):
  2744. rd_delta = delta.model_extra.get("reasoning_details")
  2745. detail_text_parts = []
  2746. for rd in rd_delta if isinstance(rd_delta, (list, tuple)) else ():
  2747. detail_text_parts.append(streamed_reasoning_detail_text(rd))
  2748. append_streamed_reasoning_detail(reasoning_details, rd)
  2749. # Details may carry the full text while ordinary reasoning is only
  2750. # a sparse fragment or a mirror. Deliver one representation per
  2751. # chunk, without rewriting either persisted/replayed field.
  2752. # Summary-part boundaries need the same repair the plain path applies:
  2753. # de-glue against the detail display's own accumulator (not
  2754. # ``reasoning_parts`` — with mirrored fields that would insert a
  2755. # spurious break on the first chunk), so the live box and the
  2756. # persisted ``reasoning_content`` agree.
  2757. detail_text = "".join(detail_text_parts)
  2758. if detail_text:
  2759. detail_text = separate_glued_reasoning_blocks(
  2760. detail_display_parts[-1] if detail_display_parts else "", detail_text)
  2761. detail_display_parts.append(detail_text)
  2762. display_reasoning = detail_text or reasoning_text
  2763. if display_reasoning:
  2764. self._emit_reasoning(display_reasoning)
  2765. if reasoning_watch.feed(reasoning_text) or detail_watch.feed(detail_text):
  2766. runaway = "reasoning"
  2767. break
  2768. # Not routed to the live display: the transport promotes a sole-payload
  2769. # refusal to content + ``content_filter`` and the loop surfaces it terminally.
  2770. delta_refusal = getattr(delta, "refusal", None)
  2771. if delta_refusal is None and isinstance(getattr(delta, "model_extra", None), dict):
  2772. delta_refusal = delta.model_extra.get("refusal")
  2773. if isinstance(delta_refusal, str) and delta_refusal:
  2774. refusal_parts.append(delta_refusal)
  2775. # Text (list-of-blocks deltas flattened once); possible echoed SSE is
  2776. # buffered until it can be judged.
  2777. delta_content = flatten_message_text(getattr(delta, "content", None), sep="")
  2778. if delta_content:
  2779. content_parts.append(delta_content)
  2780. if content_watch.feed(delta_content):
  2781. runaway = "content"
  2782. break
  2783. if tool_calls_acc:
  2784. self._route_suppressed_text(delta_content)
  2785. elif (pending_text_parts or _provider_stream_text_may_be_sse(delta_content)
  2786. # A shim cannot follow text already released to the display, so the
  2787. # whole-content re-join runs only until the first emitted delta.
  2788. or (not self.deltas_were_sent["yes"] and router_timeout_shim_may_follow("".join(content_parts)))):
  2789. pending_text_parts.append(delta_content)
  2790. pending = "".join(pending_text_parts)
  2791. if not (_provider_stream_text_may_be_sse(pending) or router_timeout_shim_may_follow(pending)):
  2792. _flush_pending_stream_text()
  2793. continue
  2794. else:
  2795. self._emit_text(delta_content)
  2796. delta_tool_calls = getattr(delta, "tool_calls", None)
  2797. if delta_tool_calls:
  2798. _flush_pending_stream_text()
  2799. for tc_delta in delta_tool_calls:
  2800. name = tool_calls.feed(tc_delta)
  2801. if name is not None:
  2802. self._emit_tool_started(name)
  2803. # Lets the stub-builder warn if streaming dies before the args
  2804. # complete instead of silently discarding the action.
  2805. self.result["partial_tool_names"].append(name)
  2806. if runaway:
  2807. _close_half_read_stream("runaway_stream_close_failed")
  2808. self._log_runaway_cut(runaway)
  2809. tool_calls.materialize()
  2810. self._close_managed_stream()
  2811. if self._stream_attempt_was_cancelled(stream_attempt_id):
  2812. raise _httpx.RemoteProtocolError(f"stream attempt {stream_attempt_id} was superseded")
  2813. if stream.final_response is not None:
  2814. return self._adopt_final_response(stream.final_response)
  2815. response = self._finish_chat_stream(stream, role, content_parts, reasoning_parts, tool_calls_acc,
  2816. "length" if runaway else finish_reason, model_name, usage_obj, flush_pending=_flush_pending_stream_text,
  2817. response_id=response_id, upstream_provider=upstream_provider, reasoning_details=reasoning_details,
  2818. refusal_parts=refusal_parts)
  2819. carriers.apply(response)
  2820. if runaway:
  2821. # Cut, not finished: the length path ends the turn on this mark instead of continuing.
  2822. response._runaway_repetition = True
  2823. return response
  2824. def _log_runaway_cut(self, channel: str) -> None:
  2825. logger.warning(
  2826. "%sStream cut: the %s turned into a runaway repetition loop (model=%s, provider=%s).",
  2827. getattr(self.agent, "log_prefix", ""), channel, self.api_kwargs.get("model", "unknown"),
  2828. getattr(self.agent, "provider", "") or "unknown")
  2829. def _adopt_final_response(self, final_response):
  2830. """Adapter returned a completed response for ``stream=True``: switch the
  2831. session to non-streaming and replay its content as deltas."""
  2832. logger.info("Streaming request returned a final response object instead of an iterator; "
  2833. "switching %s/%s to non-streaming for this session.", self.agent.provider or "unknown",
  2834. self.agent.model or "unknown")
  2835. self.agent._disable_streaming = True
  2836. return self._replay_final_response(final_response)
  2837. def _replay_final_response(self, final_response):
  2838. """Replay a completed chat-completions response's reasoning/content as deltas."""
  2839. choices = final_response.choices
  2840. message = getattr(choices[0] if isinstance(choices, (list, tuple)) and choices else None, "message", None)
  2841. if message is not None:
  2842. reasoning_text = getattr(message, "reasoning_content", None) or getattr(message, "reasoning", None)
  2843. if reasoning_text is None and isinstance(getattr(message, "model_extra", None), dict):
  2844. reasoning_text = message.model_extra.get("reasoning_content") or message.model_extra.get("reasoning")
  2845. if isinstance(reasoning_text, str) and reasoning_text:
  2846. self._emit_reasoning(reasoning_text)
  2847. content = getattr(message, "content", None)
  2848. if isinstance(content, str) and content:
  2849. self._fire_first_delta()
  2850. self.agent._fire_stream_delta(content) # not _emit_text: deltas_were_sent stays False here
  2851. return final_response
  2852. @staticmethod
  2853. def _assemble_tool_calls(tool_calls_acc, finish_reason):
  2854. """Materialize accumulated tool calls; flag truncated/unrepairable args."""
  2855. mock_tool_calls = []
  2856. has_truncated_tool_args = False
  2857. for idx in sorted(tool_calls_acc):
  2858. tc = tool_calls_acc[idx]
  2859. arguments = tc["function"]["arguments"]
  2860. if arguments and arguments.strip():
  2861. try:
  2862. json.loads(arguments)
  2863. except json.JSONDecodeError:
  2864. # A dropped stream is never repaired: closing its prefix yields valid
  2865. # JSON that silently lacks every key and digit not yet streamed.
  2866. if finish_reason is None:
  2867. has_truncated_tool_args = True
  2868. else:
  2869. # Repair before flagging (GLM via Ollama); "{}" = unrepairable.
  2870. repaired = _repair_tool_call_arguments(arguments, tc["function"]["name"] or "?")
  2871. if repaired != "{}":
  2872. arguments = repaired
  2873. else:
  2874. has_truncated_tool_args = True
  2875. # Parseable JSON does not prove that a dropped stream completed its
  2876. # action. Treat degenerate argument loops as partial calls too.
  2877. # A provider-confirmed call may legitimately write repetitive data.
  2878. if finish_reason is None and is_repetition_dominated(arguments):
  2879. logger.warning(
  2880. "Tool call '%s' has repetition-dominated arguments without a "
  2881. "finish_reason; treating as a dropped tool call.", tc["function"]["name"] or "?")
  2882. has_truncated_tool_args = True
  2883. elif finish_reason is None:
  2884. # Name arrived, zero arg bytes, no finish_reason: unflagged this
  2885. # becomes a "stop" turn executing "{}" with no retry.
  2886. has_truncated_tool_args = True
  2887. mock_tool_calls.append(SimpleNamespace(
  2888. id=tc["id"], type=tc["type"], extra_content=tc.get("extra_content"),
  2889. function=SimpleNamespace(name=tc["function"]["name"], arguments=arguments,
  2890. args_repaired=arguments != tc["function"]["arguments"],
  2891. thought_signature=tc.get("thought_signature"))))
  2892. return mock_tool_calls or None, has_truncated_tool_args
  2893. def _finish_chat_stream(self, stream, role, content_parts, reasoning_parts, tool_calls_acc, finish_reason,
  2894. model_name, usage_obj, *, flush_pending, response_id=None, upstream_provider=None, reasoning_details=None,
  2895. refusal_parts=None):
  2896. """Assemble the non-streaming-shaped response after the chunk loop. A
  2897. stream ending with no finish_reason is a drop, not a completion: return a
  2898. partial-stream stub so the loop fails fast instead of executing empty
  2899. args or stamping "stop"."""
  2900. full_content = "".join(content_parts) or None
  2901. full_reasoning = "".join(reasoning_parts) or None
  2902. if not full_reasoning and full_content:
  2903. # Inline-reasoning providers (MiniMax-M3 streams <think>…</think> in content) send no
  2904. # reasoning delta; fill the structured field from the raw content (#89647).
  2905. from agent.agent_runtime_helpers import extract_reasoning
  2906. full_reasoning = extract_reasoning(self.agent, SimpleNamespace(content=full_content))
  2907. mock_tool_calls, has_truncated_tool_args = self._assemble_tool_calls(tool_calls_acc, finish_reason)
  2908. # Zero-chunk guard: nothing usable = upstream error / malformed SSE.
  2909. if finish_reason is None and not content_parts and not reasoning_parts and not refusal_parts and not tool_calls_acc:
  2910. raise EmptyStreamError(
  2911. "Provider returned an empty stream with no finish_reason (possible upstream error or malformed SSE response).")
  2912. if has_truncated_tool_args and finish_reason is None:
  2913. # Partial args WITH finish_reason="length" is a real output cap; with NONE the
  2914. # upstream dropped mid tool-call, and stamping "length" burns 3 useless retries.
  2915. _dropped_names = [(tool_calls_acc[idx]["function"]["name"] or "?") for idx in sorted(tool_calls_acc)]
  2916. logger.warning(
  2917. "Clean EOF, no finish_reason: server ended the stream (no transport exception) while a tool "
  2918. "call's arguments were still incomplete (tools=%s). The server or a proxy closed the stream "
  2919. "cleanly; not an output-length truncation.",
  2920. _dropped_names)
  2921. return _build_partial_stream_stub(
  2922. role, full_content, full_reasoning, model_name, usage_obj, dropped_tool_names=_dropped_names or None,
  2923. clean_eof=True)
  2924. if finish_reason is None and (content_parts or reasoning_parts) and not tool_calls_acc and usage_obj is None:
  2925. # Text-only (or reasoning-only) drop: otherwise the partial text is stamped "stop"
  2926. # and the next step is lost — for reasoning-only, the clean-stop promotion in
  2927. # finish_text_response would then surface a truncated thought as the answer.
  2928. # A usage object proves the provider finished (include_usage's final chunk).
  2929. logger.warning(
  2930. "Clean EOF, no finish_reason: server ended the stream (no transport exception) after delivering "
  2931. "text with no tool calls. The server or a proxy closed the stream cleanly.")
  2932. return _build_partial_stream_stub(role, full_content, full_reasoning, model_name, usage_obj, clean_eof=True)
  2933. effective_finish_reason = "length" if has_truncated_tool_args else (finish_reason or "stop")
  2934. provider_stream_error = _provider_stream_error_from_text(
  2935. full_content or "", effective_finish_reason, response=getattr(stream, "response", None))
  2936. if provider_stream_error is not None:
  2937. raise provider_stream_error
  2938. message = SimpleNamespace(role=role, content=full_content, tool_calls=mock_tool_calls, reasoning_content=full_reasoning,
  2939. # ``normalize_response`` reads ``message.refusal`` — same contract as the non-streaming object.
  2940. refusal="".join(refusal_parts or ()) or None)
  2941. if reasoning_details:
  2942. # Only when present: _build_assistant_message's passthrough persists them
  2943. # for replay, and non-reasoning providers keep the attribute absent.
  2944. message.reasoning_details = reasoning_details
  2945. # The provider's id when the chunks carried one (chatcmpl-/gen-...): it is what a provider needs to
  2946. # look a request up. Fabricated only when the stream never sent one.
  2947. response = SimpleNamespace(id=response_id or ("stream-" + str(uuid.uuid4())), model=model_name, usage=usage_obj,
  2948. provider=upstream_provider,
  2949. choices=[SimpleNamespace(index=0, message=message, finish_reason=effective_finish_reason)])
  2950. # A held router timeout shim (#68396) is rejected by validate_response and retried;
  2951. # releasing its text here would show the provider failure as assistant output.
  2952. if not is_router_timeout_shim(response):
  2953. flush_pending()
  2954. return response
  2955. # ── anthropic_messages wire ─────────────────────────────────────────
  2956. @staticmethod
  2957. def _check_anthropic_message(message, *, tool_drop: bool = True):
  2958. """Raise EmptyStreamError for a message the stream never completed: no
  2959. content and no stop_reason (eventless -> retry), or with ``tool_drop`` a
  2960. ``tool_use`` block and no stop_reason — the SSE closed mid tool call and
  2961. its input is a partial snapshot (usually ``{}``), so raising blocks the
  2962. empty-args execution (bounded retry, or stub/continuation after text)."""
  2963. content = getattr(message, "content", None)
  2964. if not content and getattr(message, "stop_reason", None) is None:
  2965. raise EmptyStreamError(
  2966. "Provider returned an empty stream with no stop_reason (possible upstream error or malformed event stream).")
  2967. if tool_drop and getattr(message, "stop_reason", None) is None and any(
  2968. getattr(block, "type", None) == "tool_use" for block in content or []):
  2969. raise EmptyStreamError(
  2970. "Stream ended with no stop_reason while a tool_use block was still incomplete; "
  2971. "treating as a mid-tool-call stream drop (#80498).")
  2972. return message
  2973. def _call_anthropic(self, request_client):
  2974. """Stream an Anthropic Messages API response; fires delta callbacks but
  2975. returns the native Message from get_final_message(). Runs on the
  2976. per-request ``request_client`` so the watchdog can abort this socket
  2977. without closing the shared client mid-flight."""
  2978. has_tool_use = False
  2979. # No message_stop -> EmptyStreamError; saw_stream_event only picks the message.
  2980. saw_stream_event = False
  2981. saw_message_stop = False
  2982. self.last_chunk_time["t"] = time.time()
  2983. _diag = self._new_diag()
  2984. self._writer_token = self._attempt_stream_response = None
  2985. self._attempt_request_client = request_client
  2986. _stream_context = {"manager": None, "stream": None}
  2987. base_final_message = None
  2988. text_watch, thinking_watch = RunawayStreamWatch(), RunawayStreamWatch()
  2989. runaway = None
  2990. from agent import relay_llm
  2991. from agent.anthropic_adapter import normalize_stream_usage, sanitize_anthropic_kwargs
  2992. accumulator = relay_llm.AnthropicStreamAccumulator()
  2993. def _open_anthropic_stream(next_api_kwargs: dict[str, Any]):
  2994. final_kwargs = dict(next_api_kwargs)
  2995. sanitize_anthropic_kwargs(final_kwargs, log_prefix=getattr(self.agent, "log_prefix", ""))
  2996. manager = request_client.messages.stream(**final_kwargs)
  2997. _stream_context["manager"] = manager
  2998. return normalize_stream_usage(manager.__enter__())
  2999. def _anthropic_stream_created(raw_stream: Any) -> None:
  3000. _stream_context["stream"] = raw_stream
  3001. # Same wiring as the chat_completions wire: MessageStream exposes the httpx response,
  3002. # so the interrupt/stale abort can shut down THIS attempt's socket (#98974).
  3003. response = self._attempt_stream_response = getattr(raw_stream, "response", None)
  3004. # Snapshot response diagnostics now so they survive a stream dying before the first event.
  3005. self._quiet(lambda: self.agent._stream_diag_capture_response(_diag, response))
  3006. self._writer_token = claim_stream_writer(self.agent)
  3007. self._reabort_if_cancelled(response)
  3008. stream = self._set_managed_stream(relay_llm.stream(self.api_kwargs, _open_anthropic_stream,
  3009. **_relay_stream_identity(self.agent, "anthropic"), finalizer=accumulator.finalize,
  3010. on_stream_created=_anthropic_stream_created, on_chunk=accumulator.observe,
  3011. accept_chunk=lambda _event: self._writer_still_current("Anthropic streaming"),
  3012. metadata=_relay_stream_metadata(self.agent, "anthropic_messages"), defer_logical_completion=True))
  3013. try:
  3014. for event in stream:
  3015. saw_stream_event = True
  3016. self._count_chunk(_diag, event)
  3017. if self.agent._interrupt_requested:
  3018. break
  3019. event_type = getattr(event, "type", None)
  3020. if event_type == "message_stop":
  3021. saw_message_stop = True
  3022. elif event_type == "message_delta":
  3023. self._mark_finish_seen(_diag, getattr(getattr(event, "delta", None), "stop_reason", None))
  3024. elif event_type == "content_block_start":
  3025. block = getattr(event, "content_block", None)
  3026. if block and getattr(block, "type", None) == "tool_use":
  3027. has_tool_use = True
  3028. if getattr(block, "name", None):
  3029. self._emit_tool_started(block.name)
  3030. # Same as the chat_completions wire: a stream that dies inside the
  3031. # tool args is retried (no tool has run yet) instead of stubbed.
  3032. self.result["partial_tool_names"].append(block.name)
  3033. elif event_type == "content_block_delta":
  3034. delta = getattr(event, "delta", None)
  3035. delta_type = getattr(delta, "type", None) if delta else None
  3036. if delta_type == "text_delta":
  3037. text = getattr(delta, "text", "")
  3038. if text and not has_tool_use:
  3039. self._emit_text(text)
  3040. if text_watch.feed(text):
  3041. runaway = "content"
  3042. break
  3043. elif delta_type == "thinking_delta" and getattr(delta, "thinking", ""):
  3044. self._emit_reasoning(delta.thinking)
  3045. if thinking_watch.feed(delta.thinking):
  3046. runaway = "reasoning"
  3047. break
  3048. raw_stream = _stream_context["stream"]
  3049. if not self.agent._interrupt_requested and raw_stream is not None and not runaway:
  3050. if not saw_message_stop:
  3051. raise EmptyStreamError(
  3052. "Anthropic Messages stream ended before message_stop (possible upstream stream drop)."
  3053. if saw_stream_event else
  3054. "Provider returned an empty stream with no events (possible upstream error or malformed event stream).")
  3055. base_final_message = raw_stream.get_final_message()
  3056. # The SDK snapshot keeps only stop_reason/stop_sequence from message_delta; the
  3057. # refusal's stop_details (category/explanation) survives only in our accumulator.
  3058. _stop_details = accumulator.finalize().get("stop_details")
  3059. if _stop_details is not None and getattr(base_final_message, "stop_details", None) is None:
  3060. base_final_message.stop_details = _stop_details
  3061. finally:
  3062. try:
  3063. self._close_managed_stream()
  3064. finally:
  3065. manager = _stream_context["manager"]
  3066. if manager is not None:
  3067. manager.__exit__(None, None, None)
  3068. if self.agent._interrupt_requested:
  3069. return None
  3070. if runaway:
  3071. self._log_runaway_cut(runaway)
  3072. # No message_stop, so no final message: the SDK's snapshot is what streamed so far.
  3073. message = accumulator.response(_stream_context["stream"].current_message_snapshot)
  3074. message.stop_reason = "max_tokens"
  3075. message._runaway_repetition = True # same hand-off as the chat_completions wire
  3076. return message
  3077. if base_final_message is not None:
  3078. self._check_anthropic_message(base_final_message, tool_drop=False)
  3079. if not stream.output_modified:
  3080. return self._check_anthropic_message(base_final_message)
  3081. return self._check_anthropic_message(accumulator.response(base_final_message))
  3082. # ── retry loop ──────────────────────────────────────────────────────
  3083. def _retry_after_drop(self, e, attempt: int, max_retries: int, *, mid_tool_call: bool, reason: str) -> None:
  3084. """Warn about the drop and tear down the request-local client. Shared
  3085. clients are never closed from inside a request (FD-recycle hazard); the
  3086. OpenAI primary is replaced lazily."""
  3087. self.agent._emit_stream_drop(
  3088. error=e, attempt=attempt + 1, max_attempts=max_retries + 1, mid_tool_call=mid_tool_call, diag=self.clients.diag)
  3089. if self.agent._is_provider_stream_parse_error(e):
  3090. from agent.anthropic_adapter import buffer_anthropic_tool_input
  3091. buffer_anthropic_tool_input(self.api_kwargs, getattr(self.agent, "_anthropic_base_url", None))
  3092. self._cancel_current_stream_attempt(reason)
  3093. self.clients.close_once(reason)
  3094. # Exponential backoff between stream-level reconnects (back-to-back retries
  3095. # hammer a provider that just dropped us). Interruptible: /stop exits at once.
  3096. from agent.retry_utils import jittered_backoff
  3097. _wait_stream_retry_backoff(
  3098. self.agent, jittered_backoff(attempt + 1, base_delay=1.0, max_delay=4.0, jitter_ratio=0.0))
  3099. # The backoff is not the dead attempt's silence: restart the stale clock so the
  3100. # stale monitor cannot kill (and strike) a stream that has not reopened yet.
  3101. self.last_chunk_time["t"] = time.time()
  3102. def _maybe_disable_streaming(self, e) -> None:
  3103. """Flip to non-streaming for failures streaming itself cannot survive, or that
  3104. re-streaming can only repeat: the provider rejecting streams outright,
  3105. AnthropicBedrock IAM lacking InvokeModelWithResponseStream, a custom anthropic_messages
  3106. provider emitting SSE events out of order (#72833), or a gateway answering
  3107. with contentless SSE keepalive frames (a degraded gateway answers every
  3108. streaming request that way, so the retry must change channel to make progress).
  3109. llama-server stays on streaming: the router can't cancel a non-streamed request."""
  3110. if _streams_llamacpp(self.agent):
  3111. return
  3112. if _is_provider_stream_empty_frame_error(e):
  3113. self.agent._disable_streaming = True
  3114. logger.warning(
  3115. "Provider stream returned an empty SSE frame (keepalive, no payload) before any "
  3116. "delta — switching %s/%s to non-streaming for this session.",
  3117. self.agent.provider or "unknown", self.agent.model or "unknown")
  3118. # Durable channel, not _buffer_status: this recovery is expected to SUCCEED, and
  3119. # buffered retry chatter is dropped on successful recovery. Fires at most once per
  3120. # session (streaming is off from here on).
  3121. self.agent._emit_warning(
  3122. "⚠️ Provider stream returned an empty keepalive frame — retrying this turn "
  3123. "without streaming (streaming stays off for this session).")
  3124. return
  3125. from agent.anthropic_adapter import _is_stream_unavailable_error
  3126. if not _is_stream_unavailable_error(e):
  3127. return
  3128. _err_lower = str(e).lower()
  3129. _is_stream_unsupported = "stream" in _err_lower and "not supported" in _err_lower
  3130. if "unexpected event order" in _err_lower and not _is_stream_unsupported:
  3131. # Custom anthropic_messages SSE out of order (#72833): re-streaming repeats it.
  3132. # Bedrock keeps turn_recovery's sticky Converse switch instead.
  3133. if self.agent.api_mode != "anthropic_messages" or self.agent.provider == "bedrock":
  3134. return
  3135. self.agent._disable_streaming = True
  3136. self.agent._safe_print(
  3137. "\n⚠ Provider sent Anthropic stream events out of order. Switching to non-streaming.\n",
  3138. diagnostic=True,
  3139. )
  3140. return
  3141. # Remaining matches: stream rejected outright, or Bedrock IAM stream denial.
  3142. self.agent._disable_streaming = True
  3143. self.agent._safe_print(
  3144. "\n⚠ Streaming is not supported for this model/provider. Switching to non-streaming.\n"
  3145. " To avoid this delay, set display.streaming: false in config.yaml\n"
  3146. if _is_stream_unsupported else
  3147. "\n⚠ AWS IAM denied bedrock:InvokeModelWithResponseStream. Switching to non-streaming.\n"
  3148. " Grant that action to restore streaming output.\n",
  3149. diagnostic=True,
  3150. )
  3151. def _handle_stream_error(self, e: Exception, attempt: int, max_retries: int) -> bool:
  3152. """Classify a failed attempt: True = retry; False = stop with
  3153. ``result["error"]`` set (unless our own interrupt force-closed the
  3154. socket). Runs inside the ``except`` so ``logger.exception`` works."""
  3155. import httpx as _httpx
  3156. # Our own interrupt force-close: no retry/fallback/"reconnecting" (the
  3157. # poll loop raises InterruptedError).
  3158. if self._request_cancelled["value"]:
  3159. logger.debug("Streaming worker caught %s after request cancellation — exiting without retry.", type(e).__name__)
  3160. return False
  3161. e = self._as_lifetime_error(e) # a cap kill: transient, and its copy names the setting
  3162. _is_timeout = isinstance(e, (_httpx.ReadTimeout, _httpx.ConnectTimeout, _httpx.PoolTimeout, StreamLifetimeExceeded))
  3163. # ReadError: abort/reset mid-body (stale-kill shutdown under a parked reader, ECONNRESET) — retry loop owns it.
  3164. # anthropic.APIConnectionError: the Anthropic SDK wraps connect/read drops
  3165. # (incl. stale-kill aborts) in its own type, not httpx's.
  3166. _is_conn_err = isinstance(e, (_httpx.ConnectError, _httpx.ReadError, _httpx.RemoteProtocolError, ConnectionError,
  3167. *_anthropic_connection_error_types()))
  3168. _is_stream_parse_err = self.agent._is_provider_stream_parse_error(e)
  3169. _is_empty_stream = isinstance(e, EmptyStreamError)
  3170. _is_sse_conn_err = not _is_timeout and not _is_conn_err and _is_sse_connection_error(e)
  3171. _is_transient = _is_timeout or _is_conn_err or _is_sse_conn_err or _is_stream_parse_err
  3172. if not self.deltas_were_sent["yes"] and not getattr(self.agent, "_stream_options_unsupported", False) and _rejects_stream_options(e):
  3173. # Nothing streamed yet: drop the usage extension for this session and re-open.
  3174. self.agent._stream_options_unsupported = True
  3175. self._compat_retries = 1
  3176. logger.info("Endpoint rejected stream_options (HTTP %s); retrying without it for this session.",
  3177. getattr(e, "status_code", None))
  3178. self._cancel_current_stream_attempt("stream_options_rejected_retry")
  3179. self.clients.close_once("stream_options_rejected_retry")
  3180. return True
  3181. _partial_tool_in_flight = self._forget_invisible_deltas(e)
  3182. if self.deltas_were_sent["yes"]:
  3183. # Died AFTER tokens were delivered: normally no retry (would duplicate
  3184. # text). Exception: a tool call in flight — aborting discards it, so
  3185. # retry TRANSIENT errors (a "reconnecting" marker + duplicated
  3186. # preamble beats a failed action; no tool has executed yet).
  3187. if not (_partial_tool_in_flight and _is_transient and attempt < max_retries):
  3188. logger.warning("Streaming failed after partial delivery, not retrying: %s", e)
  3189. self.result["error"] = e
  3190. return False
  3191. # Marker explains the re-streamed preamble (``_emit_stream_drop`` logs the WARNING);
  3192. # reset the streamed-text buffer so it isn't double-recorded; fresh accumulators.
  3193. if self.agent._warning_presentation_enabled():
  3194. self._quiet(self.agent._fire_stream_delta, "\n\n⚠ Connection dropped mid tool-call; reconnecting…\n\n")
  3195. self._quiet(self.agent._reset_stream_delivery_tracking)
  3196. self.deltas_were_sent["yes"] = False
  3197. self.first_delta_fired["done"] = False
  3198. self._retry_after_drop(e, attempt, max_retries, mid_tool_call=True, reason="stream_mid_tool_retry_cleanup")
  3199. return True
  3200. if _is_transient or _is_empty_stream:
  3201. # Transient network / timeout error: retry with a fresh connection first.
  3202. if attempt < max_retries:
  3203. self._retry_after_drop(e, attempt, max_retries, mid_tool_call=False, reason="stream_retry_cleanup")
  3204. return True
  3205. # Exhausted: log full diagnostics (chain, headers, bytes/elapsed).
  3206. self.agent._log_stream_retry(kind="exhausted", error=e, attempt=max_retries + 1,
  3207. max_attempts=max_retries + 1, mid_tool_call=False, diag=self.clients.diag)
  3208. # Empty stream: "connection failed" would send users chasing network issues.
  3209. if _is_stream_parse_err or _is_empty_stream:
  3210. _what = ("Provider returned malformed streaming data after" if _is_stream_parse_err
  3211. else "Provider returned an empty response stream after")
  3212. self.agent._buffer_diagnostic_status(
  3213. f"❌ {_what} {max_retries + 1} attempts. The provider may be experiencing issues — try again in a moment.")
  3214. else:
  3215. from agent.stream_diag import buffer_connect_exhausted_notice
  3216. buffer_connect_exhausted_notice(self.agent, e, attempts=max_retries + 1, base_url=self.agent.base_url)
  3217. else:
  3218. self._maybe_disable_streaming(e)
  3219. logger.exception("Streaming failed before delivery: %s", e)
  3220. if self._unmask_server_error_with_nonstreaming(e):
  3221. return False
  3222. # Propagate to the main retry loop (credential rotation, fallback, backoff).
  3223. self.result["error"] = e
  3224. return False
  3225. def _unmask_server_error_with_nonstreaming(self, e: Exception) -> bool:
  3226. """One non-streaming re-issue when a 5xx killed the stream before any delta.
  3227. Some gateways validate the request only on their non-streaming path and crash
  3228. opaquely ("500 something went wrong") when streaming — the real 4xx, with its
  3229. actionable message, never reaches the user through stream retries. One
  3230. non-streaming probe per 60s window (timestamped on the agent: each outer retry
  3231. builds a fresh _StreamingCall, so an instance flag would re-probe every attempt)
  3232. surfaces it: on success the response is delivered for this turn WITHOUT latching
  3233. non-streaming (a transient gateway 500 must not permanently disable streaming);
  3234. on a probe 4xx that error REPLACES the opaque 5xx; any other probe failure keeps
  3235. the original error. The successful delivery is bracketed by its own stream
  3236. start/end pair (the failed attempt already emitted a terminal end), and a response
  3237. that cannot be replayed propagates ``e``. The probe runs on this worker thread while
  3238. the stream monitor still polls, so its stale check is suspended for the probe's
  3239. duration (the probe has its own non-streaming watchdog). Interrupts re-raise (the outer handler routes them), and a /stop that arrived before
  3240. this point suppresses the probe entirely — the loop's pre-retry interrupt check owns
  3241. that decision, so a pending stop must not buy one more request.
  3242. True = handled (caller must not overwrite result); False = propagate ``e``.
  3243. """
  3244. if getattr(self.agent, "_interrupt_requested", False):
  3245. return False
  3246. if _streams_llamacpp(self.agent):
  3247. # Same error either way, and the router can't cancel this probe once its watchdog gives up.
  3248. return False
  3249. status = _extract_status_code(e)
  3250. if status is None or status < 500 or self.deltas_were_sent["yes"]:
  3251. return False
  3252. if getattr(self.agent, "api_mode", "") not in ("", "chat_completions"):
  3253. return False # replay handles chat-completions shapes only
  3254. now = time.monotonic()
  3255. last_probe = self.agent._stream_5xx_probe_ts
  3256. if last_probe is not None and now - last_probe < _STREAM_5XX_PROBE_WINDOW_S:
  3257. return False # one probe per 60s window
  3258. self.agent._stream_5xx_probe_ts = now
  3259. probe_kwargs = {k: v for k, v in self.api_kwargs.items() if k not in ("stream", "stream_options")}
  3260. stale_timeout = self._stream_stale_timeout
  3261. self._stream_stale_timeout = float("inf") # no chunks arrive during the probe
  3262. try:
  3263. probe = interruptible_api_call(self.agent, probe_kwargs)
  3264. except (KeyboardInterrupt, InterruptedError):
  3265. raise # the outer handler routes user interrupts; never swallow them
  3266. except Exception as probe_err:
  3267. probe_status = _extract_status_code(probe_err)
  3268. if probe_status is not None and probe_status < 500:
  3269. # The provider's REAL validation error beats the opaque 5xx.
  3270. logger.info("Non-streaming unmask probe surfaced the underlying error: %s", probe_err)
  3271. self.result["error"] = probe_err
  3272. return True
  3273. logger.info("Non-streaming unmask probe failed: %s", probe_err)
  3274. return False
  3275. finally:
  3276. self._stream_stale_timeout = stale_timeout
  3277. logger.info("Streaming 5xx re-issued non-streaming successfully for %s/%s "
  3278. "(not latched: the 5xx may be transient).",
  3279. self.agent.provider or "unknown", self.agent.model or "unknown")
  3280. self._quiet(self.agent._buffer_status,
  3281. "⚠ Streaming failed with a provider server error; the non-streaming retry succeeded.")
  3282. try:
  3283. # The failed attempt already emitted its terminal on_stream_end(finished=False),
  3284. # so the recovered delivery opens and closes its OWN stream pair — consumers must
  3285. # never see deltas after that error event.
  3286. replayed = _with_stream_emitters(self.agent, lambda: self._replay_final_response(probe))
  3287. except Exception as replay_err:
  3288. # A response we cannot replay must not escape into _call()'s except block.
  3289. logger.exception("Non-streaming unmask probe response could not be replayed: %s", replay_err)
  3290. return False
  3291. self.result["response"] = replayed
  3292. return True
  3293. def _call_wire(self, stream_attempt_id: int):
  3294. if self.agent.api_mode != "anthropic_messages":
  3295. return self._call_chat_completions(stream_attempt_id)
  3296. # Per-request client so the watchdog aborts its socket, not the shared one.
  3297. request_client = self.clients.set_client(
  3298. self.agent._create_request_anthropic_client(reason="anthropic_stream_request"), kind="anthropic_messages")
  3299. return self._call_anthropic(request_client)
  3300. def _call(self):
  3301. _max_stream_retries = env_int("HERMES_STREAM_RETRIES", 2)
  3302. # The one stream_options compatibility retry (#9705) is not a network retry and must not
  3303. # consume the transient budget: on the last attempt (or HERMES_STREAM_RETRIES=0) the
  3304. # handler returned True and the loop ended with neither a response nor an error set.
  3305. self._compat_retries = 0
  3306. _stream_attempt = -1
  3307. try:
  3308. while _stream_attempt < _max_stream_retries + self._compat_retries:
  3309. _stream_attempt += 1
  3310. stream_attempt_id = self._start_stream_attempt()
  3311. # Else /stop closes the socket and the retry opens a fresh one (a full read timeout each).
  3312. if self.agent._interrupt_requested:
  3313. self._cancel_current_stream_attempt("interrupt_before_stream_retry")
  3314. raise InterruptedError("Agent interrupted before stream retry")
  3315. try:
  3316. self.result["response"] = _with_stream_emitters(
  3317. self.agent, lambda: self._call_wire(stream_attempt_id))
  3318. return # success
  3319. except Exception as e:
  3320. self._released_attempt = stream_attempt_id # unwound: kills during the backoff are not wedge kills
  3321. self._close_managed_stream()
  3322. if not self._handle_stream_error(e, _stream_attempt, _max_stream_retries):
  3323. return
  3324. if self._route_switched_under_request():
  3325. self.result["error"] = e
  3326. return
  3327. except InterruptedError as e:
  3328. # Fast pre-retry interrupt surfaces through the normal result channel.
  3329. self.result["error"] = e
  3330. return
  3331. finally:
  3332. self._close_managed_stream()
  3333. # Reuse only after a clean stream; otherwise really close (fresh pool next).
  3334. self.clients.close_once(
  3335. "stream_request_complete" if self.result["response"] is not None else "stream_error_cleanup")
  3336. # ── poll-loop monitor (heartbeat / stale kill / interrupt) ──────────
  3337. def _run_call(self):
  3338. try:
  3339. self._call()
  3340. finally:
  3341. self._call_done.set()
  3342. def _shutdown_stale_attempt_socket(self, response: Any) -> None:
  3343. """Best-effort ``shutdown()`` on the killed attempt's socket (monitor thread).
  3344. The pool sweep in ``close_once`` can miss a connection that is checked
  3345. out for the in-flight body read. ``shutdown(SHUT_RDWR)`` is FD-safe
  3346. from any thread — it wakes the owner's ``recv`` without releasing the
  3347. descriptor — so the worker unwinds and releases its own response on
  3348. the owner thread (``_call``'s ``except``/``finally``). Never
  3349. ``close()`` here: releasing a live TLS descriptor from a stranger
  3350. thread lets the kernel recycle it under the owner's SSL BIO, which is
  3351. exactly what the shutdown-only rule in ``_abort_request_slot_client``
  3352. forbids (it covers request-local clients too, #30858).
  3353. """
  3354. if response is None or response is not self._attempt_stream_response:
  3355. return
  3356. try:
  3357. from agent.agent_runtime_helpers import _shutdown_socket, _socket_from_response
  3358. sock = _socket_from_response(response)
  3359. if sock is not None:
  3360. _shutdown_socket(sock)
  3361. logger.info("Shut down the stale stream's socket to unblock the reader "
  3362. "(attempt superseded; model=%s).", self.api_kwargs.get("model", "unknown"))
  3363. return
  3364. logger.debug("Stale stream socket shutdown found no socket; pool sweep is the only abort")
  3365. except Exception:
  3366. logger.debug("Stale stream socket shutdown failed", exc_info=True)
  3367. def _uncounted_stale_attempt(self) -> int:
  3368. """The started attempt the circuit breaker (see ``_stale_streak()``) has not counted
  3369. yet, else 0. Like the non-streaming and inline watchdogs, each attempt counts once:
  3370. the stale timer re-fires every window while the worker has not dispatched yet or is
  3371. still unwinding a kill, and none of those re-kills is another unresponsive attempt."""
  3372. with self.stream_attempt_lock:
  3373. attempt = int(self.stream_attempt_state["current"])
  3374. return 0 if attempt in self._stale_counted_attempts else attempt
  3375. def _count_stale_attempt(self) -> None:
  3376. attempt = self._uncounted_stale_attempt()
  3377. if attempt:
  3378. self._stale_counted_attempts.add(attempt)
  3379. _bump_stale_streak(self.agent)
  3380. def _kill_stale_stream(self, elapsed: float) -> None:
  3381. """SSE pings but no chunks: cancel the attempt and abort the request-local
  3382. client so the retry loop opens a fresh one. The shared client is never
  3383. closed from this (stranger) thread — earlier stale-killed workers may
  3384. still be unwinding SSL BIOs (FD-recycle corruption); the OpenAI primary
  3385. is replaced lazily."""
  3386. _est_ctx = estimate_request_context_tokens(self.api_kwargs)
  3387. logger.warning(
  3388. "Stream stale for %.0fs (threshold %.0fs) — no chunks received. model=%s context=~%s tokens. Killing connection.",
  3389. elapsed, self._stream_stale_timeout, self.api_kwargs.get("model", "unknown"), f"{_est_ctx:,}",
  3390. )
  3391. self.agent._buffer_diagnostic_status(
  3392. f"⚠️ No response from provider for {int(elapsed)}s (model: {self.api_kwargs.get('model', 'unknown')}, "
  3393. f"context: ~{_est_ctx:,} tokens). Reconnecting...")
  3394. # Captured BEFORE the cancel/abort: the pool sweep can miss a checked-out
  3395. # connection, so shut down the killed attempt's own socket too — still
  3396. # shutdown-only, never close (see the helper).
  3397. _killed_response = self._attempt_stream_response
  3398. with contextlib.suppress(Exception):
  3399. self._cancel_current_stream_attempt("stale_stream_kill")
  3400. self.clients.close_once("stale_stream_kill")
  3401. self._shutdown_stale_attempt_socket(_killed_response)
  3402. self._count_stale_attempt()
  3403. self._rearm_or_abandon_after_kill(elapsed)
  3404. def _abort_for_interrupt(self, stale_elapsed: float) -> None:
  3405. """/stop seen by the monitor: mark cancelled, abort the request-local
  3406. socket, wait for the worker, flag the interrupt."""
  3407. # Once per attempt: a stale kill that already counted this attempt wins.
  3408. attempt = self._uncounted_stale_attempt()
  3409. if attempt and stale_elapsed <= self._stream_stale_timeout and _record_interrupted_provider_wait(
  3410. self.agent, stale_elapsed, response_started=self.deltas_were_sent["yes"]):
  3411. self._stale_counted_attempts.add(attempt)
  3412. # Mark cancelled BEFORE force-closing so the worker treats the forced
  3413. # transport error as a cancel, not a network error (#6600).
  3414. self._request_cancelled["value"] = True
  3415. logger.debug("Force-closing streaming httpx client due to interrupt (not a network error).")
  3416. # Same as the stale kill: the pool sweep can miss the connection checked out for the
  3417. # in-flight body read, so shut down the attempt's own socket too (#98974).
  3418. _killed_response = self._attempt_stream_response
  3419. with contextlib.suppress(Exception):
  3420. self._cancel_current_stream_attempt("stream_interrupt_abort")
  3421. # Kind-aware: only the request-local socket; the shared _anthropic_client is never closed here.
  3422. self.clients.close_once("stream_interrupt_abort")
  3423. self._shutdown_stale_attempt_socket(_killed_response)
  3424. # Let the worker unwind Relay-managed scopes first; raising first lets
  3425. # turn teardown race a still-open scope and corrupt the LIFO stack.
  3426. if self.worker is not None:
  3427. _join_worker_for_relay_teardown(self.worker, label="Streaming")
  3428. self._monitor_interrupted["yes"] = True
  3429. # ── orchestration ───────────────────────────────────────────────────
  3430. def _resolve_stale_timeout(self) -> None:
  3431. """Set ``_stream_stale_timeout``. Local endpoints (unless the env is set) get
  3432. long but FINITE patience — 900s / ``agent.local_stream_stale_timeout`` /
  3433. HERMES_LOCAL_STREAM_STALE_TIMEOUT — an infinite one stalled sessions on a
  3434. crashed endpoint forever. Cloud values scale with context size and are
  3435. floored for known reasoning models (else BrokenPipeError from the gateway)."""
  3436. base = _configured_stale_base(self.agent)
  3437. if base == 180.0 and self.agent.base_url and is_local_endpoint(self.agent.base_url):
  3438. self._stream_stale_timeout = _local_stream_stale_timeout_default()
  3439. logger.debug("Local provider detected (%s) — stale stream timeout set to %.0fs",
  3440. self.agent.base_url, self._stream_stale_timeout)
  3441. return
  3442. self._stream_stale_timeout = _cloud_stale_timeout_for(self.agent, self.api_kwargs)
  3443. def _partial_stream_stub(self, error):
  3444. """Tokens already reached the platform: a finish_reason="length" stub fires the
  3445. continuation machinery; tool_calls=None blocks executing incomplete calls.
  3446. Content may be EMPTY (dropped tool call, overflow) — the loop skips appending an
  3447. empty stub and only sends the nudge (placeholder text leaked into the stitched
  3448. response). A text-only death with 0 visible chars never gets here: the error
  3449. handler reclassifies it as undelivered (#112419)."""
  3450. _partial_text = (getattr(self.agent, "_current_streamed_assistant_text", "") or "").strip() or None
  3451. _partial_names = list(self.result.get("partial_tool_names") or [])
  3452. if _partial_names:
  3453. # User-visible warning so the user and model both know what was attempted.
  3454. _name_str = ", ".join(_partial_names[:3])
  3455. if len(_partial_names) > 3:
  3456. _name_str += f", +{len(_partial_names) - 3} more"
  3457. _warn = (f"\n\n⚠ Stream stalled mid tool-call ({_name_str}); the action was not executed. "
  3458. f"Ask me to retry if you want to continue.")
  3459. _partial_text = (_partial_text or "") + _warn # model/result bookkeeping, never gated
  3460. if self.agent._warning_presentation_enabled():
  3461. self._quiet(self.agent._fire_stream_delta, _warn) # visible immediately
  3462. logger.warning(
  3463. "Partial stream dropped tool call(s) %s after %s chars of text; surfaced warning to user: %s",
  3464. _partial_names, len(_partial_text or ""), error)
  3465. # Classify the error before it is swallowed into the stub: the loop reads the
  3466. # content-filter tag and falls back; a context overflow must not be continued at all.
  3467. _cls = None
  3468. with contextlib.suppress(Exception):
  3469. from agent.error_classifier import classify_api_error
  3470. _cls = classify_api_error(
  3471. error, provider=str(getattr(self.agent, "provider", "") or ""), model=str(getattr(self.agent, "model", "") or ""))
  3472. _reset_stale_streak(self.agent) # deltas fired => provider responsive: clear the breaker
  3473. # #106260: continuing after a context-overflow error re-sends a larger request into the
  3474. # same overflow. Return an EMPTY stub marked terminal so the loop ends the turn instead.
  3475. # Scope is context_overflow ONLY: payload_too_large (413) has its own byte-scored recovery
  3476. # owner (turn_overflow._recover_payload_too_large, #88960/#47339) that must not be bypassed.
  3477. if _cls is not None and _cls.reason == FailoverReason.context_overflow:
  3478. logger.warning(
  3479. "Partial stream ended on a context-overflow error after %s chars; "
  3480. "NOT seeding a continuation stub (transcript is already over budget): %s",
  3481. len(_partial_text or ""), error,
  3482. )
  3483. return _build_partial_stream_stub(
  3484. "assistant", None, None, getattr(self.agent, "model", "unknown"), None,
  3485. dropped_tool_names=_partial_names, overflow_terminal=True,
  3486. api_mode=getattr(self.agent, "api_mode", None),
  3487. )
  3488. if not _partial_names:
  3489. logger.warning(
  3490. "Partial stream delivered before error; returning length-truncated stub with %s chars of "
  3491. "recovered content so the loop can continue from where the stream died: %s",
  3492. len(_partial_text or ""), error)
  3493. _stub = _build_partial_stream_stub("assistant", _partial_text, None,
  3494. getattr(self.agent, "model", "unknown"), None, dropped_tool_names=_partial_names,
  3495. api_mode=getattr(self.agent, "api_mode", None))
  3496. if _cls is not None and _cls.reason == FailoverReason.content_policy_blocked:
  3497. _stub._content_filter_terminated = True
  3498. return _stub
  3499. def run(self):
  3500. """Resolve the stale timeout, run the request (worker thread or inline),
  3501. drive the heartbeat/stale/interrupt monitor, then translate the outcome."""
  3502. self._resolve_stale_timeout()
  3503. # Delegated children and cron turns run the request INLINE (a worker inside
  3504. # their nested pools wedges before the socket opens) but must still STREAM
  3505. # (edge proxies kill silent POSTs). Only the poll loop moves to a monitor
  3506. # thread, which never issues a request, so the no-worker deadlock fix holds.
  3507. self._call_done = threading.Event()
  3508. self._monitor_interrupted = {"yes": False}
  3509. if should_use_direct_api_call(self.agent):
  3510. self.worker = None
  3511. monitor = threading.Thread(
  3512. target=_context_thread_target(self._monitor_loop), name="stream-inline-monitor", daemon=True)
  3513. monitor.start()
  3514. try:
  3515. self._run_call()
  3516. finally:
  3517. monitor.join(timeout=2.0)
  3518. else:
  3519. self.worker = threading.Thread(target=_context_thread_target(self._run_call), daemon=True)
  3520. self.worker.start()
  3521. self._monitor_loop()
  3522. if self._monitor_interrupted["yes"]:
  3523. raise InterruptedError("Agent interrupted during streaming API call")
  3524. if self.agent._interrupt_requested: # worker returned early before the monitor saw the flag
  3525. raise InterruptedError("Agent interrupted during streaming API call (post-worker)")
  3526. # Read the outcome once: an abandoned worker can still write into ``self.result`` later.
  3527. error, response = self.result["error"], self.result["response"]
  3528. if self._abandon_error is not None:
  3529. # Same bounded Relay join as /stop (#81521), before the retry opens a new scope.
  3530. _join_worker_for_relay_teardown(self.worker, label="Abandoned streaming")
  3531. error, response = self._abandon_error, None
  3532. self._forget_invisible_deltas(error) # nothing visible: retry, not a stub
  3533. if error is not None:
  3534. if self.deltas_were_sent["yes"]:
  3535. return self._partial_stream_stub(error)
  3536. raise error
  3537. if response is not None:
  3538. _reset_stale_streak(self.agent) # provider proved responsive: clear the breaker
  3539. # Propagate first-chunk timing for the ``post_api_request`` hook.
  3540. if isinstance(self.clients.diag, dict) and self.clients.diag.get("first_chunk_at"):
  3541. self.agent._last_api_first_chunk_at = float(self.clients.diag["first_chunk_at"])
  3542. return response
  3543. def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=None):
  3544. """Streaming variant of _interruptible_api_call: fires the delta callbacks per
  3545. text token (tool-call turns suppress them) and returns a SimpleNamespace in
  3546. the non-streaming response shape. codex_responses delegates to the already-
  3547. streaming codex runner; cron turns and delegated children run inline."""
  3548. if agent._interrupt_requested:
  3549. raise InterruptedError("Agent interrupted before streaming API call")
  3550. if agent.api_mode == "codex_responses":
  3551. return _stream_codex_passthrough(agent, api_kwargs, on_first_delta)
  3552. if agent.api_mode == "bedrock_converse":
  3553. return _BedrockStream(agent, api_kwargs, on_first_delta).run()
  3554. # Cross-turn stale-stream circuit breaker (see ``_stale_streak()``).
  3555. _check_stale_giveup(agent)
  3556. return _StreamingCall(agent, api_kwargs, on_first_delta).run()
  3557. __all__ = [
  3558. "build_api_kwargs",
  3559. "build_assistant_message",
  3560. "cleanup_task_resources",
  3561. "handle_max_iterations",
  3562. "interruptible_api_call",
  3563. "interruptible_streaming_api_call",
  3564. "try_activate_fallback",
  3565. ]