| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765 |
- """Assorted AIAgent runtime helpers (message repair/sanitization, credential recovery, primary
- runtime restore, prompt-cache policy, client construction, model switching, tool invocation).
- Each function takes the parent ``AIAgent`` as ``agent`` except the stateless message helpers.
- ``_ra()`` resolves ``run_agent`` lazily so tests patching ``run_agent.X`` keep intercepting.
- """
- from __future__ import annotations
- import contextlib
- import copy
- import json
- import logging
- import re
- import threading
- import time
- from datetime import datetime
- from pathlib import Path
- from typing import Any, Dict, List, Optional, Tuple
- from hermes_cli.timeouts import get_provider_request_timeout
- from agent.message_sanitization import (
- _FULL_ARGS_LOG_BOUND, coalesce_tool_call_id, coerce_tool_name, tool_call_id_variants, tool_result_id_variants
- )
- from agent.message_metadata import (
- TOOL_CALL_UIDS, merge_tool_call_uids, per_occurrence_tool_call_uids, record_absorbed_message)
- from agent.cache_bounds import publish_lru
- from agent.prompt_builder import STEER_DISPLAY_KIND, steer_user_row
- from agent.tool_dispatch_helpers import _trajectory_normalize_msg, make_tool_result_message
- from agent.think_scrubber import THINK_TAG_NAMES
- from agent.trajectory import convert_scratchpad_to_think
- from agent.credential_pool import (
- STATUS_EXHAUSTED, _parse_absolute_timestamp, credential_pool_entry_serves_endpoint,
- credential_pool_matches_provider,
- )
- from agent.error_classifier import FailoverReason
- from agent.retry_utils import parse_retry_after_seconds, reset_delay_from_message
- from agent.message_metadata import MERGED_TURN_PREFIX
- from agent.turn_context import drop_stale_api_content
- from agent.agent_runtime_helpers_placeholders import _INTERRUPTED_PLACEHOLDER
- from agent.primary_runtime_snapshot import _build_primary_runtime_snapshot
- from utils import base_url_host_matches, base_url_hostname, env_var_enabled, atomic_json_write
- logger = logging.getLogger(__name__)
- _TOOL_CALL_TAG_NAMES = ("tool_call", "tool_calls", "tool_result", "function_call", "function_calls")
- # Optional XML namespace prefix: some models serialize native tool calls as <ns:function_calls>.
- _NS_PREFIX = r"(?:[\w.-]+:)?"
- _REASONING_BLOCK_PATTERNS = tuple(
- re.compile(rf"<{name}>.*?</{name}>", re.DOTALL | re.IGNORECASE) for name in THINK_TAG_NAMES
- )
- _TOOL_CALL_BLOCK_PATTERNS = tuple(
- re.compile(rf"<{_NS_PREFIX}{name}\b[^>]*>.*?</{_NS_PREFIX}{name}>", re.DOTALL | re.IGNORECASE)
- for name in _TOOL_CALL_TAG_NAMES
- )
- # Named <function name=...> blocks; boundary- and name-gated (see _THINK_STRIP_PATTERNS note).
- _NAMED_FUNCTION_BLOCK_PATTERN = re.compile(
- r'(?:(?<=^)|(?<=[\n\r.!?:]))[ \t]*'
- r'<function\b[^>]*\bname\s*=[^>]*>'
- r'(?:(?:(?!</function>).)*)</function>', re.DOTALL | re.IGNORECASE,
- )
- _UNTERMINATED_REASONING_BLOCK_PATTERN = re.compile(
- rf'(?:^|\n)[ \t]*<(?:{"|".join(THINK_TAG_NAMES)})\b[^>]*>.*$', re.DOTALL | re.IGNORECASE
- )
- _ORPHAN_REASONING_TAG_PATTERN = re.compile(
- rf'</?(?:{"|".join(THINK_TAG_NAMES)})>\s*', re.IGNORECASE
- )
- _STRAY_TOOL_CALL_CLOSER_PATTERN = re.compile(
- rf'</(?:{_NS_PREFIX}(?:{"|".join(_TOOL_CALL_TAG_NAMES)}|function))>\s*', re.IGNORECASE
- )
- # An unclosed tool call is unrecoverable (#101899), so drop its remaining block.
- # Stray argument tags only identify fragment lines, not the rest of the text
- # (#102303). Require a line-start tag (optionally glued to a bare tool name,
- # process_manage<arg_key>) or a line-ending closer (wait</arg_value>) so inline
- # prose mentions and subsequent prose survive.
- _UNTERMINATED_TOOL_CALL_PATTERN = re.compile(
- rf'(?:^|\n)[ \t]*<{_NS_PREFIX}(?:{"|".join(_TOOL_CALL_TAG_NAMES)})\b[^>]*>.*$'
- r'|(?:^|\n)[ \t]*[\w.:-]*</?arg_(?:key|value)\b[^\n]*'
- r'|(?:^|\n)[^\n<]*</arg_(?:key|value)>[ \t\r]*(?=\n|$)',
- re.DOTALL | re.IGNORECASE,
- )
- def _ra():
- """Lazy ``run_agent`` reference for test-patch routing."""
- import run_agent
- return run_agent
- AGENT_RUNTIME_POST_HOOK_TOOL_NAMES = frozenset({
- "todo_list", "session_search", "memory", "clarify", "read_terminal", "desktop_preview",
- "drive_preview", "annotate_preview", "read_window_below", "manage_connections", "manage_catalog", "setup_mcp",
- "gui_tour",
- "delegate_task",
- })
- _TRAJECTORY_SYSTEM_PROMPT = (
- "You are a function calling AI model. You are provided with function signatures within <tools> </tools> XML tags. "
- "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting "
- "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug "
- "into functions. After calling & executing the functions, you will be provided with function results within "
- "<tool_response> </tool_response> XML tags. Here are the available tools:\n"
- "<tools>\n{tools}\n</tools>\n"
- "For each function call return a JSON object, with the following pydantic model json schema for each:\n"
- "{{'title': 'FunctionCall', 'type': 'object', 'properties': {{'name': {{'title': 'Name', 'type': 'string'}}, "
- "'arguments': {{'title': 'Arguments', 'type': 'object'}}}}, 'required': ['name', 'arguments']}}\n"
- "Each function call should be enclosed within <tool_call> </tool_call> XML tags.\n"
- "Example:\n<tool_call>\n{{'name': <function-name>,'arguments': <args-dict>}}\n</tool_call>"
- )
- def _trajectory_gpt_prefix(msg: dict[str, Any]) -> str:
- """Leading ``<think>`` block from native reasoning tokens, if any."""
- if msg.get("reasoning") and msg["reasoning"].strip():
- return f"<think>\n{msg['reasoning']}\n</think>\n"
- return ""
- def _with_think_block(content: str) -> str:
- """Every gpt turn gets a <think> block (empty if none) for a consistent training format."""
- return content if "<think>" in content else "<think>\n</think>\n" + content
- def _trajectory_tool_call_turn(msg: dict[str, Any]) -> str:
- content = _trajectory_gpt_prefix(msg)
- if msg.get("content") and msg["content"].strip():
- # <REASONING_SCRATCHPAD> -> <think> (model reasons via XML when native thinking is off)
- content += convert_scratchpad_to_think(msg["content"]) + "\n"
- for tool_call in msg["tool_calls"]:
- if not tool_call or not isinstance(tool_call, dict):
- continue
- raw_args = tool_call["function"]["arguments"]
- # Arguments were validated during conversation; degrade to {} rather than abort.
- try:
- arguments = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
- except json.JSONDecodeError:
- logger.warning("Unexpected invalid JSON in trajectory conversion: %s", raw_args[:100])
- arguments = {}
- tool_call_json = {"name": tool_call["function"]["name"], "arguments": arguments}
- content += f"<tool_call>\n{json.dumps(tool_call_json, ensure_ascii=False)}\n</tool_call>\n"
- return _with_think_block(content).rstrip()
- def _trajectory_tool_responses(msg: dict[str, Any], messages: list[dict[str, Any]], start: int) -> tuple[list[str], int]:
- """Collect the ``<tool_response>`` blocks for the tool run starting at ``start``; returns ``(blocks, next_index)``."""
- tool_responses = []
- j = start
- while j < len(messages) and messages[j]["role"] == "tool":
- tool_msg = messages[j]
- tool_content = tool_msg["content"]
- try: # pretty-print tool content if it looks like JSON
- if tool_content.strip().startswith(("{", "[")):
- tool_content = json.loads(tool_content)
- except (json.JSONDecodeError, AttributeError):
- pass
- tool_index = len(tool_responses)
- tool_name = (
- msg["tool_calls"][tool_index]["function"]["name"]
- if tool_index < len(msg["tool_calls"])
- else "unknown"
- )
- payload = json.dumps(
- {"tool_call_id": tool_msg.get("tool_call_id", ""), "name": tool_name, "content": tool_content},
- ensure_ascii=False,
- )
- tool_responses.append(f"<tool_response>\n{payload}\n</tool_response>")
- j += 1
- return tool_responses, j
- def convert_to_trajectory_format(agent, messages: list[dict[str, Any]], user_query: str, completed: bool) -> list[dict[str, Any]]:
- """Convert internal message history to trajectory format for saving."""
- # Trajectories are text-only: swap image-bearing tool messages for their text_summary so ~1MB
- # base64 blobs are not embedded.
- messages = [_trajectory_normalize_msg(m) for m in messages]
- trajectory = [
- {"from": "system", "value": _TRAJECTORY_SYSTEM_PROMPT.format(tools=agent._format_tools_for_system_message())},
- {"from": "human", "value": user_query},
- ]
- # Skip messages[0] (already added). Prefill is injected at API-call time only, so no offset adjustment is needed.
- i = 1
- while i < len(messages):
- msg = messages[i]
- if msg["role"] == "assistant":
- if msg.get("tool_calls"):
- trajectory.append({"from": "gpt", "value": _trajectory_tool_call_turn(msg)})
- tool_responses, j = _trajectory_tool_responses(msg, messages, i + 1)
- if tool_responses:
- trajectory.append({"from": "tool", "value": "\n".join(tool_responses)})
- i = j - 1 # skip the tool messages just processed
- else:
- content = _trajectory_gpt_prefix(msg) + convert_scratchpad_to_think(msg["content"] or "")
- trajectory.append({"from": "gpt", "value": _with_think_block(content).strip()})
- elif msg["role"] == "user":
- trajectory.append({"from": "human", "value": msg["content"]})
- i += 1
- return trajectory
- def _prepend_corruption_marker(tool_msg: dict, marker: str) -> None:
- existing = tool_msg.get("content")
- if isinstance(existing, str) and existing.startswith(marker):
- return
- if not isinstance(existing, (str, type(None))):
- try:
- existing = json.dumps(existing)
- except TypeError:
- existing = str(existing)
- tool_msg["content"] = f"{marker}\n{existing}" if existing else marker
- # The tool result was rewritten in place; a stamped dict's persisted row is now stale.
- from agent.context_compressor import _DB_PERSISTED_MARKER
- tool_msg.pop(_DB_PERSISTED_MARKER, None)
- def _find_tool_result(messages: list, start: int, tool_call: dict) -> Optional[dict]:
- """The tool result answering ``tool_call`` in the run starting at ``start``, if any."""
- for candidate in messages[start:]:
- if not isinstance(candidate, dict) or candidate.get("role") != "tool":
- return None
- if tool_result_id_variants(candidate.get("tool_call_id")) & tool_call_id_variants(tool_call):
- return candidate
- return None
- def _cursor_skip_prefix(messages: list, cursor: Optional[dict]) -> int:
- """Length of the ``is``-identical prefix already validated on the previous call."""
- prev_prefix = cursor.get("prefix") if cursor is not None else None
- start = 0
- if isinstance(prev_prefix, list):
- while start < min(len(prev_prefix), len(messages)) and messages[start] is prev_prefix[start]:
- start += 1
- return start
- def sanitize_tool_call_arguments(
- messages: list, *, logger=None, session_id: str | None = None, cursor: Optional[dict] = None
- ) -> int:
- """Repair corrupted assistant tool-call argument JSON in-place.
- ``cursor["prefix"]`` holds strong refs (not ``id()``: address reuse aliases) to the
- messages validated last call; the ``is``-identical prefix is skipped. Safe because only
- the surrogate sanitizers mutate live dicts; every other path replaces dicts, breaking identity.
- Safety argument for skipping: a message in the matched prefix was fully scanned before — every tool_call
- argument was either already valid JSON or was rewritten to ``"{}"`` (valid). The only code paths that
- mutate ``function["arguments"]`` on live history dicts between calls are the surrogate / non-ASCII
- sanitizers, which substitute characters *inside* JSON string values and cannot invalidate JSON syntax.
- Compression, repair, undo, and steer paths replace or reorder message dicts, which breaks the identity
- match and forces a re-scan. Holding strong references (the objects themselves, not ``id()``s) makes
- address reuse aliasing (#50372-style) impossible.
- """
- log = logger or logging.getLogger(__name__)
- if not isinstance(messages, list):
- return 0
- from agent.context_compressor import _DB_PERSISTED_MARKER
- repaired = 0
- marker = _ra().AIAgent._TOOL_CALL_ARGUMENTS_CORRUPTION_MARKER
- message_index = _cursor_skip_prefix(messages, cursor)
- while message_index < len(messages):
- msg = messages[message_index]
- tool_calls = msg.get("tool_calls") if isinstance(msg, dict) and msg.get("role") == "assistant" else None
- if not isinstance(tool_calls, list) or not tool_calls:
- message_index += 1
- continue
- insert_at = message_index + 1
- for tool_call in tool_calls:
- function = tool_call.get("function") if isinstance(tool_call, dict) else None
- if not isinstance(function, dict):
- continue
- arguments = function.get("arguments")
- if arguments is None or (isinstance(arguments, str) and not arguments.strip()):
- function["arguments"] = "{}"
- msg.pop(_DB_PERSISTED_MARKER, None)
- continue
- if not isinstance(arguments, str):
- continue
- with contextlib.suppress(json.JSONDecodeError):
- json.loads(arguments)
- continue
- # Canonical ``call_id || id`` precedence so scan and stub share the id the pipeline
- # uses; bare ``id`` misses Codex call_id results and orphans a stub.
- # Keying on bare ``id`` here would fail to find a result built with ``call_id`` (Codex Responses
- # format) and insert a duplicate stub that itself becomes an orphan (#58168).
- tool_call_id = _ra().AIAgent._get_tool_call_id_static(tool_call) or None
- function_name = function.get("name", "?")
- # Log the FULL (bounded) argument string: we are about to overwrite the only copy, which
- # may hold real user content from a truncated write_file/patch.
- log.warning(
- "Corrupted tool_call arguments repaired before request "
- "(session=%s, message_index=%s, tool_call_id=%s, function=%s, "
- "original_arguments=%r)", session_id or "-", message_index, tool_call_id or "-",
- function_name, arguments[:_FULL_ARGS_LOG_BOUND],
- )
- function["arguments"] = "{}"
- # The persisted row for a stamped dict still holds the corrupted args; pop the
- # marker so the flush rewrites it (the repaired args are what the wire saw).
- msg.pop(_DB_PERSISTED_MARKER, None)
- existing_tool_msg = _find_tool_result(messages, message_index + 1, tool_call)
- if existing_tool_msg is None:
- messages.insert(
- insert_at,
- make_tool_result_message(function_name if function_name != "?" else "", marker, tool_call_id),
- )
- insert_at += 1
- else:
- _prepend_corruption_marker(existing_tool_msg, marker)
- repaired += 1
- message_index += 1
- if cursor is not None:
- # Strong refs to the objects validated this call; any divergence (compression, undo, repair,
- # steer) forces a re-scan from that index.
- cursor["prefix"] = messages[:]
- return repaired
- # Session-scoped in-flight registry for note_turn_start: the gateway caches agents per routing key
- # while the transcript is keyed by session_id, so two agent objects can run concurrent turns on one
- # session unseen by per-agent state.
- _INFLIGHT_TURNS_BY_SESSION: dict[str, tuple[str, float]] = {}
- _INFLIGHT_TURNS_LOCK = threading.Lock()
- def note_turn_start(agent, turn_id: str):
- """Tripwire: warn when a turn starts while a previous turn of the same agent or session
- (on another agent object) has not finished its persist. Does not prevent the overlap; it
- names both turn ids so the dispatch route that bypassed the busy guard is findable in logs.
- Returns the previous in-flight turn_id on overlap, else None; takes the slot either way."""
- prev = getattr(agent, "_inflight_turn_id", None)
- prev_started = getattr(agent, "_inflight_turn_started", 0.0)
- agent._inflight_turn_id = turn_id
- agent._inflight_turn_started = time.time()
- overlap = None
- if prev and prev != turn_id:
- logger.warning(
- "turn %s starting while turn %s (started %.0fs ago) has not "
- "completed its turn-end persist (session=%s) — concurrent turns "
- "on one session; transcript writes may interleave", turn_id, prev,
- time.time() - prev_started if prev_started else -1.0, getattr(agent, "session_id", None) or "-",
- )
- overlap = prev
- # Cross-agent leg: same session_id in flight under another agent object (busy guard is keyed by
- # routing key and cannot see it). Persist-disabled forks share the parent's session_id but never
- # write, so they must not register or pop here (note_turn_persisted skips them symmetrically).
- session_id = getattr(agent, "session_id", None)
- if session_id and not getattr(agent, "_persist_disabled", False):
- now = time.time()
- with _INFLIGHT_TURNS_LOCK:
- entry = _INFLIGHT_TURNS_BY_SESSION.get(session_id)
- _INFLIGHT_TURNS_BY_SESSION[session_id] = (turn_id, now)
- # Record the session id registered under: compression can rotate agent.session_id mid-turn
- # and persist must pop the slot actually held.
- agent._inflight_turn_session_id = session_id
- if entry and entry[0] not in (turn_id, prev):
- logger.warning(
- "turn %s starting while turn %s (started %.0fs ago) is still "
- "in flight on session %s under a different agent object — "
- "two routing keys are mapped to one session_id; concurrent "
- "turns on one session; transcript writes may interleave", turn_id, entry[0],
- now - entry[1] if entry[1] else -1.0, session_id,
- )
- overlap = overlap or entry[0]
- return overlap
- def note_turn_persisted(agent):
- """Clear the in-flight marker at turn-end persist (see note_turn_start). Unconditional by
- design: on a real overlap the first persist clears the second slot, so the tripwire
- under-reports rather than double-reports."""
- agent._inflight_turn_id = None
- # Persist-disabled forks never registered a slot; popping here would steal the live parent
- # turn's slot (symmetric with note_turn_start).
- if not getattr(agent, "_persist_disabled", False):
- session_id = getattr(agent, "_inflight_turn_session_id", None) or getattr(agent, "session_id", None)
- if session_id:
- with _INFLIGHT_TURNS_LOCK:
- _INFLIGHT_TURNS_BY_SESSION.pop(session_id, None)
- agent._inflight_turn_session_id = None
- def _is_codex_interim(m: dict) -> bool:
- """Codex Responses interim turn: carries its own continuation state, replayed verbatim."""
- return bool(
- m.get("codex_reasoning_items")
- or m.get("codex_message_items")
- or m.get("finish_reason") == "incomplete"
- )
- def _merge_assistant_into(prev: dict, msg: dict) -> bool:
- """Fold consecutive assistant *msg* into *prev* (union tool_calls, concat text). Returns whether *msg*'s
- text survives: multimodal (list) content is never joined."""
- from agent.context_compressor import _DB_PERSISTED_MARKER
- prev_calls = list(prev.get("tool_calls") or [])
- new_calls = list(msg.get("tool_calls") or [])
- calls_changed = False
- if new_calls:
- prev["tool_calls"] = prev_calls + new_calls
- # The absorbed turn's calls keep the per-occurrence ids they were persisted with.
- if isinstance(extra := msg.get(TOOL_CALL_UIDS), dict) and extra:
- prev[TOOL_CALL_UIDS] = merge_tool_call_uids(
- per_occurrence_tool_call_uids(
- own if isinstance(own := prev.get(TOOL_CALL_UIDS), dict) else {}, prev_calls),
- per_occurrence_tool_call_uids(extra, new_calls))
- calls_changed = True
- elif prev_calls:
- prev["tool_calls"] = prev_calls
- else:
- # Drop a stale ``tool_calls: []`` at the source: strict providers (DeepSeek v4, Kimi) 400 on
- # it and it persists into replayed history.
- # Neither turn carries tool calls, but the surviving turn may still carry a stale ``tool_calls: []``
- # from the earlier message. An empty array is semantically "no tool calls", yet strict
- # OpenAI-compatible providers (DeepSeek v4, Moonshot/Kimi) reject it with HTTP 400 ("Invalid
- # 'messages[N].tool_calls': empty array..."). Drop the key HERE, at the source:
- # ``sanitize_api_messages`` only fixes the per-call wire copy, so a ``[]`` left on the repaired turn
- # survives in the live/persisted trajectory returned to callers (gateway/WebUI transcripts, session
- # resume, subagents, cron) and is replayed on the next turn — which is how #58755 kept reproducing
- # after the chokepoint fix (#77921). Popping is non-destructive: an empty array carries no
- # information.
- calls_changed = "tool_calls" in prev
- prev.pop("tool_calls", None)
- # Concatenate plain-text content only; leave multimodal (list) content alone.
- prev_content = prev.get("content")
- new_content = msg.get("content")
- content_rewritten = False
- text_kept = not new_content # nothing to lose
- if isinstance(prev_content, str) and isinstance(new_content, str):
- text_kept = True
- joined = "\n".join(p for p in (prev_content.strip(), new_content.strip()) if p)
- prev["content"] = joined
- # A falsy new_content leaves ``joined`` == prev_content; that is not a rewrite.
- # "") strips to nothing and ``joined`` collapses back to ``prev_content`` unchanged -- that must NOT
- # count as a rewrite (wz-heng, #78063 review).
- content_rewritten = joined != prev_content
- elif not prev_content and new_content is not None:
- prev["content"] = new_content
- content_rewritten = new_content != prev_content
- text_kept = True
- # Carry reasoning_content from the later turn only if the earlier lacks it (strict thinking
- # providers need one on the merged tool-call turn).
- reasoning_carried = False
- if not prev.get("reasoning_content") and msg.get("reasoning_content"):
- prev["reasoning_content"] = msg["reasoning_content"]
- reasoning_carried = True
- # A stale ``api_content`` sidecar overrides ``content`` at API-build time and would replay
- # pre-merge bytes; drop it only when content actually changed.
- # ``prev`` may carry an ``api_content`` sidecar (the exact bytes previously sent to the API, e.g. a
- # sanitize-divergence stamp — see ``_flush_messages_to_session_db``) from BEFORE this merge. The sidecar
- # takes priority over ``content`` at API-build time (``conversation_loop``'s ``api_messages`` build
- # substitutes it back in for role ``assistant``), so leaving it in place while ``prev["content"]``
- # changes would silently replay the pre-merge bytes and discard everything this merge just concatenated
- # on — the same stale-field-survives-the-merge shape as the ``tool_calls`` gap above, just for a
- # different field. Only drop it when the merge actually changed the resulting value (e.g. the later
- # turn's content is ``None``, or either side is multimodal/list — both branches skip the reassignment
- # and ``prev["content"]`` is untouched; a falsy ``new_content`` that strips to nothing also leaves
- # ``joined`` equal to the original ``prev_content``): in those cases the sidecar is still the exact
- # bytes previously sent for the UNCHANGED content, and dropping it would break the prompt-cache replay
- # invariant for no reason (wz-heng, #78063 review).
- if content_rewritten:
- drop_stale_api_content(prev)
- # The persist marker asserts the whole row is durable (content, tool_calls, reasoning sidecar), so
- # any merged field stales it; pop it or the flush scan identity-skips the merged dict and the DB
- # keeps the pre-merge row. The caller recomputes the flush cursor for the surviving sequence.
- if content_rewritten or calls_changed or reasoning_carried:
- prev.pop(_DB_PERSISTED_MARKER, None)
- return text_kept
- def _remember_absorbed_row(survivor: dict[str, Any], dropped: dict[str, Any], *, folded: bool) -> None:
- """Retire *dropped*'s row ids onto *survivor*; record its uid as a merge witness only when *folded* (its
- text survives). An empty incoming turn still merges; stamping an empty list would change a message that
- absorbed nothing. A dropped id that equals the survivor's own live id (the display-marker merge adopts
- the plain row's id, #94486) is not an absorbed row: the survivor IS that row."""
- from agent.conversation_compression_archive import OWN_ROW, RETIRED_DURABLE_ROWS, UNNAMED_DURABLE_ROWS
- own_id = survivor.get("_row_id")
- ids = []
- row_id = dropped.get("_row_id")
- if isinstance(row_id, int) and not isinstance(row_id, bool) and row_id > 0 and row_id != own_id:
- ids.append(row_id)
- for older in dropped.get("_absorbed_row_ids") or ():
- if isinstance(older, int) and not isinstance(older, bool) and older > 0 and older not in ids:
- ids.append(older)
- if ids:
- absorbed = survivor.setdefault("_absorbed_row_ids", [])
- for row_id in ids:
- if row_id not in absorbed:
- absorbed.append(row_id)
- # The rows the retired dict counted are behind the survivor now.
- if dropped.get(UNNAMED_DURABLE_ROWS):
- survivor[UNNAMED_DURABLE_ROWS] = int(survivor.get(UNNAMED_DURABLE_ROWS) or 0) + int(dropped[UNNAMED_DURABLE_ROWS])
- # Its own row is behind the survivor now too.
- if dropped.get(RETIRED_DURABLE_ROWS):
- survivor.setdefault(RETIRED_DURABLE_ROWS, []).extend(
- {k: v for k, v in row.items() if k != OWN_ROW} for row in dropped[RETIRED_DURABLE_ROWS])
- # The uid witness claims the dropped dict's TEXT lives on in the survivor: only a fold earns it. A
- # superseded row (``folded=False``) is retired like any absorbed row but its content is discarded.
- if folded:
- record_absorbed_message(survivor, dropped)
- def _count_unnamed_row(survivor: dict[str, Any], retired: dict[str, Any]) -> None:
- """On a reload without row ids nothing names *retired*'s durable row once the repair takes the dict
- out of the list, so *survivor* counts it. A dict that counts rows was loaded too: a merge may have
- popped its marker since. Call before the merge rewrites the survivor: *retired*'s own fields are
- recorded so the commit can name its row."""
- from agent.context_compressor import _DB_PERSISTED_MARKER
- from agent.conversation_compression_archive import (
- OWN_ROW, RETIRED_DURABLE_ROWS, UNNAMED_DURABLE_ROWS, retired_row_payload)
- def loaded(message: dict[str, Any]) -> bool:
- return bool(message.get(_DB_PERSISTED_MARKER) or message.get(UNNAMED_DURABLE_ROWS))
- if loaded(survivor) and loaded(retired) and not isinstance(retired.get("_row_id"), int):
- survivor[UNNAMED_DURABLE_ROWS] = int(survivor.get(UNNAMED_DURABLE_ROWS) or 0) + 1
- # A previously folded dict already records its original row. The transfer in
- # _remember_absorbed_row retires that record; its synthetic text/calls never existed in storage.
- if not any(row.get(OWN_ROW) for row in retired.get(RETIRED_DURABLE_ROWS) or ()):
- survivor.setdefault(RETIRED_DURABLE_ROWS, []).append(retired_row_payload(retired))
- def _remember_own_row(survivor: dict[str, Any]) -> None:
- """An assistant fold rewrites *survivor*'s text, so on a reload without row ids its own row no longer
- matches it by content. Record its loaded fields first, once."""
- from agent.context_compressor import _DB_PERSISTED_MARKER
- from agent.conversation_compression_archive import OWN_ROW, RETIRED_DURABLE_ROWS, retired_row_payload
- recorded = survivor.get(RETIRED_DURABLE_ROWS) or ()
- if (survivor.get(_DB_PERSISTED_MARKER) and not isinstance(survivor.get("_row_id"), int)
- and not any(isinstance(row, dict) and row.get(OWN_ROW) for row in recorded)):
- survivor.setdefault(RETIRED_DURABLE_ROWS, []).append({**retired_row_payload(survivor), OWN_ROW: True})
- def _retire_dropped_row(kept: list[dict], dropped: dict[str, Any], leading: list[dict]) -> None:
- """A durable row the repair drops was still handed to the caller, so the survivor before it stands
- for it. Left unnamed, an in-place compaction takes it for a row another surface appended and
- re-sequences it behind the running turn. A drop with no survivor before it waits in *leading*."""
- if not kept:
- leading.append(dropped)
- elif isinstance(kept[-1], dict):
- _count_unnamed_row(kept[-1], dropped)
- _remember_absorbed_row(kept[-1], dropped, folded=False)
- def _retire_leading_drops(kept: list[dict], leading: list[dict]) -> None:
- """Rows dropped ahead of the first survivor sit right before its own row, so it stands for them."""
- for dropped in leading if kept and isinstance(kept[0], dict) else ():
- _count_unnamed_row(kept[0], dropped)
- _remember_absorbed_row(kept[0], dropped, folded=False)
- def _merge_consecutive_assistants(messages: list[dict]) -> tuple[list[dict], int]:
- """Pass 0: merge consecutive assistant turns (codex interims exempt)."""
- repairs = 0
- collapsed: list[dict] = []
- for msg in messages:
- prev = collapsed[-1] if collapsed and isinstance(collapsed[-1], dict) else None
- if (
- prev is not None and prev.get("role") == "assistant"
- and isinstance(msg, dict) and msg.get("role") == "assistant"
- and not _is_codex_interim(msg) and not _is_codex_interim(prev)
- ):
- # A provisional verification candidate is superseded, not unioned.
- if prev.get("finish_reason") in {"verification_required", "verify_hook_continue"}:
- _count_unnamed_row(msg, prev)
- _remember_absorbed_row(msg, prev, folded=False)
- collapsed[-1] = msg
- else:
- _count_unnamed_row(prev, msg)
- _remember_own_row(prev)
- _remember_absorbed_row(prev, msg, folded=_merge_assistant_into(prev, msg))
- repairs += 1
- continue
- collapsed.append(msg)
- return collapsed, repairs
- def _drop_stray_tool_results(messages: list[dict]) -> tuple[list[dict], int]:
- """Pass 1: drop tool results not following a known assistant tool call. Consumes the whole
- alias group (call_id/id/response_item_id/composite) so a duplicate keyed on a sibling
- alias is not replayed to strict providers."""
- repairs = 0
- known_tool_ids: dict[str, int] = {} # alias -> group id; reset by assistant/user turns
- # Pass 1: drop stray tool messages that don't follow a known assistant tool call. A Responses call can
- # have several equivalent spellings (call_id, id, response_item_id, or a composite ``call|item`` id), so
- # consume the whole alias group when one spelling is matched. Alias expansion lives in
- # ``agent.message_sanitization.tool_call_id_variants`` / ``tool_result_id_variants`` (single policy
- # owner) — which also handles SDK tool_call objects, preserving the #91768 dict-or-object tolerance.
- matched_tool_groups: set = set()
- next_tool_group = 0
- filtered: list[dict] = []
- leading: list[dict] = []
- for msg in messages:
- role = msg.get("role") if isinstance(msg, dict) else None
- if role in ("assistant", "user"):
- # An assistant turn starts a new tool-result run; a user turn closes it (later tool
- # messages are orphans).
- known_tool_ids = {}
- matched_tool_groups = set()
- for tc in (msg.get("tool_calls") or []) if role == "assistant" else ():
- variants = tool_call_id_variants(tc)
- if variants:
- for tc_id in variants:
- known_tool_ids.setdefault(tc_id, next_tool_group)
- next_tool_group += 1
- elif role == "tool":
- result_variants = tool_result_id_variants(msg.get("tool_call_id"))
- candidate_groups = {
- known_tool_ids[tc_id] for tc_id in result_variants
- if tc_id in known_tool_ids and known_tool_ids[tc_id] not in matched_tool_groups
- }
- if result_variants and not candidate_groups:
- _retire_dropped_row(filtered, msg, leading)
- repairs += 1
- continue
- if candidate_groups:
- matched_tool_groups.add(min(candidate_groups))
- filtered.append(msg)
- _retire_leading_drops(filtered, leading)
- return filtered, repairs
- def _prune_unanswered_tool_calls(messages: list[dict]) -> tuple[list[dict], int]:
- """Pass 2: prune tool_calls not answered in the IMMEDIATELY following tool run (a displaced
- result masks the per-call stub pass and strict providers 400). Payload-empty turns are
- dropped; codex interims exempt."""
- from agent.context_compressor import _DB_PERSISTED_MARKER
- repairs = 0
- pruned: list[dict] = []
- leading: list[dict] = []
- for i, msg in enumerate(messages):
- if not (
- isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls")
- and not _is_codex_interim(msg)
- ):
- pruned.append(msg)
- continue
- answered: set = set()
- for follower in messages[i + 1:]:
- if not (isinstance(follower, dict) and follower.get("role") == "tool"):
- break
- tid = (follower.get("tool_call_id") or "").strip()
- if tid:
- answered.update(tool_result_id_variants(tid))
- kept_calls = [tc for tc in msg["tool_calls"] if tool_call_id_variants(tc) & answered]
- if len(kept_calls) != len(msg["tool_calls"]):
- repairs += 1
- if not kept_calls and not _msg_has_payload({k: v for k, v in msg.items() if k != "tool_calls"}):
- # Pruned calls were the only payload; drop the turn (empty assistant messages 400).
- _retire_dropped_row(pruned, msg, leading)
- continue
- if kept_calls:
- msg["tool_calls"] = kept_calls
- else:
- msg.pop("tool_calls", None)
- # tool_calls is part of the persisted row; rewriting it on a stamped dict stales the
- # marker, so pop it or the flush scan skips the dict and the DB keeps the old calls.
- msg.pop(_DB_PERSISTED_MARKER, None)
- pruned.append(msg)
- _retire_leading_drops(pruned, leading)
- return pruned, repairs
- def _merge_consecutive_users(messages: list[dict]) -> tuple[list[dict], int]:
- """Pass 3: merge consecutive plain-text user messages (no user input lost)."""
- from agent.context_compressor import _DB_PERSISTED_MARKER, split_user_originated_turn
- from agent.conversation_compression_archive import MERGED_DURABLE_ROWS
- from hermes_state import SessionDB
- def _plain_text(content: Any) -> bool:
- # never rewrite the persisted row around undecodable content
- return isinstance(content, str) and not content.startswith(SessionDB._CONTENT_JSON_PREFIX)
- repairs = 0
- merged: list[dict] = []
- for msg in messages:
- prev = merged[-1] if merged and isinstance(merged[-1], dict) else None
- if (
- prev is not None and prev.get("role") == "user"
- and isinstance(msg, dict) and msg.get("role") == "user"
- # A summary carrier followed by a new user row is a deliberate durable shape after
- # retry/rewind; never mutate the persisted carrier (sanitizers merge copies later).
- and split_user_originated_turn(prev)[0] is None
- # A /steer row that ended the previous run is already persisted; merging the next
- # prompt into it would rewrite it in place and re-break replay parity.
- and prev.get("display_kind") != STEER_DISPLAY_KIND
- # Only merge plain-text content; leave multimodal (list or undecodable sentinel) content alone.
- and _plain_text(prev.get("content", "")) and _plain_text(msg.get("content", ""))
- ):
- prev_content, new_content = prev.get("content", ""), msg.get("content", "")
- merged_content = (
- (prev_content + "\n\n" + new_content) if prev_content and new_content else (prev_content or new_content)
- )
- had_api_sidecar = "api_content" in prev
- # Read before the marker is popped below. An unpersisted turn folded in ends the claim:
- # the dict no longer stands for durable rows only.
- if (prev.get(_DB_PERSISTED_MARKER) or prev.get(MERGED_DURABLE_ROWS)) and msg.get(_DB_PERSISTED_MARKER):
- prev[MERGED_DURABLE_ROWS] = int(prev.get(MERGED_DURABLE_ROWS) or 1) + 1
- else:
- prev.pop(MERGED_DURABLE_ROWS, None)
- prev["content"] = merged_content
- # The clean-text persist override must replace only the absorbed turn, never the
- # unanswered text before it; kept across replay passes (an empty turn absorbs too).
- if prev_content:
- prev[MERGED_TURN_PREFIX] = prev_content
- # Merged content invalidates the api_content sidecar; drop it so replay cannot use stale bytes.
- drop_stale_api_content(prev)
- # A display-marker row (e.g. a model-switch marker persisted as role=user on
- # purpose, #48338) merging with a plain user row must not bury the plain row's
- # addressable identity: keeping the marker's display_kind hides the merged pair
- # from every user-turn index used for rewind/submit addressing (display rows are
- # excluded), so the plain row's durable id becomes unresolvable and the client's
- # next prompt.submit fails closed with the input silently dropped (#94486). Keep
- # the pair addressable instead: drop the display classification and carry the
- # plain row's id, retiring the marker's own id onto the absorbed list. display_kind
- # never reaches providers (stripped from every outgoing copy), so the merged
- # turn's wire payload is unchanged. Deliberate scope: when BOTH rows carry
- # display_kind (two consecutive model-switch markers) the pair keeps the first
- # marker's classification and id — no plain row is swallowed there, so no
- # addressable turn is lost.
- if prev.get("display_kind") and not msg.get("display_kind"):
- marker_row_id = prev.get("_row_id")
- prev.pop("display_kind", None)
- if msg.get("_row_id") is not None:
- if isinstance(marker_row_id, int) and not isinstance(marker_row_id, bool):
- absorbed_ids = prev.setdefault("_absorbed_row_ids", [])
- if marker_row_id not in absorbed_ids:
- absorbed_ids.append(marker_row_id)
- prev["_row_id"] = msg["_row_id"]
- # display_kind is part of the persisted row; reclassifying stales it even
- # when the merged bytes reproduce the persisted content (empty absorb).
- prev.pop(_DB_PERSISTED_MARKER, None)
- # Pop the persist marker only when the durable row actually changed: a merge that
- # reproduces the persisted bytes (e.g. an empty incoming turn) keeps its stamp.
- if merged_content != prev_content or had_api_sidecar:
- prev.pop(_DB_PERSISTED_MARKER, None)
- _remember_absorbed_row(prev, msg, folded=True)
- repairs += 1
- continue
- merged.append(msg)
- return merged, repairs
- _SEQUENCE_REPAIR_PASSES = (
- _merge_consecutive_assistants, _drop_stray_tool_results, _prune_unanswered_tool_calls,
- _merge_consecutive_users,
- )
- def _normalize_sentinel_encoded_content(messages: list[dict]) -> None:
- """Decode any sentinel-encoded row content in place before the alternation passes run, so an
- image-bearing turn is never merged as text (#125299). A multimodal turn can re-enter the working set
- as its ``\\x00json:[…]`` string (e.g. after a proactive prune re-inserts history, #124102). A body
- that no longer parses stays a sentinel string; ``_merge_consecutive_users`` refuses to weld it."""
- from hermes_state import SessionDB # lazy: the persistence layer owns the sentinel codec
- for msg in messages:
- if not isinstance(msg, dict):
- continue
- content = msg.get("content")
- decoded = SessionDB._decode_content(content)
- if decoded is not content:
- msg["content"] = decoded
- # Keep any `_db_persisted` marker: re-encoding reproduces the stored scalar, and dropping
- # it makes the append-only flush re-append this user turn as a duplicate (#125331).
- def repair_message_sequence(agent, messages: list[dict]) -> int:
- """Collapse malformed role-alternation left in the live history; returns repair count.
- Providers require strict alternation after the system message (violations: silent empty
- responses or 400s); this is the pre-call belt for host-fed, resumed or replayed histories.
- Passes in order: decode any sentinel-encoded multimodal rows (so an image is not merged as text);
- merge consecutive assistant turns (BEFORE orphan detection so the merged tool_call-id union is
- known); drop stray tool results; prune unanswered tool_calls; merge consecutive user turns. A user
- turn directly after an assistant turn is valid and left alone.
- """
- if not messages:
- return 0
- _normalize_sentinel_encoded_content(messages)
- repairs = 0
- current = messages
- for repair_pass in _SEQUENCE_REPAIR_PASSES:
- current, made = repair_pass(current)
- repairs += made
- if repairs > 0:
- # Rewrite in place so persistence/return value/DB flush see the repaired sequence.
- messages[:] = current
- return repairs
- def repair_message_sequence_with_cursor(agent, messages: list[dict]) -> int:
- """Run :func:`repair_message_sequence` and keep ``_last_flushed_db_idx`` consistent. Repair
- shrinks the list in place; counting identity-preserved survivors of the flushed prefix gives
- the exact new cursor, whereas a ``min()`` clamp would skip unflushed rows (used only without a snapshot)."""
- from agent.context_compressor import _DB_PERSISTED_MARKER
- flush_cursor = getattr(agent, "_last_flushed_db_idx", None)
- flushed_ids = {id(m) for m in messages[:flush_cursor]} if isinstance(flush_cursor, int) and flush_cursor > 0 else None
- stamped_ids = {id(m) for m in messages if isinstance(m, dict) and m.get(_DB_PERSISTED_MARKER)}
- repairs = repair_message_sequence(agent, messages)
- if repairs > 0:
- # A stamped survivor that lost its marker was mutated in place by a merge/prune pass; the
- # bounded flush scan would skip past it inside the identity-matched prefix, so force a
- # full re-scan (same contract as the compressor's _flush_scan_cursor_invalidated).
- if stamped_ids and any(
- id(m) in stamped_ids and not m.get(_DB_PERSISTED_MARKER) for m in messages
- ):
- agent._db_flush_scan_prefix = None
- if hasattr(agent, "_last_flushed_db_idx"):
- if flushed_ids is not None:
- agent._last_flushed_db_idx = sum(1 for m in messages if id(m) in flushed_ids)
- else:
- agent._last_flushed_db_idx = min(agent._last_flushed_db_idx, len(messages))
- return repairs
- def _flatten_content_text(content: Any) -> str:
- """Flatten list/dict content (e.g. Anthropic-via-OpenRouter block lists) to text: a raw list
- hitting ``re.sub`` raises TypeError and the loop retries forever. Thinking/reasoning blocks
- are dropped outright; their text key varies per provider."""
- if isinstance(content, str):
- return content
- if isinstance(content, list):
- return "".join(
- part if isinstance(part, str) else part.get("text")
- for part in content
- if isinstance(part, str) or (
- isinstance(part, dict)
- and str(part.get("type") or "").strip().lower() not in {"thinking", "reasoning", "redacted_thinking"}
- and isinstance(part.get("text"), str) and part.get("text")
- )
- )
- if isinstance(content, dict):
- return str(content.get("text") or content.get("content") or "")
- return str(content)
- # Order matters: closed pairs first (case-insensitive so mixed-case tags don't fall through to the
- # unterminated pass and eat trailing content), then tool-call XML blocks, the boundary+name-gated
- # <function> block, the unterminated reasoning block, stray orphan reasoning tags, and finally stray
- # tool-call CLOSERS only (bare/unterminated <function> is kept: a truncated streaming tail may still
- # be valuable, matching OpenClaw's asymmetry).
- _THINK_STRIP_PATTERNS = (
- *_REASONING_BLOCK_PATTERNS, *_TOOL_CALL_BLOCK_PATTERNS, _NAMED_FUNCTION_BLOCK_PATTERN,
- _UNTERMINATED_REASONING_BLOCK_PATTERN, _ORPHAN_REASONING_TAG_PATTERN,
- _STRAY_TOOL_CALL_CLOSER_PATTERN, _UNTERMINATED_TOOL_CALL_PATTERN,
- )
- def strip_think_blocks(agent, content: str) -> str:
- """Remove reasoning/thinking blocks from content, returning only visible text: closed tag
- pairs, unterminated open tags at a block boundary (mirrors ``gateway/stream_consumer.py``),
- stray orphan tags (all case-insensitive variants), and standalone tool-call XML blocks some
- open models emit; ``<function>`` is boundary- and ``name=``-gated so prose mentions survive."""
- content = _flatten_content_text(content) if content else ""
- for pattern in _THINK_STRIP_PATTERNS if content else ():
- content = pattern.sub('', content)
- return content
- def sync_credential_pool_entry_id(agent) -> None:
- """Rebind ``agent._credential_pool_entry_id`` from the current pool + key. OAuth refreshes
- can replace the token before recovery runs, so the key alone cannot attribute a failure;
- the stable entry ID can. Cleared when no pool is bound."""
- pool = getattr(agent, "_credential_pool", None)
- try:
- agent._credential_pool_entry_id = (
- pool.entry_id_for_api_key(getattr(agent, "api_key", None)) if pool is not None else None
- )
- except Exception:
- agent._credential_pool_entry_id = None
- _STATUS_TO_FAILOVER_REASON = {
- 402: FailoverReason.billing, 429: FailoverReason.rate_limit, 401: FailoverReason.auth,
- 403: FailoverReason.auth,
- }
- _USAGE_LIMIT_REASON_TOKENS = ("usage_limit_reached", "gousagelimit")
- _USAGE_LIMIT_MESSAGE_TOKENS = ("usage limit reached", "usage limit has been reached")
- def _failed_credential_identity(agent, pool) -> tuple[Optional[str], Optional[str]]:
- """``(api_key_hint, credential_id)`` of the key actually dispatched, not ``pool.current()``:
- the shared pointer often points at a different healthy entry, and marking it exhausted
- can take the whole pool offline from one 429."""
- api_key_hint = getattr(agent, "api_key", None) or None
- raw_id = getattr(agent, "_credential_pool_entry_id", None)
- credential_id = raw_id if isinstance(raw_id, str) and raw_id else None
- if not api_key_hint:
- cur = pool.current()
- if cur:
- api_key_hint = getattr(cur, "runtime_api_key", None)
- if not credential_id:
- current_id = getattr(cur, "id", None)
- if isinstance(current_id, str) and current_id:
- credential_id = current_id
- return api_key_hint, credential_id
- def _is_entitlement_403(agent, status_code, error_context) -> bool:
- """Entitlement 403s look like auth failures but refresh cannot fix them. Any xai-oauth 403
- is entitlement EXCEPT xAI's stale-token signals (``[WKE=unauthenticated:...]``,
- "could not be validated"), which must stay refreshable."""
- if agent._is_entitlement_failure(error_context, status_code):
- return True
- if status_code != 403:
- return False
- haystack = " ".join(
- # Subscription/entitlement 403s look like auth failures on the wire but refresh cannot fix them —
- # the OAuth token is already valid, the account simply lacks the entitlement. Without this guard,
- # the refresh path keeps minting fresh tokens against the same unsubscribed account and the main
- # agent loop spins re-issuing the same 403 until the user Ctrl+C's. Defense-in-depth for #26847:
- # xAI's backend has been seen to 403 standard SuperGrok subscribers with bodies that don't match the
- # existing entitlement keyword set in ``_is_entitlement_failure``. Any 403 against ``xai-oauth`` is
- # treated as entitlement here so the refresh loop can't spin in those cases either. Exception
- # (#29344): xAI's ``[WKE=unauthenticated:...]`` suffix and the ``OAuth2 access token could not be
- # validated`` phrasing are xAI's authoritative "this is a stale token, not entitlement" signal. When
- # either fires we must NOT apply the catch-all override — refresh is the recoverable path for these
- # bodies, and blanket-classifying them as entitlement was the bug that left long-running TUI
- # sessions stuck on stale tokens until the user exited and reopened.
- str(error_context.get(k) or "").lower()
- for k in ("message", "reason", "code", "error")
- if isinstance(error_context, dict)
- )
- if "oauth authentication is currently not allowed for this organization" in haystack:
- return True
- provider = agent.provider or ""
- if provider == "anthropic" and getattr(agent, "api_mode", "") == "anthropic_messages":
- return True
- if provider == "xai-oauth":
- return not (
- "[wke=unauthenticated:" in haystack
- or "oauth2 access token could not be validated" in haystack
- )
- return False
- def _recover_auth_failure(agent, pool, **kwargs):
- from agent.credential_auth_recovery import recover_auth_failure
- return recover_auth_failure(agent, pool, **kwargs)
- def _recover_rate_limit(pool, *, has_retried_429, error_context, api_key_hint, credential_id, rotate_and_swap):
- # Already-exhausted credential: rotate immediately. Avoids the "cancel-between-429s" trap where
- # the local has_retried_429 resets per prompt and retries forever.
- current_entry = None
- if credential_id:
- current_entry = next((e for e in pool.entries() if e.id == credential_id), None)
- if api_key_hint:
- current_entry = current_entry or next(
- (e for e in pool.entries() if e.runtime_api_key == api_key_hint), None
- )
- if current_entry is None:
- current_entry = pool.current()
- current_last_status = getattr(current_entry, "last_status", None) if current_entry else None
- if current_last_status == STATUS_EXHAUSTED:
- _ra().logger.info(
- "Credential already exhausted (last_status=%s) — rotating immediately instead of retrying",
- current_last_status,
- )
- return (True, False) if rotate_and_swap(429, "rate limit, pre-exhausted") else (False, True)
- usage_limit_reached = False
- if error_context:
- context_reason = str(error_context.get("reason") or "").lower()
- context_message = str(error_context.get("message") or "").lower()
- usage_limit_reached = any(t in context_reason for t in _USAGE_LIMIT_REASON_TOKENS) or any(
- t in context_message for t in _USAGE_LIMIT_MESSAGE_TOKENS
- )
- if not has_retried_429 and not usage_limit_reached:
- return False, True
- return (True, False) if rotate_and_swap(429, "rate limit") else (False, True)
- def recover_with_credential_pool(
- agent, *, status_code: Optional[int], has_retried_429: bool,
- classified_reason: Optional[FailoverReason] = None,
- error_context: Optional[dict[str, Any]] = None, billing_unverified: bool = False,
- ) -> tuple[bool, bool]:
- """Attempt credential recovery via pool rotation; returns (recovered, has_retried_429).
- Rate limits: retry once, then rotate. Billing: rotate immediately. Auth: refresh before
- rotating. ``classified_reason`` beats raw HTTP codes (e.g. Anthropic 400 "out of extra
- usage"); ``billing_unverified`` gives the entry a short cooldown, not the one-hour bench."""
- pool = agent._credential_pool
- if pool is None:
- return False, has_retried_429
- # The pool belongs to the PRIMARY provider: acting on fallback errors would corrupt its state
- # and reset base_url to the primary endpoint. Empty pool provider means unscoped; empty agent
- # provider is a mismatch (swap would leave provider="" model="").
- # Defensive guard: if a fallback provider is active and its provider name doesn't match the pool's
- # provider, the pool belongs to the PRIMARY provider. Mutating it based on fallback errors would corrupt
- # the primary's credential state (see #33088) and, via _swap_credential, overwrite the agent's base_url
- # back to the primary's endpoint — every subsequent request then goes to the wrong host and 404s (see
- # #33163). The pool should only act when the agent is still on the same provider that seeded the pool.
- current_provider = (getattr(agent, "provider", "") or "").strip().lower()
- pool_provider = (getattr(pool, "provider", "") or "").strip().lower()
- if pool_provider and not credential_pool_matches_provider(
- pool, current_provider, base_url=getattr(agent, "base_url", None)
- ):
- # Same fail-closed boundary predicate as runtime binding.
- _ra().logger.warning(
- "Credential pool provider mismatch: pool=%s, agent=%s — "
- "skipping pool mutation to avoid cross-provider contamination",
- pool_provider, current_provider,
- )
- return False, has_retried_429
- api_key_hint, credential_id = _failed_credential_identity(agent, pool)
- effective_reason = classified_reason
- if effective_reason is None:
- effective_reason = _STATUS_TO_FAILOVER_REASON.get(status_code)
- def _rotate_and_swap(default_status: int, label: str) -> bool:
- """Rotate away from the failed credential; True when a new entry was swapped in."""
- rotate_status = status_code if status_code is not None else default_status
- kwargs = {
- "status_code": rotate_status,
- "error_context": error_context,
- "api_key_hint": api_key_hint,
- }
- if credential_id:
- kwargs["credential_id"] = credential_id
- # Pass classified semantics, not just the status: a billing 403 and an edge-throttle 403
- # need opposite cooldowns.
- if effective_reason is not None:
- from agent.credential_pool_model_cooldowns import pool_failure_reason
- kwargs["failure_reason"] = pool_failure_reason(effective_reason, error_context, billing_unverified)
- model = getattr(agent, "model", None)
- if isinstance(model, str) and model.strip():
- kwargs["model"] = model
- next_entry = pool.mark_exhausted_and_rotate(**kwargs)
- if next_entry is None:
- return False
- if not credential_pool_entry_serves_endpoint(next_entry, getattr(agent, "base_url", None)):
- # Mixed same-provider pool (#68237): the entry serves another endpoint and _swap_credential
- # would rebind this session to it. Treat as no recovery, like a rotation that yields nothing.
- _ra().logger.info(
- "Credential %s (%s) — pool entry %s serves another endpoint; not swapping",
- rotate_status, label, getattr(next_entry, "id", "?"),
- )
- return False
- _ra().logger.info(
- "Credential %s (%s) — rotated to pool entry %s",
- rotate_status, label, getattr(next_entry, "id", "?"),
- )
- swapped = agent._swap_credential(next_entry) is not False
- benched = next((e for e in pool.entries() if e.id == credential_id), None) if credential_id else None
- if (
- swapped
- and benched is not None
- and benched.priority < getattr(next_entry, "priority", benched.priority)
- and not getattr(agent, "_credential_pool_revert_id", None)
- and effective_reason in (FailoverReason.rate_limit, FailoverReason.billing)
- ):
- # A quota bench (429/402) lifts when the window reopens, and a fresh session's
- # select() would go straight back to this entry; arm the per-turn hook so the live
- # session does too (#114501). Only when the benched entry OUTRANKS the one we rotated
- # to: a session that was already on the fallback (preferred benched elsewhere) and
- # rotates UP once the preferred window reopened must not be pulled back down when
- # the fallback's cooldown lifts. Keep the FIRST benched entry across chained
- # rotations — it is the preferred one. Auth benches are not windows; they stay.
- agent._credential_pool_revert_id = credential_id
- return swapped
- if effective_reason == FailoverReason.upstream_rate_limit:
- # Upstream (e.g. DeepSeek behind OpenRouter) is throttling the aggregator; the credential is
- # healthy. Do not rotate/exhaust; let fallback switch models.
- upstream = (error_context or {}).get("upstream_provider") if error_context else None
- if upstream:
- _ra().logger.info(
- "Upstream provider %s rate-limited via aggregator — skipping "
- "credential rotation, deferring to fallback chain", upstream,
- )
- else:
- _ra().logger.info(
- "Upstream aggregator 429 (provider unknown) — skipping "
- "credential rotation, deferring to fallback chain"
- )
- return False, has_retried_429
- if effective_reason == FailoverReason.billing:
- # A separate pool instance may have resolved runtime credentials, leaving no ``current_id``;
- # match the key that failed, not a different account.
- return (True, False) if _rotate_and_swap(402, "billing") else (False, has_retried_429)
- if effective_reason == FailoverReason.rate_limit:
- return _recover_rate_limit(
- pool, has_retried_429=has_retried_429, error_context=error_context,
- api_key_hint=api_key_hint, credential_id=credential_id, rotate_and_swap=_rotate_and_swap,
- )
- if effective_reason == FailoverReason.model_entitlement:
- # The pool benches (credential, model) only and hands back the next entry that is not
- # benched for this model; None once every entry rejected it, so the caller falls
- # through to the single-credential handling in _mark_entitlement_rejected_model (#71970).
- return _rotate_and_swap(400, "model entitlement"), has_retried_429
- if effective_reason == FailoverReason.auth:
- return _recover_auth_failure(
- agent, pool, status_code=status_code, has_retried_429=has_retried_429,
- error_context=error_context, api_key_hint=api_key_hint, credential_id=credential_id,
- rotate_and_swap=_rotate_and_swap,
- )
- return False, has_retried_429
- def _apply_primary_runtime_fields(agent, rt: dict[str, Any]) -> None:
- """Copy the identity/transport fields of a ``_primary_runtime`` snapshot onto ``agent``
- (shared by transport recovery and turn-start restore; the caller rebuilds the client)."""
- agent.model = rt["model"]
- agent.provider = rt["provider"]
- agent.requested_provider = rt.get("requested_provider", agent.provider)
- agent.base_url = rt["base_url"] # setter updates _base_url_lower
- from hermes_cli.providers import is_actual_route
- agent.api_mode = "chat_completions" if is_actual_route(agent.provider, agent.base_url) else rt["api_mode"]
- if hasattr(agent, "_transport_cache"):
- agent._transport_cache.clear()
- agent.api_key = rt["api_key"]
- agent._reasoning_echo_flag = rt.get("reasoning_echo_flag", False)
- agent.request_overrides = dict(rt.get("request_overrides") or {})
- agent._client_kwargs = dict(rt["client_kwargs"])
- def _build_anthropic_client_from_runtime(agent, rt: dict[str, Any]) -> None:
- """Rebuild the native Anthropic client from a ``_primary_runtime`` snapshot."""
- from agent.anthropic_adapter import build_anthropic_client
- agent._anthropic_api_key = rt["anthropic_api_key"]
- agent._anthropic_base_url = rt["anthropic_base_url"]
- agent._anthropic_client = build_anthropic_client(
- rt["anthropic_api_key"], rt["anthropic_base_url"],
- timeout=get_provider_request_timeout(agent.provider, agent.model),
- )
- agent._is_anthropic_oauth = rt["is_anthropic_oauth"]
- agent.client = None
- def _rebuild_primary_client(agent, rt: dict[str, Any], *, reason: str) -> None:
- """Rebuild the primary client from a ``_primary_runtime`` snapshot (MoA facade / native Anthropic / OpenAI wire)."""
- if (agent.provider or "").strip().lower() == "moa":
- # MoA has empty client_kwargs; rebuild via the shared facade factory so the
- # reference_callback relay survives recovery.
- from agent.moa_loop import build_moa_facade
- agent.client = build_moa_facade(agent, agent.model)
- # MoA is a virtual chat-completions provider. It never has real OpenAI client kwargs; restoring it
- # after a fallback must recreate the facade, not call OpenAI() with an empty api_key. Use the shared
- # factory so the restored facade keeps the reference_callback relay wired at init — a bare
- # MoAClient() would silently stop emitting moa.reference/moa.aggregating display events (#53802).
- agent._anthropic_client = None
- elif agent.provider == "bedrock" and agent.api_mode in ("anthropic_messages", "bedrock_converse"):
- from agent.bedrock_adapter import bind_bedrock_runtime
- bind_bedrock_runtime(agent, agent.base_url, agent.api_mode)
- elif agent.api_mode == "anthropic_messages":
- _build_anthropic_client_from_runtime(agent, rt)
- else:
- agent.client = agent._create_openai_client(dict(rt["client_kwargs"]), reason=reason, shared=True)
- def try_recover_primary_transport(
- agent, api_error: Exception, *, retry_count: int, max_retries: int,
- ) -> bool:
- """Rebuild the primary client once and retry after ``max_retries`` exhaust on a transient
- transport error. Skipped for aggregators (OpenRouter, Nous) that manage retries server-side."""
- error_type = type(api_error).__name__
- if agent._fallback_activated or error_type not in _TRANSIENT_TRANSPORT_ERRORS or agent._is_openrouter_url():
- return False
- # Portal OpenAI-wire traffic rides aggregator retry infra (skip), but Portal Claude on native
- # Messages holds a local Anthropic client that needs the rebuild.
- if (
- (agent.provider or "").strip().lower() in {"nous", "nous-portal", "nousresearch"}
- and getattr(agent, "api_mode", None) != "anthropic_messages"
- ):
- return False
- try:
- # Never hard-close the shared client here: stale streaming workers may still be unwinding on
- # the old pool; _retire_shared_openai_client defers FD release to GC.
- # Retire the existing client to release stale connections. #70773: never hard-close the shared
- # client here — this runs on the conversation-loop thread while workers from stale-killed streaming
- # attempts may still be unwinding their SSL BIOs on the old pool. ``_retire_shared_openai_client``
- # shuts the sockets down (FD-safe from any thread) and defers the FD release to GC, which cannot
- # complete until every borrowing thread has unwound.
- if getattr(agent, "client", None) is not None:
- with contextlib.suppress(Exception):
- agent._retire_shared_openai_client(agent.client, reason="primary_recovery")
- rt = agent._primary_runtime
- _apply_primary_runtime_fields(agent, rt)
- _rebuild_primary_client(agent, rt, reason="primary_recovery")
- wait_time = min(3 + retry_count, 8)
- agent._vprint(
- f"{agent.log_prefix}🔁 Transient {error_type} on {agent.provider} — "
- f"rebuilt client, waiting {wait_time}s before one last primary attempt.", force=True, diagnostic=True,
- )
- time.sleep(wait_time)
- return True
- except Exception as e:
- logger.warning("Primary transport recovery failed: %s", e)
- return False
- def _merge_user_content(prev_content: Any, cur_content: Any) -> Any:
- """Merged content for two adjacent user messages (``_UNMERGEABLE`` for unknown shapes):
- string+string joins with a blank line; list sides append as separate blocks."""
- if isinstance(prev_content, str) and isinstance(cur_content, str):
- return prev_content + ("\n\n" if prev_content and cur_content else "") + cur_content
- if isinstance(prev_content, list) and isinstance(cur_content, list):
- return list(prev_content) + list(cur_content)
- if isinstance(prev_content, list) and isinstance(cur_content, str):
- return list(prev_content) + ([{"type": "text", "text": cur_content}] if cur_content else [])
- if isinstance(prev_content, str) and isinstance(cur_content, list):
- return ([{"type": "text", "text": prev_content}] if prev_content else []) + list(cur_content)
- return _UNMERGEABLE
- _UNMERGEABLE = object()
- def drop_thinking_only_and_merge_users(
- messages: list[dict[str, Any]], *, drop_codex_reasoning_items: bool = True,
- drop_nudge_marker: Optional[str] = None,
- ) -> list[dict[str, Any]]:
- """Drop thinking-only assistant turns and merge adjacent user messages left behind, on the
- per-call ``api_messages`` copy only (``agent.messages`` is never mutated). Drop-and-merge
- (not stub text) keeps history honest and preserves role alternation.
- ``drop_nudge_marker`` (#67321): user rows equal to the marker — the synthetic Codex
- continuation nudge — are dropped too once the turn has crossed to a non-Codex provider;
- doing it in this pass keeps alternation valid when the nudge sat between dropped
- reasoning-only interims and a tool result rather than next to the user's message."""
- if not messages:
- return messages
- kept = [
- m for m in messages
- if not (drop_nudge_marker is not None and m.get("role") == "user" and m.get("content") == drop_nudge_marker)
- and not _ra().AIAgent._is_thinking_only_assistant(m, drop_codex_reasoning_items=drop_codex_reasoning_items)
- ]
- dropped = len(messages) - len(kept)
- merged: list[dict[str, Any]] = []
- merges = 0
- for m in kept:
- prev = merged[-1] if merged else None
- content = _UNMERGEABLE
- if prev is not None and prev.get("role") == "user" and m.get("role") == "user":
- content = _merge_user_content(prev.get("content", ""), m.get("content", ""))
- if content is _UNMERGEABLE:
- # Not a user pair, or an unknown content shape: append separately (the latter violates
- # alternation, but is safer than raising in a hot path).
- merged.append(m)
- else:
- merged[-1] = {**prev, "content": content} # copy so caller dicts are never mutated
- merges += 1
- if dropped == 0 and merges == 0:
- return messages
- _ra().logger.debug(
- "Pre-call sanitizer: dropped %d thinking-only assistant turn(s), "
- "merged %d adjacent user message(s)", dropped, merges,
- )
- return merged
- def _primary_reset_gate_blocks(agent, rt, primary_provider, primary_runtime_base_url, matches_primary, load_primary_pool):
- """Reset-aware gate: skip a guaranteed-to-fail restore while the primary pool reports a
- future reset; fails open on any error/None. Returns ``(blocked, prefetched_pool, prefetched)``
- so the rebind step reuses the loaded pool (one auth.json read at most)."""
- prefetched_pool, prefetched = None, False
- try:
- pool = getattr(agent, "_credential_pool", None)
- if not matches_primary(pool):
- prefetched_pool = pool = load_primary_pool()
- prefetched = True
- primary_model = str(rt.get("model") or "").strip()
- next_at = getattr(pool, "next_available_at", lambda **_kwargs: None)(model=primary_model or None)
- if next_at is not None and next_at > time.time():
- if not getattr(agent, "_restore_wait_logged", False):
- agent._restore_wait_logged = True
- logger.info(
- "Primary %s rate-limited until %s; staying on fallback "
- "%s/%s until the reset elapses", primary_provider or "?",
- datetime.fromtimestamp(next_at).isoformat(timespec="seconds"), agent.provider,
- agent.model,
- )
- return True, prefetched_pool, prefetched
- except Exception:
- logger.debug("Reset-aware restore gate failed; falling back to per-turn retry", exc_info=True)
- return False, prefetched_pool, prefetched
- def _restore_runtime_capabilities(agent, rt: dict[str, Any]) -> None:
- # ``capabilities`` is the legacy key from the initial capability propagation patch.
- raw = rt["runtime_capabilities"] if "runtime_capabilities" in rt else rt.get("capabilities")
- if isinstance(raw, dict):
- agent.runtime_capabilities = dict(raw)
- elif "runtime_capabilities" in rt:
- logger.warning("Ignoring malformed runtime capabilities snapshot")
- def _rebind_primary_credential_pool(agent, primary_provider, primary_model, matches_primary, load_primary_pool, prefetched_pool, prefetched) -> None:
- """Rebind and re-select the primary credential pool after a fallback turn. A cross-provider
- fallback attaches its own pool, which would trip the provider-mismatch guard on the next
- 401/429: reload the primary pool, else clear it. The snapshot api_key may be stale after
- rotation; re-select the pool's best entry, keeping the snapshot key when none is usable."""
- pool = getattr(agent, "_credential_pool", None)
- pool_provider = str(getattr(pool, "provider", "") or "").strip().lower()
- if pool is not None and pool_provider and not matches_primary(pool):
- agent._credential_pool = None
- agent._credential_pool_entry_id = None
- try:
- # Reuse the pool the reset-aware gate already loaded (avoids a second auth.json read).
- agent._credential_pool = prefetched_pool if prefetched else load_primary_pool()
- except Exception as exc:
- logger.warning(
- "Restore could not reload primary credential pool for %s: %s", primary_provider, exc
- )
- agent._credential_pool_entry_id = None
- pool = getattr(agent, "_credential_pool", None)
- entry = pool.select(model=primary_model or None) if pool is not None and pool.has_available(model=primary_model or None) else None
- if entry is None or not (getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "")):
- return
- if matches_primary(entry):
- # _swap_credential rebuilds the client and reapplies base-url-scoped headers.
- # ``_swap_credential`` rebuilds the OpenAI/Anthropic client, reapplies base-url-scoped headers, and
- # carries the accumulated base_url / OAuth-detection fixes (#33163).
- agent._swap_credential(entry)
- logger.info(
- "Restore re-selected pool entry %s (%s)",
- getattr(entry, "id", "?"), getattr(entry, "label", "?"),
- )
- else:
- logger.info(
- "Restore skipped pool entry %s (%s): provider %s does not match primary provider %s",
- getattr(entry, "id", "?"), getattr(entry, "label", "?"),
- str(getattr(entry, "provider", "") or "").strip().lower() or "?",
- primary_provider or "?",
- )
- def _revert_credential_rotation(agent) -> None:
- """Move a live session back onto the credential a quota bench rotated it off, once the bench
- lifts. New sessions already do this through ``select()``; without it a long-lived (gateway)
- session keeps billing the fallback for its whole life (#114501). Credential-only: the
- model/base_url/compressor restore stays gated on ``_fallback_activated``."""
- revert_id = getattr(agent, "_credential_pool_revert_id", None)
- if not revert_id:
- return
- pool = getattr(agent, "_credential_pool", None)
- if pool is None or getattr(agent, "_credential_pool_entry_id", None) == revert_id:
- agent._credential_pool_revert_id = None
- return
- try:
- entry = pool.reclaim(revert_id, model=getattr(agent, "model", None))
- except Exception as exc:
- logger.warning("Credential revert check failed: %s", exc)
- return
- if entry is None:
- return # still cooling down; check again next turn
- if agent._swap_credential(entry) is not False:
- logger.info(
- "Credential %s (%s) available again — reverted pool rotation",
- getattr(entry, "id", "?"), getattr(entry, "label", "?"),
- )
- agent._credential_pool_revert_id = None
- def _primary_quota_reopened_early(agent, primary_provider, primary_model, matches_primary, load_primary_pool) -> bool:
- """True when a Codex quota window that the primary pool still has benched has reopened.
- A Codex 429 benches the entry, and arms ``_rate_limited_until``, until a ``resets_at`` that can be
- days out (weekly window). The window can reopen sooner (redeemed reset, top-up, plan change); the
- pool's throttled usage probe notices, but only on ``select()``, which a session pinned to its
- fallback never reaches. Fails closed: any doubt leaves both cooldowns in force.
- At most one check per probe interval per agent: the probe caches its verdict and nothing clears
- it on the next 429, so a "restored" answer the endpoint got wrong would otherwise restore,
- fail, and fall back again on every turn of the cached window. Checks in between could only
- replay that cached verdict, so they return before loading the pool.
- """
- if primary_provider != "openai-codex":
- return False
- from hermes_cli.auth_codex import CODEX_QUOTA_PROBE_MIN_INTERVAL_SECONDS
- now = time.monotonic()
- if now - getattr(agent, "_codex_reopen_checked_at", float("-inf")) < CODEX_QUOTA_PROBE_MIN_INTERVAL_SECONDS:
- return False
- agent._codex_reopen_checked_at = now
- try:
- pool = getattr(agent, "_credential_pool", None)
- if pool is None or not matches_primary(pool):
- pool = load_primary_pool()
- if pool is None:
- return False
- model = primary_model or None
- benched_until = pool.next_available_at(model=model)
- if benched_until is None or benched_until <= time.time():
- return False
- return pool.lift_reopened_cooldowns(model=model)
- except Exception:
- logger.debug("Early quota-reopen check failed; keeping the cooldown", exc_info=True)
- return False
- def restore_primary_runtime(agent) -> bool:
- """Restore the primary runtime at the start of a new turn so fallback stays turn-scoped
- (long-lived CLI agents and the gateway's cached agents)."""
- if not agent._fallback_activated:
- # Reset the index even without activation: a failed _try_activate_fallback() can strand
- # _fallback_index past the chain end and silently block future fallbacks (#20465).
- agent._fallback_index = 0
- _revert_credential_rotation(agent)
- return False
- rt = agent._primary_runtime
- primary_provider = str((rt or {}).get("provider") or "").strip().lower()
- primary_model = str((rt or {}).get("model") or "").strip()
- from agent.fallback_cooldown import _is_entitlement_rejected
- from hermes_cli.chat_catalog import is_known_non_chat_model
- if primary_model and (
- _is_entitlement_rejected(agent, primary_provider, primary_model)
- or is_known_non_chat_model(primary_model)
- ):
- # Unentitled (#106475) or already known non-chat: restoring would announce a recovery
- # that was never verified and re-fail every turn. Stay on the fallback.
- return False
- primary_runtime_base_url = str((rt or {}).get("base_url") or "")
- def _matches_primary(candidate) -> bool:
- return credential_pool_matches_provider(candidate, primary_provider, base_url=primary_runtime_base_url)
- def _load_primary_pool():
- """The primary's pool (entry-scoped for a custom entry); None when absent or provider-mismatched."""
- from agent.custom_entry_pool import load_runtime_pool
- loaded = load_runtime_pool(primary_provider, primary_runtime_base_url, (rt or {}).get("requested_provider"), rt or {})
- return loaded if loaded is not None and _matches_primary(loaded) else None
- if _primary_quota_reopened_early(agent, primary_provider, primary_model, _matches_primary, _load_primary_pool):
- agent._rate_limited_until = 0
- if getattr(agent, "_rate_limited_until", 0) > time.monotonic():
- return False # primary still in rate-limit cooldown, stay on fallback
- blocked, prefetched_pool, prefetched = _primary_reset_gate_blocks(
- agent, rt, primary_provider, primary_runtime_base_url, _matches_primary, _load_primary_pool
- )
- if blocked:
- return False
- agent._restore_wait_logged = False
- fallback_route = getattr(agent, "_provider_fallback_route", None)
- if not (isinstance(fallback_route, (list, tuple)) and len(fallback_route) == 2):
- fallback_route = (getattr(agent, "model", ""), getattr(agent, "provider", ""))
- previous_model, previous_provider = (str(v or "unknown") for v in fallback_route)
- provider_fallback_active = bool(getattr(agent, "_provider_fallback_active", False))
- try:
- from agent.route_binding import reinstall_primary_runtime
- reinstall_primary_runtime(
- agent, rt, primary_provider, primary_model, _matches_primary, _load_primary_pool, prefetched_pool, prefetched,
- )
- logger.info("Primary runtime restored for new turn: %s (%s)", agent.model, agent.provider)
- agent._provider_fallback_active = False
- agent._provider_fallback_route = None
- if provider_fallback_active:
- # Notification surfaces are best-effort and must never undo a successful restore.
- with contextlib.suppress(Exception):
- agent._emit_diagnostic_status(
- f"✅ Primary model restored: {agent.model} via {agent.provider}; "
- f"fallback {previous_model} via {previous_provider} is no longer active."
- )
- return True
- except Exception as e:
- logger.warning("Failed to restore primary runtime: %s", e)
- return False
- # Transient transport failures worth one more attempt with a rebuilt client / connection pool.
- _TRANSIENT_TRANSPORT_ERRORS = frozenset({
- "ReadTimeout", "ConnectTimeout", "PoolTimeout", "ConnectError", "ReadError", "RemoteProtocolError",
- "APIConnectionError", "APITimeoutError",
- })
- _INLINE_REASONING_PATTERNS = tuple(
- re.compile(rf"<{tag}>(.*?)</{tag}>", re.DOTALL | re.IGNORECASE)
- for tag in THINK_TAG_NAMES
- )
- def extract_reasoning(agent, assistant_message) -> Optional[str]:
- """Reasoning text from ``reasoning`` / ``reasoning_content`` / ``reasoning_details``
- (OpenRouter unified), else inline thinking blocks in the content; None when absent."""
- from agent.message_content import flatten_message_text
- parts: list[str] = []
- def _add(text) -> None:
- text = flatten_message_text(text, sep="")
- if text and text not in parts:
- parts.append(text)
- _add(getattr(assistant_message, "reasoning", None))
- _add(getattr(assistant_message, "reasoning_content", None))
- # reasoning_details: [{"type": "reasoning.summary", "summary": "...", ...}, ...]
- for detail in getattr(assistant_message, "reasoning_details", None) or []:
- if isinstance(detail, dict):
- _add(detail.get('summary') or detail.get('thinking') or detail.get('content') or detail.get('text'))
- # Fall back to reasoning embedded in content only when no structured field was found.
- content = getattr(assistant_message, "content", None)
- if not parts and isinstance(content, list):
- # DeepSeek V4 Pro returns typed content blocks ({"type": "thinking", ...}); dropping them
- # makes the next turn fail with HTTP 400 "thinking must be passed back".
- # Refs #21944.
- for block in content:
- if isinstance(block, dict) and block.get("type") == "thinking":
- # Non-strict OpenAI-compatible backends (Mistral via custom provider)
- # deliver the thinking value as a JSON array, not a string (#106006);
- # flatten first so .strip() never sees a list.
- _add(flatten_message_text(block.get("thinking") or block.get("text") or "", sep="").strip())
- if not parts and isinstance(content, str) and content:
- for pattern in _INLINE_REASONING_PATTERNS:
- for block in pattern.findall(content):
- _add(block.strip())
- return "\n\n".join(parts) if parts else None
- def _api_error_debug_info(error: Exception) -> dict[str, Any]:
- info: dict[str, Any] = {"type": type(error).__name__, "message": str(error)}
- info.update({
- k: v for k in ("status_code", "request_id", "code", "param", "type", "body")
- if (v := getattr(error, k, None)) is not None
- })
- response_obj = getattr(error, "response", None)
- if response_obj is not None:
- try:
- info["response_status"] = getattr(response_obj, "status_code", None)
- info["response_text"] = response_obj.text
- except Exception as e:
- _ra().logger.debug("Could not extract error response details: %s", e)
- return info
- def dump_api_request_debug(
- agent, api_kwargs: dict[str, Any], *, reason: str, error: Optional[Exception] = None
- ) -> Optional[Path]:
- """Dump the request body from api_kwargs (minus transport keys) for debugging provider 4xx failures."""
- try:
- body = {k: v for k, v in copy.deepcopy(api_kwargs).items() if v is not None and k != "timeout"}
- api_key = None
- # anthropic_messages keeps its SDK client on ``_anthropic_client`` (``client`` is None):
- # read the key from there so the dump does not say "Bearer None" (#24293).
- anthropic = agent.api_mode == "anthropic_messages"
- try:
- live = getattr(agent, "_anthropic_client", None) if anthropic else agent.client
- api_key = getattr(live, "api_key", None) or getattr(live, "auth_token", None)
- except Exception as e:
- _ra().logger.debug("Could not extract API key for debug dump: %s", e)
- endpoint = {"codex_responses": "/responses", "anthropic_messages": "/messages"}.get(
- agent.api_mode, "/chat/completions"
- )
- dump_payload: dict[str, Any] = {
- "timestamp": datetime.now().isoformat(), "session_id": agent.session_id, "reason": reason,
- "request": {
- "method": "POST", "url": f"{agent.base_url.rstrip('/')}{endpoint}",
- "headers": {
- "Authorization": f"Bearer {agent._mask_api_key_for_logs(api_key)}",
- "Content-Type": "application/json",
- },
- "body": body,
- },
- }
- if error is not None:
- dump_payload["error"] = _api_error_debug_info(error)
- # Sanitize the session ID (may come from an untrusted X-Hermes-Session-Id header) so a
- # "../"-shaped ID cannot write outside logs_dir.
- from agent.session_persistence import _safe_session_filename_component
- safe_sid = _safe_session_filename_component(agent.session_id)
- dump_file = agent.logs_dir / f"request_dump_{safe_sid}_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json"
- # Redact secrets first: this fires unconditionally on API errors and captures the full
- # request body, so context-embedded secrets would otherwise land in cleartext on disk.
- from agent.redact import redact_sensitive_text
- _serialized = json.dumps(dump_payload, ensure_ascii=False, indent=2, default=str)
- _redacted_payload = json.loads(redact_sensitive_text(_serialized, force=True))
- atomic_json_write(dump_file, _redacted_payload, default=str)
- agent._vprint(f"{agent.log_prefix}🧾 Request debug dump written to: {dump_file}")
- if env_var_enabled("HERMES_DUMP_REQUEST_STDOUT"):
- print(json.dumps(_redacted_payload, ensure_ascii=False, indent=2, default=str))
- return dump_file
- except Exception as dump_error:
- if agent.verbose_logging:
- logger.warning("Failed to dump API request debug payload: %s", dump_error)
- return None
- def _direct_native_anthropic_tool_cache_capability(
- agent, *, provider: Optional[str] = None, base_url: Optional[str] = None,
- api_mode: Optional[str] = None, model: Optional[str] = None,
- ) -> bool:
- """Return whether this resolved destination accepts native tool markers."""
- eff_base_url = base_url if base_url is not None else (agent.base_url or "")
- eff_api_mode = api_mode if api_mode is not None else (agent.api_mode or "")
- return eff_api_mode == "anthropic_messages" and base_url_hostname(eff_base_url) == "api.anthropic.com"
- # The cache_ttl tiers accepted by config; mirrored by agent_init's live-agent snapshot.
- VALID_CACHE_TTLS = ("5m", "1h")
- def cache_ttl_means_disabled(ttl: Any) -> bool:
- """True when a ``prompt_caching.cache_ttl`` value means caching off (single predicate shared
- by ``agent_init`` and the stub policy paths). Unknown values (``"2h"``, ints) are NOT a disable."""
- if ttl in VALID_CACHE_TTLS:
- return False
- return ttl is False or ttl is None or str(ttl).lower() in ("off", "false", "disabled", "no", "none")
- def _raw_cache_ttl_from_config(default: Any) -> Any:
- """Raw ``prompt_caching.cache_ttl`` config value, or ``default`` when config cannot be read."""
- try:
- from hermes_cli.config import load_config_readonly
- return (load_config_readonly().get("prompt_caching", {}) or {}).get("cache_ttl", "5m")
- except Exception:
- return default
- def prompt_caching_disabled_from_config() -> bool:
- """True when ``prompt_caching.cache_ttl`` is configured as off (same detection as ``agent_init``).
- Same disable detection as ``agent_init`` (via ``cache_ttl_means_disabled``) so stub-based policy paths
- (MoA slot decoration, auxiliary fallback replan) honor the same config contract without holding a live
- ``AIAgent`` (#76085 / #33555).
- """
- return cache_ttl_means_disabled(_raw_cache_ttl_from_config("5m"))
- def configured_cache_ttl() -> Optional[str]:
- """Configured ``prompt_caching.cache_ttl`` tier (``5m``/``1h``), else None; mirrors
- ``agent_init`` so stub paths don't regress a configured ``1h`` to 5m. ``auto`` is None here
- on purpose: stub/auxiliary calls are machine-paced, so they take the 5m tier ``None`` resolves to."""
- ttl = _raw_cache_ttl_from_config(None)
- return ttl if ttl in VALID_CACHE_TTLS else None
- def blank_cache_policy_stub(cache_disabled: Optional[bool] = None):
- """Destination-identity-blank stub for ``anthropic_prompt_cache_policy``; the sole sanctioned
- constructor so ``_cache_disabled`` is never omitted (None consults the global config)."""
- from types import SimpleNamespace
- if cache_disabled is None:
- cache_disabled = prompt_caching_disabled_from_config()
- return SimpleNamespace(provider="", base_url="", api_mode="", model="", _cache_disabled=bool(cache_disabled))
- def plan_cache_sections_for_destination(
- messages: list, tools: Optional[list], *, provider: str, base_url: str, api_mode: str,
- model: str, cache_disabled: Optional[bool] = None, cache_ttl: Optional[str] = None,
- static_system_prefix: Optional[str] = None,
- ) -> tuple[list, list]:
- """Plan request-local cache sections for one resolved destination (MoA / auxiliary senders):
- stripped copies (non-caching route) or a ``build_prompt_cache_plan`` layout; never mutates
- inputs. ``cache_disabled``/``cache_ttl`` default to live config so the operator's disable and
- tier are honored; ``static_system_prefix`` gives the system prompt the main loop's early breakpoint.
- ``cache_disabled`` threads the operator's ``prompt_caching.cache_ttl`` disable into the blank policy
- stub. When omitted, the live config is consulted so MoA/auxiliary paths cannot re-enable markers after
- the user turned caching off (#76085).
- """
- from agent.prompt_caching import (
- build_prompt_cache_plan, effective_cache_ttl, envelope_tool_part_cache_markers_supported,
- strip_anthropic_cache_control, strip_anthropic_tool_cache_control,
- )
- # The policy function reads agent.* only as fallbacks for kwargs we don't pass; blank_cache_policy_stub
- # is the only sanctioned stub so _cache_disabled cannot be left off again (#76085).
- stub = blank_cache_policy_stub(cache_disabled)
- dest = dict(provider=provider, base_url=base_url, api_mode=api_mode, model=model)
- should_cache, native_layout = anthropic_prompt_cache_policy(stub, **dest)
- if not should_cache:
- canonical_messages = copy.deepcopy(messages or [])
- strip_anthropic_cache_control(canonical_messages)
- return canonical_messages, strip_anthropic_tool_cache_control(tools)
- plan = build_prompt_cache_plan(
- messages, tools,
- # effective_cache_ttl resolves None → "5m"; cache-disabled agents never reach here.
- cache_ttl=effective_cache_ttl(cache_ttl, provider=provider, model=model),
- native_anthropic=native_layout,
- static_system_prefix=static_system_prefix if isinstance(static_system_prefix, str) else None,
- direct_native_tool_cache=_direct_native_anthropic_tool_cache_capability(stub, **dest),
- # LiteLLM-style envelope routes forward part-level markers into tool_result.content[] →
- # non-retryable 400.
- tool_part_markers=envelope_tool_part_cache_markers_supported(provider, base_url),
- )
- return plan.messages, plan.tools
- def _is_litellm_route(provider_lower: str, base_url: str) -> bool:
- """True when a route is a LiteLLM proxy: ``litellm`` as a whole delimited token (not
- substring) in the provider id or host; a path segment never qualifies."""
- return _has_litellm_token(provider_lower, ":-_/") or _has_litellm_token(base_url_hostname(base_url), ".-")
- def _has_litellm_token(value: str, delimiters: str) -> bool:
- """True when ``value`` contains ``litellm`` as a whole delimited token."""
- if not value:
- return False
- return "litellm" in value.translate(str.maketrans(delimiters, " " * len(delimiters))).split()
- def _moa_aggregator_cache_policy(agent, eff_model: str) -> tuple[bool, bool]:
- """MoA virtual provider: resolve the policy from the preset's real aggregator slot (the
- virtual provider matches no caching branch and would silently lose caching)."""
- try:
- from hermes_cli.config import load_config as _load_moa_cfg
- from hermes_cli.moa_config import resolve_moa_preset
- from hermes_cli.runtime_provider import resolve_runtime_provider
- agg = resolve_moa_preset(_load_moa_cfg().get("moa") or {}, eff_model or None).get("aggregator") or {}
- agg_provider = str(agg.get("provider") or "").strip()
- agg_model = str(agg.get("model") or "").strip()
- if agg_provider and agg_model:
- agg_base_url = agg_api_mode = ""
- with contextlib.suppress(Exception):
- rt = resolve_runtime_provider(requested=agg_provider, target_model=agg_model)
- agg_base_url = rt.get("base_url") or ""
- agg_api_mode = rt.get("api_mode") or ""
- return anthropic_prompt_cache_policy(
- agent, provider=agg_provider, base_url=agg_base_url, api_mode=agg_api_mode, model=agg_model
- )
- except Exception as _moa_exc: # pragma: no cover - defensive
- logger.debug("MoA aggregator cache-policy resolution failed: %s", _moa_exc)
- return False, False
- def _route_may_be_custom(agent, eff_provider: str, provider_lower: str, eff_base_url: str) -> bool:
- """Cheap identity gate deciding whether a custom-provider capability lookup is worth running."""
- custom_providers = getattr(agent, "_custom_providers", None)
- if custom_providers:
- # Same semantics as the capability helper (normalize_route_base_url +
- # custom_provider_aliases) so spelling differences don't drop declarations.
- from hermes_cli.providers import custom_provider_aliases
- from hermes_cli.route_identity import normalize_route_base_url
- provider_ids = {provider_lower, provider_lower.removeprefix("custom:")}
- eff_url_normalized = normalize_route_base_url(eff_base_url)
- return any(
- provider_ids & custom_provider_aliases(str(entry.get("name") or ""), str(entry.get("provider_key") or ""))
- or (eff_url_normalized and normalize_route_base_url(entry.get("base_url")) == eff_url_normalized)
- for entry in custom_providers if isinstance(entry, dict)
- )
- if custom_providers is not None:
- return False # attached empty list never matches
- # None = list not attached yet (early init or blank stub). Avoid rebuilding the list for
- # ordinary built-in routes.
- try:
- from hermes_cli.providers import get_provider
- # allow_network=False: never trigger a registry fetch from the send path; a catalog miss
- # degrades to the conservative capability lookup.
- provider_def = get_provider(eff_provider, allow_network=False)
- return provider_def is None or (
- bool(provider_def.base_url)
- and base_url_hostname(provider_def.base_url) != base_url_hostname(eff_base_url)
- )
- except Exception as _pd_exc:
- logger.debug("provider lookup failed during cache-policy pre-gate: %s", _pd_exc)
- return provider_lower.startswith("custom:")
- def anthropic_prompt_cache_policy(
- agent, *, provider: Optional[str] = None, base_url: Optional[str] = None,
- api_mode: Optional[str] = None, model: Optional[str] = None,
- ) -> tuple[bool, bool]:
- """Decide whether to apply Anthropic prompt caching; returns ``(should_cache, use_native_layout)``.
- Native layout puts markers on inner content blocks (Anthropic wire), else on the message
- envelope (OpenRouter / OpenAI-wire proxies; Qwen/Alibaba too). The operator disable is read
- from ``_cache_disabled`` (not ``_cache_ttl``, unset during init) so it survives switches
- and restores. Branch ORDER is load-bearing (see inline notes).
- Qwen / Alibaba-family models on OpenCode, OpenCode Go, and direct Alibaba (DashScope) also honour
- Anthropic-style ``cache_control`` markers on OpenAI-wire chat completions. Upstream pi-mono #3392 / pi
- #3393 documented this for opencode-go Qwen. Without markers these providers serve zero cache hits,
- re-billing the full prompt on every turn.
- """
- if getattr(agent, "_cache_disabled", False):
- return (False, False)
- eff_provider = (provider if provider is not None else agent.provider) or ""
- eff_base_url = base_url if base_url is not None else (agent.base_url or "")
- eff_api_mode = api_mode if api_mode is not None else (agent.api_mode or "")
- eff_model = (model if model is not None else agent.model) or ""
- if eff_provider.strip().lower() == "moa":
- return _moa_aggregator_cache_policy(agent, eff_model)
- if isinstance(eff_model, dict):
- eff_model = eff_model.get('model') or eff_model.get('default') or ''
- eff_model = eff_model if isinstance(eff_model, str) else str(eff_model or '')
- model_lower = eff_model.lower()
- provider_lower = eff_provider.lower()
- is_claude = "claude" in model_lower
- # Kimi/Moonshot via OpenRouter uses the same envelope cache_control as Claude; without this it
- # serves ~1% cache hits. Family matcher covers bare k1./k2. slugs.
- # Without this branch moonshotai/kimi-k2.6 falls through to (False, False), serving ~1% cache hits on
- # 64K-token prompts and re-billing the full prompt on every turn. Observed within-turn progression with
- # cache enabled: 1% → 67% → 84% → 97% (#25970). Reuses the canonical family matcher (covers bare
- # k1./k2./k25 release slugs the substring check missed).
- from agent.anthropic_endpoints import _model_name_is_kimi_family
- is_kimi = _model_name_is_kimi_family(eff_model) or "moonshot" in model_lower
- is_openrouter = base_url_host_matches(eff_base_url, "openrouter.ai")
- # Nous Portal proxies to OpenRouter; treat as OpenRouter-equivalent for cache layout.
- is_nous_portal = base_url_host_matches(eff_base_url, "nousresearch.com")
- is_anthropic_wire = eff_api_mode == "anthropic_messages"
- is_native_anthropic = is_anthropic_wire and (
- eff_provider == "anthropic" or base_url_hostname(eff_base_url) == "api.anthropic.com"
- )
- # Honor a configured route's per-model ``prompt_caching`` capability (explicit false too); only
- # for the two transports this planner handles, not Responses/Bedrock.
- supports_cache_markers = eff_api_mode in {"anthropic_messages", "chat_completions"}
- litellm_openai_wire = (
- eff_api_mode == "chat_completions" and is_claude and _is_litellm_route(provider_lower, eff_base_url)
- )
- if supports_cache_markers and (
- is_anthropic_wire
- or litellm_openai_wire
- or _route_may_be_custom(agent, eff_provider, provider_lower, eff_base_url)
- ):
- try:
- from hermes_cli.config import get_custom_provider_model_capability
- custom_prompt_caching = get_custom_provider_model_capability(
- model=eff_model, base_url=eff_base_url, capability="prompt_caching",
- custom_providers=getattr(agent, "_custom_providers", None),
- )
- if custom_prompt_caching is not None:
- # Layout follows the transport: native Messages → inner blocks; OpenAI wire → envelope.
- return custom_prompt_caching, custom_prompt_caching and is_anthropic_wire
- except Exception as _cap_exc:
- logger.debug("custom-provider prompt_caching capability lookup failed: %s", _cap_exc)
- # MiniMax-M3 uses server-side automatic prefix caching; explicit markers are dead weight.
- # Checked BEFORE the native-Anthropic return since provider="anthropic" may point at a MiniMax
- # proxy.
- is_minimax_route = (
- provider_lower in {"minimax", "minimax-cn"}
- or base_url_host_matches(eff_base_url, "api.minimax.io")
- or base_url_host_matches(eff_base_url, "api.minimaxi.com")
- )
- if is_anthropic_wire and is_minimax_route:
- from agent.model_metadata import _model_name_suggests_minimax_m3
- if _model_name_suggests_minimax_m3(eff_model):
- return False, False
- if is_native_anthropic:
- return True, True
- # Envelope layout is OpenAI-wire only; Portal Claude on native Messages must fall through to the
- # anthropic_messages branch (inner-block markers) or it serves 0% cache hits.
- if (is_openrouter or is_nous_portal) and (is_claude or is_kimi) and not is_anthropic_wire:
- return True, False
- # Nous Portal Qwen takes the envelope path too; the alibaba-family check below only matches
- # provider=opencode/alibaba and would leave Portal traffic uncached.
- if is_nous_portal and "qwen" in model_lower:
- return True, False
- if is_anthropic_wire and is_claude:
- return True, True # third-party Anthropic-compatible gateway
- # LiteLLM fronting Claude on the OpenAI wire supports cache_control but matched no grant above.
- # Claude-only: strict relays reject the block format for other models. Envelope layout: native
- # top-level markers are only relocated by the anthropic_messages adapter and 400 via LiteLLM.
- # Gated on chat_completions; codex_responses/bedrock_converse have their own handling.
- if litellm_openai_wire:
- return True, False
- # MiniMax's own models (M2.x) on its Anthropic-compatible endpoint support cache_control; opt
- # them in past the is_claude gate. M3 is excluded above.
- if is_anthropic_wire and is_minimax_route:
- return True, True
- # Qwen/Alibaba on OpenCode and DashScope accept envelope cache_control on the OpenAI wire
- # (pi-mono's "alibaba" cacheControlFormat). DeepSeek on OpenCode is excluded: its relay 400s on
- # block-array content. Family set/predicate shared with the effective_cache_ttl clamp.
- # Qwen/Alibaba on OpenCode (Zen/Go) and native DashScope: OpenAI-wire transport that accepts
- # Anthropic-style cache_control markers and rewards them with real cache hits. Without this branch
- # qwen3.6-plus on opencode-go reports 0% cached tokens and burns through the subscription on every turn.
- # OpenCode Zen's relay rejects the Anthropic-style content block format that cache markers produce
- # (content becomes a block array instead of a plain string), causing HTTP 400 (#77217).
- from agent.prompt_caching import ALIBABA_FAMILY_PROVIDERS, is_qwen_model
- if provider_lower in ALIBABA_FAMILY_PROVIDERS and is_qwen_model(model_lower):
- return True, False
- return False, False
- def _provider_supplied_client(agent, client_kwargs: dict) -> Any | None:
- """Ask the registered ProviderProfile for a custom client, if any. Resolves by provider name,
- then by ``base_url`` prefix so a URL-only runtime (``acp://…``) still reaches its profile.
- A profile that raises is logged and skipped: a third-party plugin must not be able to take
- the turn down, it can only fail to provide a client."""
- try:
- from providers import get_provider_profile
- except Exception:
- return None
- profile = None
- provider_name = (getattr(agent, "provider", "") or "").strip()
- if provider_name:
- try:
- profile = get_provider_profile(provider_name)
- except Exception:
- profile = None
- if profile is None:
- base_url = str(client_kwargs.get("base_url", "") or "").strip()
- if base_url:
- profile = _profile_for_base_url(base_url)
- if profile is None:
- return None
- try:
- return profile.create_client(**client_kwargs)
- except Exception:
- _ra().logger.warning(
- "Provider profile %r failed to create a client; falling back to the standard client path",
- getattr(profile, "name", provider_name) or "?", exc_info=True,
- )
- return None
- def _profile_for_base_url(base_url: str) -> Any | None:
- """Registered profile whose own base_url is a prefix of ``base_url`` (provider name did not
- resolve). Prefix, not equality: an ACP profile's ``acp://<vendor>`` root must also resolve a
- path or user override under the same root."""
- try:
- from providers import list_providers
- candidates = list_providers()
- except Exception:
- return None
- target = base_url.rstrip("/").lower()
- for candidate in candidates or []:
- own = str(getattr(candidate, "base_url", "") or "").rstrip("/").lower()
- if own and (target == own or target.startswith(own + "/")):
- return candidate
- return None
- def _ensure_route_client_headers(client_kwargs: dict, provider: str) -> None:
- """Defense-in-depth: recovery/restore rebuild from a snapshot without re-running header
- wiring, and a route whose provider requires its client headers (integration ids) 400s
- without them. Only ADD missing keys, never override."""
- from agent.provider_route_hooks import route_answer
- required = route_answer("client_headers", provider=provider, base_url=str(client_kwargs.get("base_url", "")),
- api_key=str(client_kwargs.get("api_key", "") or ""))
- if not required:
- return
- existing = dict(client_kwargs.get("default_headers") or {})
- existing_lower = {k.lower() for k in existing}
- for hk, hv in required.items():
- if hk.lower() not in existing_lower:
- existing[hk] = hv
- client_kwargs["default_headers"] = existing
- def _gemini_native_client(agent, client_kwargs: dict, httpx_verify, *, reason: str, shared: bool):
- """Native Gemini client when the base_url is the Gemini API, else None."""
- from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url
- base_url = str(client_kwargs.get("base_url", "") or "")
- if not is_native_gemini_base_url(base_url):
- return None
- safe_kwargs = {
- k: v for k, v in client_kwargs.items()
- if k in {"api_key", "base_url", "default_headers", "timeout", "http_client"}
- }
- if "http_client" not in safe_kwargs:
- keepalive_http = agent._build_keepalive_http_client(base_url, verify=httpx_verify)
- if keepalive_http is not None:
- safe_kwargs["http_client"] = keepalive_http
- client = GeminiNativeClient(**safe_kwargs)
- _ra().logger.info(
- "Gemini native client created (%s, shared=%s) %s", reason, shared, agent._client_log_context()
- )
- return client
- def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: bool) -> Any:
- from agent.auxiliary_client import (
- _to_openai_base_url,
- _validate_base_url,
- _validate_proxy_env_urls,
- )
- from agent.ssl_verify import resolve_httpx_verify
- # Treat client_kwargs as read-only: callers pass agent._client_kwargs, and in-place mutation
- # leaks into later requests (a torn-down httpx transport got reused).
- # Callers pass agent._client_kwargs (or shallow copies of it) in; any in-place mutation leaks back into
- # the stored dict and is reused on subsequent requests. #10933 hit this by injecting an httpx.Client
- # transport that was torn down after the first request, so the next request wrapped a closed transport
- # and raised "Cannot send a request, as the client has been closed" on every retry. The revert resolved
- # that specific path; this copy locks the contract so future transport/keepalive work can't reintroduce
- # the same class of bug.
- client_kwargs = dict(client_kwargs)
- if client_kwargs.get("base_url"):
- client_kwargs["base_url"] = _to_openai_base_url(client_kwargs["base_url"])
- try:
- from providers import get_provider_profile
- profile = get_provider_profile(getattr(agent, "provider", ""))
- if profile is not None:
- for key, value in profile.build_client_kwargs_extras(
- base_url=client_kwargs.get("base_url", "")
- ).items():
- client_kwargs.setdefault(key, value)
- except Exception:
- _ra().logger.debug("Provider client-kwargs hook skipped", exc_info=True)
- # The MoA virtual provider has no OpenAI wire endpoint; the facade *is* the client. Rebuild the
- # facade, never a native client (TypeError; relay re-wire).
- # Rebuilding a native OpenAI client while agent.provider == "moa" (client replacement, stream-retry pool
- # cleanup, credential rotation, fallback+restore) drops the facade: the next primary call either raises
- # a `_moa_prepared_request` TypeError (#78382) or, when _client_kwargs carry an unrelated relay
- # base_url, leaks the request to a foreign gateway. Rebuild the facade instead (build_moa_facade also
- # re-wires the reference relay, see #53802).
- if (getattr(agent, "provider", "") or "").strip().lower() == "moa":
- from agent.moa_loop import build_moa_facade
- return build_moa_facade(agent, getattr(agent, "model", None) or "default")
- ssl_ca_cert = client_kwargs.pop("ssl_ca_cert", None)
- ssl_verify_cfg = client_kwargs.pop("ssl_verify", None)
- httpx_verify = resolve_httpx_verify(
- ca_bundle=ssl_ca_cert, ssl_verify=ssl_verify_cfg,
- base_url=str(client_kwargs.get("base_url", "")),
- )
- _validate_proxy_env_urls()
- _validate_base_url(client_kwargs.get("base_url"))
- # Provider-supplied client (registration seam): a provider whose wire protocol is not
- # OpenAI-over-HTTP supplies its own client from ProviderProfile.create_client(). Consulted
- # before the built-in ladder so a profile registered from ~/.hermes/plugins/ or a pip entry
- # point can ship a transport without editing this function (what makes an out-of-tree ACP
- # provider possible). None (the default) falls through, so existing providers are unaffected.
- provider_client = _provider_supplied_client(agent, client_kwargs)
- if provider_client is not None:
- _ra().logger.info(
- "%s client created from provider profile (%s, shared=%s) %s",
- agent.provider, reason, shared, agent._client_log_context(),
- )
- return provider_client
- from agent.auxiliary_client import _GEMINI_NATIVE_PROVIDER_NAMES
- if agent.provider in _GEMINI_NATIVE_PROVIDER_NAMES:
- client = _gemini_native_client(agent, client_kwargs, httpx_verify, reason=reason, shared=shared)
- if client is not None:
- return client
- # TCP keepalives so dead provider connections are detected (~60s) instead of hanging in
- # CLOSE-WAIT. Injected into the local copy only, so each client gets its own httpx.Client;
- # pinned by tests/agent/test_create_openai_client_reuse.py. What IS shared across those per-client wrappers is the
- # connection pool: ``build_keepalive_http_client`` mounts a process-shared ``HTTPTransport``
- # behind a per-client view whose ``close()`` is a no-op for the pool, so a closed wrapper
- # never takes a sibling's (or the successor's) connections with it
- # (tests/agent/test_shared_http_transport.py).
- # Without this, a peer that drops mid-stream leaves the socket in a state where epoll_wait never fires,
- # ``httpx`` read timeout may not trigger, and the agent hangs until manually killed. Probes after 30s
- # idle, retry every 10s, give up after 3 → dead peer detected within ~60s. Safety against #10933: the
- # ``client_kwargs = dict(client_kwargs)`` above means this injection only lands in the local per-call
- # copy, never back into ``agent._client_kwargs``. Each ``_create_openai_client`` invocation therefore
- # gets its OWN fresh ``httpx.Client`` whose lifetime is tied to the OpenAI client it is passed to. When
- # the OpenAI client is closed (rebuild, teardown, credential rotation), the paired ``httpx.Client``
- # closes with it, and the next call constructs a fresh one — no stale closed transport can be reused.
- # Bedrock Mantle: the ``aws-sdk`` placeholder is a sentinel for IAM-chain auth, not a bearer token.
- # Every rebuild from bare ``{api_key, base_url}`` kwargs (switch_model, fallback restore, credential
- # rotation, request-scoped clients) must reinstall the SigV4 http_client or Mantle answers 401.
- if "bedrock-mantle." in str(client_kwargs.get("base_url") or ""):
- from agent.bedrock_adapter import configure_bedrock_openai_client_kwargs
- timeout = client_kwargs.get("timeout")
- configure_bedrock_openai_client_kwargs(
- client_kwargs, timeout=timeout if isinstance(timeout, (int, float)) else None,
- )
- if "http_client" not in client_kwargs:
- keepalive_http = agent._build_keepalive_http_client(client_kwargs.get("base_url", ""), verify=httpx_verify)
- if keepalive_http is not None:
- client_kwargs["http_client"] = keepalive_http
- # Retries belong to the outer conversation loop (honors Retry-After); SDK retries would
- # double-retry inside it. auxiliary_client keeps SDK retries as it isn't wrapped.
- # Delegate all rate-limit / 5xx retry to hermes's outer conversation loop, which honors Retry-After and
- # applies adaptive/jittered backoff. The OpenAI SDK default (max_retries=2) uses its own 1-2s backoff
- # that ignores Retry-After and double-retries inside our loop — the same deadlock the Anthropic clients
- # hit (#26293). This is the single chokepoint every primary OpenAI/aggregator client passes through
- # (init, switch_model, recovery, restore, request-scoped); auxiliary_client builds its own clients and
- # keeps SDK retries because it is NOT wrapped by the conversation loop.
- client_kwargs.setdefault("max_retries", 0)
- _ensure_route_client_headers(client_kwargs, str(getattr(agent, "provider", "") or ""))
- # All primary construction and recovery paths must identify Hermes to the official Codex
- # endpoint, including snapshots with custom header overrides.
- from agent.codex_headers import apply_required_codex_headers
- apply_required_codex_headers(
- client_kwargs, access_token=client_kwargs.get("api_key", ""),
- base_url=str(client_kwargs.get("base_url", "")),
- )
- # ``process_bootstrap.OpenAI`` is a lazy SDK proxy; resolved at call time so tests can patch it.
- from agent import process_bootstrap
- client = process_bootstrap.OpenAI(**client_kwargs)
- # Routing proxies name the deployment they served in a response header (#54864).
- from agent.served_model import install_served_model_capture
- install_served_model_capture(agent, client)
- _ra().logger.info("OpenAI client created (%s, shared=%s) %s", reason, shared, agent._client_log_context())
- return client
- def _apply_switched_provider_request_overrides(agent, new_provider):
- """Re-derive the switched-to provider's ``request_overrides`` (custom_providers ``extra_body``).
- Matches by provider key, base_url AND model (same rule as
- ``agent_init._merge_custom_provider_extra_body``) so a different model at the same endpoint
- never inherits another's ``extra_body``. Stale ``extra_body`` cleared; ``service_tier``/``speed`` kept."""
- from agent.agent_init import _custom_provider_extra_body_for_agent
- # Prefer the init-time cache (agent._custom_providers); reload only if absent.
- custom_providers = getattr(agent, "_custom_providers", None)
- if custom_providers is None:
- try:
- from hermes_cli.config import load_config, get_compatible_custom_providers
- custom_providers = get_compatible_custom_providers(load_config())
- except Exception:
- custom_providers = []
- new_extra_body = _custom_provider_extra_body_for_agent(
- provider=new_provider, model=getattr(agent, "model", "") or "",
- base_url=getattr(agent, "base_url", "") or "", custom_providers=custom_providers or [],
- )
- overrides = dict(getattr(agent, "request_overrides", {}) or {})
- overrides.pop("extra_body", None) # always drop the previous provider's extra_body
- if new_extra_body:
- overrides["extra_body"] = dict(new_extra_body)
- agent.request_overrides = overrides
- # Pool reload is part of the switch and must be reversible on rollback, hence the pool fields.
- _SWITCH_SNAPSHOT_FIELDS = (
- "model", "provider", "requested_provider", "base_url", "api_mode", "api_key", "client",
- "_anthropic_client", "_anthropic_api_key", "_anthropic_base_url", "_is_anthropic_oauth",
- "_config_context_length", "_reasoning_echo_flag", "runtime_capabilities",
- "_credential_pool", "_credential_pool_entry_id",
- "_codex_reasoning_replay_enabled", "_codex_reasoning_replay_rejected",
- )
- _MISSING = object()
- def _snapshot_switch_state(agent) -> dict[str, Any]:
- """Snapshot every field the swap+rebuild mutates so a failed rebuild rolls back atomically
- (else a new model name + OLD client 400s next turn). The sentinel distinguishes unset from
- None: tests build bare agents via ``__new__`` without all fields."""
- snapshot = {name: getattr(agent, name, _MISSING) for name in _SWITCH_SNAPSHOT_FIELDS}
- # Shallow-copy the dict so mutating the live one doesn't poison the rollback target.
- snapshot["_client_kwargs"] = dict(getattr(agent, "_client_kwargs", {}) or {})
- return snapshot
- def _restore_switch_snapshot(agent, snapshot: dict[str, Any]) -> None:
- for name, value in snapshot.items():
- if value is _MISSING:
- continue # attribute did not exist before the swap; don't fabricate it
- with contextlib.suppress(Exception):
- setattr(agent, name, value)
- def _resolve_switch_destination(agent, new_model, new_provider, base_url, api_mode, capabilities, old_norm, new_norm):
- """Resolve ``(api_mode, base_url, destination_capabilities)`` for the switch target."""
- from hermes_cli.providers import determine_api_mode, is_actual_route
- from agent.native_compaction import resolve_native_compaction_capabilities
- from hermes_cli.models import opencode_provider_family
- # Pass model so dual-wire providers (Nous Portal anthropic/* -> Messages) resolve correctly.
- if not api_mode:
- api_mode = determine_api_mode(new_provider, base_url, model=new_model)
- if not base_url and new_norm == "openai":
- # An omitted URL means the provider's canonical direct endpoint.
- base_url = "https://api.openai.com/v1"
- # Same-provider switches may omit base_url (e.g. credential refresh); resolve capabilities from
- # the endpoint the normalization below retains.
- effective_base_url = base_url
- if not effective_base_url and old_norm == new_norm:
- effective_base_url = getattr(agent, "base_url", "")
- if is_actual_route(new_provider, effective_base_url):
- api_mode = "chat_completions"
- if effective_base_url:
- from hermes_cli.auth import normalize_actual_base_url
- base_url = normalize_actual_base_url(effective_base_url)
- destination_capabilities = (
- dict(capabilities)
- if isinstance(capabilities, dict)
- else resolve_native_compaction_capabilities(
- model=new_model, base_url=effective_base_url, provider=new_provider,
- is_codex_backend=new_norm == "openai-codex",
- )
- )
- # Guard against a trailing /v1 on OpenCode base_url reaching the anthropic_messages client
- # (double-/v1 404); model_switch already strips it, direct callers may not.
- if (
- api_mode == "anthropic_messages"
- and opencode_provider_family(new_provider) is not None
- and isinstance(base_url, str)
- and base_url
- ):
- base_url = re.sub(r"/v1/?$", "", base_url)
- return api_mode, base_url, destination_capabilities
- def _build_switched_client(agent, new_provider, api_key, base_url, api_mode, new_norm) -> None:
- """Build the client for the switched-to destination (MoA facade / native Anthropic / OpenAI wire)."""
- if new_norm == "moa":
- from agent.moa_loop import bind_moa_runtime
- # MoA speaks only chat.completions via the MoAClient facade; the aggregator's real transport
- # is applied inside the fan-out. The binder pins api_mode so the loop never dispatches
- # client.responses.create against the facade (same pins as agent_init / fallback).
- bind_moa_runtime(agent, agent.model, api_key)
- return
- if new_provider == "bedrock" and api_mode in ("anthropic_messages", "bedrock_converse"):
- # Non-Mantle Bedrock wires authenticate through boto3, never through the generic
- # Anthropic/OpenAI builders (which would ship the ``aws-sdk`` sentinel as a credential).
- from agent.bedrock_adapter import bind_bedrock_runtime
- bind_bedrock_runtime(agent, base_url or agent.base_url, api_mode)
- return
- if api_mode == "anthropic_messages":
- from agent.anthropic_adapter import build_anthropic_client
- from agent.anthropic_credentials import resolve_anthropic_token, anthropic_route_is_oauth
- # Only fall back to ANTHROPIC_TOKEN for native Anthropic; other anthropic_messages providers
- # must never receive Anthropic credentials.
- is_native_anthropic = new_provider == "anthropic"
- effective_key = api_key or agent.api_key or (
- resolve_anthropic_token(model=getattr(agent, "model", None)) if is_native_anthropic else ""
- ) or ""
- # MiniMax OAuth: per-request callable token provider survives 15-min expiry (rationale in
- # agent_init.py).
- if new_provider == "minimax-oauth" and isinstance(effective_key, str) and effective_key:
- try:
- from hermes_cli.auth import build_minimax_oauth_token_provider
- effective_key = build_minimax_oauth_token_provider()
- except Exception as _mm_exc:
- logger.warning(
- "MiniMax OAuth: failed to install per-request token provider "
- "on switch (%s); using static bearer.", _mm_exc,
- )
- agent.api_key = agent._anthropic_api_key = effective_key
- agent._anthropic_base_url = base_url or getattr(agent, "_anthropic_base_url", None)
- agent._anthropic_client = build_anthropic_client(
- effective_key, agent._anthropic_base_url,
- timeout=get_provider_request_timeout(agent.provider, agent.model),
- )
- agent._is_anthropic_oauth = anthropic_route_is_oauth(agent._anthropic_base_url, effective_key, provider=new_provider)
- agent.client = None
- agent._client_kwargs = {}
- return
- effective_base = base_url or agent.base_url
- agent._client_kwargs = {"api_key": api_key or agent.api_key, "base_url": effective_base}
- try:
- from hermes_cli.config import (
- apply_custom_provider_tls_to_client_kwargs, get_compatible_custom_providers,
- load_config_readonly,
- )
- # Read live config, not agent._custom_providers, so mid-session ssl_ca_cert / ssl_verify
- # edits are honored.
- # Read custom_providers from live config (not the init-time snapshot on ``agent._custom_providers``)
- # so ssl_ca_cert / ssl_verify edits are honored when switching mid-session, matching the
- # context-length reload below (#15779).
- apply_custom_provider_tls_to_client_kwargs(
- agent._client_kwargs, str(effective_base or ""),
- get_compatible_custom_providers(load_config_readonly()),
- )
- except Exception:
- logger.debug("custom-provider TLS resolution skipped on switch_model", exc_info=True)
- timeout = get_provider_request_timeout(agent.provider, agent.model)
- if timeout is not None:
- agent._client_kwargs["timeout"] = timeout
- # Reapply provider headers (OpenRouter HTTP-Referer/X-Title) lost when _client_kwargs was
- # rebuilt; otherwise attribution shows "Unknown".
- agent._apply_client_headers_for_base_url(effective_base)
- agent.client = agent._create_openai_client(dict(agent._client_kwargs), reason="switch_model", shared=True)
- def _swap_switch_runtime(agent, new_model, new_provider, api_key, base_url, api_mode, old_provider, old_norm, new_norm) -> None:
- """Swap identity/transport fields, reload the pool, rebuild the client (rolled back by the caller on error)."""
- # Clear the per-config override so the new model's context window is re-resolved.
- agent._config_context_length = None
- agent.model = new_model
- agent.provider = agent.requested_provider = new_provider
- # Re-read reasoning_echo so the flag reflects the new primary model (see _reasoning_echo_opt_in).
- agent._reasoning_echo_flag = agent._read_reasoning_echo_from_config()
- # Empty base_url while the provider changes means upstream resolution failed; falling back to
- # the old provider's URL pairs the wrong host and persists via _primary_runtime. Fail loud.
- # Same-provider re-select (credential refresh) may keep the URL.
- if base_url:
- agent.base_url = base_url
- elif old_norm != new_norm:
- raise ValueError(
- f"switch_model: no base_url resolved for provider "
- f"'{new_provider}' (switching from '{old_provider}'); "
- "refusing to keep the previous provider's endpoint"
- )
- agent.api_mode = api_mode
- # New api_mode may need a different transport.
- if hasattr(agent, "_transport_cache"):
- agent._transport_cache.clear()
- from agent.turn_recovery_codex import reset_codex_reasoning_replay
- reset_codex_reasoning_replay(agent)
- if api_key:
- agent.api_key = api_key
- # Reload the credential pool on provider change: a pool with a mismatched provider makes
- # recover_with_credential_pool short-circuit. Reload failure is non-fatal.
- if old_norm != new_norm or getattr(agent, "_credential_pool", None) is None:
- # A pool bound to the old provider is worse than none: the recovery guard rejects it.
- agent._credential_pool = None
- agent._credential_pool_entry_id = None
- try:
- from agent.credential_pool import load_pool
- agent._credential_pool = load_pool(new_provider)
- except Exception as _pool_exc:
- logger.warning(
- "switch_model: credential pool reload failed for %s (%s); "
- "continuing without pool rotation this turn", new_provider, _pool_exc,
- )
- _build_switched_client(agent, new_provider, api_key, base_url, api_mode, new_norm)
- sync_credential_pool_entry_id(agent)
- def _resolve_switch_context_length(agent, snapshot):
- """Resolve the destination context length (LM Studio preload first); returns ``(custom_providers, effective_len)``."""
- custom_providers = None
- try:
- from hermes_cli.config import (
- get_compatible_custom_providers, get_custom_provider_context_length, load_config
- )
- from agent.agent_init import config_context_length_for_runtime
- switch_cfg = load_config()
- custom_providers = get_compatible_custom_providers(switch_cfg)
- # The durable ``model.context_length`` pin is re-read from live config (never carried over
- # blindly, never simply dropped): the destination IS the configured default route -> keep the
- # ceiling; it is some other route -> the scoping inside returns None. Same precedence as
- # construction, where the pin outranks custom_providers metadata (#116467).
- intent = config_context_length_for_runtime(agent, switch_cfg)
- if intent is None:
- intent = get_custom_provider_context_length(
- model=agent.model, base_url=agent.base_url, custom_providers=custom_providers
- )
- except Exception:
- intent = None
- from agent.agent_init import set_config_context_length
- set_config_context_length(agent, intent)
- runtime_len = None
- if hasattr(agent, "_ensure_lmstudio_runtime_loaded"):
- try:
- runtime_len = agent._ensure_lmstudio_runtime_loaded(intent)
- except Exception:
- _restore_switch_snapshot(agent, snapshot)
- raise
- if hasattr(agent, "_lmstudio_load_was_unverified") and agent._lmstudio_load_was_unverified(runtime_len):
- logger.warning(
- "LM Studio model activation was rejected or completed without a "
- "verifiable active context length during model switch; continuing "
- "with configured context"
- )
- effective = intent
- if hasattr(agent, "_effective_lmstudio_context_length"):
- effective = agent._effective_lmstudio_context_length(intent, runtime_len)
- return custom_providers, effective
- def _update_switch_compressor(agent, custom_providers, effective_context_length, snapshot) -> None:
- """Point the context compressor at the new model (rolls back the switch on failure)."""
- from agent.model_metadata import get_model_context_length
- if custom_providers is None:
- try:
- from hermes_cli.config import get_compatible_custom_providers, load_config
- custom_providers = get_compatible_custom_providers(load_config())
- except Exception:
- custom_providers = None
- # agent.api_key may be a callable (Azure Foundry Entra ID); get_model_context_length expects a
- # string for live probes, so coerce defensively.
- ctx_api_key = agent.api_key if isinstance(agent.api_key, str) else ""
- try:
- new_context_length = get_model_context_length(
- agent.model, base_url=agent.base_url, api_key=ctx_api_key, provider=agent.provider,
- config_context_length=effective_context_length, custom_providers=custom_providers,
- )
- agent.context_compressor.update_model(
- model=agent.model,
- context_length=new_context_length,
- base_url=agent.base_url,
- api_key=agent.api_key, # context_compressor forwards to call_llm; callable preserved
- provider=agent.provider,
- api_mode=agent.api_mode,
- )
- except Exception:
- _restore_switch_snapshot(agent, snapshot)
- raise
- # Outside the rollback guard: a probe hiccup must not undo a good switch. Eager, so the aux
- # clamp lands before the first compaction on the new window, not after it (#114707).
- from agent.conversation_compression_feasibility import revalidate_compression_feasibility
- revalidate_compression_feasibility(agent)
- def _finish_switch(agent, new_provider, old_norm, new_norm) -> None:
- """Post-switch bookkeeping: fallback reset/prune, request_overrides, billing route."""
- agent._fallback_activated = False
- agent._provider_fallback_active = False
- agent._provider_fallback_route = None
- agent._fallback_index = 0
- agent._credential_pool_revert_id = None
- # On a deliberate provider swap, prune fallback entries targeting the OLD or NEW primary;
- # otherwise a failed turn silently re-activates the provider the user just rejected.
- fallback_chain = list(getattr(agent, "_fallback_chain", []) or [])
- if old_norm and new_norm and old_norm != new_norm:
- fallback_chain = [
- entry for entry in fallback_chain
- if (entry.get("provider") or "").strip().lower() not in {old_norm, new_norm}
- ]
- agent._fallback_chain = fallback_chain
- agent._fallback_model = fallback_chain[0] if fallback_chain else None
- # Apply the switched-to provider's request_overrides (custom_providers extra_body).
- try:
- _apply_switched_provider_request_overrides(agent, new_provider)
- except Exception:
- logger.debug("switch_model: request_overrides re-derivation failed", exc_info=True)
- def _persist_switch_billing_route(agent) -> None:
- """Persist the billing route so dashboard Model cards show the post-switch provider."""
- # _session_db / session_id may be unset (tests, bare agents).
- session_db = getattr(agent, "_session_db", None)
- session_id = getattr(agent, "session_id", None)
- if session_db is None or not session_id:
- return
- try:
- session_db.update_session_billing_route(
- session_id, provider=agent.provider, base_url=agent.base_url,
- billing_mode=getattr(agent, "api_mode", None),
- )
- except Exception:
- logger.warning("Failed to persist billing route after model switch", exc_info=True)
- def switch_model(
- agent, new_model, new_provider, api_key='', base_url='', api_mode='', capabilities=None
- ):
- """Switch the model/provider in-place for a live agent (rebuild clients, caching flags,
- compressor). Mirrors ``_try_activate_fallback()`` but also updates ``_primary_runtime`` so
- the change persists across turns. A failed swap/rebuild rolls back to the pre-switch
- snapshot and re-raises (callers catch)."""
- old_model = agent.model
- old_provider = agent.provider
- # ── Reload credential pool for the new provider (issue #52727) ── Without this,
- # ``recover_with_credential_pool`` sees a ``pool.provider != agent.provider`` mismatch and
- # short-circuits, leaving the new provider with no rotation/recovery on 401/429 and burning the original
- # pool's entries. Only reload when the provider actually changed (or the pool was missing) —
- # re-selecting the same provider must not churn the pool reference. A reload failure is logged +
- # swallowed: the switch itself must still complete.
- old_norm = (old_provider or "").strip().lower()
- new_norm = (new_provider or "").strip().lower()
- api_mode, base_url, destination_capabilities = _resolve_switch_destination(
- agent, new_model, new_provider, base_url, api_mode, capabilities, old_norm, new_norm
- )
- snapshot = _snapshot_switch_state(agent)
- try:
- _swap_switch_runtime(
- agent, new_model, new_provider, api_key, base_url, api_mode, old_provider, old_norm, new_norm
- )
- except Exception:
- _restore_switch_snapshot(agent, snapshot)
- raise
- custom_providers, effective_context_length = _resolve_switch_context_length(agent, snapshot)
- # Refresh the custom-provider snapshot from the config just loaded so the prompt_caching lookup
- # sees flags added to config.yaml after session start.
- if custom_providers is not None:
- agent._custom_providers = custom_providers
- agent._use_prompt_caching, agent._use_native_cache_layout = agent._anthropic_prompt_cache_policy(
- provider=new_provider, base_url=agent.base_url, api_mode=api_mode, model=new_model
- )
- if hasattr(agent, "context_compressor") and agent.context_compressor:
- _update_switch_compressor(agent, custom_providers, effective_context_length, snapshot)
- # Re-read the per-model reasoning_effort override so it applies immediately (per-model > global;
- # YAML False = disabled).
- try:
- from hermes_constants import resolve_reasoning_config
- from hermes_cli.config import load_config as _sm_load_config
- agent.reasoning_config = resolve_reasoning_config(_sm_load_config() or {}, agent.model)
- logger.info(
- "switch_model: reasoning_config resolved for %s: %s", agent.model, agent.reasoning_config
- )
- except Exception as _reasoning_err:
- logger.debug("switch_model: could not re-resolve reasoning_config: %s", _reasoning_err)
- # Invalidate the cached system prompt so it rebuilds next turn.
- agent._cached_system_prompt = None
- # Publish the destination capability map only after every runtime setup above has succeeded.
- # Failed switches must leave the old map intact.
- agent.runtime_capabilities = destination_capabilities
- # Reset the cross-turn stale-call circuit breaker; otherwise the latched streak keeps
- # short-circuiting the freshly selected healthy provider.
- from agent.chat_completion_helpers import _reset_stale_streak
- _reset_stale_streak(agent)
- agent._primary_runtime = _build_primary_runtime_snapshot(agent, api_mode)
- _finish_switch(agent, new_provider, old_norm, new_norm)
- logger.info(
- "Model switched in-place: %s (%s) -> %s (%s)",
- old_model, old_provider, new_model, new_provider,
- )
- _persist_switch_billing_route(agent)
- def _pre_tool_block_message(agent, function_name, function_args, effective_task_id, tool_call_id, middleware_trace):
- """Plugin pre-tool-call hook verdict: ``(block_message, function_args)``; failures never block."""
- try:
- from hermes_cli.plugins import _dispatch_pre_tool_call_hooks
- block_message, modified_args = _dispatch_pre_tool_call_hooks(
- function_name, function_args, task_id=effective_task_id or "",
- session_id=getattr(agent, "session_id", "") or "", tool_call_id=tool_call_id or "",
- turn_id=getattr(agent, "_current_turn_id", "") or "",
- api_request_id=getattr(agent, "_current_api_request_id", "") or "",
- middleware_trace=list(middleware_trace),
- )
- return block_message, (modified_args if modified_args is not None else function_args)
- except Exception:
- return None, function_args
- def invoke_tool(agent, function_name: str, function_args: dict, effective_task_id: str,
- tool_call_id: Optional[str] = None, messages: list | None = None,
- pre_tool_block_checked: bool = False,
- skip_tool_request_middleware: bool = False,
- tool_request_middleware_trace: Optional[list[dict[str, Any]]] = None,
- skip_tool_execution_middleware: bool = False) -> str:
- """Invoke a single tool (agent-level or registry-dispatched) and return the result string;
- no display logic. Used by the concurrent path; the sequential path keeps its own inline
- invocation for display."""
- from agent.inline_tool_executors import (
- InlineToolContext, apply_transform_tool_result, emit_terminal_post_tool_call,
- resolve_invoke_tool_executor, tool_hook_ids
- )
- if not isinstance(function_args, dict):
- function_args = {}
- hook_ids = tool_hook_ids(agent, effective_task_id, tool_call_id)
- _tool_middleware_trace = list(tool_request_middleware_trace or [])
- try:
- from hermes_cli.middleware import apply_tool_request_middleware
- if not skip_tool_request_middleware:
- _tool_request_mw = apply_tool_request_middleware(function_name, function_args, **hook_ids)
- function_args = _tool_request_mw.payload
- _tool_middleware_trace = _tool_request_mw.trace
- except Exception as _mw_err:
- logger.debug("tool_request middleware error: %s", _mw_err)
- block_message: Optional[str] = None
- if not pre_tool_block_checked:
- block_message, function_args = _pre_tool_block_message(
- agent, function_name, function_args, effective_task_id, tool_call_id, _tool_middleware_trace
- )
- if block_message is not None:
- result = json.dumps({"error": block_message}, ensure_ascii=False)
- emit_terminal_post_tool_call(
- agent, function_name=function_name, function_args=function_args, result=result,
- effective_task_id=effective_task_id, tool_call_id=tool_call_id, status="blocked",
- error_type="plugin_block", error_message=block_message,
- middleware_trace=_tool_middleware_trace,
- )
- return result
- tool_start_time = time.monotonic()
- inline_executor = resolve_invoke_tool_executor(agent, function_name)
- if inline_executor is not None:
- inline_ctx = InlineToolContext(
- effective_task_id=effective_task_id, tool_call_id=tool_call_id, messages=messages
- )
- def _execute(next_args: dict) -> Any:
- result = inline_executor(agent, next_args, inline_ctx)
- call_args = next_args if isinstance(next_args, dict) else function_args
- duration_ms = int((time.monotonic() - tool_start_time) * 1000)
- emit_terminal_post_tool_call(
- agent, function_name=function_name, function_args=call_args,
- result=result, effective_task_id=effective_task_id, tool_call_id=tool_call_id,
- duration_ms=duration_ms, middleware_trace=_tool_middleware_trace,
- )
- return apply_transform_tool_result(
- agent, function_name=function_name, function_args=call_args, result=result,
- effective_task_id=effective_task_id, tool_call_id=tool_call_id, duration_ms=duration_ms,
- )
- else:
- def _execute(next_args: dict) -> Any:
- dispatch_kwargs = dict(
- tool_call_id=tool_call_id, session_id=agent.session_id or "",
- turn_id=getattr(agent, "_current_turn_id", "") or "",
- api_request_id=getattr(agent, "_current_api_request_id", "") or "",
- enabled_tools=list(agent.valid_tool_names) if agent.valid_tool_names else None,
- skip_pre_tool_call_hook=True, skip_tool_request_middleware=True,
- enabled_toolsets=getattr(agent, "enabled_toolsets", None),
- disabled_toolsets=getattr(agent, "disabled_toolsets", None),
- tool_request_middleware_trace=list(_tool_middleware_trace),
- )
- if skip_tool_execution_middleware:
- dispatch_kwargs["skip_tool_execution_middleware"] = True
- import model_tools
- return model_tools.handle_function_call(function_name, next_args, effective_task_id, **dispatch_kwargs)
- if skip_tool_execution_middleware:
- return _execute(function_args)
- from hermes_cli.middleware import run_tool_execution_middleware
- return run_tool_execution_middleware(
- function_name, function_args,
- lambda next_args: _execute(next_args if isinstance(next_args, dict) else function_args),
- original_args=function_args, **hook_ids,
- )
- def repair_tool_call(agent, tool_name: str) -> str | None:
- """Repair a mismatched tool name (case, separators, CamelCase, ``_tool`` suffixes twice so
- ``TodoTool_tool`` reduces fully, then fuzzy match) before aborting. Returns the repaired
- name if in valid_tool_names, else None."""
- from difflib import get_close_matches
- if not tool_name:
- return None
- # VolcEngine api/plan leaks XML attribute fragments into tool_use.name (`terminal"
- # parameter="command" ...`); trim at the first quote/angle bracket. Do NOT split on whitespace:
- # "write file" must reach ``_norm`` -> ``write_file``.
- # `terminal" parameter="command" string="true` `execute_code" parameter="code" string="true`
- # `session_search" parameter="session_id" string="true` We trim at the first unambiguous XML/quote
- # character so the rest of the repair pipeline (lowercase / snake_case / fuzzy match) can resolve the
- # cleaned name to a real tool. Crucially we DO NOT split on whitespace: legitimate inputs like "write
- # file" must keep flowing through ``_norm`` -> ``write_file`` (covered by test_space_to_underscore in
- # tests/agent/test_repair_tool_call_name.py). See #33007.
- for _xml_sep in ('"', "'", "<", ">"):
- _idx = tool_name.find(_xml_sep)
- if _idx > 0:
- tool_name = tool_name[:_idx]
- if not tool_name:
- return None
- _norm = lambda s: s.lower().replace("-", "_").replace(" ", "_")
- _camel_snake = lambda s: re.sub(r"(?<!^)(?=[A-Z])", "_", s).lower()
- def _strip_tool_suffix(s: str) -> str | None:
- lc = s.lower()
- return next((s[: -len(sfx)].rstrip("_-") for sfx in ("_tool", "-tool", "tool") if lc.endswith(sfx)), None)
- # Cheap fast-paths first.
- lowered = tool_name.lower()
- if lowered in agent.valid_tool_names:
- return lowered
- normalized = _norm(tool_name)
- if normalized in agent.valid_tool_names:
- return normalized
- cands: set[str] = {tool_name, lowered, normalized, _camel_snake(tool_name)}
- for _ in range(2): # strip trailing tool-suffix up to twice (TodoTool_tool needs it)
- extra: set[str] = set()
- for c in cands:
- stripped = _strip_tool_suffix(c)
- if stripped:
- extra.update((stripped, _norm(stripped), _camel_snake(stripped)))
- cands |= extra
- for c in cands:
- if c and c in agent.valid_tool_names:
- return c
- matches = get_close_matches(lowered, agent.valid_tool_names, n=1, cutoff=0.7)
- return matches[0] if matches else None
- # Escalate repeated heals once per session window, then stay quiet: repeated heals of the same poisoned
- # transcript used to WARNING on every send (#96870). Built-in default threshold; config.yaml's
- # ``agent.sanitizer_heal_escalation_threshold`` tunes it (<= 0 disables escalation; WARNINGs still fire per window).
- _EMPTY_HEAL_ESCALATE_AFTER = 3
- _EMPTY_HEAL_WINDOW_S = 600.0
- _empty_heal_log_state: dict[str, dict[str, Any]] = {}
- _empty_heal_log_lock = threading.Lock()
- # LRU over the log counters only (they pick WARNING vs ERROR): a forgotten session that heals again opens a fresh
- # window. The notice set and pending notices below are never evicted, so nobody is told twice or loses a notice.
- _EMPTY_HEAL_MAX_SESSIONS = 1024
- # Session keys that already received the one-time user notice. Separate from the windowed log state so a new
- # 10-minute window never re-notifies: the user is told ONCE per session, ever (#96870 — out-of-band,
- # delivery channel only, never injected into conversation context).
- _empty_heal_user_notified: set = set()
- # One-shot pending notices keyed by session, drained via ``consume_pending_sanitizer_heal_notice``
- # and delivered via the status/warning callback.
- _empty_heal_pending_notice: dict[str, str] = {}
- def _content_has_payload(content: Any) -> bool:
- if isinstance(content, str):
- return bool(content.strip())
- if not isinstance(content, list):
- return content not in (None, "")
- # Any typed block counts, as long as a text block is not itself blank.
- return any(
- (block.get("type") != "text" or (isinstance(block.get("text"), str) and block["text"].strip()))
- if isinstance(block, dict) else bool(block)
- for block in content
- )
- def _msg_has_payload(msg: dict[str, Any]) -> bool:
- """True if ``msg`` carries anything the API treats as non-empty content (text, multimodal
- blocks, tool_calls, reasoning). Role-agnostic counterpart of ``AIAgent._is_thinking_only_assistant``.
- Codex Responses item carriers persist with content:"" by design (text lives in codex_*_items
- and is replayed); treating them as payload keeps the repair from rewriting a designed-empty turn."""
- return _content_has_payload(msg.get("content")) or bool(
- msg.get("tool_calls")
- or (isinstance(msg.get("reasoning_content"), str) and msg["reasoning_content"].strip())
- or msg.get("reasoning")
- or msg.get("reasoning_details")
- or msg.get("codex_message_items")
- or msg.get("codex_reasoning_items")
- )
- def fill_empty_non_final_wire_payload(msg: dict[str, Any], *, is_final: bool) -> bool:
- """Write the interrupted placeholder onto an empty non-final wire copy; True when filled.
- Pass the per-call copy only; durable history must not be mutated."""
- if is_final or not isinstance(msg, dict) or msg.get("role") not in ("user", "assistant"):
- return False
- if _msg_has_payload(msg):
- return False
- msg["content"] = _INTERRUPTED_PLACEHOLDER
- return True
- def _session_id_for_heal_log() -> str:
- try:
- from hermes_logging import _session_context
- return str(getattr(_session_context, "session_id", None) or "")
- except Exception:
- return ""
- def _heal_escalation_threshold() -> int:
- """Escalation threshold from ``agent.sanitizer_heal_escalation_threshold``, else the module default (fail-safe on any read error)."""
- with contextlib.suppress(Exception):
- from hermes_cli.config import load_config_readonly
- raw = (load_config_readonly().get("agent", {}) or {}).get("sanitizer_heal_escalation_threshold")
- if raw is not None:
- return int(raw)
- return _EMPTY_HEAL_ESCALATE_AFTER
- def consume_pending_sanitizer_heal_notice() -> Optional[str]:
- """Drain the one-time user notice for the current session (at most one per session lifetime).
- Delivered through the status/warning callback, NEVER appended to the conversation context."""
- key = _session_id_for_heal_log() or "-"
- with _empty_heal_log_lock:
- return _empty_heal_pending_notice.pop(key, None)
- def get_sanitizer_heal_stats() -> dict[str, dict[str, Any]]:
- """Read-only per-session sanitiser heal counters (``heal_events``, ``messages_healed``, ``escalated``) for diagnostics."""
- with _empty_heal_log_lock:
- return {
- k: {
- "heal_events": v.get("total_events", v.get("count", 0)),
- "messages_healed": v.get("total_healed", 0),
- "escalated": k in _empty_heal_user_notified,
- }
- for k, v in _empty_heal_log_state.items()
- }
- def _log_empty_non_final_heal(healed: int) -> None:
- """WARNING on the first heals in a window, one ERROR at the threshold, then silent. The
- threshold also queues a ONE-TIME out-of-band user notice (drained by
- ``consume_pending_sanitizer_heal_notice``); never re-armed by a new window."""
- key = _session_id_for_heal_log() or "-"
- threshold = _heal_escalation_threshold()
- now = time.monotonic()
- with _empty_heal_log_lock:
- state = _empty_heal_log_state.pop(key, None) # re-inserted last: most recently healed
- if state is None or (now - state["window_start"]) > _EMPTY_HEAL_WINDOW_S:
- prior = state or {}
- state = {
- "count": 0, "window_start": now, "escalated": False,
- "total_events": prior.get("total_events", 0), "total_healed": prior.get("total_healed", 0),
- }
- publish_lru(_empty_heal_log_state, key, state, _EMPTY_HEAL_MAX_SESSIONS)
- state["count"] += 1
- state["total_events"] = state.get("total_events", 0) + 1
- state["total_healed"] = state.get("total_healed", 0) + healed
- count, total_events, total_healed = state["count"], state["total_events"], state["total_healed"]
- if state["escalated"]:
- return
- escalate = threshold > 0 and count >= threshold
- if escalate:
- state["escalated"] = True
- if key not in _empty_heal_user_notified:
- _empty_heal_user_notified.add(key)
- _empty_heal_pending_notice[key] = (
- "⚠️ Your session transcript required repeated repair "
- f"({total_events} heal passes so far). Replies keep "
- "working, but a corrupted turn is stuck in this "
- "session's history — run /debug share or `hermes "
- "doctor` to capture diagnostics, or /new to start a clean session."
- )
- if escalate:
- _ra().logger.error(
- "Pre-call sanitizer: repeated-heal escalation for session %s — "
- "healed %d empty non-final message(s) this send; heal pattern: "
- "%d heal events / %d messages healed this session "
- "(%d in the current session window, threshold %d). The transcript "
- "is being repaired on every send; /new drops the poisoned turns.", key, healed,
- total_events, total_healed, count, threshold,
- )
- return
- _ra().logger.warning(
- "Pre-call sanitizer: healed %d empty non-final message(s) by "
- "substituting placeholder content — an empty-content turn was in "
- "the transcript and would 400 the request ('messages must have "
- "non-empty content' / INVALID_REQUEST_BODY). Self-recovering the "
- "poisoned transcript in memory; no restart needed.", healed,
- )
- def repair_empty_non_final_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Substitute a placeholder for empty-content non-final messages on the per-call copy.
- Anthropic/litellm/Bedrock 400 on any empty non-final message and a persisted stub poisons
- every later turn; repairing the wire copy heals the session in memory. Substitution (not
- deletion) keeps role alternation and tool-call pairing intact. The final message is untouched."""
- if not messages or len(messages) < 2:
- return messages
- repaired: list[dict[str, Any]] = []
- healed = 0
- last_idx = len(messages) - 1
- for idx, msg in enumerate(messages):
- # Tool results are checked by their own pairing pass; empty ones are a separate concern.
- if idx != last_idx and isinstance(msg, dict) and msg.get("role") in ("assistant", "user") and not _msg_has_payload(msg):
- # Shallow-copy so stored history / prompt caching stays byte-stable.
- repaired.append({**msg, "content": _INTERRUPTED_PLACEHOLDER})
- healed += 1
- else:
- repaired.append(msg)
- if healed:
- _log_empty_non_final_heal(healed)
- return repaired
- return messages
- def _classify_tool_call_orphans(messages: list[dict[str, Any]]):
- """Classify orphaned tool-call / tool-result pairs; single source of truth for GLOBAL orphan
- detection. Returns ``(surviving_call_ids, result_call_ids, orphaned_results, missing_tool_calls)``;
- every id variant of a tool_call is registered so a result matching any alias survives, and
- ``orphaned_results`` are the actual dicts (filter by ``id(msg)``). ``sanitize_api_messages``
- pairs positionally instead but shares the ``*_id_variants`` alias policy."""
- assistant_call_variants = [
- (tc, variants)
- for msg in messages if msg.get("role") == "assistant"
- for tc in msg.get("tool_calls") or []
- if (variants := tool_call_id_variants(tc))
- ]
- surviving_call_ids: set[str] = set().union(*(v for _, v in assistant_call_variants))
- result_entries = [
- (msg, tool_result_id_variants(msg.get("tool_call_id"))) for msg in messages if msg.get("role") == "tool"
- ]
- result_call_ids: set[str] = set().union(*(v for _, v in result_entries))
- orphaned_results = [msg for msg, v in result_entries if v and not (v & surviving_call_ids)]
- # Orphan result variants are disjoint from every declared call, so they
- # cannot contribute a match. Reuse the union instead of scanning each result.
- missing_tool_calls = [
- tc for tc, v in assistant_call_variants if not (v & result_call_ids)
- ]
- return surviving_call_ids, result_call_ids, orphaned_results, missing_tool_calls
- def _drop_invalid_roles(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Drop messages whose role the API won't accept."""
- valid = _ra().AIAgent._VALID_API_ROLES
- for msg in messages:
- if msg.get("role") not in valid:
- _ra().logger.debug("Pre-call sanitizer: dropping message with invalid role %r", msg.get("role"))
- return [m for m in messages if m.get("role") in valid]
- def _drop_empty_tool_calls_arrays(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Strict providers 400 on ``tool_calls: []``; normalize on shallow copies so history stays byte-stable."""
- # --- Drop empty / malformed tool_calls arrays on assistant messages --- An assistant message carrying
- # ``tool_calls: []`` (an empty array) — or a non-list value under the key — is semantically identical to
- # an assistant message with no tool calls, but strict OpenAI-compatible providers reject the empty array
- # outright: DeepSeek v4 returns HTTP 400 "Invalid 'messages[N].tool_calls': empty array. Expected an
- # array with minimum length 1, but got an empty array instead." (#58755, follow-up to #56980). Empty
- # arrays reach here from session resume, host-fed histories, or the consecutive-assistant merge in
- # ``repair_message_sequence`` (which preserves a pre-existing ``[]`` on the surviving turn). This is the
- # final pre-API chokepoint, so normalize defensively — and, per the #56980 review, do it HERE on the
- # per-call copy rather than in ``repair_message_sequence``, which would destructively rewrite the
- # persisted trajectory. Shallow-copy the message before dropping the key so stored history (and prompt
- # caching) stays byte-stable.
- normalized: list[dict[str, Any]] = []
- dropped = 0
- for msg in messages:
- if (
- isinstance(msg, dict)
- and msg.get("role") == "assistant"
- # Defense-in-depth: a strict OpenAI-compatible provider (e.g. onerouter / Qwen, DeepSeek v4)
- # rejects an assistant message carrying ``tool_calls: []`` (empty array) with HTTP 400 "Empty
- # tool_calls is not supported in message." The pre-API sanitizer in agent_runtime_helpers drops
- # these, but only on the conversation_loop path — other routes can reach the wire without it.
- # For every request that serializes through this transport (conversation loop and any caller
- # using it), this is the last boundary, so normalize here. Requests built by fully separate
- # payload paths (e.g. some auxiliary clients) never pass through this layer and are out of scope
- # for it. (#58755 follow-up)
- and "tool_calls" in msg
- and not (isinstance(msg["tool_calls"], list) and msg["tool_calls"])
- ):
- msg = {k: v for k, v in msg.items() if k != "tool_calls"}
- dropped += 1
- normalized.append(msg)
- if not dropped:
- return messages
- _ra().logger.debug(
- "Pre-call sanitizer: dropped empty/invalid tool_calls on %d assistant message(s)", dropped
- )
- return normalized
- def _repair_invalid_tool_call_names(messages: list[dict[str, Any]]) -> None:
- """Coerce every ``function.name`` to the provider-safe ``^[A-Za-z0-9_-]{1,64}$``. An empty/missing
- name becomes the ``invalid_tool_call`` sentinel (dropping would unpair the anti-priming result the
- dispatch loop keeps for it); an invalid one (``multi_tool_use.parallel``, a shell command a weak
- fallback model put in ``name``) is coerced deterministically, because one such stored turn 400s
- every later request on a strict endpoint and pins the session to the fallback model (#51944).
- Tool calls are rewritten copy-on-write (an SDK object becomes a dict copy) so a shallow per-call
- copy never edits persisted history; tool results follow via ``_realign_tool_result_names``."""
- for msg in messages:
- if msg.get("role") != "assistant":
- continue
- tcs = msg.get("tool_calls") or []
- for idx, tc in enumerate(tcs):
- if isinstance(tc, dict):
- fn = tc.get("function")
- name = fn.get("name") if isinstance(fn, dict) else getattr(fn, "name", None)
- else:
- fn = getattr(tc, "function", None)
- name = getattr(fn, "name", None) if fn else None
- coerced = coerce_tool_name(name)
- if coerced == name:
- continue
- _ra().logger.warning(
- "Pre-call sanitizer: repairing tool_call with invalid function.name %r -> %r (id=%s)",
- (name or "")[:80], coerced, _ra().AIAgent._get_tool_call_id_static(tc),
- )
- if tcs is msg.get("tool_calls"):
- tcs = msg["tool_calls"] = list(tcs)
- if isinstance(tc, dict):
- fn = {**fn, "name": coerced} if isinstance(fn, dict) else {"name": coerced, "arguments": "{}"}
- tcs[idx] = {**tc, "function": fn}
- else:
- args = getattr(fn, "arguments", None) if fn is not None else None
- tcs[idx] = {
- "id": _ra().AIAgent._get_tool_call_id_static(tc),
- "type": "function",
- "function": {"name": coerced, "arguments": args if isinstance(args, str) else "{}"},
- }
- def _drop_results_without_ids(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Drop tool results with a missing/empty tool_call_id. Kept explicit (not left to the
- positional walk) for its own log line and so the final-chokepoint guarantee holds for
- callers skipping ``repair_message_sequence``."""
- kept = [
- m for m in messages
- if not (m.get("role") == "tool" and not (m.get("tool_call_id") or "").strip())
- ]
- if len(kept) != len(messages):
- _ra().logger.debug(
- "Pre-call sanitizer: dropped %d tool result(s) with missing/empty tool_call_id",
- len(messages) - len(kept),
- )
- return kept
- def _pair_tool_calls_positionally(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Positional tool_call <-> tool_result pairing: strict providers (DeepSeek v4, Kimi) require
- results IMMEDIATELY after their call. Drops positional orphans, stubs unanswered declared
- ids; matching is alias-aware."""
- # --- Positional tool_call <-> tool_result pairing --- Strict OpenAI-compatible providers (DeepSeek v4,
- # Kimi) enforce the POSITIONAL invariant: an assistant message carrying tool_calls must be IMMEDIATELY
- # followed by tool messages covering every tool_call_id. The previous implementation compared global id
- # sets, which misses the failure mode where a result exists somewhere in the transcript but not in the
- # run right after its call — an interrupted turn or a compression window can displace a result past a
- # user turn. The id then survives in the global result set, so the call looks answered, no stub is
- # injected, and the provider rejects the payload with HTTP 400 "An assistant message with 'tool_calls'
- # must be followed by tool messages responding to each 'tool_call_id' (insufficient tool messages
- # following tool_calls message)". Rewritten as a single rolling walk on the per-call copy (#94704): (a)
- # tool results that do not immediately follow an assistant message declaring their id are dropped
- # (positional orphans — includes results appearing BEFORE their call, which strict providers also
- # reject); (b) declared ids not covered by the immediately-following tool run get a stub result injected
- # at the end of that run, even when a mispositioned result exists elsewhere. Matching is variant-aware
- # (``tool_call_id_variants`` / ``tool_result_id_variants``): a result keyed on ANY alias spelling
- # (``id`` / ``call_id`` / ``response_item_id`` / composite bridge) answers the call, preserving the
- # unified alias policy from #55626/#63000/#93251.
- paired: list[dict[str, Any]] = []
- declared_calls: dict[str, tuple] = {}
- dropped = 0
- stubs = 0
- def _flush_unanswered_stubs() -> None:
- nonlocal stubs
- for key in sorted(declared_calls):
- tc, _variants = declared_calls[key]
- paired.append({
- "role": "tool", "name": _ra().AIAgent._get_tool_call_name_static(tc),
- "content": "[Result unavailable — see context summary above]",
- "tool_call_id": coalesce_tool_call_id(tc) or key,
- })
- stubs += 1
- declared_calls.clear()
- for msg in messages:
- role = msg.get("role")
- if role == "assistant":
- # A new assistant turn closes the previous tool-result run: anything still pending was
- # never answered positionally.
- _flush_unanswered_stubs()
- declared_calls = {}
- for tc in msg.get("tool_calls") or []:
- variants = tool_call_id_variants(tc)
- if variants:
- # Key on a stable representative of the alias group so a result matching ANY
- # spelling can consume the call.
- declared_calls[min(variants)] = (tc, variants)
- elif role == "tool":
- result_variants = tool_result_id_variants(msg.get("tool_call_id"))
- matched = next((k for k, (_tc, v) in declared_calls.items() if v & result_variants), None)
- if matched is None:
- dropped += 1
- continue
- # Consume so a duplicate result reusing the id is dropped (strict providers reject duplicates).
- declared_calls.pop(matched, None)
- elif role == "user":
- # A user turn closes the tool-result run; later tool messages are orphans.
- _flush_unanswered_stubs()
- paired.append(msg)
- # The transcript may end right after an unanswered assistant turn.
- _flush_unanswered_stubs()
- if dropped:
- _ra().logger.debug("Pre-call sanitizer: removed %d positionally orphaned tool result(s)", dropped)
- if stubs:
- _ra().logger.debug(
- "Pre-call sanitizer: added %d stub tool result(s) for "
- "positionally unanswered tool call(s)", stubs,
- )
- return paired if (dropped or stubs) else messages
- def _dedupe_tool_call_ids(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Deduplicate tool_call_ids (strict providers 400 on duplicates): collapse duplicates within
- an assistant message, drop results answering no OUTSTANDING call. Tracks outstanding calls
- (not ids ever seen) because llama.cpp reuses one constant id, and whole variant groups so
- alias-keyed results are not deleted."""
- outstanding: dict[str, int] = {} # every alias of an unanswered call -> its group id
- # 3. Deduplicate tool_call_ids. Strict providers (DeepSeek) reject a payload where the same tool_call_id
- # appears more than once with HTTP 400 "Duplicate value for 'tool_call_id'" (#58327). Duplicates can
- # arise from retries, crash/resume glitches, or a compression window that re-emits a tool result. This
- # is the final pre-API chokepoint, so dedup defensively here even though repair_message_sequence also
- # consumes matched ids. (a) collapse duplicate tool_calls WITHIN an assistant message (b) drop tool
- # results that answer no OUTSTANDING tool call (b) tracks outstanding calls rather than every id ever
- # seen, because ``tool_call_id`` is NOT globally unique in practice: llama.cpp emits a single constant
- # id for every tool call it ever returns (verified: three separate completions from one server all
- # carry the same id). A seen-once-drop-forever rule reads the SECOND legitimate tool result of such a
- # session as a duplicate and deletes it, so from the second tool call onward the model never sees any
- # result — it announces its next action and the turn dies with the work unfinished. Outstanding-call
- # semantics keep both protections intact: a re-emitted result still answers no pending call and is
- # still dropped, while a genuine new call that reuses the id re-arms that id first. Variant-group
- # tracking: answering or deduping one spelling consumes its siblings too. A Codex/Responses tool_call
- # registers ``id`` (fc_...), ``call_id`` (call_...), ``response_item_id``, and composite spellings
- # (#55626/#58168/#63000); tracking only the coalesced id here made a result keyed on any OTHER variant
- # look like it answered no outstanding call, so this pass deleted the very result step 2's
- # variant-aware matching had just preserved (issue #93251 — whole parallel batches vanished).
- outstanding_groups: dict[int, frozenset] = {}
- next_group_id = 0
- deduped: list[dict[str, Any]] = []
- removed = 0
- for msg in messages:
- role = msg.get("role")
- if role == "assistant" and msg.get("tool_calls"):
- kept_tcs = []
- for tc in msg.get("tool_calls") or []:
- variants = tool_call_id_variants(tc)
- if variants and variants & outstanding.keys():
- removed += 1
- continue
- if variants:
- group_id = next_group_id
- next_group_id += 1
- outstanding_groups[group_id] = variants
- for variant in variants:
- outstanding.setdefault(variant, group_id)
- kept_tcs.append(tc)
- if kept_tcs:
- msg = {**msg, "tool_calls": kept_tcs}
- elif len(kept_tcs) != len(msg.get("tool_calls") or []):
- msg = {k: v for k, v in msg.items() if k != "tool_calls"}
- elif role == "tool":
- result_variants = tool_result_id_variants(msg.get("tool_call_id"))
- candidate_groups = {outstanding[v] for v in result_variants if v in outstanding}
- if result_variants and not candidate_groups:
- removed += 1
- continue
- if candidate_groups:
- # Consume EVERY variant of the matched call; ids are re-armed by the next call reusing them.
- # Consume the whole alias group so a SECOND result replaying any sibling spelling falls into
- # the drop branch below — strict providers reject duplicate tool_call_ids with HTTP 400
- # (#58327, #66974). Credit: #55436.
- group_id = min(candidate_groups)
- for variant in outstanding_groups.pop(group_id, frozenset()):
- if outstanding.get(variant) == group_id:
- del outstanding[variant]
- deduped.append(msg)
- if not removed:
- return messages
- _ra().logger.debug(
- "Pre-call sanitizer: removed %d duplicate tool_call_id reference(s)", removed
- )
- return deduped
- def _realign_tool_result_names(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Align each tool result's wire ``name`` with its call's function name (per-call copy only):
- Google 400s on a mismatch, routine when tool_search bridges via ``tool_call``."""
- # 4. Google matches functionResponse.name against functionCall.name and rejects a mismatch with HTTP 400
- # "Request contains an invalid argument" (INVALID_ARGUMENT); behind an OpenAI-compatible gateway that
- # surfaces only as a generic "Provider returned error". When tool_search defers MCP/plugin tools the
- # model calls the bridge tool ``tool_call``, while ``make_tool_result_message()`` labels the result
- # with the unwrapped internal tool name (``mcp__github__create_issue``) that dispatch, hooks, logging,
- # and guardrails need. #72089 fixed exactly this for the native Gemini adapter, which now prefers
- # ``tool_name_by_call_id`` over the result name; requests that reach Gemini through the
- # OpenAI-compatible path (OpenRouter, Vertex/LiteLLM proxies, any OpenAI-shaped gateway) skip that
- # translation entirely and still send the internal name on the wire. Normalizing here rather than in
- # the OpenAI-compat serializer keeps it provider-agnostic: Gemini reaches Hermes under many model
- # strings and base URLs, so sniffing for "is this really Google?" is unreliable, and every other
- # provider either ignores the field or agrees with the call name. Runs on the per-call copy, so the
- # stored trajectory keeps the real tool name for the session DB and the UI — only the wire payload
- # changes. A no-op for the native Gemini path, which already resolves the same name. A result whose
- # assistant call frame is missing entirely never reaches here — pass 1 above drops it as an orphan —
- # so the only results this pass sees are ones whose call name is knowable.
- call_names: dict[str, str] = {}
- for msg in messages:
- if msg.get("role") == "assistant":
- for tc in msg.get("tool_calls") or []:
- # Strip on insert to match the lookup below so padded ids still pair.
- cid = (_ra().AIAgent._get_tool_call_id_static(tc) or "").strip()
- nm = _ra().AIAgent._get_tool_call_name_static(tc)
- if cid and nm:
- call_names[cid] = nm
- realigned: list[tuple[str, str]] = []
- aligned: list[dict[str, Any]] = []
- for msg in messages:
- if msg.get("role") == "tool":
- expected = call_names.get((msg.get("tool_call_id") or "").strip())
- current = msg.get("name")
- # Only rewrite a present, disagreeing name; clean transcripts must stay byte-identical for prompt caching.
- if expected and current and current != expected:
- msg = {**msg, "name": expected}
- realigned.append((current, expected))
- aligned.append(msg)
- if not realigned:
- return messages
- _ra().logger.debug(
- "Pre-call sanitizer: realigned %d tool result name(s) with their "
- "tool_call function name (%s)", len(realigned),
- ", ".join(f"{was} -> {now}" for was, now in realigned),
- )
- return aligned
- def sanitize_api_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Fix orphaned tool_call / tool_result pairs before every LLM call; runs unconditionally (not
- gated on the compressor). Order matters: empty non-final messages are healed first so the
- substituted turn participates in the pairing and dedup passes."""
- messages = _drop_invalid_roles(messages)
- messages = repair_empty_non_final_messages(messages)
- messages = _drop_empty_tool_calls_arrays(messages)
- _repair_invalid_tool_call_names(messages)
- messages = _drop_results_without_ids(messages)
- messages = _pair_tool_calls_positionally(messages)
- messages = _dedupe_tool_call_ids(messages)
- return _realign_tool_result_names(messages)
- _ACK_FUTURE_RE = re.compile(r"\b(i['’]ll|i will|let me|i can do that|i can help with that)\b")
- _ACK_ACTION_MARKERS = (
- "look into", "look at", "inspect", "scan", "check", "analyz", "review", "explore", "read", "open",
- "run", "test", "fix", "debug", "search", "find", "walkthrough", "report back", "summarize",
- )
- _ACK_WORKSPACE_MARKERS = (
- "directory", "current directory", "current dir", "cwd", "repo", "repository", "codebase",
- "project", "folder", "filesystem", "file tree", "files", "path",
- )
- def looks_like_codex_intermediate_ack(
- agent, user_message: Any, assistant_content: str, messages: list[dict[str, Any]],
- require_workspace: bool = True,
- ) -> bool:
- """Detect a planning/ack message that should continue instead of ending the turn.
- ``require_workspace=False`` (opt-in for all api_modes) drops the filesystem/repo reference
- requirement; short-content + no-tool-results-this-turn guardrails always apply (every row after this
- turn's own user row: ``messages`` is the whole session, #69778, and a nudge row starts no new turn).
- A response then needs a future ack + action marker, or a bounded pronounless clause + transition cue."""
- from agent.turn_context import reanchor_current_turn_user_idx as turn_user_idx
- if any(isinstance(m, dict) and m.get("role") == "tool" for m in messages[turn_user_idx(messages, user_message) + 1:]):
- return False
- assistant_text = agent._strip_think_blocks(assistant_content or "").strip().lower()
- if not assistant_text or len(assistant_text) > 1200:
- return False
- # Some tool-using providers narrate the next action in terse log style: "Brief written. Creating the
- # session now." Bare gerunds are too ambiguous ("Reading the traceback explains the failure" is a final
- # answer), so this path also requires an explicit transition cue such as "now", "on <ACP agent> via acpx",
- # "actual log", or "then launching". Completed reports and questions stay final.
- action_clause_pattern = re.compile(
- r"(?:^|[.!…—–]\s+|\n+\s*|,\s+then\s+)"
- r"(?:(?P<transition>then)\s+)?"
- r"(?P<action>(?:re)?launching|(?:re)?starting|creating|"
- r"checking|running|writing|opening|reading|inspecting|reviewing|"
- r"testing|debugging|searching|fixing)\b"
- )
- list_item_prefix_pattern = re.compile(r"(?:^|\n)\s*(?:[-*+]|\d+[.)])\s*$")
- numbered_item_pattern = re.compile(r"(?:^|\s)(\d+)[.)](?=\s)")
- status_number_pattern = re.compile(
- r"\b(?P<label>exit\s+code|attempt|step|retry)\s+"
- r"(?P<number>\d+)[.)](?=\s)"
- )
- question_pattern = re.compile(r"\?(?=\s|$|[\"'’”)\\]])")
- sentence_boundary_pattern = re.compile(
- r"(?:[.!…](?=\s|$)|\?(?=\s|$|[\"'’”)\\]])|\n+)"
- )
- allowed_trailing_action_pattern = re.compile(
- r"\s*(?:will\s+report\s+back|"
- r"then\s+(?:launching|relaunching|starting|restarting|creating)\s+"
- r"(?:it|(?:the\s+)?(?:session|worker|agent|job|process))|"
- r"(?:checking|reading|opening|inspecting)\s+(?:the\s+)?(?:log|output))"
- r"\s*[.!…]?\s*$"
- )
- launch_now_tail_pattern = re.compile(
- r"\s+(?:(?:it|(?:the\s+)?(?:session|worker|agent|job|process)|"
- r"(?:the\s+)?migration\s+file\s+in\s+the\s+repo)\s+)?"
- r"now(?:\s*\([^)]*\))?\s*$"
- )
- run_now_tail_pattern = re.compile(
- r"\s+(?:(?:it|(?:the\s+)?(?:(?:repo\s+)?suite|tests?|job|process|"
- r"server|service))\s+)?now(?:\s*\([^)]*\))?\s*$"
- )
- check_now_tail_pattern = re.compile(
- r"\s+(?:the\s+)?(?:log|output|status|service|session|job)\s+"
- r"now(?:\s*\([^)]*\))?\s*$"
- )
- launch_url_tail_pattern = re.compile(
- r"\s+now\s*[—–-]\s*see\s+https?://\S+"
- r"(?:\s+for\s+progress)?\s*$"
- )
- from providers import list_providers # "relaunching it on <an ACP agent: its name, alias or binary>"
- provider_names = sorted({n.lower() for p in list_providers() if p.auth_type == "external_process"
- for n in (p.name, *p.aliases, p.process_command) if n})
- provider_tail_pattern = re.compile(
- r"\s+(?:it|(?:the\s+)?(?:session|worker|agent|job|process))\s+"
- rf"on\s+(?:{'|'.join(map(re.escape, provider_names)) or '(?!)'})(?:\s+via\s+acpx)?\s*$"
- )
- actual_log_tail_pattern = re.compile(r"\s+(?:the\s+)?actual\s+(?:log|output)\s*$")
- health_check_tail_pattern = re.compile(
- r"\s+whether\b.*\b(?:healthy|ready|running|available|reachable|working)\s*$"
- )
- launch_with_pattern = re.compile(
- r"\s+with\s+(?:(?:corrected|updated|new)\s+"
- r"(?:arguments?|args?|options?|flags?|parameters?)|"
- r"globals?\s+before\s+(?:the\s+)?agent\s+name)\s*$"
- )
- has_pronounless_action = False
- step_numbers = {
- status_match.group("number")
- for status_match in status_number_pattern.finditer(assistant_text)
- if status_match.group("label") == "step"
- }
- has_step_sequence = len(step_numbers) >= 2
- numbering_text = status_number_pattern.sub("", assistant_text)
- numbered_markers = {
- match.group(1) for match in numbered_item_pattern.finditer(numbering_text)
- }
- has_numbered_list = has_step_sequence or bool(numbered_markers)
- if not question_pattern.search(assistant_text):
- for action_match in action_clause_pattern.finditer(assistant_text):
- action_prefix = assistant_text[: action_match.start("action")]
- if has_numbered_list or list_item_prefix_pattern.search(action_prefix):
- continue
- raw_clause_tail = assistant_text[action_match.end() :]
- boundary_match = sentence_boundary_pattern.search(raw_clause_tail)
- if boundary_match:
- clause_tail = raw_clause_tail[: boundary_match.start()]
- remaining_text = raw_clause_tail[boundary_match.end() :]
- else:
- clause_tail = raw_clause_tail
- remaining_text = ""
- if remaining_text.strip() and not allowed_trailing_action_pattern.fullmatch(
- remaining_text
- ):
- continue
- action_word = action_match.group("action")
- is_launch_action = action_word in {
- "launching",
- "relaunching",
- "starting",
- "restarting",
- "creating",
- }
- has_transition_cue = bool(
- (is_launch_action and launch_now_tail_pattern.fullmatch(clause_tail))
- or (action_word == "running" and run_now_tail_pattern.fullmatch(clause_tail))
- or (
- action_word == "checking"
- and check_now_tail_pattern.fullmatch(clause_tail)
- )
- or (is_launch_action and launch_url_tail_pattern.fullmatch(clause_tail))
- or (is_launch_action and provider_tail_pattern.fullmatch(clause_tail))
- or (
- action_word in {"checking", "reading", "opening", "inspecting"}
- and actual_log_tail_pattern.fullmatch(clause_tail)
- )
- or (
- action_word == "checking"
- and health_check_tail_pattern.fullmatch(clause_tail)
- )
- or (is_launch_action and launch_with_pattern.fullmatch(clause_tail))
- )
- if not has_transition_cue:
- continue
- has_pronounless_action = True
- break
- if not (_ACK_FUTURE_RE.search(assistant_text) or has_pronounless_action):
- return False
- if not has_pronounless_action and not any(
- marker in assistant_text for marker in _ACK_ACTION_MARKERS
- ):
- return False
- # Opted-in (all-api_mode) path: future-ack + action verb + no prior tool call suffices.
- if not require_workspace:
- return True
- # ``user_message`` may be a multi-part content list (vision via the OpenAI-compat server); a
- # list survives ``or ""`` and ``.strip()`` raises, so flatten first.
- from agent.codex_responses_adapter import _summarize_user_message_for_log
- user_text = _summarize_user_message_for_log(user_message).strip().lower()
- return (
- any(marker in user_text for marker in _ACK_WORKSPACE_MARKERS)
- or "~/" in user_text
- or "/" in user_text
- or any(marker in assistant_text for marker in _ACK_WORKSPACE_MARKERS)
- )
- # Degenerate-final detector (#103483): after real tool work a text stop whose ENTIRE answer is a
- # fragment — a stray wrong-script word ("пар" in an English conversation), a token starting
- # mid-punctuation ("?warming up") — is a provider-side collapse, not an answer, yet the loop
- # accepted it and the turn reported completed. Shape alone cannot PROVE a collapse, so this is
- # deliberately narrower than "short": a terse legitimate answer ("42", "SQLite", "report.csv",
- # "€12.50", "你好。", "Done.", ":8080", "да" to a Russian prompt) never matches, English-script
- # fragments ("the", "ing") are knowingly not covered, and the re-prompt it triggers asks for the
- # same answer again if it was complete. ``turn_finalizer._SENTENCE_END`` encodes a sibling
- # "≤ 24 chars, no terminal" heuristic for the finish explainer.
- _DEGENERATE_FINAL_MAX_CHARS = 24
- _SENTENCE_TERMINALS = (".", "!", "?", "\u3002", "\uff01", "\uff1f")
- # Punctuation no answer begins with when a letter follows ("?warming"); "$5", "#123", "-1",
- # "/tmp", ".env", "(a)", ":8080", ":)", ";;" all stay answers.
- _DEGENERATE_LEADING_PUNCT = "?!,;:)]}"
- def looks_like_degenerate_final(text: str, user_message: Any = None) -> bool:
- """Whether a text stop reads as a collapsed fragment rather than a (terse) answer.
- "Wrong script" is judged against the conversation: when the user's own message carries
- non-ASCII letters, a terse non-Latin reply ("是", "Готово") is an answer, not a collapse.
- """
- t = (text or "").strip()
- if not t or len(t) > _DEGENERATE_FINAL_MAX_CHARS or t.endswith(_SENTENCE_TERMINALS):
- return False
- if t[0] in _DEGENERATE_LEADING_PUNCT and len(t) > 1 and t[1].isalpha():
- return True
- if not any(ch.isalpha() for ch in t) or any(ch.isascii() and ch.isalnum() for ch in t):
- return False
- from agent.codex_responses_adapter import _summarize_user_message_for_log
- user_text = _summarize_user_message_for_log(user_message) if user_message else ""
- return not any(ch.isalpha() and not ch.isascii() for ch in user_text)
- def tool_results_this_turn(messages: list[dict[str, Any]]) -> int:
- """Tool-result rows after the most recent user row — whether the turn did real tool work.
- ANY user row ends the window, the continuation nudges included: that is what bounds the
- degenerate-final guard to one re-prompt per collapse. Skipping synthetic user rows here
- would turn it into a two-nudge loop.
- """
- count = 0
- for msg in reversed(messages or ()):
- if not isinstance(msg, dict):
- continue
- if msg.get("role") == "user":
- break
- if msg.get("role") == "tool":
- count += 1
- return count
- # Narrow "trailing continue-intent" detector for the stall guard (agent.stall_guards): only the
- # message TAIL announcing a next action, so mid-sentence "I will" never trips it.
- _TRAILING_CONTINUE_INTENT_RE = re.compile(
- r"(?:\blet me now\b|\bi(?:['\u2019])?ll now\b|\bi will now\b"
- r"|\bnow i(?:['\u2019]ll| will)\b|\bnext[,:] i\b)"
- r"[^.!?\n]{0,100}[.:\u2026]?\s*$", re.IGNORECASE,
- )
- # Content longer than this is a substantive reply, not a dangling ack.
- _TRAILING_CONTINUE_INTENT_MAX_CHARS = 400
- def trailing_continue_intent(text: str) -> bool:
- """Whether ``text`` is a short reply ENDING on an announced next action (stall-guard re-prompt trigger)."""
- t = (text or "").strip()
- if not t or len(t) > _TRAILING_CONTINUE_INTENT_MAX_CHARS:
- return False
- return bool(_TRAILING_CONTINUE_INTENT_RE.search(t[-160:]))
- # Broader tail detector for PROMOTED REASONING only (reasoning-only clean stop with tools offered
- # and no tool call). Visible content keeps the narrow ``let me now`` shape above because a real
- # reply legitimately says "I'll" mid-text; chain-of-thought that ENDS on a first-person plan
- # ("Let me batch the terminal calls and run them in parallel.", "I need to check the log.") is a
- # stalled model whose turn would otherwise report "complete" with zero tool calls (#111761).
- # Tail-only and anchored on the last sentence, so reasoning that merely mentions a plan before
- # stating its answer ("...Let me check. The answer is 42.") still promotes.
- # Thai (unsegmented script, so no \b after the trigger, unlike the English group) shares the same
- # tail shape: a first-person future-action marker immediately followed by more Thai text, often
- # preceded by an em/en dash rather than sentence punctuation (#116495). Trigger glosses, in
- # pattern order: "I will give you" / "I will", "next I('ll)" + one of {start,try,check,fix,send,
- # do,look}, "please let me" + one of {start,try,check,fix,send,do,look}, "I('ll)" + one of
- # {start,try,check,fix,send,do,look,run,fire}.
- _PROMOTED_REASONING_PLAN_TAIL_RE = re.compile(
- r"(?:^|[.!?:\u3002\uff01\uff1f\u2014\u2013\n]\s*|\u2026\s*)"
- r"(?:let(?:['\u2019]s| me)\b|i(?:['\u2019]ll| will| need to| should| am going to|['\u2019]m going to)\b"
- r"|next[,:]? i\b|now i(?:['\u2019]ll| will| need to)\b|first[,:]? i(?:['\u2019]ll| will| need to)\b"
- r"|\u0e08\u0e30\u0e43\u0e2b\u0e49\u0e1c\u0e21|\u0e1c\u0e21\u0e08\u0e30"
- r"|\u0e15\u0e48\u0e2d\u0e44\u0e1b(?:\u0e08\u0e30|\u0e1c\u0e21\u0e08\u0e30)"
- r"|\u0e02\u0e2d(?:\u0e40\u0e23\u0e34\u0e48\u0e21|\u0e25\u0e2d\u0e07|\u0e15\u0e23\u0e27\u0e08|\u0e41\u0e01\u0e49|\u0e2a\u0e48\u0e07|\u0e17\u0e33|\u0e14\u0e39)"
- r"|\u0e08\u0e30(?:\u0e40\u0e23\u0e34\u0e48\u0e21|\u0e25\u0e2d\u0e07|\u0e15\u0e23\u0e27\u0e08|\u0e41\u0e01\u0e49|\u0e2a\u0e48\u0e07|\u0e17\u0e33|\u0e14\u0e39|\u0e23\u0e31\u0e19|\u0e22\u0e34\u0e07))"
- r"[^.!?\n\u3002\uff01\uff1f]{0,160}(?:[.:\u2026]+)?\s*$",
- re.IGNORECASE,
- )
- def promoted_reasoning_announces_action(text: str) -> bool:
- """Whether promoted reasoning ENDS on a first-person plan to act (stall, not an answer).
- No overall length cap: the reasoning block of a stalled model is often 300-1600 chars of
- planning monologue; only the tail decides.
- """
- t = (text or "").strip()
- if not t:
- return False
- return bool(_PROMOTED_REASONING_PLAN_TAIL_RE.search(t[-240:]))
- _INTENT_ACK_ON = {"true", "always", "yes", "on"}
- _INTENT_ACK_OFF = {"false", "never", "no", "off"}
- def intent_ack_continuation_mode(agent) -> str:
- """Intent-ack continuation mode: ``"off"``, ``"codex_only"`` (workspace acks on codex_responses)
- or ``"all"``. Mirrors ``agent.tool_use_enforcement``: ``"auto"`` -> codex_only; true-ish -> all;
- false-ish -> off; ``list`` -> all when a substring matches the active model name, else off."""
- mode = getattr(agent, "_intent_ack_continuation", "auto")
- if mode is True or (isinstance(mode, str) and mode.lower() in _INTENT_ACK_ON):
- return "all"
- if mode is False or (isinstance(mode, str) and mode.lower() in _INTENT_ACK_OFF):
- return "off"
- if isinstance(mode, list):
- model_lower = (agent.model or "").lower()
- return "all" if any(p.lower() in model_lower for p in mode if isinstance(p, str)) else "off"
- # "auto" or any unrecognised value: historical codex-only behavior.
- return "codex_only" if agent.api_mode == "codex_responses" else "off"
- def copy_reasoning_content_for_api(agent, source_msg: dict, api_msg: dict) -> None:
- """Forward reasoning fields onto an API replay message; policy lives in ``agent.message_sanitization.apply_reasoning_content_policy``."""
- from agent.message_sanitization import apply_reasoning_content_policy
- apply_reasoning_content_policy(source_msg, api_msg, *agent._reasoning_replay_route())
- def reapply_reasoning_echo_for_provider(agent, api_messages: list) -> int:
- """Reconcile assistant turns' reasoning keys with the CURRENT route after a fallback switch or
- a recorded field rejection: ``api_messages`` is shaped for the primary; must-echo providers
- (DeepSeek/Kimi/MiMo) 400 without the pad, strict ones (Mistral, Cerebras, Groq) 400/422
- with any reasoning key. Idempotent; returns the number of assistant turns changed.
- * Switching TO a strict provider that rejects the field (Mistral, Cerebras, Groq, SambaNova, …):
- assistant turns built under a reasoning primary carry a ``reasoning_content`` pad (often a single space
- ``" "``), and the strict provider rejects it with HTTP 400/422 ("Extra inputs are not permitted"). This
- is the exact cross-provider fallback bug from #45655 — a DeepSeek primary pads history with ``" "``, the
- request falls back to Mistral, and Mistral 422s on the stale pad.
- """
- from agent.message_sanitization import reapply_reasoning_echo
- return reapply_reasoning_echo(api_messages, *agent._reasoning_replay_route())
- def _iter_httpx_pools_with_owner(http_client: Any):
- """Yield ``(pool, owner)`` pairs reachable from an httpx client, including mounted transports:
- keepalive and proxy configs put live connections on ``client._mounts``, which a
- ``_transport``-only walk misses.
- ``owner`` is ``None`` for a pool this client owns outright, or the ``_SharedTransport`` view
- id when the pool is process-shared with other clients
- (``process_bootstrap.build_keepalive_http_client``). Callers must then touch only the
- in-flight requests stamped with that owner.
- Walking the default transport alone makes ``force_close_tcp_sockets`` return 0 while a stream is still
- mid-recv — the interrupt logs success and the provider keeps burning the slot (#72975).
- """
- seen_pools: set[int] = set()
- try:
- transports = [getattr(http_client, "_transport", None)]
- transports += list((getattr(http_client, "_mounts", None) or {}).values())
- for transport in transports:
- if transport is None:
- continue
- # Connections live under ``_pool``; a directly mounted HTTPProxy *is* a ConnectionPool,
- # so ``_connections`` may sit on the transport itself.
- pool = getattr(transport, "_pool", None)
- if pool is None and getattr(transport, "_connections", None) is not None:
- pool = transport
- if pool is not None and id(pool) not in seen_pools:
- seen_pools.add(id(pool))
- owner = id(transport) if type(transport).__name__ == "_SharedTransport" else None
- yield pool, owner
- except Exception:
- return
- def _iter_httpx_pool_objects(http_client: Any):
- """Yield httpcore pool objects reachable from an httpx client."""
- for pool, _owner in _iter_httpx_pools_with_owner(http_client):
- yield pool
- def _connection_candidates(conn: Any):
- """Walk nested wrappers: proxy tunnels (``_connection``) plus httpx/httpcore
- stream envelopes (``_stream``/``_httpcore_stream``: BoundSyncStream →
- ResponseStream → connection byte stream → HTTP11/2 connection)."""
- seen: set[int] = set()
- stack = [conn]
- while stack:
- obj = stack.pop()
- if obj is None or id(obj) in seen:
- continue
- seen.add(id(obj))
- yield obj
- for attr in ("_connection", "_stream", "_httpcore_stream"):
- nxt = getattr(obj, attr, None)
- if nxt is not None:
- stack.append(nxt)
- def _socket_from_candidate(candidate: Any):
- """Raw socket behind a connection/stream wrapper yielded by ``_connection_candidates``."""
- stream = getattr(candidate, "_network_stream", None) or getattr(candidate, "_stream", None)
- sock = _socket_from_stream(stream) if stream is not None else None
- return sock if sock is not None else _socket_from_stream(candidate)
- def _socket_from_response(response: Any):
- """Raw socket behind an httpx response's network stream (``extensions["network_stream"]``
- first, then ``response.stream``), or None. Callers own their error handling."""
- exts = getattr(response, "extensions", None) or {}
- direct = exts.get("network_stream") if isinstance(exts, dict) else None
- for start in (direct, getattr(response, "stream", None)):
- if start is None:
- continue
- for candidate in _connection_candidates(start):
- sock = _socket_from_candidate(candidate)
- if sock is not None:
- return sock
- return None
- def _socket_from_stream(stream: Any):
- """Raw socket behind an httpcore network stream (several backends), or None."""
- sock = getattr(stream, "_sock", None)
- if sock is None and callable(getattr(stream, "get_extra_info", None)):
- with contextlib.suppress(Exception):
- sock = stream.get_extra_info("socket")
- if sock is None:
- sock = getattr(getattr(stream, "stream", None), "_sock", None)
- if sock is None and callable(getattr(getattr(stream, "_stream", None), "extra", None)):
- # anyio-backed streams expose the raw socket through SocketAttribute.raw_socket.
- with contextlib.suppress(Exception):
- from anyio.abc import SocketAttribute
- sock = stream._stream.extra(SocketAttribute.raw_socket)
- return sock
- def _iter_pool_sockets(client: Any):
- """Yield raw sockets reachable from an OpenAI/httpx client pool. Defensive over private
- httpcore internals (``conn._connection``, proxy tunnel wrappers) that vary by release; also
- walks mount transports and in-flight ``PoolRequest.connection`` objects (``_connections``
- is empty during checkout)."""
- try:
- # Some SDK wrappers *are* the httpx client; fall through so mount-aware discovery runs.
- http_client = getattr(client, "_client", None)
- pools = list(_iter_httpx_pools_with_owner(client if http_client is None else http_client))
- except Exception:
- return
- if not pools:
- return
- from agent.process_bootstrap import HERMES_TRANSPORT_OWNER_EXT
- seen: set[int] = set()
- for pool, owner in pools:
- # ``is None``, not falsiness: an empty ``_connections`` must still let us walk in-flight ``_requests``.
- raw_conns = getattr(pool, "_connections", None)
- if raw_conns is None:
- raw_conns = getattr(pool, "_pool", None)
- # A process-shared pool carries other clients' idle + in-flight connections: only this
- # client's own in-flight requests (stamped by ``_SharedTransport.handle_request``) may be
- # shut down.
- connections = [] if owner is not None else list(raw_conns or [])
- for pool_req in list(getattr(pool, "_requests", None) or []):
- if owner is not None:
- exts = getattr(getattr(pool_req, "request", None), "extensions", None) or {}
- if exts.get(HERMES_TRANSPORT_OWNER_EXT) != owner:
- continue
- conn = getattr(pool_req, "connection", None)
- if conn is not None:
- connections.append(conn)
- for conn in connections:
- for candidate in _connection_candidates(conn):
- sock = _socket_from_candidate(candidate)
- if sock is not None and id(sock) not in seen:
- seen.add(id(sock))
- yield sock
- def _set_reset_from_retry_after(context: dict[str, Any], retry_after: Any) -> None:
- if "reset_at" in context:
- return
- seconds = parse_retry_after_seconds(retry_after)
- if seconds is not None:
- context["reset_at"] = time.time() + seconds
- # OpenAI-style relative windows: "6m0s", "1.5s", "20ms", "1h2m3s" (also a bare number of seconds).
- _DURATION_COMPONENT_RE = re.compile(r"(\d+(?:\.\d+)?)(ms|h|m|s)")
- _DURATION_UNIT_SECONDS = {"h": 3600.0, "m": 60.0, "s": 1.0, "ms": 0.001}
- # Lowest-priority reset sources, after Retry-After and x-ratelimit-reset: OpenAI's per-bucket
- # durations and Anthropic's per-bucket ISO-8601 timestamps. Plain OpenAI/Anthropic 429s often
- # carry only these, and without them the retry status never names the reset window.
- _VENDOR_RESET_HEADERS = (
- "x-ratelimit-reset-requests", "x-ratelimit-reset-tokens",
- "anthropic-ratelimit-requests-reset", "anthropic-ratelimit-tokens-reset",
- )
- def _duration_string_seconds(text: str) -> Optional[float]:
- raw = text.strip().lower()
- if not raw:
- return None
- try:
- return float(raw)
- except ValueError:
- pass
- parts = _DURATION_COMPONENT_RE.findall(raw)
- if not parts or "".join(n + u for n, u in parts) != raw:
- return None
- return sum(float(n) * _DURATION_UNIT_SECONDS[u] for n, u in parts)
- def _set_reset_from_vendor_headers(context: dict[str, Any], headers: Any) -> None:
- for name in _VENDOR_RESET_HEADERS:
- value = headers.get(name)
- if not isinstance(value, str) or not value.strip():
- continue
- seconds = _duration_string_seconds(value)
- if seconds is None:
- absolute = _parse_absolute_timestamp(value)
- seconds = None if absolute is None else absolute - time.time()
- if seconds is not None and seconds > 0:
- context["reset_at"] = time.time() + seconds
- return
- def extract_api_error_context(error: Exception) -> dict[str, Any]:
- """Extract structured rate-limit details from provider errors."""
- context: dict[str, Any] = {}
- body = getattr(error, "body", None)
- payload = (body.get("error") if isinstance(body.get("error"), dict) else body) if isinstance(body, dict) else None
- if isinstance(payload, dict):
- reason = payload.get("code") or payload.get("type") or payload.get("error")
- if isinstance(reason, str) and reason.strip():
- context["reason"] = reason.strip()
- message = payload.get("message") or payload.get("error_description")
- if not message and isinstance(payload.get("error"), str):
- # xAI uses a top-level string ``error`` beside a structured ``code``.
- message = payload.get("error")
- if isinstance(message, str) and message.strip():
- context["message"] = message.strip()
- reset = next((payload.get(k) for k in ("resets_at", "reset_at") if payload.get(k) not in {None, ""}), None)
- if reset is not None:
- context["reset_at"] = reset
- elif isinstance(payload.get("resets_in_seconds"), (int, float)):
- # Codex/ChatGPT usage-limit bodies carry a relative window beside (or instead of) the epoch.
- context["reset_at"] = time.time() + float(payload["resets_in_seconds"])
- _set_reset_from_retry_after(context, payload.get("retry_after"))
- headers = getattr(getattr(error, "response", None), "headers", None)
- if headers:
- _set_reset_from_retry_after(context, headers)
- ratelimit_reset = headers.get("x-ratelimit-reset")
- if ratelimit_reset and "reset_at" not in context:
- context["reset_at"] = ratelimit_reset
- if "reset_at" not in context:
- _set_reset_from_vendor_headers(context, headers)
- if "message" not in context and str(error).strip():
- context["message"] = str(error).strip()[:500]
- if "reset_at" not in context and isinstance(context.get("message") or "", str):
- delay = reset_delay_from_message(context.get("message") or "")
- if delay is not None:
- context["reset_at"] = time.time() + delay
- return context
- def _requeue_pending_steer(agent, steer_text: str) -> None:
- """Put drained steer text back so the caller's fallback delivers it as a next-turn user message."""
- from agent.interrupt_control import requeue_drained_steer
- requeue_drained_steer(agent, steer_text)
- def apply_pending_steer_to_tool_results(agent, messages: list, num_tool_msgs: int) -> None:
- """Persist any pending /steer text as a standalone user message.
- Called at the end of a tool-call batch, before the next API call.
- The steer is emitted as a NEW ``role:"user"`` message appended after the
- last tool result (marker text included), so:
- - the model still sees the self-describing out-of-band marker (same text,
- same provenance semantics);
- - message-role alternation stays legal — ``assistant(tool_calls) → tool →
- user`` is the documented "user jumped in mid-run" pattern that
- ``repair_message_sequence`` deliberately keeps;
- - the appended dict carries no ``_DB_PERSISTED_MARKER`` yet, so the next
- ``_flush_messages_to_session_db`` writes it to the session store — the
- steer text finally becomes part of the durable transcript instead of
- being smeared onto an already-persisted tool row that append-only
- persistence never rewrites (replayed histories then diverge from the
- live request bytes and break the provider prompt cache).
- """
- if num_tool_msgs <= 0 or not messages:
- return
- steer_text = agent._drain_pending_steer()
- if not steer_text:
- return
- # Skip non-tool messages in the tail in case something else is appended at the boundary.
- tail = range(len(messages) - 1, max(len(messages) - num_tool_msgs - 1, -1), -1)
- target = next((messages[j] for j in tail if isinstance(messages[j], dict) and messages[j].get("role") == "tool"), None)
- if target is None:
- # No tool result in this batch (e.g. all skipped by interrupt);
- # requeue so the fallback path delivers it as a normal next-turn
- # user message (which persists like any other user turn).
- _requeue_pending_steer(agent, steer_text)
- return
- messages.append(steer_user_row(steer_text))
- _ra().logger.info(
- "Delivered /steer to agent after tool batch (%d chars) as new user message: %s", len(steer_text),
- steer_text[:120] + ("..." if len(steer_text) > 120 else ""),
- )
- def _shutdown_socket(sock: Any) -> None:
- """``shutdown(SHUT_RDWR)`` WITHOUT closing the FD. ``close()`` from a non-owner thread is
- unsafe: the SSL BIO caches the raw FD, the kernel recycles it, and a flushed TLS record lands
- in the wrong file (once clobbered a SQLite header). ``shutdown()`` is FD-safe from any thread.
- Already shut down / not connected / FD invalid are all benign."""
- import socket as _socket
- try:
- # Clear a blocking timeout so a hung SSL_read notices the shutdown. Still no close().
- settimeout = getattr(sock, "settimeout", None)
- if callable(settimeout):
- with contextlib.suppress(OSError):
- settimeout(0)
- sock.shutdown(_socket.SHUT_RDWR)
- except OSError:
- pass
- def force_close_tcp_sockets(client: Any) -> int:
- """Abort in-flight TCP I/O on every pool socket via ``_shutdown_socket``. Returns the count
- (logged as ``tcp_force_closed=N``)."""
- shutdown_count = 0
- try:
- for sock in _iter_pool_sockets(client):
- _shutdown_socket(sock)
- shutdown_count += 1
- except Exception as exc:
- _ra().logger.debug("Force-close TCP sockets sweep error: %s", exc)
- return shutdown_count
- __all__ = [
- "_iter_pool_sockets",
- "anthropic_prompt_cache_policy",
- "apply_pending_steer_to_tool_results",
- "blank_cache_policy_stub",
- "convert_to_trajectory_format",
- "copy_reasoning_content_for_api",
- "create_openai_client",
- "drop_thinking_only_and_merge_users",
- "dump_api_request_debug",
- "extract_api_error_context",
- "extract_reasoning",
- "force_close_tcp_sockets",
- "invoke_tool",
- "looks_like_codex_intermediate_ack",
- "plan_cache_sections_for_destination",
- "prompt_caching_disabled_from_config",
- "recover_with_credential_pool",
- "repair_message_sequence",
- "repair_tool_call",
- "restore_primary_runtime",
- "sanitize_api_messages",
- "sanitize_tool_call_arguments",
- "strip_think_blocks",
- "switch_model",
- "try_recover_primary_transport",
- ]
|