auxiliary_client.py 425 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281
  1. """Shared auxiliary client router for side tasks (compression, search, vision, ...).
  2. Text auto chain: main provider+model → OpenRouter → Nous Portal → custom endpoint →
  3. native Anthropic → direct API-key providers → None. Vision auto chain: main
  4. provider (if a supported vision backend) → OpenRouter → Nous → Anthropic → custom.
  5. ``auxiliary.free_only`` restricts the OpenRouter lane to ``:free`` SKUs. Codex OAuth is
  6. in neither chain (undocumented, shifting allow-list): main provider or explicit
  7. ``auxiliary.<task>.provider`` only. HTTP 402 in call_llm() falls through the chain.
  8. """
  9. import contextlib
  10. import contextvars
  11. import functools
  12. import hashlib
  13. import inspect
  14. import json
  15. import logging
  16. import os
  17. import re
  18. import threading
  19. import time
  20. import uuid
  21. from types import SimpleNamespace
  22. from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple, TYPE_CHECKING, Union
  23. from urllib.parse import urlparse, parse_qs, urlunparse
  24. from agent.local_prompt_progress import (
  25. PromptReadingClock, current_prompt_reading_hook, prompt_progress, prompt_reading_hook, with_prompt_progress)
  26. from agent.error_classifier import (
  27. _BILLING_PATTERNS,
  28. _OVERLOADED_PATTERNS,
  29. UNSUPPORTED_PARAM_MARKERS,
  30. is_reasoning_field_rejection,
  31. is_reasoning_required_rejection,
  32. )
  33. from agent.auxiliary_error_shapes import (
  34. _botocore_error, _exc_http_status, _is_invalid_aux_response_error, _is_statusless_structured_provider_error,
  35. )
  36. from agent.auxiliary_managed_local import _is_managed_local_endpoint
  37. from agent.auxiliary_reasoning_floor import remember_reasoning_floor, with_reasoning_floor
  38. from agent.auxiliary_structured_output import _route_key, _without_structured_output_format, remember_structured_output_rejection
  39. from agent.codex_headers import (
  40. CODEX_AUX_BASE_URL as _CODEX_AUX_BASE_URL,
  41. apply_required_codex_headers as _apply_required_codex_headers,
  42. codex_cloudflare_headers as _codex_cloudflare_headers,
  43. is_official_codex_base_url as _is_official_codex_base_url,
  44. )
  45. from agent.codex_runtime import _codex_event_has_content
  46. from agent.sdk_transform_bypass import bypass_chat_sdk_request_transform
  47. # `openai.OpenAI` is imported lazily (~240 ms cold); `OpenAI` below is a proxy
  48. # so in-module calls, `auxiliary_client.OpenAI` reads and
  49. # `patch("agent.auxiliary_client.OpenAI")` all keep working.
  50. if TYPE_CHECKING:
  51. from openai import OpenAI
  52. _OPENAI_CLS_CACHE: Optional[type] = None
  53. def _load_openai_cls() -> type:
  54. """Import and cache ``openai.OpenAI``."""
  55. global _OPENAI_CLS_CACHE
  56. if _OPENAI_CLS_CACHE is None:
  57. from openai import OpenAI as _cls
  58. _OPENAI_CLS_CACHE = _cls
  59. return _OPENAI_CLS_CACHE
  60. class _OpenAIProxy:
  61. """Lazy stand-in for ``openai.OpenAI``: forwards calls and isinstance checks, importing on first use."""
  62. __slots__ = ()
  63. def __call__(self, *args, **kwargs):
  64. return _load_openai_cls()(*args, **kwargs)
  65. def __instancecheck__(self, obj):
  66. return isinstance(obj, _load_openai_cls())
  67. def __repr__(self):
  68. return "<lazy openai.OpenAI proxy>"
  69. OpenAI = _OpenAIProxy()
  70. # Availability probe mode: check_fns only need to know whether a client is RESOLVABLE, so
  71. # inside `aux_probe_mode()` constructors return a stub instead of importing openai + building
  72. # httpx/SSL (~0.3s on CLI startup). Stubs are never cached (see _store_cached_client).
  73. _aux_probe_state = threading.local()
  74. class _AuxProbeClientStub:
  75. """Non-functional placeholder returned while `aux_probe_mode` is active."""
  76. __slots__ = ("api_key", "base_url")
  77. def __init__(self, api_key: str = "", base_url: str = "") -> None:
  78. self.api_key = api_key
  79. self.base_url = base_url
  80. def __getattr__(self, name: str) -> Any:
  81. # Loud failure if a probe stub ever leaks into a runtime call path.
  82. raise RuntimeError(
  83. f"_AuxProbeClientStub used as a real client (attribute {name!r}); "
  84. "aux_probe_mode is for availability checks only")
  85. def __repr__(self) -> str:
  86. return "<aux availability-probe client stub>"
  87. def _aux_probe_active() -> bool:
  88. return bool(getattr(_aux_probe_state, "active", False))
  89. @contextlib.contextmanager
  90. def aux_probe_mode():
  91. """Resolve provider availability without constructing real SDK clients."""
  92. prev = getattr(_aux_probe_state, "active", False)
  93. _aux_probe_state.active = True
  94. try:
  95. yield
  96. finally:
  97. _aux_probe_state.active = prev
  98. from agent.credential_pool import load_pool
  99. from agent.custom_entry_pool import entry_scoped_pool, pool_for_aux_provider
  100. from agent.message_metadata import data_use_requirement
  101. from agent.model_metadata import MINIMUM_CONTEXT_LENGTH, get_model_context_length
  102. from hermes_cli.config import get_hermes_home
  103. from hermes_cli.config_providers import _canonical_api_mode
  104. from agent.auxiliary_health import (
  105. _custom_health_base_url, _unhealthy_cache_key, fallback_candidate_quarantine_ttl,
  106. fallback_candidate_unavailable_reason,
  107. )
  108. from agent.auxiliary_unavailable import (
  109. AuxiliaryClientUnavailable, clear_nous_credential_failure, missing_provider_credentials_message,
  110. nous_credential_failure_detail, record_nous_credential_failure)
  111. from hermes_constants import OPENROUTER_BASE_URL, hermes_home_key
  112. from utils import base_url_host_matches, base_url_hostname, base_url_origin, env_float, is_truthy_value, model_forces_max_completion_tokens, normalize_proxy_env_vars
  113. logger = logging.getLogger(__name__)
  114. # resolve_provider_client fall-through dedup: misconfigured-provider warnings fire on every
  115. # retry, so only the first per process surfaces. Separate sets let tests clear each branch.
  116. _LOGGED_UNKNOWN_PROVIDER_KEYS: set = set()
  117. _LOGGED_UNHANDLED_AUTHTYPE_KEYS: set = set()
  118. _LOGGED_UNSUPPORTED_EXTPROC_KEYS: set = set()
  119. _LOGGED_UNSUPPORTED_OAUTH_KEYS: set = set()
  120. def _resolve_aux_verify(base_url: Optional[str]) -> Any:
  121. """httpx ``verify`` for an aux base_url, mirroring the main client (per-provider ``ssl_ca_cert`` /
  122. ``ssl_verify``; otherwise the OS trust store); any failure → httpx default (``True``)."""
  123. try:
  124. from agent.ssl_verify import resolve_httpx_verify
  125. from hermes_cli.config import get_custom_provider_tls_settings, load_config_readonly
  126. tls = get_custom_provider_tls_settings(str(base_url or ""), config=load_config_readonly())
  127. return resolve_httpx_verify(
  128. ca_bundle=tls.get("ssl_ca_cert"), ssl_verify=tls.get("ssl_verify"), base_url=str(base_url or ""))
  129. except Exception:
  130. return True
  131. _WARNED_KEEPALIVE_IMPORT_SKEW = False
  132. def _openai_http_client_kwargs(base_url: Optional[str], *, async_mode: bool = False) -> dict[str, Any]:
  133. """Inject keepalive httpx client with env-only proxy (not macOS system proxy)."""
  134. try:
  135. from agent.process_bootstrap import build_keepalive_http_client
  136. client = build_keepalive_http_client(
  137. str(base_url or ""), async_mode=async_mode, verify=_resolve_aux_verify(base_url))
  138. except (ImportError, AttributeError):
  139. # Version-skewed install (Desktop runtime lagging a git tree) lacks this helper:
  140. # degrade to the SDK default httpx client rather than kill the job; warn once.
  141. global _WARNED_KEEPALIVE_IMPORT_SKEW
  142. if not _WARNED_KEEPALIVE_IMPORT_SKEW:
  143. _WARNED_KEEPALIVE_IMPORT_SKEW = True
  144. logger.warning(
  145. "agent.process_bootstrap.build_keepalive_http_client is "
  146. "unavailable — mixed/stale install detected (#64333). Falling "
  147. "back to the SDK default HTTP client. Run `hermes update` (or "
  148. "reinstall the Desktop app) to resync the runtime.")
  149. client = None
  150. return {"http_client": client} if client is not None else {}
  151. def _create_openai_client(*, api_key: str, base_url: str, **kwargs: Any) -> Any:
  152. if _aux_probe_active():
  153. # Availability probe: resolved credentials/base_url are the answer.
  154. return _AuxProbeClientStub(api_key=api_key, base_url=base_url)
  155. kwargs = {**_openai_http_client_kwargs(base_url), **kwargs}
  156. _apply_required_codex_headers(kwargs, access_token=api_key, base_url=base_url)
  157. # Hermes owns aux retry/fallback policy; the SDK default (max_retries=2) would triple
  158. # wall time on a hung endpoint before Hermes sees one failure.
  159. # Hermes owns auxiliary retry + provider/model fallback policy (the same-provider transient retry in
  160. # call_llm plus the except-chain fallback). The OpenAI SDK's own default (max_retries=2 → up to 3
  161. # attempts) silently multiplies the effective wall time of every aux call by 3× on a slow/hung endpoint,
  162. # so a 120s timeout can stall ~360s before Hermes sees a single failure (issue #54465). Disable
  163. # SDK-internal retries by default and let Hermes control the budget; explicit callers can still override
  164. # via kwargs.
  165. kwargs.setdefault("max_retries", 0)
  166. return OpenAI(api_key=api_key, base_url=base_url, **kwargs)
  167. # Interrupt protection for atomic aux tasks: a compression summary killed by an ordinary
  168. # gateway interrupt degrades to a static marker, so a thread-local flag marks such calls
  169. # protected. Explicit host cancel (Ctrl+C, /stop) still overrides it, timeouts still fire.
  170. # ── Interrupt protection for atomic auxiliary tasks ────────────────────── Some auxiliary tasks must NOT be
  171. # aborted mid-flight by a gateway interrupt (e.g. an incoming user message while the agent is busy). Context
  172. # compression is the prime case: if the summary LLM call is interrupted part-way, compression falls back to
  173. # a static "summary unavailable" marker and the real handoff is lost (#23975). A thread-local flag lets such
  174. # a task mark its in-flight LLM call as interrupt-protected; the Codex Responses stream's cancellation check
  175. # honors it. TIMEOUTS still fire (a hung call must die), and all OTHER aux tasks (vision, web_extract,
  176. # title_generation, …) remain freely interruptible.
  177. _aux_interrupt_protection = threading.local()
  178. class AuxiliaryExplicitCancellation(BaseException):
  179. """Frozen signal that an auxiliary attempt was explicitly hard-cancelled. ``BaseException`` so broad
  180. ``except Exception`` retry/fallback code never treats a host stop as a transport failure; ``cause``
  181. is immutable class data so nothing re-queries a mutable host Event after the transport unwound."""
  182. cause = "explicit_host_cancel"
  183. def __init__(self) -> None:
  184. super().__init__("auxiliary request explicitly cancelled by host")
  185. def _aux_interrupt_protected() -> bool:
  186. return bool(getattr(_aux_interrupt_protection, "active", False))
  187. def _aux_interrupt_cancel_requested() -> bool:
  188. """Return whether an explicit host cancel overrides aux protection."""
  189. check = _capture_aux_cancel_check()
  190. return _captured_aux_cancel_requested(check) if check is not None else False
  191. @contextlib.contextmanager
  192. def aux_interrupt_protection(active: bool = True, cancel_check=None, cancel_event=None):
  193. """Mark this thread's aux LLM call interrupt-protected (re-entrant-safe). ``cancel_check`` /
  194. ``cancel_event`` keep an explicit host hard-cancel path (Event preferred); nested scopes inherit both."""
  195. prev = getattr(_aux_interrupt_protection, "active", False)
  196. prev_cancel_check = getattr(_aux_interrupt_protection, "cancel_check", None)
  197. prev_cancel_event = getattr(_aux_interrupt_protection, "cancel_event", None)
  198. _aux_interrupt_protection.active = active
  199. if callable(cancel_check):
  200. _aux_interrupt_protection.cancel_check = cancel_check
  201. if cancel_event is not None and callable(getattr(cancel_event, "is_set", None)):
  202. _aux_interrupt_protection.cancel_event = cancel_event
  203. try:
  204. yield
  205. finally:
  206. _aux_interrupt_protection.active = prev
  207. _aux_interrupt_protection.cancel_check = prev_cancel_check
  208. _aux_interrupt_protection.cancel_event = prev_cancel_event
  209. def _capture_aux_cancel_check() -> Optional[Callable[[], Any]]:
  210. """Capture the current explicit-cancel source on the owning request thread."""
  211. is_set = getattr(getattr(_aux_interrupt_protection, "cancel_event", None), "is_set", None)
  212. if callable(is_set):
  213. return is_set
  214. # Return the callable itself so attempt-local decision objects keep begin_timeout_cleanup().
  215. check = getattr(_aux_interrupt_protection, "cancel_check", None)
  216. return check if callable(check) else None
  217. def _captured_aux_cancel_requested(cancel_check: Callable[[], Any]) -> bool:
  218. """Read a request-thread cancellation source without leaking its failures."""
  219. try:
  220. return bool(cancel_check())
  221. except Exception:
  222. logger.debug("captured aux cancel check failed", exc_info=True)
  223. return False
  224. class _AuxiliaryCancellationDecision:
  225. """Atomically choose explicit cancellation or provider timeout per attempt."""
  226. def __init__(self, source_cancel_check: Callable[[], Any]) -> None:
  227. self._source_cancel_check = source_cancel_check
  228. self._lock = threading.Lock()
  229. self._outcome = "active"
  230. def __call__(self) -> bool:
  231. with self._lock:
  232. if self._outcome == "active" and _captured_aux_cancel_requested(self._source_cancel_check):
  233. self._outcome = "cancelled"
  234. return self._outcome == "cancelled"
  235. def begin_timeout_cleanup(self) -> bool:
  236. """Return whether timeout won and destructive cleanup is permitted."""
  237. with self._lock:
  238. if self._outcome == "active":
  239. cancelled = _captured_aux_cancel_requested(self._source_cancel_check)
  240. self._outcome = "cancelled" if cancelled else "timed_out"
  241. return self._outcome == "timed_out"
  242. # Forward-progress hooks for streamed aux calls: a fixed host deadline kills a SLOW model
  243. # streaming a big summary as hard as a HUNG one, so wire consumers tick the progress hook only
  244. # for non-empty payloads and the host extends its deadline while tokens move. Thread-local:
  245. # the call and its stream consumption run on the installing thread.
  246. _aux_progress = threading.local()
  247. _aux_dispatch = threading.local()
  248. _aux_provider_response = threading.local()
  249. # Absolute monotonic deadline of the waiting HOST. The stream's own ceiling
  250. # (_aux_stream_total_ceiling, >= the host's and started later) would otherwise leave an
  251. # orphaned stream still billing after every host-ceiling timeout.
  252. # Absolute wall-clock deadline (time.monotonic) of the HOST waiting for this auxiliary call, when it has one
  253. # (#99692). Liveness alone is not enough: a host also stops waiting at its own total ceiling, and the
  254. # streamed consumer below bounds itself only by _aux_stream_total_ceiling() — a budget derived from the aux
  255. # request timeout, which is >= the host ceiling for every configured value AND starts counting later. So the
  256. # stream that outlives its abandoned host is not an edge case; it is the guaranteed outcome of every
  257. # total-ceiling timeout.
  258. _aux_stream_deadline = threading.local()
  259. def _tick_hook(local: threading.local, label: str) -> None:
  260. """Call the thread-local hook installed on ``local``, if any. Never raises."""
  261. hook = getattr(local, "hook", None)
  262. if hook is None:
  263. return
  264. try:
  265. hook()
  266. except Exception:
  267. logger.debug("aux %s hook failed", label, exc_info=True)
  268. def _notify_aux_progress() -> None:
  269. """Tick the installed forward-progress hook, if any."""
  270. _tick_hook(_aux_progress, "progress")
  271. def _notify_aux_dispatch() -> None:
  272. """Record an actual provider dispatch without claiming response progress."""
  273. _tick_hook(_aux_dispatch, "dispatch")
  274. def _notify_aux_timing_response() -> None:
  275. """Record a content-free frame (keepalive/empty delta): counts toward
  276. ``time_to_first_progress_ms`` but must not reset a compression inactivity fence."""
  277. _tick_hook(_aux_provider_response, "provider response")
  278. def _notify_aux_provider_response() -> None:
  279. """Record a provider response/chunk, then preserve the liveness signal."""
  280. _notify_aux_timing_response()
  281. _notify_aux_progress()
  282. def _aux_progress_active() -> bool:
  283. return getattr(_aux_progress, "hook", None) is not None
  284. def _field(obj: Any, key: str, default: Any = None) -> Any:
  285. """Field access for wire objects that may be dicts or SDK/SimpleNamespace objects."""
  286. val = obj.get(key) if isinstance(obj, dict) else getattr(obj, key, None)
  287. return default if val is None else val
  288. def _anthropic_event_has_content(event: Any) -> bool:
  289. """Whether an Anthropic stream event carries a non-empty payload."""
  290. event_type = _field(event, "type")
  291. if event_type == "content_block_delta":
  292. delta = _field(event, "delta")
  293. return any(bool(_field(delta, f)) for f in ("text", "thinking", "partial_json", "signature", "citation"))
  294. if event_type == "content_block_start":
  295. block = _field(event, "content_block")
  296. return _field(block, "type") == "tool_use" and any(bool(_field(block, f)) for f in ("id", "name"))
  297. return False
  298. def _anthropic_aux_stream_event_hook() -> Callable[[Any], None]:
  299. """Per-event callback for the Anthropic aux wire: progress only for substantive payloads
  300. (keepalives must not keep a stalled summary alive), stop at the host deadline or explicit
  301. cancel. The ``TimeoutError`` text must say "timed out" so ``_is_timeout_error`` classifies it."""
  302. host_deadline = _current_aux_stream_deadline()
  303. started = time.monotonic()
  304. def _on_event(event: Any) -> None:
  305. if _anthropic_event_has_content(event):
  306. _notify_aux_provider_response()
  307. else:
  308. _notify_aux_timing_response()
  309. if _aux_interrupt_cancel_requested():
  310. raise AuxiliaryExplicitCancellation()
  311. if host_deadline is not None and time.monotonic() >= host_deadline:
  312. raise TimeoutError(
  313. "Anthropic auxiliary stream timed out at the host compression "
  314. f"deadline after {time.monotonic() - started:.0f}s (the caller already stopped waiting)")
  315. return _on_event
  316. # A dead stream fails at the no-progress window (first token AND between tokens); a live
  317. # stream re-arms per event, bounded by _aux_stream_total_ceiling().
  318. _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS = 60.0
  319. @contextlib.contextmanager
  320. def _aux_thread_local_hook(local: threading.local, hook):
  321. """Install one thread-local hook, restoring the prior on exit (non-callable = passthrough)."""
  322. previous = getattr(local, "hook", None)
  323. local.hook = hook if callable(hook) else previous
  324. try:
  325. yield
  326. finally:
  327. local.hook = previous
  328. @contextlib.contextmanager
  329. def aux_progress_hook(hook):
  330. """Install *hook* as the current thread's aux forward-progress callback (None = passthrough)."""
  331. with _aux_thread_local_hook(_aux_progress, hook):
  332. yield
  333. def _current_aux_stream_deadline() -> Optional[float]:
  334. """The waiting host's absolute monotonic deadline, if one is installed (read live from a callable)."""
  335. value = getattr(_aux_stream_deadline, "value", None)
  336. return value() if callable(value) else value
  337. @contextlib.contextmanager
  338. def aux_stream_deadline(deadline: float | Callable[[], float | None] | None):
  339. """Publish the host's absolute ``time.monotonic()`` deadline to the stream consumer.
  340. ``None`` is a passthrough; re-entrant-safe. Host->worker return leg of the progress hook:
  341. without it the isolated provider daemon streams to its own ceiling after the host stopped
  342. waiting, billing a summary the commit fence refuses.
  343. ``8207862212`` releases the compression OWNER when the fence is cancelled, but the isolated provider
  344. daemon (:func:`_run_protected_sync_provider_call`) that holds the socket keeps streaming to its own
  345. ``_aux_stream_total_ceiling`` budget — >= the host's ceiling by construction — billing an abandoned
  346. summary the commit fence is already guaranteed to refuse, and stacking one fresh orphan per turn on a
  347. session that compression never managed to shrink. See #99692.
  348. """
  349. previous = getattr(_aux_stream_deadline, "value", None)
  350. _aux_stream_deadline.value = deadline if isinstance(deadline, (int, float)) or callable(deadline) else previous
  351. try:
  352. yield
  353. finally:
  354. _aux_stream_deadline.value = previous
  355. def _run_protected_sync_provider_call(callback: Callable[[dict[str, Any]], Any], kwargs: dict[str, Any]) -> Any:
  356. """Run one protected provider callback in an attempt-isolated daemon thread.
  357. Aux clients are process-shared and cannot be closed to wake one request, so the callback (incl.
  358. stream aggregation) runs in a daemon while the owner polls cancellation; on cancel the owner
  359. unwinds at once; the daemon stops at its next streamed chunk, else under the provider timeout in ``kwargs``
  360. (it owns no transcript/commit state, never holds the session lock). Unprotected / no cancel source: direct.
  361. """
  362. source_cancel_check = _capture_aux_cancel_check()
  363. if not _aux_interrupt_protected() or not callable(source_cancel_check):
  364. return callback(kwargs)
  365. # One linearized outcome per attempt: the host Event is reused/cleared on later turns and
  366. # the Codex timeout Timer may race owner polling — same lock for both.
  367. cancel_check = _AuxiliaryCancellationDecision(source_cancel_check)
  368. if cancel_check():
  369. raise AuxiliaryExplicitCancellation()
  370. # Thread-locals do not cross into the daemon: timing hooks fire from the thread running
  371. # the callback, and the host deadline is inert unless carried along.
  372. progress_hook = getattr(_aux_progress, "hook", None)
  373. dispatch_hook = getattr(_aux_dispatch, "hook", None)
  374. provider_response_hook = getattr(_aux_provider_response, "hook", None)
  375. host_deadline = getattr(_aux_stream_deadline, "value", None)
  376. reading_hook = current_prompt_reading_hook()
  377. # #99692: the stream is consumed on the daemon below, and thread-locals do not cross that boundary — an
  378. # owner-thread-only deadline would leave the fix inert on exactly the path large-session compression
  379. # takes (protected call + hard-cancel source installed).
  380. provider_context = contextvars.copy_context()
  381. done = threading.Event()
  382. outcome: dict[str, Any] = {}
  383. def _provider_worker() -> None:
  384. try:
  385. with (
  386. aux_progress_hook(progress_hook),
  387. _aux_thread_local_hook(_aux_dispatch, dispatch_hook),
  388. _aux_thread_local_hook(_aux_provider_response, provider_response_hook),
  389. aux_stream_deadline(host_deadline), prompt_reading_hook(reading_hook),
  390. aux_interrupt_protection(cancel_check=cancel_check),
  391. ):
  392. outcome["result"] = callback(kwargs)
  393. except BaseException as exc:
  394. outcome["exception"] = exc
  395. finally:
  396. done.set()
  397. threading.Thread(
  398. target=provider_context.run, args=(_provider_worker,), name="hermes-protected-aux-provider",
  399. daemon=True).start()
  400. while True:
  401. # Check cancel before AND after each wait so it wins when result publication and the
  402. # host Event land in the same polling interval.
  403. if _captured_aux_cancel_requested(cancel_check):
  404. raise AuxiliaryExplicitCancellation()
  405. if not done.wait(0.02):
  406. continue
  407. if _captured_aux_cancel_requested(cancel_check):
  408. raise AuxiliaryExplicitCancellation()
  409. exception = outcome.get("exception")
  410. if exception is not None:
  411. raise exception
  412. return outcome.get("result")
  413. def _client_declares(client_obj: Any, flag: str) -> bool:
  414. """Whether ``client_obj`` (or its class) sets ``flag`` truthy; absent → False. Capability declaration,
  415. not isinstance, so out-of-tree clients can opt out of wrappers unimported (cf. SUPPORTS_HERMES_TOOL_CALLS)."""
  416. try:
  417. return bool(getattr(client_obj, flag, False))
  418. except Exception:
  419. return False
  420. def _safe_isinstance(obj: Any, maybe_type: Any) -> bool:
  421. """Return False instead of raising when a patched symbol is not a type."""
  422. try:
  423. return isinstance(obj, maybe_type)
  424. except TypeError:
  425. return False
  426. def _extract_url_query_params(url: str):
  427. """Extract query params from URL, return (clean_url, default_query dict or None)."""
  428. parsed = urlparse(url)
  429. if parsed.query:
  430. return urlunparse(parsed._replace(query="")), {k: v[0] for k, v in parse_qs(parsed.query).items()}
  431. return url, None
  432. # Warn only once per process about stale OPENAI_BASE_URL.
  433. _stale_base_url_warned = False
  434. # Local OpenAI-compatible servers (Ollama, vLLM, llama.cpp) route through the generic custom
  435. # provider — mirrors hermes_cli.auth._PROVIDER_ALIASES. Without this group an explicit
  436. # ``provider: ollama`` aux lane matches no registry entry and raises a misleading
  437. # ``OLLAMA_API_KEY`` error instead of using the lane's base_url (#106010).
  438. _LOCAL_SERVER_ALIASES = {
  439. "ollama": "custom", "vllm": "custom", "llamacpp": "custom",
  440. "llama.cpp": "custom", "llama-cpp": "custom",
  441. }
  442. _ALIAS_TABLES: dict[str, tuple[int, dict[str, str]]] = {} # home key -> (registry generation, table)
  443. def _provider_alias_table() -> dict[str, str]:
  444. """The alias table the main provider path resolves against (hermes_cli.auth), per home and registry
  445. generation: plugin aliases are per home. A hand-copied mirror rots silently (#106010); local-server
  446. names stay in: they guard the /v1 tail and the custom-branch no-key-borrow rule."""
  447. from providers import bound_registry_key
  448. home, generation = bound_registry_key()
  449. cached = _ALIAS_TABLES.get(home)
  450. if cached is None or cached[0] != generation:
  451. cached = (generation, dict(_LOCAL_SERVER_ALIASES))
  452. with contextlib.suppress(Exception):
  453. from hermes_cli.auth import _plugin_aliases
  454. cached[1].update(_plugin_aliases())
  455. _ALIAS_TABLES[home] = cached
  456. return cached[1]
  457. def _normalize_aux_provider(provider: Optional[str]) -> str:
  458. normalized = (provider or "auto").strip().lower()
  459. if normalized.startswith("custom:"):
  460. suffix = normalized.split(":", 1)[1].strip()
  461. if not suffix:
  462. return "custom"
  463. normalized = suffix
  464. if normalized == "codex":
  465. return "openai-codex"
  466. if normalized == "main":
  467. # Resolve to the actual main provider so named custom providers work.
  468. main_prov = (_read_main_provider() or "").strip().lower()
  469. if not main_prov or main_prov in {"auto", "main"}:
  470. return "custom"
  471. normalized = main_prov
  472. return _provider_alias_table().get(normalized, normalized)
  473. # Sentinel from _fixed_temperature_for_model(): callers strip ``temperature`` entirely.
  474. # Kimi/Moonshot manage it server-side — any value can conflict with gateway mode selection.
  475. OMIT_TEMPERATURE: object = object()
  476. def _bare_model(model: Optional[str]) -> str:
  477. """Lowercased model slug with any ``vendor/`` prefix stripped."""
  478. return (model or "").strip().lower().rsplit("/", 1)[-1]
  479. def _is_kimi_model(model: Optional[str]) -> bool:
  480. """True for any Kimi / Moonshot model that manages temperature server-side."""
  481. bare = _bare_model(model)
  482. return bare.startswith("kimi-") or bare == "kimi"
  483. def _is_arcee_trinity_thinking(model: Optional[str]) -> bool:
  484. """True for Arcee Trinity Large Thinking (direct or via OpenRouter)."""
  485. return _bare_model(model) == "trinity-large-thinking"
  486. # Codex OAuth hard-caps gpt-5.4/5.5/5.6 and gpt-6 Astra at 272K (raw API/OpenRouter expose 1.05M);
  487. # the default 50% trigger would compact at ~136K, so raise to 85% (~231K).
  488. _CODEX_GPT54_GPT55_COMPACTION_THRESHOLD = 0.85
  489. # gpt-5.3-codex-spark: Codex-OAuth-only, native 128K; 70% (~90K) leaves summary headroom.
  490. _CODEX_SPARK_COMPACTION_THRESHOLD = 0.70
  491. def _is_codex_gpt54_or_gpt55(model: Optional[str], provider: Optional[str] = None) -> bool:
  492. """True for gpt-5.4/5.5/5.6, gpt-6 Sol/Terra/Luna, gpt-6 Astra (and the Daybreak Sol alias) on the Codex OAuth route only.
  493. Other routes expose a larger window for the same slug and keep the user's threshold.
  494. Prefix-matched so ``-pro`` and dated snapshots track every 272K-capped family; ``-900k``
  495. picker variants are excluded. Astra is substring-matched (any slug containing ``astra``
  496. without ``900k``). Name kept for the ``compression.codex_gpt55_autoraise`` key.
  497. """
  498. bare = _codex_route_bare_model(model, provider)
  499. if bare is None:
  500. return False
  501. from agent.model_metadata import is_codex_context_variant
  502. if is_codex_context_variant(bare):
  503. return False
  504. if "astra" in bare:
  505. return "900k" not in bare
  506. return bare == "gpt-daybreak-blue-latest" or any(
  507. bare == fam or bare.startswith(fam + "-") or bare.startswith(fam + ".")
  508. for fam in ("gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-6-sol", "gpt-6.1-sol", "gpt-6-luna"))
  509. def _codex_route_bare_model(model: Optional[str], provider: Optional[str]) -> Optional[str]:
  510. """Lowercased bare model slug when ``provider`` is the Codex OAuth route, else None."""
  511. return _bare_model(model) if (provider or "").strip().lower() == "openai-codex" else None
  512. def _is_codex_spark(model: Optional[str], provider: Optional[str] = None) -> bool:
  513. """True for ``gpt-5.3-codex-spark`` on the Codex OAuth route (the slug exists nowhere else)."""
  514. return _codex_route_bare_model(model, provider) == "gpt-5.3-codex-spark"
  515. def _is_openai_default_temperature_only(model: Optional[str]) -> bool:
  516. """True for OpenAI reasoning families that 400 (``unsupported_value``) on any non-default
  517. ``temperature``: gpt-5.x (incl. dated snapshots, ``-pro``, ``-codex``), o1/o3/o4. The
  518. ``gpt-5-chat`` non-reasoning line still accepts it (#51083)."""
  519. bare = _bare_model(model)
  520. return bare.startswith(("gpt-5", "o1", "o3", "o4")) and not bare.startswith("gpt-5-chat")
  521. # Routes (host + model) that rejected ``temperature`` at runtime; the next call omits it up front
  522. # instead of paying the 400 round-trip again (the retry alone left #51083's first call to time out).
  523. _TEMPERATURE_REJECTED_ROUTES: set = set()
  524. def remember_temperature_rejection(
  525. provider: Optional[str], base_url: Optional[str], rejected_kwargs: dict[str, Any], error: BaseException,
  526. ) -> None:
  527. _TEMPERATURE_REJECTED_ROUTES.add((_route_key(provider, base_url), _bare_model(rejected_kwargs.get("model"))))
  528. def _fixed_temperature_for_model(
  529. model: Optional[str], base_url: Optional[str] = None, provider: Optional[str] = None,
  530. ) -> Optional[float] | object:
  531. """``OMIT_TEMPERATURE`` (drop the key; Kimi/Moonshot, OpenAI reasoning families, Gemini 3.6+, routes
  532. that already rejected it), a fixed ``float``, or ``None``."""
  533. if _is_kimi_model(model):
  534. logger.debug("Omitting temperature for Kimi model %r (server-managed)", model)
  535. return OMIT_TEMPERATURE
  536. from agent.gemini_native_adapter import gemini_sampling_deprecated
  537. if _is_openai_default_temperature_only(model) or gemini_sampling_deprecated(_bare_model(model)):
  538. logger.debug("Omitting temperature for %r (default-only, or Gemini 3.6+ ignores sampling)", model)
  539. return OMIT_TEMPERATURE
  540. if (_route_key(provider, base_url), _bare_model(model)) in _TEMPERATURE_REJECTED_ROUTES:
  541. logger.debug("Omitting temperature for %r (route rejected it earlier)", model)
  542. return OMIT_TEMPERATURE
  543. return 0.5 if _is_arcee_trinity_thinking(model) else None
  544. def _compression_threshold_for_model(
  545. model: Optional[str], provider: Optional[str] = None, *,
  546. allow_codex_gpt55_autoraise: bool = True,
  547. ) -> Optional[float]:
  548. """Per-model/route compression threshold override (fraction of context used), or None.
  549. Arcee Trinity Large Thinking → 0.75 (preserve reasoning context); Codex-route gpt-5.4/5.5/5.6/Astra
  550. → 0.85, gated by ``allow_codex_gpt55_autoraise``; Codex-route gpt-5.3-codex-spark → 0.70, ungated.
  551. """
  552. if _is_arcee_trinity_thinking(model):
  553. return 0.75
  554. if allow_codex_gpt55_autoraise and _is_codex_gpt54_or_gpt55(model, provider):
  555. return _CODEX_GPT54_GPT55_COMPACTION_THRESHOLD
  556. if _is_codex_spark(model, provider):
  557. return _CODEX_SPARK_COMPACTION_THRESHOLD
  558. return None
  559. # Aux "fast tier" families, fastest first (measured p50 titling latency). Matched as substrings
  560. # against the LIVE /v1/models catalog because pinned ids rot; rolling "-latest" aliases lead.
  561. _FAST_MODEL_FAMILIES: tuple = (
  562. "gpt-mini-latest", "gpt-nano-latest", "claude-haiku-latest", "gemini-flash-latest",
  563. "gpt-5.4-nano", "gpt-5.4-mini", "gpt-5-mini", "haiku-4.5", "gemini-3.6-flash", "flash-lite",
  564. "-nano", "-mini", "-flash", "haiku",
  565. )
  566. # Disqualifiers: reasoning variants think before answering; ":batch" is a queue; ":free" tiers
  567. # are rate-limited and slowest; embedders/modality endpoints match a rung but cannot answer.
  568. _FAST_MODEL_EXCLUDE: tuple = (
  569. "thinking", "reason", "-r1", "minilm", ":batch", ":free",
  570. "o1-", "o3-", "o4-", "codex", "audio", "-vl", "embed",
  571. "-tts", "-transcribe", "-realtime", "-image", "-search-preview",
  572. )
  573. def _model_recency_key(model_id: str) -> tuple:
  574. """Sort key putting a family's newest release first: digit runs compare numerically (plain
  575. string order picks ``gpt-3.5-mini`` over ``gpt-5.4-mini`` and breaks at 9 vs 10)."""
  576. # re.split with one capturing group alternates text, number, text, …
  577. return tuple(
  578. (1, float(part), "") if index % 2 else (0, 0.0, part)
  579. for index, part in enumerate(re.split(r"(\d+(?:\.\d+)?)", model_id.lower())) if part)
  580. def _fast_model_from_catalog(provider_id: str) -> str:
  581. """Newest ``_FAST_MODEL_FAMILIES`` match from the provider's live (cached) catalog.
  582. "" when the catalog is unavailable or holds no small model (caller falls through to the
  583. curated default). Never raises; the fetch is memory+disk cached.
  584. """
  585. is_nous = provider_id.strip().lower() == "nous"
  586. try:
  587. from hermes_cli.auth import resolve_api_key_provider_credentials
  588. from hermes_cli.models_pricing import fetch_models_with_pricing
  589. from providers import get_provider_profile
  590. # Most /v1/models endpoints are authenticated; an anonymous 401 would read as "no small
  591. # model" and pin the curated default forever.
  592. api_key, base_url = "", ""
  593. try:
  594. creds = resolve_api_key_provider_credentials(provider_id) or {}
  595. api_key = str(creds.get("api_key", "")).strip()
  596. base_url = str(creds.get("base_url", "")).strip()
  597. except Exception:
  598. # Not an API-key provider, or nothing configured; anonymous fetch may still work.
  599. logger.debug("No credentials for %s catalog", provider_id, exc_info=True)
  600. if not api_key and is_nous:
  601. # Nous is OAuth (resolver raises); anonymous reads return the full catalog.
  602. try:
  603. from hermes_cli.models_pricing import _resolve_nous_pricing_credentials
  604. api_key, base_url = _resolve_nous_pricing_credentials()
  605. except Exception:
  606. logger.debug("No Nous credentials for catalog", exc_info=True)
  607. if not base_url:
  608. base_url = str(getattr(get_provider_profile(provider_id), "base_url", "") or "")
  609. base_url = base_url.rstrip("/")
  610. if not base_url:
  611. return ""
  612. base_url = base_url.removesuffix("/v1")
  613. # Nous-only args must match the pickers' or the seeded cache loses sale chrome and
  614. # policy-catalog expiry.
  615. _nous_kwargs = {}
  616. if is_nous:
  617. from hermes_cli.models_pricing import _NOUS_CATALOG_TTL_SECONDS
  618. _nous_kwargs = {"include_sale_original": True, "cache_ttl_seconds": _NOUS_CATALOG_TTL_SECONDS}
  619. catalog = fetch_models_with_pricing(
  620. api_key=api_key or None, base_url=base_url, timeout=3.0, **_nous_kwargs) or {}
  621. except Exception:
  622. logger.debug("Fast-model catalog lookup failed for %s", provider_id, exc_info=True)
  623. return ""
  624. ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True)
  625. if is_nous:
  626. # Narrow catalog ids by org policy, as the pickers do.
  627. try:
  628. from hermes_cli.models_pricing import nous_policy_allowed_ids, restrict_to_nous_policy
  629. ids = restrict_to_nous_policy(ids, nous_policy_allowed_ids())
  630. except Exception:
  631. logger.debug("Nous policy filter unavailable", exc_info=True)
  632. for family in _FAST_MODEL_FAMILIES:
  633. for model_id in ids:
  634. lowered = model_id.lower()
  635. if family in lowered and not any(x in lowered for x in _FAST_MODEL_EXCLUDE):
  636. return model_id
  637. return ""
  638. # Default auxiliary models for direct API-key providers (cheap/fast for side tasks)
  639. def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) -> str:
  640. """Cheap auxiliary model for a provider.
  641. Ladder: (``prefer_fast`` only) live-catalog family match, then ``ProviderProfile.resolve_aux_model``;
  642. then ``default_aux_model`` (curated); then the legacy dict. ``prefer_fast`` is opt-in (titling)
  643. so other callers keep their static behaviour and cache keys.
  644. """
  645. profile = None
  646. with contextlib.suppress(Exception):
  647. from providers import get_provider_profile
  648. profile = get_provider_profile(provider_id)
  649. picked = ""
  650. if prefer_fast:
  651. picked = _fast_model_from_catalog(provider_id)
  652. if not picked and profile is not None:
  653. try:
  654. picked = profile.resolve_aux_model() or ""
  655. except Exception:
  656. logger.debug("resolve_aux_model failed for %s", provider_id, exc_info=True)
  657. if not picked and profile is not None and profile.default_aux_model:
  658. picked = profile.default_aux_model
  659. if not picked:
  660. picked = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "")
  661. # Rungs 2-4 are policy-blind; a blocked pick is refused at request time, so drop it and
  662. # let the caller keep the main model.
  663. if picked and provider_id.strip().lower() == "nous":
  664. try:
  665. from hermes_cli.models_pricing import nous_policy_allowed_ids, restrict_to_nous_policy
  666. allowed = nous_policy_allowed_ids()
  667. if allowed and not restrict_to_nous_policy([picked], allowed):
  668. return ""
  669. except Exception:
  670. logger.debug("Nous policy check unavailable", exc_info=True)
  671. return picked
  672. # Fallback for providers without ProviderProfile.default_aux_model (plus some pinned here).
  673. # New providers should set default_aux_model instead.
  674. _API_KEY_PROVIDER_AUX_MODELS_FALLBACK: dict[str, str] = {
  675. "gemini": "gemini-3.6-flash", "zai": "glm-4.5-flash", "kimi-coding": "kimi-k2-turbo-preview",
  676. "stepfun": "step-3.5-flash", "kimi-coding-cn": "kimi-k2-turbo-preview",
  677. "gmi": "google/gemini-3.1-flash-lite-preview", "anthropic": "claude-haiku-4-5-20251001",
  678. "ai-gateway": "google/gemini-3-flash", "opencode-zen": "gemini-3-flash", "opencode-go": "glm-5",
  679. "kilocode": "google/gemini-3.6-flash", "ollama-cloud": "nemotron-3-nano:30b",
  680. "tencent-tokenhub": "hy4-preview", "tencent-tokenplan": "hy4-preview",
  681. # No "deepinfra": its aux model lives on the ProviderProfile (read first).
  682. }
  683. # Legacy alias for callers not yet using _get_aux_model_for_provider().
  684. _API_KEY_PROVIDER_AUX_MODELS: dict[str, str] = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK
  685. # Tasks that may opt into ``auxiliary.<task>.prefer_fast_model``.
  686. _FAST_MODEL_TASKS: frozenset = frozenset({"title_generation"})
  687. def _task_prefers_fast_model(task: Optional[str]) -> bool:
  688. """Return whether an eligible task explicitly opts into fast-model routing."""
  689. return task in _FAST_MODEL_TASKS and is_truthy_value(
  690. _get_auxiliary_task_config(task).get("prefer_fast_model"), default=False)
  691. # Dedicated vision models for direct providers whose main chat model differs. zai: glm-5.3-flash
  692. # is the only image-capable GLM id served on every Z.AI surface (pay-as-you-go and Coding Plan,
  693. # global and CN); the former glm-5v-turbo pin 404s / 1211 "Unknown Model" on the coding endpoints
  694. # (#111429). ZaiProfile has no default_vision_model(), so dropping the pin would route vision to
  695. # the user's text-only chat model and skip Z.AI entirely.
  696. _PROVIDER_VISION_MODELS: dict[str, str] = {"xiaomi": "mimo-v2.5", "zai": "glm-5.3-flash"}
  697. def _resolve_provider_vision_default(provider: str) -> Optional[str]:
  698. """Provider default vision model id, or None: static ``_PROVIDER_VISION_MODELS`` (vision-only
  699. names absent from any catalog) win, else ``ProviderProfile.default_vision_model()``."""
  700. static = _PROVIDER_VISION_MODELS.get(provider)
  701. if static:
  702. return static
  703. try:
  704. from providers import get_provider_profile
  705. profile = get_provider_profile(provider)
  706. return profile.default_vision_model() if profile is not None else None
  707. except Exception:
  708. return None
  709. # Endpoints that reject image input: vision auto-detect skips these to the aggregator chain
  710. # instead of returning a client that 404s (Kimi Coding Plan Anthropic wire has no image_in).
  711. _PROVIDERS_WITHOUT_VISION: frozenset = frozenset({"kimi-coding", "kimi-coding-cn"})
  712. # OpenRouter app attribution (always sent). `X-Title` is what the dashboard reads.
  713. _OR_HEADERS_BASE = {
  714. "HTTP-Referer": "https://hermes-agent.nousresearch.com",
  715. "X-Title": "Hermes Agent",
  716. "X-OpenRouter-Categories": "productivity,cli-agent",
  717. }
  718. def _apply_user_default_headers(headers: dict | None) -> dict | None:
  719. """Merge user ``model.default_headers`` onto resolved headers (user wins; ``model.extra_headers``
  720. alias wins over both). Mirrors ``AIAgent._apply_user_default_headers`` so a custom endpoint behind a
  721. WAF rejecting ``User-Agent`` / ``X-Stainless-*`` works for aux calls. SECURITY: never log values."""
  722. try:
  723. from hermes_cli.config import cfg_get, load_config
  724. _cfg = load_config()
  725. user_headers = cfg_get(_cfg, "model", "default_headers")
  726. alias_headers = cfg_get(_cfg, "model", "extra_headers")
  727. if isinstance(alias_headers, dict) and alias_headers:
  728. user_headers = {**(user_headers if isinstance(user_headers, dict) else {}), **alias_headers}
  729. except Exception:
  730. return headers
  731. if not isinstance(user_headers, dict) or not user_headers:
  732. return headers
  733. merged = dict(headers or {})
  734. merged.update({str(k): str(v) for k, v in user_headers.items() if v is not None})
  735. return merged or headers
  736. def build_or_headers(or_config: dict | None = None) -> dict:
  737. """OpenRouter headers, plus response-cache headers when enabled.
  738. Precedence env > config > default: ``HERMES_OPENROUTER_CACHE`` overrides
  739. ``openrouter.response_cache``; ``HERMES_OPENROUTER_CACHE_TTL`` (1-86400 s) overrides
  740. ``openrouter.response_cache_ttl``. ``or_config=None`` reads from disk.
  741. """
  742. headers = dict(_OR_HEADERS_BASE)
  743. if or_config is None:
  744. try:
  745. from hermes_cli.config import load_config_readonly
  746. or_config = load_config_readonly().get("openrouter", {})
  747. except Exception:
  748. or_config = {}
  749. env_cache = os.environ.get("HERMES_OPENROUTER_CACHE", "").strip().lower()
  750. if not (env_cache in {"1", "true", "yes", "on"} if env_cache else or_config.get("response_cache", False)):
  751. return headers
  752. headers["X-OpenRouter-Cache"] = "true"
  753. env_ttl = os.environ.get("HERMES_OPENROUTER_CACHE_TTL", "").strip()
  754. if env_ttl:
  755. if env_ttl.isdigit() and 1 <= int(env_ttl) <= 86400:
  756. headers["X-OpenRouter-Cache-TTL"] = str(int(env_ttl))
  757. else:
  758. ttl = or_config.get("response_cache_ttl", 300)
  759. if isinstance(ttl, (int, float)) and 1 <= ttl <= 86400:
  760. headers["X-OpenRouter-Cache-TTL"] = str(int(ttl))
  761. return headers
  762. # NVIDIA NIM cloud billing attribution; host-gated because NVIDIA_BASE_URL may be a local NIM.
  763. _NVIDIA_NIM_CLOUD_HEADERS = {"X-BILLING-INVOKE-ORIGIN": "HermesAgent"}
  764. def build_nvidia_nim_headers(base_url: str | None) -> dict:
  765. """Return NVIDIA NIM cloud attribution headers for build.nvidia.com traffic."""
  766. return dict(_NVIDIA_NIM_CLOUD_HEADERS) if base_url_host_matches(str(base_url or ""), "integrate.api.nvidia.com") else {}
  767. # Vercel AI Gateway attribution (HTTP-Referer → referrerUrl, X-Title → appName).
  768. from hermes_cli.version_info import hermes_user_agent
  769. def _ai_gateway_headers() -> dict[str, str]:
  770. return {"HTTP-Referer": "https://hermes-agent.nousresearch.com", "X-Title": "Hermes Agent",
  771. "User-Agent": hermes_user_agent("HermesAgent")}
  772. # Nous Portal attribution tags, built per call. They come from agent.portal_tags so the client=
  773. # marker tracks the canonical base version — never inline a literal here.
  774. from agent.portal_tags import nous_portal_tags as _nous_portal_tags
  775. # _OPENROUTER_MODEL MUST stay a :free SKU (matching the free_only warning): this lane engages
  776. # silently, and a paid default meant spend the user never opted into. User-configured values
  777. # are honored untouched (_warn_paid_lane_once fires).
  778. _OPENROUTER_MODEL = "nvidia/nemotron-3-ultra-550b-a55b:free"
  779. _NOUS_MODEL = "google/gemini-3.6-flash"
  780. _NOUS_DEFAULT_BASE_URL = "https://inference-api.nousresearch.com/v1"
  781. _ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com"
  782. _AUTH_JSON_PATH = get_hermes_home() / "auth.json"
  783. _AUTH_JSON_PATH_AT_IMPORT = _AUTH_JSON_PATH
  784. def _auth_json_path():
  785. """Active profile's ``auth.json`` at call time (a patched ``_AUTH_JSON_PATH`` still wins). The
  786. import-time constant is the LAUNCH profile's; under multiplexing a secondary's auxiliary calls
  787. would otherwise authenticate to Nous with the default profile's token."""
  788. from hermes_cli.auth import _auth_file_path
  789. return _AUTH_JSON_PATH if _AUTH_JSON_PATH != _AUTH_JSON_PATH_AT_IMPORT else _auth_file_path()
  790. # Hosts exposing BOTH ``…/anthropic`` and a sibling OpenAI ``…/v1``. Matched on the URL *host*
  791. # only: unconditional rewrites break Anthropic-only gateways.
  792. _DUAL_SURFACE_ANTHROPIC_HOST_SUFFIXES = ("minimax.io", "minimax.chat", "minimaxi.com")
  793. _DUAL_SURFACE_ANTHROPIC_HOST_PREFIXES = ("api.minimax.",)
  794. def _is_dual_surface_anthropic_host(url: str) -> bool:
  795. """True when the URL's host is a known dual-surface (MiniMax-family) host."""
  796. try:
  797. host = (urlparse(url).hostname or "").lower()
  798. except ValueError:
  799. return False
  800. return any(
  801. host == suffix or host.endswith("." + suffix) for suffix in _DUAL_SURFACE_ANTHROPIC_HOST_SUFFIXES
  802. ) or any(host.startswith(prefix) for prefix in _DUAL_SURFACE_ANTHROPIC_HOST_PREFIXES)
  803. def _to_openai_base_url(base_url: str) -> str:
  804. """Normalize dual-surface Anthropic URLs to their OpenAI-compatible sibling.
  805. MiniMax-family: ``/anthropic`` → ``/v1``; ZAI Coding Plan → ``/coding/paas/v4`` (the general
  806. endpoint bills separately); Kimi Code ``/coding`` → ``/coding/v1`` (the OpenAI SDK path 404s
  807. without it). Anthropic-only gateways keep their path.
  808. """
  809. url = str(base_url or "").strip().rstrip("/")
  810. if base_url_hostname(url) == "api.actual.inc":
  811. from hermes_cli.auth import normalize_actual_base_url
  812. return normalize_actual_base_url(url)
  813. if url.endswith("/anthropic"):
  814. if base_url_host_matches(url, "open.bigmodel.cn") or base_url_host_matches(url, "api.z.ai"):
  815. rewritten = url[: -len("/anthropic")] + "/coding/paas/v4"
  816. logger.debug("Auxiliary client: rewrote ZAI base URL %s → %s", url, rewritten)
  817. return rewritten
  818. if _is_dual_surface_anthropic_host(url):
  819. rewritten = url[: -len("/anthropic")] + "/v1"
  820. logger.debug("Auxiliary client: rewrote dual-surface base URL %s → %s", url, rewritten)
  821. return rewritten
  822. logger.debug(
  823. "Auxiliary client: keeping Anthropic-only base URL %s (no dual-surface host match)", url)
  824. return url
  825. if base_url_host_matches(url, "api.kimi.com") and url.endswith("/coding"):
  826. rewritten = url + "/v1"
  827. logger.debug("Auxiliary client: rewrote Kimi base URL %s → %s", url, rewritten)
  828. return rewritten
  829. return url
  830. def _load_pool_with_credentials(provider: str, note: str = "") -> Optional[Any]:
  831. """``load_pool(provider)`` when it has credentials, else None (never raises)."""
  832. try:
  833. pool = load_pool(provider)
  834. except Exception as exc:
  835. logger.debug("Auxiliary client: could not load pool for %s%s: %s", provider, note, exc)
  836. return None
  837. return pool if pool and pool.has_credentials() else None
  838. def _select_pool_entry(provider: str) -> tuple[bool, Optional[Any]]:
  839. """Return (pool_exists_for_provider, selected_entry)."""
  840. pool = _load_pool_with_credentials(provider)
  841. if pool is None:
  842. return False, None
  843. try:
  844. return True, pool.select()
  845. except Exception as exc:
  846. logger.debug("Auxiliary client: could not select pool entry for %s: %s", provider, exc)
  847. return True, None
  848. def _peek_pool_entry(provider: str, pool: Any = None) -> Optional[Any]:
  849. """Best-effort current/next pool entry without mutating selection order.
  850. ``pool`` skips the disk re-read when the caller already loaded it.
  851. """
  852. if pool is None:
  853. pool = _load_pool_with_credentials(provider, " (peek)")
  854. if pool is None:
  855. return None
  856. try:
  857. current_fn = getattr(pool, "current", None)
  858. current = current_fn() if callable(current_fn) else None
  859. if current is not None:
  860. return current
  861. peek_fn = getattr(pool, "peek", None)
  862. if callable(peek_fn):
  863. return peek_fn()
  864. except Exception as exc:
  865. logger.debug("Auxiliary client: could not peek pool entry for %s: %s", provider, exc)
  866. return None
  867. def _pool_runtime_api_key(entry: Any) -> str:
  868. # runtime_api_key handles provider-specific fallback (e.g. agent_key for nous); None entry → "".
  869. key = getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "")
  870. return str(key or "").strip()
  871. def _pool_runtime_base_url(entry: Any, fallback: str = "") -> str:
  872. if entry is None:
  873. return str(fallback or "").strip().rstrip("/")
  874. if getattr(entry, "provider", None) == "nous":
  875. # Canonical auth-layer reader so the env override shares one normalization path.
  876. from hermes_cli.auth import _nous_inference_env_override
  877. env_url = _nous_inference_env_override()
  878. if env_url:
  879. return env_url
  880. # runtime_base_url is provider-aware; fall back for non-PooledCredential entries.
  881. url = (getattr(entry, "runtime_base_url", None) or getattr(entry, "inference_base_url", None)
  882. or getattr(entry, "base_url", None) or fallback)
  883. return str(url or "").strip().rstrip("/")
  884. # Hosts the aux Anthropic path may be pointed at via model.base_url; anything else falls back
  885. # to the Anthropic default so a foreign host never leaks in.
  886. _ANTHROPIC_COMPATIBLE_HOSTS = frozenset({"api.anthropic.com"})
  887. def _is_anthropic_compatible_host(url: str) -> bool:
  888. """True for native Anthropic hosts and gateways serving Messages under a ``/anthropic`` path
  889. (same convention as runtime_provider / ``_wrap_if_needed``), so a configured ``model.base_url``
  890. whose gateway holds auth is not discarded. A bare non-Anthropic base_url is False."""
  891. if not url:
  892. return False
  893. try:
  894. parsed = urlparse(url)
  895. if (parsed.hostname or "").strip().lower().rstrip(".") in _ANTHROPIC_COMPATIBLE_HOSTS:
  896. return True
  897. path = (parsed.path or "").rstrip("/").lower()
  898. return path.endswith("/anthropic") or path.endswith("/anthropic/v1")
  899. except Exception:
  900. return False
  901. def _nous_min_key_ttl_seconds() -> int:
  902. try:
  903. return max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800")))
  904. except (TypeError, ValueError):
  905. return 1800
  906. def _scoped_key_env(name: str) -> str:
  907. """Read a provider API key (or its paired base-URL) env var through the profile secret scope.
  908. In agent turns the scope's verdict is authoritative (a scoped miss must not borrow another
  909. profile's key); only the unscoped default-profile path (``UnscopedSecretError``) reads
  910. ``os.environ`` -- any other scope failure propagates instead of borrowing the ambient env.
  911. """
  912. if not name:
  913. return ""
  914. from agent.secret_scope import UnscopedSecretError, get_secret
  915. try:
  916. return (get_secret(name) or "").strip()
  917. except UnscopedSecretError:
  918. return (os.getenv(name) or "").strip()
  919. def _attempt_stream_socket(stream: Any) -> Any:
  920. """The raw socket under an SDK event stream (``stream.response`` is the ``httpx.Response``;
  921. httpcore publishes its connection as the ``network_stream`` extension), or None."""
  922. from agent.agent_runtime_helpers import _socket_from_stream
  923. extensions = getattr(getattr(stream, "response", None), "extensions", None)
  924. network_stream = extensions.get("network_stream") if isinstance(extensions, dict) else None
  925. return _socket_from_stream(network_stream) if network_stream is not None else None
  926. def _close_quietly(target: Any, failure_note: Optional[str]) -> None:
  927. """Call ``target.close()`` if present; a failure is debug-logged under ``failure_note`` (silent when None)."""
  928. close = getattr(target, "close", None)
  929. if callable(close):
  930. try:
  931. close()
  932. except Exception:
  933. if failure_note:
  934. logger.debug("Codex auxiliary: %s", failure_note, exc_info=True)
  935. # The context compressor keys its retry-ladder classification on this text (#124077).
  936. CODEX_STREAM_STALL_MARKER = "stream stalled"
  937. class _CodexStreamGuard:
  938. """Progress-aware deadline + FD-safe timeout watchdog for one Codex aux stream attempt.
  939. (1) The first substantive payload must arrive within ``no_progress_timeout`` or we fail fast into
  940. the caller's retry/fallback chain (a dead or keepalive-only zombie must not hold the budget);
  941. (2) each substantive event re-arms that window (keepalive/lifecycle frames do NOT, mirroring
  942. commit-fence gating) so a live stream is never killed by an absolute total; (3) a hard ceiling
  943. from ``_aux_stream_total_ceiling`` still terminates a pathological drip.
  944. """
  945. def __init__(
  946. self, client: Any, total_timeout: Optional[float],
  947. no_progress_timeout: Optional[float] = None,
  948. ):
  949. self._client = client
  950. self.total_timeout = total_timeout
  951. self._start = time.monotonic()
  952. # Task-scoped override (auxiliary.<task>.no_progress_timeout, #108104); falls back to the
  953. # built-in default when unset or not a positive number.
  954. if isinstance(no_progress_timeout, (int, float)) and no_progress_timeout > 0:
  955. self.no_progress_timeout = float(no_progress_timeout)
  956. else:
  957. self.no_progress_timeout = _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS
  958. # Progress-aware stream deadlines (supersedes the old single absolute kill at ``total_timeout``).
  959. # Three regimes: 1. First token: the stream must produce its first substantive payload within
  960. # ``no_progress_timeout`` (60s default) or we fail fast and let the caller's normal retry/fallback
  961. # chain run — a dead (or keepalive-only zombie) Codex stream no longer holds the full 300s
  962. # compression budget before falling back (masoria report, Aug 2026: 3 stacked 300s waits -> 20+ min
  963. # stuck on "Summarizing"). 2. Streaming: every substantive event re-arms the deadline by
  964. # ``no_progress_timeout`` — a live stream is never killed by an absolute total, so a long reasoning
  965. # summary that is actually producing tokens completes instead of timing out at 300s and falling back
  966. # (#54915's original complaint, fixed properly). Keepalive/lifecycle frames do NOT re-arm, mirroring
  967. # the commit-fence progress gating (#96707). 3. Hard ceiling: an absolute backstop from
  968. # ``_aux_stream_total_ceiling`` (max(600s, 4x configured timeout) — the same bound the streamed
  969. # chat.completions path uses) so a pathological one-token-per-59s drip still terminates.
  970. if total_timeout is not None:
  971. self.no_progress_timeout = min(self.no_progress_timeout, float(total_timeout))
  972. self.hard_deadline = self._start + _aux_stream_total_ceiling(total_timeout)
  973. # The waiting host's absolute deadline clamps the ceiling so the watchdog Timer severs
  974. # the socket the instant the host stops waiting — a stream blocked between events
  975. # can't be stopped by a per-event check.
  976. host_deadline = _current_aux_stream_deadline()
  977. if isinstance(host_deadline, (int, float)) and host_deadline < self.hard_deadline:
  978. self.hard_deadline = float(host_deadline)
  979. self._deadline_lock = threading.Lock()
  980. self._progress_deadline = self._start + self.no_progress_timeout
  981. self.saw_content = threading.Event()
  982. self.timed_out = threading.Event()
  983. # Set only when the timeout WON (not when the owner hard-cancelled first): tells the
  984. # owner's ``finally`` the shared client's FDs still need a real close.
  985. self.timeout_release_pending = threading.Event()
  986. self.stream_finished = threading.Event()
  987. self._timer = None
  988. # The owner may return on hard cancel while this attempt is still blocked in the SDK
  989. # stream. Timer threads don't inherit the worker's thread-local protection state, so
  990. # freeze the hard-cancel source before creating the timer.
  991. self._protected_cancel_check = _capture_aux_cancel_check() if _aux_interrupt_protected() else None
  992. self._attempt_stream_lock = threading.Lock()
  993. self._attempt_stream: Any = None
  994. # The request-driving thread owns the transport FDs — see _close_client_on_timeout.
  995. self._owner_tid = threading.get_ident()
  996. def effective_deadline(self) -> float:
  997. with self._deadline_lock:
  998. return min(self.hard_deadline, self._progress_deadline)
  999. def cancel_requested(self) -> bool:
  1000. """True when the frozen hard-cancel source says the owner already cancelled."""
  1001. check = self._protected_cancel_check
  1002. return callable(check) and _captured_aux_cancel_requested(check)
  1003. def adopt_stream(self, stream: Any) -> None:
  1004. with self._attempt_stream_lock:
  1005. self._attempt_stream = stream
  1006. def release_stream(self, stream: Any) -> None:
  1007. """Owner-side: close the attempt stream silently and forget it."""
  1008. _close_quietly(stream, None)
  1009. with self._attempt_stream_lock:
  1010. self._attempt_stream = None
  1011. def close_attempt_stream(self, failure_note: str) -> None:
  1012. """Wake only this attempt's stream, never the shared client. The owner thread closes it;
  1013. any other thread only ``shutdown()``s its socket, since ``Stream.close()`` would release the FD
  1014. under the owner's ``SSL_read`` (#70773, #130115). Socketless streams are closed as before."""
  1015. with self._attempt_stream_lock:
  1016. stream = self._attempt_stream
  1017. if stream is None:
  1018. return
  1019. if threading.get_ident() != self._owner_tid:
  1020. sock = _attempt_stream_socket(stream)
  1021. if sock is not None:
  1022. from agent.agent_runtime_helpers import _shutdown_socket
  1023. _shutdown_socket(sock)
  1024. return
  1025. _close_quietly(stream, failure_note)
  1026. def record_progress(self) -> None:
  1027. """Substantive payload re-arms the no-progress window; the hard ceiling never moves."""
  1028. with self._deadline_lock:
  1029. self._progress_deadline = time.monotonic() + self.no_progress_timeout
  1030. def timeout_message(self) -> str:
  1031. elapsed = time.monotonic() - self._start
  1032. if time.monotonic() >= self.hard_deadline:
  1033. return f"Codex auxiliary Responses stream exceeded {self.hard_deadline - self._start:.1f}s hard ceiling"
  1034. if not self.saw_content.is_set():
  1035. return (
  1036. "Codex auxiliary Responses stream produced no output "
  1037. f"within {float(self.no_progress_timeout):.1f}s (no-progress timeout, {elapsed:.1f}s elapsed)")
  1038. return (
  1039. f"Codex auxiliary Responses {CODEX_STREAM_STALL_MARKER}: no new output "
  1040. f"for {float(self.no_progress_timeout):.1f}s ({elapsed:.1f}s elapsed)")
  1041. def _close_client_on_timeout(self) -> None:
  1042. begin_timeout_cleanup = getattr(self._protected_cancel_check, "begin_timeout_cleanup", None)
  1043. if callable(begin_timeout_cleanup):
  1044. timeout_won = bool(begin_timeout_cleanup())
  1045. else:
  1046. timeout_won = not self.cancel_requested()
  1047. # Publish transport timeout only after the attempt-local decision is fixed, so owner
  1048. # polling cannot observe completion in between.
  1049. self.timed_out.set()
  1050. if not timeout_won:
  1051. # Owner already hard-cancelled. The OpenAI client is process-shared, so never
  1052. # close/evict it here; wake only this attempt's stream if responses.create()
  1053. # returned one, else rely on the bounded SDK timeout.
  1054. self.close_attempt_stream("cancelled attempt stream close during timeout failed")
  1055. return
  1056. # FD-ownership contract: only the thread driving the request may ``close()`` this
  1057. # client's FDs. From a stranger thread (the watchdog Timer) only ``shutdown()`` is
  1058. # FD-safe — ``close()`` releases the raw TLS fd while the owner's OpenSSL BIO still
  1059. # caches it, the kernel recycles it (e.g. into a SQLite handle), and the owner's TLS
  1060. # flush corrupts that file. The owner does the real close in its ``finally``.
  1061. # This callback has two callers — ``_check_cancelled`` on the owning thread, and the daemon watchdog
  1062. # ``threading.Timer``, which is a stranger thread. The owning thread performs the real close in the
  1063. # ``finally`` below, which is where the FD release belongs. See #70773.
  1064. self.timeout_release_pending.set()
  1065. if threading.get_ident() == self._owner_tid:
  1066. _close_quietly(self._client, "client close during timeout failed")
  1067. else:
  1068. try:
  1069. from agent.agent_runtime_helpers import force_close_tcp_sockets
  1070. shutdown_count = force_close_tcp_sockets(self._client)
  1071. logger.info(
  1072. "Codex auxiliary client aborted (timeout, tcp_force_closed=%d, "
  1073. "deferred_close=stranger_thread)", shutdown_count)
  1074. except Exception:
  1075. logger.debug("Codex auxiliary: client abort during timeout failed", exc_info=True)
  1076. # Socket shutdown only wakes a reader on a REAL transport; the owner may be blocked
  1077. # inside the SDK's event stream (or a socketless test double). Wake the
  1078. # attempt-owned stream too — from this thread that is a shutdown of its socket,
  1079. # never a close (see close_attempt_stream).
  1080. self.close_attempt_stream("attempt stream close during stranger-thread timeout failed")
  1081. # The aux client cache wraps this same client; drop the entry so the next aux call
  1082. # doesn't reuse the dead transport and fail fast.
  1083. try:
  1084. # After we close the httpx transport above, the cache must drop that entry — otherwise the next
  1085. # auxiliary call (compression retry, memory flush, etc.) reuses the dead client and fails fast
  1086. # with a connection error. See issue #23432.
  1087. _evict_cached_client_instance(self._client)
  1088. except Exception:
  1089. logger.debug("Codex auxiliary: cache eviction on timeout failed", exc_info=True)
  1090. def check_cancelled(self) -> None:
  1091. if self.total_timeout is not None and time.monotonic() >= self.effective_deadline():
  1092. if not self.timed_out.is_set():
  1093. self._close_client_on_timeout()
  1094. raise TimeoutError(self.timeout_message())
  1095. try:
  1096. from tools.interrupt import is_interrupted
  1097. # Protected atomic aux tasks (compression) must not abort on a mid-flight gateway
  1098. # interrupt (degraded fallback marker); explicit host cancel has its own exception.
  1099. if _aux_interrupt_cancel_requested():
  1100. raise AuxiliaryExplicitCancellation()
  1101. # Explicit host cancellation has its own frozen exception; timeouts above still fire and other
  1102. # aux tasks remain interruptible. See #23975.
  1103. if is_interrupted() and not _aux_interrupt_protected():
  1104. raise InterruptedError("Codex auxiliary Responses stream interrupted")
  1105. except InterruptedError:
  1106. raise
  1107. except Exception:
  1108. # Interrupt state is best-effort UX; never a new failure mode.
  1109. pass
  1110. def _watchdog_fire(self) -> None:
  1111. # Re-armable: if progress moved the deadline forward, reschedule instead of killing a
  1112. # live stream.
  1113. remaining = self.effective_deadline() - time.monotonic()
  1114. if remaining > 0:
  1115. if not (self.timed_out.is_set() or self.stream_finished.is_set()):
  1116. self._arm_timer(remaining)
  1117. return
  1118. self._close_client_on_timeout()
  1119. def _arm_timer(self, delay: float) -> None:
  1120. self._timer = t = threading.Timer(delay, self._watchdog_fire)
  1121. t.daemon = True
  1122. t.start()
  1123. def start(self) -> None:
  1124. """Arm the watchdog (when a total timeout exists) and run the first cancel check."""
  1125. if self.total_timeout:
  1126. self._arm_timer(max(self.effective_deadline() - time.monotonic(), 0.0))
  1127. self.check_cancelled()
  1128. def on_event(self, _event: Any) -> None:
  1129. # TTFP telemetry records every frame, but forward progress (compression commit fence,
  1130. # no-progress window) counts only substantive payloads — keepalives must not re-arm,
  1131. # so a zombie stream dies at the same window as a dead connection.
  1132. # #93650: keep bulk wire-format payload out of the SDK's GIL-holding request transform on auxiliary
  1133. # calls too.
  1134. if _codex_event_has_content(_event):
  1135. self.record_progress()
  1136. self.saw_content.set()
  1137. _notify_aux_provider_response()
  1138. else:
  1139. _notify_aux_timing_response()
  1140. self.check_cancelled()
  1141. def finish(self) -> None:
  1142. """Owner ``finally``: stop the watchdog and release FDs a stranger-thread timeout only shut down."""
  1143. self.stream_finished.set()
  1144. if self._timer is not None:
  1145. self._timer.cancel()
  1146. # Gated on timeout_release_pending, NOT timed_out: after a hard-cancel the shared
  1147. # client must stay usable for other sessions.
  1148. if self.timeout_release_pending.is_set():
  1149. _close_quietly(self._client, "owner-thread close after timeout failed")
  1150. class _CodexCompletionsAdapter:
  1151. """Drop-in shim routing chat.completions.create() kwargs through Codex Responses streaming."""
  1152. def __init__(self, real_client: OpenAI, model: str):
  1153. self._client = real_client
  1154. self._model = model
  1155. def _build_responses_kwargs(self, kwargs: dict[str, Any]) -> tuple[dict[str, Any], str, Any]:
  1156. """chat.completions kwargs → Responses API kwargs, ``(resp_kwargs, model, timeout)``; mirrors codex.py::build_kwargs."""
  1157. # Separate system/instructions from replayable conversation messages, then route the rest through
  1158. # the SINGLE shared chat->Responses converter used by the main agent transport
  1159. # (agent/transports/codex.py). Maintaining a private conversion loop here let chat-style messages
  1160. # with role="tool" leak straight into Responses input[] — which the Responses API rejects with
  1161. # "Invalid value: 'tool'. Supported values are: 'assistant', 'system', 'developer', and 'user'."
  1162. # (issue #5709, hit hard by flush_memories() / compression replaying real session history that
  1163. # includes assistant tool_calls + role="tool" results). The shared converter encodes assistant tool
  1164. # calls as `function_call` items and tool results as `function_call_output` items with a valid
  1165. # call_id, so every Responses path normalizes tool history identically and cannot drift.
  1166. from agent.codex_responses_adapter import (
  1167. _chat_messages_to_responses_input,
  1168. _classify_responses_issuer,
  1169. _responses_tools,
  1170. _role_message_item,
  1171. _wire_model_identity,
  1172. classify_responses_route,
  1173. )
  1174. from agent.transports.codex import _alias_wire_tools
  1175. model = kwargs.get("model", self._model)
  1176. wire_model = _wire_model_identity(model)
  1177. host = str(getattr(self._client, "base_url", "") or "")
  1178. # Same route classifier as the main transport, so the issuer stamp matches what it minted.
  1179. route = classify_responses_route(SimpleNamespace(provider=None, base_url=host))
  1180. is_xai = route.is_xai_responses
  1181. tools = kwargs.get("tools")
  1182. if tools:
  1183. # xAI Responses rejects ``pattern``/``format`` JSON Schema keywords (400); strip for
  1184. # chat_completion_helpers.py parity. Deep-copy first — sanitizers mutate inner dicts
  1185. # in place and would strip the caller's tool registry.
  1186. try:
  1187. import copy as _copy
  1188. from tools.schema_sanitizer import strip_pattern_and_format, strip_slash_enum
  1189. tools = _copy.deepcopy(list(tools))
  1190. tools, _ = strip_pattern_and_format(tools)
  1191. tools, _ = strip_slash_enum(tools)
  1192. except Exception as exc:
  1193. logger.warning(
  1194. "Auxiliary client: failed to sanitize tool schemas for "
  1195. "Codex/xAI Responses path: %s", exc,
  1196. )
  1197. # Tool schemas go through the SAME converter (``strict: False``) and the SAME reserved-name
  1198. # aliasing (OpenCode / Perplexity / xAI) as agent/transports/codex.py::build_kwargs. A private
  1199. # list here sent raw names without ``strict``, so title/compression/MoA calls 400ed on routes
  1200. # where the main loop worked (#114260). The alias map is request-local: it rides on the payload
  1201. # (``_wire_aliases``, popped in ``create()``), never on the instance — aux adapters are shared.
  1202. wire_tools, wire_aliases = _alias_wire_tools(
  1203. _responses_tools(tools),
  1204. {"provider": getattr(self._client, "_hermes_aux_effective_provider", "") or None, "base_url": host},
  1205. is_xai,
  1206. )
  1207. renamed = {original: alias for alias, original in wire_aliases.items()}
  1208. # System → ``instructions``; the rest goes through the SINGLE shared chat→Responses
  1209. # converter (a private loop here once let role="tool" leak into input[]; the shared one
  1210. # encodes tool history as function_call/function_call_output).
  1211. instructions = "You are a helpful assistant."
  1212. replay_messages: list[dict[str, Any]] = []
  1213. for msg in kwargs.get("messages", []):
  1214. content = msg.get("content") or ""
  1215. if msg.get("role", "user") == "system":
  1216. instructions = content if isinstance(content, str) else str(content)
  1217. continue
  1218. # Replayed history names each tool the way THIS request declares it.
  1219. if renamed and msg.get("tool_calls"):
  1220. msg = {**msg, "tool_calls": [
  1221. {**tc, "function": {**tc["function"], "name": renamed[tc["function"]["name"]]}}
  1222. if isinstance(tc, dict) and (tc.get("function") or {}).get("name") in renamed else tc
  1223. for tc in msg["tool_calls"]
  1224. ]}
  1225. replay_messages.append(msg)
  1226. # Endpoints that bind replayed codex_message_items ids to a backend connection 401 on a stale id
  1227. # (``replay_item_ids=False``) — same guard as build_kwargs. Aux calls
  1228. # never send ``context_management`` (main-turn feature): no compaction checkpoint.
  1229. # Auxiliary calls (context compression, flush_memories, MoA aggregation) go through this adapter
  1230. # instead of agent/transports/codex.py's build_kwargs, so they need the same guard applied
  1231. # independently. See #32716.
  1232. # Aux requests run their own model; stamp/filter reasoning provenance against it, not the main agent's.
  1233. issuer_kind = _classify_responses_issuer(base_url=host, **route._asdict())
  1234. input_items = _chat_messages_to_responses_input(
  1235. replay_messages, drop_replay_item_ids=not route.endpoint.replay_item_ids,
  1236. current_issuer_kind=issuer_kind,
  1237. current_issuer_model=wire_model, native_compaction_eligible=False,
  1238. )
  1239. resp_kwargs: dict[str, Any] = {
  1240. # Codex only knows the base slug; strip the Hermes ``-900k`` picker suffix.
  1241. "model": wire_model, "instructions": instructions,
  1242. "input": input_items or [_role_message_item("user", "")], "store": False,
  1243. }
  1244. # Forward the chat.completions timeout; otherwise a Codex stream can sit behind a
  1245. # dead-looking CLI until the user force-interrupts.
  1246. timeout = kwargs.get("timeout")
  1247. if timeout is not None:
  1248. resp_kwargs["timeout"] = timeout
  1249. # Per-request HTTP headers (OpenCode session affinity, provider request headers) map to real
  1250. # headers via the SDK kwarg — forward them.
  1251. if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
  1252. resp_kwargs["extra_headers"] = dict(kwargs["extra_headers"])
  1253. # The Codex endpoint rejects max_output_tokens/temperature (400) — omit.
  1254. extra_body = kwargs.get("extra_body") or {}
  1255. if isinstance(extra_body, dict):
  1256. # service_tier (fast mode) is a top-level Responses field; xAI's endpoint rejects it.
  1257. service_tier = extra_body.get("service_tier")
  1258. if isinstance(service_tier, str) and service_tier.strip() and not is_xai:
  1259. resp_kwargs["service_tier"] = service_tier.strip()
  1260. reasoning_cfg = extra_body.get("reasoning")
  1261. if isinstance(reasoning_cfg, dict):
  1262. # Shared per-model vocabulary with the main transport ("max" only where the model publishes it; "minimal"/"ultra"
  1263. # clamp to a listed level; ``()`` = the model takes no ``reasoning`` field at all — gpt-4o/4.1 on api.openai.com,
  1264. # #76255). ``enabled: False`` goes on the wire as ``effort: none`` where the vocabulary has it,
  1265. # since an omitted field leaves the model's default effort on (#75227).
  1266. from agent.reasoning_effort import clamp_effort
  1267. from agent.transports.codex import _codex_efforts_for_route
  1268. supported = _codex_efforts_for_route(model, host, is_codex_backend=route.is_codex_backend)
  1269. if supported and reasoning_cfg.get("enabled") is not False:
  1270. # Truthy-only: Codex 400s on e.g. {"effort": null}, so falsy → default.
  1271. effort = clamp_effort(reasoning_cfg.get("effort") or "medium", supported)
  1272. resp_kwargs["reasoning"] = {"effort": effort, "summary": "auto"}
  1273. resp_kwargs["include"] = ["reasoning.encrypted_content"]
  1274. elif "none" in supported and not is_xai:
  1275. resp_kwargs["reasoning"] = {"effort": "none"}
  1276. if wire_tools:
  1277. resp_kwargs["tools"] = wire_tools
  1278. if wire_aliases:
  1279. resp_kwargs["_wire_aliases"] = wire_aliases
  1280. # Response normalization is route-sensitive too (popped in ``create()``, never sent).
  1281. resp_kwargs["_issuer_kind"] = issuer_kind
  1282. # Stable prompt-cache routing: key is content-addressed from the static prefix
  1283. # (instructions + tool schemas) so it survives across turns, scoped by the owning
  1284. # conversation (rotation-stable logical scope, else the physical session id). Skip the
  1285. # key where the main transport does: xAI takes it in extra_body, ``prompt_cache_key=False`` endpoints opt out.
  1286. try:
  1287. # Reuse the Responses transport's single authoritative hash algorithm and session-scope
  1288. # normalization so equivalent static prefixes route to the same cache bucket across modes,
  1289. # without concentrating unrelated sessions into one shared bucket (see #78941).
  1290. from agent.transports.codex import _cache_scope_from_session_id, _content_cache_key
  1291. from agent.transports.codex import _default_prompt_cache_retention_for_request
  1292. if not is_xai and route.endpoint.prompt_cache_key and "prompt_cache_key" not in resp_kwargs:
  1293. scope = _cache_scope_from_session_id(
  1294. _runtime_main_value("cache_scope") or _runtime_main_value("session_id")
  1295. )
  1296. cache_key = _content_cache_key(resp_kwargs["instructions"], resp_kwargs.get("tools"), scope)
  1297. if cache_key:
  1298. resp_kwargs["prompt_cache_key"] = cache_key
  1299. if "prompt_cache_retention" not in resp_kwargs:
  1300. cache_retention = _default_prompt_cache_retention_for_request(model, host)
  1301. if cache_retention:
  1302. resp_kwargs["prompt_cache_retention"] = cache_retention
  1303. except Exception:
  1304. logger.debug("Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True)
  1305. # Last, like the main transport: caller extra_body must not put a rejected Astra field back.
  1306. from agent.transports.codex import _sanitize_astra_request_kwargs
  1307. _sanitize_astra_request_kwargs(resp_kwargs, model, host)
  1308. return resp_kwargs, model, timeout
  1309. def create(self, **kwargs) -> Any:
  1310. from hermes_cli.providers import is_actual_route
  1311. if is_actual_route(
  1312. getattr(self._client, "_hermes_aux_effective_provider", ""),
  1313. str(getattr(self._client, "base_url", "") or ""),
  1314. ):
  1315. raise ValueError(
  1316. "Actual requests require Chat Completions; refusing to call /responses."
  1317. )
  1318. # Low-level ``responses.create(stream=True)`` and assemble the final response ourselves
  1319. # from ``response.output_item.done``: the high-level ``responses.stream()`` rebuilds from
  1320. # ``response.completed.response.output``, which Codex returns as ``null`` (SDK crash).
  1321. resp_kwargs, model, timeout = self._build_responses_kwargs(kwargs)
  1322. wire_aliases = resp_kwargs.pop("_wire_aliases", None) or {}
  1323. issuer_kind = resp_kwargs.pop("_issuer_kind", None)
  1324. issuer_model = str(resp_kwargs.get("model") or model)
  1325. total_timeout = timeout if isinstance(timeout, (int, float)) and timeout > 0 else None
  1326. guard = _CodexStreamGuard(self._client, total_timeout, no_progress_timeout=kwargs.get("no_progress_timeout"))
  1327. try:
  1328. guard.start()
  1329. from agent.codex_runtime import _consume_codex_event_stream
  1330. from agent.sdk_transform_bypass import bypass_sdk_request_transform
  1331. # Keep bulk wire payload out of the SDK's GIL-holding request transform.
  1332. stream_kwargs = bypass_sdk_request_transform({**resp_kwargs, "stream": True})
  1333. event_stream = self._client.responses.create(**stream_kwargs)
  1334. guard.adopt_stream(event_stream)
  1335. # The timer may fire while responses.create() is blocked; if the cancelled attempt
  1336. # had no stream to close then, close it now that it is attempt-owned — never the shared client.
  1337. if guard.timed_out.is_set() and guard.cancel_requested():
  1338. guard.close_attempt_stream("late cancelled attempt stream close failed")
  1339. try:
  1340. # Some Codex-compatible hosts accept ``stream=True`` but return a completed
  1341. # Responses object (not iterable) — don't hand it to the consumer.
  1342. if hasattr(event_stream, "output"):
  1343. final = event_stream
  1344. else:
  1345. final = _consume_codex_event_stream(
  1346. event_stream, model=issuer_model, on_event=guard.on_event
  1347. )
  1348. finally:
  1349. guard.release_stream(event_stream)
  1350. if final is None:
  1351. raise RuntimeError("Codex auxiliary Responses stream did not return a final response")
  1352. from agent.auxiliary_codex_response import _parse_codex_final_response
  1353. text_parts, tool_calls_raw, usage, finish_reason = _parse_codex_final_response(
  1354. final, issuer_kind=issuer_kind, issuer_model=issuer_model,
  1355. )
  1356. # Undo only the aliases THIS request emitted, before the call reaches Hermes dispatch.
  1357. for tc in tool_calls_raw or ():
  1358. if tc.function.name in wire_aliases:
  1359. tc.function.name = wire_aliases[tc.function.name]
  1360. except Exception as exc:
  1361. if guard.timed_out.is_set():
  1362. raise TimeoutError(guard.timeout_message()) from exc
  1363. logger.debug("Codex auxiliary Responses API call failed: %s", exc)
  1364. raise
  1365. finally:
  1366. guard.finish()
  1367. # Shape the result like chat.completions.
  1368. message = SimpleNamespace(
  1369. role="assistant", content="".join(text_parts).strip() or None,
  1370. tool_calls=tool_calls_raw or None,
  1371. )
  1372. choice = SimpleNamespace(index=0, message=message, finish_reason=finish_reason)
  1373. return SimpleNamespace(choices=[choice], model=model, usage=usage)
  1374. class _ChatShim:
  1375. """Exposes ``client.chat.completions.create()`` over a sync or async adapter."""
  1376. def __init__(self, adapter: Any):
  1377. self.completions = adapter
  1378. class _AsyncCompletionsAdapter:
  1379. """Async adapter: runs the sync adapter's ``create`` via asyncio.to_thread()."""
  1380. def __init__(self, sync_adapter: Any):
  1381. self._sync = sync_adapter
  1382. async def create(self, **kwargs) -> Any:
  1383. import asyncio
  1384. return await asyncio.to_thread(self._sync.create, **kwargs)
  1385. class _AsyncAuxiliaryClientBase:
  1386. """Async-compatible wrapper matching AsyncOpenAI.chat.completions.create().
  1387. Mirrors ``_real_client`` (when the sync wrapper has one) so cache eviction by
  1388. leaf OpenAI client drops this async entry too instead of reusing a closed transport.
  1389. """
  1390. def __init__(self, sync_wrapper: Any):
  1391. self.chat = _ChatShim(_AsyncCompletionsAdapter(sync_wrapper.chat.completions))
  1392. self.api_key = sync_wrapper.api_key
  1393. self.base_url = sync_wrapper.base_url
  1394. if hasattr(sync_wrapper, "_real_client"):
  1395. # Mirror the sync wrapper's _real_client so cache eviction by leaf OpenAI client (e.g.
  1396. # _close_client_on_timeout in #23482) drops this async entry too. Without this, sync and async
  1397. # cache entries diverge on poisoning: the sync entry is evicted but the async entry keeps
  1398. # reusing the closed transport, failing every subsequent async aux call with 'Connection error'
  1399. # until the gateway restarts.
  1400. self._real_client = sync_wrapper._real_client
  1401. _AsyncAnthropicCompletionsAdapter = _AsyncCompletionsAdapter # imported by tests
  1402. class CodexAuxiliaryClient:
  1403. """OpenAI-client-compatible wrapper routing through the Codex Responses API (.api_key/.base_url for introspection)."""
  1404. def __init__(self, real_client: OpenAI, model: str):
  1405. self._real_client = real_client
  1406. self.chat = _ChatShim(_CodexCompletionsAdapter(real_client, model))
  1407. self.api_key = real_client.api_key
  1408. self.base_url = real_client.base_url
  1409. def close(self):
  1410. self._real_client.close()
  1411. class AsyncCodexAuxiliaryClient(_AsyncAuxiliaryClientBase):
  1412. pass
  1413. def _translate_anthropic_response_format(anthropic_kwargs: dict[str, Any], response_format: Any) -> None:
  1414. """Merge an OpenAI response format into Anthropic ``output_config``."""
  1415. if not isinstance(response_format, dict):
  1416. return
  1417. format_type = response_format.get("type")
  1418. if format_type == "json_schema":
  1419. json_schema = response_format.get("json_schema")
  1420. if not isinstance(json_schema, dict) or "schema" not in json_schema:
  1421. return
  1422. schema = json_schema["schema"]
  1423. elif format_type == "json_object":
  1424. # Anthropic SDK has no schema-less JSON mode; only ``json_schema``.
  1425. schema = {"type": "object"}
  1426. else:
  1427. return
  1428. output_config = anthropic_kwargs.get("output_config")
  1429. if not isinstance(output_config, dict):
  1430. output_config = {}
  1431. anthropic_kwargs["output_config"] = output_config
  1432. output_config["format"] = {"type": "json_schema", "schema": schema}
  1433. class _AnthropicCompletionsAdapter:
  1434. """OpenAI-client-compatible adapter for Anthropic Messages API."""
  1435. def __init__(self, real_client: Any, model: str, is_oauth: bool = False, base_url: str | None = None):
  1436. self._client = real_client
  1437. self._model = model
  1438. self._is_oauth = is_oauth
  1439. # Caller URL first; fall back to the SDK client's host only for Nous Portal — a blanket
  1440. # fallback would flip MiniMax/Zhipu aux adapters to third-party handling (strips thinking sigs).
  1441. self._base_url = base_url or None
  1442. if not self._base_url:
  1443. candidate = str(getattr(real_client, "base_url", "") or "") or None
  1444. if candidate:
  1445. with contextlib.suppress(Exception):
  1446. from agent.anthropic_endpoints import _is_nous_portal_endpoint
  1447. if _is_nous_portal_endpoint(candidate):
  1448. self._base_url = candidate
  1449. def create(self, **kwargs) -> Any:
  1450. from agent.anthropic_adapter import build_anthropic_kwargs, create_anthropic_message
  1451. from agent.transports import get_transport
  1452. model = kwargs.get("model", self._model)
  1453. # ZAI's Anthropic endpoint rejects max_tokens on vision models (code 1210);
  1454. # callers signal this via _skip_zai_max_tokens.
  1455. if kwargs.pop("_skip_zai_max_tokens", False):
  1456. max_tokens = None
  1457. else:
  1458. max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
  1459. temperature = kwargs.get("temperature")
  1460. # Reasoning priority: explicit per-call _reasoning_config (MoA per-slot) wins over
  1461. # extra_body.reasoning; build_anthropic_kwargs translates to ``thinking``.
  1462. reasoning_cfg = kwargs.get("_reasoning_config")
  1463. if reasoning_cfg is None:
  1464. _eb = kwargs.get("extra_body")
  1465. _rc = _eb.get("reasoning") if isinstance(_eb, dict) else None
  1466. if isinstance(_rc, dict):
  1467. reasoning_cfg = _rc
  1468. # OpenAI tool_choice (str or dict) → Anthropic-style name/mode string.
  1469. tool_choice = kwargs.get("tool_choice")
  1470. if isinstance(tool_choice, dict):
  1471. choice_type = str(tool_choice.get("type", "")).lower()
  1472. if choice_type == "function":
  1473. tool_choice = tool_choice.get("function", {}).get("name")
  1474. else:
  1475. tool_choice = choice_type if choice_type in {"auto", "required", "none"} else None
  1476. elif not isinstance(tool_choice, str):
  1477. tool_choice = None
  1478. anthropic_kwargs = build_anthropic_kwargs(
  1479. model=model, messages=kwargs.get("messages", []), tools=kwargs.get("tools"),
  1480. max_tokens=max_tokens, reasoning_config=reasoning_cfg, tool_choice=tool_choice,
  1481. is_oauth=self._is_oauth,
  1482. # Portal routes on ``anthropic/<slug>`` ids and replays signed thinking
  1483. # keyed off base_url; omitting it breaks Portal model resolution.
  1484. base_url=self._base_url,
  1485. )
  1486. # Opus 4.7+ rejects non-default temperature/top_p/top_k; build_anthropic_kwargs
  1487. # also strips these as a safety net — keep both layers.
  1488. if temperature is not None:
  1489. from agent.anthropic_adapter import _forbids_sampling_params
  1490. if not _forbids_sampling_params(model):
  1491. anthropic_kwargs["temperature"] = temperature
  1492. # Per-request HTTP headers (OpenCode session affinity) — the Anthropic SDK accepts
  1493. # ``extra_headers`` on messages.create/stream too.
  1494. if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
  1495. anthropic_kwargs["extra_headers"] = {
  1496. **(anthropic_kwargs.get("extra_headers") or {}),
  1497. **kwargs["extra_headers"],
  1498. }
  1499. # response_format: top-level gets the same translation as the extra_body form; when both
  1500. # are present the extra_body form wins. Passthrough excludes ``reasoning``/``response_format``
  1501. # (already TRANSLATED to native fields — raw would 400 on strict gateways) and ``_`` Hermes plumbing.
  1502. # The adapter builds the Messages body from a fixed allow-list of kwargs, so before this an
  1503. # unrecognized top-level kwarg was dropped on the floor: the request succeeded but the schema
  1504. # contract silently became prompt compliance (#85626 review, point 2).
  1505. top_level_response_format = kwargs.get("response_format")
  1506. if top_level_response_format is not None:
  1507. _translate_anthropic_response_format(anthropic_kwargs, top_level_response_format)
  1508. caller_extra_body = kwargs.get("extra_body")
  1509. if caller_extra_body and isinstance(caller_extra_body, dict):
  1510. _translate_anthropic_response_format(anthropic_kwargs, caller_extra_body.get("response_format"))
  1511. passthrough = {
  1512. k: v for k, v in caller_extra_body.items()
  1513. if k not in {"reasoning", "response_format"} and not str(k).startswith("_")
  1514. }
  1515. if passthrough:
  1516. existing = anthropic_kwargs.get("extra_body") or {}
  1517. if not isinstance(existing, dict):
  1518. existing = {}
  1519. anthropic_kwargs["extra_body"] = {**existing, **passthrough}
  1520. response = create_anthropic_message(
  1521. self._client,
  1522. anthropic_kwargs,
  1523. # Record provider-response timing every event, but tick forward progress only for
  1524. # substantive payloads so keepalives can't hold a stalled summary open. None keeps
  1525. # the fast get_final_message path.
  1526. on_stream_event=(_anthropic_aux_stream_event_hook() if _aux_progress_active() else None),
  1527. )
  1528. _nr = get_transport("anthropic_messages").normalize_response(response, strip_tool_prefix=self._is_oauth)
  1529. usage = None
  1530. if hasattr(response, "usage") and response.usage:
  1531. prompt_tokens = getattr(response.usage, "input_tokens", 0) or 0
  1532. completion_tokens = getattr(response.usage, "output_tokens", 0) or 0
  1533. usage = SimpleNamespace(
  1534. prompt_tokens=prompt_tokens, completion_tokens=completion_tokens,
  1535. total_tokens=getattr(response.usage, "total_tokens", 0) or (prompt_tokens + completion_tokens),
  1536. )
  1537. # ToolCall already duck-types as OpenAI shape via properties.
  1538. choice = SimpleNamespace(
  1539. index=0,
  1540. message=SimpleNamespace(content=_nr.content, tool_calls=_nr.tool_calls, reasoning=_nr.reasoning),
  1541. finish_reason=_nr.finish_reason,
  1542. )
  1543. return SimpleNamespace(choices=[choice], model=model, usage=usage)
  1544. class AnthropicAuxiliaryClient:
  1545. """OpenAI-client-compatible wrapper over a native Anthropic client."""
  1546. def __init__(self, real_client: Any, model: str, api_key: str, base_url: str, is_oauth: bool = False):
  1547. self._real_client = real_client
  1548. self.chat = _ChatShim(_AnthropicCompletionsAdapter(real_client, model, is_oauth=is_oauth, base_url=base_url))
  1549. self.api_key = api_key
  1550. self.base_url = base_url
  1551. def close(self):
  1552. close_fn = getattr(self._real_client, "close", None)
  1553. if callable(close_fn):
  1554. close_fn()
  1555. class AsyncAnthropicAuxiliaryClient(_AsyncAuxiliaryClientBase):
  1556. pass
  1557. class _BedrockCompletionsAdapter:
  1558. """Translates ``chat.completions.create(**kwargs)`` into Bedrock Converse."""
  1559. def __init__(self, region: str, model: str):
  1560. self._region = region
  1561. self._model = model
  1562. def create(self, **kwargs) -> Any:
  1563. from agent.bedrock_adapter import call_converse
  1564. model = kwargs.get("model", self._model)
  1565. max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
  1566. # OpenAI accepts ``stop`` as str or list; Converse requires a list.
  1567. stop = kwargs.get("stop")
  1568. if isinstance(stop, str):
  1569. stop = [stop]
  1570. if kwargs.get("tool_choice") is not None:
  1571. # Converse toolChoice isn't wired through call_converse(); surface the drop.
  1572. logger.debug(
  1573. "BedrockAuxiliaryClient: tool_choice=%r not supported by the "
  1574. "Converse shim — ignored.", kwargs.get("tool_choice"),
  1575. )
  1576. if kwargs.get("stream"):
  1577. # Converse streaming isn't wired here; call_llm's streaming consumer
  1578. # detects a final object and downgrades to non-live output.
  1579. logger.debug(
  1580. "BedrockAuxiliaryClient: stream=True requested for %s — returning a complete response "
  1581. "(Converse shim does not stream); caller downgrades to non-streaming.", model,
  1582. )
  1583. response = call_converse(
  1584. region=self._region, model=model, messages=kwargs.get("messages", []), tools=kwargs.get("tools"),
  1585. # Converse specifically defaults to the model maximum when omitted.
  1586. # Truthiness mirrors the Anthropic shim: explicit 0 means omit.
  1587. max_tokens=int(max_tokens) if max_tokens else None, temperature=kwargs.get("temperature"),
  1588. top_p=kwargs.get("top_p"), stop_sequences=stop,
  1589. )
  1590. # Converse is complete-response here: mark provider progress only after
  1591. # return so TTFP reflects real Bedrock latency, not dispatch/setup.
  1592. _notify_aux_provider_response()
  1593. return response
  1594. class BedrockAuxiliaryClient:
  1595. """OpenAI-client-compatible wrapper over AWS Bedrock Converse API."""
  1596. def __init__(self, region: str, model: str):
  1597. self._region = region
  1598. self._model = model
  1599. self.chat = _ChatShim(_BedrockCompletionsAdapter(region, model))
  1600. self.api_key = "aws-sdk"
  1601. self.base_url = f"https://bedrock-runtime.{region}.amazonaws.com"
  1602. def close(self):
  1603. pass
  1604. class AsyncBedrockAuxiliaryClient(_AsyncAuxiliaryClientBase):
  1605. pass
  1606. def _endpoint_speaks_anthropic_messages(base_url: str) -> bool:
  1607. """True if ``base_url`` speaks Anthropic Messages, not OpenAI chat.completions.
  1608. Mirrors ``hermes_cli.runtime_provider._detect_api_mode_for_url`` so aux and main agree: any
  1609. ``/anthropic`` URL (MiniMax, Zhipu, LiteLLM), ``api.kimi.com/coding`` (chat 404s), ``api.anthropic.com``.
  1610. """
  1611. normalized = (base_url or "").strip().lower().rstrip("/")
  1612. if not normalized:
  1613. return False
  1614. if urlparse(normalized).path.rstrip("/").endswith(("/anthropic", "/anthropic/v1")):
  1615. return True
  1616. hostname = base_url_hostname(normalized)
  1617. return hostname == "api.anthropic.com" or bool(hostname == "api.kimi.com" and "/coding" in normalized)
  1618. def _maybe_wrap_anthropic(
  1619. client_obj: Any, model: str, api_key: str, base_url: str, api_mode: Optional[str] = None
  1620. ) -> Any:
  1621. """Rewrap a plain OpenAI client in ``AnthropicAuxiliaryClient`` when the endpoint speaks Anthropic Messages.
  1622. Single transport-correction chokepoint at the end of every ``resolve_provider_client`` branch; returns
  1623. ``client_obj`` unchanged for probe stubs/specialized adapters, OpenAI-wire, explicit non-Anthropic
  1624. ``api_mode``, or missing ``anthropic`` SDK.
  1625. """
  1626. # Anthropic/Bedrock/Codex wrappers, plus any client declaring HERMES_SKIP_TRANSPORT_WRAP
  1627. # (native/ACP shims, in-tree or plugin), must never be re-dispatched through a wire adapter —
  1628. # a class-attribute declaration rather than isinstance so this hot path never imports them.
  1629. if (
  1630. isinstance(client_obj, _AuxProbeClientStub)
  1631. or _safe_isinstance(client_obj, (AnthropicAuxiliaryClient, BedrockAuxiliaryClient, CodexAuxiliaryClient))
  1632. or _client_declares(client_obj, "HERMES_SKIP_TRANSPORT_WRAP")
  1633. ):
  1634. return client_obj
  1635. # Explicit non-anthropic api_mode wins over URL heuristics.
  1636. if api_mode != "anthropic_messages" and (api_mode or not _endpoint_speaks_anthropic_messages(base_url)):
  1637. return client_obj
  1638. try:
  1639. from agent.anthropic_adapter import build_anthropic_client
  1640. except ImportError:
  1641. logger.warning(
  1642. "Endpoint %s speaks Anthropic Messages but the anthropic SDK is "
  1643. "not installed — falling back to OpenAI-wire (will likely 404).",
  1644. base_url,
  1645. )
  1646. return client_obj
  1647. try:
  1648. real_client = build_anthropic_client(api_key, base_url)
  1649. except Exception as exc:
  1650. logger.warning(
  1651. "Failed to build Anthropic client for %s (%s) — falling back to "
  1652. "OpenAI-wire client.", base_url, exc,
  1653. )
  1654. return client_obj
  1655. logger.debug(
  1656. "Auxiliary transport: wrapping client in AnthropicAuxiliaryClient "
  1657. "(model=%s, base_url=%s, api_mode=%s)",
  1658. model, base_url[:60] if base_url else "", api_mode or "auto-detected",
  1659. )
  1660. return AnthropicAuxiliaryClient(real_client, model, api_key, base_url, is_oauth=False)
  1661. def _read_nous_auth() -> Optional[dict]:
  1662. """Nous provider state dict from the credential pool or ~/.hermes/auth.json; None when not active with tokens."""
  1663. pool_present, entry = _select_pool_entry("nous")
  1664. if pool_present:
  1665. if entry is None:
  1666. return None
  1667. return {
  1668. "access_token": getattr(entry, "access_token", ""),
  1669. "refresh_token": getattr(entry, "refresh_token", None),
  1670. "agent_key": getattr(entry, "agent_key", None),
  1671. "inference_base_url": _pool_runtime_base_url(entry, _NOUS_DEFAULT_BASE_URL),
  1672. "portal_base_url": getattr(entry, "portal_base_url", None),
  1673. "client_id": getattr(entry, "client_id", None),
  1674. "scope": getattr(entry, "scope", None),
  1675. "token_type": getattr(entry, "token_type", "Bearer"),
  1676. "source": "pool",
  1677. }
  1678. try:
  1679. auth_path = _auth_json_path()
  1680. if not auth_path.is_file():
  1681. return None
  1682. data = json.loads(auth_path.read_text(encoding="utf-8-sig"))
  1683. if data.get("active_provider") != "nous":
  1684. return None
  1685. provider = data.get("providers", {}).get("nous", {})
  1686. # Must have at least an access_token or agent_key.
  1687. if not provider.get("agent_key") and not provider.get("access_token"):
  1688. return None
  1689. return provider
  1690. except Exception as exc:
  1691. logger.debug("Could not read Nous auth: %s", exc)
  1692. return None
  1693. def _nous_api_key(provider: dict) -> str:
  1694. """Extract a usable Nous inference JWT from stored auth state."""
  1695. from hermes_cli.auth import _nous_invoke_jwt_is_usable
  1696. for token_key, expiry_key in (("agent_key", "agent_key_expires_at"), ("access_token", "expires_at")):
  1697. token = provider.get(token_key)
  1698. if not isinstance(token, str) or not token.strip():
  1699. continue
  1700. if _nous_invoke_jwt_is_usable(token, scope=provider.get("scope"), expires_at=provider.get(expiry_key)):
  1701. return token
  1702. return ""
  1703. def _resolve_nous_pool_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[str, str]]:
  1704. """Resolve Nous auxiliary credentials from the selected pool entry."""
  1705. try:
  1706. from hermes_cli.auth import _agent_key_is_usable
  1707. pool = load_pool("nous")
  1708. except Exception as exc:
  1709. logger.debug("Auxiliary Nous pool credential resolution failed: %s", exc)
  1710. return None
  1711. if not pool or not pool.has_credentials():
  1712. return None
  1713. try:
  1714. entry = pool.select()
  1715. except Exception as exc:
  1716. logger.debug("Auxiliary Nous pool selection failed: %s", exc)
  1717. return None
  1718. if entry is None:
  1719. return None
  1720. def _entry_state(e: Any) -> dict[str, Any]:
  1721. return {k: getattr(e, k, None) for k in (
  1722. "agent_key", "agent_key_expires_at", "access_token", "expires_at", "scope")}
  1723. if force_refresh or not _agent_key_is_usable(_entry_state(entry), _nous_min_key_ttl_seconds()):
  1724. try:
  1725. refreshed = pool.try_refresh_current()
  1726. except Exception as exc:
  1727. logger.debug("Auxiliary Nous pool refresh failed: %s", exc)
  1728. refreshed = None
  1729. if refreshed is None:
  1730. return None
  1731. entry = refreshed
  1732. api_key = _nous_api_key(_entry_state(entry))
  1733. base_url = _pool_runtime_base_url(entry, _NOUS_DEFAULT_BASE_URL)
  1734. if not api_key or not base_url:
  1735. return None
  1736. return api_key, base_url
  1737. def _resolve_nous_runtime_api(
  1738. *, force_refresh: bool = False, stale_access_token: Optional[str] = None
  1739. ) -> Optional[tuple[str, str]]:
  1740. """Fresh Nous runtime credentials (pool first, then auth store + JWT refresh) — mirrors the main
  1741. agent's 401 recovery. ``stale_access_token`` is the bearer that just 401'd; with ``force_refresh``
  1742. it lets the auth store adopt a sibling process's rotation instead of re-POSTing the shared grant."""
  1743. pooled = _resolve_nous_pool_runtime_api(force_refresh=force_refresh)
  1744. if pooled is not None:
  1745. return pooled
  1746. try:
  1747. from hermes_cli.auth import resolve_nous_runtime_credentials
  1748. creds = resolve_nous_runtime_credentials(
  1749. timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15),
  1750. force_refresh=force_refresh,
  1751. stale_access_token=stale_access_token or None,
  1752. )
  1753. except Exception as exc:
  1754. # Kept at WARNING (once per message) and remembered: the ladder falls back silently, and
  1755. # without this the goal judge only ever saw "judge error: RuntimeError" (#42177).
  1756. record_nous_credential_failure(exc)
  1757. return None
  1758. clear_nous_credential_failure()
  1759. return _creds_pair(creds)
  1760. def _creds_pair(creds: dict[str, Any]) -> Optional[tuple[str, str]]:
  1761. """``(api_key, base_url)`` from a runtime-credentials dict, or None when either is missing."""
  1762. api_key = str(creds.get("api_key") or "").strip()
  1763. base_url = str(creds.get("base_url") or "").strip().rstrip("/")
  1764. if not api_key or not base_url:
  1765. return None
  1766. return api_key, base_url
  1767. def _resolve_xai_oauth_for_aux() -> Optional[tuple[str, str]]:
  1768. """Fresh xAI OAuth (api_key, base_url) for aux clients, or None.
  1769. Pool first (some xAI OAuth logins exist only as pool entries), then the singleton auth-store resolver.
  1770. """
  1771. try:
  1772. from hermes_cli.auth import DEFAULT_XAI_OAUTH_BASE_URL, _xai_validate_inference_base_url
  1773. pool = load_pool("xai-oauth")
  1774. if pool and pool.has_credentials():
  1775. entry = pool.select()
  1776. if entry is not None:
  1777. api_key = str(
  1778. getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "") or ""
  1779. ).strip()
  1780. _url = lambda v: str(v or "").strip().rstrip("/")
  1781. base_url = _xai_validate_inference_base_url(
  1782. _url(_scoped_key_env("HERMES_XAI_BASE_URL"))
  1783. or _url(_scoped_key_env("XAI_BASE_URL"))
  1784. or _url(getattr(entry, "runtime_base_url", None))
  1785. or _url(getattr(entry, "base_url", None)),
  1786. fallback=DEFAULT_XAI_OAUTH_BASE_URL,
  1787. )
  1788. if api_key and base_url:
  1789. return api_key, base_url
  1790. except Exception as exc:
  1791. logger.debug("Auxiliary xAI OAuth pool credential resolution failed: %s", exc)
  1792. try:
  1793. from hermes_cli.auth import resolve_xai_oauth_runtime_credentials
  1794. creds = resolve_xai_oauth_runtime_credentials()
  1795. except Exception as exc:
  1796. logger.debug("Auxiliary xAI OAuth runtime credential resolution failed: %s", exc)
  1797. return None
  1798. return _creds_pair(creds)
  1799. def _resolve_codex_credential_and_base() -> tuple[Optional[str], str]:
  1800. """``(token, base_url)`` taken from ONE authority, so a Codex key is only ever sent to the host
  1801. it belongs to (#121486): the profile-scoped ``HERMES_CODEX_BASE_URL`` wins; otherwise a pooled
  1802. key goes where that pool entry routes (row URL / ``model.base_url``) and the auth.json OAuth
  1803. token goes to the ChatGPT default. ``(None, <base>)`` without a usable token."""
  1804. override = _codex_base_url_override()
  1805. pool_present, entry = _select_pool_entry("openai-codex")
  1806. if pool_present:
  1807. token = _pool_runtime_api_key(entry)
  1808. if token:
  1809. if override:
  1810. return token, override
  1811. # Same route rule as the chat path (row URL, else ``model.base_url``); never empty.
  1812. from hermes_cli.auth_codex import _codex_pool_route_base_url
  1813. return token, _codex_pool_route_base_url(_pool_runtime_base_url(entry))
  1814. # No usable pool token: auth.json only (re-selecting could pair another row's key with the default).
  1815. return _read_codex_singleton_token(), override or _CODEX_AUX_BASE_URL
  1816. def _read_codex_singleton_token() -> Optional[str]:
  1817. """The profile's auth.json Codex access token (expired JWTs skipped), else None."""
  1818. try:
  1819. from hermes_cli.auth import _read_codex_tokens
  1820. access_token = _read_codex_tokens().get("tokens", {}).get("access_token")
  1821. if not isinstance(access_token, str) or not access_token.strip():
  1822. return None
  1823. # Expired JWTs would block the auto chain and prevent fallback to working providers.
  1824. try:
  1825. import base64
  1826. payload = access_token.split(".")[1]
  1827. payload += "=" * (-len(payload) % 4)
  1828. exp = json.loads(base64.urlsafe_b64decode(payload)).get("exp", 0)
  1829. if exp and time.time() > exp:
  1830. logger.debug("Codex access token expired (exp=%s), skipping", exp)
  1831. return None
  1832. except Exception:
  1833. pass # Non-JWT token or decode error — use as-is
  1834. return access_token.strip()
  1835. except Exception as exc:
  1836. logger.debug("Could not read Codex auth for auxiliary client: %s", exc)
  1837. return None
  1838. def _resolve_api_key_provider() -> tuple[Optional[OpenAI], Optional[str]]:
  1839. """Try each API-key provider in PROVIDER_REGISTRY order; (client, model) or (None, None)."""
  1840. try:
  1841. from hermes_cli.auth import PROVIDER_REGISTRY, resolve_api_key_provider_credentials
  1842. except ImportError:
  1843. logger.debug("Could not import PROVIDER_REGISTRY for API-key fallback")
  1844. return None, None
  1845. from hermes_cli.auth_plugin_providers import canonical_registry_rows
  1846. for provider_id, pconfig in canonical_registry_rows():
  1847. if pconfig.auth_type != "api_key":
  1848. continue
  1849. if _is_provider_unhealthy(provider_id):
  1850. logger.debug("Auxiliary api-key chain: %s is unhealthy, skipping", provider_id)
  1851. continue
  1852. if provider_id == "anthropic":
  1853. # Explicit-config gate: Claude Code credentials must not silently become aux fallback.
  1854. with contextlib.suppress(ImportError):
  1855. from hermes_cli.auth import is_provider_explicitly_configured
  1856. if not is_provider_explicitly_configured("anthropic"):
  1857. continue
  1858. return _try_anthropic()
  1859. from providers import get_provider_profile
  1860. if getattr(get_provider_profile(provider_id), "ambient_credentials", False):
  1861. # Explicit-config gate: ambient credentials (another tool's login) must not silently become aux fallback (#114740).
  1862. from hermes_cli.auth import is_provider_explicitly_configured
  1863. if not is_provider_explicitly_configured(provider_id):
  1864. continue
  1865. model = _get_aux_model_for_provider(provider_id) or None
  1866. if model is None:
  1867. continue # skip provider if we don't know a valid aux model
  1868. pool_present, entry = _select_pool_entry(provider_id)
  1869. if pool_present:
  1870. api_key = _pool_runtime_api_key(entry)
  1871. if not api_key:
  1872. continue
  1873. raw_base_url = _pool_runtime_base_url(entry, pconfig.inference_base_url) or pconfig.inference_base_url
  1874. via = " via pool"
  1875. else:
  1876. creds = resolve_api_key_provider_credentials(provider_id)
  1877. api_key = str(creds.get("api_key", "")).strip()
  1878. if not api_key:
  1879. continue
  1880. raw_base_url = str(creds.get("base_url", "")).strip().rstrip("/") or pconfig.inference_base_url
  1881. via = ""
  1882. # The session's own endpoint wins for its provider: the key was issued for that gateway, and
  1883. # sending it to the registry default 401s, then quarantines the provider the main model is on.
  1884. runtime = _normalize_main_runtime(None)
  1885. if runtime.get("provider") == provider_id and runtime.get("base_url"):
  1886. raw_base_url = runtime["base_url"].rstrip("/")
  1887. if isinstance(runtime.get("api_key"), str) and runtime["api_key"]:
  1888. api_key = runtime["api_key"]
  1889. via = " (session endpoint)"
  1890. logger.debug("Auxiliary text client: %s (%s)%s", pconfig.name, model, via)
  1891. # Native Gemini, else OpenAI-wire + Anthropic rewrap.
  1892. base_url = _to_openai_base_url(raw_base_url)
  1893. if provider_id == "gemini":
  1894. from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url
  1895. if is_native_gemini_base_url(base_url):
  1896. return GeminiNativeClient(api_key=api_key, base_url=base_url), model
  1897. if base_url_host_matches(base_url, "api.kimi.com"):
  1898. headers = {"User-Agent": "claude-code/0.1.0"}
  1899. elif base_url_host_matches(base_url, "integrate.api.nvidia.com"):
  1900. headers = build_nvidia_nim_headers(base_url)
  1901. else:
  1902. headers = _route_client_headers(base_url, provider_id) or _profile_default_headers(provider_id)
  1903. extra = {"default_headers": headers} if headers else {}
  1904. merged = _apply_user_default_headers(extra.get("default_headers"))
  1905. if merged:
  1906. extra["default_headers"] = merged
  1907. client = _create_openai_client(api_key=api_key, base_url=base_url, **extra)
  1908. return _maybe_wrap_anthropic(client, model, api_key, raw_base_url), model
  1909. return None, None
  1910. def _route_client_headers(base_url: str, provider: str, *, is_vision: bool = False) -> Optional[dict]:
  1911. from agent.provider_route_hooks import route_answer
  1912. headers = route_answer("client_headers", provider=provider or "", base_url=base_url, has_images=is_vision)
  1913. return dict(headers) if headers else None
  1914. def _endpoint_default_headers(
  1915. base_url: str, provider: str, *, is_vision: bool = False, xai: bool = False,
  1916. ) -> Optional[dict]:
  1917. """Provider-specific client headers by endpoint host, merged with user ``model.default_headers``.
  1918. Kimi Code needs the claude-code User-Agent; a profile's ``client_headers`` (``is_vision`` = image
  1919. client), NVIDIA NIM and (optionally) xAI have their own; else the provider profile.
  1920. """
  1921. if base_url_host_matches(base_url, "api.kimi.com"):
  1922. headers: dict = {"User-Agent": "claude-code/0.1.0"}
  1923. elif route := _route_client_headers(base_url, provider, is_vision=is_vision):
  1924. headers = route
  1925. elif base_url_host_matches(base_url, "integrate.api.nvidia.com"):
  1926. headers = dict(build_nvidia_nim_headers(base_url))
  1927. elif xai and base_url_host_matches(base_url, "x.ai"):
  1928. from tools.xai_http import hermes_xai_default_headers
  1929. headers = dict(hermes_xai_default_headers())
  1930. else:
  1931. headers = _profile_default_headers(provider) or {}
  1932. return _apply_user_default_headers(headers or None) or None
  1933. def _profile_default_headers(provider: str) -> Optional[dict]:
  1934. """Client-level attribution headers from the provider profile (e.g. GMI User-Agent), or None."""
  1935. if not provider:
  1936. return None
  1937. with contextlib.suppress(Exception):
  1938. from providers import get_provider_profile
  1939. profile = get_provider_profile(provider)
  1940. if profile and profile.default_headers:
  1941. return dict(profile.default_headers)
  1942. return None
  1943. # Provider resolution helpers
  1944. _paid_lane_warned: set = set()
  1945. def _is_free_model(model: Optional[str]) -> bool:
  1946. """True when ``model`` is a free SKU (``:free`` suffix or ``stealth/`` prefix) — naming-convention trust."""
  1947. if not model:
  1948. return False
  1949. normalized = str(model).strip()
  1950. return normalized.endswith(":free") or normalized.startswith("stealth/")
  1951. def _aux_openrouter_settings() -> tuple[bool, str]:
  1952. """Read (free_only, openrouter_model) from config; (False, _OPENROUTER_MODEL) on failure."""
  1953. try:
  1954. from hermes_cli.config import cfg_get, load_config_readonly
  1955. cfg = load_config_readonly()
  1956. free_only = bool(cfg_get(cfg, "auxiliary", "free_only", default=False))
  1957. val = cfg_get(cfg, "auxiliary", "openrouter_model")
  1958. model = val.strip() if isinstance(val, str) and val.strip() else _OPENROUTER_MODEL
  1959. return free_only, model
  1960. except Exception:
  1961. return False, _OPENROUTER_MODEL
  1962. def _warn_paid_lane_once(model: str) -> None:
  1963. """Log a WARNING the first time a non-free OpenRouter model is engaged."""
  1964. if model in _paid_lane_warned:
  1965. return
  1966. _paid_lane_warned.add(model)
  1967. logger.warning(
  1968. "Auxiliary client: PAID lane engaged for auxiliary task — OpenRouter fallback model %r is not "
  1969. "a :free SKU and may incur real spend. Set auxiliary.free_only: true to restrict auxiliary "
  1970. "fallbacks to free models, or auxiliary.openrouter_model to a :free model.", model,
  1971. )
  1972. def _try_openrouter(explicit_api_key: Optional[str | Callable[[], str]] = None, model: str | None = None,
  1973. explicit_base_url: Optional[str] = None) -> tuple[Optional[OpenAI], Optional[str]]:
  1974. free_only, cfg_model = _aux_openrouter_settings()
  1975. or_model = model or cfg_model
  1976. if free_only and not _is_free_model(or_model):
  1977. logger.warning(
  1978. "Auxiliary client: auxiliary.free_only is enabled but the OpenRouter fallback model %r is "
  1979. "not a :free SKU — skipping the OpenRouter fallback. Set auxiliary.openrouter_model to a "
  1980. ":free model (e.g. nvidia/nemotron-3-ultra-550b-a55b:free) or disable auxiliary.free_only.",
  1981. or_model,
  1982. )
  1983. return None, None
  1984. if not _is_free_model(or_model):
  1985. _warn_paid_lane_once(or_model)
  1986. # A caller-supplied endpoint (fallback_providers entry, custom_providers entry) is
  1987. # authoritative over both the pool row and the canonical host (#121359).
  1988. override_url = (explicit_base_url or "").strip().rstrip("/")
  1989. pool_present, entry = _select_pool_entry("openrouter")
  1990. if pool_present:
  1991. or_key = explicit_api_key or _pool_runtime_api_key(entry)
  1992. if or_key:
  1993. base_url = override_url or _pool_runtime_base_url(entry, OPENROUTER_BASE_URL) or OPENROUTER_BASE_URL
  1994. logger.debug("Auxiliary client: OpenRouter via pool")
  1995. return _create_openai_client(
  1996. api_key=or_key, base_url=base_url, default_headers=build_or_headers()
  1997. ), or_model
  1998. # Exhausted pool: fall through to OPENROUTER_API_KEY rather than fail.
  1999. logger.debug("Auxiliary client: OpenRouter pool exhausted, trying OPENROUTER_API_KEY")
  2000. or_key = explicit_api_key or _scoped_key_env("OPENROUTER_API_KEY")
  2001. if not or_key:
  2002. _mark_provider_unhealthy(
  2003. "openrouter", ttl=60, reason=_describe_openrouter_unavailable(or_model), level=logging.DEBUG)
  2004. return None, None
  2005. logger.debug("Auxiliary client: OpenRouter")
  2006. return _create_openai_client(
  2007. api_key=or_key, base_url=override_url or OPENROUTER_BASE_URL, default_headers=build_or_headers()
  2008. ), or_model
  2009. def _describe_openrouter_unavailable(model: str | None = None) -> str:
  2010. """Return the policy or credential reason OpenRouter was unavailable."""
  2011. free_only, cfg_model = _aux_openrouter_settings()
  2012. or_model = model or cfg_model
  2013. if free_only and not _is_free_model(or_model):
  2014. return (
  2015. f"auxiliary.free_only rejected non-free model {or_model!r}; "
  2016. "the request was skipped before provider availability checks"
  2017. )
  2018. pool_present, entry = _select_pool_entry("openrouter")
  2019. if pool_present:
  2020. if entry is None:
  2021. return "OpenRouter credential pool has no usable entries (credentials may be exhausted)"
  2022. if not _pool_runtime_api_key(entry):
  2023. return "OpenRouter credential pool entry is missing a runtime API key"
  2024. if not _scoped_key_env("OPENROUTER_API_KEY"):
  2025. return "OPENROUTER_API_KEY not set"
  2026. return "no usable OpenRouter credentials found"
  2027. def _try_nous(vision: bool = False) -> tuple[Optional[OpenAI], Optional[str]]:
  2028. nous = _read_nous_auth()
  2029. runtime = _resolve_nous_runtime_api(force_refresh=False)
  2030. if runtime is None and not nous:
  2031. logger.warning("Auxiliary Nous client unavailable: no Nous authentication found (run: hermes auth).")
  2032. _mark_provider_unhealthy("nous", ttl=60, reason="no Nous authentication found", level=logging.DEBUG)
  2033. return None, None
  2034. if runtime is None and nous:
  2035. logger.debug("Auxiliary Nous: runtime JWT refresh failed; checking stored auth.json token.")
  2036. if runtime is not None:
  2037. api_key, base_url = runtime
  2038. else:
  2039. api_key = _nous_api_key(nous or {})
  2040. if not api_key:
  2041. logger.warning(
  2042. "Auxiliary Nous client unavailable: no usable inference JWT found "
  2043. "(run: hermes auth add nous)."
  2044. )
  2045. _mark_provider_unhealthy("nous", ttl=60, reason="no usable Nous inference JWT", level=logging.DEBUG)
  2046. return None, None
  2047. base_url = str(
  2048. (nous or {}).get("inference_base_url") or _scoped_key_env("NOUS_INFERENCE_BASE_URL") or _NOUS_DEFAULT_BASE_URL
  2049. ).rstrip("/")
  2050. with contextlib.suppress(Exception):
  2051. from agent.nous_rate_guard import nous_rate_limit_remaining
  2052. from hermes_cli.anon_auth import is_anonymous_request
  2053. anonymous = is_anonymous_request("nous", api_key)
  2054. remaining = nous_rate_limit_remaining(anonymous=anonymous)
  2055. if remaining is not None and remaining > 0:
  2056. logger.debug("Auxiliary: skipping Nous Portal (rate-limited, resets in %.0fs)", remaining)
  2057. # The health marker is provider-wide, so a full-length anonymous cooldown would
  2058. # outlive signing in mid-cooldown; bound it instead of re-resolving credentials
  2059. # (auth store lock, pool read) on every auxiliary call for the cooldown's duration.
  2060. _mark_provider_unhealthy(
  2061. "nous", ttl=min(remaining, 60.0) if anonymous else remaining,
  2062. reason="Nous Portal rate-limited", level=logging.INFO)
  2063. return None, None
  2064. lane = "vision" if vision else "text"
  2065. # The free tier's host serves exactly one model, for every lane: asking it for the Portal's
  2066. # recommended aux model is a guaranteed 429 ``model_not_free``. Pin the route's model instead.
  2067. # Vision rides the same id (the backing model is multimodal; a backing that is not answers
  2068. # the request with the upstream's own error, which the ladder handles like any other).
  2069. from hermes_cli.anon_auth import GUEST_MODEL, route_is_welcome_host
  2070. if route_is_welcome_host(base_url):
  2071. logger.debug("Auxiliary/%s: Nous free tier; using %s", lane, GUEST_MODEL)
  2072. return _create_openai_client(api_key=api_key, base_url=base_url), GUEST_MODEL
  2073. logger.debug("Auxiliary client: Nous Portal")
  2074. # Portal recommended-models is authoritative (tier-aware); _NOUS_MODEL when unreachable/null.
  2075. # Probes skip the lookup: exact model is irrelevant and it hits the network.
  2076. model = _NOUS_MODEL
  2077. if not _aux_probe_active():
  2078. try:
  2079. from hermes_cli.models import get_nous_recommended_aux_model
  2080. recommended = get_nous_recommended_aux_model(vision=vision)
  2081. if recommended:
  2082. model = recommended
  2083. logger.debug("Auxiliary/%s: using Portal-recommended model %s", lane, model)
  2084. else:
  2085. logger.debug("Auxiliary/%s: no Portal recommendation, falling back to %s", lane, model)
  2086. except Exception as exc:
  2087. logger.debug(
  2088. "Auxiliary/%s: recommended-models lookup failed (%s); "
  2089. "falling back to %s",
  2090. lane, exc, model,
  2091. )
  2092. return _create_openai_client(api_key=api_key, base_url=base_url), model
  2093. def _refresh_nous_recommended_model(*, vision: bool, stale_model: Optional[str]) -> Optional[str]:
  2094. """Fresh Portal recommended model after a stale-model 404 (long-lived processes pin dropped models).
  2095. Returns the fresh recommendation, else ``_NOUS_MODEL``, whichever differs from ``stale_model``; None if neither.
  2096. """
  2097. stale = (stale_model or "").strip().lower()
  2098. fresh: Optional[str] = None
  2099. try:
  2100. from hermes_cli.models import get_nous_recommended_aux_model
  2101. fresh = get_nous_recommended_aux_model(vision=vision, force_refresh=True)
  2102. except Exception as exc:
  2103. logger.debug("Nous recommended-model refresh failed (%s); using default %s", exc, _NOUS_MODEL)
  2104. if fresh and fresh.strip().lower() != stale:
  2105. return fresh
  2106. return _NOUS_MODEL if _NOUS_MODEL.strip().lower() != stale else None
  2107. def _read_main_field(field: str, *, readonly: bool, lower: bool = False) -> str:
  2108. """Main ``model.<field>``: runtime override (``set_runtime_main``) first, then config.yaml.
  2109. The override wins so "active main model" gates see the live CLI/gateway runtime, not the persisted
  2110. default. ``readonly`` picks ``load_config_readonly`` (model/provider) vs ``load_config`` (api_key/base_url).
  2111. """
  2112. override = _runtime_main_value(field)
  2113. if isinstance(override, str) and override.strip():
  2114. value = override.strip()
  2115. return value.lower() if lower else value
  2116. with contextlib.suppress(Exception):
  2117. from hermes_cli import config as _cfg_mod
  2118. cfg = (_cfg_mod.load_config_readonly if readonly else _cfg_mod.load_config)()
  2119. model_cfg = cfg.get("model", {})
  2120. if field == "model" and isinstance(model_cfg, str) and model_cfg.strip():
  2121. return model_cfg.strip()
  2122. if isinstance(model_cfg, dict):
  2123. value = model_cfg.get("default" if field == "model" else field, "")
  2124. if isinstance(value, str) and value.strip():
  2125. value = value.strip()
  2126. return value.lower() if lower else value
  2127. return ""
  2128. # Module-level callables (tests patch them): model/provider (lowercased) read the readonly config;
  2129. # api_key/base_url read the full config so ``custom`` aux tasks can inherit main creds.
  2130. _read_main_model = functools.partial(_read_main_field, "model", readonly=True)
  2131. _read_main_provider = functools.partial(_read_main_field, "provider", readonly=True, lower=True)
  2132. _read_main_api_key = functools.partial(_read_main_field, "api_key", readonly=False)
  2133. _read_main_base_url = functools.partial(_read_main_field, "base_url", readonly=False)
  2134. def _resolve_moa_aggregator(preset_name: Optional[str]) -> tuple[Optional[str], Optional[str]]:
  2135. """MoA preset → aggregator (provider, model); (None, None) if unresolvable. None/"" = default preset.
  2136. "moa" is virtual — aux tasks skip the fan-out and use the aggregator slot; shared so lookup can't drift.
  2137. """
  2138. try:
  2139. from hermes_cli.config import load_config
  2140. from hermes_cli.moa_config import resolve_moa_preset
  2141. preset = resolve_moa_preset(load_config().get("moa") or {}, preset_name or None)
  2142. agg = preset.get("aggregator") or {}
  2143. agg_provider = str(agg.get("provider") or "").strip()
  2144. agg_model = str(agg.get("model") or "").strip()
  2145. if agg_provider and agg_model and agg_provider.lower() != "moa":
  2146. return agg_provider, agg_model
  2147. except Exception:
  2148. logger.debug("MoA aggregator resolution failed for preset %r", preset_name, exc_info=True)
  2149. return None, None
  2150. def _read_main_model_for_aux() -> str:
  2151. """Main model with MoA presets unwrapped to the aggregator's model; "" when unresolvable (a preset name would 400)."""
  2152. model = _read_main_model()
  2153. if (_read_main_provider() or "").strip().lower() == "moa":
  2154. _, agg_model = _resolve_moa_aggregator(model)
  2155. return agg_model or ""
  2156. return model
  2157. def _read_main_api_key_if_same_origin(aux_base_url: str) -> str | Callable[[], str]:
  2158. """Main api_key only when *aux_base_url* has the main base_url's exact origin.
  2159. Unconditional inheritance would leak the credential to any misconfigured host; mismatch keeps ``no-key-required`` → 401.
  2160. Origin, not hostname: another scheme (``http://``) or port on the same host is a different endpoint.
  2161. Anchor and key come from ONE source: the live runtime a turn bound, else config.yaml. The
  2162. per-field readers fall back to config field by field, so a keyless or key_cmd live main would
  2163. pair its own base_url with config's key and send that key to the live endpoint.
  2164. Origins are compared before any key is read: the client cache calls this on every keyless
  2165. ``custom`` lookup, and a mismatch must not pay for a config.yaml key read.
  2166. """
  2167. aux_origin = base_url_origin(aux_base_url)
  2168. if not aux_origin[1]:
  2169. return ""
  2170. live = _normalize_main_runtime(None)
  2171. if live.get("base_url") or live.get("api_key"):
  2172. if aux_origin != base_url_origin(live.get("base_url", "")):
  2173. return ""
  2174. return live.get("api_key", "")
  2175. if aux_origin != base_url_origin(_read_main_base_url()):
  2176. return ""
  2177. return _read_main_api_key()
  2178. # The live main runtime, per context: overlapping gateway sessions make a process-global unsafe.
  2179. _RUNTIME_MAIN_CONTEXT: contextvars.ContextVar[Optional[dict[str, Any]]] = (
  2180. contextvars.ContextVar("auxiliary_runtime_main", default=None)
  2181. )
  2182. _RELAY_AUX_CALL_CONTEXT: contextvars.ContextVar[Optional[dict[str, Any]]] = (
  2183. contextvars.ContextVar("auxiliary_relay_call", default=None)
  2184. )
  2185. @contextlib.contextmanager
  2186. def _relay_aux_call_scope(args: tuple, kwargs: dict):
  2187. """Bind a fresh relay call context for one auxiliary call; mark it failed on any exception."""
  2188. task = args[0] if args else kwargs.get("task")
  2189. token = _RELAY_AUX_CALL_CONTEXT.set({
  2190. "task": str(task or "unknown"),
  2191. "request_id": f"aux-{uuid.uuid4().hex}",
  2192. "attempt_count": 0,
  2193. "provider": "",
  2194. "model": "",
  2195. "response_model": None,
  2196. "api_mode": "chat_completions",
  2197. })
  2198. try:
  2199. yield
  2200. except BaseException as exc:
  2201. _fail_relay_auxiliary_call(exc)
  2202. raise
  2203. finally:
  2204. _RELAY_AUX_CALL_CONTEXT.reset(token)
  2205. def _relay_auxiliary_call(callback):
  2206. """Give every physical retry in one auxiliary call a shared Relay identity."""
  2207. @functools.wraps(callback)
  2208. def wrapped(*args, **kwargs):
  2209. with _relay_aux_call_scope(args, kwargs):
  2210. return callback(*args, **kwargs)
  2211. return wrapped
  2212. def _relay_auxiliary_call_async(callback):
  2213. """Async counterpart to :func:`_relay_auxiliary_call`."""
  2214. @functools.wraps(callback)
  2215. async def wrapped(*args, **kwargs):
  2216. with _relay_aux_call_scope(args, kwargs):
  2217. return await callback(*args, **kwargs)
  2218. return wrapped
  2219. def _set_relay_auxiliary_route(provider: str | None, model: str | None, api_mode: str | None) -> None:
  2220. context = _RELAY_AUX_CALL_CONTEXT.get()
  2221. if context is None:
  2222. return
  2223. context["provider"] = str(provider or "auxiliary")
  2224. context["model"] = str(model or "unknown")
  2225. context["response_model"] = None
  2226. context["api_mode"] = str(api_mode or "chat_completions")
  2227. def _record_route_info(
  2228. route_info: Optional[dict[str, str]], provider: Optional[str], model: Optional[str]
  2229. ) -> None:
  2230. """Expose the concrete route selected for one auxiliary call."""
  2231. if route_info is not None:
  2232. route_info["provider"] = provider or "auto"
  2233. route_info["model"] = model or "default"
  2234. def _relay_auxiliary_metadata(
  2235. *, provider: str | None = None, api_mode: str | None = None
  2236. ) -> tuple[str, str, dict[str, Any]] | None:
  2237. context = _RELAY_AUX_CALL_CONTEXT.get()
  2238. if context is None:
  2239. return None
  2240. attempt_count = int(context.get("attempt_count") or 0)
  2241. context["attempt_count"] = attempt_count + 1
  2242. provider_name = str(provider or context.get("provider") or "auxiliary")
  2243. model_name = str(context.get("model") or "unknown")
  2244. return provider_name, model_name, {
  2245. "api_mode": str(api_mode or context.get("api_mode") or "chat_completions"),
  2246. "api_request_id": str(context["request_id"]),
  2247. "call_role": f"auxiliary:{context['task']}",
  2248. "retry_count": attempt_count,
  2249. "auxiliary_task": str(context["task"]),
  2250. }
  2251. def _relay_sync_completion(
  2252. client: Any, kwargs: dict[str, Any], *, provider: str | None = None,
  2253. api_mode: str | None = None, create: Callable[[dict[str, Any]], Any] | None = None,
  2254. ) -> Any:
  2255. from agent.auxiliary_wire import prepare_chat_messages, relay_chat_metadata
  2256. kwargs = prepare_chat_messages(client, kwargs)
  2257. # The progress hook is installed per TASK, so every attempt (retries, recovery rungs, fallbacks)
  2258. # must stream through _create_with_progress or the compression watchdog sees silence (#98466).
  2259. # Recovery rungs / credential retries keep the task's ``no_progress_timeout`` window; the
  2260. # first-token window uses this attempt's provider (fallbacks name theirs) for its stale timeout.
  2261. relay_context = _RELAY_AUX_CALL_CONTEXT.get() or {}
  2262. task = relay_context.get("task")
  2263. relay_context["stream_provider"] = provider or relay_context.get("provider")
  2264. callback = create or (lambda request: _create_with_progress(client, request, task))
  2265. route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
  2266. # Isolate only the provider callback so the owning thread can unwind its lease/DB
  2267. # transaction on hard cancel without touching the shared client.
  2268. if route is None:
  2269. return _run_protected_sync_provider_call(callback, kwargs)
  2270. provider_name, fallback_model, metadata = route
  2271. from agent import relay_llm
  2272. from agent.auxiliary_hooks import run_with_aux_hooks
  2273. model_name = str(kwargs.get("model") or fallback_model)
  2274. try:
  2275. return run_with_aux_hooks(
  2276. lambda: relay_llm.execute_current(
  2277. kwargs, lambda request: _run_protected_sync_provider_call(callback, request),
  2278. name=provider_name, model_name=model_name, metadata=relay_chat_metadata(metadata),
  2279. defer_logical_completion=True,
  2280. ),
  2281. aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
  2282. provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""),
  2283. )
  2284. except Exception as exc:
  2285. _note_relay_auxiliary_error(exc)
  2286. raise
  2287. async def _relay_async_completion(
  2288. client: Any, kwargs: dict[str, Any], *, provider: str | None = None,
  2289. api_mode: str | None = None, create: Callable[[dict[str, Any]], Any] | None = None,
  2290. ) -> Any:
  2291. from agent.auxiliary_wire import prepare_chat_messages, relay_chat_metadata
  2292. kwargs = prepare_chat_messages(client, kwargs)
  2293. # Async twin of the seam default above (#98466).
  2294. callback = create or (lambda request: _acreate_with_progress(client, request))
  2295. route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
  2296. if route is None:
  2297. return await callback(kwargs)
  2298. provider_name, fallback_model, metadata = route
  2299. from agent import relay_llm
  2300. from agent.auxiliary_hooks import arun_with_aux_hooks
  2301. model_name = str(kwargs.get("model") or fallback_model)
  2302. try:
  2303. return await arun_with_aux_hooks(
  2304. lambda: relay_llm.execute_current_async(
  2305. kwargs, callback, name=provider_name, model_name=model_name,
  2306. metadata=relay_chat_metadata(metadata), defer_logical_completion=True,
  2307. ),
  2308. aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
  2309. provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""),
  2310. )
  2311. except Exception as exc:
  2312. _note_relay_auxiliary_error(exc)
  2313. raise
  2314. def _relay_sync_stream(
  2315. client: Any, kwargs: dict[str, Any], *, provider: str | None = None, api_mode: str | None = None
  2316. ) -> Any:
  2317. from agent.auxiliary_wire import prepare_chat_messages, relay_chat_metadata
  2318. kwargs = prepare_chat_messages(client, kwargs)
  2319. # The bypass runs inside the provider callback, AFTER Relay has seen (and possibly
  2320. # rewritten) the real conversation; applying it to `kwargs` would hand Relay an empty one.
  2321. create = lambda request: client.chat.completions.create(**bypass_chat_sdk_request_transform(request, client))
  2322. route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
  2323. if route is None:
  2324. return create(kwargs)
  2325. provider_name, fallback_model, metadata = route
  2326. from agent import relay_llm
  2327. from agent.auxiliary_hooks import run_with_aux_hooks
  2328. model_name = str(kwargs.get("model") or fallback_model)
  2329. return run_with_aux_hooks(
  2330. lambda: relay_llm.stream_current(
  2331. kwargs, create, name=provider_name, model_name=model_name, finalizer=dict,
  2332. metadata=relay_chat_metadata(metadata), completed_response_predicate=lambda value: hasattr(value, "choices"),
  2333. ),
  2334. aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
  2335. provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""), streaming=True,
  2336. )
  2337. def _runtime_main_value(field: str) -> Any:
  2338. """Read one field of the current context's main runtime ("" when unbound)."""
  2339. runtime = _RUNTIME_MAIN_CONTEXT.get()
  2340. return (runtime.get(field) or "") if isinstance(runtime, dict) else ""
  2341. def set_runtime_main(
  2342. provider: str, model: str, *, requested_provider: str = "", base_url: str = "",
  2343. api_key: Any = "", api_mode: str = "", auth_mode: str = "", session_id: str = "",
  2344. cache_scope: str = "",
  2345. ) -> contextvars.Token:
  2346. """Record the current context's live main runtime for auxiliary routing.
  2347. Context-local so concurrent gateway sessions don't clobber each other.
  2348. ``cache_scope`` is the rotation-stable logical cache scope (compression- lineage root —
  2349. agent/prompt_cache_scope.py) resolved once per turn by turn_context; auxiliary Responses calls prefer it
  2350. over ``session_id`` for prompt_cache_key derivation (#79017).
  2351. """
  2352. runtime = {
  2353. "provider": (provider or "").strip().lower(),
  2354. "requested_provider": (requested_provider or "").strip().lower(),
  2355. "model": (model or "").strip(),
  2356. "base_url": (base_url or "").strip(),
  2357. "api_key": _normalize_api_key(api_key),
  2358. "api_mode": (api_mode or "").strip(),
  2359. "auth_mode": (auth_mode or "").strip().lower(),
  2360. "session_id": (session_id or "").strip(),
  2361. "cache_scope": (cache_scope or "").strip(),
  2362. }
  2363. return _RUNTIME_MAIN_CONTEXT.set(runtime)
  2364. def reset_runtime_main(token: contextvars.Token) -> None:
  2365. """Restore the runtime binding that preceded one scoped turn."""
  2366. if token is None:
  2367. return
  2368. try:
  2369. _RUNTIME_MAIN_CONTEXT.reset(token)
  2370. except (RuntimeError, ValueError):
  2371. pass # Tokens can't be reset from a copied Context (workers inherit values, not token ownership).
  2372. @contextlib.contextmanager
  2373. def scoped_runtime_main(main_runtime: Optional[dict[str, Any]]):
  2374. """Temporarily bind an explicit runtime for the current context."""
  2375. runtime = _normalize_main_runtime(main_runtime)
  2376. token = _RUNTIME_MAIN_CONTEXT.set(runtime or None)
  2377. try:
  2378. yield runtime
  2379. finally:
  2380. _RUNTIME_MAIN_CONTEXT.reset(token)
  2381. def clear_runtime_main() -> None:
  2382. """Clear the runtime override in the current context."""
  2383. _RUNTIME_MAIN_CONTEXT.set(None)
  2384. def _resolve_custom_runtime() -> tuple[Optional[str], Optional[str], Optional[str]]:
  2385. """Resolve the active custom/main endpoint like the main CLI (env OPENAI_BASE_URL or config-saved)."""
  2386. try:
  2387. from hermes_cli.auth import AuthError
  2388. from hermes_cli.runtime_provider import resolve_runtime_provider
  2389. runtime = resolve_runtime_provider(requested="custom")
  2390. except AuthError as exc:
  2391. # Bare 'custom' with nothing configured fails fast in the main resolver: there is no
  2392. # custom endpoint, so do NOT fall through to a stale env OPENAI_BASE_URL that the main
  2393. # resolver deliberately never consults.
  2394. logger.debug("Auxiliary client: no custom endpoint configured: %s", exc)
  2395. return None, None, None
  2396. except Exception as exc:
  2397. logger.debug("Auxiliary client: custom runtime resolution failed: %s", exc)
  2398. runtime = None
  2399. if not isinstance(runtime, dict):
  2400. # Base URL is per-profile like the key one line below (a scoped key must not hit the default's proxy).
  2401. openai_base = _scoped_key_env("OPENAI_BASE_URL").rstrip("/")
  2402. if not openai_base:
  2403. return None, None, None
  2404. runtime = {"base_url": openai_base, "api_key": _scoped_key_env("OPENAI_API_KEY")}
  2405. custom_base = runtime.get("base_url")
  2406. custom_key = runtime.get("api_key")
  2407. custom_mode = runtime.get("api_mode")
  2408. if not isinstance(custom_base, str) or not custom_base.strip():
  2409. return None, None, None
  2410. custom_base = custom_base.strip().rstrip("/")
  2411. if base_url_host_matches(custom_base, "openrouter.ai"):
  2412. return None, None, None # requested='custom' falls back to OpenRouter when unconfigured.
  2413. # Local servers (Ollama, vLLM, ...) ignore auth but the SDK needs a non-empty key.
  2414. # Use a placeholder key — the OpenAI SDK requires a non-empty string but local servers ignore the
  2415. # Authorization header. Same fix as cli.py _ensure_runtime_credentials() (PR #2556).
  2416. if not isinstance(custom_key, str) or not custom_key.strip():
  2417. custom_key = "no-key-required"
  2418. if not isinstance(custom_mode, str) or not custom_mode.strip():
  2419. custom_mode = None
  2420. return custom_base, custom_key.strip(), custom_mode
  2421. def _current_custom_base_url() -> str:
  2422. custom_base, _, _ = _resolve_custom_runtime()
  2423. return custom_base or ""
  2424. def _validate_proxy_env_urls() -> None:
  2425. """Fail fast on malformed proxy env URLs (a shell typo like ``:6153export`` otherwise surfaces as a cryptic httpx ``Invalid port``)."""
  2426. from urllib.parse import urlparse
  2427. normalize_proxy_env_vars()
  2428. for key in ("HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "https_proxy", "http_proxy", "all_proxy"):
  2429. value = str(os.environ.get(key) or "").strip()
  2430. if not value:
  2431. continue
  2432. try:
  2433. parsed = urlparse(value)
  2434. if parsed.scheme:
  2435. _ = parsed.port # raises ValueError for e.g. '6153export'
  2436. except ValueError as exc:
  2437. raise RuntimeError(
  2438. f"Malformed proxy environment variable {key}={value!r}. "
  2439. "Fix or unset your proxy settings and try again."
  2440. ) from exc
  2441. def _validate_base_url(base_url: str) -> None:
  2442. """Reject obviously broken custom endpoint URLs before they reach httpx."""
  2443. from urllib.parse import urlparse
  2444. candidate = str(base_url or "").strip()
  2445. if not candidate or candidate.startswith("acp://"):
  2446. return
  2447. try:
  2448. parsed = urlparse(candidate)
  2449. if parsed.scheme in {"http", "https"}:
  2450. _ = parsed.port # raises ValueError for malformed ports
  2451. except ValueError as exc:
  2452. raise RuntimeError(
  2453. f"Malformed custom endpoint URL: {candidate!r}. "
  2454. "Run `hermes setup` or `hermes model` and enter a valid http(s) base URL."
  2455. ) from exc
  2456. def _try_custom_endpoint() -> tuple[Optional[Any], Optional[str]]:
  2457. runtime = _resolve_custom_runtime()
  2458. custom_base, custom_key, custom_mode = (*runtime, None) if len(runtime) == 2 else runtime
  2459. if not custom_base or not custom_key:
  2460. return None, None
  2461. if custom_base.lower().startswith(_CODEX_AUX_BASE_URL.lower()):
  2462. return None, None
  2463. model = _read_main_model_for_aux() or "gpt-4o-mini"
  2464. logger.debug("Auxiliary client: custom endpoint (%s, api_mode=%s)", model, custom_mode or "chat_completions")
  2465. _clean_base, _dq = _extract_url_query_params(custom_base)
  2466. _extra = {"default_query": _dq} if _dq else {}
  2467. # User model.default_headers override SDK fingerprint headers (as on the main client) for strict gateways/WAFs.
  2468. _custom_headers = _apply_user_default_headers(None)
  2469. if _custom_headers:
  2470. _extra["default_headers"] = _custom_headers
  2471. if custom_mode == "codex_responses":
  2472. real_client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
  2473. return CodexAuxiliaryClient(real_client, model), model
  2474. if custom_mode == "anthropic_messages":
  2475. # OAuth identity only when the host is exactly api.anthropic.com (key_cmd callable included,
  2476. # #114967); third-party Anthropic-compatible gateways never get it.
  2477. try:
  2478. from agent.anthropic_adapter import build_anthropic_client
  2479. from agent.anthropic_credentials import anthropic_route_is_oauth
  2480. real_client = build_anthropic_client(custom_key, custom_base)
  2481. except ImportError:
  2482. logger.warning(
  2483. "Custom endpoint declares api_mode=anthropic_messages but the "
  2484. "anthropic SDK is not installed — falling back to OpenAI-wire."
  2485. )
  2486. return _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra), model
  2487. return AnthropicAuxiliaryClient(real_client, model, custom_key, custom_base,
  2488. is_oauth=anthropic_route_is_oauth(custom_base, custom_key)), model
  2489. # URL-based anthropic detection for custom endpoints without explicit api_mode.
  2490. _fallback_client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
  2491. return _maybe_wrap_anthropic(_fallback_client, model, custom_key, custom_base, custom_mode), model
  2492. def _build_xai_oauth_aux_client(model: str) -> tuple[Optional[Any], Optional[str]]:
  2493. """CodexAuxiliaryClient for xAI Grok OAuth (Responses API); (None, None) if not authed.
  2494. Caller must pass an explicit model — a pinned Grok default would rot as xAI's allowlist drifts.
  2495. """
  2496. if not model:
  2497. logger.warning(
  2498. "Auxiliary client: xai-oauth requested without a model; "
  2499. "pass model explicitly (auxiliary.<task>.model in config.yaml)."
  2500. )
  2501. return None, None
  2502. resolved = _resolve_xai_oauth_for_aux()
  2503. if resolved is None:
  2504. return None, None
  2505. api_key, base_url = resolved
  2506. logger.debug("Auxiliary client: xAI OAuth (%s via Responses API)", model)
  2507. from tools.xai_http import hermes_xai_default_headers
  2508. real_client = _create_openai_client(
  2509. api_key=api_key, base_url=base_url, default_headers=hermes_xai_default_headers()
  2510. )
  2511. return CodexAuxiliaryClient(real_client, model), model
  2512. def _codex_base_url_override() -> str:
  2513. """Profile-scoped ``HERMES_CODEX_BASE_URL`` (same read as the API-key env vars: under a
  2514. multiplexer the routed profile's .env decides the endpoint, never a sibling's process env)."""
  2515. return _scoped_key_env("HERMES_CODEX_BASE_URL").rstrip("/")
  2516. def _build_codex_client(model: str) -> tuple[Optional[Any], Optional[str]]:
  2517. """CodexAuxiliaryClient for an explicit model; (None, None) without a Codex OAuth token.
  2518. No auto-selected default: the Codex model allow-list is undocumented and drifts.
  2519. """
  2520. if not model:
  2521. logger.warning(
  2522. "Auxiliary client: openai-codex requested without a model; "
  2523. "pass model explicitly (auxiliary.<task>.model in config.yaml)."
  2524. )
  2525. return None, None
  2526. codex_token, base_url = _resolve_codex_credential_and_base()
  2527. if not codex_token:
  2528. return None, None
  2529. logger.debug("Auxiliary client: Codex OAuth (%s via Responses API)", model)
  2530. real_client = _create_openai_client(
  2531. api_key=codex_token, base_url=base_url,
  2532. default_headers=_codex_cloudflare_headers(codex_token, base_url=base_url),
  2533. )
  2534. return CodexAuxiliaryClient(real_client, model), model
  2535. def _try_azure_foundry(
  2536. *, model: Optional[str] = None, explicit_api_key: Optional[str] = None,
  2537. explicit_base_url: Optional[str] = None, api_mode: Optional[str] = None,
  2538. ) -> tuple[Optional[Any], Optional[str]]:
  2539. """Azure Foundry aux client via the main agent's ``_resolve_azure_foundry_runtime`` (api_key vs Entra
  2540. callable bearer, per-model api_mode, base_url overrides). Returns ``(client, model)`` or ``(None, None)``."""
  2541. try:
  2542. from hermes_cli.runtime_provider import _resolve_azure_foundry_runtime
  2543. from hermes_cli.auth import AuthError
  2544. from hermes_cli.config import load_config_readonly
  2545. except ImportError:
  2546. return None, None
  2547. try:
  2548. cfg = load_config_readonly()
  2549. model_cfg = cfg.get("model") if isinstance(cfg, dict) else {}
  2550. if not isinstance(model_cfg, dict):
  2551. model_cfg = {}
  2552. except Exception:
  2553. model_cfg = {}
  2554. try:
  2555. runtime = _resolve_azure_foundry_runtime(
  2556. requested_provider="azure-foundry", model_cfg=model_cfg,
  2557. explicit_api_key=explicit_api_key, explicit_base_url=explicit_base_url,
  2558. target_model=model,
  2559. )
  2560. except AuthError as exc:
  2561. logger.debug("Auxiliary azure-foundry: %s", exc)
  2562. return None, None
  2563. except Exception as exc:
  2564. logger.debug("Auxiliary azure-foundry runtime error: %s", exc)
  2565. return None, None
  2566. api_key = runtime.get("api_key")
  2567. base_url = str(runtime.get("base_url", "") or "")
  2568. runtime_api_mode = api_mode or runtime.get("api_mode") or "chat_completions"
  2569. # api_key may be a callable token provider; bail only on None/"".
  2570. if not (callable(api_key) or api_key) or not base_url:
  2571. return None, None
  2572. final_model = _normalize_resolved_model(model or str(model_cfg.get("default") or ""), "azure-foundry")
  2573. if not final_model:
  2574. # No fallback aux model for Azure (needs a deployment name): let the auto chain fall through instead of 404ing.
  2575. logger.debug(
  2576. "Auxiliary azure-foundry: no model resolved (model=%r, default=%r)",
  2577. model, model_cfg.get("default"),
  2578. )
  2579. return None, None
  2580. # The SDK drops api-version query params from the base URL; pass via default_query.
  2581. _clean_base, _dq = _extract_url_query_params(base_url)
  2582. extra: dict[str, Any] = {"default_query": _dq} if _dq else {}
  2583. client = _create_openai_client(api_key=api_key, base_url=_clean_base, **extra)
  2584. if runtime_api_mode == "codex_responses":
  2585. return CodexAuxiliaryClient(client, final_model), final_model
  2586. if runtime_api_mode == "anthropic_messages":
  2587. # api_key forwarded verbatim (string or Entra callable; build_anthropic_client installs the bearer hook).
  2588. return _maybe_wrap_anthropic(client, final_model, api_key, base_url, runtime_api_mode), final_model
  2589. return client, final_model
  2590. def _try_anthropic(explicit_api_key: Optional[str | Callable[[], str]] = None,
  2591. explicit_base_url: Optional[str] = None) -> tuple[Optional[Any], Optional[str]]:
  2592. try:
  2593. from agent.anthropic_adapter import build_anthropic_client
  2594. from agent.anthropic_credentials import resolve_anthropic_token
  2595. except ImportError:
  2596. return None, None
  2597. pool_present, entry = _select_pool_entry("anthropic")
  2598. if pool_present and entry is not None:
  2599. token = explicit_api_key or _pool_runtime_api_key(entry)
  2600. else:
  2601. # Pool absent/empty: legacy resolver so a dead pool entry can't wedge aux tasks when a standalone credential exists.
  2602. entry = None
  2603. token = explicit_api_key or resolve_anthropic_token()
  2604. if not token:
  2605. return None, None
  2606. # Honor config.yaml model.base_url only when provider is anthropic AND the URL is
  2607. # Anthropic-compatible; a foreign host (Codex, OpenRouter) would 401 every aux call.
  2608. base_url = _pool_runtime_base_url(entry, _ANTHROPIC_DEFAULT_BASE_URL) if pool_present else _ANTHROPIC_DEFAULT_BASE_URL
  2609. with contextlib.suppress(Exception):
  2610. from hermes_cli.config import load_config_readonly
  2611. cfg = load_config_readonly()
  2612. model_cfg = cfg.get("model")
  2613. if isinstance(model_cfg, dict):
  2614. cfg_provider = str(model_cfg.get("provider") or "").strip().lower()
  2615. if cfg_provider == "anthropic":
  2616. cfg_base_url = (model_cfg.get("base_url") or "").strip().rstrip("/")
  2617. if cfg_base_url and _is_anthropic_compatible_host(cfg_base_url):
  2618. base_url = cfg_base_url
  2619. # A caller-supplied endpoint (fallback_providers entry, custom_providers entry) wins over
  2620. # both the pool row and config.yaml, under the same Anthropic-compatible host rule the
  2621. # primary path applies. A foreign host is REFUSED outright rather than silently demoted to
  2622. # the canonical host: continuing would send the explicit credential to a target the caller
  2623. # did not ask for (#121359).
  2624. override_url = (explicit_base_url or "").strip().rstrip("/")
  2625. if override_url:
  2626. if not _is_anthropic_compatible_host(override_url):
  2627. logger.warning(
  2628. "Auxiliary client: refusing anthropic explicit base_url %r — not an "
  2629. "Anthropic-compatible host; no client built and no request sent.",
  2630. override_url,
  2631. )
  2632. return None, None
  2633. base_url = override_url
  2634. from agent.anthropic_credentials import _is_oauth_token
  2635. is_oauth = _is_oauth_token(token)
  2636. model = _get_aux_model_for_provider("anthropic") or "claude-haiku-4-5-20251001"
  2637. if _aux_probe_active():
  2638. # Probe: token + adapter import resolved; skip real client construction.
  2639. return _AuxProbeClientStub(api_key="", base_url=base_url), model
  2640. logger.debug("Auxiliary client: Anthropic native (%s) at %s (oauth=%s)", model, base_url, is_oauth)
  2641. try:
  2642. real_client = build_anthropic_client(token, base_url)
  2643. except ImportError:
  2644. return None, None # Adapter imports fine but the anthropic SDK itself is missing.
  2645. return AnthropicAuxiliaryClient(real_client, model, token, base_url, is_oauth=is_oauth), model
  2646. _MAIN_RUNTIME_FIELDS = ("provider", "model", "base_url", "api_key", "api_mode", "auth_mode")
  2647. _MAIN_RUNTIME_CONTEXT_FIELDS = _MAIN_RUNTIME_FIELDS + (
  2648. "requested_provider", "session_id", "cache_scope",
  2649. )
  2650. def _normalize_main_runtime(main_runtime: Optional[dict[str, Any]]) -> dict[str, Any]:
  2651. """Return a sanitized copy of a live main-runtime override.
  2652. ``api_key`` may be a zero-arg callable (Entra ID token provider, accepted by the OpenAI SDK)
  2653. — preserved as-is so aux clients share main-agent auth.
  2654. """
  2655. if main_runtime is None:
  2656. main_runtime = _RUNTIME_MAIN_CONTEXT.get()
  2657. if not isinstance(main_runtime, dict):
  2658. return {}
  2659. normalized: dict[str, Any] = {}
  2660. for field in _MAIN_RUNTIME_CONTEXT_FIELDS:
  2661. value = main_runtime.get(field)
  2662. if field == "api_key" and callable(value) and not isinstance(value, str):
  2663. normalized[field] = value
  2664. elif isinstance(value, str) and value.strip():
  2665. normalized[field] = value.strip()
  2666. for identity_field in ("provider", "requested_provider"):
  2667. identity = normalized.get(identity_field)
  2668. if isinstance(identity, str):
  2669. normalized[identity_field] = identity.lower()
  2670. return normalized
  2671. def _get_provider_chain() -> list[tuple]:
  2672. """Ordered provider detection chain, built at call time so ``_try_*`` patches are picked up.
  2673. ``openai-codex`` is deliberately absent (shifting allow-list breaks guessed-model fallback).
  2674. """
  2675. return [
  2676. ("openrouter", _try_openrouter), ("nous", _try_nous),
  2677. ("local/custom", _try_custom_endpoint), ("api-key", _resolve_api_key_provider),
  2678. ]
  2679. # "Recently 402'd" unhealthy-provider cache: a depleted provider stays so for hours, so hiding it
  2680. # for a TTL saves an RTT per aux call. In-process only (profiles may use different keys).
  2681. _AUX_UNHEALTHY_TTL_SECONDS = 600 # 10 minutes
  2682. _aux_unhealthy_until: dict[Any, float] = {}
  2683. _aux_unhealthy_logged_at: dict[Any, float] = {}
  2684. _aux_unhealthy_reason: dict[Any, str] = {}
  2685. # resolved_provider / explicit-config names → chain labels.
  2686. _AUX_UNHEALTHY_LABEL_ALIASES = {
  2687. "openrouter": "openrouter", "nous": "nous", "custom": "local/custom",
  2688. "local/custom": "local/custom", "openai-codex": "openai-codex", "codex": "openai-codex",
  2689. }
  2690. def _normalize_chain_label(provider: str) -> str:
  2691. """resolved_provider → chain label; unknown API-key providers fall back to the lowercased input."""
  2692. if not provider:
  2693. return ""
  2694. p = str(provider).strip().lower()
  2695. return _AUX_UNHEALTHY_LABEL_ALIASES.get(p, p)
  2696. _AUX_UNHEALTHY_PAYMENT_REASON = "payment / credit error"
  2697. def _mark_provider_unhealthy(
  2698. provider: str, ttl: Optional[float] = None, *, base_url: Optional[str] = None,
  2699. reason: str = _AUX_UNHEALTHY_PAYMENT_REASON, level: int = logging.WARNING,
  2700. ) -> None:
  2701. """Hide one provider endpoint until the TTL expires. ``reason`` is what the log says and what the
  2702. skip line echoes: absent credentials are an expected state (DEBUG), a confirmed 402 is a fault
  2703. (WARNING) — the old fixed payment wording sent local-only users chasing billing (#64144)."""
  2704. label = _normalize_chain_label(provider)
  2705. if not label:
  2706. return
  2707. key = _unhealthy_cache_key(label, base_url)
  2708. ttl = _AUX_UNHEALTHY_TTL_SECONDS if ttl is None else ttl
  2709. expires_at = time.time() + ttl
  2710. _aux_unhealthy_until[key] = expires_at
  2711. _aux_unhealthy_reason[key] = reason
  2712. logger.log(
  2713. level,
  2714. "Auxiliary: marking %s unhealthy for %ds (%s). "
  2715. "Subsequent auxiliary calls will skip it until %s.",
  2716. label, int(ttl), reason, time.strftime("%H:%M:%S", time.localtime(expires_at)),
  2717. )
  2718. def _is_provider_unhealthy(label: str, base_url: Optional[str] = None) -> bool:
  2719. """True iff this provider endpoint is unhealthy and unexpired; lazily evicts expired entries."""
  2720. if not label:
  2721. return False
  2722. key = _unhealthy_cache_key(label, base_url)
  2723. expires_at = _aux_unhealthy_until.get(key)
  2724. if expires_at is None:
  2725. return False
  2726. if time.time() >= expires_at:
  2727. _aux_unhealthy_until.pop(key, None)
  2728. _aux_unhealthy_logged_at.pop(key, None)
  2729. _aux_unhealthy_reason.pop(key, None)
  2730. return False
  2731. return True
  2732. def _log_skip_unhealthy(
  2733. label: str, task: Optional[str] = None, *, base_url: Optional[str] = None,
  2734. ) -> None:
  2735. """Log a skipped unhealthy provider at most once per minute per label."""
  2736. now = time.time()
  2737. key = _unhealthy_cache_key(label, base_url)
  2738. if now - _aux_unhealthy_logged_at.get(key, 0.0) >= 60:
  2739. _aux_unhealthy_logged_at[key] = now
  2740. expires_at = _aux_unhealthy_until.get(key, now)
  2741. logger.info(
  2742. "Auxiliary %s: skipping %s (%s, retry in %ds)",
  2743. task or "call", label, _aux_unhealthy_reason.get(key, _AUX_UNHEALTHY_PAYMENT_REASON),
  2744. max(0, int(expires_at - now)),
  2745. )
  2746. def _reset_aux_unhealthy_cache() -> None:
  2747. """Clear the unhealthy cache (tests / explicit user reset)."""
  2748. _aux_unhealthy_until.clear()
  2749. _aux_unhealthy_logged_at.clear()
  2750. _aux_unhealthy_reason.clear()
  2751. def _contains_any(text: str, needles: tuple[str, ...]) -> bool:
  2752. """True when any needle is a substring of ``text``."""
  2753. return any(kw in text for kw in needles)
  2754. # Billing-body markers (credit exhaustion wrapped in 402/403/404/429 bodies). The main classifier's
  2755. # ``_BILLING_PATTERNS`` is the single source (#107166: the two lists had drifted, so OpenRouter's org
  2756. # "Budget limit exceeded" / "Key limit exceeded" 403s were billing for the main loop but not for the aux
  2757. # ladder, and aux fallback never fired); the aux list only adds broader phrasings and daily/weekly quota
  2758. # exhaustion (functionally credit exhaustion; "resource exhausted" is the Vertex/gRPC quota phrasing —
  2759. # also serialized by SDK wrappers and NIM as RESOURCE_EXHAUSTED / ResourceExhausted / resource-exhausted).
  2760. _PAYMENT_KEYWORDS = _BILLING_PATTERNS + (
  2761. "credits", "insufficient funds", "can only afford", "billing",
  2762. "isn't available on the free tier", "key limit exceeded", "budget limit",
  2763. "requires a subscription", "upgrade for access", "upgrade for higher limits",
  2764. "reached your session usage limit", "quota exceeded", "quota_exceeded",
  2765. "too many tokens per day", "daily limit", "tokens per day", "daily quota", "resource exhausted",
  2766. "resource_exhausted", "resource-exhausted", "resourceexhausted",
  2767. "weekly usage limit", "weekly limit",
  2768. )
  2769. def _is_payment_error(exc: Exception) -> bool:
  2770. """Payment/credit/quota exhaustion: HTTP 402, or a billing/quota body on 403/404/429/no-status."""
  2771. status = getattr(exc, "status_code", None)
  2772. return status == 402 or (
  2773. status in {403, 404, 429, None} and _contains_any(str(exc).lower(), _PAYMENT_KEYWORDS)
  2774. )
  2775. def _nous_portal_account_has_fresh_paid_access() -> bool:
  2776. """Return True only when the fresh Nous account API says paid access is allowed."""
  2777. try:
  2778. from hermes_cli.nous_account import get_nous_portal_account_info
  2779. return get_nous_portal_account_info(force_fresh=True).paid_service_access is True
  2780. except Exception as exc:
  2781. logger.debug("Auxiliary Nous paid-entitlement refresh check failed: %s", exc)
  2782. return False
  2783. _RATE_LIMIT_KEYWORDS = (
  2784. "rate limit", "rate_limit", "too many requests", "try again", "retry after", "resets in"
  2785. )
  2786. _RATE_LIMIT_BILLING_KEYWORDS = (
  2787. "credits", "insufficient funds", "billing", "payment required", "can only afford",
  2788. "out of funds", "run out of funds", "balance_depleted", "no usable credits",
  2789. "model_not_supported_on_free_tier", "not available on the free tier", "isn't available on the free tier",
  2790. )
  2791. def _is_overloaded_error(exc: Exception) -> bool:
  2792. """Server busy, credential fine ("at capacity upstream … not your API key's rate limit"): back off,
  2793. never bench the credential (#108349). Same phrase table as the main classifier."""
  2794. return _contains_any(str(exc).lower(), _OVERLOADED_PATTERNS)
  2795. def _is_rate_limit_error(exc: Exception) -> bool:
  2796. """429 rate limit (not billing/quota, which _is_payment_error owns).
  2797. OpenAI's RateLimitError may omit .status_code — matched by class name. A generic 429 without
  2798. billing keywords counts as a rate limit.
  2799. """
  2800. # (PR #8023 pattern)
  2801. if type(exc).__name__ == "RateLimitError" or _botocore_error(exc)[1] == "ThrottlingException":
  2802. return True
  2803. if _exc_http_status(exc) != 429:
  2804. return False
  2805. err_lower = str(exc).lower()
  2806. return _contains_any(err_lower, _RATE_LIMIT_KEYWORDS) or not _contains_any(err_lower, _RATE_LIMIT_BILLING_KEYWORDS)
  2807. def _is_timeout_error(exc: Exception) -> bool:
  2808. """Full-budget request timeout, distinct from a fast connection drop.
  2809. A timeout burns the whole ``timeout`` budget, so a same-provider retry on the compression
  2810. path doubles wall time; fast drops stay on the retry path.
  2811. """
  2812. with contextlib.suppress(ImportError):
  2813. from openai import APITimeoutError
  2814. if isinstance(exc, APITimeoutError):
  2815. return True
  2816. return "Timeout" in type(exc).__name__ or "timed out" in str(exc).lower()
  2817. def _is_connection_error(exc: Exception) -> bool:
  2818. """Connection/network errors (endpoint unreachable), as opposed to 4xx/5xx API errors."""
  2819. with contextlib.suppress(ImportError):
  2820. from openai import APIConnectionError, APITimeoutError
  2821. if isinstance(exc, (APIConnectionError, APITimeoutError)):
  2822. return True
  2823. if _contains_any(type(exc).__name__, ("Connection", "Timeout", "DNS", "SSL")):
  2824. return True
  2825. return _contains_any(str(exc).lower(), (
  2826. "connection refused", "name or service not known", "no route to host",
  2827. "network is unreachable", "timed out", "connection reset",
  2828. # httpcore/httpx premature stream close — transient, retry/reroute.
  2829. "incomplete chunked read", "peer closed connection", "response ended prematurely",
  2830. "unexpected eof", "remoteprotocolerror", "localprotocolerror",
  2831. ))
  2832. def _is_transient_transport_error(exc: Exception) -> bool:
  2833. """One-off transport blip worth retrying on the SAME provider: connection/stream-close errors plus pure 5xx/408.
  2834. Deliberately narrow: payment/auth/rate-limit errors switch provider, refresh creds, or rotate the pool.
  2835. """
  2836. if _is_connection_error(exc):
  2837. return True
  2838. status = _exc_http_status(exc)
  2839. return isinstance(status, int) and (status == 408 or 500 <= status < 600)
  2840. _DEFAULT_TRANSIENT_RETRIES = 2
  2841. _TRANSIENT_RETRY_BACKOFF_BASE = 1.0 # Backoff base (seconds); overridable so tests can zero it out.
  2842. def _transient_retry_count() -> int:
  2843. """Same-provider retries for a transient blip: ``auxiliary.transient_retries``
  2844. (default 2), clamped to [0, 6]; config-read failures fall back to default."""
  2845. try:
  2846. from hermes_cli.config import cfg_get, load_config
  2847. val = cfg_get(load_config(), "auxiliary", "transient_retries")
  2848. return _DEFAULT_TRANSIENT_RETRIES if val is None else max(0, min(int(val), 6))
  2849. except Exception:
  2850. return _DEFAULT_TRANSIENT_RETRIES
  2851. def _is_auth_error(exc: Exception) -> bool:
  2852. """Auth failures that should trigger provider-specific refresh."""
  2853. status = getattr(exc, "status_code", None)
  2854. if status == 401:
  2855. return True
  2856. err_lower = str(exc).lower()
  2857. if "error code: 401" in err_lower or "authenticationerror" in type(exc).__name__.lower():
  2858. return True
  2859. # xAI returns 403 "unauthenticated:bad-credentials" for expired OAuth tokens — semantically a 401.
  2860. return "bad-credentials" in err_lower and (status == 403 or "unauthenticated" in err_lower)
  2861. def _is_unsupported_parameter_error(exc: Exception, param: str) -> bool:
  2862. """Provider 400 for an unsupported request parameter: the parameter name plus a generic
  2863. unsupported/unknown/unrecognized marker, so call sites can retry without the key."""
  2864. param_lower = (param or "").lower()
  2865. if not param_lower:
  2866. return False
  2867. err_lower = str(exc).lower()
  2868. return param_lower in err_lower and _contains_any(err_lower, UNSUPPORTED_PARAM_MARKERS)
  2869. def _is_structured_output_rejection(exc: Exception) -> bool:
  2870. """Provider 400/422 rejecting the structured-output field, on either wire: OpenAI ``response_format``
  2871. (incl. vLLM's ``guided_grammar``/xgrammar failures) or Anthropic ``output_config.format`` ("Extra inputs
  2872. are not permitted"). Callers tolerate an unconstrained reply, so the reaction is one retry without it."""
  2873. status = getattr(exc, "status_code", None)
  2874. if status is not None and status not in {400, 422}:
  2875. return False
  2876. err_lower = str(exc).lower()
  2877. # vLLM grammar-backend failures name the translated parameter, not ours.
  2878. if _contains_any(err_lower, ("guided_grammar", "xgrammar", "compile_grammar_error")):
  2879. return True
  2880. if "extra inputs are not permitted" in err_lower and (
  2881. "response_format" in err_lower or "output_config" in err_lower
  2882. ):
  2883. return True
  2884. if "response_format" in err_lower and "unavailable" in err_lower:
  2885. return True
  2886. # Gateways that validate the request body with a strict pydantic model reject the
  2887. # OBJECT-form json_schema by shape ("str type expected" on response_format.json_schema,
  2888. # 422) rather than by naming the feature. The field is what they refuse; the retry
  2889. # without it is the same remedy, so treat the shape error as a rejection too.
  2890. if "response_format" in err_lower and "json_schema" in err_lower:
  2891. return True
  2892. # Gemini native names its own generationConfig keys, never ours: "Function calling with a response
  2893. # mime type: 'application/json' is unsupported" (pre-Gemini-3 + tools via a proxy), or an
  2894. # "Unknown name"/"Invalid value" 400 on response_schema / response_json_schema for a schema the
  2895. # surface cannot express. Same remedy: one retry without the format.
  2896. if _contains_any(err_lower, ("response mime type", "response_schema", "response_json_schema")):
  2897. return True
  2898. return _is_unsupported_parameter_error(exc, "response_format") or _is_unsupported_parameter_error(exc, "output_config")
  2899. def _is_reasoning_field_rejection(exc: Exception) -> bool:
  2900. """Provider 400 rejecting a reasoning wire control by name (``reasoning_effort``, ``reasoning``,
  2901. ``thinking``/``think``). Chat-only models behind OpenAI-compatible relays reject the top-level
  2902. ``reasoning_effort: none`` a disabled ``reasoning_config`` projects on the custom profile
  2903. ("Unrecognized request argument supplied: reasoning_effort", #112781); the route default is
  2904. the right answer for such a model, so the reaction is one retry without any reasoning field."""
  2905. status = getattr(exc, "status_code", None)
  2906. if status is not None and status not in {400, 422}:
  2907. return False
  2908. return is_reasoning_field_rejection(str(exc))
  2909. def _is_reasoning_required_rejection(exc: Exception) -> bool:
  2910. """Provider 400 refusing to switch reasoning OFF ("Reasoning is mandatory for this endpoint and cannot
  2911. be disabled"): the field is understood, only the disable is refused, so the rung steps the effort up to
  2912. the floor instead of dropping the field (agent/auxiliary_reasoning_floor.py)."""
  2913. status = getattr(exc, "status_code", None)
  2914. if status is not None and status not in {400, 422}:
  2915. return False
  2916. return is_reasoning_required_rejection(str(exc))
  2917. def _without_reasoning_fields(kwargs: dict) -> Optional[dict]:
  2918. """Copy *kwargs* without reasoning wire controls (top-level ``reasoning_effort``, the adapter's
  2919. private ``_reasoning_config`` and every ``extra_body`` reasoning key); None when nothing was
  2920. removed, so call sites don't retry an unchanged request."""
  2921. retry_kwargs = dict(kwargs)
  2922. changed = retry_kwargs.pop("reasoning_effort", None) is not None
  2923. changed = retry_kwargs.pop("_reasoning_config", None) is not None or changed
  2924. extra_body = retry_kwargs.get("extra_body")
  2925. if isinstance(extra_body, dict):
  2926. remaining = {k: v for k, v in extra_body.items() if str(k).strip().lower() not in _PROFILE_REASONING_KEYS}
  2927. if len(remaining) != len(extra_body):
  2928. if remaining:
  2929. retry_kwargs["extra_body"] = remaining
  2930. else:
  2931. retry_kwargs.pop("extra_body", None)
  2932. changed = True
  2933. return retry_kwargs if changed else None
  2934. def _is_model_not_found_error(exc: Exception) -> bool:
  2935. """"Requested model doesn't exist" (404 / invalid model) — typically a long-lived process pinned a
  2936. since-dropped model. Excludes billing keywords, which :func:`_is_payment_error` owns."""
  2937. status = getattr(exc, "status_code", None)
  2938. err_lower = str(exc).lower()
  2939. if _contains_any(err_lower, (
  2940. "credits", "insufficient funds", "billing", "out of funds", "balance_depleted",
  2941. "no usable credits", "free tier", "free-tier", "not available on the free tier",
  2942. )):
  2943. return False
  2944. if status not in {404, 400, None}:
  2945. return False
  2946. return _contains_any(err_lower, (
  2947. "model does not exist", "does not exist in our configuration", "openrouter catalog",
  2948. "is not a valid model", "no such model", "model not found",
  2949. "the model `", # OpenAI-style: "The model `X` does not exist"
  2950. "model_not_found", "unknown model",
  2951. ))
  2952. def _is_model_incompatible_error(exc: Exception) -> bool:
  2953. """"This route cannot serve this model" 400 (capability mismatch, e.g. a Codex/ChatGPT-account
  2954. fallback asked to run a non-OpenAI model). Auth/payment predicates don't fire, so this keeps the
  2955. chain going instead of aborting. Excludes billing 400s and not-found 400s."""
  2956. status = getattr(exc, "status_code", None)
  2957. if status not in {400, None}:
  2958. return False
  2959. err_lower = str(exc).lower()
  2960. if _is_model_not_found_error(exc):
  2961. return False
  2962. # Billing keywords checked directly: _is_payment_error is status-gated and misses 400-coded billing bodies.
  2963. if _contains_any(err_lower, (
  2964. "credits", "insufficient funds", "billing", "out of funds", "balance_depleted",
  2965. "no usable credits", "payment required", "free tier", "free-tier",
  2966. "not available on the free tier", "model_not_supported_on_free_tier", "quota",
  2967. )):
  2968. return False
  2969. return _contains_any(err_lower, (
  2970. "is not supported when using", # codex/ChatGPT-account model gating
  2971. "model is not supported", "not supported with this", "not supported for this account",
  2972. "model_not_supported", "does not support this model", "unsupported model",
  2973. ))
  2974. # Tasks on a user-visible critical path (compression blocks resuming an oversized session; vision
  2975. # stalls the serialised turn queue). A same-provider retry after a full-budget timeout costs another
  2976. # whole ``timeout`` window, so they skip straight to fallback; fast blips still retry.
  2977. # Fast blips (a streaming-close or a 5xx) still retry, since those are cheap. See issue #54465 for the
  2978. # compression case. Title generation joins them because its retries multiplied the user's
  2979. # ``auxiliary.title_generation.timeout`` (~4x: three full windows plus backoff) on a slow local
  2980. # model, and the auto-title thread outlived the deadline the user thought they had set (#89445, #66251).
  2981. _TIMEOUT_NO_RETRY_TASKS = frozenset({"compression", "vision", "title_generation"})
  2982. def _should_skip_same_provider_retry(task: Optional[str], exc: Exception) -> bool:
  2983. """True when a transient error on a critical-path task should go straight to fallback.
  2984. Carve-out: a fast first-token fail (dead stream within the no-progress window, zero output —
  2985. see ``_timeout_message``) is cheap and keeps the same-provider retry; mid-stream stalls and
  2986. hard-ceiling timeouts skip to fallback.
  2987. """
  2988. return task in _TIMEOUT_NO_RETRY_TASKS and _is_timeout_error(exc) and "no-progress timeout" not in str(exc)
  2989. def _evict_cached_clients(provider: str) -> None:
  2990. """Drop this profile's cached auxiliary clients for a provider so fresh creds are used.
  2991. Scoped to the calling profile (``hermes_home_key()`` is the first key slot): a rotation in
  2992. one profile must not drop another profile's client for the same provider in a multiplexing
  2993. gateway, since that profile's credentials did not change. Entries are popped, not closed:
  2994. a concurrent caller may be mid-request on the shared client (closing it raises ReadError /
  2995. "client has been closed" for them); the dropped client is retired by GC like the FIFO
  2996. overflow path in ``_get_cached_client``.
  2997. """
  2998. normalized = _normalize_aux_provider(provider)
  2999. home = hermes_home_key()
  3000. with _client_cache_lock:
  3001. for key in [key for key in _client_cache
  3002. if key[0] == home and _normalize_aux_provider(str(key[1])) == normalized]:
  3003. _client_cache.pop(key, None)
  3004. def _evict_cached_client_instance(target: Any) -> bool:
  3005. """Drop cache entries whose stored client (or its ``_real_client``) is *target*; True if any evicted.
  3006. Used when a cached client is poisoned (closed transport after a timeout). Async wrappers must
  3007. expose the same ``_real_client`` as their sync sibling or the async entry survives.
  3008. """
  3009. if target is None:
  3010. return False
  3011. evicted = False
  3012. with _client_cache_lock:
  3013. for key, entry in list(_client_cache.items()):
  3014. cached = entry[0] if entry is not None else None
  3015. if cached is not None and (cached is target or getattr(cached, "_real_client", None) is target):
  3016. del _client_cache[key]
  3017. evicted = True
  3018. return evicted
  3019. def _pool_credential_digest(pool: Any, entry: Any = None) -> str:
  3020. """Secret-safe digest over the runtime key(s) backing the cache hint.
  3021. When ``entry`` (the peeked selection) carries a key, only THAT key is digested: rotating a
  3022. sibling account must not churn this entry's cached client. Otherwise every pool entry
  3023. contributes, taken WITHOUT availability filters (``pool.entries()``, not ``peek()``): an entry
  3024. benched by a per-model cooldown still counts, so a token rotated while the cooldown is active
  3025. changes the digest too (#113022 point 5). Only the blake2b of the keys enters the cache key.
  3026. """
  3027. keys = [k for k in (_pool_runtime_api_key(entry),) if k] if entry is not None else []
  3028. if not keys:
  3029. try:
  3030. entries_fn = getattr(pool, "entries", None)
  3031. entries = entries_fn() if callable(entries_fn) else []
  3032. except Exception as exc:
  3033. logger.debug("Auxiliary client: could not list pool entries for digest: %s", exc)
  3034. return ""
  3035. keys = sorted(k for k in (_pool_runtime_api_key(e) for e in entries or ()) if k)
  3036. if not keys:
  3037. return ""
  3038. digest = hashlib.blake2b(digest_size=16)
  3039. for key in keys:
  3040. digest.update(key.encode("utf-8"))
  3041. digest.update(b"\0")
  3042. return digest.hexdigest()
  3043. def _pool_cache_hint(provider: str, *, main_runtime: Optional[dict[str, Any]] = None) -> str:
  3044. """Return a cache discriminator that follows the pooled credential, not just its entry id.
  3045. ``<provider>:<entry id>:<digest of that entry's key>`` when ``peek()`` names an entry,
  3046. ``<provider>::<digest of every entry's key>`` when every entry is filtered out (cooldown)
  3047. but keyed entries exist. A token rotated or
  3048. refreshed by another process yields a new client instead of a 401 round trip on the cached
  3049. one; the stale entry ages out through the non-closing FIFO cap (#113022).
  3050. """
  3051. normalized = _normalize_aux_provider(provider)
  3052. if normalized == "auto":
  3053. runtime = _normalize_main_runtime(main_runtime)
  3054. normalized = _normalize_aux_provider(runtime.get("provider") or _read_main_provider())
  3055. if normalized in {"", "auto", "custom"}:
  3056. return ""
  3057. pool = _load_pool_with_credentials(normalized, " (cache hint)")
  3058. if pool is None:
  3059. return ""
  3060. entry = _peek_pool_entry(normalized, pool)
  3061. digest = _pool_credential_digest(pool, entry)
  3062. entry_id = str(getattr(entry, "id", "") or "").strip() if entry is not None else ""
  3063. if not entry_id and not digest:
  3064. return ""
  3065. return f"{normalized}:{entry_id}:{digest}"
  3066. # Ordered (host, provider) tables for inferring a backend from a client base URL (then profile ``hostnames``).
  3067. _POOL_PROVIDER_BY_HOST = (
  3068. ("chatgpt.com", "openai-codex"), ("openrouter.ai", "openrouter"),
  3069. ("inference-api.nousresearch.com", "nous"), ("api.anthropic.com", "anthropic"),
  3070. ("api.kimi.com", "kimi-coding"), ("api.x.ai", "xai-oauth"),
  3071. )
  3072. _AUTH_REFRESH_PROVIDER_BY_HOST = (
  3073. ("chatgpt.com", "openai-codex"),
  3074. ("api.anthropic.com", "anthropic"), ("inference-api.nousresearch.com", "nous"),
  3075. # An aux call that inherits the main xai-oauth route arrives as "auto"; without this row the
  3076. # 403 bad-credentials rung skipped the refresh and benched the only grant (#84845).
  3077. ("api.x.ai", "xai-oauth"),
  3078. )
  3079. def _provider_for_host(base_url: str, table: tuple[tuple[str, str], ...]) -> Optional[str]:
  3080. """First provider in ``table`` whose host matches ``base_url``, else a profile claiming the host."""
  3081. for host, provider in table:
  3082. if base_url_host_matches(base_url, host):
  3083. return provider
  3084. from providers import profile_for_host
  3085. return getattr(profile_for_host(base_url), "name", None)
  3086. def _recoverable_pool_provider(
  3087. resolved_provider: str, client: Any, main_runtime: Optional[dict[str, Any]] = None
  3088. ) -> Optional[str]:
  3089. """Infer which provider pool can recover the current auxiliary client.
  3090. None when the client targets a different host than the session's configured endpoint for that
  3091. provider: a rejection there says nothing about the key, so rotating/quarantining it would kill a
  3092. working credential (Miho report — proxy users)."""
  3093. normalized = _normalize_aux_provider(resolved_provider)
  3094. base = str(getattr(client, "base_url", "") or "")
  3095. runtime = _normalize_main_runtime(main_runtime)
  3096. rt_base = str(runtime.get("base_url") or "")
  3097. rt_key = runtime.get("api_key")
  3098. client_key = getattr(client, "api_key", None)
  3099. # Only the SESSION's own key is shielded, and only when it was sent somewhere other than the
  3100. # session's origin (scheme+host+port — a port or HTTPS→HTTP change is a different trust boundary).
  3101. # An independently owned auxiliary pool keeps rotating at its own origin.
  3102. if (base and rt_base and normalized == runtime.get("provider")
  3103. and isinstance(rt_key, str) and rt_key and client_key == rt_key
  3104. and base_url_origin(base) != base_url_origin(rt_base)):
  3105. logger.info("Auxiliary: %s rejected the session key at %s, but the session's endpoint is %s — "
  3106. "endpoint mismatch, not a dead key; skipping credential rotation",
  3107. normalized, base_url_hostname(base), base_url_hostname(rt_base))
  3108. return None
  3109. if normalized not in {"", "auto", "custom"}: # a pinned ``custom:<name>`` keeps its entry's identity
  3110. return pinned if (pinned := str(resolved_provider or "").strip().lower()).startswith("custom:") else normalized
  3111. known = _provider_for_host(base, _POOL_PROVIDER_BY_HOST)
  3112. if known is not None:
  3113. return known
  3114. # Providers outside the table (e.g. opencode-go): match base URL against registered
  3115. # api_key providers so pool rotation works for them too.
  3116. if main_runtime:
  3117. runtime = _normalize_main_runtime(main_runtime)
  3118. rt_provider = runtime.get("provider", "")
  3119. if rt_provider and rt_provider not in {"", "auto", "custom"}:
  3120. with contextlib.suppress(Exception):
  3121. from hermes_cli.auth import PROVIDER_REGISTRY
  3122. pconfig = PROVIDER_REGISTRY.get(rt_provider)
  3123. if pconfig and getattr(pconfig, "auth_type", None) == "api_key":
  3124. # The pool's key was issued for the endpoint the main runtime actually uses; a
  3125. # rejection at any other host (registry default vs configured proxy) says nothing
  3126. # about that key, so it must not be marked exhausted.
  3127. rt_base = str(runtime.get("base_url") or getattr(pconfig, "inference_base_url", "") or "").rstrip("/")
  3128. if rt_base and base_url_host_matches(base, base_url_hostname(rt_base)):
  3129. return rt_provider
  3130. return None
  3131. def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str = "") -> bool:
  3132. """Try same-provider credential-pool recovery for auxiliary calls.
  3133. ``failed_api_key`` lets mark_exhausted_and_rotate identify the right pool entry even if
  3134. another process already rotated (current() would be None).
  3135. """
  3136. normalized = _normalize_aux_provider(provider)
  3137. try:
  3138. pool = pool_for_aux_provider(provider, normalized, load_pool)
  3139. except Exception as load_exc:
  3140. logger.debug("Auxiliary client: could not load pool for %s recovery: %s", normalized, load_exc)
  3141. return False
  3142. if not pool or not pool.has_credentials():
  3143. return False
  3144. status_code = getattr(exc, "status_code", None)
  3145. def _rotate(fallback_status: int) -> bool:
  3146. error_context: dict[str, Any] = {"message": str(exc)}
  3147. if status_code is not None:
  3148. error_context["status_code"] = status_code
  3149. next_entry = pool.mark_exhausted_and_rotate(
  3150. status_code=status_code if status_code is not None else fallback_status,
  3151. error_context=error_context, api_key_hint=failed_api_key or None,
  3152. )
  3153. if next_entry is None:
  3154. return False
  3155. _evict_cached_clients(normalized)
  3156. return True
  3157. if _is_auth_error(exc):
  3158. if pool.try_refresh_current() is not None:
  3159. _evict_cached_clients(normalized)
  3160. return True
  3161. return _rotate(401)
  3162. if _is_payment_error(exc):
  3163. return _rotate(402)
  3164. if _is_rate_limit_error(exc) and not _is_overloaded_error(exc):
  3165. return _rotate(429)
  3166. return False
  3167. def _prepare_same_provider_retry(
  3168. *, task: Optional[str], resolved_provider: str, resolved_model: Optional[str],
  3169. resolved_base_url: Optional[str], resolved_api_key: Optional[str],
  3170. resolved_api_mode: Optional[str], main_runtime: Optional[dict[str, Any]],
  3171. final_model: Optional[str], messages: list, temperature: Optional[float],
  3172. max_tokens: Optional[int], tools: Optional[list], effective_timeout: float,
  3173. effective_extra_body: dict, reasoning_config: Optional[dict], async_mode: bool,
  3174. extra_headers: Optional[dict[str, str]] = None,
  3175. ) -> tuple[Any, dict[str, Any]]:
  3176. """Rebuild (client, request kwargs) for a same-provider retry after credential recovery."""
  3177. if task == "vision":
  3178. effective_provider, retry_client, retry_model = resolve_vision_provider_client(
  3179. provider=resolved_provider, model=final_model, base_url=resolved_base_url,
  3180. api_key=resolved_api_key, async_mode=async_mode,
  3181. )
  3182. else:
  3183. retry_client, retry_model = _get_cached_client(
  3184. resolved_provider, resolved_model, async_mode=async_mode, base_url=resolved_base_url,
  3185. api_key=resolved_api_key, api_mode=resolved_api_mode, main_runtime=main_runtime,
  3186. )
  3187. effective_provider = _effective_provider_for_client(retry_client, resolved_provider)
  3188. if retry_client is None:
  3189. raise RuntimeError(
  3190. f"Auxiliary {task or 'call'}: provider {resolved_provider} could not be rebuilt after recovery"
  3191. )
  3192. retry_base = str(getattr(retry_client, "base_url", "") or "")
  3193. retry_kwargs = _build_call_kwargs(
  3194. effective_provider or resolved_provider, retry_model or final_model, messages,
  3195. temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout,
  3196. extra_body=effective_extra_body, reasoning_config=reasoning_config,
  3197. base_url=retry_base or resolved_base_url, task=task,
  3198. )
  3199. # Preserve per-request attribution headers (a profile's ``request_headers``) across the rebuilt-client
  3200. # retry: dropping them would let a recovery retry silently lose capability gating (#60293).
  3201. if extra_headers:
  3202. retry_kwargs["extra_headers"] = dict(extra_headers)
  3203. if _is_anthropic_compat_endpoint(resolved_provider, retry_base):
  3204. retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"])
  3205. return retry_client, retry_kwargs
  3206. def _retry_same_provider_sync(*, resolved_provider: str, resolved_api_mode: Optional[str], task: Optional[str], **prep) -> Any:
  3207. retry_client, retry_kwargs = _prepare_same_provider_retry(
  3208. task=task, resolved_provider=resolved_provider, resolved_api_mode=resolved_api_mode, async_mode=False, **prep,
  3209. )
  3210. return _validate_llm_response(
  3211. _relay_sync_completion(retry_client, retry_kwargs, provider=resolved_provider, api_mode=resolved_api_mode), task,
  3212. )
  3213. async def _retry_same_provider_async(*, resolved_provider: str, resolved_api_mode: Optional[str], task: Optional[str], **prep) -> Any:
  3214. import asyncio
  3215. retry_client, retry_kwargs = _prepare_same_provider_retry(
  3216. task=task, resolved_provider=resolved_provider, resolved_api_mode=resolved_api_mode, async_mode=True, **prep,
  3217. )
  3218. return await asyncio.to_thread(
  3219. _validate_llm_response,
  3220. await _relay_async_completion(retry_client, retry_kwargs, provider=resolved_provider, api_mode=resolved_api_mode),
  3221. task,
  3222. )
  3223. def _creds_have_api_key(creds: dict[str, Any]) -> bool:
  3224. return bool(str(creds.get("api_key", "") or "").strip())
  3225. def _refresh_codex_credentials() -> bool:
  3226. from hermes_cli.auth import resolve_codex_runtime_credentials
  3227. return _creds_have_api_key(resolve_codex_runtime_credentials(force_refresh=True))
  3228. def _refresh_nous_credentials() -> bool:
  3229. from hermes_cli.auth import resolve_nous_runtime_credentials
  3230. return _creds_have_api_key(resolve_nous_runtime_credentials(
  3231. timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15), force_refresh=True
  3232. ))
  3233. def _refresh_anthropic_credentials(failed_api_key: str = "") -> bool:
  3234. from agent.anthropic_credentials import read_claude_code_credentials, _refresh_oauth_token
  3235. token = failed_api_key
  3236. if not token:
  3237. return False
  3238. pool = load_pool("anthropic")
  3239. if pool.entry_id_for_api_key(token):
  3240. return pool.try_refresh_matching(api_key_hint=token) is not None
  3241. creds = read_claude_code_credentials()
  3242. # Never spend an ambient login's refresh rotation for another request's key.
  3243. if isinstance(creds, dict) and creds.get("accessToken") == token and creds.get("refreshToken"):
  3244. return bool(_refresh_oauth_token(creds))
  3245. return False
  3246. def _refresh_xai_oauth_credentials() -> bool:
  3247. """Pool-level refresh first, then the singleton auth-store resolver."""
  3248. pool = load_pool("xai-oauth")
  3249. if pool and pool.has_credentials():
  3250. pool.select()
  3251. refreshed = pool.try_refresh_current()
  3252. if refreshed is not None and str(getattr(refreshed, "runtime_api_key", "") or "").strip():
  3253. return True
  3254. from hermes_cli.auth import resolve_xai_oauth_runtime_credentials
  3255. return _creds_have_api_key(resolve_xai_oauth_runtime_credentials(force_refresh=True))
  3256. def _refresh_vertex_credentials() -> bool:
  3257. """Mirrors run_agent's Vertex refresh; the cache key ignores the rotating bearer, so
  3258. without the eviction that follows, a ~1h-expired aux Vertex client 401s forever."""
  3259. from agent.vertex_adapter import get_vertex_config
  3260. token, base_url = get_vertex_config()
  3261. return bool(isinstance(token, str) and token.strip() and isinstance(base_url, str) and base_url.strip())
  3262. # Each refresher returns True when a usable credential exists; the caller then evicts cached clients.
  3263. _CREDENTIAL_REFRESHERS: dict[str, Callable[..., bool]] = {
  3264. "openai-codex": _refresh_codex_credentials,
  3265. "nous": _refresh_nous_credentials, "anthropic": _refresh_anthropic_credentials,
  3266. "xai-oauth": _refresh_xai_oauth_credentials, "vertex": _refresh_vertex_credentials,
  3267. }
  3268. def _refresh_provider_credentials(provider: str, *, failed_api_key: str = "") -> bool:
  3269. """Refresh short-lived credentials for OAuth-backed auxiliary providers."""
  3270. normalized = _normalize_aux_provider(provider)
  3271. refresher = _CREDENTIAL_REFRESHERS.get(normalized)
  3272. if refresher is None:
  3273. from providers import get_provider_profile
  3274. profile = get_provider_profile(normalized)
  3275. if profile is None:
  3276. return False
  3277. refresher = lambda: profile.refresh_runtime_credentials(reason="auxiliary_unauthorized")
  3278. try:
  3279. if not (refresher(failed_api_key) if normalized == "anthropic" else refresher()):
  3280. return False
  3281. _evict_cached_clients(normalized)
  3282. return True
  3283. except Exception as exc:
  3284. logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc)
  3285. return False
  3286. def _auth_refresh_provider_for_route(
  3287. resolved_provider: Optional[str], client_base_url: str, effective_provider: str = "",
  3288. ) -> str:
  3289. """Provider whose short-lived credentials should be refreshed; auto-routed calls keep
  3290. ``resolved_provider == "auto"``, so infer the backend from the client's base URL."""
  3291. normalized = _normalize_aux_provider(resolved_provider)
  3292. if normalized and normalized != "auto":
  3293. return normalized
  3294. host_provider = _provider_for_host(client_base_url, _AUTH_REFRESH_PROVIDER_BY_HOST)
  3295. # The auto client already knows it runs on the host's API-key sibling (``xai`` on api.x.ai):
  3296. # an XAI_API_KEY 401 must not spend a stale ``xai-oauth`` grant's refresh and switch routes.
  3297. if host_provider and host_provider == f"{_normalize_aux_provider(effective_provider)}-oauth":
  3298. return normalized
  3299. return host_provider or normalized
  3300. def _fallback_chain_entry(task: Optional[str], fb_label: str) -> Optional[dict[str, Any]]:
  3301. """Resolve the ``fallback_chain`` entry a ``fallback_chain[<i>](<provider>)`` label points at,
  3302. or None when the label is not a configured-chain candidate or the index no longer resolves."""
  3303. if not task or not fb_label:
  3304. return None
  3305. m = re.match(r"fallback_chain\[(\d+)\]", fb_label)
  3306. if not m:
  3307. return None
  3308. try:
  3309. chain = _get_auxiliary_task_config(task).get("fallback_chain")
  3310. entry = chain[int(m.group(1))] if isinstance(chain, list) else None
  3311. except Exception:
  3312. return None
  3313. return entry if isinstance(entry, dict) else None
  3314. def _coerce_positive_timeout(raw: Any) -> Optional[float]:
  3315. """Coerce a config ``timeout`` to a positive float, or None (rejects bools, which are ints)."""
  3316. if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw > 0:
  3317. return float(raw)
  3318. return None
  3319. def _fallback_entry_timeout(task: Optional[str], fb_label: str) -> Optional[float]:
  3320. """Per-entry ``timeout`` for a configured fallback candidate, or None (keep the task-level
  3321. timeout). Inheriting the primary's deadline used to kill healthy-but-slower fallbacks.
  3322. A fallback candidate previously inherited the exact timeout the primary provider was called with. When
  3323. that deadline was tuned for the primary (or the primary simply consumed its whole budget before failing
  3324. over), the fallback aborted on the same clock even when independently healthy — a 163k-token compression
  3325. that needs ~90s on the fallback died at the primary's 30s deadline every turn (#62452).
  3326. """
  3327. entry = _fallback_chain_entry(task, fb_label)
  3328. return _coerce_positive_timeout(entry.get("timeout") if entry else None)
  3329. def _fallback_provider_from_label(label: str) -> str:
  3330. """Recover the provider identifier from a fallback display label."""
  3331. match = re.match(r"(?:fallback_chain\[\d+\]|fallback_providers\[\d+\]|main-agent)\(([^)]+)\)$", label or "")
  3332. return match.group(1).strip() if match else str(label or "").strip()
  3333. class _FallbackDestination(NamedTuple):
  3334. provider: str
  3335. base_url: str
  3336. api_mode: Optional[str]
  3337. model: Optional[str]
  3338. def _complete_fallback_destination(
  3339. provider: str, base_url: str, api_mode: Optional[str], model: Optional[str]
  3340. ) -> _FallbackDestination:
  3341. if not api_mode:
  3342. if _endpoint_speaks_anthropic_messages(base_url):
  3343. api_mode = "anthropic_messages"
  3344. else:
  3345. with contextlib.suppress(Exception):
  3346. from hermes_cli.runtime_provider import resolve_runtime_provider
  3347. runtime = resolve_runtime_provider(
  3348. requested=provider, explicit_base_url=base_url or None, target_model=model or ""
  3349. )
  3350. api_mode = str(runtime.get("api_mode") or "").strip() or None
  3351. return _FallbackDestination(provider, base_url, api_mode, model)
  3352. def _fallback_destination_from_entry(
  3353. entry: dict[str, Any], fb_client: Any, fb_model: Optional[str]
  3354. ) -> _FallbackDestination:
  3355. provider = str(entry.get("provider") or "").strip()
  3356. base_url = str(entry.get("base_url") or getattr(fb_client, "base_url", "") or "").strip()
  3357. api_mode = str(entry.get("api_mode") or entry.get("transport") or "").strip() or None
  3358. model = fb_model or str(entry.get("model") or "").strip() or None
  3359. return _complete_fallback_destination(provider, base_url, api_mode, model)
  3360. def _fallback_destination(
  3361. task: Optional[str], fb_client: Any, fb_model: Optional[str], fb_label: str
  3362. ) -> _FallbackDestination:
  3363. """Route identity of a fallback request: attached destination, else configured entry, else label."""
  3364. attached = getattr(fb_client, "_hermes_fallback_destination", None)
  3365. if isinstance(attached, _FallbackDestination):
  3366. return attached
  3367. entry = _fallback_chain_entry(task, fb_label)
  3368. if entry is not None:
  3369. return _fallback_destination_from_entry(entry, fb_client, fb_model)
  3370. return _complete_fallback_destination(
  3371. _fallback_provider_from_label(fb_label), str(getattr(fb_client, "base_url", "") or ""), None, fb_model,
  3372. )
  3373. def _replan_synchronous_cache_sections(
  3374. messages: list, tools: Optional[list], *, destination: _FallbackDestination
  3375. ) -> tuple[list, list]:
  3376. """Plan one destination at the configured TTL (1h must not regress to 5m); one-shots never cache."""
  3377. from agent.agent_runtime_helpers import configured_cache_ttl, plan_cache_sections_for_destination
  3378. from agent.prompt_caching import is_one_shot_request
  3379. return plan_cache_sections_for_destination(
  3380. messages, tools, provider=destination.provider, base_url=destination.base_url,
  3381. api_mode=destination.api_mode or "", model=destination.model or "", cache_ttl=configured_cache_ttl(),
  3382. cache_disabled=True if is_one_shot_request(messages, tools) else None,
  3383. )
  3384. def _fallback_request_kwargs(
  3385. destination: _FallbackDestination, *, task: Optional[str], messages: list,
  3386. tools: Optional[list], temperature: Optional[float], max_tokens: Optional[int],
  3387. effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
  3388. fallback_entry: dict, task_config: dict, apply_fast_lane: bool,
  3389. ) -> dict[str, Any]:
  3390. """Build request kwargs for one fallback destination (cache-section replan + fast-lane cap)."""
  3391. fallback_max_tokens, fallback_extra_body = max_tokens, effective_extra_body
  3392. if apply_fast_lane:
  3393. fallback_max_tokens, fallback_extra_body = _compression_fast_lane_controls(
  3394. task, actual_provider=destination.provider, actual_model=destination.model,
  3395. requested_provider=fallback_entry.get("provider"),
  3396. requested_model=fallback_entry.get("model"), route_config=fallback_entry,
  3397. leak_guard_config=task_config, max_tokens=max_tokens, extra_body=effective_extra_body,
  3398. )
  3399. fallback_messages, fallback_tools = _replan_synchronous_cache_sections(messages, tools, destination=destination)
  3400. fb_kwargs = _build_call_kwargs(
  3401. destination.provider, destination.model, fallback_messages,
  3402. temperature=temperature, max_tokens=fallback_max_tokens, tools=fallback_tools, timeout=effective_timeout,
  3403. extra_body=fallback_extra_body, reasoning_config=reasoning_config, base_url=destination.base_url, task=task)
  3404. return fb_kwargs
  3405. def _plan_fallback_candidate(
  3406. fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str],
  3407. effective_timeout: float, apply_fast_lane: bool, **request,
  3408. ) -> tuple[_FallbackDestination, dict[str, Any], Callable[[str, Any, Optional[str]], dict[str, Any]]]:
  3409. """Resolve the destination + first-attempt kwargs for a fallback candidate.
  3410. Returns ``(destination, kwargs, rebuild)`` where ``rebuild(provider, client, model)`` produces
  3411. kwargs for the credential-refreshed retry destination. A configured-chain entry's own
  3412. ``timeout`` overrides ``effective_timeout``.
  3413. """
  3414. fb_timeout = _fallback_entry_timeout(task, fb_label)
  3415. if fb_timeout is not None and fb_timeout != effective_timeout:
  3416. logger.info(
  3417. "Auxiliary %s: %s using its configured timeout %.0fs "
  3418. "(task-level was %.0fs)",
  3419. task or "call", fb_label, fb_timeout, effective_timeout,
  3420. )
  3421. effective_timeout = fb_timeout
  3422. destination = _fallback_destination(task, fb_client, fb_model, fb_label)
  3423. task_config = _get_auxiliary_task_config(task) if task == "compression" else {}
  3424. fallback_entry = _fallback_chain_entry(task, fb_label) or {}
  3425. common = dict(
  3426. task=task, effective_timeout=effective_timeout, fallback_entry=fallback_entry,
  3427. task_config=task_config, apply_fast_lane=apply_fast_lane, **request,
  3428. )
  3429. def _rebuild(provider: str, client: Any, model: Optional[str]) -> tuple[_FallbackDestination, dict[str, Any]]:
  3430. retry_destination = _FallbackDestination(
  3431. provider, destination.base_url or str(getattr(client, "base_url", "") or ""),
  3432. destination.api_mode, model or destination.model,
  3433. )
  3434. return retry_destination, _fallback_request_kwargs(retry_destination, **common)
  3435. return destination, _fallback_request_kwargs(destination, **common), _rebuild
  3436. def _quarantine_fallback_candidate(
  3437. task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *,
  3438. base_url: str = "", tag: str = "", reason: Optional[str] = None,
  3439. ) -> None:
  3440. """The candidate cannot serve this walk (``reason`` = its ``_FALLBACK_REASONS`` capacity label,
  3441. None = dead token): mark it unhealthy so the ordered re-walk skips it and the caller moves on to
  3442. the next entry. Transient classes get a short hold, payment/quota and dead tokens the long one."""
  3443. _mark_provider_unhealthy(
  3444. fb_provider or fb_label, ttl=fallback_candidate_quarantine_ttl(reason),
  3445. base_url=base_url, reason=reason or "stale fallback credential")
  3446. why = f"is out of capacity ({reason})" if reason else "has a stale/unrefreshable credential"
  3447. logger.warning("Auxiliary %s%s: fallback candidate %s %s (%s) — skipping to next fallback",
  3448. task or "call", tag, fb_label, why, fb_err)
  3449. def _plan_fallback_auth_retry(
  3450. destination: _FallbackDestination,
  3451. rebuild: Callable[[str, Any, Optional[str]], tuple[_FallbackDestination, dict[str, Any]]], *,
  3452. async_mode: bool,
  3453. failed_api_key: str = "",
  3454. ) -> tuple[str, Optional[tuple[Any, dict[str, Any], _FallbackDestination]]]:
  3455. """After an auth error on a fallback candidate: refresh credentials and rebuild the request.
  3456. Returns ``(refresh_provider, retry)``; ``retry`` = ``(client, kwargs, destination)`` or None."""
  3457. fb_provider = _auth_refresh_provider_for_route(destination.provider, destination.base_url)
  3458. refresh_kwargs = {"failed_api_key": failed_api_key} if fb_provider == "anthropic" else {}
  3459. if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider, **refresh_kwargs):
  3460. retry_client, retry_model = _get_cached_client(
  3461. fb_provider, destination.model, **({"async_mode": True} if async_mode else {}),
  3462. base_url=destination.base_url or None, api_mode=destination.api_mode,
  3463. )
  3464. if retry_client is not None:
  3465. retry_destination, retry_kwargs = rebuild(fb_provider, retry_client, retry_model)
  3466. return fb_provider, (retry_client, retry_kwargs, retry_destination)
  3467. return fb_provider, None
  3468. def _call_fallback_candidate_sync(
  3469. fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str], messages: list,
  3470. temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
  3471. effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
  3472. ) -> Optional[Any]:
  3473. """Call one fallback candidate with stale-credential recovery: on an auth error refresh its
  3474. credentials and retry once with a rebuilt client; if that also auth-fails, quarantine the
  3475. provider and return None so the caller moves on. A capacity error (quota/rate-limit 429, 402,
  3476. connection, route-incompatible model, malformed response) also quarantines and returns None so
  3477. the ordered chain advances to the next configured entry (#106367); other errors raise.
  3478. ``effective_timeout`` is the task-level deadline; a configured-chain candidate with its own ``timeout``
  3479. entry gets that instead, so a fallback tuned differently from the primary is allowed its own budget
  3480. (#62452).
  3481. """
  3482. destination, fb_kwargs, rebuild = _plan_fallback_candidate(
  3483. fb_client, fb_model, fb_label, task=task, effective_timeout=effective_timeout,
  3484. apply_fast_lane=True, messages=messages, tools=tools, temperature=temperature,
  3485. max_tokens=max_tokens, effective_extra_body=effective_extra_body,
  3486. reasoning_config=reasoning_config,
  3487. )
  3488. def _send(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
  3489. return _validate_llm_response(
  3490. _relay_sync_completion(
  3491. client, request_kwargs, provider=dest.provider, api_mode=dest.api_mode,
  3492. create=lambda request: _create_with_progress(
  3493. client, request, task,
  3494. force_stream=_provider_requires_stream(dest.provider, dest.base_url),
  3495. ),
  3496. ),
  3497. task,
  3498. )
  3499. from agent.auxiliary_fallback_recovery import send_with_parameter_rungs
  3500. def _send_recovering(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
  3501. return send_with_parameter_rungs(
  3502. lambda c, kw: _send(c, kw, dest), client, request_kwargs, task=task)
  3503. try:
  3504. return _send_recovering(fb_client, fb_kwargs, destination)
  3505. except Exception as fb_err:
  3506. if not _is_auth_error(fb_err):
  3507. capacity = fallback_candidate_unavailable_reason(fb_err)
  3508. if capacity is None:
  3509. raise
  3510. _quarantine_fallback_candidate(
  3511. task, fb_label, destination.provider, fb_err, base_url=destination.base_url,
  3512. reason=capacity)
  3513. return None
  3514. fb_provider, retry = _plan_fallback_auth_retry(
  3515. destination, rebuild, async_mode=False, failed_api_key=getattr(fb_client, "api_key", ""))
  3516. failed_destination = destination
  3517. if retry is not None:
  3518. failed_destination = retry[2]
  3519. try:
  3520. return _send_recovering(*retry)
  3521. except Exception as retry_err:
  3522. if not _is_auth_error(retry_err) and fallback_candidate_unavailable_reason(retry_err) is None:
  3523. raise
  3524. _quarantine_fallback_candidate(
  3525. task, fb_label, fb_provider, fb_err, base_url=failed_destination.base_url,
  3526. )
  3527. return None
  3528. async def _call_fallback_candidate_async(
  3529. fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str], messages: list,
  3530. temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
  3531. effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
  3532. ) -> Optional[Any]:
  3533. """Async mirror of :func:`_call_fallback_candidate_sync` (no fast-lane cap on this wire)."""
  3534. destination, fb_kwargs, rebuild = _plan_fallback_candidate(
  3535. fb_client, fb_model, fb_label, task=task, effective_timeout=effective_timeout,
  3536. apply_fast_lane=False, messages=messages, tools=tools, temperature=temperature,
  3537. max_tokens=max_tokens, effective_extra_body=effective_extra_body,
  3538. reasoning_config=reasoning_config,
  3539. )
  3540. async def _send(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
  3541. import asyncio
  3542. return await asyncio.to_thread(
  3543. _validate_llm_response,
  3544. await _relay_async_completion(client, request_kwargs, provider=dest.provider, api_mode=dest.api_mode),
  3545. task,
  3546. )
  3547. from agent.auxiliary_fallback_recovery import send_with_parameter_rungs_async
  3548. async def _send_recovering(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
  3549. return await send_with_parameter_rungs_async(
  3550. lambda c, kw: _send(c, kw, dest), client, request_kwargs, task=task)
  3551. try:
  3552. return await _send_recovering(fb_client, fb_kwargs, destination)
  3553. except Exception as fb_err:
  3554. if not _is_auth_error(fb_err):
  3555. capacity = fallback_candidate_unavailable_reason(fb_err)
  3556. if capacity is None:
  3557. raise
  3558. _quarantine_fallback_candidate(
  3559. task, fb_label, destination.provider, fb_err, base_url=destination.base_url,
  3560. tag=" (async)", reason=capacity)
  3561. return None
  3562. fb_provider, retry = _plan_fallback_auth_retry(
  3563. destination, rebuild, async_mode=True, failed_api_key=getattr(fb_client, "api_key", ""))
  3564. failed_destination = destination
  3565. if retry is not None:
  3566. failed_destination = retry[2]
  3567. try:
  3568. return await _send_recovering(*retry)
  3569. except Exception as retry_err:
  3570. if not _is_auth_error(retry_err) and fallback_candidate_unavailable_reason(retry_err) is None:
  3571. raise
  3572. _quarantine_fallback_candidate(
  3573. task, fb_label, fb_provider, fb_err,
  3574. base_url=failed_destination.base_url, tag=" (async)",
  3575. )
  3576. return None
  3577. def _try_payment_fallback(
  3578. failed_provider: str, task: str | None = None, reason: str = "payment error", *,
  3579. failed_base_url: str = "", failure_scope: Any = None, main_runtime: Optional[dict[str, Any]] = None,
  3580. ) -> tuple[Optional[Any], Optional[str], str]:
  3581. """Try the auto-detection chain after a payment/credit or connection error, skipping the failed
  3582. provider (and the main-provider path when it maps to the same backend). Returns (client, model, label) or (None, None, "")."""
  3583. skip = failed_provider.lower().strip()
  3584. # The SESSION's provider decides whether discovery is allowed: a live `/model xai-oauth` session
  3585. # over a persisted ``provider: auto`` is a selection, so the disk value alone is not the answer.
  3586. main_provider = _normalize_main_runtime(main_runtime).get("provider") or _read_main_provider()
  3587. if not _discovery_chain_allowed(main_provider, task):
  3588. return None, None, ""
  3589. skip_labels = {skip}
  3590. if main_provider and main_provider.lower() in skip:
  3591. skip_labels.add(main_provider.lower())
  3592. skip_chain_labels = {_normalize_chain_label(s) for s in skip_labels}
  3593. skip_backend = _failed_backend_skip(
  3594. failed_provider, None, failed_base_url=failed_base_url, failure_scope=failure_scope)
  3595. tried = []
  3596. for label, try_fn in _get_provider_chain():
  3597. candidate_base_url = _custom_health_base_url(label)
  3598. if (not failed_base_url and label in skip_chain_labels) or skip_backend(
  3599. label, None, candidate_base_url):
  3600. continue
  3601. if _is_provider_unhealthy(label, candidate_base_url):
  3602. _log_skip_unhealthy(label, task, base_url=candidate_base_url)
  3603. tried.append(f"{label} (unhealthy)")
  3604. continue
  3605. client, model = try_fn()
  3606. if client is not None:
  3607. logger.info("Auxiliary %s: %s on %s — falling back to %s (%s)",
  3608. task or "call", reason, failed_provider, label, model or "default")
  3609. return client, model, label
  3610. tried.append(label)
  3611. logger.warning("Auxiliary %s: %s on %s and no fallback available (tried: %s)",
  3612. task or "call", reason, failed_provider, ", ".join(tried))
  3613. return None, None, ""
  3614. def _failed_backend_skip(
  3615. failed_provider: str, failed_model: Optional[str], *, failed_base_url: str = "",
  3616. failure_scope: Any = None,
  3617. ) -> Callable[..., bool]:
  3618. """Predicate ``skip(provider, model, base_url="")`` → True when a candidate must be skipped for the failed
  3619. route. Scope: ``failed_model`` → model-scoped (only that deployment; timeout/connection/rate-limit);
  3620. None → credential-wide (whole provider; auth/payment)."""
  3621. from agent.backend_identity import BackendIdentity, FailureScope, should_skip_candidate
  3622. skip_model = (failed_model or "").strip().lower() or None
  3623. failed_ident = BackendIdentity.build(
  3624. provider=failed_provider, model=skip_model, base_url=failed_base_url)
  3625. failure_scope = failure_scope or (FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL)
  3626. def _skip(provider: str, model: Optional[str], base_url: str = "") -> bool:
  3627. return should_skip_candidate(
  3628. BackendIdentity.build(provider=provider, model=model, base_url=base_url), failed_ident, failure_scope,
  3629. )
  3630. return _skip
  3631. def _try_main_agent_model_fallback(
  3632. failed_provider: str, task: str | None = None, reason: str = "error",
  3633. failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
  3634. ) -> tuple[Optional[Any], Optional[str], str]:
  3635. """Last-resort fallback to the main agent provider + model after the configured chain is exhausted.
  3636. ``failed_model`` scoping per ``_failed_backend_skip``; same-URL custom endpoints serve many models,
  3637. so a hung aux model says nothing about the main model's health. Returns (client, model, label) or (None, None, "")."""
  3638. main_provider = (_read_main_provider() or "").strip()
  3639. main_model = (_read_main_model() or "").strip()
  3640. if main_provider.lower() == "moa":
  3641. # MoA virtual provider: fall back to the preset's aggregator (the acting model).
  3642. _agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
  3643. if not _agg_provider or not _agg_model:
  3644. return None, None, ""
  3645. main_provider, main_model = _agg_provider, _agg_model
  3646. if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
  3647. return None, None, ""
  3648. if task == "vision" and (
  3649. main_provider in _PROVIDERS_WITHOUT_VISION or not _main_model_supports_vision(main_provider, main_model)):
  3650. # Same capability gate as the auto-route (_vision_main_provider_client): handing an image to a
  3651. # text-only main model turns a transient 429 into a guaranteed 400 (#108349).
  3652. logger.info("Auxiliary vision: %s on %s — main agent provider %s accepts no image input, not falling back",
  3653. reason, failed_provider, main_provider)
  3654. return None, None, ""
  3655. main_base_url = _custom_health_base_url(main_provider)
  3656. if _failed_backend_skip(
  3657. failed_provider, failed_model, failed_base_url=failed_base_url,
  3658. failure_scope=failure_scope)(main_provider, main_model, main_base_url):
  3659. return None, None, ""
  3660. if _is_provider_unhealthy(main_provider, main_base_url):
  3661. _log_skip_unhealthy(main_provider, task, base_url=main_base_url)
  3662. return None, None, ""
  3663. try:
  3664. client, resolved_model = resolve_provider_client(provider=main_provider, model=main_model)
  3665. except Exception:
  3666. client, resolved_model = None, None
  3667. if client is None:
  3668. return None, None, ""
  3669. label = f"main-agent({main_provider})"
  3670. logger.info("Auxiliary %s: %s on %s — falling back to main agent model %s (%s)",
  3671. task or "call", reason, failed_provider, label, resolved_model or main_model)
  3672. return client, resolved_model or main_model, label
  3673. # Context-window screening for runtime fallback chains: the startup feasibility check filters
  3674. # too-small aux models; runtime chains must too, or compression stops at a reachable-but-too-small
  3675. # candidate. ``None`` (unknown) passes through.
  3676. # ── Context-window screening for runtime fallback chains (issue #52392) ── When the runtime auxiliary
  3677. # fallback chain selects a candidate that is reachable but has a context window smaller than the compression
  3678. # task requires, the call errors out instead of continuing to the next, viable candidate. The startup
  3679. # feasibility check in ``agent.conversation_compression.check_compression_model_feasibility`` already
  3680. # filters too-small auxiliary models at startup, but the runtime fallback chain
  3681. # (``_try_configured_fallback_chain`` and ``_try_main_fallback_chain``) does not apply the same filter, so
  3682. # compression can stop at the first alive door even if the room behind it is too small. The helpers below
  3683. # screen each candidate by its effective context window before it is returned. ``None`` results from
  3684. # ``get_model_context_length`` are passed through (we cannot prove a model is too small, so we do not block
  3685. # it). This preserves the existing fallback surface for unrecognised/custom models while closing the gap on
  3686. # the well-known ones.
  3687. def _task_minimum_context_length(task: Optional[str]) -> Optional[int]:
  3688. """Minimum context length for an auxiliary task; None = no floor (only ``compression`` has one)."""
  3689. return MINIMUM_CONTEXT_LENGTH if task == "compression" else None
  3690. def _candidate_context_window(provider: str, model: str, base_url: str = "", api_key: str = "") -> Optional[int]:
  3691. """Best-effort context window for a fallback candidate; ``None`` = unknown (never raises; callers pass it through)."""
  3692. if not model:
  3693. return None
  3694. try:
  3695. ctx = get_model_context_length(model, base_url=base_url, api_key=api_key, provider=provider)
  3696. except Exception as exc:
  3697. logger.debug("Auxiliary fallback: could not resolve context window for %s/%s: %s", provider, model, exc)
  3698. return None
  3699. return ctx if isinstance(ctx, int) and ctx > 0 else None
  3700. def _context_too_small(
  3701. entry: dict[str, Any], provider: str, model: str, min_ctx: Optional[int], *,
  3702. task: Optional[str], label: str, name_model: bool = False,
  3703. ) -> Optional[str]:
  3704. """Screen one fallback candidate by context window; returns the ``tried`` note when it is too small."""
  3705. if min_ctx is None:
  3706. return None
  3707. fb_ctx = _candidate_context_window(
  3708. provider, model, base_url=str(entry.get("base_url") or ""), api_key=_fallback_entry_api_key(entry) or "")
  3709. if fb_ctx is None or fb_ctx >= min_ctx:
  3710. return None
  3711. if name_model:
  3712. logger.info("Auxiliary %s: skipping %s (%s context=%d < min=%d), continuing chain",
  3713. task, label, model, fb_ctx, min_ctx)
  3714. else:
  3715. logger.info("Auxiliary %s: skipping %s (context=%d < min=%d), continuing chain",
  3716. task or "call", label, fb_ctx, min_ctx)
  3717. return f"{label} (context too small: {fb_ctx}<{min_ctx})"
  3718. def _try_configured_fallback_chain(
  3719. task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None, *,
  3720. failed_base_url: str = "", failure_scope: Any = None,
  3721. ) -> tuple[Optional[Any], Optional[str], str]:
  3722. """Try auxiliary.<task>.fallback_chain entries in order (each needs ``provider``; model/base_url/api_key optional).
  3723. ``failed_model`` scoping per ``_failed_backend_skip`` (sibling models on the same provider still
  3724. run after a model-scoped failure). Returns (client, model, provider_label) or (None, None, "")."""
  3725. if not task:
  3726. return None, None, ""
  3727. chain = _get_auxiliary_task_config(task).get("fallback_chain")
  3728. if not chain or not isinstance(chain, list):
  3729. return None, None, ""
  3730. skip = _failed_backend_skip(
  3731. failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
  3732. tried = []
  3733. min_ctx = _task_minimum_context_length(task)
  3734. for i, entry in enumerate(chain):
  3735. if not isinstance(entry, dict):
  3736. continue
  3737. fb_provider = str(entry.get("provider", "")).strip()
  3738. if not fb_provider:
  3739. continue
  3740. fb_model_raw = str(entry.get("model", "")).strip()
  3741. fb_base_url = _custom_health_base_url(fb_provider, entry.get("base_url"))
  3742. if skip(fb_provider, fb_model_raw, fb_base_url):
  3743. continue
  3744. if _is_provider_unhealthy(fb_provider, fb_base_url):
  3745. _log_skip_unhealthy(fb_provider, task, base_url=fb_base_url)
  3746. tried.append(f"fallback_chain[{i}]({fb_provider}) (unhealthy)")
  3747. continue
  3748. fb_model = fb_model_raw or None
  3749. label = f"fallback_chain[{i}]({fb_provider})"
  3750. try:
  3751. fb_client, resolved_model = _resolve_fallback_entry(entry)
  3752. except Exception:
  3753. fb_client, resolved_model = None, None
  3754. if fb_client is not None:
  3755. too_small = _context_too_small(
  3756. entry, fb_provider, resolved_model, min_ctx, task=task, label=label, name_model=True,
  3757. ) if resolved_model else None
  3758. if too_small:
  3759. tried.append(too_small)
  3760. continue
  3761. logger.info("Auxiliary %s: %s on %s — configured fallback to %s (%s)",
  3762. task, reason, failed_provider, label, resolved_model or fb_model or "default")
  3763. return fb_client, resolved_model or fb_model, label
  3764. tried.append(label)
  3765. if tried:
  3766. logger.debug("Auxiliary %s: configured fallback_chain exhausted (tried: %s)", task, ", ".join(tried))
  3767. return None, None, ""
  3768. def _try_configured_fallback_for_unavailable_client(
  3769. task: Optional[str], failed_provider: str
  3770. ) -> tuple[Optional[Any], Optional[str], str]:
  3771. """Task fallback_chain when an explicit aux provider cannot build a client (no key/OAuth/pool creds);
  3772. stops at the per-task chain — the main-agent model stays the runtime last resort."""
  3773. explicit = (failed_provider or "").strip().lower()
  3774. if not task or not explicit or explicit in {"auto"}:
  3775. return None, None, ""
  3776. return _try_configured_fallback_chain(task, explicit, reason="provider unavailable")
  3777. def _fallback_entry_api_key(entry: dict[str, Any]) -> Optional[str]:
  3778. """Resolve inline or env-backed API key via the secret-scope-aware resolver (no raw os.getenv under multiplexing)."""
  3779. from hermes_cli.fallback_config import resolve_entry_api_key
  3780. return resolve_entry_api_key(entry)
  3781. def _resolve_fallback_entry(entry: dict[str, Any]) -> tuple[Optional[Any], Optional[str]]:
  3782. """Resolve one fallback entry through the central provider router."""
  3783. provider = str(entry.get("provider") or "").strip()
  3784. model = str(entry.get("model") or "").strip() or None
  3785. if not provider or not model:
  3786. return None, None
  3787. client, resolved_model = resolve_provider_client(
  3788. provider, model=model, explicit_base_url=str(entry.get("base_url") or "").strip() or None,
  3789. explicit_api_key=_fallback_entry_api_key(entry),
  3790. api_mode=str(entry.get("api_mode") or entry.get("transport") or "").strip() or None,
  3791. )
  3792. if client is not None:
  3793. with contextlib.suppress(Exception):
  3794. client._hermes_fallback_destination = _fallback_destination_from_entry(entry, client, resolved_model)
  3795. return client, resolved_model
  3796. def _try_main_fallback_chain(
  3797. task: Optional[str], failed_provider: str = "", reason: str = "error", *,
  3798. failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
  3799. ) -> tuple[Optional[Any], Optional[str], str]:
  3800. """Top-level main-agent fallback chain for a ``provider: auto`` auxiliary call: auto tasks honour the
  3801. user's main fallback policy before the built-in discovery chain; read via ``get_fallback_chain`` so
  3802. ``fallback_providers`` and legacy ``fallback_model`` keep the main agent's order."""
  3803. try:
  3804. from hermes_cli.config import load_config_readonly
  3805. from hermes_cli.fallback_config import get_fallback_chain
  3806. chain = get_fallback_chain(load_config_readonly())
  3807. except Exception as exc:
  3808. logger.debug("Auxiliary %s: could not load main fallback chain: %s", task or "call", exc)
  3809. return None, None, ""
  3810. if not chain:
  3811. return None, None, ""
  3812. skip = _failed_backend_skip(
  3813. failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
  3814. tried: list[str] = []
  3815. min_ctx = _task_minimum_context_length(task)
  3816. for i, entry in enumerate(chain):
  3817. if not isinstance(entry, dict):
  3818. continue
  3819. fb_provider = str(entry.get("provider") or "").strip()
  3820. fb_model = str(entry.get("model") or "").strip()
  3821. if not fb_provider or not fb_model:
  3822. continue
  3823. fb_norm = fb_provider.lower()
  3824. label = f"fallback_providers[{i}]({fb_provider})"
  3825. fb_base_url = _custom_health_base_url(fb_provider, entry.get("base_url"))
  3826. if fb_norm == "auto" or skip(fb_provider, fb_model, fb_base_url):
  3827. tried.append(f"{label} (skipped)")
  3828. continue
  3829. if _is_provider_unhealthy(fb_norm, fb_base_url):
  3830. _log_skip_unhealthy(fb_norm, task, base_url=fb_base_url)
  3831. tried.append(f"{label} (unhealthy)")
  3832. continue
  3833. try:
  3834. fb_client, resolved_model = _resolve_fallback_entry(entry)
  3835. except Exception as exc:
  3836. logger.debug("Auxiliary %s: main fallback %s failed to resolve: %s", task or "call", label, exc)
  3837. fb_client, resolved_model = None, None
  3838. if fb_client is not None:
  3839. too_small = _context_too_small(
  3840. entry, fb_provider, resolved_model or fb_model, min_ctx, task=task, label=label,
  3841. )
  3842. if too_small:
  3843. tried.append(too_small)
  3844. continue
  3845. logger.info("Auxiliary %s: %s on %s — main fallback chain to %s (%s)",
  3846. task or "call", reason, failed_provider or "auto", label, resolved_model or fb_model)
  3847. return fb_client, resolved_model or fb_model, fb_provider
  3848. tried.append(label)
  3849. if tried:
  3850. logger.debug("Auxiliary %s: main fallback chain exhausted (tried: %s)", task or "call", ", ".join(tried))
  3851. return None, None, ""
  3852. def _warn_stale_openai_base_url(runtime_provider: str) -> None:
  3853. """Warn once when OPENAI_BASE_URL is set but config.yaml names a non-custom provider (a stale
  3854. ~/.hermes/.env value after `hermes model` poisons routing)."""
  3855. global _stale_base_url_warned
  3856. if _stale_base_url_warned:
  3857. return
  3858. _env_base = os.getenv("OPENAI_BASE_URL", "").strip()
  3859. _cfg_provider = runtime_provider or _read_main_provider()
  3860. if (_env_base and _cfg_provider and _cfg_provider != "custom" and not _cfg_provider.startswith("custom:")):
  3861. logger.warning(
  3862. "OPENAI_BASE_URL is set (%s) but model.provider is '%s'. "
  3863. "Auxiliary clients may route to the wrong endpoint. "
  3864. "Run: hermes model to reconfigure, or remove "
  3865. "OPENAI_BASE_URL from ~/.hermes/.env",
  3866. _env_base, _cfg_provider,
  3867. )
  3868. _stale_base_url_warned = True
  3869. def _main_route_target(runtime: dict[str, Any], task: Optional[str]) -> tuple[str, str, str, Any, str]:
  3870. """Step-1 target: (provider, model, base_url, api_key, api_mode) of the main runtime, after the
  3871. fast-model opt-in and the MoA aggregator substitution."""
  3872. main_provider = str(runtime.get("provider", "") or _read_main_provider() or "")
  3873. main_model = str(runtime.get("model") or _read_main_model() or "")
  3874. runtime_base_url = str(runtime.get("base_url") or "")
  3875. runtime_api_key = runtime.get("api_key", "")
  3876. runtime_api_mode = str(runtime.get("api_mode") or "")
  3877. # Latency-critical tasks (titling only) opt in to the provider's fast model. Opt-in only:
  3878. # every settings surface defines "auto" as the main model.
  3879. if _task_prefers_fast_model(task) and main_provider and main_provider not in {"auto", ""}:
  3880. fast_model = _get_aux_model_for_provider(main_provider, prefer_fast=True)
  3881. if fast_model and fast_model != main_model:
  3882. logger.debug("Auxiliary task %s: preferring fast model %s over main model %s",
  3883. task, fast_model, main_model)
  3884. main_model = fast_model
  3885. # MoA virtual provider: the preset name is not a wire model; run aux on the aggregator and drop
  3886. # the facade's "moa://local" base_url / placeholder key so it uses its own credentials.
  3887. if main_provider == "moa":
  3888. _agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
  3889. if _agg_provider and _agg_model:
  3890. main_provider, main_model = _agg_provider, _agg_model
  3891. runtime_base_url = runtime_api_key = runtime_api_mode = ""
  3892. return main_provider, main_model, runtime_base_url, runtime_api_key, runtime_api_mode
  3893. def _try_main_provider_route(
  3894. main_provider: str, main_model: str, runtime_base_url: str, runtime_api_key: Any, runtime_api_mode: str,
  3895. ) -> Optional[tuple[Any, str, str]]:
  3896. """Step 1: route aux onto the main provider + main model; None if unusable."""
  3897. if not (main_provider and main_model and main_provider not in {"auto", ""}):
  3898. return None
  3899. resolved_provider = main_provider
  3900. explicit_base_url = runtime_base_url or None
  3901. health_base_url = _custom_health_base_url(main_provider, explicit_base_url)
  3902. explicit_api_key = None
  3903. if runtime_base_url and main_provider == "custom":
  3904. # Anonymous custom endpoint — pass through explicit base_url + api_key.
  3905. explicit_api_key = runtime_api_key or None
  3906. elif main_provider.startswith("custom:"):
  3907. # Named custom provider (custom_providers / providers dict entry).
  3908. _has_named_entry = False
  3909. with contextlib.suppress(ImportError):
  3910. from hermes_cli.runtime_provider import _get_named_custom_provider
  3911. _has_named_entry = _get_named_custom_provider(main_provider) is not None
  3912. if _has_named_entry:
  3913. # KEEP the full ``custom:<name>`` so the named arm honours the entry's api_mode
  3914. # (collapsing to "custom" strips /anthropic → 404s). base_url/api_key come from the entry.
  3915. explicit_base_url = None
  3916. elif runtime_base_url:
  3917. # Config-less named custom provider (live runtime only): anonymous custom arm + runtime key.
  3918. # See #34777.
  3919. resolved_provider = "custom"
  3920. explicit_api_key = runtime_api_key or None
  3921. elif runtime_api_key:
  3922. explicit_api_key = runtime_api_key
  3923. elif runtime_api_key:
  3924. # Pin aux to the main session's working key, not a re-selected (maybe exhausted) pool key.
  3925. explicit_api_key = runtime_api_key
  3926. # Skip if the main provider was recently 402'd (unhealthy TTL bounds the bypass).
  3927. main_chain_label = _normalize_chain_label(resolved_provider)
  3928. if main_chain_label and _is_provider_unhealthy(main_chain_label, health_base_url):
  3929. _log_skip_unhealthy(main_chain_label, base_url=health_base_url)
  3930. return None
  3931. client, resolved = resolve_provider_client(
  3932. resolved_provider, main_model, explicit_base_url=explicit_base_url,
  3933. explicit_api_key=explicit_api_key, api_mode=runtime_api_mode or None,
  3934. )
  3935. if client is None:
  3936. return None
  3937. logger.info("Auxiliary auto-detect: using main provider %s (%s)", main_provider, resolved or main_model)
  3938. return client, resolved or main_model, resolved_provider
  3939. def _discovery_chain_allowed(main_provider: str, task: Optional[str] = None) -> bool:
  3940. """The built-in discovery chain is a convenience for installs with NO selected main provider.
  3941. Once the user picked one, every auxiliary route must be a provider they configured (main,
  3942. ``auxiliary.<task>``, ``fallback_providers``); guessing "whatever else is logged in" bills an
  3943. account they never pointed this session at (xAI OAuth session with a dead token → every
  3944. compression silently charged to a Nous Portal balance)."""
  3945. if (main_provider or "").strip().lower() in {"", "auto"}:
  3946. return True
  3947. logger.warning(
  3948. "Auxiliary %s: main provider %s is unavailable and no fallback_chain / fallback_providers is "
  3949. "configured — refusing to guess another logged-in provider. Re-authenticate (`hermes model`) "
  3950. "or declare a fallback.", task or "call", main_provider)
  3951. return False
  3952. def _try_discovery_chain() -> tuple[Optional[OpenAI], Optional[str], str]:
  3953. """Step 3: hardcoded aggregator/fallback chain, skipping unhealthy providers."""
  3954. tried = []
  3955. for label, try_fn in _get_provider_chain():
  3956. candidate_base_url = _custom_health_base_url(label)
  3957. if _is_provider_unhealthy(label, candidate_base_url):
  3958. _log_skip_unhealthy(label, base_url=candidate_base_url)
  3959. tried.append(f"{label} (unhealthy)")
  3960. continue
  3961. client, model = try_fn()
  3962. if client is not None:
  3963. if tried:
  3964. logger.info("Auxiliary auto-detect: using %s (%s) — skipped: %s",
  3965. label, model or "default", ", ".join(tried))
  3966. else:
  3967. logger.info("Auxiliary auto-detect: using %s (%s)", label, model or "default")
  3968. return client, model, label
  3969. tried.append(label)
  3970. logger.warning("Auxiliary auto-detect: no provider available (tried: %s). "
  3971. "Compression, summarization, and memory flush will not work. "
  3972. "Set OPENROUTER_API_KEY or configure a local model in config.yaml.", ", ".join(tried))
  3973. return None, None, ""
  3974. def _resolve_auto_route(
  3975. main_runtime: Optional[dict[str, Any]] = None, task: Optional[str] = None
  3976. ) -> tuple[Optional[OpenAI], Optional[str], str]:
  3977. """Full auto-detection chain, including the selected provider identity. Priority: (1) main provider +
  3978. main model, regardless of provider type ("auto" means "my main model for side tasks too"; explicit
  3979. per-task overrides still win); (2) configured fallback policy — task chain, then the main agent's
  3980. top-level chain; (3) OpenRouter → Nous → custom → Codex → API-key providers, only with no policy
  3981. and no working main client."""
  3982. runtime = _normalize_main_runtime(main_runtime)
  3983. _warn_stale_openai_base_url(runtime.get("provider", ""))
  3984. main_provider, main_model, base_url, api_key, api_mode = _main_route_target(runtime, task)
  3985. routed = _try_main_provider_route(main_provider, main_model, base_url, api_key, api_mode)
  3986. if routed is not None:
  3987. return routed
  3988. if task:
  3989. fb_client, fb_model, fb_label = _try_configured_fallback_chain(
  3990. task, main_provider or "auto", reason="main provider unavailable")
  3991. if fb_client is not None:
  3992. return fb_client, fb_model, _fallback_provider_from_label(fb_label)
  3993. fb_client, fb_model, fb_label = _try_main_fallback_chain(
  3994. task, main_provider or "auto", reason="main provider unavailable")
  3995. if fb_client is not None:
  3996. return fb_client, fb_model, fb_label
  3997. if not _discovery_chain_allowed(main_provider, task):
  3998. return None, None, ""
  3999. return _try_discovery_chain()
  4000. def _effective_provider_for_client(client: Any, fallback: str) -> str:
  4001. """Return the concrete provider selected for an auto-routed client."""
  4002. effective_provider = getattr(client, "_hermes_aux_effective_provider", "")
  4003. if isinstance(effective_provider, str) and effective_provider:
  4004. return effective_provider
  4005. return str(fallback or "")
  4006. # Centralized Provider Router: resolve_provider_client() is the single entry point for building a configured
  4007. # client (auth, base URL, headers, API format) from (provider, model). Never read auth env vars ad-hoc.
  4008. def _to_async_client(sync_client, model: str, is_vision: bool = False):
  4009. """Sync client → async counterpart, preserving Codex routing (``is_vision`` keeps the endpoint's vision headers)."""
  4010. from openai import AsyncOpenAI
  4011. if isinstance(sync_client, _AuxProbeClientStub):
  4012. return sync_client, model
  4013. if isinstance(sync_client, CodexAuxiliaryClient):
  4014. return AsyncCodexAuxiliaryClient(sync_client), model
  4015. if isinstance(sync_client, AnthropicAuxiliaryClient):
  4016. return AsyncAnthropicAuxiliaryClient(sync_client), model
  4017. if isinstance(sync_client, BedrockAuxiliaryClient):
  4018. return AsyncBedrockAuxiliaryClient(sync_client), model
  4019. with contextlib.suppress(ImportError):
  4020. from agent.gemini_native_adapter import GeminiNativeClient, AsyncGeminiNativeClient
  4021. if isinstance(sync_client, GeminiNativeClient):
  4022. return AsyncGeminiNativeClient(sync_client), model
  4023. # ACP shims (subprocess, not an HTTP pool) are already async-safe and opt out of the wrapper.
  4024. if _client_declares(sync_client, "HERMES_SKIP_ASYNC_WRAP"):
  4025. return sync_client, model
  4026. sync_base_url = str(sync_client.base_url)
  4027. # A key_cmd/Entra client keeps its credential in the SDK's per-request provider slot, not in
  4028. # ``.api_key`` (which stays ""); rebuilding from the snapshot alone ships NO Authorization
  4029. # header. Configured default_headers (a named entry's extra_headers) are likewise carried over,
  4030. # merged last exactly as the SDK merges them on the sync client. See #109595.
  4031. from agent.auxiliary_async_rebuild import async_api_key, configured_default_headers
  4032. async_kwargs = {"api_key": async_api_key(sync_client), "base_url": sync_base_url}
  4033. if base_url_host_matches(sync_base_url, "openrouter.ai"):
  4034. headers = _apply_user_default_headers(build_or_headers())
  4035. elif _is_official_codex_base_url(sync_base_url):
  4036. headers = _apply_user_default_headers(_codex_cloudflare_headers(sync_client.api_key, base_url=sync_base_url))
  4037. else:
  4038. # Provider for the profile-header fallback is inferred from the hostname.
  4039. try:
  4040. from agent.model_metadata import _infer_provider_from_url
  4041. inferred = _infer_provider_from_url(sync_base_url) or ""
  4042. except Exception:
  4043. inferred = ""
  4044. headers = _endpoint_default_headers(sync_base_url, inferred, is_vision=is_vision, xai=True)
  4045. headers = {**(headers or {}), **configured_default_headers(sync_client)}
  4046. if headers:
  4047. async_kwargs["default_headers"] = headers
  4048. _apply_required_codex_headers(async_kwargs, access_token=sync_client.api_key, base_url=sync_base_url)
  4049. async_kwargs = {**_openai_http_client_kwargs(sync_base_url, async_mode=True), **async_kwargs}
  4050. # Hermes owns the auxiliary retry/timeout budget; disable SDK-internal retries.
  4051. # See #54465.
  4052. async_kwargs.setdefault("max_retries", 0)
  4053. return AsyncOpenAI(**async_kwargs), model
  4054. def _normalize_resolved_model(model_name: Optional[str], provider: str) -> Optional[str]:
  4055. """Normalize a resolved model for the provider that will receive it."""
  4056. if not model_name:
  4057. return model_name
  4058. try:
  4059. from hermes_cli.model_normalize import normalize_model_for_provider
  4060. return normalize_model_for_provider(model_name, provider)
  4061. except Exception:
  4062. return model_name
  4063. def _named_custom_api_key(custom_entry: dict[str, Any], provider: str, custom_base: str) -> Any:
  4064. """Credential for a named custom provider: inline api_key → key_env → key_cmd → credential pool → placeholder.
  4065. Aux resolves named custom providers here, not via _resolve_named_custom_runtime, so key_cmd must be
  4066. honoured at the same precedence or every aux call 401s."""
  4067. custom_key: Any = (custom_entry.get("api_key") or "").strip()
  4068. custom_key_env = (custom_entry.get("key_env") or custom_entry.get("api_key_env") or "").strip()
  4069. if not custom_key and custom_key_env:
  4070. custom_key = _scoped_key_env(custom_key_env)
  4071. custom_key_cmd = str(custom_entry.get("key_cmd", "") or "").strip()
  4072. if custom_key_cmd:
  4073. from agent.command_token_source import build_command_token_provider
  4074. custom_key = build_command_token_provider(custom_key_cmd, custom_entry.get("name") or provider) or custom_key
  4075. if not custom_key:
  4076. with contextlib.suppress(Exception):
  4077. from agent.credential_pool import custom_provider_pool_key_candidates
  4078. pool_name = custom_entry.get("provider_key") or custom_entry.get("name") or provider
  4079. for pool_key in custom_provider_pool_key_candidates(custom_base, pool_name):
  4080. try:
  4081. pool = load_pool(pool_key)
  4082. except Exception:
  4083. continue
  4084. if (pool := entry_scoped_pool(pool, pool_key, custom_entry)) is None or not pool.has_credentials():
  4085. continue
  4086. if (pool_entry := pool.select()) is None:
  4087. continue
  4088. pool_api_key = getattr(pool_entry, "runtime_api_key", None) or getattr(pool_entry, "access_token", "") or ""
  4089. if str(pool_api_key).strip():
  4090. custom_key = str(pool_api_key).strip()
  4091. break
  4092. return custom_key or "no-key-required"
  4093. def _build_bedrock_client(provider: str, model: Optional[str], *, raw_codex: bool) -> tuple[Optional[Any], Optional[str]]:
  4094. """AWS Bedrock: Claude → Anthropic Bedrock SDK (prompt caching, thinking); bare in-Region OpenAI IDs
  4095. → Mantle Responses; everything else, incl. OpenAI ``us.``/``global.`` profiles, → Converse API."""
  4096. try:
  4097. from agent.bedrock_adapter import (
  4098. has_aws_credentials, is_anthropic_bedrock_model, resolve_bedrock_runtime_region,
  4099. bedrock_openai_uses_mantle, bedrock_openai_base_url, resolve_bedrock_bearer_token,
  4100. configure_bedrock_openai_client_kwargs,
  4101. )
  4102. from agent.anthropic_adapter import build_anthropic_bedrock_client
  4103. except ImportError:
  4104. logger.warning("resolve_provider_client: bedrock requested but boto3, httpx/openai, or anthropic SDK not installed")
  4105. return None, None
  4106. if not has_aws_credentials():
  4107. logger.debug("resolve_provider_client: bedrock requested but no AWS credentials found")
  4108. return None, None
  4109. # Region must match the main runtime's resolution (bedrock.region in config first, then
  4110. # env/profile) so aux calls never leave the primary runtime's configured region.
  4111. # See #53880, #65076.
  4112. region = resolve_bedrock_runtime_region()
  4113. default_model = "anthropic.claude-haiku-4-5-20251001-v1:0"
  4114. final_model = _normalize_resolved_model(model or default_model, provider) or default_model
  4115. if bedrock_openai_uses_mantle(final_model):
  4116. # Module-level lazy ``OpenAI`` proxy on purpose so tests can patch("agent.auxiliary_client.OpenAI").
  4117. client_kwargs: dict[str, Any] = {
  4118. "api_key": resolve_bedrock_bearer_token() or "aws-sdk",
  4119. "base_url": bedrock_openai_base_url(region),
  4120. }
  4121. configure_bedrock_openai_client_kwargs(client_kwargs)
  4122. client = OpenAI(**client_kwargs)
  4123. logger.debug("resolve_provider_client: bedrock-openai (%s, %s)", final_model, region)
  4124. return (client if raw_codex else CodexAuxiliaryClient(client, final_model)), final_model
  4125. base_url = f"https://bedrock-runtime.{region}.amazonaws.com"
  4126. if is_anthropic_bedrock_model(final_model):
  4127. try:
  4128. real_client = build_anthropic_bedrock_client(region)
  4129. except ImportError as exc:
  4130. logger.warning("resolve_provider_client: cannot create Bedrock client: %s", exc)
  4131. return None, None
  4132. client = AnthropicAuxiliaryClient(real_client, final_model, api_key="aws-sdk", base_url=base_url)
  4133. logger.debug("resolve_provider_client: bedrock anthropic (%s, %s)", final_model, region)
  4134. else:
  4135. client = BedrockAuxiliaryClient(region, final_model)
  4136. logger.debug("resolve_provider_client: bedrock converse (%s, %s)", final_model, region)
  4137. return client, final_model
  4138. def _build_vertex_client(provider: str, model: Optional[str]) -> tuple[Optional[Any], Optional[str]]:
  4139. """Google Vertex AI: Gemini via the OpenAI-compatible endpoint with an OAuth2 bearer (standard OpenAI client)."""
  4140. try:
  4141. from agent.vertex_adapter import get_vertex_config, has_vertex_credentials
  4142. except ImportError:
  4143. logger.warning("resolve_provider_client: vertex requested but google-auth not installed")
  4144. return None, None
  4145. if not has_vertex_credentials():
  4146. logger.debug("resolve_provider_client: vertex requested but no GCP credentials found")
  4147. return None, None
  4148. token, base_url = get_vertex_config()
  4149. if not token or not base_url:
  4150. logger.warning("resolve_provider_client: vertex requested but could not mint token / resolve project")
  4151. return None, None
  4152. final_model = _normalize_resolved_model(model or "google/gemini-3-flash-preview", provider)
  4153. try:
  4154. # Aliased import: a bare `from openai import OpenAI` would shadow the module-level lazy proxy.
  4155. from openai import OpenAI as _VertexOpenAI
  4156. client = _VertexOpenAI(api_key=token, base_url=base_url)
  4157. except Exception as exc:
  4158. logger.warning("resolve_provider_client: cannot create Vertex client: %s", exc)
  4159. return None, None
  4160. logger.debug("resolve_provider_client: vertex (%s)", final_model)
  4161. return client, final_model
  4162. class _ResolveRequest(NamedTuple):
  4163. """Normalized resolve_provider_client() arguments shared by the per-provider branch helpers."""
  4164. provider: str
  4165. original_provider: str
  4166. model: Optional[str]
  4167. async_mode: bool
  4168. raw_codex: bool
  4169. explicit_base_url: Optional[str]
  4170. explicit_api_key: Optional[str | Callable[[], str]]
  4171. api_mode: Optional[str]
  4172. main_runtime: Optional[dict[str, Any]]
  4173. is_vision: bool
  4174. task: Optional[str]
  4175. _ResolveResult = tuple[Optional[Any], Optional[str]]
  4176. def _normalize_api_key(raw: Any) -> str | Callable[[], str]:
  4177. """A key_cmd/Entra callable passes through uncalled; strings are stripped; anything else is ''."""
  4178. if callable(raw) and not isinstance(raw, str):
  4179. return raw
  4180. return raw.strip() if isinstance(raw, str) else ""
  4181. def _log_once_debug(seen: set, key: Any, msg: str, *args: Any) -> None:
  4182. """Debug-log ``msg`` the first time ``key`` is seen so per-call retries stay silent."""
  4183. if key not in seen:
  4184. seen.add(key)
  4185. logger.debug(msg, *args)
  4186. def _is_actual_auxiliary_route(req: _ResolveRequest, base_url: str) -> bool:
  4187. from hermes_cli.auth import normalize_actual_base_url
  4188. from hermes_cli.providers import is_actual_route
  4189. from hermes_cli.route_identity import normalize_route_base_url
  4190. if is_actual_route(req.provider, base_url):
  4191. return True
  4192. runtime = _normalize_main_runtime(req.main_runtime)
  4193. return bool(
  4194. base_url
  4195. and is_actual_route(runtime.get("provider", ""), runtime.get("base_url", ""))
  4196. and normalize_route_base_url(normalize_actual_base_url(base_url))
  4197. == normalize_route_base_url(
  4198. normalize_actual_base_url(runtime.get("base_url", ""))
  4199. )
  4200. )
  4201. def _wrap_transport(req: _ResolveRequest, client_obj: Any, final_model_str: str,
  4202. base_url_str: str = "", api_key_str: str = ""):
  4203. """Wrap a plain OpenAI client in the right transport adapter; specialized wrappers pass through.
  4204. Codex (Responses API): explicit ``api_mode=codex_responses``, else — with no
  4205. explicit api_mode — api.openai.com + codex model. Anthropic (Messages): ``api_mode=anthropic_messages``,
  4206. any ``/anthropic`` suffix, ``api.kimi.com/coding``, or ``api.anthropic.com``."""
  4207. if _is_actual_auxiliary_route(req, base_url_str):
  4208. client = (
  4209. client_obj._real_client
  4210. if isinstance(client_obj, CodexAuxiliaryClient)
  4211. else client_obj
  4212. )
  4213. client._hermes_aux_effective_provider = "actual"
  4214. return client
  4215. # OpenCode relay targets pick the wire per model; a task/provider-level api_mode is stale for
  4216. # every other model (#98799), so it is re-derived here like the main runtime does.
  4217. from agent.opencode_affinity import opencode_transport
  4218. _oc_mode, _oc_base = opencode_transport(req.provider, final_model_str, base_url_str)
  4219. if _oc_mode:
  4220. req, base_url_str = req._replace(api_mode=_oc_mode), _oc_base
  4221. needs_codex = not (
  4222. isinstance(client_obj, CodexAuxiliaryClient) or req.raw_codex
  4223. ) and (
  4224. req.api_mode == "codex_responses"
  4225. or (
  4226. not req.api_mode
  4227. and base_url_hostname(base_url_str) == "api.openai.com"
  4228. and "codex" in (final_model_str or "").lower()
  4229. )
  4230. )
  4231. if needs_codex:
  4232. logger.debug("resolve_provider_client: wrapping client in CodexAuxiliaryClient "
  4233. "(api_mode=%s, model=%s, base_url=%s)",
  4234. req.api_mode or "auto-detected", final_model_str, base_url_str[:60] if base_url_str else "")
  4235. return CodexAuxiliaryClient(client_obj, final_model_str)
  4236. # A profile that declares the Messages wire (commandcode-anthropic) is on it whatever the URL
  4237. # looks like; the same declaration gates ``_reasoning_config`` in _build_call_kwargs.
  4238. api_mode = req.api_mode or _profile_declared_messages_wire(req.provider)
  4239. return _maybe_wrap_anthropic(client_obj, final_model_str, api_key_str, base_url_str, api_mode)
  4240. def _profile_declared_messages_wire(provider: str) -> Optional[str]:
  4241. """``"anthropic_messages"`` when the registered profile declares that api_mode, else None."""
  4242. from providers import get_provider_profile
  4243. profile = get_provider_profile(str(provider or "").strip().lower())
  4244. return "anthropic_messages" if profile is not None and profile.api_mode == "anthropic_messages" else None
  4245. def _route_client(req: _ResolveRequest, client_obj: Any, final_model_str: Optional[str]) -> _ResolveResult:
  4246. """Return (client, model), converting to the async wrapper when ``req.async_mode``."""
  4247. if req.async_mode:
  4248. return _to_async_client(client_obj, final_model_str, is_vision=req.is_vision)
  4249. return client_obj, final_model_str
  4250. def _route_or_warn(req: _ResolveRequest, client: Any, default: Optional[str], unavailable_msg: str, *args: Any) -> _ResolveResult:
  4251. """Route ``client`` on ``req.model or default``; warn and return (None, None) when the provider produced no client."""
  4252. if client is None:
  4253. logger.warning(unavailable_msg, *args)
  4254. return None, None
  4255. return _route_client(req, client, _normalize_resolved_model(req.model or default, req.provider))
  4256. def _resolve_auto_branch(req: _ResolveRequest) -> _ResolveResult:
  4257. """Auto: try all providers in priority order; tag the client with the effective provider (survives cache reuse)."""
  4258. client, resolved, effective_provider = _resolve_auto_route(main_runtime=req.main_runtime, task=req.task)
  4259. if client is None:
  4260. return None, None
  4261. model = req.model
  4262. # An OpenRouter-format model override won't work on a non-OpenRouter provider (e.g. local
  4263. # server); drop it for the provider's default.
  4264. if model and "/" in model and resolved and "/" not in resolved:
  4265. logger.debug("Dropping OpenRouter-format model %r for non-OpenRouter "
  4266. "auxiliary provider (using %r instead)", model, resolved)
  4267. model = None
  4268. routed_client, routed_model = _route_client(req, client, model or resolved)
  4269. if routed_client is not None and effective_provider:
  4270. try:
  4271. routed_client._hermes_aux_effective_provider = effective_provider
  4272. except (AttributeError, TypeError):
  4273. logger.debug("Auxiliary client %s cannot retain effective provider %s",
  4274. type(routed_client).__name__, effective_provider)
  4275. return routed_client, routed_model
  4276. def _resolve_openrouter_branch(req: _ResolveRequest) -> _ResolveResult:
  4277. """OpenRouter."""
  4278. client, default = _try_openrouter(explicit_api_key=req.explicit_api_key, model=req.model,
  4279. explicit_base_url=req.explicit_base_url)
  4280. if client is None:
  4281. logger.warning("resolve_provider_client: openrouter requested but %s",
  4282. _describe_openrouter_unavailable(model=req.model))
  4283. return None, None
  4284. return _route_client(req, client, _normalize_resolved_model(req.model or default, req.provider))
  4285. def _resolve_nous_branch(req: _ResolveRequest) -> _ResolveResult:
  4286. """Nous Portal (OAuth)."""
  4287. model = req.model
  4288. # Vision: caller flag, _PROVIDER_VISION_MODELS override, or a known vision id.
  4289. client, default = _try_nous(vision=(req.is_vision or model in _PROVIDER_VISION_MODELS.values()
  4290. or (model or "").strip().lower() == "mimo-v2-omni"))
  4291. if client is None:
  4292. logger.warning("resolve_provider_client: nous requested but Nous Portal not configured (run: hermes auth)")
  4293. return None, None
  4294. final_model = _normalize_resolved_model(model or default, req.provider)
  4295. # Dual-wire: anthropic/* → /v1/messages, else /chat/completions. Derive from the catalog id
  4296. # (not a stale api_mode) so aux matches the main agent.
  4297. from hermes_cli.providers import nous_api_mode
  4298. client = _maybe_wrap_anthropic(
  4299. client, final_model, str(getattr(client, "api_key", "") or ""),
  4300. str(getattr(client, "base_url", "") or ""), nous_api_mode(final_model),
  4301. )
  4302. return _route_client(req, client, final_model)
  4303. def _resolve_openai_codex_branch(req: _ResolveRequest) -> _ResolveResult:
  4304. """OpenAI Codex (OAuth → Responses API)."""
  4305. model = req.model
  4306. if not model:
  4307. logger.warning("resolve_provider_client: openai-codex requested without a "
  4308. "model; pass model explicitly (e.g. model.model in config.yaml "
  4309. "or auxiliary.<task>.model for per-task aux routing).")
  4310. return None, None
  4311. no_token_msg = "resolve_provider_client: openai-codex requested but no Codex OAuth token found (run: hermes model)"
  4312. if req.raw_codex:
  4313. # Raw OpenAI client for callers needing responses.stream() (main agent loop).
  4314. codex_token, base_url = _resolve_codex_credential_and_base()
  4315. if not codex_token:
  4316. logger.warning(no_token_msg)
  4317. return None, None
  4318. raw_client = _create_openai_client(api_key=codex_token, base_url=base_url,
  4319. default_headers=_codex_cloudflare_headers(codex_token, base_url=base_url))
  4320. return raw_client, _normalize_resolved_model(model, req.provider)
  4321. client, default = _build_codex_client(model)
  4322. return _route_or_warn(req, client, default, no_token_msg)
  4323. def _resolve_xai_oauth_branch(req: _ResolveRequest) -> _ResolveResult:
  4324. """xAI Grok OAuth (device code → Responses API). Without this branch xai-oauth falls to the generic
  4325. oauth_external arm, returns (None, None), and silently re-routes every aux task to the Step-2 fallback."""
  4326. client, default = _build_xai_oauth_aux_client(req.model)
  4327. return _route_or_warn(req, client, default,
  4328. "resolve_provider_client: xai-oauth requested but no xAI "
  4329. "OAuth token found (run: hermes model -> xAI Grok OAuth — SuperGrok / Premium+)")
  4330. def _resolve_custom_branch(req: _ResolveRequest) -> _ResolveResult:
  4331. """Custom endpoint (OPENAI_BASE_URL + OPENAI_API_KEY)."""
  4332. provider, model, main_runtime = req.provider, req.model, req.main_runtime
  4333. # wrap_base: base for the Anthropic-wrap decision. anthropic_messages must keep the raw
  4334. # /anthropic base while the plain OpenAI client uses the /v1-rewritten custom_base (never
  4335. # /anthropic/chat/completions). Empty means "use custom_base".
  4336. custom_base = custom_key = wrap_base = ""
  4337. if req.explicit_base_url:
  4338. custom_base = _to_openai_base_url(req.explicit_base_url).strip()
  4339. # Ollama/vLLM/llama.cpp serve the OpenAI surface under /v1; a bare host posts to the
  4340. # native API and 404s (#106010). Only the alias group gets the tail — ``custom`` is verbatim.
  4341. if req.original_provider in _LOCAL_SERVER_ALIASES and not urlparse(custom_base).path.strip("/"):
  4342. custom_base = custom_base.rstrip("/") + "/v1"
  4343. if req.api_mode == "anthropic_messages":
  4344. wrap_base = (req.explicit_base_url or "").strip().rstrip("/")
  4345. if req.original_provider in _LOCAL_SERVER_ALIASES:
  4346. # SECURITY: a local-server alias never borrows OPENAI_API_KEY or the main key —
  4347. # the alias means "this is my own server"; sending an OpenAI secret to whatever
  4348. # host base_url names is never intended. Explicit api_key or the placeholder only.
  4349. custom_key = _normalize_api_key(req.explicit_api_key) or "no-key-required"
  4350. else:
  4351. custom_key = (
  4352. _normalize_api_key(req.explicit_api_key)
  4353. or _scoped_key_env("OPENAI_API_KEY")
  4354. or _read_main_api_key_if_same_origin(custom_base)
  4355. or "no-key-required" # local servers don't need auth
  4356. )
  4357. if not custom_base:
  4358. logger.warning("resolve_provider_client: explicit custom endpoint requested but base_url is empty")
  4359. return None, None
  4360. elif main_runtime:
  4361. # Reuse main_runtime's concrete base_url + api_key for a named custom provider;
  4362. # re-resolving from bare "custom" loses the name and lands on the wrong provider.
  4363. # Re-resolution loses the provider name and falls back to OpenRouter or a wrong API-key provider —
  4364. # the main agent already solved this, we just need to reuse its answer. (#45472)
  4365. _main_base = str(main_runtime.get("base_url") or "").strip().rstrip("/")
  4366. _main_key = _normalize_api_key(main_runtime.get("api_key"))
  4367. if _main_base and _main_key:
  4368. custom_base, custom_key = _main_base, _main_key
  4369. if custom_base and custom_key:
  4370. if _is_actual_auxiliary_route(req, custom_base):
  4371. from hermes_cli.auth import normalize_actual_base_url
  4372. custom_base = normalize_actual_base_url(custom_base)
  4373. final_model = _normalize_resolved_model(
  4374. model or (main_runtime.get("model") if main_runtime else None) or "gpt-4o-mini", provider,
  4375. )
  4376. _clean_base, _dq = _extract_url_query_params(custom_base)
  4377. extra = {"default_query": _dq} if _dq else {}
  4378. from hermes_cli.config import get_custom_provider_extra_headers # this endpoint's entry, as agent_init
  4379. _custom_headers = {**(_endpoint_default_headers(custom_base, provider, is_vision=req.is_vision) or {}),
  4380. **get_custom_provider_extra_headers(custom_base)}
  4381. if _custom_headers:
  4382. extra["default_headers"] = _custom_headers
  4383. client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **extra)
  4384. client = _wrap_transport(req, client, final_model, wrap_base or custom_base, custom_key)
  4385. return _route_client(req, client, final_model)
  4386. # Try custom first, then API-key providers (Codex excluded here:
  4387. # falling through to Codex with no model is a stale-constant trap).
  4388. for try_fn in (_try_custom_endpoint, _resolve_api_key_provider):
  4389. client, default = try_fn()
  4390. if client is not None:
  4391. final_model = _normalize_resolved_model(model or default, provider)
  4392. # ``client.api_key`` may be a callable (Azure Entra bearer provider);
  4393. # wrapping decisions only need base_url + api_mode.
  4394. _raw_ckey = getattr(client, "api_key", "")
  4395. _ckey = "" if (callable(_raw_ckey) and not isinstance(_raw_ckey, str)) else str(_raw_ckey or "")
  4396. client = _wrap_transport(req, client, final_model, str(getattr(client, "base_url", "") or ""), _ckey)
  4397. return _route_client(req, client, final_model)
  4398. logger.warning("resolve_provider_client: custom/main requested but no endpoint credentials found")
  4399. return None, None
  4400. def _named_custom_openai_wire_client(custom_base: str, custom_key: Any, extra_headers: Optional[dict[str, str]] = None):
  4401. """Plain OpenAI client on the /v1 equivalent of a named custom entry's base URL.
  4402. ``extra_headers`` is the entry's own header block (gateway routing tags, proxy auth): the main
  4403. runtime lifts it onto every request, so aux must too (#109595). SECURITY: never log the values."""
  4404. _clean_base, _dq = _extract_url_query_params(_to_openai_base_url(custom_base))
  4405. _extra = {"default_query": _dq} if _dq else {}
  4406. _headers = _apply_user_default_headers(None)
  4407. if extra_headers:
  4408. _headers = {**(_headers or {}), **extra_headers}
  4409. if _headers:
  4410. _extra["default_headers"] = _headers
  4411. return _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
  4412. def _resolve_named_custom_branch(req: _ResolveRequest) -> Optional[_ResolveResult]:
  4413. """Named custom provider (config.yaml providers dict / custom_providers list); None if no entry matches."""
  4414. from hermes_cli.runtime_provider import _get_named_custom_provider
  4415. provider = req.provider
  4416. # If the raw name is an alias (``kimi`` → ``kimi-coding``) and a custom_providers entry exists
  4417. # under it, the custom entry wins over alias rewriting. Only for aliases, so entries matching a
  4418. # canonical name (e.g. ``nous``) still defer to the built-in.
  4419. custom_entry = None
  4420. if req.original_provider and req.original_provider != provider:
  4421. custom_entry = _get_named_custom_provider(req.original_provider)
  4422. if custom_entry is None:
  4423. custom_entry = _get_named_custom_provider(provider)
  4424. if not custom_entry:
  4425. return None
  4426. # A per-task/explicit base_url or api_key composes OVER the named entry's defaults: the entry supplies
  4427. # whatever the caller left blank, never replaces what the caller set (compression prompts carry
  4428. # conversation history, so a silently swapped destination is a data-routing bug, not a nuisance).
  4429. custom_base = (req.explicit_base_url or custom_entry.get("base_url") or "").strip()
  4430. custom_key = _normalize_api_key(req.explicit_api_key) or _named_custom_api_key(custom_entry, provider, custom_base)
  4431. if custom_key == "no-key-required":
  4432. logger.warning("resolve_provider_client: named custom provider %r has no resolvable "
  4433. "api_key — request will be sent with placeholder no-key-required "
  4434. "and will 401 on auth-required endpoints", custom_entry.get("name") or provider)
  4435. # Actual's wire protocol takes precedence over persisted task/provider modes.
  4436. entry_api_mode = (req.api_mode or custom_entry.get("api_mode") or "").strip()
  4437. if _is_actual_auxiliary_route(req, custom_base):
  4438. from hermes_cli.auth import normalize_actual_base_url
  4439. custom_base = normalize_actual_base_url(custom_base)
  4440. entry_api_mode = "chat_completions"
  4441. if not custom_base:
  4442. logger.warning("resolve_provider_client: named custom provider %r has no base_url", provider)
  4443. return None, None
  4444. from hermes_cli.config import normalize_extra_headers
  4445. entry_headers = normalize_extra_headers(custom_entry.get("extra_headers"))
  4446. final_model = _normalize_resolved_model(
  4447. req.model
  4448. or custom_entry.get("model")
  4449. or (req.main_runtime.get("model") if req.main_runtime else None)
  4450. or _read_main_model_for_aux()
  4451. or "gpt-4o-mini",
  4452. provider,
  4453. )
  4454. # An OpenCode-family entry (``opencode-go-bridge``, #85589) persisted the api_mode of whichever
  4455. # model was selected at save time; the relay picks the wire per model (#98799).
  4456. from agent.opencode_affinity import opencode_transport
  4457. _oc_mode, _oc_base = opencode_transport(provider, final_model, custom_base)
  4458. if _oc_mode:
  4459. entry_api_mode, custom_base = _oc_mode, _oc_base
  4460. logger.debug("resolve_provider_client: named custom provider %r (%s, api_mode=%s)",
  4461. provider, final_model, entry_api_mode or "chat_completions")
  4462. # anthropic_messages: route via AnthropicAuxiliaryClient (mirrors _try_custom_endpoint);
  4463. # the Anthropic SDK sees the original (un-rewritten) URL.
  4464. # Mirrors the anonymous-custom branch in _try_custom_endpoint(). See #15033.
  4465. if entry_api_mode == "anthropic_messages":
  4466. try:
  4467. from agent.anthropic_adapter import build_anthropic_client
  4468. from agent.anthropic_credentials import anthropic_route_is_oauth
  4469. real_client = build_anthropic_client(custom_key, custom_base)
  4470. if entry_headers:
  4471. # Same entry headers as the two OpenAI-wire arms; ``with_options`` merges onto the
  4472. # beta/credential-Omit headers the builder installed (#109595).
  4473. real_client = real_client.with_options(default_headers=entry_headers)
  4474. except ImportError:
  4475. logger.warning("Named custom provider %r declares api_mode=anthropic_messages but the anthropic SDK "
  4476. "is not installed — falling back to OpenAI-wire.", provider)
  4477. return _route_client(req, _named_custom_openai_wire_client(custom_base, custom_key, entry_headers), final_model)
  4478. return _route_client(
  4479. req, AnthropicAuxiliaryClient(real_client, final_model, custom_key, custom_base,
  4480. is_oauth=anthropic_route_is_oauth(custom_base, custom_key)), final_model)
  4481. client = _named_custom_openai_wire_client(custom_base, custom_key, entry_headers)
  4482. # codex_responses, or auto-detect via _wrap_transport (which reads the task-level api_mode).
  4483. if entry_api_mode == "codex_responses":
  4484. client = CodexAuxiliaryClient(client, final_model)
  4485. else:
  4486. client = _wrap_transport(req, client, final_model, custom_base, custom_key)
  4487. return _route_client(req, client, final_model)
  4488. def _resolve_azure_foundry_branch(req: _ResolveRequest) -> _ResolveResult:
  4489. """Azure Foundry via the runtime resolver: the generic PROVIDER_REGISTRY path only knows the static
  4490. AZURE_FOUNDRY_API_KEY env var, missing ``auth_mode: entra_id`` (callable bearer) and config base_url overrides."""
  4491. client, default_model = _try_azure_foundry(model=req.model, explicit_api_key=req.explicit_api_key,
  4492. explicit_base_url=req.explicit_base_url, api_mode=req.api_mode)
  4493. return _route_or_warn(req, client, default_model,
  4494. "resolve_provider_client: azure-foundry requested but "
  4495. "runtime resolution failed (run: hermes doctor for diagnostics)")
  4496. def _api_key_profile_supplied_client(provider: str, **client_kwargs: Any) -> Any | None:
  4497. """Registered profile's own client for an ``api_key`` aux route, or ``None``.
  4498. Same registration seam as ``agent_runtime_helpers._provider_supplied_client`` (main agent)
  4499. and the ``external_process`` branch below: a profile whose wire protocol is not
  4500. OpenAI-over-HTTP overrides ``ProviderProfile.create_client()`` to supply its transport.
  4501. A profile that raises is logged and skipped — a third-party plugin can only fail to
  4502. provide a client, never take the auxiliary resolution down."""
  4503. try:
  4504. from providers import get_provider_profile
  4505. profile = get_provider_profile(provider)
  4506. except Exception:
  4507. return None
  4508. if profile is None:
  4509. return None
  4510. try:
  4511. return profile.create_client(**client_kwargs)
  4512. except Exception:
  4513. logger.warning("resolve_provider_client: provider profile %r failed to create an "
  4514. "auxiliary client; falling back to the standard client path",
  4515. provider, exc_info=True)
  4516. return None
  4517. def _resolve_api_key_branch(req: _ResolveRequest, pconfig: Any, resolve_creds: Callable) -> _ResolveResult:
  4518. """PROVIDER_REGISTRY ``api_key`` providers (Anthropic via its own resolver), honouring explicit overrides."""
  4519. provider = req.provider
  4520. if provider == "anthropic":
  4521. client, default_model = _try_anthropic(explicit_api_key=req.explicit_api_key,
  4522. explicit_base_url=req.explicit_base_url)
  4523. return _route_or_warn(req, client, default_model,
  4524. "resolve_provider_client: anthropic requested but no Anthropic credentials found")
  4525. creds = resolve_creds(provider)
  4526. api_key = str(creds.get("api_key", "")).strip()
  4527. # Explicit api_key override (fallback_model / custom_providers entry) lets callers
  4528. # authenticate where no built-in credential is registered for this alias.
  4529. api_key = _normalize_api_key(req.explicit_api_key) or api_key
  4530. raw_base_url = str(creds.get("base_url", "")).strip().rstrip("/") or pconfig.inference_base_url
  4531. if req.explicit_base_url:
  4532. raw_base_url = req.explicit_base_url.strip().rstrip("/")
  4533. if provider == "actual":
  4534. with contextlib.suppress(Exception):
  4535. from hermes_cli.auth import (
  4536. ACTUAL_LOCAL_NOAUTH_PLACEHOLDER, is_actual_local_base_url, normalize_actual_base_url
  4537. )
  4538. raw_base_url = normalize_actual_base_url(raw_base_url)
  4539. if not api_key and is_actual_local_base_url(raw_base_url):
  4540. api_key = ACTUAL_LOCAL_NOAUTH_PLACEHOLDER
  4541. if not api_key:
  4542. logger.debug("resolve_provider_client: provider %s has no API key configured (tried: %s)",
  4543. provider, ", ".join(pconfig.api_key_env_vars))
  4544. return None, None
  4545. base_url = _to_openai_base_url(raw_base_url)
  4546. # Explicit base_url override: a fallback_model/custom_providers entry pointing a built-in name elsewhere.
  4547. if req.explicit_base_url and provider != "actual":
  4548. base_url = _to_openai_base_url(req.explicit_base_url.strip().rstrip("/"))
  4549. final_model = _normalize_resolved_model(req.model or _get_aux_model_for_provider(provider), provider)
  4550. # Consulted before the built-in gemini/OpenAI ladder so a registered native transport wins (#112384).
  4551. profile_client = _api_key_profile_supplied_client(provider, api_key=api_key, base_url=base_url)
  4552. if profile_client is not None:
  4553. logger.debug("resolve_provider_client: %s native client from provider profile (%s)", provider, final_model)
  4554. return _route_client(req, profile_client, final_model)
  4555. if provider == "gemini":
  4556. from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url
  4557. if is_native_gemini_base_url(base_url):
  4558. client = GeminiNativeClient(api_key=api_key, base_url=base_url)
  4559. logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
  4560. return _route_client(req, client, final_model)
  4561. headers = _endpoint_default_headers(base_url, provider, is_vision=req.is_vision, xai=True)
  4562. client = _create_openai_client(api_key=api_key, base_url=base_url, **({"default_headers": headers} if headers else {}))
  4563. from agent.provider_route_hooks import model_api_mode # a Responses-only model: call_llm() streams responses
  4564. from providers import get_provider_profile
  4565. profile = get_provider_profile(provider) if final_model and not req.raw_codex else None
  4566. if profile is not None and model_api_mode(profile, final_model) == "codex_responses":
  4567. logger.debug("resolve_provider_client: %s model %s needs Responses API — wrapping", provider, final_model)
  4568. client = CodexAuxiliaryClient(client, final_model)
  4569. # api_mode handling for any API-key provider (direct OpenAI + codex model) and Anthropic-wire
  4570. # endpoints (api.kimi.com/coding, /anthropic gateways) without per-provider branches.
  4571. client = _wrap_transport(req, client, final_model, raw_base_url, api_key)
  4572. logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
  4573. return _route_client(req, client, final_model)
  4574. def _resolve_external_process_branch(req: _ResolveRequest, creds: dict[str, Any]) -> _ResolveResult:
  4575. """PROVIDER_REGISTRY ``external_process`` providers, served via their registered profile."""
  4576. provider = req.provider
  4577. final_model = _normalize_resolved_model(
  4578. req.model or (req.main_runtime.get("model") if req.main_runtime else None) or _read_main_model_for_aux(),
  4579. provider,
  4580. )
  4581. # Keyed on the registered profile, not a provider name, so an out-of-tree ACP provider reaches
  4582. # the auxiliary path (compression, vision, background review) exactly like the in-tree one.
  4583. try:
  4584. from providers import get_provider_profile as _get_provider_profile
  4585. _extproc_profile = _get_provider_profile(provider)
  4586. except Exception:
  4587. _extproc_profile = None
  4588. if _extproc_profile is not None:
  4589. api_key = str(creds.get("api_key", "")).strip()
  4590. base_url = str(creds.get("base_url", "")).strip()
  4591. if not final_model:
  4592. logger.warning("resolve_provider_client: %s requested but no model was provided or configured", provider)
  4593. return None, None
  4594. if not api_key or not base_url:
  4595. logger.warning("resolve_provider_client: %s requested but external process credentials are incomplete", provider)
  4596. return None, None
  4597. try:
  4598. client = _extproc_profile.create_client(
  4599. api_key=api_key, base_url=base_url,
  4600. command=str(creds.get("command", "")).strip() or None, args=list(creds.get("args") or []))
  4601. except Exception:
  4602. logger.warning("resolve_provider_client: profile %r failed to create an external-process client",
  4603. provider, exc_info=True)
  4604. client = None
  4605. if client is not None:
  4606. logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
  4607. return _route_client(req, client, final_model)
  4608. _log_once_debug(_LOGGED_UNSUPPORTED_EXTPROC_KEYS, provider,
  4609. "resolve_provider_client: external-process provider %s not "
  4610. "directly supported", provider)
  4611. return None, None
  4612. def _resolve_registry_branch(req: _ResolveRequest) -> _ResolveResult:
  4613. """PROVIDER_REGISTRY providers, dispatched on ``auth_type``; unknown providers log once."""
  4614. provider = req.provider
  4615. try:
  4616. from hermes_cli.auth import (
  4617. PROVIDER_REGISTRY, resolve_api_key_provider_credentials,
  4618. resolve_external_process_provider_credentials,
  4619. )
  4620. except ImportError:
  4621. logger.debug("hermes_cli.auth not available for provider %s", provider)
  4622. return None, None
  4623. pconfig = PROVIDER_REGISTRY.get(provider)
  4624. if pconfig is None:
  4625. _log_once_debug(_LOGGED_UNKNOWN_PROVIDER_KEYS, provider,
  4626. "resolve_provider_client: unknown provider %r", provider)
  4627. return None, None
  4628. auth_type = pconfig.auth_type
  4629. if auth_type == "api_key":
  4630. return _resolve_api_key_branch(req, pconfig, resolve_api_key_provider_credentials)
  4631. if auth_type == "external_process":
  4632. return _resolve_external_process_branch(req, resolve_external_process_provider_credentials(provider))
  4633. from agent.auxiliary_client_registry import REGISTRY_AUTHTYPE_ARMS
  4634. arm = REGISTRY_AUTHTYPE_ARMS.get(auth_type)
  4635. if arm is not None:
  4636. return arm(req)
  4637. # The first occurrence surfaces a real schema-drift bug; per-call retries stay silent.
  4638. _log_once_debug(_LOGGED_UNHANDLED_AUTHTYPE_KEYS, (auth_type, provider),
  4639. "resolve_provider_client: unhandled auth_type %s for %s",
  4640. auth_type, provider)
  4641. return None, None
  4642. # Explicit providers with a dedicated branch; anything else falls through to named custom
  4643. # providers → azure-foundry → PROVIDER_REGISTRY (order preserved from the original if-chain).
  4644. _EXPLICIT_PROVIDER_BRANCHES: dict[str, Callable[[_ResolveRequest], _ResolveResult]] = {
  4645. "auto": _resolve_auto_branch,
  4646. "openrouter": _resolve_openrouter_branch,
  4647. "nous": _resolve_nous_branch,
  4648. "openai-codex": _resolve_openai_codex_branch,
  4649. "xai-oauth": _resolve_xai_oauth_branch,
  4650. "custom": _resolve_custom_branch,
  4651. }
  4652. def resolve_provider_client(
  4653. provider: str, model: str | None = None, async_mode: bool = False, raw_codex: bool = False,
  4654. explicit_base_url: str | None = None, explicit_api_key: Optional[str | Callable[[], str]] = None,
  4655. api_mode: str | None = None, main_runtime: Optional[dict[str, Any]] = None, is_vision: bool = False,
  4656. task: Optional[str] = None,
  4657. ) -> tuple[Optional[Any], Optional[str]]:
  4658. """Central router: return a configured client (auth, base URL, API format) for a provider + optional model.
  4659. The client always exposes ``.chat.completions.create()``; Codex/Responses providers get an adapter.
  4660. ``provider``: built-in name, ``custom:<name>``, "custom" (OPENAI_BASE_URL + OPENAI_API_KEY) or "auto"
  4661. (full auto-detection chain). ``model=None`` → provider's default aux model. ``raw_codex`` → bare OpenAI
  4662. client for ``responses.stream()`` callers. ``api_mode`` forces "codex_responses"/"chat_completions"/
  4663. "anthropic_messages" instead of auto-detect. Returns (client, resolved_model) or (None, None)."""
  4664. _validate_proxy_env_urls()
  4665. # Keep the pre-alias name so a custom_providers entry named like a built-in alias
  4666. # (e.g. "kimi" → "kimi-coding") is still reachable via the named-custom branch.
  4667. original_provider = (provider or "").strip().lower()
  4668. provider = _normalize_aux_provider(provider)
  4669. api_mode = _canonical_api_mode(str(api_mode or "")).lower() or None
  4670. # MoA chokepoint: "moa" is not an HTTP provider; resolve to the aggregator so direct callers don't
  4671. # dead-end in unknown-provider. Unresolvable preset → leave untouched for the normal diagnostic.
  4672. if provider == "moa":
  4673. _agg_provider, _agg_model = _resolve_moa_aggregator(model)
  4674. if _agg_provider and _agg_model:
  4675. original_provider = _agg_provider.strip().lower()
  4676. provider = _normalize_aux_provider(_agg_provider)
  4677. model = _agg_model
  4678. # The moa:// facade endpoint/key belong to the virtual runtime, not the aggregator.
  4679. if explicit_base_url and str(explicit_base_url).lower().startswith("moa://"):
  4680. explicit_base_url = None
  4681. explicit_api_key = None
  4682. from agent.auxiliary_local_runtime import bare_llamacpp_endpoint
  4683. local = bare_llamacpp_endpoint(original_provider, explicit_base_url, explicit_api_key)
  4684. if local is not None:
  4685. if not local[0]:
  4686. logger.warning("resolve_provider_client: %s requested but no local llama.cpp server is running", original_provider)
  4687. return None, None
  4688. explicit_base_url, explicit_api_key = local
  4689. # Model for concrete providers: caller ``model`` → catalog default (empty for OAuth-gated providers whose
  4690. # lists drift) → configured main model (MoA → aggregator), keeping OAuth aux tasks off the Step-2 fallback.
  4691. # Excluded: ``auto`` (a stale main slug could pair with any picked provider) and Nous + vision (the
  4692. # Portal's tier-aware vision recommendation must win over a text-only model).
  4693. if not model and provider != "auto" and not (provider == "nous" and is_vision):
  4694. # ``auto`` is intentionally excluded: `_resolve_auto_route(main_runtime=...)` returns the model paired
  4695. # with the provider it actually selected. Pre-filling an auto call from `_read_main_model()` can
  4696. # leak a stale process-global runtime into a different provider (for example Claude model slug on
  4697. # Codex OAuth) and override that correctly resolved model. 1. ``model`` argument (caller knew what
  4698. # they wanted) 2. Provider's catalog default — cheap/fast model the provider registered via
  4699. # ``ProviderProfile.default_aux_model`` or the legacy ``_API_KEY_PROVIDER_AUX_MODELS_FALLBACK``
  4700. # dict. 3. User's main model from ``model.model`` in config.yaml. This is the load-bearing step for
  4701. # OAuth providers: an xai-oauth user with grok-4.3 configured gets grok-4.3 for title generation
  4702. # instead of silently dropping to whatever Step-2 fallback (#31845). When the main provider is MoA,
  4703. # ``_read_main_model_for_aux()`` substitutes the preset's aggregator model — the preset NAME is
  4704. # never a valid wire model id, so unset aux models default to the preset's acting model instead.
  4705. # Each provider branch below sees a non-empty ``model`` whenever the user has *anything* configured
  4706. # — no provider-specific empty-model guards needed. When the user has NOTHING configured (fresh
  4707. # install, main_model also empty), the branches still hit their own missing-credentials returns and
  4708. # ``_resolve_auto_route`` falls through to the Step-2 chain as before. Do NOT pre-fill a blank ``auto``
  4709. # request from the config/main default here. Claude model sent to Codex after the main lane fell
  4710. # back to gpt-5.5). Let _resolve_auto_route() return the actual current runtime model when the caller did
  4711. # not explicitly request one. (# compression-current-model) Nous + vision is the one carve-out: the
  4712. # branch below resolves its model from the Portal's tier-aware vision recommendation
  4713. # (``_try_nous(vision= True)``), and ``final_model = model or default`` means anything pre-filled
  4714. # here wins over that. The main chat model is routinely text-only (e.g. a ``:free`` chat SKU), so
  4715. # pre-filling it sends the image to a model that cannot accept one and the Portal 404s. Leave
  4716. # ``model`` unset and let the Portal slot through; only an explicit caller model may override it.
  4717. model = _get_aux_model_for_provider(provider) or _read_main_model_for_aux() or model
  4718. req = _ResolveRequest(
  4719. provider, original_provider, model, async_mode, raw_codex,
  4720. explicit_base_url, explicit_api_key, api_mode, main_runtime, is_vision, task,
  4721. )
  4722. branch = _EXPLICIT_PROVIDER_BRANCHES.get(provider)
  4723. alias_identity = original_provider.removeprefix("custom:")
  4724. # A configured provider whose name collides with a local-server alias is still a named
  4725. # provider. Resolve it before the alias's generic ``custom`` branch, which otherwise loses
  4726. # the configured base_url and can fall through to an unrelated credentialed provider.
  4727. if branch is None or alias_identity in _LOCAL_SERVER_ALIASES:
  4728. try:
  4729. result = _resolve_named_custom_branch(req)
  4730. except ImportError:
  4731. result = None
  4732. if result is not None:
  4733. return result
  4734. if branch is not None:
  4735. return branch(req)
  4736. if provider == "azure-foundry":
  4737. return _resolve_azure_foundry_branch(req)
  4738. return _resolve_registry_branch(req)
  4739. # ── Public API ──────────────────────────────────────────────────────────────
  4740. def get_text_auxiliary_client(task: str = "", *, main_runtime: Optional[dict[str, Any]] = None) -> tuple[Optional[OpenAI], Optional[str]]:
  4741. """Return (client, default_model_slug) for text-only aux tasks; ``task`` selects auxiliary.<task> overrides."""
  4742. provider, model, base_url, api_key, api_mode = _resolve_task_provider_model(task or None)
  4743. return resolve_provider_client(
  4744. provider, model=model, explicit_base_url=base_url, explicit_api_key=api_key,
  4745. api_mode=api_mode, main_runtime=main_runtime,
  4746. )
  4747. _VISION_AUTO_PROVIDER_ORDER = ("openrouter", "nous", "deepinfra")
  4748. def _main_model_supports_vision(provider: str, model: Optional[str]) -> bool:
  4749. """True when ``provider``/``model`` is known to accept image input; unknown capability → True (attempt the call)."""
  4750. try:
  4751. from agent.image_routing import _lookup_supports_vision
  4752. from hermes_cli.config import load_config_readonly
  4753. except ImportError:
  4754. return True
  4755. try:
  4756. supports = _lookup_supports_vision(provider, model, load_config_readonly())
  4757. except Exception: # pragma: no cover - defensive
  4758. return True
  4759. return True if supports is None else bool(supports)
  4760. def _normalize_vision_provider(provider: Optional[str]) -> str:
  4761. return _normalize_aux_provider(provider)
  4762. def _deepinfra_strict_vision_backend(model: Optional[str]) -> tuple[Optional[Any], Optional[str]]:
  4763. """DeepInfra vision: default model is discovered live via default_vision_model() so no hardcoded id can rot."""
  4764. vision_model = model or _resolve_provider_vision_default("deepinfra")
  4765. if not vision_model:
  4766. logger.debug("Vision auto-detect: deepinfra catalog unreachable or returned no vision-tagged models — skipping")
  4767. return None, None
  4768. return resolve_provider_client("deepinfra", vision_model, is_vision=True)
  4769. # Strict (explicitly requested) vision backends by normalized provider name. nous MUST go
  4770. # through resolve_provider_client so anthropic/* picks wrap onto /v1/messages (a bare _try_nous
  4771. # client 404s). openai-codex has no safe default model; callers set auxiliary.<task>.model.
  4772. _STRICT_VISION_BACKENDS: dict[str, Callable[[Optional[str]], tuple[Optional[Any], Optional[str]]]] = {
  4773. "openrouter": lambda model: _try_openrouter(model=model),
  4774. "nous": lambda model: resolve_provider_client("nous", model, is_vision=True),
  4775. "openai-codex": lambda model: resolve_provider_client("openai-codex", model, is_vision=True),
  4776. "anthropic": lambda model: _try_anthropic(),
  4777. "deepinfra": _deepinfra_strict_vision_backend,
  4778. "custom": lambda model: _try_custom_endpoint(),
  4779. }
  4780. def _resolve_strict_vision_backend(provider: str, model: Optional[str] = None) -> tuple[Optional[Any], Optional[str]]:
  4781. backend = _STRICT_VISION_BACKENDS.get(_normalize_vision_provider(provider))
  4782. return backend(model) if backend is not None else (None, None)
  4783. def get_available_vision_backends() -> list[str]:
  4784. """Available vision backends in auto-selection order (active provider → OpenRouter → Nous → DeepInfra).
  4785. Single source of truth for setup, tool gating, and runtime auto-routing.
  4786. """
  4787. available: list[str] = []
  4788. main_provider = _read_main_provider()
  4789. if main_provider and main_provider not in {"auto", ""}:
  4790. if main_provider in _VISION_AUTO_PROVIDER_ORDER:
  4791. main_ok = _resolve_strict_vision_backend(main_provider)[0] is not None
  4792. else:
  4793. main_ok = resolve_provider_client(main_provider, _read_main_model())[0] is not None
  4794. if main_ok:
  4795. available.append(main_provider)
  4796. for p in _VISION_AUTO_PROVIDER_ORDER: # skip if already covered by main provider
  4797. if p not in available and _resolve_strict_vision_backend(p)[0] is not None:
  4798. available.append(p)
  4799. return available
  4800. def _finalize_vision_client(
  4801. resolved_provider: str, sync_client: Any, default_model: Optional[str],
  4802. resolved_model: Optional[str], async_mode: bool,
  4803. ) -> tuple[Optional[str], Optional[Any], Optional[str]]:
  4804. """Apply the explicit model override (and async wrapping) to a resolved vision client."""
  4805. if sync_client is None:
  4806. return resolved_provider, None, None
  4807. final_model = resolved_model or default_model
  4808. if async_mode:
  4809. async_client, async_model = _to_async_client(sync_client, final_model, is_vision=True)
  4810. return resolved_provider, async_client, async_model
  4811. return resolved_provider, sync_client, final_model
  4812. def _vision_main_provider_client(
  4813. main_provider: str, main_model: str, runtime: dict[str, Any], resolved_model: Optional[str],
  4814. resolved_api_mode: Optional[str],
  4815. ) -> tuple[Optional[Any], Optional[str]]:
  4816. """Auto-detect step 1: try the main provider; (None, None) falls through to the aggregator chain."""
  4817. # A provider vision default (static override or catalog discovery) is a *known* multimodal
  4818. # model; the pinned chat model usually isn't, so only fall back to it when no default exists.
  4819. provider_vision_default = _resolve_provider_vision_default(main_provider)
  4820. vision_model = provider_vision_default or main_model
  4821. if main_provider == "nous":
  4822. # Nous picks its vision model from Portal tier-aware slots inside _try_nous(vision=True);
  4823. # passing the chat model would override that and 404. Only auxiliary.vision.model may.
  4824. sync_client, default_model = _resolve_strict_vision_backend(main_provider, resolved_model or provider_vision_default)
  4825. if sync_client is None:
  4826. return None, None
  4827. logger.info("Vision auto-detect: using main provider %s (%s)", main_provider, default_model or resolved_model or main_model)
  4828. return sync_client, default_model
  4829. if main_provider in _PROVIDERS_WITHOUT_VISION: # endpoint rejects image input entirely
  4830. logger.debug("Vision auto-detect: skipping main provider %s (no vision support) — falling through to aggregator chain", main_provider)
  4831. return None, None
  4832. if not _main_model_supports_vision(main_provider, vision_model):
  4833. # Known text-only model. Log only the provider name (CodeQL clear-text-logging FPs).
  4834. logger.debug(
  4835. "Vision auto-detect: skipping main provider %s (reports no vision capability) — falling through to aggregator chain",
  4836. main_provider,
  4837. )
  4838. return None, None
  4839. # Custom endpoints carry no built-in base_url/api_key: recover the live main endpoint from
  4840. # set_runtime_main() or, with no live runtime recorded, the configured custom endpoint.
  4841. rpc_base_url = rpc_api_key = None
  4842. rpc_api_mode = resolved_api_mode
  4843. if main_provider == "custom" or main_provider.startswith("custom:"):
  4844. if runtime.get("base_url"):
  4845. custom_base, custom_key, custom_mode = runtime.get("base_url"), runtime.get("api_key") or None, runtime.get("api_mode")
  4846. else:
  4847. custom_base, custom_key, custom_mode = _resolve_custom_runtime()
  4848. if custom_base:
  4849. rpc_base_url, rpc_api_key = custom_base, custom_key
  4850. rpc_api_mode = resolved_api_mode or custom_mode or None
  4851. rpc_client, rpc_model = resolve_provider_client(
  4852. main_provider, vision_model, api_mode=rpc_api_mode, explicit_base_url=rpc_base_url,
  4853. explicit_api_key=rpc_api_key, main_runtime=runtime, is_vision=True)
  4854. if rpc_client is None:
  4855. return None, None
  4856. logger.info("Vision auto-detect: using main provider %s (%s)", main_provider, rpc_model or vision_model)
  4857. return rpc_client, rpc_model or vision_model
  4858. def _vision_auto_route(
  4859. runtime: dict[str, Any], resolved_model: Optional[str], resolved_api_mode: Optional[str],
  4860. async_mode: bool,
  4861. ) -> tuple[Optional[str], Optional[Any], Optional[str]]:
  4862. """Auto-detect order: 1. main provider + model, 2. OpenRouter, 3. Nous Portal, 4. DeepInfra, 5. stop."""
  4863. main_provider = str(runtime.get("provider") or _read_main_provider())
  4864. main_model = str(runtime.get("model") or _read_main_model())
  4865. if main_provider.strip().lower() == "moa":
  4866. # MoA main_model is a preset NAME, not a wire model — unwrap to the preset's aggregator
  4867. # slot. The moa:// facade endpoint belongs to the virtual provider, not the real one.
  4868. _agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
  4869. if _agg_provider and _agg_model:
  4870. main_provider, main_model = _agg_provider, _agg_model
  4871. runtime = dict(runtime, base_url="", api_key="", api_mode="")
  4872. if main_provider and main_provider not in {"auto", "", "moa"}:
  4873. client, default_model = _vision_main_provider_client(main_provider, main_model, runtime, resolved_model, resolved_api_mode)
  4874. if client is not None:
  4875. return _finalize_vision_client(main_provider, client, default_model, resolved_model, async_mode)
  4876. # Aggregators use their dedicated vision model, not the user's main model.
  4877. for candidate in _VISION_AUTO_PROVIDER_ORDER:
  4878. if candidate == main_provider:
  4879. continue # already tried above
  4880. sync_client, default_model = _resolve_strict_vision_backend(candidate)
  4881. if sync_client is not None:
  4882. return _finalize_vision_client(candidate, sync_client, default_model, resolved_model, async_mode)
  4883. logger.debug("Auxiliary vision client: none available")
  4884. return None, None, None
  4885. # ZAI vision must use the OpenAI-compatible endpoint: the Anthropic wire rejects max_tokens on
  4886. # multimodal calls (error 1210).
  4887. _ZAI_OPENAI_VISION_URLS = ("https://open.bigmodel.cn/api/paas/v4", "https://api.z.ai/api/paas/v4")
  4888. def resolve_vision_provider_client(
  4889. provider: Optional[str] = None, model: Optional[str] = None, *, base_url: Optional[str] = None,
  4890. api_key: Optional[str] = None, async_mode: bool = False,
  4891. main_runtime: Optional[dict[str, Any]] = None,
  4892. ) -> tuple[Optional[str], Optional[Any], Optional[str]]:
  4893. """Resolve the client actually used for vision tasks.
  4894. Direct endpoint overrides beat provider selection; explicit providers may force
  4895. experimental backends; auto mode only tries backends known to work.
  4896. """
  4897. runtime = _normalize_main_runtime(main_runtime)
  4898. requested, resolved_model, resolved_base_url, resolved_api_key, resolved_api_mode = _resolve_task_provider_model(
  4899. "vision", provider, model, base_url, api_key
  4900. )
  4901. # The raw name keeps a llama.cpp alias distinguishable from bare ``custom`` for the last resolve.
  4902. raw_requested, requested = requested, _normalize_vision_provider(requested)
  4903. if resolved_base_url:
  4904. provider_for_base_override = requested if requested and requested not in {"", "auto"} else "custom"
  4905. client, final_model = resolve_provider_client(
  4906. provider_for_base_override, model=resolved_model, async_mode=async_mode,
  4907. explicit_base_url=resolved_base_url, explicit_api_key=resolved_api_key,
  4908. api_mode=resolved_api_mode, main_runtime=runtime,
  4909. )
  4910. return provider_for_base_override, client, (final_model if client is not None else None)
  4911. if requested == "auto":
  4912. return _vision_auto_route(runtime, resolved_model, resolved_api_mode, async_mode)
  4913. if requested in _VISION_AUTO_PROVIDER_ORDER:
  4914. sync_client, default_model = _resolve_strict_vision_backend(requested, resolved_model)
  4915. return _finalize_vision_client(requested, sync_client, default_model, resolved_model, async_mode)
  4916. if requested == "zai":
  4917. for _zai_url in _ZAI_OPENAI_VISION_URLS:
  4918. client, final_model = _get_cached_client(
  4919. requested, resolved_model, async_mode, base_url=_zai_url,
  4920. api_key=resolved_api_key or None, api_mode="chat_completions", main_runtime=runtime,
  4921. is_vision=True,
  4922. )
  4923. if client is not None:
  4924. return _finalize_vision_client(requested, client, final_model, resolved_model, async_mode)
  4925. # Fallback: try without explicit base_url (old behavior)
  4926. client, final_model = _get_cached_client(
  4927. raw_requested, resolved_model, async_mode, api_mode=resolved_api_mode, main_runtime=runtime, is_vision=True,
  4928. )
  4929. return requested, client, (final_model if client is not None else None)
  4930. def auxiliary_max_tokens_param(value: int, *, model: Optional[str] = None) -> dict:
  4931. """Max-tokens kwarg for the auxiliary provider: direct OpenAI, a custom endpoint whose profile declares
  4932. it, and newer OpenAI-family models (by ``model`` name: custom endpoints fronting gpt-5.x) need max_completion_tokens."""
  4933. from agent.provider_route_hooks import route_answer
  4934. _custom_base = _current_custom_base_url()
  4935. direct_openai_family = not _scoped_key_env("OPENROUTER_API_KEY") and _read_nous_auth() is None and (
  4936. base_url_hostname(_custom_base) == "api.openai.com"
  4937. or route_answer("max_tokens_param", base_url=_custom_base, model=model or "") == "max_completion_tokens")
  4938. if direct_openai_family or model_forces_max_completion_tokens(model):
  4939. return {"max_completion_tokens": value}
  4940. return {"max_tokens": value}
  4941. # ── Centralized LLM Call API: call_llm()/async_call_llm() own resolve → cached client → shape
  4942. # request → call → return. Every auxiliary LLM consumer should use these.
  4943. # Client cache: (provider, async_mode, base_url, api_key, api_mode, runtime_key) -> (client, default_model, loop)
  4944. # Loop identity is NOT part of the key: stale-loop entries are replaced in place on async hits,
  4945. # bounding growth to one entry per provider config (avoids fd accumulation in gateways).
  4946. # This bounds cache growth to one entry per unique provider config rather than one per (config ×
  4947. # event-loop), which previously caused unbounded fd accumulation in long-running gateway processes (#10200).
  4948. _client_cache: dict[tuple, tuple] = {}
  4949. _client_cache_lock = threading.Lock()
  4950. _CLIENT_CACHE_MAX_SIZE = 64 # safety belt — evict oldest when exceeded
  4951. class _CallableCacheDiscriminator:
  4952. """Hash a credential callback by identity without exposing its state."""
  4953. __slots__ = ("_callback",)
  4954. def __init__(self, callback: Any) -> None:
  4955. self._callback = callback # retained so its id cannot be reused while cached
  4956. def __hash__(self) -> int:
  4957. return id(self._callback)
  4958. def __eq__(self, other: object) -> bool:
  4959. return isinstance(other, _CallableCacheDiscriminator) and self._callback is other._callback
  4960. def __repr__(self) -> str:
  4961. return "<callable-api-key>"
  4962. def _runtime_cache_discriminator(field: str, value: Any) -> Any:
  4963. """Return a hashable, secret-safe runtime cache-key component."""
  4964. if field == "api_key" and callable(value):
  4965. return _CallableCacheDiscriminator(value)
  4966. if field == "api_key" and isinstance(value, str) and value:
  4967. return ("api-key-digest", hashlib.blake2b(value.encode("utf-8"), digest_size=16).digest())
  4968. return value
  4969. def _client_cache_key(
  4970. provider: str, *, async_mode: bool, base_url: Optional[str] = None,
  4971. api_key: Optional[str] = None, api_mode: Optional[str] = None,
  4972. main_runtime: Optional[dict[str, Any]] = None, is_vision: bool = False,
  4973. task: Optional[str] = None, model: Optional[str] = None,
  4974. ) -> tuple:
  4975. runtime = _normalize_main_runtime(main_runtime)
  4976. # `auto` resolves through the main runtime and task-specific policy, so both join the key.
  4977. runtime_key = tuple(_runtime_cache_discriminator(f, runtime.get(f, "")) for f in _MAIN_RUNTIME_FIELDS) if provider == "auto" else ()
  4978. task_key = (task or "", _task_prefers_fast_model(task)) if provider == "auto" else ""
  4979. pool_hint = _pool_cache_hint(provider, main_runtime=main_runtime)
  4980. # Model MUST be in the key: concurrent calls to the same endpoint with different models would
  4981. # share an entry, and the second builder's _store_cached_client would close the first's client.
  4982. model_key = model or runtime.get("model", "")
  4983. api_key_key = _runtime_cache_discriminator("api_key", api_key or "")
  4984. # Profile home leads the key: callers that omit api_key (pool / Nous auth.json paths) would
  4985. # otherwise share one client across multiplex profiles holding different credentials.
  4986. return (hermes_home_key(), provider, async_mode, base_url or "", api_key_key, api_mode or "", runtime_key, is_vision, task_key, pool_hint, model_key,
  4987. _borrowed_main_credential_key(provider, base_url, api_key, runtime))
  4988. def _borrowed_main_credential_key(provider: str, base_url: Optional[str], api_key: Any, runtime: dict[str, Any]) -> tuple:
  4989. """What a keyless ``custom`` route borrows from the main runtime when its client is built.
  4990. The client keeps that credential for its lifetime, so it joins the cache key: otherwise a
  4991. later runtime (another session, a ``/model`` switch) is served the earlier one's key.
  4992. """
  4993. if _normalize_aux_provider(provider) != "custom":
  4994. return ()
  4995. if not base_url:
  4996. # This shape takes the runtime's endpoint and key even when an explicit key was passed.
  4997. return (runtime.get("base_url", ""), _runtime_cache_discriminator("api_key", runtime.get("api_key", "")))
  4998. # Same normalization as the client build, which treats a blank explicit key as keyless.
  4999. if _normalize_api_key(api_key):
  5000. return ()
  5001. borrowed = _read_main_api_key_if_same_origin(_to_openai_base_url(base_url).strip())
  5002. return (_runtime_cache_discriminator("api_key", borrowed),)
  5003. def _current_event_loop() -> Any:
  5004. """``asyncio.get_event_loop()`` or None when no loop can be obtained (async cache-key binding)."""
  5005. try:
  5006. import asyncio as _aio
  5007. return _aio.get_event_loop()
  5008. except RuntimeError:
  5009. return None
  5010. def _store_cached_client(cache_key: tuple, client: Any, default_model: Optional[str], *, bound_loop: Any = None) -> None:
  5011. if isinstance(client, _AuxProbeClientStub):
  5012. return # probe stubs must never be cached — the next hit would get a dud client
  5013. with _client_cache_lock:
  5014. old_entry = _client_cache.get(cache_key)
  5015. if old_entry is not None and old_entry[0] is not client:
  5016. _close_cached_client(old_entry[0])
  5017. _client_cache[cache_key] = (client, default_model, bound_loop)
  5018. def _refresh_nous_auxiliary_client(
  5019. *, cache_provider: str, model: Optional[str], async_mode: bool, base_url: Optional[str] = None,
  5020. api_key: Optional[str] = None, api_mode: Optional[str] = None,
  5021. main_runtime: Optional[dict[str, Any]] = None, is_vision: bool = False,
  5022. lookup_model: Optional[str] = None, lookup_task: Optional[str] = None,
  5023. ) -> tuple[Optional[Any], Optional[str]]:
  5024. """Refresh Nous runtime creds, rebuild the client, and replace the cache entry.
  5025. ``model`` is the resolved wire model stored as the entry's usable model and returned. The
  5026. cache KEY MUST be built from ``lookup_model``/``lookup_task`` — the model and task as passed
  5027. to ``_get_cached_client`` when the stale client was acquired — so the fresh client overwrites
  5028. the exact entry the stale one is served from. Keying on the resolved model or an empty task
  5029. would leave the expired client immortal and every auxiliary call 401ing forever.
  5030. See #56889.
  5031. For ``provider == "auto"`` the task participates in the cache key (task-specific fallback policy), so it
  5032. MUST be carried into the key here for the same reason as ``lookup_model``; otherwise an auto-provider
  5033. client refreshed on a 401 lands under the ``task=""`` key while the stale entry survives under the
  5034. task-scoped key (#58894).
  5035. """
  5036. runtime = _resolve_nous_runtime_api(force_refresh=True, stale_access_token=api_key)
  5037. if runtime is None:
  5038. return None, model
  5039. fresh_key, fresh_base_url = runtime
  5040. sync_client = _create_openai_client(api_key=fresh_key, base_url=fresh_base_url)
  5041. current_loop = _current_event_loop() if async_mode else None
  5042. if async_mode:
  5043. client, final_model = _to_async_client(sync_client, model or "", is_vision=is_vision)
  5044. else:
  5045. client, final_model = sync_client, model
  5046. cache_key = _client_cache_key(
  5047. cache_provider, async_mode=async_mode, base_url=base_url, api_key=api_key,
  5048. api_mode=api_mode, main_runtime=main_runtime, is_vision=is_vision, task=lookup_task,
  5049. model=lookup_model,
  5050. )
  5051. _store_cached_client(cache_key, client, final_model, bound_loop=current_loop)
  5052. return client, final_model
  5053. def neuter_async_httpx_del() -> None:
  5054. """Monkey-patch ``AsyncHttpxClientWrapper.__del__`` to be a no-op.
  5055. The SDK's ``__del__`` schedules ``aclose()`` on the *running* loop, but the transport is
  5056. bound to the loop the client was created on; when that loop is dead this raises "Event loop
  5057. is closed" into prompt_toolkit's loop. Safe because cached clients are closed explicitly and
  5058. the OS reaps the rest. Call once at CLI startup, before any ``AsyncOpenAI`` is created.
  5059. """
  5060. try:
  5061. from openai._base_client import AsyncHttpxClientWrapper
  5062. AsyncHttpxClientWrapper.__del__ = lambda self: None # type: ignore[assignment]
  5063. except (ImportError, AttributeError):
  5064. pass # Graceful degradation if the SDK changes its internals
  5065. def _force_close_async_httpx(client: Any) -> None:
  5066. """Mark the httpx AsyncClient inside an AsyncOpenAI client as closed so ``__del__`` won't
  5067. schedule ``aclose()`` on a dead loop. Skips the full async close — the OS drops connections."""
  5068. with contextlib.suppress(Exception):
  5069. from httpx._client import ClientState
  5070. inner = getattr(client, "_client", None)
  5071. if inner is not None and not getattr(inner, "is_closed", True):
  5072. inner._state = ClientState.CLOSED
  5073. def _schedule_async_close(close_result: Any, client: Any) -> None:
  5074. """Finish an async close without leaking an unawaited coroutine."""
  5075. async def _await_close() -> None:
  5076. try:
  5077. await close_result
  5078. except Exception:
  5079. pass
  5080. finally:
  5081. _force_close_async_httpx(client)
  5082. runner = _await_close()
  5083. try:
  5084. import asyncio as _aio
  5085. try:
  5086. loop = _aio.get_running_loop()
  5087. except RuntimeError:
  5088. _aio.run(runner)
  5089. else:
  5090. task = loop.create_task(runner)
  5091. def _consume(completed_task) -> None:
  5092. with contextlib.suppress(BaseException):
  5093. completed_task.exception()
  5094. task.add_done_callback(_consume)
  5095. runner = None
  5096. except Exception:
  5097. if runner is not None:
  5098. with contextlib.suppress(Exception):
  5099. runner.close()
  5100. _force_close_async_httpx(client)
  5101. def _close_cached_client(client: Any, *, close_async: bool = False) -> None:
  5102. """Close one cached client, awaiting async transports only when safe."""
  5103. if client is None:
  5104. return
  5105. close_fn = getattr(client, "close", None)
  5106. if not callable(close_fn):
  5107. _force_close_async_httpx(client)
  5108. return
  5109. try:
  5110. close_result = close_fn()
  5111. except Exception:
  5112. _force_close_async_httpx(client)
  5113. return
  5114. if inspect.isawaitable(close_result):
  5115. if close_async:
  5116. _schedule_async_close(close_result, client)
  5117. else:
  5118. # Never await a client owned by another live loop; close the coroutine (no
  5119. # unawaited warning) and neuter the transport.
  5120. with contextlib.suppress(Exception):
  5121. close_result.close()
  5122. _force_close_async_httpx(client)
  5123. return
  5124. _force_close_async_httpx(client)
  5125. def shutdown_cached_clients() -> None:
  5126. """Close all cached clients; call at CLI shutdown *before* the loop closes.
  5127. Snapshot+clear under the lock, close outside it: async teardown can block while an owner
  5128. loop drains, and holding the lock would convoy every caller.
  5129. """
  5130. with _client_cache_lock:
  5131. clients = [(entry[0], entry[2]) for entry in _client_cache.values() if entry[0] is not None]
  5132. _client_cache.clear()
  5133. try:
  5134. import asyncio as _aio
  5135. running_loop = _aio.get_running_loop()
  5136. except RuntimeError:
  5137. running_loop = None
  5138. for client, owner_loop in clients:
  5139. # A live foreign loop owns its transport — neuter only and let it finish teardown.
  5140. # Closed loops and the current loop are safe to drain here.
  5141. close_async = owner_loop is not None and (owner_loop.is_closed() or owner_loop is running_loop)
  5142. _close_cached_client(client, close_async=close_async)
  5143. def cleanup_stale_async_clients() -> None:
  5144. """Force-close cached async clients whose loop is closed; call after each agent turn
  5145. (defense-in-depth behind ``neuter_async_httpx_del``)."""
  5146. with _client_cache_lock:
  5147. stale = [(key, entry[0]) for key, entry in _client_cache.items() if entry[2] is not None and entry[2].is_closed()]
  5148. for key, _client in stale:
  5149. del _client_cache[key]
  5150. for _key, client in stale:
  5151. _close_cached_client(client, close_async=True)
  5152. def _compat_model(client: Any, model: Optional[str], cached_default: Optional[str]) -> Optional[str]:
  5153. """Keep slash-bearing model IDs only for cached clients that accept ``vendor/model`` (OpenRouter
  5154. or a slash-bearing default). Mirrors the resolve_provider_client() guard, which cache hits skip."""
  5155. if model and "/" in model:
  5156. accepts_slash = any(
  5157. obj and base_url_host_matches(str(getattr(obj, "base_url", "") or ""), "openrouter.ai")
  5158. for obj in (client, getattr(client, "_client", None), getattr(client, "client", None))
  5159. ) or bool(cached_default and "/" in cached_default)
  5160. if not accepts_slash:
  5161. return cached_default
  5162. return model or cached_default
  5163. def _get_cached_client(
  5164. provider: str, model: str | None = None, async_mode: bool = False, base_url: str | None = None,
  5165. api_key: str | None = None, api_mode: str | None = None, main_runtime: Optional[dict[str, Any]] = None,
  5166. is_vision: bool = False, task: Optional[str] = None,
  5167. ) -> tuple[Optional[Any], Optional[str]]:
  5168. """Get or create a cached client for the given provider.
  5169. Async clients bind to the loop they were created on, so every async hit validates the cached
  5170. loop is the current, open loop; stale entries are replaced in place (bounded, no cross-loop reuse).
  5171. This keeps cache size bounded to one entry per unique provider config, preventing the fd-exhaustion that
  5172. previously occurred in long-running gateways where recycled worker threads created unbounded entries
  5173. (#10200).
  5174. """
  5175. # A bare llama.cpp alias keys on the live local endpoint: a restarted server (new port/key)
  5176. # must not be served the old client, and a stopped one resolves to nothing, not a cloud client.
  5177. from agent.auxiliary_local_runtime import bare_llamacpp_endpoint
  5178. local = bare_llamacpp_endpoint(provider, base_url, api_key)
  5179. if local is not None:
  5180. if not local[0]:
  5181. return None, None
  5182. base_url, api_key = local
  5183. current_loop = _current_event_loop() if async_mode else None
  5184. runtime = _normalize_main_runtime(main_runtime)
  5185. cache_key = _client_cache_key(
  5186. provider, async_mode=async_mode, base_url=base_url, api_key=api_key, api_mode=api_mode,
  5187. main_runtime=main_runtime, is_vision=is_vision, task=task, model=model,
  5188. )
  5189. with _client_cache_lock:
  5190. if cache_key in _client_cache:
  5191. cached_client, cached_default, cached_loop = _client_cache[cache_key]
  5192. loop_ok = not async_mode or (
  5193. cached_loop is not None and cached_loop is current_loop and not cached_loop.is_closed()
  5194. )
  5195. if loop_ok:
  5196. return cached_client, _compat_model(cached_client, model, cached_default)
  5197. # Stale async entry — evict. Only a closed owner loop may be awaited here; a live
  5198. # foreign loop stays force-neutered.
  5199. _close_cached_client(cached_client, close_async=cached_loop is not None and cached_loop.is_closed())
  5200. del _client_cache[cache_key]
  5201. # Build outside the lock. For pool-backed providers derive the key from the pool entry:
  5202. # resolve_api_key_provider_credentials prefers env vars, which would bypass pool rotation
  5203. # and retry an exhausted key. A pinned ``custom:<name>`` peeks only its entry's rows.
  5204. effective_api_key = api_key
  5205. if not effective_api_key:
  5206. _pool = pool_for_aux_provider(provider, _normalize_aux_provider(provider), lambda k: _load_pool_with_credentials(k, " (peek)"))
  5207. if _pool is not None and (_pe := _peek_pool_entry(provider, _pool)) is not None:
  5208. effective_api_key = _pool_runtime_api_key(_pe) or api_key
  5209. client, default_model = resolve_provider_client(
  5210. provider, model, async_mode, explicit_base_url=base_url, explicit_api_key=effective_api_key,
  5211. api_mode=api_mode, main_runtime=runtime, is_vision=is_vision, task=task,
  5212. )
  5213. if client is not None and _aux_probe_active():
  5214. # Availability probes answer "resolvable?" and must leave the cache untouched: the
  5215. # probe stub (bare, or wrapped in a Codex/Anthropic adapter whose leaf is the stub)
  5216. # shares the runtime key, and a cached one is served to every later caller — the
  5217. # next probe dies in _compat_model() on stub attribute access, so check_fns flip to
  5218. # False and vision tools vanish for the process lifetime (#87654).
  5219. return client, model or default_model
  5220. if client is not None:
  5221. with _client_cache_lock:
  5222. if cache_key not in _client_cache:
  5223. # FIFO safety-belt eviction. Do NOT close evicted clients: another caller may be
  5224. # mid-request on one; refcount/GC handles it.
  5225. while len(_client_cache) >= _CLIENT_CACHE_MAX_SIZE:
  5226. del _client_cache[next(iter(_client_cache))]
  5227. _client_cache[cache_key] = (client, default_model, current_loop)
  5228. else:
  5229. built_client = client
  5230. client, default_model, _ = _client_cache[cache_key]
  5231. # Race loser was never exposed to a caller — safe to close now.
  5232. _close_cached_client(built_client, close_async=async_mode)
  5233. return client, _compat_model(client, model, default_model)
  5234. # MoA virtual provider: an *explicit* `provider: moa` override (either the caller-passed `provider` arg or
  5235. # `auxiliary.<task>.provider` in config.yaml) reaches this function directly — it never goes through
  5236. # _resolve_auto_route(), which only unwraps the *implicit* "main provider is moa" case (#53827). Left as-is, "moa"
  5237. # is returned verbatim and resolve_provider_client() looks it up in PROVIDER_REGISTRY (which has no "moa"
  5238. # entry — it's not a real HTTP provider), falls to the unknown-provider dead end, and call_llm surfaces a
  5239. # nonsensical "MOA_API_KEY environment variable" error for a provider that was never meant to be reached
  5240. # over the wire. Auxiliary tasks don't need the reference fan-out — resolve to the preset's aggregator slot
  5241. # instead, exactly like the implicit path does (shared helper: _resolve_moa_aggregator).
  5242. def _unwrap_moa_provider(prov: str, mdl: Optional[str]) -> tuple[str, Optional[str]]:
  5243. """Resolve an *explicit* ``provider: moa`` to its preset's aggregator slot (_resolve_auto_route()
  5244. only unwraps the implicit case; "moa" isn't in PROVIDER_REGISTRY and would dead-end)."""
  5245. if prov.strip().lower() != "moa":
  5246. return prov, mdl
  5247. agg_provider, agg_model = _resolve_moa_aggregator(mdl)
  5248. if agg_provider and agg_model:
  5249. return agg_provider, agg_model
  5250. return prov, mdl
  5251. def _preserve_provider_with_base_url(prov: Optional[str]) -> bool:
  5252. """True when a first-class provider keeps its identity alongside an explicit base_url."""
  5253. normalized = str(prov or "").strip().lower()
  5254. if normalized in {"", "auto", "custom"} or normalized.startswith("custom:"):
  5255. return False
  5256. if normalized in _LOCAL_SERVER_ALIASES:
  5257. return True # the custom branch applies the /v1 tail only when it still sees the alias
  5258. # #76602 — two independent lookups, each guarded by its own try/except so a partial
  5259. # catalog-load failure in either path doesn't suppress the other. A user-defined
  5260. # ``providers:`` entry keeps its name alongside an explicit base_url so the named-custom
  5261. # branch resolves the entry's key/transport instead of the anonymous ``custom`` downgrade
  5262. # (which sends ``no-key-required`` and 401s on auth-required endpoints).
  5263. if _builtin_provider_present(normalized):
  5264. return True
  5265. if _named_custom_provider_present(normalized):
  5266. return True
  5267. return False
  5268. def _builtin_provider_present(name: str) -> bool:
  5269. """*name* is a built-in provider. Never raises, so a failing catalog cannot suppress the parallel
  5270. user-defined lookup in ``_preserve_provider_with_base_url`` (#76602)."""
  5271. try:
  5272. from hermes_cli.providers import get_provider
  5273. return get_provider(name) is not None
  5274. except Exception:
  5275. # Catalog unavailable (early import/test paths): still keep the high-risk provider-backed routes.
  5276. return (name in {"anthropic", "minimax-oauth", "nous", "openai-codex", "qwen-oauth", "xai-oauth"}
  5277. or _profile_credentials_bypass_custom(name))
  5278. def _profile_credentials_bypass_custom(name: str) -> bool:
  5279. """A profile (canonical name) whose credentials ``custom`` cannot carry: ambient, or an external process's."""
  5280. try:
  5281. from providers import get_provider_profile
  5282. profile = get_provider_profile(name)
  5283. except Exception:
  5284. logger.debug("provider profile lookup for %s failed", name, exc_info=True)
  5285. return False
  5286. return profile is not None and profile.name == name and (
  5287. profile.ambient_credentials or profile.auth_type == "external_process")
  5288. def _named_custom_provider_present(name: str) -> bool:
  5289. """Look up *name* in the user-defined ``providers:`` section of
  5290. config.yaml, returning False when the config is unavailable or
  5291. fails to load.
  5292. Used by ``_preserve_provider_with_base_url`` so a user-defined
  5293. provider remains preserved even when the built-in registry raises
  5294. (parallel lookup; each side fails independently — #76602).
  5295. """
  5296. try:
  5297. from hermes_cli.runtime_provider import _get_named_custom_provider
  5298. return _get_named_custom_provider(name) is not None
  5299. except Exception:
  5300. # Config not loaded yet (early import paths, tests) — fail closed:
  5301. # never widen True just because the import / load failed.
  5302. return False
  5303. def _resolve_task_provider_model(
  5304. task: str | None = None, provider: str | None = None, model: str | None = None, base_url: Optional[str] = None,
  5305. api_key: Optional[str] = None,
  5306. ) -> tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]]:
  5307. """Determine (provider, model, base_url, api_key, api_mode) for a call.
  5308. Priority: explicit args > config auxiliary.{task}.* > "auto". A bare base_url means custom,
  5309. but a first-class provider + base_url keeps the provider identity so its auth/transport
  5310. shaping still applies. api_mode is "chat_completions", "codex_responses", or None (auto).
  5311. """
  5312. cfg_provider = cfg_model = cfg_base_url = cfg_api_key = resolved_api_mode = None
  5313. if task:
  5314. task_config = _get_auxiliary_task_config(task)
  5315. cfg_provider = str(task_config.get("provider", "")).strip() or None
  5316. cfg_model = str(task_config.get("model", "")).strip() or None
  5317. cfg_base_url = str(task_config.get("base_url", "")).strip() or None
  5318. cfg_api_key = str(task_config.get("api_key", "")).strip() or None
  5319. if not cfg_api_key: # key_env → env var when api_key is not set directly
  5320. cfg_key_env = str(task_config.get("key_env") or task_config.get("api_key_env") or "").strip()
  5321. if cfg_key_env:
  5322. cfg_api_key = _scoped_key_env(cfg_key_env) or None
  5323. # User-facing spellings (``responses``, ``anthropic``, …) canonicalize here so every
  5324. # branch downstream compares against the transport names only (#39750).
  5325. resolved_api_mode = _canonical_api_mode(str(task_config.get("api_mode") or "")).lower() or None
  5326. # 'auto' is a sentinel ("inherit / auto-detect"), not a model id — leaking it to the wire
  5327. # yields a 200 with an error-text body that consumers accept as output. The explicit `model`
  5328. # kwarg needs the same normalization: MoA slots forward preset `model:` fields through it.
  5329. if model and model.lower() == "auto":
  5330. model = None
  5331. if cfg_model and cfg_model.lower() == "auto":
  5332. cfg_model = None
  5333. resolved_model = model or cfg_model
  5334. # Any moa:// facade endpoint belongs to the facade, not the aggregator's real provider —
  5335. # drop it (mirrors _resolve_auto_route()).
  5336. if provider and str(provider).strip().lower() == "moa":
  5337. provider, resolved_model = _unwrap_moa_provider(provider, resolved_model)
  5338. if provider and provider.lower() != "moa":
  5339. base_url = None
  5340. api_key = None
  5341. elif cfg_provider and str(cfg_provider).strip().lower() == "moa":
  5342. cfg_provider, cfg_model = _unwrap_moa_provider(cfg_provider, resolved_model)
  5343. if cfg_provider and cfg_provider.lower() != "moa":
  5344. resolved_model = cfg_model
  5345. cfg_base_url = None
  5346. cfg_api_key = None
  5347. # One shared alias table with resolve_runtime_provider(): ``provider: openai`` routes the same
  5348. # way here (compression/vision/title) and on the runtime path (background review, curator, MoA).
  5349. from hermes_cli.runtime_provider_custom import expand_direct_api_alias
  5350. if provider:
  5351. provider, base_url = expand_direct_api_alias(provider, base_url)
  5352. if cfg_provider:
  5353. cfg_provider, cfg_base_url = expand_direct_api_alias(cfg_provider, cfg_base_url)
  5354. # An explicit provider without base_url adopts the task's configured endpoint (same or
  5355. # unnamed provider) so the early return below carries it. Explicit "auto" is excluded — it
  5356. # must keep flowing through auto-resolution.
  5357. # See #58515.
  5358. if provider and provider != "auto" and not base_url and cfg_base_url and cfg_provider in (None, provider):
  5359. base_url = cfg_base_url
  5360. if not api_key:
  5361. api_key = cfg_api_key
  5362. if base_url:
  5363. kept = provider if _preserve_provider_with_base_url(provider) else "custom"
  5364. return kept, resolved_model, base_url, api_key, resolved_api_mode
  5365. if provider:
  5366. return provider, resolved_model, base_url, api_key, resolved_api_mode
  5367. if cfg_base_url and cfg_api_key:
  5368. kept = cfg_provider if str(cfg_provider or "").strip().lower() in _LOCAL_SERVER_ALIASES else "custom"
  5369. return kept, resolved_model, cfg_base_url, cfg_api_key, resolved_api_mode
  5370. if cfg_base_url and cfg_provider and cfg_provider != "auto":
  5371. # base_url without api_key: keep the provider so it can resolve credentials from env
  5372. # vars instead of locking into "custom".
  5373. return cfg_provider, resolved_model, cfg_base_url, None, resolved_api_mode
  5374. if cfg_provider and cfg_provider != "auto":
  5375. return cfg_provider, resolved_model, cfg_base_url, cfg_api_key, resolved_api_mode
  5376. return "auto", resolved_model, None, None, resolved_api_mode
  5377. _DEFAULT_AUX_TIMEOUT = 30.0
  5378. # Reasoning compression models can exceed the default 120 s config timeout, falling back to the
  5379. # deterministic marker. Bounded *floor* for config-derived compression timeouts only; never
  5380. # overrides an explicit per-call timeout.
  5381. # Compression summarises large conversation histories; a reasoning auxiliary model (e.g. Codex / GPT-5.5)
  5382. # can legitimately take longer than the default ``auxiliary.compression.timeout`` (120 s), causing the
  5383. # stream to time out and the compressor to fall back to the deterministic context marker (#54915). A floor
  5384. # is harmless for fast compression models (they finish before the deadline) and is a minimum, so a higher
  5385. # config value is kept unchanged.
  5386. _COMPRESSION_TIMEOUT_FLOOR_SECONDS = 300.0
  5387. # Read-time resolution of auxiliary.<task> (plugin defaults, inherit_from) lives in its own module;
  5388. # re-exported here because callers and tests reach it as agent.auxiliary_client._get_auxiliary_task_config.
  5389. from agent.auxiliary_task_config import _get_auxiliary_task_config
  5390. class CompressionFastLane(NamedTuple):
  5391. """Explicit, non-reasoning compression route."""
  5392. certified_non_reasoning: bool
  5393. reasoning_config: Optional[dict[str, Any]]
  5394. def _fast_lane_config_fields(config: dict[str, Any]) -> tuple[str, str, bool]:
  5395. """Only explicit reasoning disablement certifies a non-reasoning route."""
  5396. from hermes_constants import parse_reasoning_effort
  5397. provider = str(config.get("provider") or "").strip().lower()
  5398. model = str(config.get("model") or "").strip()
  5399. parsed_effort = parse_reasoning_effort(config.get("reasoning_effort"))
  5400. non_reasoning = parsed_effort is not None and parsed_effort.get("enabled") is False
  5401. return provider, model, non_reasoning
  5402. def resolve_compression_fast_lane(
  5403. actual_provider: str, actual_model: Optional[str], *, requested_provider: Optional[str] = None,
  5404. requested_model: Optional[str] = None, route_config: Optional[dict[str, Any]] = None,
  5405. ) -> CompressionFastLane:
  5406. """Certify explicit non-reasoning settings only on the matching destination."""
  5407. config = route_config if route_config is not None else _get_auxiliary_task_config("compression")
  5408. cfg_provider, cfg_model, non_reasoning = _fast_lane_config_fields(config)
  5409. provider = str(requested_provider or "").strip().lower() or cfg_provider
  5410. model = str(requested_model or "").strip() or cfg_model
  5411. explicit_route = provider not in {"", "auto"} and model.lower() not in {"", "auto"}
  5412. actual_norm = _normalize_aux_provider(_fallback_provider_from_label(str(actual_provider or "")))
  5413. provider_matches = actual_norm == _normalize_aux_provider(provider)
  5414. model_matches = str(actual_model or "").strip().lower() == model.lower()
  5415. if explicit_route and provider_matches and model_matches and non_reasoning:
  5416. return CompressionFastLane(True, {"enabled": False, "effort": "none"})
  5417. return CompressionFastLane(False, None)
  5418. def _compression_config_claims_fast_lane(config: dict[str, Any]) -> bool:
  5419. """Whether task config declares fast-only controls that cannot leak."""
  5420. provider, model, non_reasoning = _fast_lane_config_fields(config)
  5421. return provider not in {"", "auto"} and model.lower() not in {"", "auto"} and non_reasoning
  5422. def _compression_fast_lane_controls(
  5423. task: str | None, *, actual_provider: str, actual_model: str | None,
  5424. requested_provider: str | None, requested_model: str | None, route_config: dict[str, Any],
  5425. leak_guard_config: dict[str, Any], max_tokens: int | None, extra_body: dict[str, Any],
  5426. ) -> tuple[int | None, dict[str, Any]]:
  5427. """Apply the certified compression controls to one resolved route."""
  5428. if task != "compression" or max_tokens is not None:
  5429. return max_tokens, extra_body
  5430. body = dict(extra_body)
  5431. lane = resolve_compression_fast_lane(
  5432. actual_provider, actual_model, requested_provider=requested_provider, requested_model=requested_model, route_config=route_config,
  5433. )
  5434. if lane.reasoning_config is not None:
  5435. if "reasoning" not in body:
  5436. body["reasoning"] = lane.reasoning_config
  5437. elif _compression_config_claims_fast_lane(leak_guard_config):
  5438. body.pop("reasoning", None)
  5439. return max_tokens, body
  5440. def _get_task_no_progress_timeout(task: str) -> Optional[float]:
  5441. """``auxiliary.<task>.no_progress_timeout`` from config, or None when unset/invalid
  5442. (the Codex and chat-stream watchdogs then keep the built-in ``_AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS``
  5443. default). Lets an operator widen the substantive-progress window independently of the
  5444. overall request timeout — see #108104."""
  5445. if not task:
  5446. return None
  5447. raw = _get_auxiliary_task_config(task).get("no_progress_timeout")
  5448. if raw is None:
  5449. return None
  5450. try:
  5451. value = float(raw)
  5452. except (ValueError, TypeError):
  5453. value = 0.0
  5454. if isinstance(raw, bool) or value <= 0:
  5455. # Fail clearly: a typo here silently leaving the 60s default is exactly the
  5456. # "why did my 600s request abort after 60s" confusion the key exists to remove.
  5457. logger.warning(
  5458. "auxiliary.%s.no_progress_timeout=%r is not a positive number of seconds; "
  5459. "using the built-in %.0fs default", task, raw, _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS)
  5460. return None
  5461. return value
  5462. def _get_task_timeout(task: str, default: float = _DEFAULT_AUX_TIMEOUT) -> float:
  5463. """``auxiliary.<task>.timeout`` from config, else *default*."""
  5464. if not task:
  5465. return default
  5466. raw = _get_auxiliary_task_config(task).get("timeout")
  5467. if raw is not None:
  5468. with contextlib.suppress(ValueError, TypeError):
  5469. return float(raw)
  5470. return default
  5471. def _effective_aux_timeout(task: str, timeout: Optional[float]) -> float:
  5472. """Explicit ``timeout`` wins, else config; compression gets a floor so a reasoning model
  5473. summarising a large context isn't cut off."""
  5474. if timeout is not None:
  5475. return timeout
  5476. effective = _get_task_timeout(task)
  5477. return max(effective, _COMPRESSION_TIMEOUT_FLOOR_SECONDS) if task == "compression" else effective
  5478. def _with_custom_endpoint_extra_body(
  5479. extra_body: Optional[dict], provider: str, model: Optional[str], base_url: Optional[str],
  5480. ) -> Optional[dict]:
  5481. """Layer the destination custom provider's ``extra_body`` UNDER the task/caller body.
  5482. The main agent merges a ``custom_providers`` / ``providers:`` entry's ``extra_body`` into every
  5483. request to that endpoint (``agent_init._merge_custom_provider_extra_body``); an aux request routed
  5484. to the same entry must carry it too, or a proxy that 400s without e.g. a ``user`` field breaks
  5485. smart approval, titles and compression (#103738). Resolved per destination with the agent's own
  5486. matcher, so a fallback to another provider never inherits it; ``auxiliary.<task>.extra_body`` and
  5487. caller keys win on conflict, as request_overrides win over the entry on the main path."""
  5488. if not base_url:
  5489. return extra_body
  5490. try:
  5491. from agent.agent_init import _custom_provider_extra_body_for_agent
  5492. from hermes_cli.config import get_compatible_custom_providers, load_config_readonly
  5493. inherited = _custom_provider_extra_body_for_agent(
  5494. provider=provider or "", model=model or "", base_url=str(base_url),
  5495. custom_providers=get_compatible_custom_providers(load_config_readonly()),
  5496. )
  5497. except Exception:
  5498. logger.debug("custom provider extra_body lookup failed for aux request", exc_info=True)
  5499. return extra_body
  5500. if not inherited:
  5501. return extra_body
  5502. return {**inherited, **(extra_body or {})}
  5503. def _get_task_extra_body(task: str) -> dict[str, Any]:
  5504. """Shallow copy of ``auxiliary.<task>.extra_body`` with ``reasoning_effort`` folded into
  5505. ``reasoning`` unless one is configured (more specific wins). MoA tasks are excluded: their
  5506. reasoning depth is per-slot in the preset."""
  5507. task_config = _get_auxiliary_task_config(task)
  5508. raw = task_config.get("extra_body")
  5509. result = dict(raw) if isinstance(raw, dict) else {}
  5510. if "reasoning" in result:
  5511. return result
  5512. effort = task_config.get("reasoning_effort")
  5513. if effort is None or effort == "":
  5514. return result
  5515. if task in ("moa_reference", "moa_aggregator"):
  5516. logger.warning(
  5517. "auxiliary.%s.reasoning_effort is not supported — MoA reasoning depth is per-slot: set reasoning_effort "
  5518. "on the preset's reference_models entries / aggregator instead (moa.presets.<name>...). Ignoring.",
  5519. task,
  5520. )
  5521. return result
  5522. from hermes_constants import parse_reasoning_effort
  5523. parsed = parse_reasoning_effort(effort)
  5524. if parsed is not None:
  5525. result["reasoning"] = parsed
  5526. else:
  5527. logger.warning(
  5528. "auxiliary.%s.reasoning_effort %r is not a valid level (none, minimal, low, medium, high, xhigh, max, ultra) — ignoring",
  5529. task, effort,
  5530. )
  5531. return result
  5532. # Per-task concurrency limiting: many sessions can spawn background aux calls, and during provider incidents
  5533. # each retries / fans out across the fallback chain, multiplying request volume on already-degraded endpoints.
  5534. # A per-task semaphore caps in-flight calls so retry amplification stays bounded. See #23324.
  5535. # Keyed by profile home as well: the limit is the profile's ``auxiliary.<task>.max_concurrency``, and two
  5536. # multiplexed profiles with different limits would otherwise rebuild (and reset) one shared semaphore.
  5537. _aux_sync_semaphores: dict[tuple[str, str], tuple[int, threading.BoundedSemaphore]] = {}
  5538. _aux_async_semaphores: dict[Any, dict] = {} # loop -> {(home, task): (limit, sem)} until closed: cache_bounds.live_loop_table
  5539. _aux_sem_lock = threading.Lock()
  5540. def _get_task_max_concurrency(task: Optional[str]) -> Optional[int]:
  5541. """``auxiliary.<task>.max_concurrency`` as a positive int, or None. Vision uses this key for
  5542. its encode/resize CPU pool; its LLM calls stay concurrent."""
  5543. if not task or task == "vision":
  5544. return None
  5545. try:
  5546. value = int(_get_auxiliary_task_config(task).get("max_concurrency"))
  5547. except (TypeError, ValueError): # missing (None) or malformed
  5548. return None
  5549. return value if value > 0 else None
  5550. def _cached_semaphore(store: dict, key: Any, limit: int, factory: Callable[[int], Any]) -> Any:
  5551. """Return the cached semaphore for ``key``, rebuilding it when the limit changed."""
  5552. with _aux_sem_lock:
  5553. entry = store.get(key)
  5554. if entry is None or entry[0] != limit:
  5555. store[key] = entry = (limit, factory(limit))
  5556. return entry[1]
  5557. def _acquire_sync_aux_semaphore(task: Optional[str]) -> Optional[threading.BoundedSemaphore]:
  5558. """Get a per-task sync semaphore, rebuilding it after a config change."""
  5559. limit = _get_task_max_concurrency(task)
  5560. if limit is None:
  5561. return None
  5562. from hermes_constants import hermes_home_key
  5563. return _cached_semaphore(_aux_sync_semaphores, (hermes_home_key(), task), limit, threading.BoundedSemaphore)
  5564. def _acquire_async_aux_semaphore(task: Optional[str]):
  5565. """Get a per-task, per-event-loop async semaphore after config lookup."""
  5566. limit = _get_task_max_concurrency(task)
  5567. if limit is None:
  5568. return None
  5569. import asyncio
  5570. try:
  5571. loop = asyncio.get_running_loop()
  5572. except RuntimeError:
  5573. return None
  5574. from agent.cache_bounds import live_loop_table
  5575. from hermes_constants import hermes_home_key
  5576. tables = live_loop_table(_aux_async_semaphores, loop, _aux_sem_lock)
  5577. return _cached_semaphore(tables, (hermes_home_key(), task), limit, asyncio.Semaphore)
  5578. def _reset_aux_semaphores() -> None:
  5579. """Drop cached semaphores (test helper)."""
  5580. with _aux_sem_lock:
  5581. _aux_sync_semaphores.clear()
  5582. _aux_async_semaphores.clear()
  5583. # Anthropic-compatible endpoints reached via the OpenAI SDK wrapper; their image content blocks
  5584. # must use Anthropic format.
  5585. _ANTHROPIC_COMPAT_PROVIDERS = frozenset({"minimax", "minimax-oauth", "minimax-cn"})
  5586. def _is_anthropic_compat_endpoint(provider: str, base_url: str) -> bool:
  5587. """True for known Anthropic-compatible providers or any ``/anthropic`` URL path."""
  5588. return provider in _ANTHROPIC_COMPAT_PROVIDERS or "/anthropic" in (base_url or "").lower()
  5589. # OpenAI block type → (Anthropic block type, default media type for data: URLs). MiniMax's
  5590. # Anthropic-compatible endpoint wants type="video" (not "video_url"/"input_video") with the same
  5591. # ``source`` shape as "image".
  5592. _ANTHROPIC_MEDIA_BLOCKS = {"image_url": ("image", "image/png"), "video_url": ("video", "video/mp4")}
  5593. def _convert_openai_images_to_anthropic(messages: list) -> list:
  5594. """Convert OpenAI ``image_url``/``video_url`` blocks to Anthropic ``image``/``video``;
  5595. only list-content messages with such blocks change."""
  5596. converted = []
  5597. for msg in messages:
  5598. content = msg.get("content")
  5599. if not isinstance(content, list):
  5600. converted.append(msg)
  5601. continue
  5602. new_content = []
  5603. changed = False
  5604. for block in content:
  5605. block_type = block.get("type")
  5606. if block_type not in _ANTHROPIC_MEDIA_BLOCKS:
  5607. new_content.append(block)
  5608. continue
  5609. url = (block.get(block_type) or {}).get("url", "")
  5610. anth_type, media_type = _ANTHROPIC_MEDIA_BLOCKS[block_type]
  5611. if url.startswith("data:"):
  5612. header, _, b64data = url.partition(",")
  5613. if ":" in header and ";" in header:
  5614. media_type = header.split(":", 1)[1].split(";", 1)[0]
  5615. source = {"type": "base64", "media_type": media_type, "data": b64data}
  5616. else:
  5617. source = {"type": "url", "url": url}
  5618. new_content.append({"type": anth_type, "source": source})
  5619. changed = True
  5620. converted.append({**msg, "content": new_content} if changed else msg)
  5621. return converted
  5622. _PROFILE_REASONING_KEYS = {
  5623. "reasoning", "reasoning_effort", "thinking", "thinking_config", "thinkingconfig",
  5624. "thinking_budget", "thinkingbudget", "enable_thinking", "think", "verbosity",
  5625. }
  5626. def _contains_profile_reasoning_fields(value: Any) -> bool:
  5627. """Return whether a profile payload contains a reasoning wire control (recursive)."""
  5628. if not isinstance(value, dict):
  5629. return False
  5630. return any(
  5631. str(key).strip().lower() in _PROFILE_REASONING_KEYS or _contains_profile_reasoning_fields(nested)
  5632. for key, nested in value.items()
  5633. )
  5634. _NOUS_PROVIDER_NAMES = frozenset({"nous", "nous-portal", "nousresearch"})
  5635. def _nous_on_messages_wire(provider_norm: str, model: str) -> bool:
  5636. """True when a Nous Portal route serves ``model`` over /v1/messages (dual-wire catalog)."""
  5637. if provider_norm not in _NOUS_PROVIDER_NAMES:
  5638. return False
  5639. from hermes_cli.providers import nous_api_mode
  5640. return nous_api_mode(model) == "anthropic_messages"
  5641. _NVIDIA_PROVIDER_NAMES = {"nvidia", "nvidia-nim", "nim", "build-nvidia", "nemotron"}
  5642. _GEMINI_NATIVE_PROVIDER_NAMES = {"gemini", "google", "google-gemini", "google-ai-studio"}
  5643. def _is_gemini_native_route(provider_norm: str, effective_base: str) -> bool:
  5644. """Gemini native by provider name, else (best-effort) by base URL shape."""
  5645. if provider_norm in _GEMINI_NATIVE_PROVIDER_NAMES:
  5646. return True
  5647. if not effective_base:
  5648. return False
  5649. try:
  5650. from agent.gemini_native_adapter import is_native_gemini_base_url
  5651. return is_native_gemini_base_url(effective_base)
  5652. except Exception:
  5653. return False
  5654. def _forwards_max_tokens(provider: str, provider_norm: str, model: str, effective_base: str, task: Optional[str]) -> bool:
  5655. """Whether an explicit max_tokens is forwarded on this route.
  5656. No default cap elsewhere (omitted = provider default; avoids max_completion_tokens / ZAI-vision
  5657. quirks). Forward only where mandatory or honored: Anthropic Messages wire (400 without it);
  5658. NVIDIA NIM (empty choices[] when omitted); MoA reference slots; Gemini native (fixed 65,535
  5659. ceiling otherwise); OpenRouter (budgets the FULL window when omitted → 402 on low credit);
  5660. managed local llama-server (uncapped decode with no EOS burns the GPU to the context window).
  5661. """
  5662. return (
  5663. _is_anthropic_compat_endpoint(provider, effective_base)
  5664. or _nous_on_messages_wire(provider_norm, model)
  5665. or provider_norm in _NVIDIA_PROVIDER_NAMES
  5666. or base_url_host_matches(effective_base, "integrate.api.nvidia.com")
  5667. or str(task) == "moa_reference"
  5668. or _is_gemini_native_route(provider_norm, effective_base)
  5669. or provider_norm == "openrouter"
  5670. or base_url_host_matches(effective_base, "openrouter.ai")
  5671. or _is_managed_local_endpoint(effective_base)
  5672. )
  5673. def _dedupe_tool_names(tools: list, provider: str, model: str) -> list:
  5674. """Drop duplicate tool names (Vertex/Azure/Bedrock 400 on them) with a warning."""
  5675. seen: set = set()
  5676. deduped: list = []
  5677. for tool in tools:
  5678. name = (tool.get("function") or {}).get("name", "")
  5679. if name and name in seen:
  5680. logger.warning("_build_call_kwargs: duplicate tool name '%s' removed (provider=%s model=%s)", name, provider, model)
  5681. continue
  5682. if name:
  5683. seen.add(name)
  5684. deduped.append(tool)
  5685. return deduped
  5686. class _ProfileProjection(NamedTuple):
  5687. body: dict[str, Any]
  5688. reasoning_extra: dict[str, Any]
  5689. top_level: dict[str, Any]
  5690. handles_reasoning: bool
  5691. messages_wire: bool = False
  5692. def _routes_to_custom_endpoint(provider_norm: str) -> bool:
  5693. """True when a profile-less provider name is really a configured OpenAI-compatible custom endpoint.
  5694. Keyed ``providers:`` / ``custom_providers`` entries (by bare key, or ``main``/``auto`` resolving to
  5695. one). An unconfigured name with only an explicit base_url keeps the generic nested fallback: the
  5696. operator declared nothing about that endpoint's wire, and the fireworks control contract pins it.
  5697. """
  5698. name = _normalize_aux_provider(provider_norm)
  5699. if name == "custom":
  5700. return True
  5701. from hermes_cli.runtime_provider import _get_named_custom_provider
  5702. return _get_named_custom_provider(name) is not None
  5703. def _project_provider_profile(
  5704. provider: str, provider_norm: str, model: str, effective_base: str, reasoning_config: Optional[dict], data_use: frozenset[str],
  5705. ) -> _ProfileProjection:
  5706. """Provider profile's extra_body / kwargs projection; partial on failure."""
  5707. body: dict[str, Any] = {}
  5708. reasoning_extra: dict[str, Any] = {}
  5709. top_level: dict[str, Any] = {}
  5710. handles_reasoning = False
  5711. messages_wire = False
  5712. try:
  5713. from providers import get_provider_profile
  5714. from providers.base import ProviderProfile
  5715. profile = get_provider_profile(provider_norm)
  5716. if profile is None and _routes_to_custom_endpoint(provider_norm):
  5717. # A keyed ``providers:`` entry referenced by its bare key (or via ``main``/``auto``) is
  5718. # the same OpenAI-compatible custom endpoint the main path already projects with the
  5719. # ``custom`` profile (``custom:<key>`` falls back inside get_provider_profile). Without
  5720. # it the generic nested ``extra_body.reasoning`` fallback below ships to a strict
  5721. # gateway that only accepts top-level ``reasoning_effort`` -> 400 (#75089).
  5722. profile = get_provider_profile("custom")
  5723. if profile is not None:
  5724. messages_wire = profile.api_mode == "anthropic_messages"
  5725. body = profile.build_extra_body(model=model, base_url=effective_base, reasoning_config=reasoning_config, data_use=data_use) or {}
  5726. reasoning_extra, top_level = profile.build_api_kwargs_extras(
  5727. reasoning_config=reasoning_config, supports_reasoning=reasoning_config is not None,
  5728. model=model, base_url=effective_base,
  5729. )
  5730. reasoning_extra = reasoning_extra or {}
  5731. top_level = top_level or {}
  5732. handles_reasoning = (
  5733. type(profile).build_api_kwargs_extras is not ProviderProfile.build_api_kwargs_extras
  5734. or _contains_profile_reasoning_fields(body)
  5735. or _contains_profile_reasoning_fields(reasoning_extra)
  5736. or _contains_profile_reasoning_fields(top_level)
  5737. )
  5738. except Exception as exc:
  5739. logger.debug("_build_call_kwargs: provider profile projection failed for %s: %s", provider, exc)
  5740. return _ProfileProjection(body, reasoning_extra, top_level, handles_reasoning, messages_wire)
  5741. def _merge_aux_extra_body(
  5742. extra_body: Optional[dict], projection: _ProfileProjection, reasoning_config: Optional[dict], provider_norm: str,
  5743. ) -> dict[str, Any]:
  5744. """Caller extra_body + profile body/reasoning + generic reasoning fallback + Nous tags."""
  5745. merged_extra = dict(extra_body or {})
  5746. caller_reasoning_fields = {
  5747. key: value for key, value in merged_extra.items()
  5748. if str(key).strip().lower() in _PROFILE_REASONING_KEYS and str(key).strip().lower() != "reasoning"
  5749. }
  5750. caller_disabled = isinstance(reasoning_config, dict) and reasoning_config.get("enabled") is False
  5751. if caller_disabled:
  5752. # The caller's thinking-off beats ``auxiliary.<task>.reasoning_effort`` (folded into
  5753. # ``extra_body.reasoning`` by _get_task_extra_body). Dropped BEFORE the profile merge so a
  5754. # profile that projects disabled reasoning onto its own wire (custom: top-level
  5755. # ``reasoning_effort=none``) never ships beside a task-level ``reasoning.effort`` — strict
  5756. # gateways 400 on the contradiction (#114020) — while a profile whose disabled shape IS
  5757. # ``extra_body.reasoning`` (OpenRouter) still lands it below.
  5758. merged_extra.pop("reasoning", None)
  5759. merged_extra.update(projection.body)
  5760. merged_extra.update(projection.reasoning_extra)
  5761. # Profiles supply route defaults, but an explicit vendor wire control in the task/call config
  5762. # is already provider-specific and must not be replaced by that default.
  5763. merged_extra.update(caller_reasoning_fields)
  5764. if not projection.handles_reasoning:
  5765. # ``reasoning_config`` is already clamped to the OpenAI-compat wire by _build_call_kwargs.
  5766. from agent.reasoning_effort import generic_nested_reasoning
  5767. merged_extra.update(generic_nested_reasoning(reasoning_config))
  5768. # Caller/task ``extra_body.reasoning`` (``auxiliary.<task>.reasoning_effort`` folds in here via
  5769. # _get_task_extra_body) takes the same wire clamp: Hermes-only ``ultra`` never reaches the
  5770. # OpenAI-compat wire from any aux task (#112010).
  5771. if isinstance(merged_extra.get("reasoning"), dict):
  5772. from agent.reasoning_effort import clamp_reasoning_config
  5773. merged_extra["reasoning"] = clamp_reasoning_config(merged_extra["reasoning"])
  5774. # Portal tags + sticky session_id fallback when the profile didn't supply them; session_id
  5775. # keeps aux calls on the main turn's upstream instance (cache warmth) — tags alone are not
  5776. # enough on /v1/messages.
  5777. if provider_norm in _NOUS_PROVIDER_NAMES:
  5778. if "tags" not in merged_extra:
  5779. merged_extra["tags"] = _nous_portal_tags()
  5780. if "session_id" not in merged_extra:
  5781. try:
  5782. from agent.portal_tags import get_conversation_context
  5783. sticky_key = get_conversation_context()
  5784. except Exception:
  5785. sticky_key = None
  5786. if sticky_key:
  5787. merged_extra["session_id"] = sticky_key
  5788. return merged_extra
  5789. def _build_call_kwargs(
  5790. provider: str, model: str, messages: list, temperature: Optional[float] = None,
  5791. max_tokens: Optional[int] = None, tools: Optional[list] = None, timeout: float = 30.0,
  5792. extra_body: Optional[dict] = None, reasoning_config: Optional[dict] = None,
  5793. base_url: Optional[str] = None, task: Optional[str] = None,
  5794. no_progress_timeout: Optional[float] = None,
  5795. ) -> dict:
  5796. """Build kwargs for .chat.completions.create() with model/provider adjustments.
  5797. ``no_progress_timeout`` is a Codex-Responses-only extra (consumed by
  5798. ``_CodexCompletionsAdapter.create``'s ``**kwargs`` catch-all); callers must only pass it
  5799. when the resolved client is a ``CodexAuxiliaryClient`` — real SDK clients don't accept it."""
  5800. kwargs: dict[str, Any] = {"model": model, "messages": messages, "timeout": timeout}
  5801. if no_progress_timeout is not None:
  5802. kwargs["no_progress_timeout"] = no_progress_timeout
  5803. effective_base = base_url or (_current_custom_base_url() if provider == "custom" else "")
  5804. extra_body = _with_custom_endpoint_extra_body(extra_body, provider, model, effective_base)
  5805. # Per-model fixed/omitted temperature, then Opus 4.7+ sampling bans: it rejects any
  5806. # non-default temperature/top_p/top_k, so drop silently rather than 400 when the aux model flips.
  5807. fixed_temperature = _fixed_temperature_for_model(model, effective_base, provider)
  5808. if fixed_temperature is OMIT_TEMPERATURE:
  5809. temperature = None # strip — let server choose
  5810. elif fixed_temperature is not None:
  5811. temperature = fixed_temperature
  5812. if temperature is not None:
  5813. from agent.anthropic_adapter import _forbids_sampling_params
  5814. if not _forbids_sampling_params(model):
  5815. kwargs["temperature"] = temperature
  5816. provider_norm = str(provider or "").strip().lower()
  5817. if max_tokens is not None and _forwards_max_tokens(provider, provider_norm, model, effective_base, task):
  5818. kwargs.update(auxiliary_max_tokens_param(max_tokens, model=model)) # picks max_completion_tokens where needed
  5819. if tools:
  5820. kwargs["tools"] = _dedupe_tool_names(tools, provider, model)
  5821. # Provider profiles are the source of truth for reasoning wire shapes (top-level, nested body,
  5822. # or extra_body.reasoning); providers without a reasoning-aware profile keep the generic
  5823. # ``extra_body.reasoning`` fallback. Clamp Hermes-internal levels (``ultra``) to the
  5824. # OpenAI-compat wire ONCE here, before either path sees the config — the same entry clamp the
  5825. # main transport applies (#89503); MoA aggregator/reference and aux calls 400'd without it (#112010).
  5826. from agent.reasoning_effort import clamp_reasoning_config
  5827. from agent.auxiliary_reasoning_floor import known_reasoning_floor
  5828. if isinstance(extra_body, dict):
  5829. task_reasoning = extra_body.get("reasoning")
  5830. if isinstance(task_reasoning, dict) and "enabled" in task_reasoning:
  5831. extra_body = dict(extra_body)
  5832. extra_body.pop("reasoning")
  5833. if reasoning_config is None:
  5834. reasoning_config = task_reasoning
  5835. reasoning_config = clamp_reasoning_config(known_reasoning_floor(reasoning_config, provider_norm, effective_base, model, task))
  5836. projection = _project_provider_profile(provider, provider_norm, model, effective_base, reasoning_config, data_use_requirement(messages))
  5837. kwargs.update(projection.top_level)
  5838. merged_extra = _merge_aux_extra_body(extra_body, projection, reasoning_config, provider_norm)
  5839. if "response_format" in merged_extra:
  5840. from agent.auxiliary_structured_output import without_unsupported_response_format
  5841. merged_extra = without_unsupported_response_format(merged_extra, provider_norm, effective_base, model, task)
  5842. if merged_extra:
  5843. kwargs["extra_body"] = merged_extra
  5844. # Anthropic Messages adapters take reasoning via a private kwarg that plain OpenAI SDK clients
  5845. # would reject; Portal Claude is dual-wire, so include it only when the catalog id selects
  5846. # /v1/messages. A profile declaring api_mode=anthropic_messages (commandcode-anthropic) is on
  5847. # that wire regardless of URL shape — once it overrides build_api_kwargs_extras the generic
  5848. # ``extra_body.reasoning`` fallback the adapter used to read is gone, so this is the adapter's
  5849. # only path. _wrap_transport wraps such providers on the same declaration.
  5850. if reasoning_config and isinstance(reasoning_config, dict):
  5851. raw_base = base_url or ""
  5852. if (
  5853. provider_norm == "anthropic" or projection.messages_wire or _nous_on_messages_wire(provider_norm, model)
  5854. or _endpoint_speaks_anthropic_messages(raw_base) or _is_anthropic_compat_endpoint(provider_norm, raw_base)
  5855. ):
  5856. kwargs["_reasoning_config"] = dict(reasoning_config)
  5857. # Conversation affinity (OpenCode relay, opt-in custom-provider header) — same key as the main
  5858. # turn so compression/title/vision calls stay on the conversation's warm backend.
  5859. from agent.opencode_affinity import merge_session_affinity_headers
  5860. return merge_session_affinity_headers(kwargs, provider, base_url, _runtime_main_value("session_id") or None)
  5861. def _validate_llm_response(
  5862. response: Any, task: Optional[str] = None, provider: Optional[str] = None, base_url: Optional[str] = None,
  5863. ) -> Any:
  5864. """Validate the .choices[0].message shape (fail fast, not a downstream AttributeError).
  5865. Also the single aux-usage accounting chokepoint: every successful non-streaming response
  5866. passes here exactly once; *provider*/*base_url* are optional hints. Async callers
  5867. await this off-loop because accounting may commit through synchronous worker RPC.
  5868. See #7264.
  5869. Recording is best-effort and never affects validation. *provider*/*base_url* are optional accounting
  5870. hints — fallback-path calls omit them and the row keeps the model (read from the response itself) with
  5871. an empty route. See #23270.
  5872. """
  5873. if response is None:
  5874. raise RuntimeError(
  5875. f"Auxiliary {task or 'call'}: LLM returned None response"
  5876. )
  5877. response = _unwrap_data_envelope(response, task)
  5878. from agent.aux_accounting import record_aux_usage
  5879. record_aux_usage(response, task, provider=provider, base_url=base_url)
  5880. # Adapter SimpleNamespace responses are fine — they have .choices[0].message.
  5881. try:
  5882. choices = response.choices
  5883. if not choices or not hasattr(choices[0], "message"):
  5884. raise AttributeError("missing choices[0].message")
  5885. except (AttributeError, TypeError, IndexError) as exc:
  5886. recovered = _recover_aux_response_message(response)
  5887. if recovered is None:
  5888. invalid = RuntimeError(
  5889. f"Auxiliary {task or 'call'}: LLM returned invalid response (type={type(response).__name__}): "
  5890. f"{str(response)[:120]!r}. Expected object with .choices[0].message — check provider "
  5891. f"adapter or custom endpoint compatibility."
  5892. )
  5893. # An HTTP-200 body carrying a provider ``error`` (aggregators relay upstream failures
  5894. # this way): keep it for the classifier without changing how the ladder routes this.
  5895. from agent import auxiliary_call_outcome
  5896. setattr(invalid, auxiliary_call_outcome.PROVIDER_ERROR_ATTR,
  5897. auxiliary_call_outcome.embedded_provider_error(response))
  5898. _note_relay_auxiliary_error(invalid)
  5899. raise invalid from exc
  5900. response = recovered
  5901. from agent.transports.chat_completions import is_router_timeout_shim
  5902. if is_router_timeout_shim(response):
  5903. # HTTP-200 router failure shim (#68396): invalid like a malformed shape so the
  5904. # auxiliary fallback chain moves to the next candidate instead of titling with it.
  5905. shim = RuntimeError(f"Auxiliary {task or 'call'}: provider returned a timeout shim instead of a completion")
  5906. _note_relay_auxiliary_error(shim)
  5907. raise shim
  5908. # Retain the provider-reported model for terminal relay route attribution.
  5909. context = _RELAY_AUX_CALL_CONTEXT.get()
  5910. if context is not None:
  5911. model = _field(response, "model")
  5912. if isinstance(model, str) and model.strip():
  5913. context["response_model"] = model
  5914. _complete_relay_auxiliary_call()
  5915. return response
  5916. def _complete_relay_auxiliary_call(*, outcome: str = "success", error_class: Optional[str] = None) -> None:
  5917. """Close one auxiliary logical call after acceptance or terminal failure. A success
  5918. reports the last attempt error it recovered from (``none`` when the first attempt held)."""
  5919. context = _RELAY_AUX_CALL_CONTEXT.get()
  5920. if context is None:
  5921. return
  5922. from agent import relay_llm
  5923. relay_llm.complete_logical_call(
  5924. str(context.get("request_id") or ""), outcome=outcome,
  5925. model_name=str(context.get("model") or "unknown"),
  5926. provider_name=str(context.get("provider") or "auxiliary"),
  5927. response_model_name=context.get("response_model"),
  5928. error_class=error_class or context.get("error_class") or "none",
  5929. )
  5930. def _note_relay_auxiliary_error(exc: BaseException) -> None:
  5931. """Remember one failed attempt's classified reason on the logical call."""
  5932. context = _RELAY_AUX_CALL_CONTEXT.get()
  5933. if context is not None and isinstance(exc, Exception):
  5934. from agent import auxiliary_call_outcome
  5935. context["error_class"] = auxiliary_call_outcome.error_class(
  5936. exc, provider=str(context.get("provider") or ""), model=str(context.get("model") or ""))
  5937. def _fail_relay_auxiliary_call(exc: BaseException) -> None:
  5938. """Close a call that ended in ``exc`` without replacing it: a Hermes abort is ``cancelled``,
  5939. anything else ``failed`` with the classifier's reason for the error that ended it."""
  5940. try:
  5941. from agent import auxiliary_call_outcome
  5942. if auxiliary_call_outcome.is_cancellation(exc):
  5943. _complete_relay_auxiliary_call(outcome="cancelled", error_class="none")
  5944. return
  5945. _note_relay_auxiliary_error(exc)
  5946. _complete_relay_auxiliary_call(outcome="failed")
  5947. except Exception:
  5948. logger.warning("Relay auxiliary failure finalization failed", exc_info=True)
  5949. def _unwrap_data_envelope(response: Any, task: str | None = None) -> Any:
  5950. """Unwrap gateway envelopes like {"data": {<chat completion>}, "success": true}.
  5951. Some OpenAI-compatible gateways (e.g. api.cline.bot) wrap non-streaming
  5952. JSON bodies in a data/success envelope. The SDK leniently parses this
  5953. into a ChatCompletion with choices=None and keeps the envelope keys as
  5954. extra fields, so the real completion is reachable at ``response.data``.
  5955. Error envelopes ({"success": false, "data": {"error": ...}}) raise the
  5956. provider's actual error instead of a generic invalid-response error.
  5957. """
  5958. if _field(response, "choices"):
  5959. return response
  5960. data = _field(response, "data")
  5961. if not isinstance(data, dict):
  5962. return response
  5963. if _field(response, "success") is False:
  5964. err = data.get("error") or data
  5965. msg = err.get("message") if isinstance(err, dict) else None
  5966. raise RuntimeError(
  5967. f"Auxiliary {task or 'call'}: provider returned error envelope: "
  5968. f"{msg or err}"
  5969. )
  5970. if not data.get("choices"):
  5971. return response
  5972. try:
  5973. return type(response).model_validate(data)
  5974. except (AttributeError, TypeError, ValueError):
  5975. # No model_validate (SimpleNamespace) or the inner payload fails
  5976. # strict validation (pydantic ValidationError is a ValueError).
  5977. pass
  5978. choices = []
  5979. for ch in data["choices"]:
  5980. msg = ch.get("message") if isinstance(ch, dict) else None
  5981. if isinstance(msg, dict):
  5982. choices.append(SimpleNamespace(
  5983. message=SimpleNamespace(**msg),
  5984. finish_reason=ch.get("finish_reason") or "stop",
  5985. ))
  5986. if not choices:
  5987. return response
  5988. return SimpleNamespace(
  5989. id=data.get("id", ""),
  5990. model=data.get("model", ""),
  5991. object=data.get("object", "chat.completion"),
  5992. choices=choices,
  5993. usage=data.get("usage"),
  5994. )
  5995. def _recover_aux_response_message(response: Any) -> Optional[Any]:
  5996. """Synthesize chat-completions shape from Responses-style text (``output_text``,
  5997. ``output`` items) that some compatible endpoints return outside ``choices``."""
  5998. text = _extract_aux_response_text(response)
  5999. if not text:
  6000. return None
  6001. choice = SimpleNamespace(message=SimpleNamespace(content=text), finish_reason=getattr(response, "finish_reason", None) or "stop")
  6002. try:
  6003. response.choices = [choice]
  6004. return response
  6005. except Exception:
  6006. return SimpleNamespace(
  6007. id=getattr(response, "id", ""), model=getattr(response, "model", ""),
  6008. object=getattr(response, "object", "chat.completion"), choices=[choice],
  6009. usage=getattr(response, "usage", None),
  6010. )
  6011. def _extract_aux_response_text(response: Any) -> str:
  6012. """Text from Responses-style ``output_text`` or ``output[].content[].text``."""
  6013. output_text = _field(response, "output_text")
  6014. if isinstance(output_text, str) and output_text.strip():
  6015. return output_text.strip()
  6016. output = _field(response, "output")
  6017. if not isinstance(output, list):
  6018. return ""
  6019. parts: list[str] = []
  6020. for item in output:
  6021. item_type = _field(item, "type")
  6022. if item_type and item_type != "message":
  6023. continue
  6024. for part in (_field(item, "content") or []):
  6025. if _field(part, "type") in {"output_text", "text", None}:
  6026. text = _field(part, "text")
  6027. if isinstance(text, str) and text.strip():
  6028. parts.append(text.strip())
  6029. return "\n".join(parts).strip()
  6030. # Streamed aggregation for progress-hooked aux calls: ``timeout`` becomes an inter-chunk idle
  6031. # timeout (httpx read timeout is per read), each chunk ticks outer watchdogs; the total ceiling
  6032. # bounds trickles.
  6033. _AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0
  6034. _AUX_STREAM_CEILING_MULTIPLIER = 4.0
  6035. def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float:
  6036. """Absolute wall-clock bound for a streamed aux call; generous by design (the idle
  6037. timeout is the real guard — this only stops a one-token-per-idle-window trickle)."""
  6038. try:
  6039. timeout = float(effective_timeout) if effective_timeout is not None else 0.0
  6040. except (TypeError, ValueError):
  6041. timeout = 0.0
  6042. return max(_AUX_STREAM_CEILING_FLOOR_SECONDS, _AUX_STREAM_CEILING_MULTIPLIER * timeout)
  6043. def _client_streams_internally(client: Any) -> bool:
  6044. """Adapters that stream inside .create() tick the hook themselves (Codex, Anthropic) or
  6045. cannot stream (Bedrock); none accept ``stream=True`` from us."""
  6046. return isinstance(client, (CodexAuxiliaryClient, AnthropicAuxiliaryClient, BedrockAuxiliaryClient))
  6047. def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool:
  6048. """Providers that only accept streaming (non-stream = 400): Tencent Copilot, any
  6049. ``auxiliary.stream_only_base_urls`` substring, and the managed local llama-server
  6050. (streamed for cancellation — it only notices a dead client on socket write)."""
  6051. _url = str(base_url or "").lower()
  6052. if not _url:
  6053. return False
  6054. if base_url_host_matches(_url, "copilot.tencent.com") or _is_managed_local_endpoint(_url):
  6055. return True
  6056. try:
  6057. from hermes_cli.config import load_config
  6058. markers = (load_config() or {}).get("auxiliary", {}).get("stream_only_base_urls") or []
  6059. if isinstance(markers, (list, tuple)):
  6060. return any(
  6061. isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url
  6062. for marker in markers)
  6063. except Exception:
  6064. pass # Config read is best-effort; never break an aux call over it.
  6065. return False
  6066. _AFFORDABLE_TOKENS_RE = re.compile(r"can only afford\s+([0-9][0-9,]*)", re.IGNORECASE)
  6067. # Below the floor the affordable budget can't fit a useful aux output — treat as exhaustion;
  6068. # the margin keeps provider-side token-count rounding from 402-ing the retry.
  6069. _AFFORDABLE_RETRY_FLOOR_TOKENS = 512
  6070. # See #49785.
  6071. _AFFORDABLE_RETRY_MARGIN_TOKENS = 64
  6072. def _affordable_max_tokens_from_error(exc: Exception) -> Optional[int]:
  6073. """Affordable output budget (minus margin) from an OpenRouter credit-limited 402
  6074. ("...but can only afford 7117": credit exists, the cap was too large); ``None``
  6075. when no count is present or the budget is too small to be useful."""
  6076. if not _is_payment_error(exc):
  6077. return None
  6078. match = _AFFORDABLE_TOKENS_RE.search(str(exc))
  6079. if not match:
  6080. return None
  6081. try:
  6082. affordable = int(match.group(1).replace(",", ""))
  6083. except (TypeError, ValueError):
  6084. return None
  6085. capped = affordable - _AFFORDABLE_RETRY_MARGIN_TOKENS
  6086. return capped if capped >= _AFFORDABLE_RETRY_FLOOR_TOKENS else None
  6087. def _create_with_progress(
  6088. client: Any, kwargs: dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
  6089. ) -> Any:
  6090. """Credit-aware :func:`_create_with_progress_once`: a 402 naming an affordable
  6091. budget retries ONCE with that cap (only ever lowering); anything else re-raises."""
  6092. try:
  6093. return _create_with_progress_once(client, kwargs, task, force_stream=force_stream)
  6094. except Exception as exc:
  6095. affordable = _affordable_max_tokens_from_error(exc)
  6096. if affordable is None:
  6097. raise
  6098. existing_cap = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
  6099. if isinstance(existing_cap, (int, float)) and 0 < existing_cap <= affordable:
  6100. raise # Already within budget — the error is something else; don't spin.
  6101. retry_kwargs = dict(kwargs)
  6102. retry_kwargs.pop("max_tokens", None)
  6103. retry_kwargs.pop("max_completion_tokens", None)
  6104. retry_kwargs.update(
  6105. auxiliary_max_tokens_param(affordable, model=str(kwargs.get("model") or "") or None))
  6106. logger.info("Auxiliary %s: credit-limited 402 (affordable=%d tokens); "
  6107. "retrying once with a clamped output cap instead of failing: %s",
  6108. task or "call", affordable, exc)
  6109. return _create_with_progress_once(client, retry_kwargs, task, force_stream=force_stream)
  6110. def _stream_request_plan(kwargs: dict[str, Any], client: Any = None) -> tuple[dict[str, Any], str, float]:
  6111. """(stream kwargs, model name, total ceiling) for a streamed re-aggregation. Requests to the
  6112. managed llama-server also ask for prompt-reading progress, which keeps the watchdogs fed
  6113. through a long prefill."""
  6114. stream_kwargs = dict(kwargs)
  6115. stream_kwargs["stream"] = True
  6116. stream_kwargs["stream_options"] = {"include_usage": True}
  6117. if _is_managed_local_endpoint(str(getattr(client, "base_url", "") or "")):
  6118. stream_kwargs = with_prompt_progress(stream_kwargs)
  6119. return (stream_kwargs, str(kwargs.get("model") or ""),
  6120. _aux_stream_total_ceiling(kwargs.get("timeout")))
  6121. def _create_with_progress_once(
  6122. client: Any, kwargs: dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
  6123. ) -> Any:
  6124. """create() that streams (and re-aggregates, ticking the hook per substantive chunk) when a
  6125. progress hook is active or the provider is stream-only; plain ``create(**kwargs)`` otherwise
  6126. or when the adapter streams internally. Streaming rejections fall back to a plain call —
  6127. except under ``force_stream``.
  6128. Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when neither trigger applies (every
  6129. existing caller/task) or when the client's wire adapter streams internally. With a hook + a
  6130. chunk-capable client, the request is sent with ``stream=True`` and aggregated, ticking the hook only for
  6131. substantive chunks. The configured ``timeout`` acts per stream read (idle) rather than as a total
  6132. budget, and outer liveness watchdogs see tokens moving. ``force_stream=True`` (stream-only providers
  6133. such as Tencent Copilot — credit @kudi88, PR #60686) takes the same streamed path even without a hook.
  6134. Providers that reject the streamed request fall back to the plain non-streaming call — except under
  6135. ``force_stream``, where a stream-only provider rejects the plain call by definition, so the original
  6136. error is surfaced to the normal recovery chains instead.
  6137. """
  6138. kwargs = bypass_chat_sdk_request_transform(kwargs, client)
  6139. _notify_aux_dispatch()
  6140. # Dispatch alone is not forward progress: a 401/retry/fallback dispatch must not
  6141. # reset the compression inactivity fence, or a zero-output attempt runs to the
  6142. # total ceiling instead of idling out (#114938). Progress ticks only for
  6143. # substantive stream payloads or a completed usable response.
  6144. if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client):
  6145. response = client.chat.completions.create(**kwargs)
  6146. if not _client_streams_internally(client):
  6147. _notify_aux_provider_response()
  6148. return response
  6149. stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs, client)
  6150. try:
  6151. chunks = client.chat.completions.create(**stream_kwargs)
  6152. except Exception as exc:
  6153. # Genuine provider failures aren't streaming's fault — surface unchanged so the
  6154. # recovery chains see the same error as a plain call.
  6155. if (force_stream or _is_transient_transport_error(exc) or _is_auth_error(exc)
  6156. or _is_payment_error(exc) or _is_rate_limit_error(exc)):
  6157. raise
  6158. # Possibly a streaming-specific rejection: retry non-streaming once; a genuinely bad
  6159. # request reproduces the real error for the except-chains.
  6160. logger.debug("Auxiliary %s: streamed request failed (%s); retrying non-streaming",
  6161. task or "call", exc)
  6162. _notify_aux_dispatch()
  6163. response = client.chat.completions.create(**kwargs)
  6164. _notify_aux_provider_response()
  6165. return response
  6166. # Some shims (MoA quiet mode, defensive adapters) return a complete response despite
  6167. # stream=True; it counts as provider response + forward progress.
  6168. if hasattr(chunks, "choices"):
  6169. _notify_aux_provider_response()
  6170. return chunks
  6171. from agent.auxiliary_stream_watchdog import chat_stream_windows
  6172. return _aggregate_chat_stream(
  6173. chunks, model=model, total_ceiling=total_ceiling, no_progress=chat_stream_windows(client, kwargs, task))
  6174. def _aggregate_chat_stream(
  6175. chunks: Any, *, model: str = "", total_ceiling: Optional[float] = None,
  6176. no_progress: Optional[tuple[float, Optional[float]]] = None,
  6177. ) -> Any:
  6178. """Consume a chunk stream into a complete response; TimeoutError (phrased "timed out" so
  6179. ``_is_timeout_error`` matches) past *total_ceiling* or the *no_progress* windows (#100501)."""
  6180. from agent.auxiliary_stream_watchdog import _close_chunk_stream, consume_chat_stream
  6181. acc = _ChatStreamAccumulator(
  6182. model=model, total_ceiling=total_ceiling, host_deadline=_current_aux_stream_deadline)
  6183. try:
  6184. consume_chat_stream(chunks, acc, no_progress)
  6185. finally:
  6186. _close_chunk_stream(chunks)
  6187. return acc.finish()
  6188. # Reasoning-detail fields whose non-empty text counts as forward progress.
  6189. _REASONING_DETAIL_TEXT_FIELDS = ("summary", "thinking", "content", "text")
  6190. class _ChatStreamAccumulator:
  6191. """Shared per-chunk accumulation so sync and async aggregation cannot drift."""
  6192. def __init__(self, model: str = "", total_ceiling: Optional[float] = None,
  6193. host_deadline: float | Callable[[], float | None] | None = None):
  6194. self._started = time.monotonic()
  6195. # llama-server prompt reading doesn't count toward the ceiling (``local_prompt_progress``).
  6196. self._prompt_reading = PromptReadingClock(self._started)
  6197. self._total_ceiling = total_ceiling
  6198. # Absolute instant the waiting host gives up, checked as well as (not instead of) the ceiling, which
  6199. # still bounds callers with no host deadline; unaffected by pre-construction dispatch/TTFT (#99692).
  6200. self._host_deadline = host_deadline
  6201. self.content_parts: list[str] = []
  6202. self.reasoning_parts: list[str] = []
  6203. self.reasoning_details: list[Any] = []
  6204. self.tool_calls_acc: dict[int, dict[str, Any]] = {}
  6205. self.finish_reason = self.usage = None
  6206. self.resp_id = ""
  6207. self.resp_model = model or ""
  6208. def _check_deadlines(self) -> None:
  6209. """Raise TimeoutError past the total ceiling or the host deadline; a host hard cancel (/stop, session
  6210. boundary) stops the stream at its next chunk instead of letting the daemon bill on to the deadline."""
  6211. if _aux_interrupt_cancel_requested():
  6212. raise AuxiliaryExplicitCancellation()
  6213. now = time.monotonic()
  6214. if self._total_ceiling is not None and (now - self._started - self._prompt_reading.seconds) >= self._total_ceiling:
  6215. raise TimeoutError(f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s "
  6216. "total ceiling (stream still open but over budget)")
  6217. host_deadline = self._host_deadline() if callable(self._host_deadline) else self._host_deadline
  6218. if host_deadline is not None and now >= host_deadline:
  6219. raise TimeoutError("Auxiliary streamed call timed out at the host compression "
  6220. f"deadline after {time.monotonic() - self._started:.0f}s "
  6221. "(the caller already stopped waiting; streaming on would only "
  6222. "pin its session lease)")
  6223. def _feed_reasoning_details(self, delta: Any) -> bool:
  6224. """Collect ``reasoning_details`` (OpenRouter-style thinking); True only when a detail
  6225. carries text, so structural/signed envelopes can't keep a stall alive."""
  6226. reasoning_details = getattr(delta, "reasoning_details", None)
  6227. if reasoning_details is None:
  6228. model_extra = getattr(delta, "model_extra", None)
  6229. if isinstance(model_extra, dict):
  6230. reasoning_details = model_extra.get("reasoning_details")
  6231. if not isinstance(reasoning_details, list):
  6232. return False
  6233. made_progress = False
  6234. for detail in reasoning_details:
  6235. self.reasoning_details.append(detail)
  6236. if isinstance(detail, dict) and any(
  6237. isinstance(detail.get(f), str) and detail[f] for f in _REASONING_DETAIL_TEXT_FIELDS):
  6238. made_progress = True
  6239. return made_progress
  6240. def _feed_tool_calls(self, delta: Any) -> bool:
  6241. """Merge tool-call fragments by index; True when any fragment carried data."""
  6242. made_progress = False
  6243. for tc in (getattr(delta, "tool_calls", None) or []):
  6244. idx = getattr(tc, "index", 0) or 0
  6245. acc = self.tool_calls_acc.setdefault(idx, {"id": "", "name": "", "arguments": []})
  6246. if getattr(tc, "id", None):
  6247. acc["id"] = tc.id
  6248. made_progress = True
  6249. fn = getattr(tc, "function", None)
  6250. if fn is not None:
  6251. if getattr(fn, "name", None):
  6252. acc["name"] = fn.name
  6253. made_progress = True
  6254. if getattr(fn, "arguments", None):
  6255. acc["arguments"].append(fn.arguments)
  6256. made_progress = True
  6257. return made_progress
  6258. def feed(self, chunk: Any) -> bool:
  6259. # Every frame records transport timing (TTFP); only a substantive payload ticks the
  6260. # forward-progress hook that keeps compression alive (and is reported as True).
  6261. _notify_aux_timing_response()
  6262. progress = prompt_progress(chunk)
  6263. # Count the reading before the deadline check it extends.
  6264. read_more = progress is not None and self._prompt_reading.tick(progress)
  6265. self._check_deadlines()
  6266. self.resp_id = getattr(chunk, "id", None) or self.resp_id
  6267. self.resp_model = getattr(chunk, "model", None) or self.resp_model
  6268. chunk_usage = getattr(chunk, "usage", None)
  6269. if chunk_usage:
  6270. self.usage = chunk_usage
  6271. if progress is not None:
  6272. # llama-server is still reading the prompt: new tokens read keep the compression fence
  6273. # alive. It is no first token, so the stream watchdog keeps its pre-token patience: a
  6274. # busy server reports a batch every minute or more.
  6275. if read_more:
  6276. _notify_aux_progress()
  6277. return False
  6278. choices = getattr(chunk, "choices", None) or []
  6279. if not choices:
  6280. return False
  6281. choice = choices[0]
  6282. self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason
  6283. delta = getattr(choice, "delta", None)
  6284. if delta is None:
  6285. return False
  6286. made_progress = False
  6287. from agent.message_content import flatten_message_text
  6288. piece = flatten_message_text(getattr(delta, "content", None), sep="")
  6289. if piece:
  6290. self.content_parts.append(piece)
  6291. made_progress = True
  6292. reasoning_piece = getattr(delta, "reasoning", None) or getattr(delta, "reasoning_content", None)
  6293. if reasoning_piece is None and isinstance(getattr(delta, "model_extra", None), dict):
  6294. reasoning_piece = delta.model_extra.get("reasoning") or delta.model_extra.get("reasoning_content")
  6295. reasoning_piece = flatten_message_text(reasoning_piece, sep="")
  6296. if reasoning_piece:
  6297. self.reasoning_parts.append(reasoning_piece)
  6298. made_progress = True
  6299. # Evaluate both unconditionally: they accumulate state, not just progress.
  6300. made_progress |= self._feed_reasoning_details(delta)
  6301. made_progress |= self._feed_tool_calls(delta)
  6302. if made_progress:
  6303. _notify_aux_progress()
  6304. return made_progress
  6305. def finish(self) -> Any:
  6306. tool_calls = None
  6307. if self.tool_calls_acc:
  6308. tool_calls = [
  6309. SimpleNamespace(id=acc["id"], type="function", function=SimpleNamespace(
  6310. name=acc["name"], arguments="".join(acc["arguments"])))
  6311. for _idx, acc in sorted(self.tool_calls_acc.items())]
  6312. message = SimpleNamespace(
  6313. role="assistant", content="".join(self.content_parts), tool_calls=tool_calls,
  6314. reasoning="".join(self.reasoning_parts) or None,
  6315. reasoning_details=self.reasoning_details or None,
  6316. )
  6317. choice = SimpleNamespace(index=0, message=message, finish_reason=self.finish_reason or "stop")
  6318. return SimpleNamespace(id=self.resp_id, model=self.resp_model, object="chat.completion",
  6319. choices=[choice], usage=self.usage)
  6320. async def _aggregate_chat_stream_async(
  6321. chunks: Any, *, model: str = "", total_ceiling: Optional[float] = None
  6322. ) -> Any:
  6323. """Async mirror of :func:`_aggregate_chat_stream` (AsyncOpenAI streams need ``async for``)."""
  6324. acc = _ChatStreamAccumulator(
  6325. model=model, total_ceiling=total_ceiling, host_deadline=_current_aux_stream_deadline)
  6326. try:
  6327. async for chunk in chunks:
  6328. acc.feed(chunk)
  6329. finally:
  6330. from agent.auxiliary_stream_watchdog import _close_chunk_stream
  6331. pending = _close_chunk_stream(chunks, allow_aclose=True)
  6332. if pending is not None:
  6333. with contextlib.suppress(Exception):
  6334. await pending
  6335. return acc.finish()
  6336. async def _acreate_with_stream(client: Any, kwargs: dict[str, Any], task: Optional[str] = None) -> Any:
  6337. """Async create() for stream-only providers: ``stream=True`` + aggregate the async chunks."""
  6338. stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs, client)
  6339. chunks = await client.chat.completions.create(**stream_kwargs)
  6340. if hasattr(chunks, "choices"): # shims may hand back a complete response despite stream=True
  6341. return chunks
  6342. return await _aggregate_chat_stream_async(chunks, model=model, total_ceiling=total_ceiling)
  6343. def _async_client_streams_internally(client: Any) -> bool:
  6344. """Async twin of :func:`_client_streams_internally` (the async adapters are separate classes)."""
  6345. return isinstance(client, (AsyncCodexAuxiliaryClient, AsyncAnthropicAuxiliaryClient, AsyncBedrockAuxiliaryClient))
  6346. async def _acreate_with_progress(
  6347. client: Any, kwargs: dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
  6348. ) -> Any:
  6349. """Async :func:`_create_with_progress`: stream + re-aggregate (ticking the hook per substantive
  6350. chunk) when a progress hook is active or the provider is stream-only; plain create otherwise."""
  6351. kwargs = bypass_chat_sdk_request_transform(kwargs, client)
  6352. _notify_aux_dispatch()
  6353. # Same contract as the sync twin (#114938): dispatch alone is not progress.
  6354. if (not _aux_progress_active() and not force_stream) or _async_client_streams_internally(client):
  6355. response = await client.chat.completions.create(**kwargs)
  6356. if not _async_client_streams_internally(client):
  6357. _notify_aux_provider_response()
  6358. return response
  6359. stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs, client)
  6360. try:
  6361. chunks = await client.chat.completions.create(**stream_kwargs)
  6362. except Exception as exc:
  6363. # Only a rejected stream NEGOTIATION falls back to a plain call (mirrors the sync wrapper); a
  6364. # failure mid-consumption below reaches the classified recovery ladder instead of silently
  6365. # re-sending the whole prompt non-streaming.
  6366. if (force_stream or _is_transient_transport_error(exc) or _is_auth_error(exc)
  6367. or _is_payment_error(exc) or _is_rate_limit_error(exc)):
  6368. raise
  6369. logger.debug("Auxiliary %s: streamed async request failed (%s); retrying non-streaming",
  6370. task or "call", exc)
  6371. _notify_aux_dispatch()
  6372. response = await client.chat.completions.create(**kwargs)
  6373. _notify_aux_provider_response()
  6374. return response
  6375. if hasattr(chunks, "choices"): # shims may hand back a complete response despite stream=True
  6376. _notify_aux_provider_response()
  6377. return chunks
  6378. return await _aggregate_chat_stream_async(chunks, model=model, total_ceiling=total_ceiling)
  6379. # Shared request head + recovery ladder for call_llm / async_call_llm: the entry points differ
  6380. # only in how a request is awaited, so route resolution and the ordered recovery ladder are
  6381. # written once. The ladder is a generator yielding ``_LadderStep`` requests and receiving the
  6382. # response (or thrown exception), so rung ORDER and accept/re-raise contracts match on both wires.
  6383. class _ResolvedAuxRoute(NamedTuple):
  6384. client: Any
  6385. final_model: Optional[str]
  6386. resolved_provider: str
  6387. effective_provider: str
  6388. def _resolve_call_client(
  6389. task: Optional[str], *, provider: Optional[str], model: Optional[str], base_url: Optional[str],
  6390. api_key: Optional[str], resolved_provider: str, resolved_model: Optional[str],
  6391. resolved_base_url: Optional[str], resolved_api_key: Optional[str],
  6392. resolved_api_mode: Optional[str], main_runtime: Optional[dict[str, Any]], async_mode: bool,
  6393. ) -> _ResolvedAuxRoute:
  6394. """Resolve the client for one aux call: vision chain, or cached text client with the
  6395. explicit-provider fallback_chain / auto-chain rescue; RuntimeError when nothing is configured."""
  6396. effective_provider = resolved_provider
  6397. if task == "vision":
  6398. effective_provider, client, final_model = resolve_vision_provider_client(
  6399. provider=resolved_provider if resolved_provider != "auto" else provider,
  6400. model=resolved_model or model, base_url=resolved_base_url or base_url,
  6401. api_key=resolved_api_key or api_key, async_mode=async_mode, main_runtime=main_runtime,
  6402. )
  6403. if client is None and resolved_provider != "auto" and not resolved_base_url:
  6404. logger.warning("Vision provider %s unavailable, falling back to auto vision backends",
  6405. resolved_provider)
  6406. effective_provider, client, final_model = resolve_vision_provider_client(
  6407. provider="auto", model=resolved_model, async_mode=async_mode,
  6408. main_runtime=main_runtime)
  6409. if client is not None:
  6410. resolved_provider = effective_provider or resolved_provider
  6411. else:
  6412. client, final_model = _get_cached_client(
  6413. resolved_provider, resolved_model, async_mode=async_mode, base_url=resolved_base_url,
  6414. api_key=resolved_api_key, api_mode=resolved_api_mode, main_runtime=main_runtime,
  6415. task=task)
  6416. effective_provider = _effective_provider_for_client(client, resolved_provider)
  6417. if client is None:
  6418. # Explicit provider with no credentials: honor the task fallback_chain before
  6419. # raising (fallback entries may use OAuth / credential-pool auth).
  6420. _explicit = (resolved_provider or "").strip().lower()
  6421. if _explicit and _explicit not in {"auto", "openrouter", "custom"}:
  6422. fb_client, fb_model, fb_label = _try_configured_fallback_for_unavailable_client(
  6423. task, _explicit)
  6424. if fb_client is None:
  6425. nous_detail = nous_credential_failure_detail() if _explicit == "nous" else None
  6426. raise AuxiliaryClientUnavailable(
  6427. nous_detail or missing_provider_credentials_message(_explicit))
  6428. client, final_model = fb_client, fb_model
  6429. if async_mode:
  6430. client, final_model = _to_async_client(
  6431. fb_client, fb_model or "", is_vision=(task == "vision"))
  6432. resolved_provider = fb_label or resolved_provider
  6433. effective_provider = resolved_provider
  6434. # Auto/custom with no credentials: walk the full auto chain (not just OpenRouter).
  6435. # model=None so each provider uses its own default.
  6436. if client is None and not resolved_base_url:
  6437. logger.info("Auxiliary %s: provider %s unavailable, trying auto-detection chain",
  6438. task or "call", resolved_provider)
  6439. client, final_model = _get_cached_client(
  6440. "auto", async_mode=async_mode, main_runtime=main_runtime, task=task)
  6441. effective_provider = _effective_provider_for_client(client, "auto")
  6442. if client is None:
  6443. raise AuxiliaryClientUnavailable(f"No LLM provider configured for task={task} "
  6444. f"provider={resolved_provider}. Run: hermes setup")
  6445. return _ResolvedAuxRoute(client, final_model, resolved_provider, effective_provider)
  6446. class _PreparedAuxRequest(NamedTuple):
  6447. client: Any
  6448. final_model: Optional[str]
  6449. kwargs: dict[str, Any]
  6450. resolved_provider: str
  6451. request_provider: str
  6452. resolved_model: Optional[str]
  6453. resolved_base_url: Optional[str]
  6454. resolved_api_key: Optional[str]
  6455. resolved_api_mode: Optional[str]
  6456. effective_timeout: float
  6457. effective_extra_body: dict[str, Any]
  6458. base_info: str
  6459. def _prepare_aux_request(
  6460. task: Optional[str], *, provider: Optional[str], model: Optional[str], base_url: Optional[str],
  6461. api_key: Optional[str], main_runtime: dict[str, Any], messages: list,
  6462. temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
  6463. timeout: Optional[float], extra_body: Optional[dict], reasoning_config: Optional[dict],
  6464. extra_headers: Optional[dict[str, str]], api_mode: Optional[str],
  6465. route_info: Optional[dict[str, str]], async_mode: bool,
  6466. ) -> _PreparedAuxRequest:
  6467. """Shared head of call_llm/async_call_llm: resolve route + client, publish it, build request kwargs.
  6468. Sync-only: compression fast lane, per-request ``extra_headers``, and ``base_info`` falling
  6469. back to the resolved base_url when the client exposes none."""
  6470. resolved_provider, resolved_model, resolved_base_url, resolved_api_key, resolved_api_mode = _resolve_task_provider_model(
  6471. task, provider, model, base_url, api_key)
  6472. if api_mode:
  6473. resolved_api_mode = api_mode
  6474. effective_extra_body = _get_task_extra_body(task)
  6475. effective_extra_body.update(extra_body or {})
  6476. client, final_model, resolved_provider, effective_provider = _resolve_call_client(
  6477. task, provider=provider, model=model, base_url=base_url, api_key=api_key,
  6478. resolved_provider=resolved_provider, resolved_model=resolved_model,
  6479. resolved_base_url=resolved_base_url, resolved_api_key=resolved_api_key,
  6480. resolved_api_mode=resolved_api_mode, main_runtime=main_runtime, async_mode=async_mode,
  6481. )
  6482. effective_timeout = _effective_aux_timeout(task, timeout)
  6483. # Codex-Responses-only: real SDK clients reject an unrecognized ``no_progress_timeout``
  6484. # kwarg, so only resolve/forward it when the route is actually a Codex stream (#108104).
  6485. no_progress_timeout = (
  6486. _get_task_no_progress_timeout(task)
  6487. if isinstance(client, (CodexAuxiliaryClient, AsyncCodexAuxiliaryClient)) else None
  6488. )
  6489. request_provider = effective_provider or resolved_provider
  6490. if not async_mode:
  6491. compression_config = _get_auxiliary_task_config(task) if task == "compression" else {} # authoritative: no aux route
  6492. _, effective_extra_body = _compression_fast_lane_controls(
  6493. task, actual_provider=request_provider, actual_model=final_model,
  6494. requested_provider=provider, requested_model=model, route_config=compression_config,
  6495. leak_guard_config=compression_config, max_tokens=max_tokens,
  6496. extra_body=effective_extra_body,
  6497. )
  6498. _set_relay_auxiliary_route(request_provider, final_model, resolved_api_mode)
  6499. _record_route_info(route_info, _fallback_provider_from_label(request_provider), final_model)
  6500. if async_mode:
  6501. base_info = str(getattr(client, "base_url", "") or "")
  6502. else:
  6503. base_info = str(getattr(client, "base_url", resolved_base_url) or "")
  6504. if task:
  6505. logger.info("Auxiliary %s: using %s (%s)%s",
  6506. task, request_provider or "auto", final_model or "default",
  6507. f" at {base_info}" if base_info and "openrouter" not in base_info else "")
  6508. # Client's actual base_url so endpoint-specific temperature overrides work on
  6509. # auto-detected routes (api.moonshot.ai vs api.kimi.com/coding).
  6510. kwargs = _build_call_kwargs(
  6511. request_provider, final_model, messages, temperature=temperature, max_tokens=max_tokens,
  6512. tools=tools, timeout=effective_timeout, extra_body=effective_extra_body,
  6513. reasoning_config=reasoning_config, base_url=base_info or resolved_base_url, task=task,
  6514. no_progress_timeout=no_progress_timeout)
  6515. if extra_headers:
  6516. kwargs["extra_headers"] = dict(extra_headers)
  6517. # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)
  6518. client_base = str(getattr(client, "base_url", "") or "")
  6519. if _is_anthropic_compat_endpoint(request_provider, client_base):
  6520. kwargs["messages"] = _convert_openai_images_to_anthropic(kwargs["messages"])
  6521. return _PreparedAuxRequest(
  6522. client, final_model, kwargs, resolved_provider, request_provider, resolved_model,
  6523. resolved_base_url, resolved_api_key, resolved_api_mode, effective_timeout,
  6524. effective_extra_body, base_info)
  6525. class _LadderStep(NamedTuple):
  6526. """A provider request the ladder asks its driver to perform. kind: "call" (client, kwargs) |
  6527. "retry_same_provider" (provider, model) | "fallback" (fb_client, fb_model, fb_label)."""
  6528. kind: str
  6529. args: tuple
  6530. # Ordered (predicate, reason) pairs for the provider-fallback rung: first match
  6531. # wins, so a payment-flavoured 429 reads as "payment error", not "rate limit".
  6532. _FALLBACK_REASONS: tuple[tuple[Callable[[Exception], bool], str], ...] = (
  6533. (_is_auth_error, "auth error"), (_is_payment_error, "payment error"),
  6534. (_is_rate_limit_error, "rate limit"), (_is_model_incompatible_error, "model incompatible with route"),
  6535. (_is_invalid_aux_response_error, "invalid provider response"),
  6536. # A status-less in-stream ``error`` event (SSE committed 200) is a route failure (#101538).
  6537. (_is_statusless_structured_provider_error, "structured provider error"),
  6538. # Before the connection-error rung (its superset): a full-budget timeout must be named as one, or
  6539. # a slow local model reads as an unreachable endpoint (#89445).
  6540. (_is_timeout_error, "request timed out"), (_is_connection_error, "connection error"),
  6541. )
  6542. def _rung(step: _LadderStep, accept: Callable[[Exception], bool]):
  6543. """One ladder rung: perform ``step``; yields ``(response, None)`` on success,
  6544. ``(None, exc)`` when ``accept(exc)`` lets the next rung handle it, else re-raises."""
  6545. try:
  6546. result = yield step
  6547. except Exception as exc:
  6548. if not accept(exc):
  6549. raise
  6550. return None, exc
  6551. return result, None
  6552. def _param_rung_accepts(exc: Exception) -> bool:
  6553. """After a parameter-strip retry: fall through to the max_tokens/payment/auth
  6554. chains with the stripped kwargs; re-raise anything those chains won't handle.
  6555. A 429 on the retry is the credential/provider-fallback rungs' job, so it falls
  6556. through too (the pre-ladder max_tokens rung accepted rate limits)."""
  6557. return (_is_payment_error(exc) or _is_connection_error(exc) or _is_auth_error(exc)
  6558. or _is_rate_limit_error(exc) or _is_statusless_structured_provider_error(exc)
  6559. or "max_tokens" in str(exc) or "unsupported_parameter" in str(exc)
  6560. # Parameter rungs chain in any order (a reasoning-strip retry can 400 on temperature,
  6561. # a temperature-strip retry on max_tokens), and a route-gating 400 after a strip still
  6562. # reaches the provider-fallback rung.
  6563. or _is_unsupported_parameter_error(exc, "temperature")
  6564. or _is_reasoning_field_rejection(exc) or _is_reasoning_required_rejection(exc)
  6565. or _is_structured_output_rejection(exc)
  6566. or _is_model_incompatible_error(exc))
  6567. def _credential_rung_accepts(exc: Exception) -> bool:
  6568. return _is_auth_error(exc) or _is_payment_error(exc) or _is_rate_limit_error(exc)
  6569. # Immutable route context shared by the recovery rungs.
  6570. class _LadderRoute(NamedTuple):
  6571. client: Any
  6572. task: Optional[str]
  6573. tag: str
  6574. async_mode: bool
  6575. base_info: str
  6576. resolved_provider: str
  6577. resolved_model: Optional[str]
  6578. resolved_base_url: Optional[str]
  6579. resolved_api_key: Optional[str]
  6580. resolved_api_mode: Optional[str]
  6581. final_model: Optional[str]
  6582. main_runtime: Optional[dict[str, Any]]
  6583. route_info: Optional[dict[str, str]]
  6584. timeout: Optional[float]
  6585. def _without_temperature(kwargs: dict) -> Optional[dict]:
  6586. """Copy *kwargs* without ``temperature``; None when it was not sent."""
  6587. return {k: v for k, v in kwargs.items() if k != "temperature"} if "temperature" in kwargs else None
  6588. def _without_max_tokens(kwargs: dict) -> dict:
  6589. """Copy *kwargs* without either output cap. Unlike the other strips this never returns None: a
  6590. route can translate the caller's cap into a field the wire kwargs no longer show (Codex
  6591. Responses), and the provider's own gateway may inject one — the 400 still names ``max_tokens``
  6592. and the identical request completes on retry (registry class #89897/#90257)."""
  6593. return {k: v for k, v in kwargs.items() if k not in ("max_tokens", "max_completion_tokens")}
  6594. def _is_max_tokens_rejection(exc: Exception, client: Any) -> bool:
  6595. err_str = str(exc)
  6596. # ZAI vision models reject max_tokens with code 1210 and a message that never
  6597. # mentions "max_tokens", so detect it explicitly.
  6598. is_zai_param_error = "1210" in err_str and "bigmodel" in str(getattr(client, "base_url", ""))
  6599. return ("max_tokens" in err_str or "unsupported_parameter" in err_str
  6600. or _is_unsupported_parameter_error(exc, "max_tokens") or is_zai_param_error)
  6601. def _parameter_rungs(client: Any, max_tokens: Optional[int]) -> tuple:
  6602. """Ordered ``(matches, strip, log message, remember)`` parameter rungs; ``strip`` returns None
  6603. when the field was not on the wire, so an unchanged request is never re-sent; ``remember``
  6604. (optional) records the rejection per route so the next call omits the field up front."""
  6605. return (
  6606. (lambda exc: _is_unsupported_parameter_error(exc, "temperature"), _without_temperature,
  6607. "provider rejected temperature; retrying without it", remember_temperature_rejection),
  6608. (_is_structured_output_rejection, _without_structured_output_format,
  6609. "provider rejected the structured-output format field; retrying without it "
  6610. "(schema enforcement degrades to prompt compliance)", remember_structured_output_rejection),
  6611. # A chat-only model on an OpenAI-compatible relay rejects the profile's thinking-off encoding
  6612. # (top-level ``reasoning_effort: none``), and strict-schema gateways reject the generic
  6613. # ``extra_body.reasoning`` fallback outright (#109774); the caller only wanted "no thinking",
  6614. # so retry with every reasoning field omitted and let the route default apply (#112781).
  6615. # The endpoint refuses the *disable* rather than the field (Nous Portal gpt-6-astra: "Reasoning is
  6616. # mandatory ... cannot be disabled"): step the effort up to the floor and remember the route so
  6617. # the next thinking-off aux call starts there. Ordered before the strip so a floor that still
  6618. # 400s falls through to it.
  6619. (_is_reasoning_required_rejection, with_reasoning_floor,
  6620. "provider requires reasoning; retrying at the floor effort", remember_reasoning_floor),
  6621. (_is_reasoning_field_rejection, _without_reasoning_fields,
  6622. "provider rejected the reasoning field; retrying without it (route default applies)", None),
  6623. (lambda exc: max_tokens is not None and _is_max_tokens_rejection(exc, client), _without_max_tokens,
  6624. "provider rejected the output cap; retrying without it", None),
  6625. )
  6626. def _ladder_parameter_rungs(
  6627. first_err: Exception, route: _LadderRoute, kwargs: dict[str, Any], max_tokens: Optional[int],
  6628. ):
  6629. """Parameter rungs: retry without temperature / structured-output format / reasoning field /
  6630. max_tokens. Rungs chain in whichever order the provider raises them (reasoning models reject
  6631. temperature AND max_tokens; a reasoning-strip retry can then trip temperature, #78273), each
  6632. field stripped at most once, so a request with N rejected fields recovers in N retries.
  6633. Returns ``(response, None, kwargs)`` or ``(None, narrowed_err, stripped_kwargs)``."""
  6634. client, task, tag = route.client, route.task, route.tag
  6635. rungs = list(_parameter_rungs(client, max_tokens))
  6636. while rungs:
  6637. hit = next((rung for rung in rungs
  6638. if rung[0](first_err) and rung[1](kwargs) is not None), None)
  6639. if hit is None:
  6640. break
  6641. rungs.remove(hit)
  6642. _matches, strip, message, remember = hit
  6643. retry_kwargs = strip(kwargs)
  6644. logger.info("Auxiliary %s%s: %s: %s", task or "call", tag, message, first_err)
  6645. rejection = first_err
  6646. resp, first_err = yield from _rung(
  6647. _LadderStep("call", (client, retry_kwargs)), _param_rung_accepts)
  6648. if first_err is None:
  6649. if remember is not None:
  6650. # Same key _build_call_kwargs looks up (base_info or resolved_base_url), so the
  6651. # memory hits when the client exposes no base_url but the task resolved one.
  6652. remember(route.resolved_provider, route.base_info or route.resolved_base_url,
  6653. kwargs, rejection)
  6654. return resp, None, retry_kwargs
  6655. kwargs = retry_kwargs
  6656. return None, first_err, kwargs
  6657. def _refreshed_nous_step(route: _LadderRoute, kwargs: dict[str, Any], message: str) -> Optional[_LadderStep]:
  6658. """Rebuild the Nous client after a credential event; None when nothing refreshed."""
  6659. refreshed_client, refreshed_model = _refresh_nous_auxiliary_client(
  6660. cache_provider=route.resolved_provider or "nous", model=route.final_model,
  6661. lookup_model=route.resolved_model, lookup_task=route.task, async_mode=route.async_mode,
  6662. base_url=route.resolved_base_url, api_key=route.resolved_api_key,
  6663. api_mode=route.resolved_api_mode, main_runtime=route.main_runtime,
  6664. is_vision=(route.task == "vision"),
  6665. )
  6666. if refreshed_client is None:
  6667. return None
  6668. logger.info(message, route.task or "call", route.tag)
  6669. if refreshed_model and refreshed_model != kwargs.get("model"):
  6670. kwargs["model"] = refreshed_model
  6671. return _LadderStep("call", (refreshed_client, kwargs))
  6672. def _ladder_nous_rungs(
  6673. first_err: Exception, route: _LadderRoute, kwargs: dict[str, Any], client_is_nous: bool,
  6674. ):
  6675. """Nous-only rungs: stale-model self-heal, paid-account refresh, 401 refresh.
  6676. Returns ``(response, None)`` or ``(None, first_err)`` to fall through."""
  6677. client, task, tag = route.client, route.task, route.tag
  6678. # A long-lived process can pin a Portal model since dropped from the catalog (every call
  6679. # 404s); force a fresh Portal fetch and retry once.
  6680. if _is_model_not_found_error(first_err) and client_is_nous:
  6681. healed_model = _refresh_nous_recommended_model(
  6682. vision=(task == "vision"), stale_model=kwargs.get("model"))
  6683. if healed_model and healed_model != kwargs.get("model"):
  6684. logger.warning("Auxiliary %s%s: model %r no longer in Nous catalog; "
  6685. "retrying with refreshed recommendation %r",
  6686. task or "call", tag, kwargs.get("model"), healed_model)
  6687. kwargs["model"] = healed_model
  6688. resp, first_err = yield from _rung(_LadderStep("call", (client, kwargs)), lambda exc: True)
  6689. if first_err is None:
  6690. return resp, None
  6691. # Auth refresh parity with the main agent.
  6692. if _is_payment_error(first_err) and client_is_nous and _nous_portal_account_has_fresh_paid_access():
  6693. step = _refreshed_nous_step(
  6694. route, kwargs,
  6695. "Auxiliary %s%s: refreshed Nous runtime credentials after paid account check, retrying")
  6696. if step is not None:
  6697. resp, first_err = yield from _rung(
  6698. step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
  6699. if first_err is None:
  6700. return resp, None
  6701. if _is_auth_error(first_err) and client_is_nous:
  6702. step = _refreshed_nous_step(
  6703. route, kwargs, "Auxiliary %s%s: refreshed Nous runtime credentials after 401, retrying")
  6704. if step is not None:
  6705. resp, first_err = yield from _rung(
  6706. step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
  6707. if first_err is None:
  6708. return resp, None
  6709. return None, first_err
  6710. def _ladder_credential_rungs(
  6711. first_err: Exception, route: _LadderRoute, kwargs: dict[str, Any], client_is_nous: bool,
  6712. ):
  6713. """OAuth credential refresh + same-provider retry, then credential-pool rotation.
  6714. Returns ``(response, None)`` or ``(None, first_err)`` to fall through."""
  6715. client, task, tag, resolved_provider = route.client, route.task, route.tag, route.resolved_provider
  6716. auth_refresh_provider = _auth_refresh_provider_for_route(
  6717. resolved_provider, route.base_info, _effective_provider_for_client(client, ""))
  6718. if (_is_auth_error(first_err) and auth_refresh_provider not in {"auto", "", None}
  6719. and not client_is_nous):
  6720. refresh_kwargs = ({"failed_api_key": getattr(client, "api_key", "")}
  6721. if auth_refresh_provider == "anthropic" else {})
  6722. if _refresh_provider_credentials(auth_refresh_provider, **refresh_kwargs):
  6723. if auth_refresh_provider != _normalize_aux_provider(resolved_provider):
  6724. # The stale client is cached under the route label (e.g. "auto"), not the
  6725. # concrete backend we refreshed.
  6726. _evict_cached_clients(resolved_provider)
  6727. logger.info("Auxiliary %s%s: refreshed %s credentials after auth error, retrying",
  6728. task or "call", tag, auth_refresh_provider)
  6729. step = _LadderStep(
  6730. "retry_same_provider",
  6731. (auth_refresh_provider, route.resolved_model or route.final_model))
  6732. resp, first_err = yield from _rung(
  6733. step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
  6734. if first_err is None:
  6735. return resp, None
  6736. # ``first_err`` is now the retry's own failure, not the original auth error: the
  6737. # pool gate below and the ladder tail's eviction check both read this narrowed
  6738. # value. An unclaimed failure (e.g. a 500) re-raised out of ``_rung`` above
  6739. # instead, since the provider-fallback rung only acts on ``_FALLBACK_REASONS``.
  6740. pool_provider = _recoverable_pool_provider(resolved_provider, client, main_runtime=route.main_runtime)
  6741. # Capture the exact key used so recovery finds the right pool entry even if another
  6742. # process rotated the pool meanwhile (current() would be None).
  6743. _client_api_key = str(getattr(client, "api_key", "") or "")
  6744. # Gate on the narrowed error: a connection failure from the retry above arrives here
  6745. # unaccepted on purpose (a fresh key cannot fix an unreachable endpoint), so rotation
  6746. # is skipped and ``first_err`` is handed to the provider-fallback chain as-is.
  6747. if pool_provider and _credential_rung_accepts(first_err):
  6748. recovery_err = first_err
  6749. # No backoff retry for a payment/quota error: the same exhausted key stays refused.
  6750. if _is_rate_limit_error(first_err) and not _is_payment_error(first_err):
  6751. from agent.auxiliary_rate_limit_backoff import rate_limit_retry_rungs
  6752. resp, recovery_err = yield from rate_limit_retry_rungs(first_err, client, kwargs, f"{task or 'call'}{tag}")
  6753. if recovery_err is None:
  6754. return resp, None
  6755. if _recover_provider_pool(pool_provider, recovery_err, failed_api_key=_client_api_key):
  6756. logger.info("Auxiliary %s%s: recovered %s via credential-pool rotation after %s",
  6757. task or "call", tag, pool_provider, type(recovery_err).__name__)
  6758. try:
  6759. return (yield _LadderStep(
  6760. "retry_same_provider", (resolved_provider, route.resolved_model))), None
  6761. except Exception as retry2_err:
  6762. # Rotated key also hit a wall: mark it now so concurrent processes skip it,
  6763. # then fall through to the provider fallback.
  6764. if (_is_payment_error(retry2_err) or _is_auth_error(retry2_err)
  6765. or _is_rate_limit_error(retry2_err)):
  6766. _recover_provider_pool(pool_provider, retry2_err)
  6767. first_err = retry2_err
  6768. else:
  6769. raise
  6770. return None, first_err
  6771. def _next_fallback_after_quarantine(
  6772. task: Optional[str], resolved_provider: str, is_auto: bool, route: _LadderRoute,
  6773. failed_model: Optional[str], failure_scope: Any, *, task_chain_only: bool = False,
  6774. ) -> tuple[Optional[Any], Optional[str], str]:
  6775. """Next candidate after a fallback entry was quarantined mid-request (dead credential or a
  6776. capacity error): remaining configured entries (task chain, then main chain on auto) before the
  6777. discovery chain. ``task_chain_only`` (explicit-provider auth error) stops at the task chain —
  6778. the user never opted that task into discovery or the main model."""
  6779. reason = "fallback candidate unavailable"
  6780. fb = _try_configured_fallback_chain(
  6781. task, resolved_provider or "auto", reason=reason, failed_model=failed_model,
  6782. failed_base_url=route.base_info, failure_scope=failure_scope)
  6783. if task_chain_only:
  6784. return fb
  6785. if fb[0] is None and is_auto:
  6786. fb = _try_main_fallback_chain(
  6787. task, resolved_provider or "auto", reason=reason, failed_model=failed_model,
  6788. failed_base_url=route.base_info, failure_scope=failure_scope)
  6789. if fb[0] is None:
  6790. fb = _try_payment_fallback(
  6791. resolved_provider, task, reason=reason, failed_base_url=route.base_info,
  6792. failure_scope=failure_scope, main_runtime=route.main_runtime)
  6793. return fb
  6794. def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):
  6795. """Last rung: other providers (per-task chain; then auto: main fallback chain + discovery
  6796. chain, explicit: main-agent-model net). Returns the response or None.
  6797. Capacity errors (payment/quota, connection, exhausted 429, model incompatible, malformed
  6798. response) bypass the explicit-provider gate — the provider cannot serve this request
  6799. regardless of user intent. Auth errors from an explicit provider may only use the task's
  6800. own configured fallback_chain; they never imply an unconfigured provider hop."""
  6801. task, tag, resolved_provider = route.task, route.tag, route.resolved_provider
  6802. # Respect explicit provider choice for transient errors (auth, request validation, etc.) but fall back when
  6803. # the provider cannot serve due to capacity: payment/quota exhaustion (a daily-token 429 like a 402, #26803),
  6804. # connection failures, and a 429 after retries are exhausted (#52228) are not request constraints.
  6805. is_auto = resolved_provider in {"auto", "", None}
  6806. reason = next((label for predicate, label in _FALLBACK_REASONS if predicate(first_err)), None)
  6807. is_capacity_error = any(
  6808. predicate(first_err) for predicate, label in _FALLBACK_REASONS if label != "auth error")
  6809. task_chain = _get_auxiliary_task_config(task).get("fallback_chain") if task else None
  6810. has_task_fallback_chain = isinstance(task_chain, list) and bool(task_chain)
  6811. explicit_auth_with_task_chain = (
  6812. reason == "auth error" and not is_auto and has_task_fallback_chain
  6813. )
  6814. if reason is None or not (is_auto or is_capacity_error or explicit_auth_with_task_chain):
  6815. return None
  6816. if reason == "payment error":
  6817. # Mark the concrete backend (not the "auto" label) unhealthy so later aux calls skip
  6818. # it instead of paying another doomed RTT.
  6819. _mark_provider_unhealthy(
  6820. _recoverable_pool_provider(resolved_provider, route.client, main_runtime=route.main_runtime)
  6821. or resolved_provider, base_url=route.base_info)
  6822. if reason == "request timed out":
  6823. # WARNING, naming the endpoint, the budget and the knob: the only other trace of a slow
  6824. # local model is the fallback provider's complaint about a model it never had (#89445).
  6825. stalled = "Auxiliary chat stream" in str(first_err)
  6826. logger.warning("Auxiliary %s%s: request to %s %s (raise auxiliary.%s.%s for slow or reasoning "
  6827. "models) on %s, trying fallback", task or "call", tag, route.base_info or resolved_provider,
  6828. first_err if stalled else f"timed out after {route.timeout}s", task or "call",
  6829. "no_progress_timeout" if stalled else "timeout", resolved_provider)
  6830. else:
  6831. logger.info("Auxiliary %s%s: %s on %s (%s), trying fallback",
  6832. task or "call", tag, reason, resolved_provider, first_err)
  6833. # Skip only the failed model for model-specific failures; 401/402 are provider-wide, so
  6834. # auth keeps skipping the credential surface, while billing is scoped to the endpoint:
  6835. # separate custom URLs can carry separate credentials (or no billing relationship at all).
  6836. _chain_failed_model = None if reason in ("auth error", "payment error") else route.final_model
  6837. from agent.backend_identity import FailureScope
  6838. _chain_failure_scope = (
  6839. FailureScope.ENDPOINT
  6840. if reason == "payment error" and _custom_health_base_url(resolved_provider, route.base_info)
  6841. else None
  6842. )
  6843. fb_client, fb_model, fb_label = _try_configured_fallback_chain(
  6844. task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
  6845. failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
  6846. if fb_client is None and is_auto:
  6847. fb_client, fb_model, fb_label = _try_main_fallback_chain(
  6848. task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
  6849. failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
  6850. if fb_client is None:
  6851. fb_client, fb_model, fb_label = _try_payment_fallback(
  6852. resolved_provider, task, reason=reason, failed_base_url=route.base_info,
  6853. failure_scope=_chain_failure_scope, main_runtime=route.main_runtime)
  6854. elif fb_client is None and not explicit_auth_with_task_chain:
  6855. fb_client, fb_model, fb_label = _try_main_agent_model_fallback(
  6856. resolved_provider, task, reason=reason, failed_model=_chain_failed_model,
  6857. failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
  6858. # Ordered walk: a candidate that returns None was quarantined (dead credential or a capacity
  6859. # error such as a quota 429) and is now unhealthy, so re-walking the CONFIGURED chains first
  6860. # lands on the next entry, then discovery where the selection policy allows it (#106367).
  6861. # Bounded: each pass quarantines its candidate and a re-selected lane ends the walk (≤1 try each).
  6862. # Candidates overwrite route_info; on exhaustion the PRIMARY error is raised, so restore its route.
  6863. tried_lanes: set = set()
  6864. primary_route = dict(route.route_info) if route.route_info is not None else None
  6865. while fb_client is not None:
  6866. lane = (fb_label, fb_model, str(getattr(fb_client, "base_url", "") or ""))
  6867. if lane in tried_lanes:
  6868. break
  6869. tried_lanes.add(lane)
  6870. _record_route_info(route.route_info, _fallback_provider_from_label(fb_label), fb_model)
  6871. fb_resp = yield _LadderStep("fallback", (fb_client, fb_model, fb_label))
  6872. if fb_resp is not None:
  6873. if route.route_info is not None: # tell the caller its own route failed, not only where it landed
  6874. route.route_info.update(failed_model=route.final_model or "", failure=f"{reason}: {first_err}",
  6875. fallback_label=fb_label)
  6876. return fb_resp
  6877. fb_client, fb_model, fb_label = _next_fallback_after_quarantine(
  6878. task, resolved_provider, is_auto, route, _chain_failed_model, _chain_failure_scope,
  6879. task_chain_only=explicit_auth_with_task_chain)
  6880. # All fallback layers exhausted: one user-visible warning (#26882), then the caller re-raises.
  6881. logger.warning("Auxiliary %s%s: %s on %s and all fallbacks exhausted "
  6882. "(fallback_chain + main agent model). Raising the primary error.",
  6883. task or "call", tag, reason, resolved_provider)
  6884. if primary_route is not None and route.route_info is not None:
  6885. route.route_info.update(primary_route)
  6886. return None
  6887. def _aux_recovery_ladder(
  6888. first_err: Exception, *, client: Any, kwargs: dict[str, Any], task: Optional[str],
  6889. async_mode: bool, base_info: str, resolved_provider: str, resolved_model: Optional[str],
  6890. resolved_base_url: Optional[str], resolved_api_key: Optional[str],
  6891. resolved_api_mode: Optional[str], final_model: Optional[str], max_tokens: Optional[int],
  6892. main_runtime: Optional[dict[str, Any]], route_info: Optional[dict[str, str]],
  6893. ):
  6894. """Ordered recovery rungs after the primary request failed (generator): parameter
  6895. strips → Nous heal/refresh → credential refresh/pool rotation → provider fallback.
  6896. Each rung returns a response, narrows ``first_err`` and falls through, or re-raises.
  6897. Raises the narrowed ``first_err`` when exhausted (after evicting a connection-poisoned client)."""
  6898. tag = " (async)" if async_mode else ""
  6899. route = _LadderRoute(
  6900. client, task, tag, async_mode, base_info, resolved_provider, resolved_model,
  6901. resolved_base_url, resolved_api_key, resolved_api_mode, final_model, main_runtime, route_info,
  6902. kwargs.get("timeout"))
  6903. resp, first_err, kwargs = yield from _ladder_parameter_rungs(first_err, route, kwargs, max_tokens)
  6904. if first_err is None:
  6905. return resp
  6906. client_is_nous = (resolved_provider == "nous"
  6907. or base_url_host_matches(base_info, "inference-api.nousresearch.com"))
  6908. resp, first_err = yield from _ladder_nous_rungs(first_err, route, kwargs, client_is_nous)
  6909. if first_err is None:
  6910. return resp
  6911. resp, first_err = yield from _ladder_credential_rungs(first_err, route, kwargs, client_is_nous)
  6912. if first_err is None:
  6913. return resp
  6914. resp = yield from _ladder_provider_fallback(first_err, route)
  6915. if resp is not None:
  6916. return resp
  6917. # Connection/timeout errors poison the cached client (closed transport, half-read
  6918. # stream); evict so the next aux call rebuilds a fresh one.
  6919. # Reached only when no fallback answered, so the next auxiliary call rebuilds a fresh
  6920. # client instead of reusing the dead one. ``first_err`` is the narrowed error from the
  6921. # rungs above, not necessarily the original one. See issue #23432.
  6922. # Mirror the sync path: drop poisoned clients on connection/timeout so the next aux call rebuilds. See
  6923. # issue #23432.
  6924. if _is_connection_error(first_err):
  6925. try:
  6926. _evict_cached_client_instance(client)
  6927. except Exception:
  6928. logger.debug("Auxiliary%s: cache eviction after connection error failed",
  6929. tag, exc_info=True)
  6930. # The narrowed error is the actionable one (e.g. a 404 "requires credits" from the
  6931. # retry after a healed 401), so surface it rather than the original.
  6932. raise first_err
  6933. def _drive_ladder(ladder, perform: Callable[[_LadderStep], Any]) -> Any:
  6934. """Run a ladder generator, feeding each step's result (or exception) back in; a ``wait`` step sleeps."""
  6935. try:
  6936. step = next(ladder)
  6937. while True:
  6938. try:
  6939. result = time.sleep(step.args[0]) if step.kind == "wait" else perform(step)
  6940. except Exception as exc:
  6941. step = ladder.throw(exc)
  6942. else:
  6943. step = ladder.send(result)
  6944. except StopIteration as stop:
  6945. return stop.value
  6946. async def _drive_ladder_async(ladder, perform: Callable[[_LadderStep], Any]) -> Any:
  6947. """Async twin of :func:`_drive_ladder` (``perform`` is awaited)."""
  6948. import asyncio
  6949. try:
  6950. step = next(ladder)
  6951. while True:
  6952. try:
  6953. result = await (asyncio.sleep(step.args[0]) if step.kind == "wait" else perform(step))
  6954. except Exception as exc:
  6955. step = ladder.throw(exc)
  6956. else:
  6957. step = ladder.send(result)
  6958. except StopIteration as stop:
  6959. return stop.value
  6960. def _elapsed_ms(started_at: float, now: Optional[float] = None) -> int:
  6961. """Whole milliseconds since ``started_at`` (clamped at 0)."""
  6962. return max(0, int(((time.monotonic() if now is None else now) - started_at) * 1000))
  6963. def _stamp_latency_once(latency_info: Optional[dict[str, int]], key: str, started_at: float) -> None:
  6964. """Record ``key`` in ``latency_info`` the first time it fires."""
  6965. if latency_info is not None and key not in latency_info:
  6966. latency_info[key] = _elapsed_ms(started_at)
  6967. @_relay_auxiliary_call
  6968. def call_llm(
  6969. task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
  6970. api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
  6971. temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
  6972. timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
  6973. extra_headers: Optional[dict[str, str]] = None, api_mode: str | None = None, stream: bool = False,
  6974. stream_options: dict | None = None, route_info: Optional[dict[str, str]] = None,
  6975. latency_info: Optional[dict[str, int]] = None,
  6976. ) -> Any:
  6977. """Run an auxiliary LLM request, applying the configured task limit."""
  6978. queue_started_at = time.monotonic()
  6979. semaphore = _acquire_sync_aux_semaphore(task)
  6980. if semaphore is not None:
  6981. semaphore.acquire()
  6982. request_started_at = time.monotonic()
  6983. if latency_info is not None:
  6984. latency_info["queue_wait_ms"] = _elapsed_ms(queue_started_at, request_started_at)
  6985. prior_progress_hook = getattr(_aux_progress, "hook", None)
  6986. try:
  6987. with (
  6988. scoped_runtime_main(main_runtime),
  6989. aux_progress_hook(
  6990. prior_progress_hook
  6991. if callable(prior_progress_hook)
  6992. else ((lambda: None) if latency_info is not None else None)
  6993. ),
  6994. _aux_thread_local_hook(_aux_dispatch, functools.partial(
  6995. _stamp_latency_once, latency_info, "provider_dispatch_ms", request_started_at)),
  6996. _aux_thread_local_hook(_aux_provider_response, functools.partial(
  6997. _stamp_latency_once, latency_info, "time_to_first_progress_ms", request_started_at)),
  6998. ):
  6999. response = _call_llm_impl(
  7000. task=task, provider=provider, model=model, base_url=base_url, api_key=api_key,
  7001. main_runtime=main_runtime, messages=messages, temperature=temperature,
  7002. max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
  7003. reasoning_config=reasoning_config, extra_headers=extra_headers, api_mode=api_mode,
  7004. stream=stream, stream_options=stream_options, route_info=route_info,
  7005. )
  7006. if stream and semaphore is not None:
  7007. stream_semaphore = semaphore
  7008. semaphore = None
  7009. return _release_sync_semaphore_after_stream(response, stream_semaphore)
  7010. return response
  7011. finally:
  7012. if latency_info is not None:
  7013. latency_info["summary_generation_ms"] = _elapsed_ms(request_started_at)
  7014. if semaphore is not None:
  7015. semaphore.release()
  7016. def _release_sync_semaphore_after_stream(stream: Any, semaphore: threading.BoundedSemaphore):
  7017. """Release a permit only after a streaming response is consumed or closed."""
  7018. try:
  7019. yield from stream
  7020. finally:
  7021. try:
  7022. close = getattr(stream, "close", None)
  7023. if callable(close):
  7024. close()
  7025. finally:
  7026. semaphore.release()
  7027. def _plan_aux_call(
  7028. task: Optional[str], *, async_mode: bool, provider: Optional[str], model: Optional[str],
  7029. base_url: Optional[str], api_key: Optional[str], main_runtime: Optional[dict[str, Any]],
  7030. messages: list, temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
  7031. timeout: Optional[float], extra_body: Optional[dict], reasoning_config: Optional[dict],
  7032. extra_headers: Optional[dict[str, str]], api_mode: Optional[str],
  7033. route_info: Optional[dict[str, str]],
  7034. ) -> tuple[_PreparedAuxRequest, dict[str, Any], dict[str, Any]]:
  7035. """Shared head of both call impls: prepare the request and bundle the kwargs the recovery
  7036. drivers pass to ``_retry_same_provider_*`` / ``_call_fallback_candidate_*``. One immutable
  7037. runtime snapshot for keying/resolution/retries/fallbacks, so a concurrent /model switch
  7038. can't mix key and client from different runtimes."""
  7039. main_runtime = _normalize_main_runtime(main_runtime)
  7040. req = _prepare_aux_request(
  7041. task, provider=provider, model=model, base_url=base_url, api_key=api_key,
  7042. main_runtime=main_runtime, messages=messages, temperature=temperature,
  7043. max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
  7044. reasoning_config=reasoning_config, extra_headers=extra_headers,
  7045. api_mode=api_mode, route_info=route_info, async_mode=async_mode,
  7046. )
  7047. candidate_kwargs = dict(
  7048. task=task, messages=messages, temperature=temperature, max_tokens=max_tokens,
  7049. tools=tools, effective_timeout=req.effective_timeout,
  7050. effective_extra_body=req.effective_extra_body, reasoning_config=reasoning_config,
  7051. )
  7052. retry_kwargs = dict(
  7053. candidate_kwargs, resolved_base_url=req.resolved_base_url,
  7054. resolved_api_key=req.resolved_api_key, resolved_api_mode=req.resolved_api_mode,
  7055. main_runtime=main_runtime, final_model=req.final_model, extra_headers=extra_headers,
  7056. )
  7057. return req, retry_kwargs, candidate_kwargs
  7058. def _should_retry_same_provider(task: Optional[str], exc: Exception, tag: str) -> bool:
  7059. """True when ``exc`` is a transient transport blip worth a same-provider retry; critical-path
  7060. tasks skip it on a full-budget timeout (``_should_skip_same_provider_retry``) and go straight
  7061. to fallback."""
  7062. if not _is_transient_transport_error(exc):
  7063. return False
  7064. if _should_skip_same_provider_retry(task, exc):
  7065. logger.info("Auxiliary %s%s: timeout on the critical path; "
  7066. "skipping same-provider retry and falling back: %s", task, tag, exc)
  7067. return False
  7068. return True
  7069. def _ladder_step_call(
  7070. step: _LadderStep, req: _PreparedAuxRequest, retry_kwargs: dict[str, Any], candidate_kwargs: dict[str, Any],
  7071. ) -> tuple[str, tuple, dict[str, Any]]:
  7072. """Resolve a ladder step into ``(kind, args, kwargs)`` for the sync/async performer."""
  7073. if step.kind == "call":
  7074. return "call", step.args, dict(provider=req.resolved_provider, api_mode=req.resolved_api_mode)
  7075. if step.kind == "retry_same_provider":
  7076. retry_provider, retry_model = step.args
  7077. return "retry", (), dict(retry_kwargs, resolved_provider=retry_provider, resolved_model=retry_model)
  7078. return "fallback", step.args, candidate_kwargs
  7079. def _start_recovery_ladder(
  7080. first_err: Exception, req: _PreparedAuxRequest, retry_kwargs: dict[str, Any], *,
  7081. task: Optional[str], async_mode: bool, route_info: Optional[dict[str, str]],
  7082. ):
  7083. """Build the recovery-ladder generator for a failed primary request."""
  7084. return _aux_recovery_ladder(
  7085. first_err, client=req.client, kwargs=req.kwargs, task=task, async_mode=async_mode,
  7086. base_info=req.base_info, resolved_provider=req.resolved_provider,
  7087. resolved_model=req.resolved_model, resolved_base_url=req.resolved_base_url,
  7088. resolved_api_key=req.resolved_api_key, resolved_api_mode=req.resolved_api_mode,
  7089. final_model=req.final_model, max_tokens=retry_kwargs["max_tokens"],
  7090. main_runtime=retry_kwargs["main_runtime"], route_info=route_info)
  7091. def _call_llm_impl(
  7092. task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
  7093. api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
  7094. temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
  7095. timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
  7096. extra_headers: Optional[dict[str, str]] = None, api_mode: str | None = None, stream: bool = False,
  7097. stream_options: dict | None = None, route_info: Optional[dict[str, str]] = None,
  7098. ) -> Any:
  7099. """Centralized synchronous LLM call: resolve provider/model, auth, kwargs, fallbacks.
  7100. task: aux task whose provider:model comes from config (ignored if provider set); api_mode
  7101. overrides task config; timeout=None reads auxiliary.{task}.timeout; extra_headers override
  7102. client defaults. stream=True returns the raw SDK stream (caller consumes/falls back)
  7103. instead of a validated response. RuntimeError if no provider is configured."""
  7104. req, retry_kwargs, candidate_kwargs = _plan_aux_call(
  7105. task, async_mode=False, provider=provider, model=model, base_url=base_url,
  7106. api_key=api_key, main_runtime=main_runtime, messages=messages,
  7107. temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=timeout,
  7108. extra_body=extra_body, reasoning_config=reasoning_config,
  7109. extra_headers=extra_headers, api_mode=api_mode, route_info=route_info,
  7110. )
  7111. client, kwargs, request_provider = req.client, req.kwargs, req.request_provider
  7112. # Streaming path (MoA aggregator): return the raw SDK stream, skipping validation and
  7113. # the fallback chain (they assume a complete response); the caller owns reassembly/fallback.
  7114. if stream:
  7115. kwargs["stream"] = True
  7116. if stream_options:
  7117. kwargs["stream_options"] = stream_options
  7118. if task == "moa_aggregator" and isinstance(client, CodexAuxiliaryClient):
  7119. # Responses-shim clients consume the stream internally and return a completed
  7120. # object Relay's managed stream would iterate; the MoA facade wraps it as one chunk.
  7121. return client.chat.completions.create(**kwargs)
  7122. return _relay_sync_stream(client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode)
  7123. def _primary(**validate_kw: Any) -> Any:
  7124. # Retry on the same provider for a transient transport blip (connection reset / streaming-close /
  7125. # incomplete chunked read / 5xx / 408) before the except-chain below escalates to provider/model
  7126. # fallback. A dropped connection shouldn't abandon an otherwise-healthy provider — this especially
  7127. # matters for pinned auxiliary calls like MoA reference advisors, where "fallback to another
  7128. # provider" is not a meaningful recovery (the advisor is a specific model), so a transient blip that
  7129. # isn't retried simply loses that advisor for the turn (root of the run2 double-advisor "Connection
  7130. # error" collapse — a genuine upstream blip hitting both parallel advisors at once). Attempts are
  7131. # bounded and use exponential backoff. Count is configurable via auxiliary.transient_retries
  7132. # (default 2 retries → 3 total attempts); a second/third failure or any non-transient error falls
  7133. # through to ``first_err`` and the existing fallback handling unchanged. Unified home for the
  7134. # transient retry every auxiliary task shares. (PR #16587)
  7135. return _validate_llm_response(
  7136. _relay_sync_completion(
  7137. client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode,
  7138. create=lambda request: _create_with_progress(
  7139. client, request, task,
  7140. force_stream=_provider_requires_stream(
  7141. request_provider, req.base_info or req.resolved_base_url),
  7142. ),
  7143. ),
  7144. task, **validate_kw,
  7145. )
  7146. try:
  7147. # Bounded same-provider retry (exponential backoff, auxiliary.transient_retries) for
  7148. # transient blips before escalating to fallback — a dropped connection shouldn't
  7149. # abandon a healthy provider (matters for pinned MoA advisors).
  7150. try:
  7151. return _primary(provider=request_provider, base_url=req.base_info)
  7152. except Exception as transient_err:
  7153. if not _should_retry_same_provider(task, transient_err, ""):
  7154. raise
  7155. _max_transient_retries = _transient_retry_count()
  7156. _last_transient = transient_err
  7157. for _attempt in range(1, _max_transient_retries + 1):
  7158. _backoff = min(_TRANSIENT_RETRY_BACKOFF_BASE * (2.0 ** (_attempt - 1)), 8.0)
  7159. logger.info("Auxiliary %s: transient transport error (attempt %d/%d); "
  7160. "retrying same provider after %.1fs before fallback: %s",
  7161. task or "call", _attempt, _max_transient_retries, _backoff, _last_transient)
  7162. time.sleep(_backoff)
  7163. try:
  7164. return _primary()
  7165. except Exception as retry_transient:
  7166. if not _is_transient_transport_error(retry_transient):
  7167. raise
  7168. _last_transient = retry_transient
  7169. raise _last_transient
  7170. except Exception as first_err:
  7171. def _perform(step: _LadderStep) -> Any:
  7172. kind, args, kw = _ladder_step_call(step, req, retry_kwargs, candidate_kwargs)
  7173. if kind == "call":
  7174. return _validate_llm_response(_relay_sync_completion(*args, **kw), task)
  7175. if kind == "retry":
  7176. return _retry_same_provider_sync(**kw)
  7177. return _call_fallback_candidate_sync(*args, **kw)
  7178. return _drive_ladder(
  7179. _start_recovery_ladder(first_err, req, retry_kwargs, task=task, async_mode=False, route_info=route_info),
  7180. _perform)
  7181. def _coerce_llm_message(response):
  7182. """Pull a message (dict, object, or str) out of a response-or-message value: dict-shaped
  7183. responses/bare messages (compression, proxies) and ChatCompletion objects; MagicMock
  7184. ``reasoning_*`` attrs are deliberately not strings."""
  7185. if response is None or isinstance(response, str):
  7186. return response
  7187. if isinstance(response, dict):
  7188. if "choices" not in response:
  7189. return response
  7190. choices = response.get("choices") or []
  7191. else:
  7192. choices = getattr(response, "choices", None)
  7193. if not choices:
  7194. return response
  7195. return _message_field(choices[0], "message") if choices else None
  7196. def _message_field(msg, name):
  7197. return msg.get(name) if isinstance(msg, dict) else getattr(msg, name, None)
  7198. def extract_content_or_reasoning(response, *, max_reasoning_chars: int | None = None) -> str:
  7199. """Extract content from an LLM response, falling back to reasoning fields.
  7200. Order: ``content`` (inline think blocks stripped) → ``reasoning``/``reasoning_content`` →
  7201. ``reasoning_details`` (OpenRouter array). Accepts a response or bare message;
  7202. ``max_reasoning_chars`` bounds a reasoning fallback so unbounded chain-of-thought can't
  7203. become the compaction summary. Returns ``""`` if nothing found."""
  7204. msg = _coerce_llm_message(response)
  7205. if msg is None:
  7206. return ""
  7207. if isinstance(msg, str):
  7208. return msg.strip()
  7209. raw = _message_field(msg, "content")
  7210. if not isinstance(raw, str):
  7211. raw = str(raw) if raw else ""
  7212. content = raw.strip()
  7213. if content:
  7214. # Same precompiled closed-pair patterns as strip_think_blocks.
  7215. from agent.agent_runtime_helpers import _REASONING_BLOCK_PATTERNS
  7216. cleaned = content
  7217. for pattern in _REASONING_BLOCK_PATTERNS:
  7218. cleaned = pattern.sub("", cleaned)
  7219. cleaned = cleaned.strip()
  7220. if cleaned:
  7221. return cleaned
  7222. # Content is empty or reasoning-only — try structured reasoning fields
  7223. reasoning_parts: list[str] = []
  7224. for field in ("reasoning", "reasoning_content"):
  7225. val = _message_field(msg, field)
  7226. if val and isinstance(val, str) and val.strip() and val not in reasoning_parts:
  7227. reasoning_parts.append(val.strip())
  7228. details = _message_field(msg, "reasoning_details")
  7229. if details and isinstance(details, list):
  7230. for detail in details:
  7231. if isinstance(detail, dict):
  7232. summary = detail.get("summary") or detail.get("content") or detail.get("text")
  7233. if summary and summary not in reasoning_parts:
  7234. reasoning_parts.append(summary.strip() if isinstance(summary, str) else str(summary))
  7235. if not reasoning_parts:
  7236. return ""
  7237. text = "\n\n".join(reasoning_parts)
  7238. if max_reasoning_chars is not None and len(text) > max_reasoning_chars:
  7239. logger.warning("fell back to reasoning fields (%d chars); truncating to %d",
  7240. len(text), max_reasoning_chars)
  7241. return text[:max_reasoning_chars]
  7242. return text
  7243. @_relay_auxiliary_call_async
  7244. async def async_call_llm(
  7245. task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
  7246. api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
  7247. temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
  7248. timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
  7249. route_info: Optional[dict[str, str]] = None,
  7250. ) -> Any:
  7251. """Run an asynchronous auxiliary LLM request under the configured limit."""
  7252. semaphore = _acquire_async_aux_semaphore(task)
  7253. if semaphore is not None:
  7254. await semaphore.acquire()
  7255. try:
  7256. with scoped_runtime_main(main_runtime):
  7257. return await _async_call_llm_impl(
  7258. task=task, provider=provider, model=model, base_url=base_url, api_key=api_key,
  7259. main_runtime=main_runtime, messages=messages, temperature=temperature,
  7260. max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
  7261. reasoning_config=reasoning_config, route_info=route_info,
  7262. )
  7263. finally:
  7264. if semaphore is not None:
  7265. semaphore.release()
  7266. async def _async_call_llm_impl(
  7267. task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
  7268. api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
  7269. temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
  7270. timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
  7271. route_info: Optional[dict[str, str]] = None,
  7272. ) -> Any:
  7273. """Centralized asynchronous LLM call; see call_llm() for full documentation.
  7274. No per-request header / api_mode override on the async entry point."""
  7275. req, retry_kwargs, candidate_kwargs = _plan_aux_call(
  7276. task, async_mode=True, provider=provider, model=model, base_url=base_url,
  7277. api_key=api_key, main_runtime=main_runtime, messages=messages,
  7278. temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=timeout,
  7279. extra_body=extra_body, reasoning_config=reasoning_config,
  7280. extra_headers=None, api_mode=None, route_info=route_info,
  7281. )
  7282. client, kwargs, request_provider = req.client, req.kwargs, req.request_provider
  7283. try:
  7284. # Retry ONCE on the same provider for a transient blip before fallback (see call_llm()).
  7285. # (PR #16587)
  7286. _force_stream_async = _provider_requires_stream(request_provider, req.base_info or req.resolved_base_url)
  7287. async def _acreate(_kwargs: dict[str, Any]) -> Any:
  7288. return await _acreate_with_progress(client, _kwargs, task, force_stream=_force_stream_async)
  7289. async def _primary(**validate_kw: Any) -> Any:
  7290. import asyncio
  7291. return await asyncio.to_thread(
  7292. _validate_llm_response,
  7293. await _relay_async_completion(
  7294. client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode,
  7295. create=_acreate),
  7296. task, **validate_kw)
  7297. try:
  7298. return await _primary(provider=request_provider, base_url=req.base_info)
  7299. except Exception as transient_err:
  7300. # The async Codex adapter wraps the sync stream via to_thread: same TimeoutError here.
  7301. if not _should_retry_same_provider(task, transient_err, " (async)"):
  7302. raise
  7303. logger.info("Auxiliary %s (async): transient transport error; retrying "
  7304. "once on the same provider before fallback: %s", task or "call", transient_err)
  7305. return await _primary()
  7306. except Exception as first_err:
  7307. async def _perform(step: _LadderStep) -> Any:
  7308. kind, args, kw = _ladder_step_call(step, req, retry_kwargs, candidate_kwargs)
  7309. if kind == "call":
  7310. import asyncio
  7311. return await asyncio.to_thread(_validate_llm_response, await _relay_async_completion(*args, **kw), task)
  7312. if kind == "retry":
  7313. return await _retry_same_provider_async(**kw)
  7314. fb_client, fb_model, fb_label = args
  7315. fb_client, _ = _to_async_client(fb_client, fb_model or "", is_vision=(task == "vision"))
  7316. return await _call_fallback_candidate_async(fb_client, fb_model, fb_label, **kw)
  7317. return await _drive_ladder_async(
  7318. _start_recovery_ladder(first_err, req, retry_kwargs, task=task, async_mode=True, route_info=route_info),
  7319. _perform)