| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281 |
- """Shared auxiliary client router for side tasks (compression, search, vision, ...).
- Text auto chain: main provider+model → OpenRouter → Nous Portal → custom endpoint →
- native Anthropic → direct API-key providers → None. Vision auto chain: main
- provider (if a supported vision backend) → OpenRouter → Nous → Anthropic → custom.
- ``auxiliary.free_only`` restricts the OpenRouter lane to ``:free`` SKUs. Codex OAuth is
- in neither chain (undocumented, shifting allow-list): main provider or explicit
- ``auxiliary.<task>.provider`` only. HTTP 402 in call_llm() falls through the chain.
- """
- import contextlib
- import contextvars
- import functools
- import hashlib
- import inspect
- import json
- import logging
- import os
- import re
- import threading
- import time
- import uuid
- from types import SimpleNamespace
- from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple, TYPE_CHECKING, Union
- from urllib.parse import urlparse, parse_qs, urlunparse
- from agent.local_prompt_progress import (
- PromptReadingClock, current_prompt_reading_hook, prompt_progress, prompt_reading_hook, with_prompt_progress)
- from agent.error_classifier import (
- _BILLING_PATTERNS,
- _OVERLOADED_PATTERNS,
- UNSUPPORTED_PARAM_MARKERS,
- is_reasoning_field_rejection,
- is_reasoning_required_rejection,
- )
- from agent.auxiliary_error_shapes import (
- _botocore_error, _exc_http_status, _is_invalid_aux_response_error, _is_statusless_structured_provider_error,
- )
- from agent.auxiliary_managed_local import _is_managed_local_endpoint
- from agent.auxiliary_reasoning_floor import remember_reasoning_floor, with_reasoning_floor
- from agent.auxiliary_structured_output import _route_key, _without_structured_output_format, remember_structured_output_rejection
- from agent.codex_headers import (
- CODEX_AUX_BASE_URL as _CODEX_AUX_BASE_URL,
- apply_required_codex_headers as _apply_required_codex_headers,
- codex_cloudflare_headers as _codex_cloudflare_headers,
- is_official_codex_base_url as _is_official_codex_base_url,
- )
- from agent.codex_runtime import _codex_event_has_content
- from agent.sdk_transform_bypass import bypass_chat_sdk_request_transform
- # `openai.OpenAI` is imported lazily (~240 ms cold); `OpenAI` below is a proxy
- # so in-module calls, `auxiliary_client.OpenAI` reads and
- # `patch("agent.auxiliary_client.OpenAI")` all keep working.
- if TYPE_CHECKING:
- from openai import OpenAI
- _OPENAI_CLS_CACHE: Optional[type] = None
- def _load_openai_cls() -> type:
- """Import and cache ``openai.OpenAI``."""
- global _OPENAI_CLS_CACHE
- if _OPENAI_CLS_CACHE is None:
- from openai import OpenAI as _cls
- _OPENAI_CLS_CACHE = _cls
- return _OPENAI_CLS_CACHE
- class _OpenAIProxy:
- """Lazy stand-in for ``openai.OpenAI``: forwards calls and isinstance checks, importing on first use."""
- __slots__ = ()
- def __call__(self, *args, **kwargs):
- return _load_openai_cls()(*args, **kwargs)
- def __instancecheck__(self, obj):
- return isinstance(obj, _load_openai_cls())
- def __repr__(self):
- return "<lazy openai.OpenAI proxy>"
- OpenAI = _OpenAIProxy()
- # Availability probe mode: check_fns only need to know whether a client is RESOLVABLE, so
- # inside `aux_probe_mode()` constructors return a stub instead of importing openai + building
- # httpx/SSL (~0.3s on CLI startup). Stubs are never cached (see _store_cached_client).
- _aux_probe_state = threading.local()
- class _AuxProbeClientStub:
- """Non-functional placeholder returned while `aux_probe_mode` is active."""
- __slots__ = ("api_key", "base_url")
- def __init__(self, api_key: str = "", base_url: str = "") -> None:
- self.api_key = api_key
- self.base_url = base_url
- def __getattr__(self, name: str) -> Any:
- # Loud failure if a probe stub ever leaks into a runtime call path.
- raise RuntimeError(
- f"_AuxProbeClientStub used as a real client (attribute {name!r}); "
- "aux_probe_mode is for availability checks only")
- def __repr__(self) -> str:
- return "<aux availability-probe client stub>"
- def _aux_probe_active() -> bool:
- return bool(getattr(_aux_probe_state, "active", False))
- @contextlib.contextmanager
- def aux_probe_mode():
- """Resolve provider availability without constructing real SDK clients."""
- prev = getattr(_aux_probe_state, "active", False)
- _aux_probe_state.active = True
- try:
- yield
- finally:
- _aux_probe_state.active = prev
- from agent.credential_pool import load_pool
- from agent.custom_entry_pool import entry_scoped_pool, pool_for_aux_provider
- from agent.message_metadata import data_use_requirement
- from agent.model_metadata import MINIMUM_CONTEXT_LENGTH, get_model_context_length
- from hermes_cli.config import get_hermes_home
- from hermes_cli.config_providers import _canonical_api_mode
- from agent.auxiliary_health import (
- _custom_health_base_url, _unhealthy_cache_key, fallback_candidate_quarantine_ttl,
- fallback_candidate_unavailable_reason,
- )
- from agent.auxiliary_unavailable import (
- AuxiliaryClientUnavailable, clear_nous_credential_failure, missing_provider_credentials_message,
- nous_credential_failure_detail, record_nous_credential_failure)
- from hermes_constants import OPENROUTER_BASE_URL, hermes_home_key
- from utils import base_url_host_matches, base_url_hostname, base_url_origin, env_float, is_truthy_value, model_forces_max_completion_tokens, normalize_proxy_env_vars
- logger = logging.getLogger(__name__)
- # resolve_provider_client fall-through dedup: misconfigured-provider warnings fire on every
- # retry, so only the first per process surfaces. Separate sets let tests clear each branch.
- _LOGGED_UNKNOWN_PROVIDER_KEYS: set = set()
- _LOGGED_UNHANDLED_AUTHTYPE_KEYS: set = set()
- _LOGGED_UNSUPPORTED_EXTPROC_KEYS: set = set()
- _LOGGED_UNSUPPORTED_OAUTH_KEYS: set = set()
- def _resolve_aux_verify(base_url: Optional[str]) -> Any:
- """httpx ``verify`` for an aux base_url, mirroring the main client (per-provider ``ssl_ca_cert`` /
- ``ssl_verify``; otherwise the OS trust store); any failure → httpx default (``True``)."""
- try:
- from agent.ssl_verify import resolve_httpx_verify
- from hermes_cli.config import get_custom_provider_tls_settings, load_config_readonly
- tls = get_custom_provider_tls_settings(str(base_url or ""), config=load_config_readonly())
- return resolve_httpx_verify(
- ca_bundle=tls.get("ssl_ca_cert"), ssl_verify=tls.get("ssl_verify"), base_url=str(base_url or ""))
- except Exception:
- return True
- _WARNED_KEEPALIVE_IMPORT_SKEW = False
- def _openai_http_client_kwargs(base_url: Optional[str], *, async_mode: bool = False) -> dict[str, Any]:
- """Inject keepalive httpx client with env-only proxy (not macOS system proxy)."""
- try:
- from agent.process_bootstrap import build_keepalive_http_client
- client = build_keepalive_http_client(
- str(base_url or ""), async_mode=async_mode, verify=_resolve_aux_verify(base_url))
- except (ImportError, AttributeError):
- # Version-skewed install (Desktop runtime lagging a git tree) lacks this helper:
- # degrade to the SDK default httpx client rather than kill the job; warn once.
- global _WARNED_KEEPALIVE_IMPORT_SKEW
- if not _WARNED_KEEPALIVE_IMPORT_SKEW:
- _WARNED_KEEPALIVE_IMPORT_SKEW = True
- logger.warning(
- "agent.process_bootstrap.build_keepalive_http_client is "
- "unavailable — mixed/stale install detected (#64333). Falling "
- "back to the SDK default HTTP client. Run `hermes update` (or "
- "reinstall the Desktop app) to resync the runtime.")
- client = None
- return {"http_client": client} if client is not None else {}
- def _create_openai_client(*, api_key: str, base_url: str, **kwargs: Any) -> Any:
- if _aux_probe_active():
- # Availability probe: resolved credentials/base_url are the answer.
- return _AuxProbeClientStub(api_key=api_key, base_url=base_url)
- kwargs = {**_openai_http_client_kwargs(base_url), **kwargs}
- _apply_required_codex_headers(kwargs, access_token=api_key, base_url=base_url)
- # Hermes owns aux retry/fallback policy; the SDK default (max_retries=2) would triple
- # wall time on a hung endpoint before Hermes sees one failure.
- # Hermes owns auxiliary retry + provider/model fallback policy (the same-provider transient retry in
- # call_llm plus the except-chain fallback). The OpenAI SDK's own default (max_retries=2 → up to 3
- # attempts) silently multiplies the effective wall time of every aux call by 3× on a slow/hung endpoint,
- # so a 120s timeout can stall ~360s before Hermes sees a single failure (issue #54465). Disable
- # SDK-internal retries by default and let Hermes control the budget; explicit callers can still override
- # via kwargs.
- kwargs.setdefault("max_retries", 0)
- return OpenAI(api_key=api_key, base_url=base_url, **kwargs)
- # Interrupt protection for atomic aux tasks: a compression summary killed by an ordinary
- # gateway interrupt degrades to a static marker, so a thread-local flag marks such calls
- # protected. Explicit host cancel (Ctrl+C, /stop) still overrides it, timeouts still fire.
- # ── Interrupt protection for atomic auxiliary tasks ────────────────────── Some auxiliary tasks must NOT be
- # aborted mid-flight by a gateway interrupt (e.g. an incoming user message while the agent is busy). Context
- # compression is the prime case: if the summary LLM call is interrupted part-way, compression falls back to
- # a static "summary unavailable" marker and the real handoff is lost (#23975). A thread-local flag lets such
- # a task mark its in-flight LLM call as interrupt-protected; the Codex Responses stream's cancellation check
- # honors it. TIMEOUTS still fire (a hung call must die), and all OTHER aux tasks (vision, web_extract,
- # title_generation, …) remain freely interruptible.
- _aux_interrupt_protection = threading.local()
- class AuxiliaryExplicitCancellation(BaseException):
- """Frozen signal that an auxiliary attempt was explicitly hard-cancelled. ``BaseException`` so broad
- ``except Exception`` retry/fallback code never treats a host stop as a transport failure; ``cause``
- is immutable class data so nothing re-queries a mutable host Event after the transport unwound."""
- cause = "explicit_host_cancel"
- def __init__(self) -> None:
- super().__init__("auxiliary request explicitly cancelled by host")
- def _aux_interrupt_protected() -> bool:
- return bool(getattr(_aux_interrupt_protection, "active", False))
- def _aux_interrupt_cancel_requested() -> bool:
- """Return whether an explicit host cancel overrides aux protection."""
- check = _capture_aux_cancel_check()
- return _captured_aux_cancel_requested(check) if check is not None else False
- @contextlib.contextmanager
- def aux_interrupt_protection(active: bool = True, cancel_check=None, cancel_event=None):
- """Mark this thread's aux LLM call interrupt-protected (re-entrant-safe). ``cancel_check`` /
- ``cancel_event`` keep an explicit host hard-cancel path (Event preferred); nested scopes inherit both."""
- prev = getattr(_aux_interrupt_protection, "active", False)
- prev_cancel_check = getattr(_aux_interrupt_protection, "cancel_check", None)
- prev_cancel_event = getattr(_aux_interrupt_protection, "cancel_event", None)
- _aux_interrupt_protection.active = active
- if callable(cancel_check):
- _aux_interrupt_protection.cancel_check = cancel_check
- if cancel_event is not None and callable(getattr(cancel_event, "is_set", None)):
- _aux_interrupt_protection.cancel_event = cancel_event
- try:
- yield
- finally:
- _aux_interrupt_protection.active = prev
- _aux_interrupt_protection.cancel_check = prev_cancel_check
- _aux_interrupt_protection.cancel_event = prev_cancel_event
- def _capture_aux_cancel_check() -> Optional[Callable[[], Any]]:
- """Capture the current explicit-cancel source on the owning request thread."""
- is_set = getattr(getattr(_aux_interrupt_protection, "cancel_event", None), "is_set", None)
- if callable(is_set):
- return is_set
- # Return the callable itself so attempt-local decision objects keep begin_timeout_cleanup().
- check = getattr(_aux_interrupt_protection, "cancel_check", None)
- return check if callable(check) else None
- def _captured_aux_cancel_requested(cancel_check: Callable[[], Any]) -> bool:
- """Read a request-thread cancellation source without leaking its failures."""
- try:
- return bool(cancel_check())
- except Exception:
- logger.debug("captured aux cancel check failed", exc_info=True)
- return False
- class _AuxiliaryCancellationDecision:
- """Atomically choose explicit cancellation or provider timeout per attempt."""
- def __init__(self, source_cancel_check: Callable[[], Any]) -> None:
- self._source_cancel_check = source_cancel_check
- self._lock = threading.Lock()
- self._outcome = "active"
- def __call__(self) -> bool:
- with self._lock:
- if self._outcome == "active" and _captured_aux_cancel_requested(self._source_cancel_check):
- self._outcome = "cancelled"
- return self._outcome == "cancelled"
- def begin_timeout_cleanup(self) -> bool:
- """Return whether timeout won and destructive cleanup is permitted."""
- with self._lock:
- if self._outcome == "active":
- cancelled = _captured_aux_cancel_requested(self._source_cancel_check)
- self._outcome = "cancelled" if cancelled else "timed_out"
- return self._outcome == "timed_out"
- # Forward-progress hooks for streamed aux calls: a fixed host deadline kills a SLOW model
- # streaming a big summary as hard as a HUNG one, so wire consumers tick the progress hook only
- # for non-empty payloads and the host extends its deadline while tokens move. Thread-local:
- # the call and its stream consumption run on the installing thread.
- _aux_progress = threading.local()
- _aux_dispatch = threading.local()
- _aux_provider_response = threading.local()
- # Absolute monotonic deadline of the waiting HOST. The stream's own ceiling
- # (_aux_stream_total_ceiling, >= the host's and started later) would otherwise leave an
- # orphaned stream still billing after every host-ceiling timeout.
- # Absolute wall-clock deadline (time.monotonic) of the HOST waiting for this auxiliary call, when it has one
- # (#99692). Liveness alone is not enough: a host also stops waiting at its own total ceiling, and the
- # streamed consumer below bounds itself only by _aux_stream_total_ceiling() — a budget derived from the aux
- # request timeout, which is >= the host ceiling for every configured value AND starts counting later. So the
- # stream that outlives its abandoned host is not an edge case; it is the guaranteed outcome of every
- # total-ceiling timeout.
- _aux_stream_deadline = threading.local()
- def _tick_hook(local: threading.local, label: str) -> None:
- """Call the thread-local hook installed on ``local``, if any. Never raises."""
- hook = getattr(local, "hook", None)
- if hook is None:
- return
- try:
- hook()
- except Exception:
- logger.debug("aux %s hook failed", label, exc_info=True)
- def _notify_aux_progress() -> None:
- """Tick the installed forward-progress hook, if any."""
- _tick_hook(_aux_progress, "progress")
- def _notify_aux_dispatch() -> None:
- """Record an actual provider dispatch without claiming response progress."""
- _tick_hook(_aux_dispatch, "dispatch")
- def _notify_aux_timing_response() -> None:
- """Record a content-free frame (keepalive/empty delta): counts toward
- ``time_to_first_progress_ms`` but must not reset a compression inactivity fence."""
- _tick_hook(_aux_provider_response, "provider response")
- def _notify_aux_provider_response() -> None:
- """Record a provider response/chunk, then preserve the liveness signal."""
- _notify_aux_timing_response()
- _notify_aux_progress()
- def _aux_progress_active() -> bool:
- return getattr(_aux_progress, "hook", None) is not None
- def _field(obj: Any, key: str, default: Any = None) -> Any:
- """Field access for wire objects that may be dicts or SDK/SimpleNamespace objects."""
- val = obj.get(key) if isinstance(obj, dict) else getattr(obj, key, None)
- return default if val is None else val
- def _anthropic_event_has_content(event: Any) -> bool:
- """Whether an Anthropic stream event carries a non-empty payload."""
- event_type = _field(event, "type")
- if event_type == "content_block_delta":
- delta = _field(event, "delta")
- return any(bool(_field(delta, f)) for f in ("text", "thinking", "partial_json", "signature", "citation"))
- if event_type == "content_block_start":
- block = _field(event, "content_block")
- return _field(block, "type") == "tool_use" and any(bool(_field(block, f)) for f in ("id", "name"))
- return False
- def _anthropic_aux_stream_event_hook() -> Callable[[Any], None]:
- """Per-event callback for the Anthropic aux wire: progress only for substantive payloads
- (keepalives must not keep a stalled summary alive), stop at the host deadline or explicit
- cancel. The ``TimeoutError`` text must say "timed out" so ``_is_timeout_error`` classifies it."""
- host_deadline = _current_aux_stream_deadline()
- started = time.monotonic()
- def _on_event(event: Any) -> None:
- if _anthropic_event_has_content(event):
- _notify_aux_provider_response()
- else:
- _notify_aux_timing_response()
- if _aux_interrupt_cancel_requested():
- raise AuxiliaryExplicitCancellation()
- if host_deadline is not None and time.monotonic() >= host_deadline:
- raise TimeoutError(
- "Anthropic auxiliary stream timed out at the host compression "
- f"deadline after {time.monotonic() - started:.0f}s (the caller already stopped waiting)")
- return _on_event
- # A dead stream fails at the no-progress window (first token AND between tokens); a live
- # stream re-arms per event, bounded by _aux_stream_total_ceiling().
- _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS = 60.0
- @contextlib.contextmanager
- def _aux_thread_local_hook(local: threading.local, hook):
- """Install one thread-local hook, restoring the prior on exit (non-callable = passthrough)."""
- previous = getattr(local, "hook", None)
- local.hook = hook if callable(hook) else previous
- try:
- yield
- finally:
- local.hook = previous
- @contextlib.contextmanager
- def aux_progress_hook(hook):
- """Install *hook* as the current thread's aux forward-progress callback (None = passthrough)."""
- with _aux_thread_local_hook(_aux_progress, hook):
- yield
- def _current_aux_stream_deadline() -> Optional[float]:
- """The waiting host's absolute monotonic deadline, if one is installed (read live from a callable)."""
- value = getattr(_aux_stream_deadline, "value", None)
- return value() if callable(value) else value
- @contextlib.contextmanager
- def aux_stream_deadline(deadline: float | Callable[[], float | None] | None):
- """Publish the host's absolute ``time.monotonic()`` deadline to the stream consumer.
- ``None`` is a passthrough; re-entrant-safe. Host->worker return leg of the progress hook:
- without it the isolated provider daemon streams to its own ceiling after the host stopped
- waiting, billing a summary the commit fence refuses.
- ``8207862212`` releases the compression OWNER when the fence is cancelled, but the isolated provider
- daemon (:func:`_run_protected_sync_provider_call`) that holds the socket keeps streaming to its own
- ``_aux_stream_total_ceiling`` budget — >= the host's ceiling by construction — billing an abandoned
- summary the commit fence is already guaranteed to refuse, and stacking one fresh orphan per turn on a
- session that compression never managed to shrink. See #99692.
- """
- previous = getattr(_aux_stream_deadline, "value", None)
- _aux_stream_deadline.value = deadline if isinstance(deadline, (int, float)) or callable(deadline) else previous
- try:
- yield
- finally:
- _aux_stream_deadline.value = previous
- def _run_protected_sync_provider_call(callback: Callable[[dict[str, Any]], Any], kwargs: dict[str, Any]) -> Any:
- """Run one protected provider callback in an attempt-isolated daemon thread.
- Aux clients are process-shared and cannot be closed to wake one request, so the callback (incl.
- stream aggregation) runs in a daemon while the owner polls cancellation; on cancel the owner
- unwinds at once; the daemon stops at its next streamed chunk, else under the provider timeout in ``kwargs``
- (it owns no transcript/commit state, never holds the session lock). Unprotected / no cancel source: direct.
- """
- source_cancel_check = _capture_aux_cancel_check()
- if not _aux_interrupt_protected() or not callable(source_cancel_check):
- return callback(kwargs)
- # One linearized outcome per attempt: the host Event is reused/cleared on later turns and
- # the Codex timeout Timer may race owner polling — same lock for both.
- cancel_check = _AuxiliaryCancellationDecision(source_cancel_check)
- if cancel_check():
- raise AuxiliaryExplicitCancellation()
- # Thread-locals do not cross into the daemon: timing hooks fire from the thread running
- # the callback, and the host deadline is inert unless carried along.
- progress_hook = getattr(_aux_progress, "hook", None)
- dispatch_hook = getattr(_aux_dispatch, "hook", None)
- provider_response_hook = getattr(_aux_provider_response, "hook", None)
- host_deadline = getattr(_aux_stream_deadline, "value", None)
- reading_hook = current_prompt_reading_hook()
- # #99692: the stream is consumed on the daemon below, and thread-locals do not cross that boundary — an
- # owner-thread-only deadline would leave the fix inert on exactly the path large-session compression
- # takes (protected call + hard-cancel source installed).
- provider_context = contextvars.copy_context()
- done = threading.Event()
- outcome: dict[str, Any] = {}
- def _provider_worker() -> None:
- try:
- with (
- aux_progress_hook(progress_hook),
- _aux_thread_local_hook(_aux_dispatch, dispatch_hook),
- _aux_thread_local_hook(_aux_provider_response, provider_response_hook),
- aux_stream_deadline(host_deadline), prompt_reading_hook(reading_hook),
- aux_interrupt_protection(cancel_check=cancel_check),
- ):
- outcome["result"] = callback(kwargs)
- except BaseException as exc:
- outcome["exception"] = exc
- finally:
- done.set()
- threading.Thread(
- target=provider_context.run, args=(_provider_worker,), name="hermes-protected-aux-provider",
- daemon=True).start()
- while True:
- # Check cancel before AND after each wait so it wins when result publication and the
- # host Event land in the same polling interval.
- if _captured_aux_cancel_requested(cancel_check):
- raise AuxiliaryExplicitCancellation()
- if not done.wait(0.02):
- continue
- if _captured_aux_cancel_requested(cancel_check):
- raise AuxiliaryExplicitCancellation()
- exception = outcome.get("exception")
- if exception is not None:
- raise exception
- return outcome.get("result")
- def _client_declares(client_obj: Any, flag: str) -> bool:
- """Whether ``client_obj`` (or its class) sets ``flag`` truthy; absent → False. Capability declaration,
- not isinstance, so out-of-tree clients can opt out of wrappers unimported (cf. SUPPORTS_HERMES_TOOL_CALLS)."""
- try:
- return bool(getattr(client_obj, flag, False))
- except Exception:
- return False
- def _safe_isinstance(obj: Any, maybe_type: Any) -> bool:
- """Return False instead of raising when a patched symbol is not a type."""
- try:
- return isinstance(obj, maybe_type)
- except TypeError:
- return False
- def _extract_url_query_params(url: str):
- """Extract query params from URL, return (clean_url, default_query dict or None)."""
- parsed = urlparse(url)
- if parsed.query:
- return urlunparse(parsed._replace(query="")), {k: v[0] for k, v in parse_qs(parsed.query).items()}
- return url, None
- # Warn only once per process about stale OPENAI_BASE_URL.
- _stale_base_url_warned = False
- # Local OpenAI-compatible servers (Ollama, vLLM, llama.cpp) route through the generic custom
- # provider — mirrors hermes_cli.auth._PROVIDER_ALIASES. Without this group an explicit
- # ``provider: ollama`` aux lane matches no registry entry and raises a misleading
- # ``OLLAMA_API_KEY`` error instead of using the lane's base_url (#106010).
- _LOCAL_SERVER_ALIASES = {
- "ollama": "custom", "vllm": "custom", "llamacpp": "custom",
- "llama.cpp": "custom", "llama-cpp": "custom",
- }
- _ALIAS_TABLES: dict[str, tuple[int, dict[str, str]]] = {} # home key -> (registry generation, table)
- def _provider_alias_table() -> dict[str, str]:
- """The alias table the main provider path resolves against (hermes_cli.auth), per home and registry
- generation: plugin aliases are per home. A hand-copied mirror rots silently (#106010); local-server
- names stay in: they guard the /v1 tail and the custom-branch no-key-borrow rule."""
- from providers import bound_registry_key
- home, generation = bound_registry_key()
- cached = _ALIAS_TABLES.get(home)
- if cached is None or cached[0] != generation:
- cached = (generation, dict(_LOCAL_SERVER_ALIASES))
- with contextlib.suppress(Exception):
- from hermes_cli.auth import _plugin_aliases
- cached[1].update(_plugin_aliases())
- _ALIAS_TABLES[home] = cached
- return cached[1]
- def _normalize_aux_provider(provider: Optional[str]) -> str:
- normalized = (provider or "auto").strip().lower()
- if normalized.startswith("custom:"):
- suffix = normalized.split(":", 1)[1].strip()
- if not suffix:
- return "custom"
- normalized = suffix
- if normalized == "codex":
- return "openai-codex"
- if normalized == "main":
- # Resolve to the actual main provider so named custom providers work.
- main_prov = (_read_main_provider() or "").strip().lower()
- if not main_prov or main_prov in {"auto", "main"}:
- return "custom"
- normalized = main_prov
- return _provider_alias_table().get(normalized, normalized)
- # Sentinel from _fixed_temperature_for_model(): callers strip ``temperature`` entirely.
- # Kimi/Moonshot manage it server-side — any value can conflict with gateway mode selection.
- OMIT_TEMPERATURE: object = object()
- def _bare_model(model: Optional[str]) -> str:
- """Lowercased model slug with any ``vendor/`` prefix stripped."""
- return (model or "").strip().lower().rsplit("/", 1)[-1]
- def _is_kimi_model(model: Optional[str]) -> bool:
- """True for any Kimi / Moonshot model that manages temperature server-side."""
- bare = _bare_model(model)
- return bare.startswith("kimi-") or bare == "kimi"
- def _is_arcee_trinity_thinking(model: Optional[str]) -> bool:
- """True for Arcee Trinity Large Thinking (direct or via OpenRouter)."""
- return _bare_model(model) == "trinity-large-thinking"
- # Codex OAuth hard-caps gpt-5.4/5.5/5.6 and gpt-6 Astra at 272K (raw API/OpenRouter expose 1.05M);
- # the default 50% trigger would compact at ~136K, so raise to 85% (~231K).
- _CODEX_GPT54_GPT55_COMPACTION_THRESHOLD = 0.85
- # gpt-5.3-codex-spark: Codex-OAuth-only, native 128K; 70% (~90K) leaves summary headroom.
- _CODEX_SPARK_COMPACTION_THRESHOLD = 0.70
- def _is_codex_gpt54_or_gpt55(model: Optional[str], provider: Optional[str] = None) -> bool:
- """True for gpt-5.4/5.5/5.6, gpt-6 Sol/Terra/Luna, gpt-6 Astra (and the Daybreak Sol alias) on the Codex OAuth route only.
- Other routes expose a larger window for the same slug and keep the user's threshold.
- Prefix-matched so ``-pro`` and dated snapshots track every 272K-capped family; ``-900k``
- picker variants are excluded. Astra is substring-matched (any slug containing ``astra``
- without ``900k``). Name kept for the ``compression.codex_gpt55_autoraise`` key.
- """
- bare = _codex_route_bare_model(model, provider)
- if bare is None:
- return False
- from agent.model_metadata import is_codex_context_variant
- if is_codex_context_variant(bare):
- return False
- if "astra" in bare:
- return "900k" not in bare
- return bare == "gpt-daybreak-blue-latest" or any(
- bare == fam or bare.startswith(fam + "-") or bare.startswith(fam + ".")
- for fam in ("gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-6-sol", "gpt-6.1-sol", "gpt-6-luna"))
- def _codex_route_bare_model(model: Optional[str], provider: Optional[str]) -> Optional[str]:
- """Lowercased bare model slug when ``provider`` is the Codex OAuth route, else None."""
- return _bare_model(model) if (provider or "").strip().lower() == "openai-codex" else None
- def _is_codex_spark(model: Optional[str], provider: Optional[str] = None) -> bool:
- """True for ``gpt-5.3-codex-spark`` on the Codex OAuth route (the slug exists nowhere else)."""
- return _codex_route_bare_model(model, provider) == "gpt-5.3-codex-spark"
- def _is_openai_default_temperature_only(model: Optional[str]) -> bool:
- """True for OpenAI reasoning families that 400 (``unsupported_value``) on any non-default
- ``temperature``: gpt-5.x (incl. dated snapshots, ``-pro``, ``-codex``), o1/o3/o4. The
- ``gpt-5-chat`` non-reasoning line still accepts it (#51083)."""
- bare = _bare_model(model)
- return bare.startswith(("gpt-5", "o1", "o3", "o4")) and not bare.startswith("gpt-5-chat")
- # Routes (host + model) that rejected ``temperature`` at runtime; the next call omits it up front
- # instead of paying the 400 round-trip again (the retry alone left #51083's first call to time out).
- _TEMPERATURE_REJECTED_ROUTES: set = set()
- def remember_temperature_rejection(
- provider: Optional[str], base_url: Optional[str], rejected_kwargs: dict[str, Any], error: BaseException,
- ) -> None:
- _TEMPERATURE_REJECTED_ROUTES.add((_route_key(provider, base_url), _bare_model(rejected_kwargs.get("model"))))
- def _fixed_temperature_for_model(
- model: Optional[str], base_url: Optional[str] = None, provider: Optional[str] = None,
- ) -> Optional[float] | object:
- """``OMIT_TEMPERATURE`` (drop the key; Kimi/Moonshot, OpenAI reasoning families, Gemini 3.6+, routes
- that already rejected it), a fixed ``float``, or ``None``."""
- if _is_kimi_model(model):
- logger.debug("Omitting temperature for Kimi model %r (server-managed)", model)
- return OMIT_TEMPERATURE
- from agent.gemini_native_adapter import gemini_sampling_deprecated
- if _is_openai_default_temperature_only(model) or gemini_sampling_deprecated(_bare_model(model)):
- logger.debug("Omitting temperature for %r (default-only, or Gemini 3.6+ ignores sampling)", model)
- return OMIT_TEMPERATURE
- if (_route_key(provider, base_url), _bare_model(model)) in _TEMPERATURE_REJECTED_ROUTES:
- logger.debug("Omitting temperature for %r (route rejected it earlier)", model)
- return OMIT_TEMPERATURE
- return 0.5 if _is_arcee_trinity_thinking(model) else None
- def _compression_threshold_for_model(
- model: Optional[str], provider: Optional[str] = None, *,
- allow_codex_gpt55_autoraise: bool = True,
- ) -> Optional[float]:
- """Per-model/route compression threshold override (fraction of context used), or None.
- Arcee Trinity Large Thinking → 0.75 (preserve reasoning context); Codex-route gpt-5.4/5.5/5.6/Astra
- → 0.85, gated by ``allow_codex_gpt55_autoraise``; Codex-route gpt-5.3-codex-spark → 0.70, ungated.
- """
- if _is_arcee_trinity_thinking(model):
- return 0.75
- if allow_codex_gpt55_autoraise and _is_codex_gpt54_or_gpt55(model, provider):
- return _CODEX_GPT54_GPT55_COMPACTION_THRESHOLD
- if _is_codex_spark(model, provider):
- return _CODEX_SPARK_COMPACTION_THRESHOLD
- return None
- # Aux "fast tier" families, fastest first (measured p50 titling latency). Matched as substrings
- # against the LIVE /v1/models catalog because pinned ids rot; rolling "-latest" aliases lead.
- _FAST_MODEL_FAMILIES: tuple = (
- "gpt-mini-latest", "gpt-nano-latest", "claude-haiku-latest", "gemini-flash-latest",
- "gpt-5.4-nano", "gpt-5.4-mini", "gpt-5-mini", "haiku-4.5", "gemini-3.6-flash", "flash-lite",
- "-nano", "-mini", "-flash", "haiku",
- )
- # Disqualifiers: reasoning variants think before answering; ":batch" is a queue; ":free" tiers
- # are rate-limited and slowest; embedders/modality endpoints match a rung but cannot answer.
- _FAST_MODEL_EXCLUDE: tuple = (
- "thinking", "reason", "-r1", "minilm", ":batch", ":free",
- "o1-", "o3-", "o4-", "codex", "audio", "-vl", "embed",
- "-tts", "-transcribe", "-realtime", "-image", "-search-preview",
- )
- def _model_recency_key(model_id: str) -> tuple:
- """Sort key putting a family's newest release first: digit runs compare numerically (plain
- string order picks ``gpt-3.5-mini`` over ``gpt-5.4-mini`` and breaks at 9 vs 10)."""
- # re.split with one capturing group alternates text, number, text, …
- return tuple(
- (1, float(part), "") if index % 2 else (0, 0.0, part)
- for index, part in enumerate(re.split(r"(\d+(?:\.\d+)?)", model_id.lower())) if part)
- def _fast_model_from_catalog(provider_id: str) -> str:
- """Newest ``_FAST_MODEL_FAMILIES`` match from the provider's live (cached) catalog.
- "" when the catalog is unavailable or holds no small model (caller falls through to the
- curated default). Never raises; the fetch is memory+disk cached.
- """
- is_nous = provider_id.strip().lower() == "nous"
- try:
- from hermes_cli.auth import resolve_api_key_provider_credentials
- from hermes_cli.models_pricing import fetch_models_with_pricing
- from providers import get_provider_profile
- # Most /v1/models endpoints are authenticated; an anonymous 401 would read as "no small
- # model" and pin the curated default forever.
- api_key, base_url = "", ""
- try:
- creds = resolve_api_key_provider_credentials(provider_id) or {}
- api_key = str(creds.get("api_key", "")).strip()
- base_url = str(creds.get("base_url", "")).strip()
- except Exception:
- # Not an API-key provider, or nothing configured; anonymous fetch may still work.
- logger.debug("No credentials for %s catalog", provider_id, exc_info=True)
- if not api_key and is_nous:
- # Nous is OAuth (resolver raises); anonymous reads return the full catalog.
- try:
- from hermes_cli.models_pricing import _resolve_nous_pricing_credentials
- api_key, base_url = _resolve_nous_pricing_credentials()
- except Exception:
- logger.debug("No Nous credentials for catalog", exc_info=True)
- if not base_url:
- base_url = str(getattr(get_provider_profile(provider_id), "base_url", "") or "")
- base_url = base_url.rstrip("/")
- if not base_url:
- return ""
- base_url = base_url.removesuffix("/v1")
- # Nous-only args must match the pickers' or the seeded cache loses sale chrome and
- # policy-catalog expiry.
- _nous_kwargs = {}
- if is_nous:
- from hermes_cli.models_pricing import _NOUS_CATALOG_TTL_SECONDS
- _nous_kwargs = {"include_sale_original": True, "cache_ttl_seconds": _NOUS_CATALOG_TTL_SECONDS}
- catalog = fetch_models_with_pricing(
- api_key=api_key or None, base_url=base_url, timeout=3.0, **_nous_kwargs) or {}
- except Exception:
- logger.debug("Fast-model catalog lookup failed for %s", provider_id, exc_info=True)
- return ""
- ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True)
- if is_nous:
- # Narrow catalog ids by org policy, as the pickers do.
- try:
- from hermes_cli.models_pricing import nous_policy_allowed_ids, restrict_to_nous_policy
- ids = restrict_to_nous_policy(ids, nous_policy_allowed_ids())
- except Exception:
- logger.debug("Nous policy filter unavailable", exc_info=True)
- for family in _FAST_MODEL_FAMILIES:
- for model_id in ids:
- lowered = model_id.lower()
- if family in lowered and not any(x in lowered for x in _FAST_MODEL_EXCLUDE):
- return model_id
- return ""
- # Default auxiliary models for direct API-key providers (cheap/fast for side tasks)
- def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) -> str:
- """Cheap auxiliary model for a provider.
- Ladder: (``prefer_fast`` only) live-catalog family match, then ``ProviderProfile.resolve_aux_model``;
- then ``default_aux_model`` (curated); then the legacy dict. ``prefer_fast`` is opt-in (titling)
- so other callers keep their static behaviour and cache keys.
- """
- profile = None
- with contextlib.suppress(Exception):
- from providers import get_provider_profile
- profile = get_provider_profile(provider_id)
- picked = ""
- if prefer_fast:
- picked = _fast_model_from_catalog(provider_id)
- if not picked and profile is not None:
- try:
- picked = profile.resolve_aux_model() or ""
- except Exception:
- logger.debug("resolve_aux_model failed for %s", provider_id, exc_info=True)
- if not picked and profile is not None and profile.default_aux_model:
- picked = profile.default_aux_model
- if not picked:
- picked = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "")
- # Rungs 2-4 are policy-blind; a blocked pick is refused at request time, so drop it and
- # let the caller keep the main model.
- if picked and provider_id.strip().lower() == "nous":
- try:
- from hermes_cli.models_pricing import nous_policy_allowed_ids, restrict_to_nous_policy
- allowed = nous_policy_allowed_ids()
- if allowed and not restrict_to_nous_policy([picked], allowed):
- return ""
- except Exception:
- logger.debug("Nous policy check unavailable", exc_info=True)
- return picked
- # Fallback for providers without ProviderProfile.default_aux_model (plus some pinned here).
- # New providers should set default_aux_model instead.
- _API_KEY_PROVIDER_AUX_MODELS_FALLBACK: dict[str, str] = {
- "gemini": "gemini-3.6-flash", "zai": "glm-4.5-flash", "kimi-coding": "kimi-k2-turbo-preview",
- "stepfun": "step-3.5-flash", "kimi-coding-cn": "kimi-k2-turbo-preview",
- "gmi": "google/gemini-3.1-flash-lite-preview", "anthropic": "claude-haiku-4-5-20251001",
- "ai-gateway": "google/gemini-3-flash", "opencode-zen": "gemini-3-flash", "opencode-go": "glm-5",
- "kilocode": "google/gemini-3.6-flash", "ollama-cloud": "nemotron-3-nano:30b",
- "tencent-tokenhub": "hy4-preview", "tencent-tokenplan": "hy4-preview",
- # No "deepinfra": its aux model lives on the ProviderProfile (read first).
- }
- # Legacy alias for callers not yet using _get_aux_model_for_provider().
- _API_KEY_PROVIDER_AUX_MODELS: dict[str, str] = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK
- # Tasks that may opt into ``auxiliary.<task>.prefer_fast_model``.
- _FAST_MODEL_TASKS: frozenset = frozenset({"title_generation"})
- def _task_prefers_fast_model(task: Optional[str]) -> bool:
- """Return whether an eligible task explicitly opts into fast-model routing."""
- return task in _FAST_MODEL_TASKS and is_truthy_value(
- _get_auxiliary_task_config(task).get("prefer_fast_model"), default=False)
- # Dedicated vision models for direct providers whose main chat model differs. zai: glm-5.3-flash
- # is the only image-capable GLM id served on every Z.AI surface (pay-as-you-go and Coding Plan,
- # global and CN); the former glm-5v-turbo pin 404s / 1211 "Unknown Model" on the coding endpoints
- # (#111429). ZaiProfile has no default_vision_model(), so dropping the pin would route vision to
- # the user's text-only chat model and skip Z.AI entirely.
- _PROVIDER_VISION_MODELS: dict[str, str] = {"xiaomi": "mimo-v2.5", "zai": "glm-5.3-flash"}
- def _resolve_provider_vision_default(provider: str) -> Optional[str]:
- """Provider default vision model id, or None: static ``_PROVIDER_VISION_MODELS`` (vision-only
- names absent from any catalog) win, else ``ProviderProfile.default_vision_model()``."""
- static = _PROVIDER_VISION_MODELS.get(provider)
- if static:
- return static
- try:
- from providers import get_provider_profile
- profile = get_provider_profile(provider)
- return profile.default_vision_model() if profile is not None else None
- except Exception:
- return None
- # Endpoints that reject image input: vision auto-detect skips these to the aggregator chain
- # instead of returning a client that 404s (Kimi Coding Plan Anthropic wire has no image_in).
- _PROVIDERS_WITHOUT_VISION: frozenset = frozenset({"kimi-coding", "kimi-coding-cn"})
- # OpenRouter app attribution (always sent). `X-Title` is what the dashboard reads.
- _OR_HEADERS_BASE = {
- "HTTP-Referer": "https://hermes-agent.nousresearch.com",
- "X-Title": "Hermes Agent",
- "X-OpenRouter-Categories": "productivity,cli-agent",
- }
- def _apply_user_default_headers(headers: dict | None) -> dict | None:
- """Merge user ``model.default_headers`` onto resolved headers (user wins; ``model.extra_headers``
- alias wins over both). Mirrors ``AIAgent._apply_user_default_headers`` so a custom endpoint behind a
- WAF rejecting ``User-Agent`` / ``X-Stainless-*`` works for aux calls. SECURITY: never log values."""
- try:
- from hermes_cli.config import cfg_get, load_config
- _cfg = load_config()
- user_headers = cfg_get(_cfg, "model", "default_headers")
- alias_headers = cfg_get(_cfg, "model", "extra_headers")
- if isinstance(alias_headers, dict) and alias_headers:
- user_headers = {**(user_headers if isinstance(user_headers, dict) else {}), **alias_headers}
- except Exception:
- return headers
- if not isinstance(user_headers, dict) or not user_headers:
- return headers
- merged = dict(headers or {})
- merged.update({str(k): str(v) for k, v in user_headers.items() if v is not None})
- return merged or headers
- def build_or_headers(or_config: dict | None = None) -> dict:
- """OpenRouter headers, plus response-cache headers when enabled.
- Precedence env > config > default: ``HERMES_OPENROUTER_CACHE`` overrides
- ``openrouter.response_cache``; ``HERMES_OPENROUTER_CACHE_TTL`` (1-86400 s) overrides
- ``openrouter.response_cache_ttl``. ``or_config=None`` reads from disk.
- """
- headers = dict(_OR_HEADERS_BASE)
- if or_config is None:
- try:
- from hermes_cli.config import load_config_readonly
- or_config = load_config_readonly().get("openrouter", {})
- except Exception:
- or_config = {}
- env_cache = os.environ.get("HERMES_OPENROUTER_CACHE", "").strip().lower()
- if not (env_cache in {"1", "true", "yes", "on"} if env_cache else or_config.get("response_cache", False)):
- return headers
- headers["X-OpenRouter-Cache"] = "true"
- env_ttl = os.environ.get("HERMES_OPENROUTER_CACHE_TTL", "").strip()
- if env_ttl:
- if env_ttl.isdigit() and 1 <= int(env_ttl) <= 86400:
- headers["X-OpenRouter-Cache-TTL"] = str(int(env_ttl))
- else:
- ttl = or_config.get("response_cache_ttl", 300)
- if isinstance(ttl, (int, float)) and 1 <= ttl <= 86400:
- headers["X-OpenRouter-Cache-TTL"] = str(int(ttl))
- return headers
- # NVIDIA NIM cloud billing attribution; host-gated because NVIDIA_BASE_URL may be a local NIM.
- _NVIDIA_NIM_CLOUD_HEADERS = {"X-BILLING-INVOKE-ORIGIN": "HermesAgent"}
- def build_nvidia_nim_headers(base_url: str | None) -> dict:
- """Return NVIDIA NIM cloud attribution headers for build.nvidia.com traffic."""
- return dict(_NVIDIA_NIM_CLOUD_HEADERS) if base_url_host_matches(str(base_url or ""), "integrate.api.nvidia.com") else {}
- # Vercel AI Gateway attribution (HTTP-Referer → referrerUrl, X-Title → appName).
- from hermes_cli.version_info import hermes_user_agent
- def _ai_gateway_headers() -> dict[str, str]:
- return {"HTTP-Referer": "https://hermes-agent.nousresearch.com", "X-Title": "Hermes Agent",
- "User-Agent": hermes_user_agent("HermesAgent")}
- # Nous Portal attribution tags, built per call. They come from agent.portal_tags so the client=
- # marker tracks the canonical base version — never inline a literal here.
- from agent.portal_tags import nous_portal_tags as _nous_portal_tags
- # _OPENROUTER_MODEL MUST stay a :free SKU (matching the free_only warning): this lane engages
- # silently, and a paid default meant spend the user never opted into. User-configured values
- # are honored untouched (_warn_paid_lane_once fires).
- _OPENROUTER_MODEL = "nvidia/nemotron-3-ultra-550b-a55b:free"
- _NOUS_MODEL = "google/gemini-3.6-flash"
- _NOUS_DEFAULT_BASE_URL = "https://inference-api.nousresearch.com/v1"
- _ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com"
- _AUTH_JSON_PATH = get_hermes_home() / "auth.json"
- _AUTH_JSON_PATH_AT_IMPORT = _AUTH_JSON_PATH
- def _auth_json_path():
- """Active profile's ``auth.json`` at call time (a patched ``_AUTH_JSON_PATH`` still wins). The
- import-time constant is the LAUNCH profile's; under multiplexing a secondary's auxiliary calls
- would otherwise authenticate to Nous with the default profile's token."""
- from hermes_cli.auth import _auth_file_path
- return _AUTH_JSON_PATH if _AUTH_JSON_PATH != _AUTH_JSON_PATH_AT_IMPORT else _auth_file_path()
- # Hosts exposing BOTH ``…/anthropic`` and a sibling OpenAI ``…/v1``. Matched on the URL *host*
- # only: unconditional rewrites break Anthropic-only gateways.
- _DUAL_SURFACE_ANTHROPIC_HOST_SUFFIXES = ("minimax.io", "minimax.chat", "minimaxi.com")
- _DUAL_SURFACE_ANTHROPIC_HOST_PREFIXES = ("api.minimax.",)
- def _is_dual_surface_anthropic_host(url: str) -> bool:
- """True when the URL's host is a known dual-surface (MiniMax-family) host."""
- try:
- host = (urlparse(url).hostname or "").lower()
- except ValueError:
- return False
- return any(
- host == suffix or host.endswith("." + suffix) for suffix in _DUAL_SURFACE_ANTHROPIC_HOST_SUFFIXES
- ) or any(host.startswith(prefix) for prefix in _DUAL_SURFACE_ANTHROPIC_HOST_PREFIXES)
- def _to_openai_base_url(base_url: str) -> str:
- """Normalize dual-surface Anthropic URLs to their OpenAI-compatible sibling.
- MiniMax-family: ``/anthropic`` → ``/v1``; ZAI Coding Plan → ``/coding/paas/v4`` (the general
- endpoint bills separately); Kimi Code ``/coding`` → ``/coding/v1`` (the OpenAI SDK path 404s
- without it). Anthropic-only gateways keep their path.
- """
- url = str(base_url or "").strip().rstrip("/")
- if base_url_hostname(url) == "api.actual.inc":
- from hermes_cli.auth import normalize_actual_base_url
- return normalize_actual_base_url(url)
- if url.endswith("/anthropic"):
- if base_url_host_matches(url, "open.bigmodel.cn") or base_url_host_matches(url, "api.z.ai"):
- rewritten = url[: -len("/anthropic")] + "/coding/paas/v4"
- logger.debug("Auxiliary client: rewrote ZAI base URL %s → %s", url, rewritten)
- return rewritten
- if _is_dual_surface_anthropic_host(url):
- rewritten = url[: -len("/anthropic")] + "/v1"
- logger.debug("Auxiliary client: rewrote dual-surface base URL %s → %s", url, rewritten)
- return rewritten
- logger.debug(
- "Auxiliary client: keeping Anthropic-only base URL %s (no dual-surface host match)", url)
- return url
- if base_url_host_matches(url, "api.kimi.com") and url.endswith("/coding"):
- rewritten = url + "/v1"
- logger.debug("Auxiliary client: rewrote Kimi base URL %s → %s", url, rewritten)
- return rewritten
- return url
- def _load_pool_with_credentials(provider: str, note: str = "") -> Optional[Any]:
- """``load_pool(provider)`` when it has credentials, else None (never raises)."""
- try:
- pool = load_pool(provider)
- except Exception as exc:
- logger.debug("Auxiliary client: could not load pool for %s%s: %s", provider, note, exc)
- return None
- return pool if pool and pool.has_credentials() else None
- def _select_pool_entry(provider: str) -> tuple[bool, Optional[Any]]:
- """Return (pool_exists_for_provider, selected_entry)."""
- pool = _load_pool_with_credentials(provider)
- if pool is None:
- return False, None
- try:
- return True, pool.select()
- except Exception as exc:
- logger.debug("Auxiliary client: could not select pool entry for %s: %s", provider, exc)
- return True, None
- def _peek_pool_entry(provider: str, pool: Any = None) -> Optional[Any]:
- """Best-effort current/next pool entry without mutating selection order.
- ``pool`` skips the disk re-read when the caller already loaded it.
- """
- if pool is None:
- pool = _load_pool_with_credentials(provider, " (peek)")
- if pool is None:
- return None
- try:
- current_fn = getattr(pool, "current", None)
- current = current_fn() if callable(current_fn) else None
- if current is not None:
- return current
- peek_fn = getattr(pool, "peek", None)
- if callable(peek_fn):
- return peek_fn()
- except Exception as exc:
- logger.debug("Auxiliary client: could not peek pool entry for %s: %s", provider, exc)
- return None
- def _pool_runtime_api_key(entry: Any) -> str:
- # runtime_api_key handles provider-specific fallback (e.g. agent_key for nous); None entry → "".
- key = getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "")
- return str(key or "").strip()
- def _pool_runtime_base_url(entry: Any, fallback: str = "") -> str:
- if entry is None:
- return str(fallback or "").strip().rstrip("/")
- if getattr(entry, "provider", None) == "nous":
- # Canonical auth-layer reader so the env override shares one normalization path.
- from hermes_cli.auth import _nous_inference_env_override
- env_url = _nous_inference_env_override()
- if env_url:
- return env_url
- # runtime_base_url is provider-aware; fall back for non-PooledCredential entries.
- url = (getattr(entry, "runtime_base_url", None) or getattr(entry, "inference_base_url", None)
- or getattr(entry, "base_url", None) or fallback)
- return str(url or "").strip().rstrip("/")
- # Hosts the aux Anthropic path may be pointed at via model.base_url; anything else falls back
- # to the Anthropic default so a foreign host never leaks in.
- _ANTHROPIC_COMPATIBLE_HOSTS = frozenset({"api.anthropic.com"})
- def _is_anthropic_compatible_host(url: str) -> bool:
- """True for native Anthropic hosts and gateways serving Messages under a ``/anthropic`` path
- (same convention as runtime_provider / ``_wrap_if_needed``), so a configured ``model.base_url``
- whose gateway holds auth is not discarded. A bare non-Anthropic base_url is False."""
- if not url:
- return False
- try:
- parsed = urlparse(url)
- if (parsed.hostname or "").strip().lower().rstrip(".") in _ANTHROPIC_COMPATIBLE_HOSTS:
- return True
- path = (parsed.path or "").rstrip("/").lower()
- return path.endswith("/anthropic") or path.endswith("/anthropic/v1")
- except Exception:
- return False
- def _nous_min_key_ttl_seconds() -> int:
- try:
- return max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800")))
- except (TypeError, ValueError):
- return 1800
- def _scoped_key_env(name: str) -> str:
- """Read a provider API key (or its paired base-URL) env var through the profile secret scope.
- In agent turns the scope's verdict is authoritative (a scoped miss must not borrow another
- profile's key); only the unscoped default-profile path (``UnscopedSecretError``) reads
- ``os.environ`` -- any other scope failure propagates instead of borrowing the ambient env.
- """
- if not name:
- return ""
- from agent.secret_scope import UnscopedSecretError, get_secret
- try:
- return (get_secret(name) or "").strip()
- except UnscopedSecretError:
- return (os.getenv(name) or "").strip()
- def _attempt_stream_socket(stream: Any) -> Any:
- """The raw socket under an SDK event stream (``stream.response`` is the ``httpx.Response``;
- httpcore publishes its connection as the ``network_stream`` extension), or None."""
- from agent.agent_runtime_helpers import _socket_from_stream
- extensions = getattr(getattr(stream, "response", None), "extensions", None)
- network_stream = extensions.get("network_stream") if isinstance(extensions, dict) else None
- return _socket_from_stream(network_stream) if network_stream is not None else None
- def _close_quietly(target: Any, failure_note: Optional[str]) -> None:
- """Call ``target.close()`` if present; a failure is debug-logged under ``failure_note`` (silent when None)."""
- close = getattr(target, "close", None)
- if callable(close):
- try:
- close()
- except Exception:
- if failure_note:
- logger.debug("Codex auxiliary: %s", failure_note, exc_info=True)
- # The context compressor keys its retry-ladder classification on this text (#124077).
- CODEX_STREAM_STALL_MARKER = "stream stalled"
- class _CodexStreamGuard:
- """Progress-aware deadline + FD-safe timeout watchdog for one Codex aux stream attempt.
- (1) The first substantive payload must arrive within ``no_progress_timeout`` or we fail fast into
- the caller's retry/fallback chain (a dead or keepalive-only zombie must not hold the budget);
- (2) each substantive event re-arms that window (keepalive/lifecycle frames do NOT, mirroring
- commit-fence gating) so a live stream is never killed by an absolute total; (3) a hard ceiling
- from ``_aux_stream_total_ceiling`` still terminates a pathological drip.
- """
- def __init__(
- self, client: Any, total_timeout: Optional[float],
- no_progress_timeout: Optional[float] = None,
- ):
- self._client = client
- self.total_timeout = total_timeout
- self._start = time.monotonic()
- # Task-scoped override (auxiliary.<task>.no_progress_timeout, #108104); falls back to the
- # built-in default when unset or not a positive number.
- if isinstance(no_progress_timeout, (int, float)) and no_progress_timeout > 0:
- self.no_progress_timeout = float(no_progress_timeout)
- else:
- self.no_progress_timeout = _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS
- # Progress-aware stream deadlines (supersedes the old single absolute kill at ``total_timeout``).
- # Three regimes: 1. First token: the stream must produce its first substantive payload within
- # ``no_progress_timeout`` (60s default) or we fail fast and let the caller's normal retry/fallback
- # chain run — a dead (or keepalive-only zombie) Codex stream no longer holds the full 300s
- # compression budget before falling back (masoria report, Aug 2026: 3 stacked 300s waits -> 20+ min
- # stuck on "Summarizing"). 2. Streaming: every substantive event re-arms the deadline by
- # ``no_progress_timeout`` — a live stream is never killed by an absolute total, so a long reasoning
- # summary that is actually producing tokens completes instead of timing out at 300s and falling back
- # (#54915's original complaint, fixed properly). Keepalive/lifecycle frames do NOT re-arm, mirroring
- # the commit-fence progress gating (#96707). 3. Hard ceiling: an absolute backstop from
- # ``_aux_stream_total_ceiling`` (max(600s, 4x configured timeout) — the same bound the streamed
- # chat.completions path uses) so a pathological one-token-per-59s drip still terminates.
- if total_timeout is not None:
- self.no_progress_timeout = min(self.no_progress_timeout, float(total_timeout))
- self.hard_deadline = self._start + _aux_stream_total_ceiling(total_timeout)
- # The waiting host's absolute deadline clamps the ceiling so the watchdog Timer severs
- # the socket the instant the host stops waiting — a stream blocked between events
- # can't be stopped by a per-event check.
- host_deadline = _current_aux_stream_deadline()
- if isinstance(host_deadline, (int, float)) and host_deadline < self.hard_deadline:
- self.hard_deadline = float(host_deadline)
- self._deadline_lock = threading.Lock()
- self._progress_deadline = self._start + self.no_progress_timeout
- self.saw_content = threading.Event()
- self.timed_out = threading.Event()
- # Set only when the timeout WON (not when the owner hard-cancelled first): tells the
- # owner's ``finally`` the shared client's FDs still need a real close.
- self.timeout_release_pending = threading.Event()
- self.stream_finished = threading.Event()
- self._timer = None
- # The owner may return on hard cancel while this attempt is still blocked in the SDK
- # stream. Timer threads don't inherit the worker's thread-local protection state, so
- # freeze the hard-cancel source before creating the timer.
- self._protected_cancel_check = _capture_aux_cancel_check() if _aux_interrupt_protected() else None
- self._attempt_stream_lock = threading.Lock()
- self._attempt_stream: Any = None
- # The request-driving thread owns the transport FDs — see _close_client_on_timeout.
- self._owner_tid = threading.get_ident()
- def effective_deadline(self) -> float:
- with self._deadline_lock:
- return min(self.hard_deadline, self._progress_deadline)
- def cancel_requested(self) -> bool:
- """True when the frozen hard-cancel source says the owner already cancelled."""
- check = self._protected_cancel_check
- return callable(check) and _captured_aux_cancel_requested(check)
- def adopt_stream(self, stream: Any) -> None:
- with self._attempt_stream_lock:
- self._attempt_stream = stream
- def release_stream(self, stream: Any) -> None:
- """Owner-side: close the attempt stream silently and forget it."""
- _close_quietly(stream, None)
- with self._attempt_stream_lock:
- self._attempt_stream = None
- def close_attempt_stream(self, failure_note: str) -> None:
- """Wake only this attempt's stream, never the shared client. The owner thread closes it;
- any other thread only ``shutdown()``s its socket, since ``Stream.close()`` would release the FD
- under the owner's ``SSL_read`` (#70773, #130115). Socketless streams are closed as before."""
- with self._attempt_stream_lock:
- stream = self._attempt_stream
- if stream is None:
- return
- if threading.get_ident() != self._owner_tid:
- sock = _attempt_stream_socket(stream)
- if sock is not None:
- from agent.agent_runtime_helpers import _shutdown_socket
- _shutdown_socket(sock)
- return
- _close_quietly(stream, failure_note)
- def record_progress(self) -> None:
- """Substantive payload re-arms the no-progress window; the hard ceiling never moves."""
- with self._deadline_lock:
- self._progress_deadline = time.monotonic() + self.no_progress_timeout
- def timeout_message(self) -> str:
- elapsed = time.monotonic() - self._start
- if time.monotonic() >= self.hard_deadline:
- return f"Codex auxiliary Responses stream exceeded {self.hard_deadline - self._start:.1f}s hard ceiling"
- if not self.saw_content.is_set():
- return (
- "Codex auxiliary Responses stream produced no output "
- f"within {float(self.no_progress_timeout):.1f}s (no-progress timeout, {elapsed:.1f}s elapsed)")
- return (
- f"Codex auxiliary Responses {CODEX_STREAM_STALL_MARKER}: no new output "
- f"for {float(self.no_progress_timeout):.1f}s ({elapsed:.1f}s elapsed)")
- def _close_client_on_timeout(self) -> None:
- begin_timeout_cleanup = getattr(self._protected_cancel_check, "begin_timeout_cleanup", None)
- if callable(begin_timeout_cleanup):
- timeout_won = bool(begin_timeout_cleanup())
- else:
- timeout_won = not self.cancel_requested()
- # Publish transport timeout only after the attempt-local decision is fixed, so owner
- # polling cannot observe completion in between.
- self.timed_out.set()
- if not timeout_won:
- # Owner already hard-cancelled. The OpenAI client is process-shared, so never
- # close/evict it here; wake only this attempt's stream if responses.create()
- # returned one, else rely on the bounded SDK timeout.
- self.close_attempt_stream("cancelled attempt stream close during timeout failed")
- return
- # FD-ownership contract: only the thread driving the request may ``close()`` this
- # client's FDs. From a stranger thread (the watchdog Timer) only ``shutdown()`` is
- # FD-safe — ``close()`` releases the raw TLS fd while the owner's OpenSSL BIO still
- # caches it, the kernel recycles it (e.g. into a SQLite handle), and the owner's TLS
- # flush corrupts that file. The owner does the real close in its ``finally``.
- # This callback has two callers — ``_check_cancelled`` on the owning thread, and the daemon watchdog
- # ``threading.Timer``, which is a stranger thread. The owning thread performs the real close in the
- # ``finally`` below, which is where the FD release belongs. See #70773.
- self.timeout_release_pending.set()
- if threading.get_ident() == self._owner_tid:
- _close_quietly(self._client, "client close during timeout failed")
- else:
- try:
- from agent.agent_runtime_helpers import force_close_tcp_sockets
- shutdown_count = force_close_tcp_sockets(self._client)
- logger.info(
- "Codex auxiliary client aborted (timeout, tcp_force_closed=%d, "
- "deferred_close=stranger_thread)", shutdown_count)
- except Exception:
- logger.debug("Codex auxiliary: client abort during timeout failed", exc_info=True)
- # Socket shutdown only wakes a reader on a REAL transport; the owner may be blocked
- # inside the SDK's event stream (or a socketless test double). Wake the
- # attempt-owned stream too — from this thread that is a shutdown of its socket,
- # never a close (see close_attempt_stream).
- self.close_attempt_stream("attempt stream close during stranger-thread timeout failed")
- # The aux client cache wraps this same client; drop the entry so the next aux call
- # doesn't reuse the dead transport and fail fast.
- try:
- # After we close the httpx transport above, the cache must drop that entry — otherwise the next
- # auxiliary call (compression retry, memory flush, etc.) reuses the dead client and fails fast
- # with a connection error. See issue #23432.
- _evict_cached_client_instance(self._client)
- except Exception:
- logger.debug("Codex auxiliary: cache eviction on timeout failed", exc_info=True)
- def check_cancelled(self) -> None:
- if self.total_timeout is not None and time.monotonic() >= self.effective_deadline():
- if not self.timed_out.is_set():
- self._close_client_on_timeout()
- raise TimeoutError(self.timeout_message())
- try:
- from tools.interrupt import is_interrupted
- # Protected atomic aux tasks (compression) must not abort on a mid-flight gateway
- # interrupt (degraded fallback marker); explicit host cancel has its own exception.
- if _aux_interrupt_cancel_requested():
- raise AuxiliaryExplicitCancellation()
- # Explicit host cancellation has its own frozen exception; timeouts above still fire and other
- # aux tasks remain interruptible. See #23975.
- if is_interrupted() and not _aux_interrupt_protected():
- raise InterruptedError("Codex auxiliary Responses stream interrupted")
- except InterruptedError:
- raise
- except Exception:
- # Interrupt state is best-effort UX; never a new failure mode.
- pass
- def _watchdog_fire(self) -> None:
- # Re-armable: if progress moved the deadline forward, reschedule instead of killing a
- # live stream.
- remaining = self.effective_deadline() - time.monotonic()
- if remaining > 0:
- if not (self.timed_out.is_set() or self.stream_finished.is_set()):
- self._arm_timer(remaining)
- return
- self._close_client_on_timeout()
- def _arm_timer(self, delay: float) -> None:
- self._timer = t = threading.Timer(delay, self._watchdog_fire)
- t.daemon = True
- t.start()
- def start(self) -> None:
- """Arm the watchdog (when a total timeout exists) and run the first cancel check."""
- if self.total_timeout:
- self._arm_timer(max(self.effective_deadline() - time.monotonic(), 0.0))
- self.check_cancelled()
- def on_event(self, _event: Any) -> None:
- # TTFP telemetry records every frame, but forward progress (compression commit fence,
- # no-progress window) counts only substantive payloads — keepalives must not re-arm,
- # so a zombie stream dies at the same window as a dead connection.
- # #93650: keep bulk wire-format payload out of the SDK's GIL-holding request transform on auxiliary
- # calls too.
- if _codex_event_has_content(_event):
- self.record_progress()
- self.saw_content.set()
- _notify_aux_provider_response()
- else:
- _notify_aux_timing_response()
- self.check_cancelled()
- def finish(self) -> None:
- """Owner ``finally``: stop the watchdog and release FDs a stranger-thread timeout only shut down."""
- self.stream_finished.set()
- if self._timer is not None:
- self._timer.cancel()
- # Gated on timeout_release_pending, NOT timed_out: after a hard-cancel the shared
- # client must stay usable for other sessions.
- if self.timeout_release_pending.is_set():
- _close_quietly(self._client, "owner-thread close after timeout failed")
- class _CodexCompletionsAdapter:
- """Drop-in shim routing chat.completions.create() kwargs through Codex Responses streaming."""
- def __init__(self, real_client: OpenAI, model: str):
- self._client = real_client
- self._model = model
- def _build_responses_kwargs(self, kwargs: dict[str, Any]) -> tuple[dict[str, Any], str, Any]:
- """chat.completions kwargs → Responses API kwargs, ``(resp_kwargs, model, timeout)``; mirrors codex.py::build_kwargs."""
- # Separate system/instructions from replayable conversation messages, then route the rest through
- # the SINGLE shared chat->Responses converter used by the main agent transport
- # (agent/transports/codex.py). Maintaining a private conversion loop here let chat-style messages
- # with role="tool" leak straight into Responses input[] — which the Responses API rejects with
- # "Invalid value: 'tool'. Supported values are: 'assistant', 'system', 'developer', and 'user'."
- # (issue #5709, hit hard by flush_memories() / compression replaying real session history that
- # includes assistant tool_calls + role="tool" results). The shared converter encodes assistant tool
- # calls as `function_call` items and tool results as `function_call_output` items with a valid
- # call_id, so every Responses path normalizes tool history identically and cannot drift.
- from agent.codex_responses_adapter import (
- _chat_messages_to_responses_input,
- _classify_responses_issuer,
- _responses_tools,
- _role_message_item,
- _wire_model_identity,
- classify_responses_route,
- )
- from agent.transports.codex import _alias_wire_tools
- model = kwargs.get("model", self._model)
- wire_model = _wire_model_identity(model)
- host = str(getattr(self._client, "base_url", "") or "")
- # Same route classifier as the main transport, so the issuer stamp matches what it minted.
- route = classify_responses_route(SimpleNamespace(provider=None, base_url=host))
- is_xai = route.is_xai_responses
- tools = kwargs.get("tools")
- if tools:
- # xAI Responses rejects ``pattern``/``format`` JSON Schema keywords (400); strip for
- # chat_completion_helpers.py parity. Deep-copy first — sanitizers mutate inner dicts
- # in place and would strip the caller's tool registry.
- try:
- import copy as _copy
- from tools.schema_sanitizer import strip_pattern_and_format, strip_slash_enum
- tools = _copy.deepcopy(list(tools))
- tools, _ = strip_pattern_and_format(tools)
- tools, _ = strip_slash_enum(tools)
- except Exception as exc:
- logger.warning(
- "Auxiliary client: failed to sanitize tool schemas for "
- "Codex/xAI Responses path: %s", exc,
- )
- # Tool schemas go through the SAME converter (``strict: False``) and the SAME reserved-name
- # aliasing (OpenCode / Perplexity / xAI) as agent/transports/codex.py::build_kwargs. A private
- # list here sent raw names without ``strict``, so title/compression/MoA calls 400ed on routes
- # where the main loop worked (#114260). The alias map is request-local: it rides on the payload
- # (``_wire_aliases``, popped in ``create()``), never on the instance — aux adapters are shared.
- wire_tools, wire_aliases = _alias_wire_tools(
- _responses_tools(tools),
- {"provider": getattr(self._client, "_hermes_aux_effective_provider", "") or None, "base_url": host},
- is_xai,
- )
- renamed = {original: alias for alias, original in wire_aliases.items()}
- # System → ``instructions``; the rest goes through the SINGLE shared chat→Responses
- # converter (a private loop here once let role="tool" leak into input[]; the shared one
- # encodes tool history as function_call/function_call_output).
- instructions = "You are a helpful assistant."
- replay_messages: list[dict[str, Any]] = []
- for msg in kwargs.get("messages", []):
- content = msg.get("content") or ""
- if msg.get("role", "user") == "system":
- instructions = content if isinstance(content, str) else str(content)
- continue
- # Replayed history names each tool the way THIS request declares it.
- if renamed and msg.get("tool_calls"):
- msg = {**msg, "tool_calls": [
- {**tc, "function": {**tc["function"], "name": renamed[tc["function"]["name"]]}}
- if isinstance(tc, dict) and (tc.get("function") or {}).get("name") in renamed else tc
- for tc in msg["tool_calls"]
- ]}
- replay_messages.append(msg)
- # Endpoints that bind replayed codex_message_items ids to a backend connection 401 on a stale id
- # (``replay_item_ids=False``) — same guard as build_kwargs. Aux calls
- # never send ``context_management`` (main-turn feature): no compaction checkpoint.
- # Auxiliary calls (context compression, flush_memories, MoA aggregation) go through this adapter
- # instead of agent/transports/codex.py's build_kwargs, so they need the same guard applied
- # independently. See #32716.
- # Aux requests run their own model; stamp/filter reasoning provenance against it, not the main agent's.
- issuer_kind = _classify_responses_issuer(base_url=host, **route._asdict())
- input_items = _chat_messages_to_responses_input(
- replay_messages, drop_replay_item_ids=not route.endpoint.replay_item_ids,
- current_issuer_kind=issuer_kind,
- current_issuer_model=wire_model, native_compaction_eligible=False,
- )
- resp_kwargs: dict[str, Any] = {
- # Codex only knows the base slug; strip the Hermes ``-900k`` picker suffix.
- "model": wire_model, "instructions": instructions,
- "input": input_items or [_role_message_item("user", "")], "store": False,
- }
- # Forward the chat.completions timeout; otherwise a Codex stream can sit behind a
- # dead-looking CLI until the user force-interrupts.
- timeout = kwargs.get("timeout")
- if timeout is not None:
- resp_kwargs["timeout"] = timeout
- # Per-request HTTP headers (OpenCode session affinity, provider request headers) map to real
- # headers via the SDK kwarg — forward them.
- if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
- resp_kwargs["extra_headers"] = dict(kwargs["extra_headers"])
- # The Codex endpoint rejects max_output_tokens/temperature (400) — omit.
- extra_body = kwargs.get("extra_body") or {}
- if isinstance(extra_body, dict):
- # service_tier (fast mode) is a top-level Responses field; xAI's endpoint rejects it.
- service_tier = extra_body.get("service_tier")
- if isinstance(service_tier, str) and service_tier.strip() and not is_xai:
- resp_kwargs["service_tier"] = service_tier.strip()
- reasoning_cfg = extra_body.get("reasoning")
- if isinstance(reasoning_cfg, dict):
- # Shared per-model vocabulary with the main transport ("max" only where the model publishes it; "minimal"/"ultra"
- # clamp to a listed level; ``()`` = the model takes no ``reasoning`` field at all — gpt-4o/4.1 on api.openai.com,
- # #76255). ``enabled: False`` goes on the wire as ``effort: none`` where the vocabulary has it,
- # since an omitted field leaves the model's default effort on (#75227).
- from agent.reasoning_effort import clamp_effort
- from agent.transports.codex import _codex_efforts_for_route
- supported = _codex_efforts_for_route(model, host, is_codex_backend=route.is_codex_backend)
- if supported and reasoning_cfg.get("enabled") is not False:
- # Truthy-only: Codex 400s on e.g. {"effort": null}, so falsy → default.
- effort = clamp_effort(reasoning_cfg.get("effort") or "medium", supported)
- resp_kwargs["reasoning"] = {"effort": effort, "summary": "auto"}
- resp_kwargs["include"] = ["reasoning.encrypted_content"]
- elif "none" in supported and not is_xai:
- resp_kwargs["reasoning"] = {"effort": "none"}
- if wire_tools:
- resp_kwargs["tools"] = wire_tools
- if wire_aliases:
- resp_kwargs["_wire_aliases"] = wire_aliases
- # Response normalization is route-sensitive too (popped in ``create()``, never sent).
- resp_kwargs["_issuer_kind"] = issuer_kind
- # Stable prompt-cache routing: key is content-addressed from the static prefix
- # (instructions + tool schemas) so it survives across turns, scoped by the owning
- # conversation (rotation-stable logical scope, else the physical session id). Skip the
- # key where the main transport does: xAI takes it in extra_body, ``prompt_cache_key=False`` endpoints opt out.
- try:
- # Reuse the Responses transport's single authoritative hash algorithm and session-scope
- # normalization so equivalent static prefixes route to the same cache bucket across modes,
- # without concentrating unrelated sessions into one shared bucket (see #78941).
- from agent.transports.codex import _cache_scope_from_session_id, _content_cache_key
- from agent.transports.codex import _default_prompt_cache_retention_for_request
- if not is_xai and route.endpoint.prompt_cache_key and "prompt_cache_key" not in resp_kwargs:
- scope = _cache_scope_from_session_id(
- _runtime_main_value("cache_scope") or _runtime_main_value("session_id")
- )
- cache_key = _content_cache_key(resp_kwargs["instructions"], resp_kwargs.get("tools"), scope)
- if cache_key:
- resp_kwargs["prompt_cache_key"] = cache_key
- if "prompt_cache_retention" not in resp_kwargs:
- cache_retention = _default_prompt_cache_retention_for_request(model, host)
- if cache_retention:
- resp_kwargs["prompt_cache_retention"] = cache_retention
- except Exception:
- logger.debug("Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True)
- # Last, like the main transport: caller extra_body must not put a rejected Astra field back.
- from agent.transports.codex import _sanitize_astra_request_kwargs
- _sanitize_astra_request_kwargs(resp_kwargs, model, host)
- return resp_kwargs, model, timeout
- def create(self, **kwargs) -> Any:
- from hermes_cli.providers import is_actual_route
- if is_actual_route(
- getattr(self._client, "_hermes_aux_effective_provider", ""),
- str(getattr(self._client, "base_url", "") or ""),
- ):
- raise ValueError(
- "Actual requests require Chat Completions; refusing to call /responses."
- )
- # Low-level ``responses.create(stream=True)`` and assemble the final response ourselves
- # from ``response.output_item.done``: the high-level ``responses.stream()`` rebuilds from
- # ``response.completed.response.output``, which Codex returns as ``null`` (SDK crash).
- resp_kwargs, model, timeout = self._build_responses_kwargs(kwargs)
- wire_aliases = resp_kwargs.pop("_wire_aliases", None) or {}
- issuer_kind = resp_kwargs.pop("_issuer_kind", None)
- issuer_model = str(resp_kwargs.get("model") or model)
- total_timeout = timeout if isinstance(timeout, (int, float)) and timeout > 0 else None
- guard = _CodexStreamGuard(self._client, total_timeout, no_progress_timeout=kwargs.get("no_progress_timeout"))
- try:
- guard.start()
- from agent.codex_runtime import _consume_codex_event_stream
- from agent.sdk_transform_bypass import bypass_sdk_request_transform
- # Keep bulk wire payload out of the SDK's GIL-holding request transform.
- stream_kwargs = bypass_sdk_request_transform({**resp_kwargs, "stream": True})
- event_stream = self._client.responses.create(**stream_kwargs)
- guard.adopt_stream(event_stream)
- # The timer may fire while responses.create() is blocked; if the cancelled attempt
- # had no stream to close then, close it now that it is attempt-owned — never the shared client.
- if guard.timed_out.is_set() and guard.cancel_requested():
- guard.close_attempt_stream("late cancelled attempt stream close failed")
- try:
- # Some Codex-compatible hosts accept ``stream=True`` but return a completed
- # Responses object (not iterable) — don't hand it to the consumer.
- if hasattr(event_stream, "output"):
- final = event_stream
- else:
- final = _consume_codex_event_stream(
- event_stream, model=issuer_model, on_event=guard.on_event
- )
- finally:
- guard.release_stream(event_stream)
- if final is None:
- raise RuntimeError("Codex auxiliary Responses stream did not return a final response")
- from agent.auxiliary_codex_response import _parse_codex_final_response
- text_parts, tool_calls_raw, usage, finish_reason = _parse_codex_final_response(
- final, issuer_kind=issuer_kind, issuer_model=issuer_model,
- )
- # Undo only the aliases THIS request emitted, before the call reaches Hermes dispatch.
- for tc in tool_calls_raw or ():
- if tc.function.name in wire_aliases:
- tc.function.name = wire_aliases[tc.function.name]
- except Exception as exc:
- if guard.timed_out.is_set():
- raise TimeoutError(guard.timeout_message()) from exc
- logger.debug("Codex auxiliary Responses API call failed: %s", exc)
- raise
- finally:
- guard.finish()
- # Shape the result like chat.completions.
- message = SimpleNamespace(
- role="assistant", content="".join(text_parts).strip() or None,
- tool_calls=tool_calls_raw or None,
- )
- choice = SimpleNamespace(index=0, message=message, finish_reason=finish_reason)
- return SimpleNamespace(choices=[choice], model=model, usage=usage)
- class _ChatShim:
- """Exposes ``client.chat.completions.create()`` over a sync or async adapter."""
- def __init__(self, adapter: Any):
- self.completions = adapter
- class _AsyncCompletionsAdapter:
- """Async adapter: runs the sync adapter's ``create`` via asyncio.to_thread()."""
- def __init__(self, sync_adapter: Any):
- self._sync = sync_adapter
- async def create(self, **kwargs) -> Any:
- import asyncio
- return await asyncio.to_thread(self._sync.create, **kwargs)
- class _AsyncAuxiliaryClientBase:
- """Async-compatible wrapper matching AsyncOpenAI.chat.completions.create().
- Mirrors ``_real_client`` (when the sync wrapper has one) so cache eviction by
- leaf OpenAI client drops this async entry too instead of reusing a closed transport.
- """
- def __init__(self, sync_wrapper: Any):
- self.chat = _ChatShim(_AsyncCompletionsAdapter(sync_wrapper.chat.completions))
- self.api_key = sync_wrapper.api_key
- self.base_url = sync_wrapper.base_url
- if hasattr(sync_wrapper, "_real_client"):
- # Mirror the sync wrapper's _real_client so cache eviction by leaf OpenAI client (e.g.
- # _close_client_on_timeout in #23482) drops this async entry too. Without this, sync and async
- # cache entries diverge on poisoning: the sync entry is evicted but the async entry keeps
- # reusing the closed transport, failing every subsequent async aux call with 'Connection error'
- # until the gateway restarts.
- self._real_client = sync_wrapper._real_client
- _AsyncAnthropicCompletionsAdapter = _AsyncCompletionsAdapter # imported by tests
- class CodexAuxiliaryClient:
- """OpenAI-client-compatible wrapper routing through the Codex Responses API (.api_key/.base_url for introspection)."""
- def __init__(self, real_client: OpenAI, model: str):
- self._real_client = real_client
- self.chat = _ChatShim(_CodexCompletionsAdapter(real_client, model))
- self.api_key = real_client.api_key
- self.base_url = real_client.base_url
- def close(self):
- self._real_client.close()
- class AsyncCodexAuxiliaryClient(_AsyncAuxiliaryClientBase):
- pass
- def _translate_anthropic_response_format(anthropic_kwargs: dict[str, Any], response_format: Any) -> None:
- """Merge an OpenAI response format into Anthropic ``output_config``."""
- if not isinstance(response_format, dict):
- return
- format_type = response_format.get("type")
- if format_type == "json_schema":
- json_schema = response_format.get("json_schema")
- if not isinstance(json_schema, dict) or "schema" not in json_schema:
- return
- schema = json_schema["schema"]
- elif format_type == "json_object":
- # Anthropic SDK has no schema-less JSON mode; only ``json_schema``.
- schema = {"type": "object"}
- else:
- return
- output_config = anthropic_kwargs.get("output_config")
- if not isinstance(output_config, dict):
- output_config = {}
- anthropic_kwargs["output_config"] = output_config
- output_config["format"] = {"type": "json_schema", "schema": schema}
- class _AnthropicCompletionsAdapter:
- """OpenAI-client-compatible adapter for Anthropic Messages API."""
- def __init__(self, real_client: Any, model: str, is_oauth: bool = False, base_url: str | None = None):
- self._client = real_client
- self._model = model
- self._is_oauth = is_oauth
- # Caller URL first; fall back to the SDK client's host only for Nous Portal — a blanket
- # fallback would flip MiniMax/Zhipu aux adapters to third-party handling (strips thinking sigs).
- self._base_url = base_url or None
- if not self._base_url:
- candidate = str(getattr(real_client, "base_url", "") or "") or None
- if candidate:
- with contextlib.suppress(Exception):
- from agent.anthropic_endpoints import _is_nous_portal_endpoint
- if _is_nous_portal_endpoint(candidate):
- self._base_url = candidate
- def create(self, **kwargs) -> Any:
- from agent.anthropic_adapter import build_anthropic_kwargs, create_anthropic_message
- from agent.transports import get_transport
- model = kwargs.get("model", self._model)
- # ZAI's Anthropic endpoint rejects max_tokens on vision models (code 1210);
- # callers signal this via _skip_zai_max_tokens.
- if kwargs.pop("_skip_zai_max_tokens", False):
- max_tokens = None
- else:
- max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
- temperature = kwargs.get("temperature")
- # Reasoning priority: explicit per-call _reasoning_config (MoA per-slot) wins over
- # extra_body.reasoning; build_anthropic_kwargs translates to ``thinking``.
- reasoning_cfg = kwargs.get("_reasoning_config")
- if reasoning_cfg is None:
- _eb = kwargs.get("extra_body")
- _rc = _eb.get("reasoning") if isinstance(_eb, dict) else None
- if isinstance(_rc, dict):
- reasoning_cfg = _rc
- # OpenAI tool_choice (str or dict) → Anthropic-style name/mode string.
- tool_choice = kwargs.get("tool_choice")
- if isinstance(tool_choice, dict):
- choice_type = str(tool_choice.get("type", "")).lower()
- if choice_type == "function":
- tool_choice = tool_choice.get("function", {}).get("name")
- else:
- tool_choice = choice_type if choice_type in {"auto", "required", "none"} else None
- elif not isinstance(tool_choice, str):
- tool_choice = None
- anthropic_kwargs = build_anthropic_kwargs(
- model=model, messages=kwargs.get("messages", []), tools=kwargs.get("tools"),
- max_tokens=max_tokens, reasoning_config=reasoning_cfg, tool_choice=tool_choice,
- is_oauth=self._is_oauth,
- # Portal routes on ``anthropic/<slug>`` ids and replays signed thinking
- # keyed off base_url; omitting it breaks Portal model resolution.
- base_url=self._base_url,
- )
- # Opus 4.7+ rejects non-default temperature/top_p/top_k; build_anthropic_kwargs
- # also strips these as a safety net — keep both layers.
- if temperature is not None:
- from agent.anthropic_adapter import _forbids_sampling_params
- if not _forbids_sampling_params(model):
- anthropic_kwargs["temperature"] = temperature
- # Per-request HTTP headers (OpenCode session affinity) — the Anthropic SDK accepts
- # ``extra_headers`` on messages.create/stream too.
- if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
- anthropic_kwargs["extra_headers"] = {
- **(anthropic_kwargs.get("extra_headers") or {}),
- **kwargs["extra_headers"],
- }
- # response_format: top-level gets the same translation as the extra_body form; when both
- # are present the extra_body form wins. Passthrough excludes ``reasoning``/``response_format``
- # (already TRANSLATED to native fields — raw would 400 on strict gateways) and ``_`` Hermes plumbing.
- # The adapter builds the Messages body from a fixed allow-list of kwargs, so before this an
- # unrecognized top-level kwarg was dropped on the floor: the request succeeded but the schema
- # contract silently became prompt compliance (#85626 review, point 2).
- top_level_response_format = kwargs.get("response_format")
- if top_level_response_format is not None:
- _translate_anthropic_response_format(anthropic_kwargs, top_level_response_format)
- caller_extra_body = kwargs.get("extra_body")
- if caller_extra_body and isinstance(caller_extra_body, dict):
- _translate_anthropic_response_format(anthropic_kwargs, caller_extra_body.get("response_format"))
- passthrough = {
- k: v for k, v in caller_extra_body.items()
- if k not in {"reasoning", "response_format"} and not str(k).startswith("_")
- }
- if passthrough:
- existing = anthropic_kwargs.get("extra_body") or {}
- if not isinstance(existing, dict):
- existing = {}
- anthropic_kwargs["extra_body"] = {**existing, **passthrough}
- response = create_anthropic_message(
- self._client,
- anthropic_kwargs,
- # Record provider-response timing every event, but tick forward progress only for
- # substantive payloads so keepalives can't hold a stalled summary open. None keeps
- # the fast get_final_message path.
- on_stream_event=(_anthropic_aux_stream_event_hook() if _aux_progress_active() else None),
- )
- _nr = get_transport("anthropic_messages").normalize_response(response, strip_tool_prefix=self._is_oauth)
- usage = None
- if hasattr(response, "usage") and response.usage:
- prompt_tokens = getattr(response.usage, "input_tokens", 0) or 0
- completion_tokens = getattr(response.usage, "output_tokens", 0) or 0
- usage = SimpleNamespace(
- prompt_tokens=prompt_tokens, completion_tokens=completion_tokens,
- total_tokens=getattr(response.usage, "total_tokens", 0) or (prompt_tokens + completion_tokens),
- )
- # ToolCall already duck-types as OpenAI shape via properties.
- choice = SimpleNamespace(
- index=0,
- message=SimpleNamespace(content=_nr.content, tool_calls=_nr.tool_calls, reasoning=_nr.reasoning),
- finish_reason=_nr.finish_reason,
- )
- return SimpleNamespace(choices=[choice], model=model, usage=usage)
- class AnthropicAuxiliaryClient:
- """OpenAI-client-compatible wrapper over a native Anthropic client."""
- def __init__(self, real_client: Any, model: str, api_key: str, base_url: str, is_oauth: bool = False):
- self._real_client = real_client
- self.chat = _ChatShim(_AnthropicCompletionsAdapter(real_client, model, is_oauth=is_oauth, base_url=base_url))
- self.api_key = api_key
- self.base_url = base_url
- def close(self):
- close_fn = getattr(self._real_client, "close", None)
- if callable(close_fn):
- close_fn()
- class AsyncAnthropicAuxiliaryClient(_AsyncAuxiliaryClientBase):
- pass
- class _BedrockCompletionsAdapter:
- """Translates ``chat.completions.create(**kwargs)`` into Bedrock Converse."""
- def __init__(self, region: str, model: str):
- self._region = region
- self._model = model
- def create(self, **kwargs) -> Any:
- from agent.bedrock_adapter import call_converse
- model = kwargs.get("model", self._model)
- max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
- # OpenAI accepts ``stop`` as str or list; Converse requires a list.
- stop = kwargs.get("stop")
- if isinstance(stop, str):
- stop = [stop]
- if kwargs.get("tool_choice") is not None:
- # Converse toolChoice isn't wired through call_converse(); surface the drop.
- logger.debug(
- "BedrockAuxiliaryClient: tool_choice=%r not supported by the "
- "Converse shim — ignored.", kwargs.get("tool_choice"),
- )
- if kwargs.get("stream"):
- # Converse streaming isn't wired here; call_llm's streaming consumer
- # detects a final object and downgrades to non-live output.
- logger.debug(
- "BedrockAuxiliaryClient: stream=True requested for %s — returning a complete response "
- "(Converse shim does not stream); caller downgrades to non-streaming.", model,
- )
- response = call_converse(
- region=self._region, model=model, messages=kwargs.get("messages", []), tools=kwargs.get("tools"),
- # Converse specifically defaults to the model maximum when omitted.
- # Truthiness mirrors the Anthropic shim: explicit 0 means omit.
- max_tokens=int(max_tokens) if max_tokens else None, temperature=kwargs.get("temperature"),
- top_p=kwargs.get("top_p"), stop_sequences=stop,
- )
- # Converse is complete-response here: mark provider progress only after
- # return so TTFP reflects real Bedrock latency, not dispatch/setup.
- _notify_aux_provider_response()
- return response
- class BedrockAuxiliaryClient:
- """OpenAI-client-compatible wrapper over AWS Bedrock Converse API."""
- def __init__(self, region: str, model: str):
- self._region = region
- self._model = model
- self.chat = _ChatShim(_BedrockCompletionsAdapter(region, model))
- self.api_key = "aws-sdk"
- self.base_url = f"https://bedrock-runtime.{region}.amazonaws.com"
- def close(self):
- pass
- class AsyncBedrockAuxiliaryClient(_AsyncAuxiliaryClientBase):
- pass
- def _endpoint_speaks_anthropic_messages(base_url: str) -> bool:
- """True if ``base_url`` speaks Anthropic Messages, not OpenAI chat.completions.
- Mirrors ``hermes_cli.runtime_provider._detect_api_mode_for_url`` so aux and main agree: any
- ``/anthropic`` URL (MiniMax, Zhipu, LiteLLM), ``api.kimi.com/coding`` (chat 404s), ``api.anthropic.com``.
- """
- normalized = (base_url or "").strip().lower().rstrip("/")
- if not normalized:
- return False
- if urlparse(normalized).path.rstrip("/").endswith(("/anthropic", "/anthropic/v1")):
- return True
- hostname = base_url_hostname(normalized)
- return hostname == "api.anthropic.com" or bool(hostname == "api.kimi.com" and "/coding" in normalized)
- def _maybe_wrap_anthropic(
- client_obj: Any, model: str, api_key: str, base_url: str, api_mode: Optional[str] = None
- ) -> Any:
- """Rewrap a plain OpenAI client in ``AnthropicAuxiliaryClient`` when the endpoint speaks Anthropic Messages.
- Single transport-correction chokepoint at the end of every ``resolve_provider_client`` branch; returns
- ``client_obj`` unchanged for probe stubs/specialized adapters, OpenAI-wire, explicit non-Anthropic
- ``api_mode``, or missing ``anthropic`` SDK.
- """
- # Anthropic/Bedrock/Codex wrappers, plus any client declaring HERMES_SKIP_TRANSPORT_WRAP
- # (native/ACP shims, in-tree or plugin), must never be re-dispatched through a wire adapter —
- # a class-attribute declaration rather than isinstance so this hot path never imports them.
- if (
- isinstance(client_obj, _AuxProbeClientStub)
- or _safe_isinstance(client_obj, (AnthropicAuxiliaryClient, BedrockAuxiliaryClient, CodexAuxiliaryClient))
- or _client_declares(client_obj, "HERMES_SKIP_TRANSPORT_WRAP")
- ):
- return client_obj
- # Explicit non-anthropic api_mode wins over URL heuristics.
- if api_mode != "anthropic_messages" and (api_mode or not _endpoint_speaks_anthropic_messages(base_url)):
- return client_obj
- try:
- from agent.anthropic_adapter import build_anthropic_client
- except ImportError:
- logger.warning(
- "Endpoint %s speaks Anthropic Messages but the anthropic SDK is "
- "not installed — falling back to OpenAI-wire (will likely 404).",
- base_url,
- )
- return client_obj
- try:
- real_client = build_anthropic_client(api_key, base_url)
- except Exception as exc:
- logger.warning(
- "Failed to build Anthropic client for %s (%s) — falling back to "
- "OpenAI-wire client.", base_url, exc,
- )
- return client_obj
- logger.debug(
- "Auxiliary transport: wrapping client in AnthropicAuxiliaryClient "
- "(model=%s, base_url=%s, api_mode=%s)",
- model, base_url[:60] if base_url else "", api_mode or "auto-detected",
- )
- return AnthropicAuxiliaryClient(real_client, model, api_key, base_url, is_oauth=False)
- def _read_nous_auth() -> Optional[dict]:
- """Nous provider state dict from the credential pool or ~/.hermes/auth.json; None when not active with tokens."""
- pool_present, entry = _select_pool_entry("nous")
- if pool_present:
- if entry is None:
- return None
- return {
- "access_token": getattr(entry, "access_token", ""),
- "refresh_token": getattr(entry, "refresh_token", None),
- "agent_key": getattr(entry, "agent_key", None),
- "inference_base_url": _pool_runtime_base_url(entry, _NOUS_DEFAULT_BASE_URL),
- "portal_base_url": getattr(entry, "portal_base_url", None),
- "client_id": getattr(entry, "client_id", None),
- "scope": getattr(entry, "scope", None),
- "token_type": getattr(entry, "token_type", "Bearer"),
- "source": "pool",
- }
- try:
- auth_path = _auth_json_path()
- if not auth_path.is_file():
- return None
- data = json.loads(auth_path.read_text(encoding="utf-8-sig"))
- if data.get("active_provider") != "nous":
- return None
- provider = data.get("providers", {}).get("nous", {})
- # Must have at least an access_token or agent_key.
- if not provider.get("agent_key") and not provider.get("access_token"):
- return None
- return provider
- except Exception as exc:
- logger.debug("Could not read Nous auth: %s", exc)
- return None
- def _nous_api_key(provider: dict) -> str:
- """Extract a usable Nous inference JWT from stored auth state."""
- from hermes_cli.auth import _nous_invoke_jwt_is_usable
- for token_key, expiry_key in (("agent_key", "agent_key_expires_at"), ("access_token", "expires_at")):
- token = provider.get(token_key)
- if not isinstance(token, str) or not token.strip():
- continue
- if _nous_invoke_jwt_is_usable(token, scope=provider.get("scope"), expires_at=provider.get(expiry_key)):
- return token
- return ""
- def _resolve_nous_pool_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[str, str]]:
- """Resolve Nous auxiliary credentials from the selected pool entry."""
- try:
- from hermes_cli.auth import _agent_key_is_usable
- pool = load_pool("nous")
- except Exception as exc:
- logger.debug("Auxiliary Nous pool credential resolution failed: %s", exc)
- return None
- if not pool or not pool.has_credentials():
- return None
- try:
- entry = pool.select()
- except Exception as exc:
- logger.debug("Auxiliary Nous pool selection failed: %s", exc)
- return None
- if entry is None:
- return None
- def _entry_state(e: Any) -> dict[str, Any]:
- return {k: getattr(e, k, None) for k in (
- "agent_key", "agent_key_expires_at", "access_token", "expires_at", "scope")}
- if force_refresh or not _agent_key_is_usable(_entry_state(entry), _nous_min_key_ttl_seconds()):
- try:
- refreshed = pool.try_refresh_current()
- except Exception as exc:
- logger.debug("Auxiliary Nous pool refresh failed: %s", exc)
- refreshed = None
- if refreshed is None:
- return None
- entry = refreshed
- api_key = _nous_api_key(_entry_state(entry))
- base_url = _pool_runtime_base_url(entry, _NOUS_DEFAULT_BASE_URL)
- if not api_key or not base_url:
- return None
- return api_key, base_url
- def _resolve_nous_runtime_api(
- *, force_refresh: bool = False, stale_access_token: Optional[str] = None
- ) -> Optional[tuple[str, str]]:
- """Fresh Nous runtime credentials (pool first, then auth store + JWT refresh) — mirrors the main
- agent's 401 recovery. ``stale_access_token`` is the bearer that just 401'd; with ``force_refresh``
- it lets the auth store adopt a sibling process's rotation instead of re-POSTing the shared grant."""
- pooled = _resolve_nous_pool_runtime_api(force_refresh=force_refresh)
- if pooled is not None:
- return pooled
- try:
- from hermes_cli.auth import resolve_nous_runtime_credentials
- creds = resolve_nous_runtime_credentials(
- timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15),
- force_refresh=force_refresh,
- stale_access_token=stale_access_token or None,
- )
- except Exception as exc:
- # Kept at WARNING (once per message) and remembered: the ladder falls back silently, and
- # without this the goal judge only ever saw "judge error: RuntimeError" (#42177).
- record_nous_credential_failure(exc)
- return None
- clear_nous_credential_failure()
- return _creds_pair(creds)
- def _creds_pair(creds: dict[str, Any]) -> Optional[tuple[str, str]]:
- """``(api_key, base_url)`` from a runtime-credentials dict, or None when either is missing."""
- api_key = str(creds.get("api_key") or "").strip()
- base_url = str(creds.get("base_url") or "").strip().rstrip("/")
- if not api_key or not base_url:
- return None
- return api_key, base_url
- def _resolve_xai_oauth_for_aux() -> Optional[tuple[str, str]]:
- """Fresh xAI OAuth (api_key, base_url) for aux clients, or None.
- Pool first (some xAI OAuth logins exist only as pool entries), then the singleton auth-store resolver.
- """
- try:
- from hermes_cli.auth import DEFAULT_XAI_OAUTH_BASE_URL, _xai_validate_inference_base_url
- pool = load_pool("xai-oauth")
- if pool and pool.has_credentials():
- entry = pool.select()
- if entry is not None:
- api_key = str(
- getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "") or ""
- ).strip()
- _url = lambda v: str(v or "").strip().rstrip("/")
- base_url = _xai_validate_inference_base_url(
- _url(_scoped_key_env("HERMES_XAI_BASE_URL"))
- or _url(_scoped_key_env("XAI_BASE_URL"))
- or _url(getattr(entry, "runtime_base_url", None))
- or _url(getattr(entry, "base_url", None)),
- fallback=DEFAULT_XAI_OAUTH_BASE_URL,
- )
- if api_key and base_url:
- return api_key, base_url
- except Exception as exc:
- logger.debug("Auxiliary xAI OAuth pool credential resolution failed: %s", exc)
- try:
- from hermes_cli.auth import resolve_xai_oauth_runtime_credentials
- creds = resolve_xai_oauth_runtime_credentials()
- except Exception as exc:
- logger.debug("Auxiliary xAI OAuth runtime credential resolution failed: %s", exc)
- return None
- return _creds_pair(creds)
- def _resolve_codex_credential_and_base() -> tuple[Optional[str], str]:
- """``(token, base_url)`` taken from ONE authority, so a Codex key is only ever sent to the host
- it belongs to (#121486): the profile-scoped ``HERMES_CODEX_BASE_URL`` wins; otherwise a pooled
- key goes where that pool entry routes (row URL / ``model.base_url``) and the auth.json OAuth
- token goes to the ChatGPT default. ``(None, <base>)`` without a usable token."""
- override = _codex_base_url_override()
- pool_present, entry = _select_pool_entry("openai-codex")
- if pool_present:
- token = _pool_runtime_api_key(entry)
- if token:
- if override:
- return token, override
- # Same route rule as the chat path (row URL, else ``model.base_url``); never empty.
- from hermes_cli.auth_codex import _codex_pool_route_base_url
- return token, _codex_pool_route_base_url(_pool_runtime_base_url(entry))
- # No usable pool token: auth.json only (re-selecting could pair another row's key with the default).
- return _read_codex_singleton_token(), override or _CODEX_AUX_BASE_URL
- def _read_codex_singleton_token() -> Optional[str]:
- """The profile's auth.json Codex access token (expired JWTs skipped), else None."""
- try:
- from hermes_cli.auth import _read_codex_tokens
- access_token = _read_codex_tokens().get("tokens", {}).get("access_token")
- if not isinstance(access_token, str) or not access_token.strip():
- return None
- # Expired JWTs would block the auto chain and prevent fallback to working providers.
- try:
- import base64
- payload = access_token.split(".")[1]
- payload += "=" * (-len(payload) % 4)
- exp = json.loads(base64.urlsafe_b64decode(payload)).get("exp", 0)
- if exp and time.time() > exp:
- logger.debug("Codex access token expired (exp=%s), skipping", exp)
- return None
- except Exception:
- pass # Non-JWT token or decode error — use as-is
- return access_token.strip()
- except Exception as exc:
- logger.debug("Could not read Codex auth for auxiliary client: %s", exc)
- return None
- def _resolve_api_key_provider() -> tuple[Optional[OpenAI], Optional[str]]:
- """Try each API-key provider in PROVIDER_REGISTRY order; (client, model) or (None, None)."""
- try:
- from hermes_cli.auth import PROVIDER_REGISTRY, resolve_api_key_provider_credentials
- except ImportError:
- logger.debug("Could not import PROVIDER_REGISTRY for API-key fallback")
- return None, None
- from hermes_cli.auth_plugin_providers import canonical_registry_rows
- for provider_id, pconfig in canonical_registry_rows():
- if pconfig.auth_type != "api_key":
- continue
- if _is_provider_unhealthy(provider_id):
- logger.debug("Auxiliary api-key chain: %s is unhealthy, skipping", provider_id)
- continue
- if provider_id == "anthropic":
- # Explicit-config gate: Claude Code credentials must not silently become aux fallback.
- with contextlib.suppress(ImportError):
- from hermes_cli.auth import is_provider_explicitly_configured
- if not is_provider_explicitly_configured("anthropic"):
- continue
- return _try_anthropic()
- from providers import get_provider_profile
- if getattr(get_provider_profile(provider_id), "ambient_credentials", False):
- # Explicit-config gate: ambient credentials (another tool's login) must not silently become aux fallback (#114740).
- from hermes_cli.auth import is_provider_explicitly_configured
- if not is_provider_explicitly_configured(provider_id):
- continue
- model = _get_aux_model_for_provider(provider_id) or None
- if model is None:
- continue # skip provider if we don't know a valid aux model
- pool_present, entry = _select_pool_entry(provider_id)
- if pool_present:
- api_key = _pool_runtime_api_key(entry)
- if not api_key:
- continue
- raw_base_url = _pool_runtime_base_url(entry, pconfig.inference_base_url) or pconfig.inference_base_url
- via = " via pool"
- else:
- creds = resolve_api_key_provider_credentials(provider_id)
- api_key = str(creds.get("api_key", "")).strip()
- if not api_key:
- continue
- raw_base_url = str(creds.get("base_url", "")).strip().rstrip("/") or pconfig.inference_base_url
- via = ""
- # The session's own endpoint wins for its provider: the key was issued for that gateway, and
- # sending it to the registry default 401s, then quarantines the provider the main model is on.
- runtime = _normalize_main_runtime(None)
- if runtime.get("provider") == provider_id and runtime.get("base_url"):
- raw_base_url = runtime["base_url"].rstrip("/")
- if isinstance(runtime.get("api_key"), str) and runtime["api_key"]:
- api_key = runtime["api_key"]
- via = " (session endpoint)"
- logger.debug("Auxiliary text client: %s (%s)%s", pconfig.name, model, via)
- # Native Gemini, else OpenAI-wire + Anthropic rewrap.
- base_url = _to_openai_base_url(raw_base_url)
- if provider_id == "gemini":
- from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url
- if is_native_gemini_base_url(base_url):
- return GeminiNativeClient(api_key=api_key, base_url=base_url), model
- if base_url_host_matches(base_url, "api.kimi.com"):
- headers = {"User-Agent": "claude-code/0.1.0"}
- elif base_url_host_matches(base_url, "integrate.api.nvidia.com"):
- headers = build_nvidia_nim_headers(base_url)
- else:
- headers = _route_client_headers(base_url, provider_id) or _profile_default_headers(provider_id)
- extra = {"default_headers": headers} if headers else {}
- merged = _apply_user_default_headers(extra.get("default_headers"))
- if merged:
- extra["default_headers"] = merged
- client = _create_openai_client(api_key=api_key, base_url=base_url, **extra)
- return _maybe_wrap_anthropic(client, model, api_key, raw_base_url), model
- return None, None
- def _route_client_headers(base_url: str, provider: str, *, is_vision: bool = False) -> Optional[dict]:
- from agent.provider_route_hooks import route_answer
- headers = route_answer("client_headers", provider=provider or "", base_url=base_url, has_images=is_vision)
- return dict(headers) if headers else None
- def _endpoint_default_headers(
- base_url: str, provider: str, *, is_vision: bool = False, xai: bool = False,
- ) -> Optional[dict]:
- """Provider-specific client headers by endpoint host, merged with user ``model.default_headers``.
- Kimi Code needs the claude-code User-Agent; a profile's ``client_headers`` (``is_vision`` = image
- client), NVIDIA NIM and (optionally) xAI have their own; else the provider profile.
- """
- if base_url_host_matches(base_url, "api.kimi.com"):
- headers: dict = {"User-Agent": "claude-code/0.1.0"}
- elif route := _route_client_headers(base_url, provider, is_vision=is_vision):
- headers = route
- elif base_url_host_matches(base_url, "integrate.api.nvidia.com"):
- headers = dict(build_nvidia_nim_headers(base_url))
- elif xai and base_url_host_matches(base_url, "x.ai"):
- from tools.xai_http import hermes_xai_default_headers
- headers = dict(hermes_xai_default_headers())
- else:
- headers = _profile_default_headers(provider) or {}
- return _apply_user_default_headers(headers or None) or None
- def _profile_default_headers(provider: str) -> Optional[dict]:
- """Client-level attribution headers from the provider profile (e.g. GMI User-Agent), or None."""
- if not provider:
- return None
- with contextlib.suppress(Exception):
- from providers import get_provider_profile
- profile = get_provider_profile(provider)
- if profile and profile.default_headers:
- return dict(profile.default_headers)
- return None
- # Provider resolution helpers
- _paid_lane_warned: set = set()
- def _is_free_model(model: Optional[str]) -> bool:
- """True when ``model`` is a free SKU (``:free`` suffix or ``stealth/`` prefix) — naming-convention trust."""
- if not model:
- return False
- normalized = str(model).strip()
- return normalized.endswith(":free") or normalized.startswith("stealth/")
- def _aux_openrouter_settings() -> tuple[bool, str]:
- """Read (free_only, openrouter_model) from config; (False, _OPENROUTER_MODEL) on failure."""
- try:
- from hermes_cli.config import cfg_get, load_config_readonly
- cfg = load_config_readonly()
- free_only = bool(cfg_get(cfg, "auxiliary", "free_only", default=False))
- val = cfg_get(cfg, "auxiliary", "openrouter_model")
- model = val.strip() if isinstance(val, str) and val.strip() else _OPENROUTER_MODEL
- return free_only, model
- except Exception:
- return False, _OPENROUTER_MODEL
- def _warn_paid_lane_once(model: str) -> None:
- """Log a WARNING the first time a non-free OpenRouter model is engaged."""
- if model in _paid_lane_warned:
- return
- _paid_lane_warned.add(model)
- logger.warning(
- "Auxiliary client: PAID lane engaged for auxiliary task — OpenRouter fallback model %r is not "
- "a :free SKU and may incur real spend. Set auxiliary.free_only: true to restrict auxiliary "
- "fallbacks to free models, or auxiliary.openrouter_model to a :free model.", model,
- )
- def _try_openrouter(explicit_api_key: Optional[str | Callable[[], str]] = None, model: str | None = None,
- explicit_base_url: Optional[str] = None) -> tuple[Optional[OpenAI], Optional[str]]:
- free_only, cfg_model = _aux_openrouter_settings()
- or_model = model or cfg_model
- if free_only and not _is_free_model(or_model):
- logger.warning(
- "Auxiliary client: auxiliary.free_only is enabled but the OpenRouter fallback model %r is "
- "not a :free SKU — skipping the OpenRouter fallback. Set auxiliary.openrouter_model to a "
- ":free model (e.g. nvidia/nemotron-3-ultra-550b-a55b:free) or disable auxiliary.free_only.",
- or_model,
- )
- return None, None
- if not _is_free_model(or_model):
- _warn_paid_lane_once(or_model)
- # A caller-supplied endpoint (fallback_providers entry, custom_providers entry) is
- # authoritative over both the pool row and the canonical host (#121359).
- override_url = (explicit_base_url or "").strip().rstrip("/")
- pool_present, entry = _select_pool_entry("openrouter")
- if pool_present:
- or_key = explicit_api_key or _pool_runtime_api_key(entry)
- if or_key:
- base_url = override_url or _pool_runtime_base_url(entry, OPENROUTER_BASE_URL) or OPENROUTER_BASE_URL
- logger.debug("Auxiliary client: OpenRouter via pool")
- return _create_openai_client(
- api_key=or_key, base_url=base_url, default_headers=build_or_headers()
- ), or_model
- # Exhausted pool: fall through to OPENROUTER_API_KEY rather than fail.
- logger.debug("Auxiliary client: OpenRouter pool exhausted, trying OPENROUTER_API_KEY")
- or_key = explicit_api_key or _scoped_key_env("OPENROUTER_API_KEY")
- if not or_key:
- _mark_provider_unhealthy(
- "openrouter", ttl=60, reason=_describe_openrouter_unavailable(or_model), level=logging.DEBUG)
- return None, None
- logger.debug("Auxiliary client: OpenRouter")
- return _create_openai_client(
- api_key=or_key, base_url=override_url or OPENROUTER_BASE_URL, default_headers=build_or_headers()
- ), or_model
- def _describe_openrouter_unavailable(model: str | None = None) -> str:
- """Return the policy or credential reason OpenRouter was unavailable."""
- free_only, cfg_model = _aux_openrouter_settings()
- or_model = model or cfg_model
- if free_only and not _is_free_model(or_model):
- return (
- f"auxiliary.free_only rejected non-free model {or_model!r}; "
- "the request was skipped before provider availability checks"
- )
- pool_present, entry = _select_pool_entry("openrouter")
- if pool_present:
- if entry is None:
- return "OpenRouter credential pool has no usable entries (credentials may be exhausted)"
- if not _pool_runtime_api_key(entry):
- return "OpenRouter credential pool entry is missing a runtime API key"
- if not _scoped_key_env("OPENROUTER_API_KEY"):
- return "OPENROUTER_API_KEY not set"
- return "no usable OpenRouter credentials found"
- def _try_nous(vision: bool = False) -> tuple[Optional[OpenAI], Optional[str]]:
- nous = _read_nous_auth()
- runtime = _resolve_nous_runtime_api(force_refresh=False)
- if runtime is None and not nous:
- logger.warning("Auxiliary Nous client unavailable: no Nous authentication found (run: hermes auth).")
- _mark_provider_unhealthy("nous", ttl=60, reason="no Nous authentication found", level=logging.DEBUG)
- return None, None
- if runtime is None and nous:
- logger.debug("Auxiliary Nous: runtime JWT refresh failed; checking stored auth.json token.")
- if runtime is not None:
- api_key, base_url = runtime
- else:
- api_key = _nous_api_key(nous or {})
- if not api_key:
- logger.warning(
- "Auxiliary Nous client unavailable: no usable inference JWT found "
- "(run: hermes auth add nous)."
- )
- _mark_provider_unhealthy("nous", ttl=60, reason="no usable Nous inference JWT", level=logging.DEBUG)
- return None, None
- base_url = str(
- (nous or {}).get("inference_base_url") or _scoped_key_env("NOUS_INFERENCE_BASE_URL") or _NOUS_DEFAULT_BASE_URL
- ).rstrip("/")
- with contextlib.suppress(Exception):
- from agent.nous_rate_guard import nous_rate_limit_remaining
- from hermes_cli.anon_auth import is_anonymous_request
- anonymous = is_anonymous_request("nous", api_key)
- remaining = nous_rate_limit_remaining(anonymous=anonymous)
- if remaining is not None and remaining > 0:
- logger.debug("Auxiliary: skipping Nous Portal (rate-limited, resets in %.0fs)", remaining)
- # The health marker is provider-wide, so a full-length anonymous cooldown would
- # outlive signing in mid-cooldown; bound it instead of re-resolving credentials
- # (auth store lock, pool read) on every auxiliary call for the cooldown's duration.
- _mark_provider_unhealthy(
- "nous", ttl=min(remaining, 60.0) if anonymous else remaining,
- reason="Nous Portal rate-limited", level=logging.INFO)
- return None, None
- lane = "vision" if vision else "text"
- # The free tier's host serves exactly one model, for every lane: asking it for the Portal's
- # recommended aux model is a guaranteed 429 ``model_not_free``. Pin the route's model instead.
- # Vision rides the same id (the backing model is multimodal; a backing that is not answers
- # the request with the upstream's own error, which the ladder handles like any other).
- from hermes_cli.anon_auth import GUEST_MODEL, route_is_welcome_host
- if route_is_welcome_host(base_url):
- logger.debug("Auxiliary/%s: Nous free tier; using %s", lane, GUEST_MODEL)
- return _create_openai_client(api_key=api_key, base_url=base_url), GUEST_MODEL
- logger.debug("Auxiliary client: Nous Portal")
- # Portal recommended-models is authoritative (tier-aware); _NOUS_MODEL when unreachable/null.
- # Probes skip the lookup: exact model is irrelevant and it hits the network.
- model = _NOUS_MODEL
- if not _aux_probe_active():
- try:
- from hermes_cli.models import get_nous_recommended_aux_model
- recommended = get_nous_recommended_aux_model(vision=vision)
- if recommended:
- model = recommended
- logger.debug("Auxiliary/%s: using Portal-recommended model %s", lane, model)
- else:
- logger.debug("Auxiliary/%s: no Portal recommendation, falling back to %s", lane, model)
- except Exception as exc:
- logger.debug(
- "Auxiliary/%s: recommended-models lookup failed (%s); "
- "falling back to %s",
- lane, exc, model,
- )
- return _create_openai_client(api_key=api_key, base_url=base_url), model
- def _refresh_nous_recommended_model(*, vision: bool, stale_model: Optional[str]) -> Optional[str]:
- """Fresh Portal recommended model after a stale-model 404 (long-lived processes pin dropped models).
- Returns the fresh recommendation, else ``_NOUS_MODEL``, whichever differs from ``stale_model``; None if neither.
- """
- stale = (stale_model or "").strip().lower()
- fresh: Optional[str] = None
- try:
- from hermes_cli.models import get_nous_recommended_aux_model
- fresh = get_nous_recommended_aux_model(vision=vision, force_refresh=True)
- except Exception as exc:
- logger.debug("Nous recommended-model refresh failed (%s); using default %s", exc, _NOUS_MODEL)
- if fresh and fresh.strip().lower() != stale:
- return fresh
- return _NOUS_MODEL if _NOUS_MODEL.strip().lower() != stale else None
- def _read_main_field(field: str, *, readonly: bool, lower: bool = False) -> str:
- """Main ``model.<field>``: runtime override (``set_runtime_main``) first, then config.yaml.
- The override wins so "active main model" gates see the live CLI/gateway runtime, not the persisted
- default. ``readonly`` picks ``load_config_readonly`` (model/provider) vs ``load_config`` (api_key/base_url).
- """
- override = _runtime_main_value(field)
- if isinstance(override, str) and override.strip():
- value = override.strip()
- return value.lower() if lower else value
- with contextlib.suppress(Exception):
- from hermes_cli import config as _cfg_mod
- cfg = (_cfg_mod.load_config_readonly if readonly else _cfg_mod.load_config)()
- model_cfg = cfg.get("model", {})
- if field == "model" and isinstance(model_cfg, str) and model_cfg.strip():
- return model_cfg.strip()
- if isinstance(model_cfg, dict):
- value = model_cfg.get("default" if field == "model" else field, "")
- if isinstance(value, str) and value.strip():
- value = value.strip()
- return value.lower() if lower else value
- return ""
- # Module-level callables (tests patch them): model/provider (lowercased) read the readonly config;
- # api_key/base_url read the full config so ``custom`` aux tasks can inherit main creds.
- _read_main_model = functools.partial(_read_main_field, "model", readonly=True)
- _read_main_provider = functools.partial(_read_main_field, "provider", readonly=True, lower=True)
- _read_main_api_key = functools.partial(_read_main_field, "api_key", readonly=False)
- _read_main_base_url = functools.partial(_read_main_field, "base_url", readonly=False)
- def _resolve_moa_aggregator(preset_name: Optional[str]) -> tuple[Optional[str], Optional[str]]:
- """MoA preset → aggregator (provider, model); (None, None) if unresolvable. None/"" = default preset.
- "moa" is virtual — aux tasks skip the fan-out and use the aggregator slot; shared so lookup can't drift.
- """
- try:
- from hermes_cli.config import load_config
- from hermes_cli.moa_config import resolve_moa_preset
- preset = resolve_moa_preset(load_config().get("moa") or {}, preset_name or None)
- agg = preset.get("aggregator") or {}
- agg_provider = str(agg.get("provider") or "").strip()
- agg_model = str(agg.get("model") or "").strip()
- if agg_provider and agg_model and agg_provider.lower() != "moa":
- return agg_provider, agg_model
- except Exception:
- logger.debug("MoA aggregator resolution failed for preset %r", preset_name, exc_info=True)
- return None, None
- def _read_main_model_for_aux() -> str:
- """Main model with MoA presets unwrapped to the aggregator's model; "" when unresolvable (a preset name would 400)."""
- model = _read_main_model()
- if (_read_main_provider() or "").strip().lower() == "moa":
- _, agg_model = _resolve_moa_aggregator(model)
- return agg_model or ""
- return model
- def _read_main_api_key_if_same_origin(aux_base_url: str) -> str | Callable[[], str]:
- """Main api_key only when *aux_base_url* has the main base_url's exact origin.
- Unconditional inheritance would leak the credential to any misconfigured host; mismatch keeps ``no-key-required`` → 401.
- Origin, not hostname: another scheme (``http://``) or port on the same host is a different endpoint.
- Anchor and key come from ONE source: the live runtime a turn bound, else config.yaml. The
- per-field readers fall back to config field by field, so a keyless or key_cmd live main would
- pair its own base_url with config's key and send that key to the live endpoint.
- Origins are compared before any key is read: the client cache calls this on every keyless
- ``custom`` lookup, and a mismatch must not pay for a config.yaml key read.
- """
- aux_origin = base_url_origin(aux_base_url)
- if not aux_origin[1]:
- return ""
- live = _normalize_main_runtime(None)
- if live.get("base_url") or live.get("api_key"):
- if aux_origin != base_url_origin(live.get("base_url", "")):
- return ""
- return live.get("api_key", "")
- if aux_origin != base_url_origin(_read_main_base_url()):
- return ""
- return _read_main_api_key()
- # The live main runtime, per context: overlapping gateway sessions make a process-global unsafe.
- _RUNTIME_MAIN_CONTEXT: contextvars.ContextVar[Optional[dict[str, Any]]] = (
- contextvars.ContextVar("auxiliary_runtime_main", default=None)
- )
- _RELAY_AUX_CALL_CONTEXT: contextvars.ContextVar[Optional[dict[str, Any]]] = (
- contextvars.ContextVar("auxiliary_relay_call", default=None)
- )
- @contextlib.contextmanager
- def _relay_aux_call_scope(args: tuple, kwargs: dict):
- """Bind a fresh relay call context for one auxiliary call; mark it failed on any exception."""
- task = args[0] if args else kwargs.get("task")
- token = _RELAY_AUX_CALL_CONTEXT.set({
- "task": str(task or "unknown"),
- "request_id": f"aux-{uuid.uuid4().hex}",
- "attempt_count": 0,
- "provider": "",
- "model": "",
- "response_model": None,
- "api_mode": "chat_completions",
- })
- try:
- yield
- except BaseException as exc:
- _fail_relay_auxiliary_call(exc)
- raise
- finally:
- _RELAY_AUX_CALL_CONTEXT.reset(token)
- def _relay_auxiliary_call(callback):
- """Give every physical retry in one auxiliary call a shared Relay identity."""
- @functools.wraps(callback)
- def wrapped(*args, **kwargs):
- with _relay_aux_call_scope(args, kwargs):
- return callback(*args, **kwargs)
- return wrapped
- def _relay_auxiliary_call_async(callback):
- """Async counterpart to :func:`_relay_auxiliary_call`."""
- @functools.wraps(callback)
- async def wrapped(*args, **kwargs):
- with _relay_aux_call_scope(args, kwargs):
- return await callback(*args, **kwargs)
- return wrapped
- def _set_relay_auxiliary_route(provider: str | None, model: str | None, api_mode: str | None) -> None:
- context = _RELAY_AUX_CALL_CONTEXT.get()
- if context is None:
- return
- context["provider"] = str(provider or "auxiliary")
- context["model"] = str(model or "unknown")
- context["response_model"] = None
- context["api_mode"] = str(api_mode or "chat_completions")
- def _record_route_info(
- route_info: Optional[dict[str, str]], provider: Optional[str], model: Optional[str]
- ) -> None:
- """Expose the concrete route selected for one auxiliary call."""
- if route_info is not None:
- route_info["provider"] = provider or "auto"
- route_info["model"] = model or "default"
- def _relay_auxiliary_metadata(
- *, provider: str | None = None, api_mode: str | None = None
- ) -> tuple[str, str, dict[str, Any]] | None:
- context = _RELAY_AUX_CALL_CONTEXT.get()
- if context is None:
- return None
- attempt_count = int(context.get("attempt_count") or 0)
- context["attempt_count"] = attempt_count + 1
- provider_name = str(provider or context.get("provider") or "auxiliary")
- model_name = str(context.get("model") or "unknown")
- return provider_name, model_name, {
- "api_mode": str(api_mode or context.get("api_mode") or "chat_completions"),
- "api_request_id": str(context["request_id"]),
- "call_role": f"auxiliary:{context['task']}",
- "retry_count": attempt_count,
- "auxiliary_task": str(context["task"]),
- }
- def _relay_sync_completion(
- client: Any, kwargs: dict[str, Any], *, provider: str | None = None,
- api_mode: str | None = None, create: Callable[[dict[str, Any]], Any] | None = None,
- ) -> Any:
- from agent.auxiliary_wire import prepare_chat_messages, relay_chat_metadata
- kwargs = prepare_chat_messages(client, kwargs)
- # The progress hook is installed per TASK, so every attempt (retries, recovery rungs, fallbacks)
- # must stream through _create_with_progress or the compression watchdog sees silence (#98466).
- # Recovery rungs / credential retries keep the task's ``no_progress_timeout`` window; the
- # first-token window uses this attempt's provider (fallbacks name theirs) for its stale timeout.
- relay_context = _RELAY_AUX_CALL_CONTEXT.get() or {}
- task = relay_context.get("task")
- relay_context["stream_provider"] = provider or relay_context.get("provider")
- callback = create or (lambda request: _create_with_progress(client, request, task))
- route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
- # Isolate only the provider callback so the owning thread can unwind its lease/DB
- # transaction on hard cancel without touching the shared client.
- if route is None:
- return _run_protected_sync_provider_call(callback, kwargs)
- provider_name, fallback_model, metadata = route
- from agent import relay_llm
- from agent.auxiliary_hooks import run_with_aux_hooks
- model_name = str(kwargs.get("model") or fallback_model)
- try:
- return run_with_aux_hooks(
- lambda: relay_llm.execute_current(
- kwargs, lambda request: _run_protected_sync_provider_call(callback, request),
- name=provider_name, model_name=model_name, metadata=relay_chat_metadata(metadata),
- defer_logical_completion=True,
- ),
- aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
- provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""),
- )
- except Exception as exc:
- _note_relay_auxiliary_error(exc)
- raise
- async def _relay_async_completion(
- client: Any, kwargs: dict[str, Any], *, provider: str | None = None,
- api_mode: str | None = None, create: Callable[[dict[str, Any]], Any] | None = None,
- ) -> Any:
- from agent.auxiliary_wire import prepare_chat_messages, relay_chat_metadata
- kwargs = prepare_chat_messages(client, kwargs)
- # Async twin of the seam default above (#98466).
- callback = create or (lambda request: _acreate_with_progress(client, request))
- route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
- if route is None:
- return await callback(kwargs)
- provider_name, fallback_model, metadata = route
- from agent import relay_llm
- from agent.auxiliary_hooks import arun_with_aux_hooks
- model_name = str(kwargs.get("model") or fallback_model)
- try:
- return await arun_with_aux_hooks(
- lambda: relay_llm.execute_current_async(
- kwargs, callback, name=provider_name, model_name=model_name,
- metadata=relay_chat_metadata(metadata), defer_logical_completion=True,
- ),
- aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
- provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""),
- )
- except Exception as exc:
- _note_relay_auxiliary_error(exc)
- raise
- def _relay_sync_stream(
- client: Any, kwargs: dict[str, Any], *, provider: str | None = None, api_mode: str | None = None
- ) -> Any:
- from agent.auxiliary_wire import prepare_chat_messages, relay_chat_metadata
- kwargs = prepare_chat_messages(client, kwargs)
- # The bypass runs inside the provider callback, AFTER Relay has seen (and possibly
- # rewritten) the real conversation; applying it to `kwargs` would hand Relay an empty one.
- create = lambda request: client.chat.completions.create(**bypass_chat_sdk_request_transform(request, client))
- route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
- if route is None:
- return create(kwargs)
- provider_name, fallback_model, metadata = route
- from agent import relay_llm
- from agent.auxiliary_hooks import run_with_aux_hooks
- model_name = str(kwargs.get("model") or fallback_model)
- return run_with_aux_hooks(
- lambda: relay_llm.stream_current(
- kwargs, create, name=provider_name, model_name=model_name, finalizer=dict,
- metadata=relay_chat_metadata(metadata), completed_response_predicate=lambda value: hasattr(value, "choices"),
- ),
- aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
- provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""), streaming=True,
- )
- def _runtime_main_value(field: str) -> Any:
- """Read one field of the current context's main runtime ("" when unbound)."""
- runtime = _RUNTIME_MAIN_CONTEXT.get()
- return (runtime.get(field) or "") if isinstance(runtime, dict) else ""
- def set_runtime_main(
- provider: str, model: str, *, requested_provider: str = "", base_url: str = "",
- api_key: Any = "", api_mode: str = "", auth_mode: str = "", session_id: str = "",
- cache_scope: str = "",
- ) -> contextvars.Token:
- """Record the current context's live main runtime for auxiliary routing.
- Context-local so concurrent gateway sessions don't clobber each other.
- ``cache_scope`` is the rotation-stable logical cache scope (compression- lineage root —
- agent/prompt_cache_scope.py) resolved once per turn by turn_context; auxiliary Responses calls prefer it
- over ``session_id`` for prompt_cache_key derivation (#79017).
- """
- runtime = {
- "provider": (provider or "").strip().lower(),
- "requested_provider": (requested_provider or "").strip().lower(),
- "model": (model or "").strip(),
- "base_url": (base_url or "").strip(),
- "api_key": _normalize_api_key(api_key),
- "api_mode": (api_mode or "").strip(),
- "auth_mode": (auth_mode or "").strip().lower(),
- "session_id": (session_id or "").strip(),
- "cache_scope": (cache_scope or "").strip(),
- }
- return _RUNTIME_MAIN_CONTEXT.set(runtime)
- def reset_runtime_main(token: contextvars.Token) -> None:
- """Restore the runtime binding that preceded one scoped turn."""
- if token is None:
- return
- try:
- _RUNTIME_MAIN_CONTEXT.reset(token)
- except (RuntimeError, ValueError):
- pass # Tokens can't be reset from a copied Context (workers inherit values, not token ownership).
- @contextlib.contextmanager
- def scoped_runtime_main(main_runtime: Optional[dict[str, Any]]):
- """Temporarily bind an explicit runtime for the current context."""
- runtime = _normalize_main_runtime(main_runtime)
- token = _RUNTIME_MAIN_CONTEXT.set(runtime or None)
- try:
- yield runtime
- finally:
- _RUNTIME_MAIN_CONTEXT.reset(token)
- def clear_runtime_main() -> None:
- """Clear the runtime override in the current context."""
- _RUNTIME_MAIN_CONTEXT.set(None)
- def _resolve_custom_runtime() -> tuple[Optional[str], Optional[str], Optional[str]]:
- """Resolve the active custom/main endpoint like the main CLI (env OPENAI_BASE_URL or config-saved)."""
- try:
- from hermes_cli.auth import AuthError
- from hermes_cli.runtime_provider import resolve_runtime_provider
- runtime = resolve_runtime_provider(requested="custom")
- except AuthError as exc:
- # Bare 'custom' with nothing configured fails fast in the main resolver: there is no
- # custom endpoint, so do NOT fall through to a stale env OPENAI_BASE_URL that the main
- # resolver deliberately never consults.
- logger.debug("Auxiliary client: no custom endpoint configured: %s", exc)
- return None, None, None
- except Exception as exc:
- logger.debug("Auxiliary client: custom runtime resolution failed: %s", exc)
- runtime = None
- if not isinstance(runtime, dict):
- # Base URL is per-profile like the key one line below (a scoped key must not hit the default's proxy).
- openai_base = _scoped_key_env("OPENAI_BASE_URL").rstrip("/")
- if not openai_base:
- return None, None, None
- runtime = {"base_url": openai_base, "api_key": _scoped_key_env("OPENAI_API_KEY")}
- custom_base = runtime.get("base_url")
- custom_key = runtime.get("api_key")
- custom_mode = runtime.get("api_mode")
- if not isinstance(custom_base, str) or not custom_base.strip():
- return None, None, None
- custom_base = custom_base.strip().rstrip("/")
- if base_url_host_matches(custom_base, "openrouter.ai"):
- return None, None, None # requested='custom' falls back to OpenRouter when unconfigured.
- # Local servers (Ollama, vLLM, ...) ignore auth but the SDK needs a non-empty key.
- # Use a placeholder key — the OpenAI SDK requires a non-empty string but local servers ignore the
- # Authorization header. Same fix as cli.py _ensure_runtime_credentials() (PR #2556).
- if not isinstance(custom_key, str) or not custom_key.strip():
- custom_key = "no-key-required"
- if not isinstance(custom_mode, str) or not custom_mode.strip():
- custom_mode = None
- return custom_base, custom_key.strip(), custom_mode
- def _current_custom_base_url() -> str:
- custom_base, _, _ = _resolve_custom_runtime()
- return custom_base or ""
- def _validate_proxy_env_urls() -> None:
- """Fail fast on malformed proxy env URLs (a shell typo like ``:6153export`` otherwise surfaces as a cryptic httpx ``Invalid port``)."""
- from urllib.parse import urlparse
- normalize_proxy_env_vars()
- for key in ("HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "https_proxy", "http_proxy", "all_proxy"):
- value = str(os.environ.get(key) or "").strip()
- if not value:
- continue
- try:
- parsed = urlparse(value)
- if parsed.scheme:
- _ = parsed.port # raises ValueError for e.g. '6153export'
- except ValueError as exc:
- raise RuntimeError(
- f"Malformed proxy environment variable {key}={value!r}. "
- "Fix or unset your proxy settings and try again."
- ) from exc
- def _validate_base_url(base_url: str) -> None:
- """Reject obviously broken custom endpoint URLs before they reach httpx."""
- from urllib.parse import urlparse
- candidate = str(base_url or "").strip()
- if not candidate or candidate.startswith("acp://"):
- return
- try:
- parsed = urlparse(candidate)
- if parsed.scheme in {"http", "https"}:
- _ = parsed.port # raises ValueError for malformed ports
- except ValueError as exc:
- raise RuntimeError(
- f"Malformed custom endpoint URL: {candidate!r}. "
- "Run `hermes setup` or `hermes model` and enter a valid http(s) base URL."
- ) from exc
- def _try_custom_endpoint() -> tuple[Optional[Any], Optional[str]]:
- runtime = _resolve_custom_runtime()
- custom_base, custom_key, custom_mode = (*runtime, None) if len(runtime) == 2 else runtime
- if not custom_base or not custom_key:
- return None, None
- if custom_base.lower().startswith(_CODEX_AUX_BASE_URL.lower()):
- return None, None
- model = _read_main_model_for_aux() or "gpt-4o-mini"
- logger.debug("Auxiliary client: custom endpoint (%s, api_mode=%s)", model, custom_mode or "chat_completions")
- _clean_base, _dq = _extract_url_query_params(custom_base)
- _extra = {"default_query": _dq} if _dq else {}
- # User model.default_headers override SDK fingerprint headers (as on the main client) for strict gateways/WAFs.
- _custom_headers = _apply_user_default_headers(None)
- if _custom_headers:
- _extra["default_headers"] = _custom_headers
- if custom_mode == "codex_responses":
- real_client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
- return CodexAuxiliaryClient(real_client, model), model
- if custom_mode == "anthropic_messages":
- # OAuth identity only when the host is exactly api.anthropic.com (key_cmd callable included,
- # #114967); third-party Anthropic-compatible gateways never get it.
- try:
- from agent.anthropic_adapter import build_anthropic_client
- from agent.anthropic_credentials import anthropic_route_is_oauth
- real_client = build_anthropic_client(custom_key, custom_base)
- except ImportError:
- logger.warning(
- "Custom endpoint declares api_mode=anthropic_messages but the "
- "anthropic SDK is not installed — falling back to OpenAI-wire."
- )
- return _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra), model
- return AnthropicAuxiliaryClient(real_client, model, custom_key, custom_base,
- is_oauth=anthropic_route_is_oauth(custom_base, custom_key)), model
- # URL-based anthropic detection for custom endpoints without explicit api_mode.
- _fallback_client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
- return _maybe_wrap_anthropic(_fallback_client, model, custom_key, custom_base, custom_mode), model
- def _build_xai_oauth_aux_client(model: str) -> tuple[Optional[Any], Optional[str]]:
- """CodexAuxiliaryClient for xAI Grok OAuth (Responses API); (None, None) if not authed.
- Caller must pass an explicit model — a pinned Grok default would rot as xAI's allowlist drifts.
- """
- if not model:
- logger.warning(
- "Auxiliary client: xai-oauth requested without a model; "
- "pass model explicitly (auxiliary.<task>.model in config.yaml)."
- )
- return None, None
- resolved = _resolve_xai_oauth_for_aux()
- if resolved is None:
- return None, None
- api_key, base_url = resolved
- logger.debug("Auxiliary client: xAI OAuth (%s via Responses API)", model)
- from tools.xai_http import hermes_xai_default_headers
- real_client = _create_openai_client(
- api_key=api_key, base_url=base_url, default_headers=hermes_xai_default_headers()
- )
- return CodexAuxiliaryClient(real_client, model), model
- def _codex_base_url_override() -> str:
- """Profile-scoped ``HERMES_CODEX_BASE_URL`` (same read as the API-key env vars: under a
- multiplexer the routed profile's .env decides the endpoint, never a sibling's process env)."""
- return _scoped_key_env("HERMES_CODEX_BASE_URL").rstrip("/")
- def _build_codex_client(model: str) -> tuple[Optional[Any], Optional[str]]:
- """CodexAuxiliaryClient for an explicit model; (None, None) without a Codex OAuth token.
- No auto-selected default: the Codex model allow-list is undocumented and drifts.
- """
- if not model:
- logger.warning(
- "Auxiliary client: openai-codex requested without a model; "
- "pass model explicitly (auxiliary.<task>.model in config.yaml)."
- )
- return None, None
- codex_token, base_url = _resolve_codex_credential_and_base()
- if not codex_token:
- return None, None
- logger.debug("Auxiliary client: Codex OAuth (%s via Responses API)", model)
- real_client = _create_openai_client(
- api_key=codex_token, base_url=base_url,
- default_headers=_codex_cloudflare_headers(codex_token, base_url=base_url),
- )
- return CodexAuxiliaryClient(real_client, model), model
- def _try_azure_foundry(
- *, model: Optional[str] = None, explicit_api_key: Optional[str] = None,
- explicit_base_url: Optional[str] = None, api_mode: Optional[str] = None,
- ) -> tuple[Optional[Any], Optional[str]]:
- """Azure Foundry aux client via the main agent's ``_resolve_azure_foundry_runtime`` (api_key vs Entra
- callable bearer, per-model api_mode, base_url overrides). Returns ``(client, model)`` or ``(None, None)``."""
- try:
- from hermes_cli.runtime_provider import _resolve_azure_foundry_runtime
- from hermes_cli.auth import AuthError
- from hermes_cli.config import load_config_readonly
- except ImportError:
- return None, None
- try:
- cfg = load_config_readonly()
- model_cfg = cfg.get("model") if isinstance(cfg, dict) else {}
- if not isinstance(model_cfg, dict):
- model_cfg = {}
- except Exception:
- model_cfg = {}
- try:
- runtime = _resolve_azure_foundry_runtime(
- requested_provider="azure-foundry", model_cfg=model_cfg,
- explicit_api_key=explicit_api_key, explicit_base_url=explicit_base_url,
- target_model=model,
- )
- except AuthError as exc:
- logger.debug("Auxiliary azure-foundry: %s", exc)
- return None, None
- except Exception as exc:
- logger.debug("Auxiliary azure-foundry runtime error: %s", exc)
- return None, None
- api_key = runtime.get("api_key")
- base_url = str(runtime.get("base_url", "") or "")
- runtime_api_mode = api_mode or runtime.get("api_mode") or "chat_completions"
- # api_key may be a callable token provider; bail only on None/"".
- if not (callable(api_key) or api_key) or not base_url:
- return None, None
- final_model = _normalize_resolved_model(model or str(model_cfg.get("default") or ""), "azure-foundry")
- if not final_model:
- # No fallback aux model for Azure (needs a deployment name): let the auto chain fall through instead of 404ing.
- logger.debug(
- "Auxiliary azure-foundry: no model resolved (model=%r, default=%r)",
- model, model_cfg.get("default"),
- )
- return None, None
- # The SDK drops api-version query params from the base URL; pass via default_query.
- _clean_base, _dq = _extract_url_query_params(base_url)
- extra: dict[str, Any] = {"default_query": _dq} if _dq else {}
- client = _create_openai_client(api_key=api_key, base_url=_clean_base, **extra)
- if runtime_api_mode == "codex_responses":
- return CodexAuxiliaryClient(client, final_model), final_model
- if runtime_api_mode == "anthropic_messages":
- # api_key forwarded verbatim (string or Entra callable; build_anthropic_client installs the bearer hook).
- return _maybe_wrap_anthropic(client, final_model, api_key, base_url, runtime_api_mode), final_model
- return client, final_model
- def _try_anthropic(explicit_api_key: Optional[str | Callable[[], str]] = None,
- explicit_base_url: Optional[str] = None) -> tuple[Optional[Any], Optional[str]]:
- try:
- from agent.anthropic_adapter import build_anthropic_client
- from agent.anthropic_credentials import resolve_anthropic_token
- except ImportError:
- return None, None
- pool_present, entry = _select_pool_entry("anthropic")
- if pool_present and entry is not None:
- token = explicit_api_key or _pool_runtime_api_key(entry)
- else:
- # Pool absent/empty: legacy resolver so a dead pool entry can't wedge aux tasks when a standalone credential exists.
- entry = None
- token = explicit_api_key or resolve_anthropic_token()
- if not token:
- return None, None
- # Honor config.yaml model.base_url only when provider is anthropic AND the URL is
- # Anthropic-compatible; a foreign host (Codex, OpenRouter) would 401 every aux call.
- base_url = _pool_runtime_base_url(entry, _ANTHROPIC_DEFAULT_BASE_URL) if pool_present else _ANTHROPIC_DEFAULT_BASE_URL
- with contextlib.suppress(Exception):
- from hermes_cli.config import load_config_readonly
- cfg = load_config_readonly()
- model_cfg = cfg.get("model")
- if isinstance(model_cfg, dict):
- cfg_provider = str(model_cfg.get("provider") or "").strip().lower()
- if cfg_provider == "anthropic":
- cfg_base_url = (model_cfg.get("base_url") or "").strip().rstrip("/")
- if cfg_base_url and _is_anthropic_compatible_host(cfg_base_url):
- base_url = cfg_base_url
- # A caller-supplied endpoint (fallback_providers entry, custom_providers entry) wins over
- # both the pool row and config.yaml, under the same Anthropic-compatible host rule the
- # primary path applies. A foreign host is REFUSED outright rather than silently demoted to
- # the canonical host: continuing would send the explicit credential to a target the caller
- # did not ask for (#121359).
- override_url = (explicit_base_url or "").strip().rstrip("/")
- if override_url:
- if not _is_anthropic_compatible_host(override_url):
- logger.warning(
- "Auxiliary client: refusing anthropic explicit base_url %r — not an "
- "Anthropic-compatible host; no client built and no request sent.",
- override_url,
- )
- return None, None
- base_url = override_url
- from agent.anthropic_credentials import _is_oauth_token
- is_oauth = _is_oauth_token(token)
- model = _get_aux_model_for_provider("anthropic") or "claude-haiku-4-5-20251001"
- if _aux_probe_active():
- # Probe: token + adapter import resolved; skip real client construction.
- return _AuxProbeClientStub(api_key="", base_url=base_url), model
- logger.debug("Auxiliary client: Anthropic native (%s) at %s (oauth=%s)", model, base_url, is_oauth)
- try:
- real_client = build_anthropic_client(token, base_url)
- except ImportError:
- return None, None # Adapter imports fine but the anthropic SDK itself is missing.
- return AnthropicAuxiliaryClient(real_client, model, token, base_url, is_oauth=is_oauth), model
- _MAIN_RUNTIME_FIELDS = ("provider", "model", "base_url", "api_key", "api_mode", "auth_mode")
- _MAIN_RUNTIME_CONTEXT_FIELDS = _MAIN_RUNTIME_FIELDS + (
- "requested_provider", "session_id", "cache_scope",
- )
- def _normalize_main_runtime(main_runtime: Optional[dict[str, Any]]) -> dict[str, Any]:
- """Return a sanitized copy of a live main-runtime override.
- ``api_key`` may be a zero-arg callable (Entra ID token provider, accepted by the OpenAI SDK)
- — preserved as-is so aux clients share main-agent auth.
- """
- if main_runtime is None:
- main_runtime = _RUNTIME_MAIN_CONTEXT.get()
- if not isinstance(main_runtime, dict):
- return {}
- normalized: dict[str, Any] = {}
- for field in _MAIN_RUNTIME_CONTEXT_FIELDS:
- value = main_runtime.get(field)
- if field == "api_key" and callable(value) and not isinstance(value, str):
- normalized[field] = value
- elif isinstance(value, str) and value.strip():
- normalized[field] = value.strip()
- for identity_field in ("provider", "requested_provider"):
- identity = normalized.get(identity_field)
- if isinstance(identity, str):
- normalized[identity_field] = identity.lower()
- return normalized
- def _get_provider_chain() -> list[tuple]:
- """Ordered provider detection chain, built at call time so ``_try_*`` patches are picked up.
- ``openai-codex`` is deliberately absent (shifting allow-list breaks guessed-model fallback).
- """
- return [
- ("openrouter", _try_openrouter), ("nous", _try_nous),
- ("local/custom", _try_custom_endpoint), ("api-key", _resolve_api_key_provider),
- ]
- # "Recently 402'd" unhealthy-provider cache: a depleted provider stays so for hours, so hiding it
- # for a TTL saves an RTT per aux call. In-process only (profiles may use different keys).
- _AUX_UNHEALTHY_TTL_SECONDS = 600 # 10 minutes
- _aux_unhealthy_until: dict[Any, float] = {}
- _aux_unhealthy_logged_at: dict[Any, float] = {}
- _aux_unhealthy_reason: dict[Any, str] = {}
- # resolved_provider / explicit-config names → chain labels.
- _AUX_UNHEALTHY_LABEL_ALIASES = {
- "openrouter": "openrouter", "nous": "nous", "custom": "local/custom",
- "local/custom": "local/custom", "openai-codex": "openai-codex", "codex": "openai-codex",
- }
- def _normalize_chain_label(provider: str) -> str:
- """resolved_provider → chain label; unknown API-key providers fall back to the lowercased input."""
- if not provider:
- return ""
- p = str(provider).strip().lower()
- return _AUX_UNHEALTHY_LABEL_ALIASES.get(p, p)
- _AUX_UNHEALTHY_PAYMENT_REASON = "payment / credit error"
- def _mark_provider_unhealthy(
- provider: str, ttl: Optional[float] = None, *, base_url: Optional[str] = None,
- reason: str = _AUX_UNHEALTHY_PAYMENT_REASON, level: int = logging.WARNING,
- ) -> None:
- """Hide one provider endpoint until the TTL expires. ``reason`` is what the log says and what the
- skip line echoes: absent credentials are an expected state (DEBUG), a confirmed 402 is a fault
- (WARNING) — the old fixed payment wording sent local-only users chasing billing (#64144)."""
- label = _normalize_chain_label(provider)
- if not label:
- return
- key = _unhealthy_cache_key(label, base_url)
- ttl = _AUX_UNHEALTHY_TTL_SECONDS if ttl is None else ttl
- expires_at = time.time() + ttl
- _aux_unhealthy_until[key] = expires_at
- _aux_unhealthy_reason[key] = reason
- logger.log(
- level,
- "Auxiliary: marking %s unhealthy for %ds (%s). "
- "Subsequent auxiliary calls will skip it until %s.",
- label, int(ttl), reason, time.strftime("%H:%M:%S", time.localtime(expires_at)),
- )
- def _is_provider_unhealthy(label: str, base_url: Optional[str] = None) -> bool:
- """True iff this provider endpoint is unhealthy and unexpired; lazily evicts expired entries."""
- if not label:
- return False
- key = _unhealthy_cache_key(label, base_url)
- expires_at = _aux_unhealthy_until.get(key)
- if expires_at is None:
- return False
- if time.time() >= expires_at:
- _aux_unhealthy_until.pop(key, None)
- _aux_unhealthy_logged_at.pop(key, None)
- _aux_unhealthy_reason.pop(key, None)
- return False
- return True
- def _log_skip_unhealthy(
- label: str, task: Optional[str] = None, *, base_url: Optional[str] = None,
- ) -> None:
- """Log a skipped unhealthy provider at most once per minute per label."""
- now = time.time()
- key = _unhealthy_cache_key(label, base_url)
- if now - _aux_unhealthy_logged_at.get(key, 0.0) >= 60:
- _aux_unhealthy_logged_at[key] = now
- expires_at = _aux_unhealthy_until.get(key, now)
- logger.info(
- "Auxiliary %s: skipping %s (%s, retry in %ds)",
- task or "call", label, _aux_unhealthy_reason.get(key, _AUX_UNHEALTHY_PAYMENT_REASON),
- max(0, int(expires_at - now)),
- )
- def _reset_aux_unhealthy_cache() -> None:
- """Clear the unhealthy cache (tests / explicit user reset)."""
- _aux_unhealthy_until.clear()
- _aux_unhealthy_logged_at.clear()
- _aux_unhealthy_reason.clear()
- def _contains_any(text: str, needles: tuple[str, ...]) -> bool:
- """True when any needle is a substring of ``text``."""
- return any(kw in text for kw in needles)
- # Billing-body markers (credit exhaustion wrapped in 402/403/404/429 bodies). The main classifier's
- # ``_BILLING_PATTERNS`` is the single source (#107166: the two lists had drifted, so OpenRouter's org
- # "Budget limit exceeded" / "Key limit exceeded" 403s were billing for the main loop but not for the aux
- # ladder, and aux fallback never fired); the aux list only adds broader phrasings and daily/weekly quota
- # exhaustion (functionally credit exhaustion; "resource exhausted" is the Vertex/gRPC quota phrasing —
- # also serialized by SDK wrappers and NIM as RESOURCE_EXHAUSTED / ResourceExhausted / resource-exhausted).
- _PAYMENT_KEYWORDS = _BILLING_PATTERNS + (
- "credits", "insufficient funds", "can only afford", "billing",
- "isn't available on the free tier", "key limit exceeded", "budget limit",
- "requires a subscription", "upgrade for access", "upgrade for higher limits",
- "reached your session usage limit", "quota exceeded", "quota_exceeded",
- "too many tokens per day", "daily limit", "tokens per day", "daily quota", "resource exhausted",
- "resource_exhausted", "resource-exhausted", "resourceexhausted",
- "weekly usage limit", "weekly limit",
- )
- def _is_payment_error(exc: Exception) -> bool:
- """Payment/credit/quota exhaustion: HTTP 402, or a billing/quota body on 403/404/429/no-status."""
- status = getattr(exc, "status_code", None)
- return status == 402 or (
- status in {403, 404, 429, None} and _contains_any(str(exc).lower(), _PAYMENT_KEYWORDS)
- )
- def _nous_portal_account_has_fresh_paid_access() -> bool:
- """Return True only when the fresh Nous account API says paid access is allowed."""
- try:
- from hermes_cli.nous_account import get_nous_portal_account_info
- return get_nous_portal_account_info(force_fresh=True).paid_service_access is True
- except Exception as exc:
- logger.debug("Auxiliary Nous paid-entitlement refresh check failed: %s", exc)
- return False
- _RATE_LIMIT_KEYWORDS = (
- "rate limit", "rate_limit", "too many requests", "try again", "retry after", "resets in"
- )
- _RATE_LIMIT_BILLING_KEYWORDS = (
- "credits", "insufficient funds", "billing", "payment required", "can only afford",
- "out of funds", "run out of funds", "balance_depleted", "no usable credits",
- "model_not_supported_on_free_tier", "not available on the free tier", "isn't available on the free tier",
- )
- def _is_overloaded_error(exc: Exception) -> bool:
- """Server busy, credential fine ("at capacity upstream … not your API key's rate limit"): back off,
- never bench the credential (#108349). Same phrase table as the main classifier."""
- return _contains_any(str(exc).lower(), _OVERLOADED_PATTERNS)
- def _is_rate_limit_error(exc: Exception) -> bool:
- """429 rate limit (not billing/quota, which _is_payment_error owns).
- OpenAI's RateLimitError may omit .status_code — matched by class name. A generic 429 without
- billing keywords counts as a rate limit.
- """
- # (PR #8023 pattern)
- if type(exc).__name__ == "RateLimitError" or _botocore_error(exc)[1] == "ThrottlingException":
- return True
- if _exc_http_status(exc) != 429:
- return False
- err_lower = str(exc).lower()
- return _contains_any(err_lower, _RATE_LIMIT_KEYWORDS) or not _contains_any(err_lower, _RATE_LIMIT_BILLING_KEYWORDS)
- def _is_timeout_error(exc: Exception) -> bool:
- """Full-budget request timeout, distinct from a fast connection drop.
- A timeout burns the whole ``timeout`` budget, so a same-provider retry on the compression
- path doubles wall time; fast drops stay on the retry path.
- """
- with contextlib.suppress(ImportError):
- from openai import APITimeoutError
- if isinstance(exc, APITimeoutError):
- return True
- return "Timeout" in type(exc).__name__ or "timed out" in str(exc).lower()
- def _is_connection_error(exc: Exception) -> bool:
- """Connection/network errors (endpoint unreachable), as opposed to 4xx/5xx API errors."""
- with contextlib.suppress(ImportError):
- from openai import APIConnectionError, APITimeoutError
- if isinstance(exc, (APIConnectionError, APITimeoutError)):
- return True
- if _contains_any(type(exc).__name__, ("Connection", "Timeout", "DNS", "SSL")):
- return True
- return _contains_any(str(exc).lower(), (
- "connection refused", "name or service not known", "no route to host",
- "network is unreachable", "timed out", "connection reset",
- # httpcore/httpx premature stream close — transient, retry/reroute.
- "incomplete chunked read", "peer closed connection", "response ended prematurely",
- "unexpected eof", "remoteprotocolerror", "localprotocolerror",
- ))
- def _is_transient_transport_error(exc: Exception) -> bool:
- """One-off transport blip worth retrying on the SAME provider: connection/stream-close errors plus pure 5xx/408.
- Deliberately narrow: payment/auth/rate-limit errors switch provider, refresh creds, or rotate the pool.
- """
- if _is_connection_error(exc):
- return True
- status = _exc_http_status(exc)
- return isinstance(status, int) and (status == 408 or 500 <= status < 600)
- _DEFAULT_TRANSIENT_RETRIES = 2
- _TRANSIENT_RETRY_BACKOFF_BASE = 1.0 # Backoff base (seconds); overridable so tests can zero it out.
- def _transient_retry_count() -> int:
- """Same-provider retries for a transient blip: ``auxiliary.transient_retries``
- (default 2), clamped to [0, 6]; config-read failures fall back to default."""
- try:
- from hermes_cli.config import cfg_get, load_config
- val = cfg_get(load_config(), "auxiliary", "transient_retries")
- return _DEFAULT_TRANSIENT_RETRIES if val is None else max(0, min(int(val), 6))
- except Exception:
- return _DEFAULT_TRANSIENT_RETRIES
- def _is_auth_error(exc: Exception) -> bool:
- """Auth failures that should trigger provider-specific refresh."""
- status = getattr(exc, "status_code", None)
- if status == 401:
- return True
- err_lower = str(exc).lower()
- if "error code: 401" in err_lower or "authenticationerror" in type(exc).__name__.lower():
- return True
- # xAI returns 403 "unauthenticated:bad-credentials" for expired OAuth tokens — semantically a 401.
- return "bad-credentials" in err_lower and (status == 403 or "unauthenticated" in err_lower)
- def _is_unsupported_parameter_error(exc: Exception, param: str) -> bool:
- """Provider 400 for an unsupported request parameter: the parameter name plus a generic
- unsupported/unknown/unrecognized marker, so call sites can retry without the key."""
- param_lower = (param or "").lower()
- if not param_lower:
- return False
- err_lower = str(exc).lower()
- return param_lower in err_lower and _contains_any(err_lower, UNSUPPORTED_PARAM_MARKERS)
- def _is_structured_output_rejection(exc: Exception) -> bool:
- """Provider 400/422 rejecting the structured-output field, on either wire: OpenAI ``response_format``
- (incl. vLLM's ``guided_grammar``/xgrammar failures) or Anthropic ``output_config.format`` ("Extra inputs
- are not permitted"). Callers tolerate an unconstrained reply, so the reaction is one retry without it."""
- status = getattr(exc, "status_code", None)
- if status is not None and status not in {400, 422}:
- return False
- err_lower = str(exc).lower()
- # vLLM grammar-backend failures name the translated parameter, not ours.
- if _contains_any(err_lower, ("guided_grammar", "xgrammar", "compile_grammar_error")):
- return True
- if "extra inputs are not permitted" in err_lower and (
- "response_format" in err_lower or "output_config" in err_lower
- ):
- return True
- if "response_format" in err_lower and "unavailable" in err_lower:
- return True
- # Gateways that validate the request body with a strict pydantic model reject the
- # OBJECT-form json_schema by shape ("str type expected" on response_format.json_schema,
- # 422) rather than by naming the feature. The field is what they refuse; the retry
- # without it is the same remedy, so treat the shape error as a rejection too.
- if "response_format" in err_lower and "json_schema" in err_lower:
- return True
- # Gemini native names its own generationConfig keys, never ours: "Function calling with a response
- # mime type: 'application/json' is unsupported" (pre-Gemini-3 + tools via a proxy), or an
- # "Unknown name"/"Invalid value" 400 on response_schema / response_json_schema for a schema the
- # surface cannot express. Same remedy: one retry without the format.
- if _contains_any(err_lower, ("response mime type", "response_schema", "response_json_schema")):
- return True
- return _is_unsupported_parameter_error(exc, "response_format") or _is_unsupported_parameter_error(exc, "output_config")
- def _is_reasoning_field_rejection(exc: Exception) -> bool:
- """Provider 400 rejecting a reasoning wire control by name (``reasoning_effort``, ``reasoning``,
- ``thinking``/``think``). Chat-only models behind OpenAI-compatible relays reject the top-level
- ``reasoning_effort: none`` a disabled ``reasoning_config`` projects on the custom profile
- ("Unrecognized request argument supplied: reasoning_effort", #112781); the route default is
- the right answer for such a model, so the reaction is one retry without any reasoning field."""
- status = getattr(exc, "status_code", None)
- if status is not None and status not in {400, 422}:
- return False
- return is_reasoning_field_rejection(str(exc))
- def _is_reasoning_required_rejection(exc: Exception) -> bool:
- """Provider 400 refusing to switch reasoning OFF ("Reasoning is mandatory for this endpoint and cannot
- be disabled"): the field is understood, only the disable is refused, so the rung steps the effort up to
- the floor instead of dropping the field (agent/auxiliary_reasoning_floor.py)."""
- status = getattr(exc, "status_code", None)
- if status is not None and status not in {400, 422}:
- return False
- return is_reasoning_required_rejection(str(exc))
- def _without_reasoning_fields(kwargs: dict) -> Optional[dict]:
- """Copy *kwargs* without reasoning wire controls (top-level ``reasoning_effort``, the adapter's
- private ``_reasoning_config`` and every ``extra_body`` reasoning key); None when nothing was
- removed, so call sites don't retry an unchanged request."""
- retry_kwargs = dict(kwargs)
- changed = retry_kwargs.pop("reasoning_effort", None) is not None
- changed = retry_kwargs.pop("_reasoning_config", None) is not None or changed
- extra_body = retry_kwargs.get("extra_body")
- if isinstance(extra_body, dict):
- remaining = {k: v for k, v in extra_body.items() if str(k).strip().lower() not in _PROFILE_REASONING_KEYS}
- if len(remaining) != len(extra_body):
- if remaining:
- retry_kwargs["extra_body"] = remaining
- else:
- retry_kwargs.pop("extra_body", None)
- changed = True
- return retry_kwargs if changed else None
- def _is_model_not_found_error(exc: Exception) -> bool:
- """"Requested model doesn't exist" (404 / invalid model) — typically a long-lived process pinned a
- since-dropped model. Excludes billing keywords, which :func:`_is_payment_error` owns."""
- status = getattr(exc, "status_code", None)
- err_lower = str(exc).lower()
- if _contains_any(err_lower, (
- "credits", "insufficient funds", "billing", "out of funds", "balance_depleted",
- "no usable credits", "free tier", "free-tier", "not available on the free tier",
- )):
- return False
- if status not in {404, 400, None}:
- return False
- return _contains_any(err_lower, (
- "model does not exist", "does not exist in our configuration", "openrouter catalog",
- "is not a valid model", "no such model", "model not found",
- "the model `", # OpenAI-style: "The model `X` does not exist"
- "model_not_found", "unknown model",
- ))
- def _is_model_incompatible_error(exc: Exception) -> bool:
- """"This route cannot serve this model" 400 (capability mismatch, e.g. a Codex/ChatGPT-account
- fallback asked to run a non-OpenAI model). Auth/payment predicates don't fire, so this keeps the
- chain going instead of aborting. Excludes billing 400s and not-found 400s."""
- status = getattr(exc, "status_code", None)
- if status not in {400, None}:
- return False
- err_lower = str(exc).lower()
- if _is_model_not_found_error(exc):
- return False
- # Billing keywords checked directly: _is_payment_error is status-gated and misses 400-coded billing bodies.
- if _contains_any(err_lower, (
- "credits", "insufficient funds", "billing", "out of funds", "balance_depleted",
- "no usable credits", "payment required", "free tier", "free-tier",
- "not available on the free tier", "model_not_supported_on_free_tier", "quota",
- )):
- return False
- return _contains_any(err_lower, (
- "is not supported when using", # codex/ChatGPT-account model gating
- "model is not supported", "not supported with this", "not supported for this account",
- "model_not_supported", "does not support this model", "unsupported model",
- ))
- # Tasks on a user-visible critical path (compression blocks resuming an oversized session; vision
- # stalls the serialised turn queue). A same-provider retry after a full-budget timeout costs another
- # whole ``timeout`` window, so they skip straight to fallback; fast blips still retry.
- # Fast blips (a streaming-close or a 5xx) still retry, since those are cheap. See issue #54465 for the
- # compression case. Title generation joins them because its retries multiplied the user's
- # ``auxiliary.title_generation.timeout`` (~4x: three full windows plus backoff) on a slow local
- # model, and the auto-title thread outlived the deadline the user thought they had set (#89445, #66251).
- _TIMEOUT_NO_RETRY_TASKS = frozenset({"compression", "vision", "title_generation"})
- def _should_skip_same_provider_retry(task: Optional[str], exc: Exception) -> bool:
- """True when a transient error on a critical-path task should go straight to fallback.
- Carve-out: a fast first-token fail (dead stream within the no-progress window, zero output —
- see ``_timeout_message``) is cheap and keeps the same-provider retry; mid-stream stalls and
- hard-ceiling timeouts skip to fallback.
- """
- return task in _TIMEOUT_NO_RETRY_TASKS and _is_timeout_error(exc) and "no-progress timeout" not in str(exc)
- def _evict_cached_clients(provider: str) -> None:
- """Drop this profile's cached auxiliary clients for a provider so fresh creds are used.
- Scoped to the calling profile (``hermes_home_key()`` is the first key slot): a rotation in
- one profile must not drop another profile's client for the same provider in a multiplexing
- gateway, since that profile's credentials did not change. Entries are popped, not closed:
- a concurrent caller may be mid-request on the shared client (closing it raises ReadError /
- "client has been closed" for them); the dropped client is retired by GC like the FIFO
- overflow path in ``_get_cached_client``.
- """
- normalized = _normalize_aux_provider(provider)
- home = hermes_home_key()
- with _client_cache_lock:
- for key in [key for key in _client_cache
- if key[0] == home and _normalize_aux_provider(str(key[1])) == normalized]:
- _client_cache.pop(key, None)
- def _evict_cached_client_instance(target: Any) -> bool:
- """Drop cache entries whose stored client (or its ``_real_client``) is *target*; True if any evicted.
- Used when a cached client is poisoned (closed transport after a timeout). Async wrappers must
- expose the same ``_real_client`` as their sync sibling or the async entry survives.
- """
- if target is None:
- return False
- evicted = False
- with _client_cache_lock:
- for key, entry in list(_client_cache.items()):
- cached = entry[0] if entry is not None else None
- if cached is not None and (cached is target or getattr(cached, "_real_client", None) is target):
- del _client_cache[key]
- evicted = True
- return evicted
- def _pool_credential_digest(pool: Any, entry: Any = None) -> str:
- """Secret-safe digest over the runtime key(s) backing the cache hint.
- When ``entry`` (the peeked selection) carries a key, only THAT key is digested: rotating a
- sibling account must not churn this entry's cached client. Otherwise every pool entry
- contributes, taken WITHOUT availability filters (``pool.entries()``, not ``peek()``): an entry
- benched by a per-model cooldown still counts, so a token rotated while the cooldown is active
- changes the digest too (#113022 point 5). Only the blake2b of the keys enters the cache key.
- """
- keys = [k for k in (_pool_runtime_api_key(entry),) if k] if entry is not None else []
- if not keys:
- try:
- entries_fn = getattr(pool, "entries", None)
- entries = entries_fn() if callable(entries_fn) else []
- except Exception as exc:
- logger.debug("Auxiliary client: could not list pool entries for digest: %s", exc)
- return ""
- keys = sorted(k for k in (_pool_runtime_api_key(e) for e in entries or ()) if k)
- if not keys:
- return ""
- digest = hashlib.blake2b(digest_size=16)
- for key in keys:
- digest.update(key.encode("utf-8"))
- digest.update(b"\0")
- return digest.hexdigest()
- def _pool_cache_hint(provider: str, *, main_runtime: Optional[dict[str, Any]] = None) -> str:
- """Return a cache discriminator that follows the pooled credential, not just its entry id.
- ``<provider>:<entry id>:<digest of that entry's key>`` when ``peek()`` names an entry,
- ``<provider>::<digest of every entry's key>`` when every entry is filtered out (cooldown)
- but keyed entries exist. A token rotated or
- refreshed by another process yields a new client instead of a 401 round trip on the cached
- one; the stale entry ages out through the non-closing FIFO cap (#113022).
- """
- normalized = _normalize_aux_provider(provider)
- if normalized == "auto":
- runtime = _normalize_main_runtime(main_runtime)
- normalized = _normalize_aux_provider(runtime.get("provider") or _read_main_provider())
- if normalized in {"", "auto", "custom"}:
- return ""
- pool = _load_pool_with_credentials(normalized, " (cache hint)")
- if pool is None:
- return ""
- entry = _peek_pool_entry(normalized, pool)
- digest = _pool_credential_digest(pool, entry)
- entry_id = str(getattr(entry, "id", "") or "").strip() if entry is not None else ""
- if not entry_id and not digest:
- return ""
- return f"{normalized}:{entry_id}:{digest}"
- # Ordered (host, provider) tables for inferring a backend from a client base URL (then profile ``hostnames``).
- _POOL_PROVIDER_BY_HOST = (
- ("chatgpt.com", "openai-codex"), ("openrouter.ai", "openrouter"),
- ("inference-api.nousresearch.com", "nous"), ("api.anthropic.com", "anthropic"),
- ("api.kimi.com", "kimi-coding"), ("api.x.ai", "xai-oauth"),
- )
- _AUTH_REFRESH_PROVIDER_BY_HOST = (
- ("chatgpt.com", "openai-codex"),
- ("api.anthropic.com", "anthropic"), ("inference-api.nousresearch.com", "nous"),
- # An aux call that inherits the main xai-oauth route arrives as "auto"; without this row the
- # 403 bad-credentials rung skipped the refresh and benched the only grant (#84845).
- ("api.x.ai", "xai-oauth"),
- )
- def _provider_for_host(base_url: str, table: tuple[tuple[str, str], ...]) -> Optional[str]:
- """First provider in ``table`` whose host matches ``base_url``, else a profile claiming the host."""
- for host, provider in table:
- if base_url_host_matches(base_url, host):
- return provider
- from providers import profile_for_host
- return getattr(profile_for_host(base_url), "name", None)
- def _recoverable_pool_provider(
- resolved_provider: str, client: Any, main_runtime: Optional[dict[str, Any]] = None
- ) -> Optional[str]:
- """Infer which provider pool can recover the current auxiliary client.
- None when the client targets a different host than the session's configured endpoint for that
- provider: a rejection there says nothing about the key, so rotating/quarantining it would kill a
- working credential (Miho report — proxy users)."""
- normalized = _normalize_aux_provider(resolved_provider)
- base = str(getattr(client, "base_url", "") or "")
- runtime = _normalize_main_runtime(main_runtime)
- rt_base = str(runtime.get("base_url") or "")
- rt_key = runtime.get("api_key")
- client_key = getattr(client, "api_key", None)
- # Only the SESSION's own key is shielded, and only when it was sent somewhere other than the
- # session's origin (scheme+host+port — a port or HTTPS→HTTP change is a different trust boundary).
- # An independently owned auxiliary pool keeps rotating at its own origin.
- if (base and rt_base and normalized == runtime.get("provider")
- and isinstance(rt_key, str) and rt_key and client_key == rt_key
- and base_url_origin(base) != base_url_origin(rt_base)):
- logger.info("Auxiliary: %s rejected the session key at %s, but the session's endpoint is %s — "
- "endpoint mismatch, not a dead key; skipping credential rotation",
- normalized, base_url_hostname(base), base_url_hostname(rt_base))
- return None
- if normalized not in {"", "auto", "custom"}: # a pinned ``custom:<name>`` keeps its entry's identity
- return pinned if (pinned := str(resolved_provider or "").strip().lower()).startswith("custom:") else normalized
- known = _provider_for_host(base, _POOL_PROVIDER_BY_HOST)
- if known is not None:
- return known
- # Providers outside the table (e.g. opencode-go): match base URL against registered
- # api_key providers so pool rotation works for them too.
- if main_runtime:
- runtime = _normalize_main_runtime(main_runtime)
- rt_provider = runtime.get("provider", "")
- if rt_provider and rt_provider not in {"", "auto", "custom"}:
- with contextlib.suppress(Exception):
- from hermes_cli.auth import PROVIDER_REGISTRY
- pconfig = PROVIDER_REGISTRY.get(rt_provider)
- if pconfig and getattr(pconfig, "auth_type", None) == "api_key":
- # The pool's key was issued for the endpoint the main runtime actually uses; a
- # rejection at any other host (registry default vs configured proxy) says nothing
- # about that key, so it must not be marked exhausted.
- rt_base = str(runtime.get("base_url") or getattr(pconfig, "inference_base_url", "") or "").rstrip("/")
- if rt_base and base_url_host_matches(base, base_url_hostname(rt_base)):
- return rt_provider
- return None
- def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str = "") -> bool:
- """Try same-provider credential-pool recovery for auxiliary calls.
- ``failed_api_key`` lets mark_exhausted_and_rotate identify the right pool entry even if
- another process already rotated (current() would be None).
- """
- normalized = _normalize_aux_provider(provider)
- try:
- pool = pool_for_aux_provider(provider, normalized, load_pool)
- except Exception as load_exc:
- logger.debug("Auxiliary client: could not load pool for %s recovery: %s", normalized, load_exc)
- return False
- if not pool or not pool.has_credentials():
- return False
- status_code = getattr(exc, "status_code", None)
- def _rotate(fallback_status: int) -> bool:
- error_context: dict[str, Any] = {"message": str(exc)}
- if status_code is not None:
- error_context["status_code"] = status_code
- next_entry = pool.mark_exhausted_and_rotate(
- status_code=status_code if status_code is not None else fallback_status,
- error_context=error_context, api_key_hint=failed_api_key or None,
- )
- if next_entry is None:
- return False
- _evict_cached_clients(normalized)
- return True
- if _is_auth_error(exc):
- if pool.try_refresh_current() is not None:
- _evict_cached_clients(normalized)
- return True
- return _rotate(401)
- if _is_payment_error(exc):
- return _rotate(402)
- if _is_rate_limit_error(exc) and not _is_overloaded_error(exc):
- return _rotate(429)
- return False
- def _prepare_same_provider_retry(
- *, task: Optional[str], resolved_provider: str, resolved_model: Optional[str],
- resolved_base_url: Optional[str], resolved_api_key: Optional[str],
- resolved_api_mode: Optional[str], main_runtime: Optional[dict[str, Any]],
- final_model: Optional[str], messages: list, temperature: Optional[float],
- max_tokens: Optional[int], tools: Optional[list], effective_timeout: float,
- effective_extra_body: dict, reasoning_config: Optional[dict], async_mode: bool,
- extra_headers: Optional[dict[str, str]] = None,
- ) -> tuple[Any, dict[str, Any]]:
- """Rebuild (client, request kwargs) for a same-provider retry after credential recovery."""
- if task == "vision":
- effective_provider, retry_client, retry_model = resolve_vision_provider_client(
- provider=resolved_provider, model=final_model, base_url=resolved_base_url,
- api_key=resolved_api_key, async_mode=async_mode,
- )
- else:
- retry_client, retry_model = _get_cached_client(
- resolved_provider, resolved_model, async_mode=async_mode, base_url=resolved_base_url,
- api_key=resolved_api_key, api_mode=resolved_api_mode, main_runtime=main_runtime,
- )
- effective_provider = _effective_provider_for_client(retry_client, resolved_provider)
- if retry_client is None:
- raise RuntimeError(
- f"Auxiliary {task or 'call'}: provider {resolved_provider} could not be rebuilt after recovery"
- )
- retry_base = str(getattr(retry_client, "base_url", "") or "")
- retry_kwargs = _build_call_kwargs(
- effective_provider or resolved_provider, retry_model or final_model, messages,
- temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout,
- extra_body=effective_extra_body, reasoning_config=reasoning_config,
- base_url=retry_base or resolved_base_url, task=task,
- )
- # Preserve per-request attribution headers (a profile's ``request_headers``) across the rebuilt-client
- # retry: dropping them would let a recovery retry silently lose capability gating (#60293).
- if extra_headers:
- retry_kwargs["extra_headers"] = dict(extra_headers)
- if _is_anthropic_compat_endpoint(resolved_provider, retry_base):
- retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"])
- return retry_client, retry_kwargs
- def _retry_same_provider_sync(*, resolved_provider: str, resolved_api_mode: Optional[str], task: Optional[str], **prep) -> Any:
- retry_client, retry_kwargs = _prepare_same_provider_retry(
- task=task, resolved_provider=resolved_provider, resolved_api_mode=resolved_api_mode, async_mode=False, **prep,
- )
- return _validate_llm_response(
- _relay_sync_completion(retry_client, retry_kwargs, provider=resolved_provider, api_mode=resolved_api_mode), task,
- )
- async def _retry_same_provider_async(*, resolved_provider: str, resolved_api_mode: Optional[str], task: Optional[str], **prep) -> Any:
- import asyncio
- retry_client, retry_kwargs = _prepare_same_provider_retry(
- task=task, resolved_provider=resolved_provider, resolved_api_mode=resolved_api_mode, async_mode=True, **prep,
- )
- return await asyncio.to_thread(
- _validate_llm_response,
- await _relay_async_completion(retry_client, retry_kwargs, provider=resolved_provider, api_mode=resolved_api_mode),
- task,
- )
- def _creds_have_api_key(creds: dict[str, Any]) -> bool:
- return bool(str(creds.get("api_key", "") or "").strip())
- def _refresh_codex_credentials() -> bool:
- from hermes_cli.auth import resolve_codex_runtime_credentials
- return _creds_have_api_key(resolve_codex_runtime_credentials(force_refresh=True))
- def _refresh_nous_credentials() -> bool:
- from hermes_cli.auth import resolve_nous_runtime_credentials
- return _creds_have_api_key(resolve_nous_runtime_credentials(
- timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15), force_refresh=True
- ))
- def _refresh_anthropic_credentials(failed_api_key: str = "") -> bool:
- from agent.anthropic_credentials import read_claude_code_credentials, _refresh_oauth_token
- token = failed_api_key
- if not token:
- return False
- pool = load_pool("anthropic")
- if pool.entry_id_for_api_key(token):
- return pool.try_refresh_matching(api_key_hint=token) is not None
- creds = read_claude_code_credentials()
- # Never spend an ambient login's refresh rotation for another request's key.
- if isinstance(creds, dict) and creds.get("accessToken") == token and creds.get("refreshToken"):
- return bool(_refresh_oauth_token(creds))
- return False
- def _refresh_xai_oauth_credentials() -> bool:
- """Pool-level refresh first, then the singleton auth-store resolver."""
- pool = load_pool("xai-oauth")
- if pool and pool.has_credentials():
- pool.select()
- refreshed = pool.try_refresh_current()
- if refreshed is not None and str(getattr(refreshed, "runtime_api_key", "") or "").strip():
- return True
- from hermes_cli.auth import resolve_xai_oauth_runtime_credentials
- return _creds_have_api_key(resolve_xai_oauth_runtime_credentials(force_refresh=True))
- def _refresh_vertex_credentials() -> bool:
- """Mirrors run_agent's Vertex refresh; the cache key ignores the rotating bearer, so
- without the eviction that follows, a ~1h-expired aux Vertex client 401s forever."""
- from agent.vertex_adapter import get_vertex_config
- token, base_url = get_vertex_config()
- return bool(isinstance(token, str) and token.strip() and isinstance(base_url, str) and base_url.strip())
- # Each refresher returns True when a usable credential exists; the caller then evicts cached clients.
- _CREDENTIAL_REFRESHERS: dict[str, Callable[..., bool]] = {
- "openai-codex": _refresh_codex_credentials,
- "nous": _refresh_nous_credentials, "anthropic": _refresh_anthropic_credentials,
- "xai-oauth": _refresh_xai_oauth_credentials, "vertex": _refresh_vertex_credentials,
- }
- def _refresh_provider_credentials(provider: str, *, failed_api_key: str = "") -> bool:
- """Refresh short-lived credentials for OAuth-backed auxiliary providers."""
- normalized = _normalize_aux_provider(provider)
- refresher = _CREDENTIAL_REFRESHERS.get(normalized)
- if refresher is None:
- from providers import get_provider_profile
- profile = get_provider_profile(normalized)
- if profile is None:
- return False
- refresher = lambda: profile.refresh_runtime_credentials(reason="auxiliary_unauthorized")
- try:
- if not (refresher(failed_api_key) if normalized == "anthropic" else refresher()):
- return False
- _evict_cached_clients(normalized)
- return True
- except Exception as exc:
- logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc)
- return False
- def _auth_refresh_provider_for_route(
- resolved_provider: Optional[str], client_base_url: str, effective_provider: str = "",
- ) -> str:
- """Provider whose short-lived credentials should be refreshed; auto-routed calls keep
- ``resolved_provider == "auto"``, so infer the backend from the client's base URL."""
- normalized = _normalize_aux_provider(resolved_provider)
- if normalized and normalized != "auto":
- return normalized
- host_provider = _provider_for_host(client_base_url, _AUTH_REFRESH_PROVIDER_BY_HOST)
- # The auto client already knows it runs on the host's API-key sibling (``xai`` on api.x.ai):
- # an XAI_API_KEY 401 must not spend a stale ``xai-oauth`` grant's refresh and switch routes.
- if host_provider and host_provider == f"{_normalize_aux_provider(effective_provider)}-oauth":
- return normalized
- return host_provider or normalized
- def _fallback_chain_entry(task: Optional[str], fb_label: str) -> Optional[dict[str, Any]]:
- """Resolve the ``fallback_chain`` entry a ``fallback_chain[<i>](<provider>)`` label points at,
- or None when the label is not a configured-chain candidate or the index no longer resolves."""
- if not task or not fb_label:
- return None
- m = re.match(r"fallback_chain\[(\d+)\]", fb_label)
- if not m:
- return None
- try:
- chain = _get_auxiliary_task_config(task).get("fallback_chain")
- entry = chain[int(m.group(1))] if isinstance(chain, list) else None
- except Exception:
- return None
- return entry if isinstance(entry, dict) else None
- def _coerce_positive_timeout(raw: Any) -> Optional[float]:
- """Coerce a config ``timeout`` to a positive float, or None (rejects bools, which are ints)."""
- if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw > 0:
- return float(raw)
- return None
- def _fallback_entry_timeout(task: Optional[str], fb_label: str) -> Optional[float]:
- """Per-entry ``timeout`` for a configured fallback candidate, or None (keep the task-level
- timeout). Inheriting the primary's deadline used to kill healthy-but-slower fallbacks.
- A fallback candidate previously inherited the exact timeout the primary provider was called with. When
- that deadline was tuned for the primary (or the primary simply consumed its whole budget before failing
- over), the fallback aborted on the same clock even when independently healthy — a 163k-token compression
- that needs ~90s on the fallback died at the primary's 30s deadline every turn (#62452).
- """
- entry = _fallback_chain_entry(task, fb_label)
- return _coerce_positive_timeout(entry.get("timeout") if entry else None)
- def _fallback_provider_from_label(label: str) -> str:
- """Recover the provider identifier from a fallback display label."""
- match = re.match(r"(?:fallback_chain\[\d+\]|fallback_providers\[\d+\]|main-agent)\(([^)]+)\)$", label or "")
- return match.group(1).strip() if match else str(label or "").strip()
- class _FallbackDestination(NamedTuple):
- provider: str
- base_url: str
- api_mode: Optional[str]
- model: Optional[str]
- def _complete_fallback_destination(
- provider: str, base_url: str, api_mode: Optional[str], model: Optional[str]
- ) -> _FallbackDestination:
- if not api_mode:
- if _endpoint_speaks_anthropic_messages(base_url):
- api_mode = "anthropic_messages"
- else:
- with contextlib.suppress(Exception):
- from hermes_cli.runtime_provider import resolve_runtime_provider
- runtime = resolve_runtime_provider(
- requested=provider, explicit_base_url=base_url or None, target_model=model or ""
- )
- api_mode = str(runtime.get("api_mode") or "").strip() or None
- return _FallbackDestination(provider, base_url, api_mode, model)
- def _fallback_destination_from_entry(
- entry: dict[str, Any], fb_client: Any, fb_model: Optional[str]
- ) -> _FallbackDestination:
- provider = str(entry.get("provider") or "").strip()
- base_url = str(entry.get("base_url") or getattr(fb_client, "base_url", "") or "").strip()
- api_mode = str(entry.get("api_mode") or entry.get("transport") or "").strip() or None
- model = fb_model or str(entry.get("model") or "").strip() or None
- return _complete_fallback_destination(provider, base_url, api_mode, model)
- def _fallback_destination(
- task: Optional[str], fb_client: Any, fb_model: Optional[str], fb_label: str
- ) -> _FallbackDestination:
- """Route identity of a fallback request: attached destination, else configured entry, else label."""
- attached = getattr(fb_client, "_hermes_fallback_destination", None)
- if isinstance(attached, _FallbackDestination):
- return attached
- entry = _fallback_chain_entry(task, fb_label)
- if entry is not None:
- return _fallback_destination_from_entry(entry, fb_client, fb_model)
- return _complete_fallback_destination(
- _fallback_provider_from_label(fb_label), str(getattr(fb_client, "base_url", "") or ""), None, fb_model,
- )
- def _replan_synchronous_cache_sections(
- messages: list, tools: Optional[list], *, destination: _FallbackDestination
- ) -> tuple[list, list]:
- """Plan one destination at the configured TTL (1h must not regress to 5m); one-shots never cache."""
- from agent.agent_runtime_helpers import configured_cache_ttl, plan_cache_sections_for_destination
- from agent.prompt_caching import is_one_shot_request
- return plan_cache_sections_for_destination(
- messages, tools, provider=destination.provider, base_url=destination.base_url,
- api_mode=destination.api_mode or "", model=destination.model or "", cache_ttl=configured_cache_ttl(),
- cache_disabled=True if is_one_shot_request(messages, tools) else None,
- )
- def _fallback_request_kwargs(
- destination: _FallbackDestination, *, task: Optional[str], messages: list,
- tools: Optional[list], temperature: Optional[float], max_tokens: Optional[int],
- effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
- fallback_entry: dict, task_config: dict, apply_fast_lane: bool,
- ) -> dict[str, Any]:
- """Build request kwargs for one fallback destination (cache-section replan + fast-lane cap)."""
- fallback_max_tokens, fallback_extra_body = max_tokens, effective_extra_body
- if apply_fast_lane:
- fallback_max_tokens, fallback_extra_body = _compression_fast_lane_controls(
- task, actual_provider=destination.provider, actual_model=destination.model,
- requested_provider=fallback_entry.get("provider"),
- requested_model=fallback_entry.get("model"), route_config=fallback_entry,
- leak_guard_config=task_config, max_tokens=max_tokens, extra_body=effective_extra_body,
- )
- fallback_messages, fallback_tools = _replan_synchronous_cache_sections(messages, tools, destination=destination)
- fb_kwargs = _build_call_kwargs(
- destination.provider, destination.model, fallback_messages,
- temperature=temperature, max_tokens=fallback_max_tokens, tools=fallback_tools, timeout=effective_timeout,
- extra_body=fallback_extra_body, reasoning_config=reasoning_config, base_url=destination.base_url, task=task)
- return fb_kwargs
- def _plan_fallback_candidate(
- fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str],
- effective_timeout: float, apply_fast_lane: bool, **request,
- ) -> tuple[_FallbackDestination, dict[str, Any], Callable[[str, Any, Optional[str]], dict[str, Any]]]:
- """Resolve the destination + first-attempt kwargs for a fallback candidate.
- Returns ``(destination, kwargs, rebuild)`` where ``rebuild(provider, client, model)`` produces
- kwargs for the credential-refreshed retry destination. A configured-chain entry's own
- ``timeout`` overrides ``effective_timeout``.
- """
- fb_timeout = _fallback_entry_timeout(task, fb_label)
- if fb_timeout is not None and fb_timeout != effective_timeout:
- logger.info(
- "Auxiliary %s: %s using its configured timeout %.0fs "
- "(task-level was %.0fs)",
- task or "call", fb_label, fb_timeout, effective_timeout,
- )
- effective_timeout = fb_timeout
- destination = _fallback_destination(task, fb_client, fb_model, fb_label)
- task_config = _get_auxiliary_task_config(task) if task == "compression" else {}
- fallback_entry = _fallback_chain_entry(task, fb_label) or {}
- common = dict(
- task=task, effective_timeout=effective_timeout, fallback_entry=fallback_entry,
- task_config=task_config, apply_fast_lane=apply_fast_lane, **request,
- )
- def _rebuild(provider: str, client: Any, model: Optional[str]) -> tuple[_FallbackDestination, dict[str, Any]]:
- retry_destination = _FallbackDestination(
- provider, destination.base_url or str(getattr(client, "base_url", "") or ""),
- destination.api_mode, model or destination.model,
- )
- return retry_destination, _fallback_request_kwargs(retry_destination, **common)
- return destination, _fallback_request_kwargs(destination, **common), _rebuild
- def _quarantine_fallback_candidate(
- task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *,
- base_url: str = "", tag: str = "", reason: Optional[str] = None,
- ) -> None:
- """The candidate cannot serve this walk (``reason`` = its ``_FALLBACK_REASONS`` capacity label,
- None = dead token): mark it unhealthy so the ordered re-walk skips it and the caller moves on to
- the next entry. Transient classes get a short hold, payment/quota and dead tokens the long one."""
- _mark_provider_unhealthy(
- fb_provider or fb_label, ttl=fallback_candidate_quarantine_ttl(reason),
- base_url=base_url, reason=reason or "stale fallback credential")
- why = f"is out of capacity ({reason})" if reason else "has a stale/unrefreshable credential"
- logger.warning("Auxiliary %s%s: fallback candidate %s %s (%s) — skipping to next fallback",
- task or "call", tag, fb_label, why, fb_err)
- def _plan_fallback_auth_retry(
- destination: _FallbackDestination,
- rebuild: Callable[[str, Any, Optional[str]], tuple[_FallbackDestination, dict[str, Any]]], *,
- async_mode: bool,
- failed_api_key: str = "",
- ) -> tuple[str, Optional[tuple[Any, dict[str, Any], _FallbackDestination]]]:
- """After an auth error on a fallback candidate: refresh credentials and rebuild the request.
- Returns ``(refresh_provider, retry)``; ``retry`` = ``(client, kwargs, destination)`` or None."""
- fb_provider = _auth_refresh_provider_for_route(destination.provider, destination.base_url)
- refresh_kwargs = {"failed_api_key": failed_api_key} if fb_provider == "anthropic" else {}
- if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider, **refresh_kwargs):
- retry_client, retry_model = _get_cached_client(
- fb_provider, destination.model, **({"async_mode": True} if async_mode else {}),
- base_url=destination.base_url or None, api_mode=destination.api_mode,
- )
- if retry_client is not None:
- retry_destination, retry_kwargs = rebuild(fb_provider, retry_client, retry_model)
- return fb_provider, (retry_client, retry_kwargs, retry_destination)
- return fb_provider, None
- def _call_fallback_candidate_sync(
- fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str], messages: list,
- temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
- effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
- ) -> Optional[Any]:
- """Call one fallback candidate with stale-credential recovery: on an auth error refresh its
- credentials and retry once with a rebuilt client; if that also auth-fails, quarantine the
- provider and return None so the caller moves on. A capacity error (quota/rate-limit 429, 402,
- connection, route-incompatible model, malformed response) also quarantines and returns None so
- the ordered chain advances to the next configured entry (#106367); other errors raise.
- ``effective_timeout`` is the task-level deadline; a configured-chain candidate with its own ``timeout``
- entry gets that instead, so a fallback tuned differently from the primary is allowed its own budget
- (#62452).
- """
- destination, fb_kwargs, rebuild = _plan_fallback_candidate(
- fb_client, fb_model, fb_label, task=task, effective_timeout=effective_timeout,
- apply_fast_lane=True, messages=messages, tools=tools, temperature=temperature,
- max_tokens=max_tokens, effective_extra_body=effective_extra_body,
- reasoning_config=reasoning_config,
- )
- def _send(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
- return _validate_llm_response(
- _relay_sync_completion(
- client, request_kwargs, provider=dest.provider, api_mode=dest.api_mode,
- create=lambda request: _create_with_progress(
- client, request, task,
- force_stream=_provider_requires_stream(dest.provider, dest.base_url),
- ),
- ),
- task,
- )
- from agent.auxiliary_fallback_recovery import send_with_parameter_rungs
- def _send_recovering(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
- return send_with_parameter_rungs(
- lambda c, kw: _send(c, kw, dest), client, request_kwargs, task=task)
- try:
- return _send_recovering(fb_client, fb_kwargs, destination)
- except Exception as fb_err:
- if not _is_auth_error(fb_err):
- capacity = fallback_candidate_unavailable_reason(fb_err)
- if capacity is None:
- raise
- _quarantine_fallback_candidate(
- task, fb_label, destination.provider, fb_err, base_url=destination.base_url,
- reason=capacity)
- return None
- fb_provider, retry = _plan_fallback_auth_retry(
- destination, rebuild, async_mode=False, failed_api_key=getattr(fb_client, "api_key", ""))
- failed_destination = destination
- if retry is not None:
- failed_destination = retry[2]
- try:
- return _send_recovering(*retry)
- except Exception as retry_err:
- if not _is_auth_error(retry_err) and fallback_candidate_unavailable_reason(retry_err) is None:
- raise
- _quarantine_fallback_candidate(
- task, fb_label, fb_provider, fb_err, base_url=failed_destination.base_url,
- )
- return None
- async def _call_fallback_candidate_async(
- fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str], messages: list,
- temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
- effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
- ) -> Optional[Any]:
- """Async mirror of :func:`_call_fallback_candidate_sync` (no fast-lane cap on this wire)."""
- destination, fb_kwargs, rebuild = _plan_fallback_candidate(
- fb_client, fb_model, fb_label, task=task, effective_timeout=effective_timeout,
- apply_fast_lane=False, messages=messages, tools=tools, temperature=temperature,
- max_tokens=max_tokens, effective_extra_body=effective_extra_body,
- reasoning_config=reasoning_config,
- )
- async def _send(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
- import asyncio
- return await asyncio.to_thread(
- _validate_llm_response,
- await _relay_async_completion(client, request_kwargs, provider=dest.provider, api_mode=dest.api_mode),
- task,
- )
- from agent.auxiliary_fallback_recovery import send_with_parameter_rungs_async
- async def _send_recovering(client: Any, request_kwargs: dict[str, Any], dest: _FallbackDestination) -> Any:
- return await send_with_parameter_rungs_async(
- lambda c, kw: _send(c, kw, dest), client, request_kwargs, task=task)
- try:
- return await _send_recovering(fb_client, fb_kwargs, destination)
- except Exception as fb_err:
- if not _is_auth_error(fb_err):
- capacity = fallback_candidate_unavailable_reason(fb_err)
- if capacity is None:
- raise
- _quarantine_fallback_candidate(
- task, fb_label, destination.provider, fb_err, base_url=destination.base_url,
- tag=" (async)", reason=capacity)
- return None
- fb_provider, retry = _plan_fallback_auth_retry(
- destination, rebuild, async_mode=True, failed_api_key=getattr(fb_client, "api_key", ""))
- failed_destination = destination
- if retry is not None:
- failed_destination = retry[2]
- try:
- return await _send_recovering(*retry)
- except Exception as retry_err:
- if not _is_auth_error(retry_err) and fallback_candidate_unavailable_reason(retry_err) is None:
- raise
- _quarantine_fallback_candidate(
- task, fb_label, fb_provider, fb_err,
- base_url=failed_destination.base_url, tag=" (async)",
- )
- return None
- def _try_payment_fallback(
- failed_provider: str, task: str | None = None, reason: str = "payment error", *,
- failed_base_url: str = "", failure_scope: Any = None, main_runtime: Optional[dict[str, Any]] = None,
- ) -> tuple[Optional[Any], Optional[str], str]:
- """Try the auto-detection chain after a payment/credit or connection error, skipping the failed
- provider (and the main-provider path when it maps to the same backend). Returns (client, model, label) or (None, None, "")."""
- skip = failed_provider.lower().strip()
- # The SESSION's provider decides whether discovery is allowed: a live `/model xai-oauth` session
- # over a persisted ``provider: auto`` is a selection, so the disk value alone is not the answer.
- main_provider = _normalize_main_runtime(main_runtime).get("provider") or _read_main_provider()
- if not _discovery_chain_allowed(main_provider, task):
- return None, None, ""
- skip_labels = {skip}
- if main_provider and main_provider.lower() in skip:
- skip_labels.add(main_provider.lower())
- skip_chain_labels = {_normalize_chain_label(s) for s in skip_labels}
- skip_backend = _failed_backend_skip(
- failed_provider, None, failed_base_url=failed_base_url, failure_scope=failure_scope)
- tried = []
- for label, try_fn in _get_provider_chain():
- candidate_base_url = _custom_health_base_url(label)
- if (not failed_base_url and label in skip_chain_labels) or skip_backend(
- label, None, candidate_base_url):
- continue
- if _is_provider_unhealthy(label, candidate_base_url):
- _log_skip_unhealthy(label, task, base_url=candidate_base_url)
- tried.append(f"{label} (unhealthy)")
- continue
- client, model = try_fn()
- if client is not None:
- logger.info("Auxiliary %s: %s on %s — falling back to %s (%s)",
- task or "call", reason, failed_provider, label, model or "default")
- return client, model, label
- tried.append(label)
- logger.warning("Auxiliary %s: %s on %s and no fallback available (tried: %s)",
- task or "call", reason, failed_provider, ", ".join(tried))
- return None, None, ""
- def _failed_backend_skip(
- failed_provider: str, failed_model: Optional[str], *, failed_base_url: str = "",
- failure_scope: Any = None,
- ) -> Callable[..., bool]:
- """Predicate ``skip(provider, model, base_url="")`` → True when a candidate must be skipped for the failed
- route. Scope: ``failed_model`` → model-scoped (only that deployment; timeout/connection/rate-limit);
- None → credential-wide (whole provider; auth/payment)."""
- from agent.backend_identity import BackendIdentity, FailureScope, should_skip_candidate
- skip_model = (failed_model or "").strip().lower() or None
- failed_ident = BackendIdentity.build(
- provider=failed_provider, model=skip_model, base_url=failed_base_url)
- failure_scope = failure_scope or (FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL)
- def _skip(provider: str, model: Optional[str], base_url: str = "") -> bool:
- return should_skip_candidate(
- BackendIdentity.build(provider=provider, model=model, base_url=base_url), failed_ident, failure_scope,
- )
- return _skip
- def _try_main_agent_model_fallback(
- failed_provider: str, task: str | None = None, reason: str = "error",
- failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
- ) -> tuple[Optional[Any], Optional[str], str]:
- """Last-resort fallback to the main agent provider + model after the configured chain is exhausted.
- ``failed_model`` scoping per ``_failed_backend_skip``; same-URL custom endpoints serve many models,
- so a hung aux model says nothing about the main model's health. Returns (client, model, label) or (None, None, "")."""
- main_provider = (_read_main_provider() or "").strip()
- main_model = (_read_main_model() or "").strip()
- if main_provider.lower() == "moa":
- # MoA virtual provider: fall back to the preset's aggregator (the acting model).
- _agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
- if not _agg_provider or not _agg_model:
- return None, None, ""
- main_provider, main_model = _agg_provider, _agg_model
- if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
- return None, None, ""
- if task == "vision" and (
- main_provider in _PROVIDERS_WITHOUT_VISION or not _main_model_supports_vision(main_provider, main_model)):
- # Same capability gate as the auto-route (_vision_main_provider_client): handing an image to a
- # text-only main model turns a transient 429 into a guaranteed 400 (#108349).
- logger.info("Auxiliary vision: %s on %s — main agent provider %s accepts no image input, not falling back",
- reason, failed_provider, main_provider)
- return None, None, ""
- main_base_url = _custom_health_base_url(main_provider)
- if _failed_backend_skip(
- failed_provider, failed_model, failed_base_url=failed_base_url,
- failure_scope=failure_scope)(main_provider, main_model, main_base_url):
- return None, None, ""
- if _is_provider_unhealthy(main_provider, main_base_url):
- _log_skip_unhealthy(main_provider, task, base_url=main_base_url)
- return None, None, ""
- try:
- client, resolved_model = resolve_provider_client(provider=main_provider, model=main_model)
- except Exception:
- client, resolved_model = None, None
- if client is None:
- return None, None, ""
- label = f"main-agent({main_provider})"
- logger.info("Auxiliary %s: %s on %s — falling back to main agent model %s (%s)",
- task or "call", reason, failed_provider, label, resolved_model or main_model)
- return client, resolved_model or main_model, label
- # Context-window screening for runtime fallback chains: the startup feasibility check filters
- # too-small aux models; runtime chains must too, or compression stops at a reachable-but-too-small
- # candidate. ``None`` (unknown) passes through.
- # ── Context-window screening for runtime fallback chains (issue #52392) ── When the runtime auxiliary
- # fallback chain selects a candidate that is reachable but has a context window smaller than the compression
- # task requires, the call errors out instead of continuing to the next, viable candidate. The startup
- # feasibility check in ``agent.conversation_compression.check_compression_model_feasibility`` already
- # filters too-small auxiliary models at startup, but the runtime fallback chain
- # (``_try_configured_fallback_chain`` and ``_try_main_fallback_chain``) does not apply the same filter, so
- # compression can stop at the first alive door even if the room behind it is too small. The helpers below
- # screen each candidate by its effective context window before it is returned. ``None`` results from
- # ``get_model_context_length`` are passed through (we cannot prove a model is too small, so we do not block
- # it). This preserves the existing fallback surface for unrecognised/custom models while closing the gap on
- # the well-known ones.
- def _task_minimum_context_length(task: Optional[str]) -> Optional[int]:
- """Minimum context length for an auxiliary task; None = no floor (only ``compression`` has one)."""
- return MINIMUM_CONTEXT_LENGTH if task == "compression" else None
- def _candidate_context_window(provider: str, model: str, base_url: str = "", api_key: str = "") -> Optional[int]:
- """Best-effort context window for a fallback candidate; ``None`` = unknown (never raises; callers pass it through)."""
- if not model:
- return None
- try:
- ctx = get_model_context_length(model, base_url=base_url, api_key=api_key, provider=provider)
- except Exception as exc:
- logger.debug("Auxiliary fallback: could not resolve context window for %s/%s: %s", provider, model, exc)
- return None
- return ctx if isinstance(ctx, int) and ctx > 0 else None
- def _context_too_small(
- entry: dict[str, Any], provider: str, model: str, min_ctx: Optional[int], *,
- task: Optional[str], label: str, name_model: bool = False,
- ) -> Optional[str]:
- """Screen one fallback candidate by context window; returns the ``tried`` note when it is too small."""
- if min_ctx is None:
- return None
- fb_ctx = _candidate_context_window(
- provider, model, base_url=str(entry.get("base_url") or ""), api_key=_fallback_entry_api_key(entry) or "")
- if fb_ctx is None or fb_ctx >= min_ctx:
- return None
- if name_model:
- logger.info("Auxiliary %s: skipping %s (%s context=%d < min=%d), continuing chain",
- task, label, model, fb_ctx, min_ctx)
- else:
- logger.info("Auxiliary %s: skipping %s (context=%d < min=%d), continuing chain",
- task or "call", label, fb_ctx, min_ctx)
- return f"{label} (context too small: {fb_ctx}<{min_ctx})"
- def _try_configured_fallback_chain(
- task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None, *,
- failed_base_url: str = "", failure_scope: Any = None,
- ) -> tuple[Optional[Any], Optional[str], str]:
- """Try auxiliary.<task>.fallback_chain entries in order (each needs ``provider``; model/base_url/api_key optional).
- ``failed_model`` scoping per ``_failed_backend_skip`` (sibling models on the same provider still
- run after a model-scoped failure). Returns (client, model, provider_label) or (None, None, "")."""
- if not task:
- return None, None, ""
- chain = _get_auxiliary_task_config(task).get("fallback_chain")
- if not chain or not isinstance(chain, list):
- return None, None, ""
- skip = _failed_backend_skip(
- failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
- tried = []
- min_ctx = _task_minimum_context_length(task)
- for i, entry in enumerate(chain):
- if not isinstance(entry, dict):
- continue
- fb_provider = str(entry.get("provider", "")).strip()
- if not fb_provider:
- continue
- fb_model_raw = str(entry.get("model", "")).strip()
- fb_base_url = _custom_health_base_url(fb_provider, entry.get("base_url"))
- if skip(fb_provider, fb_model_raw, fb_base_url):
- continue
- if _is_provider_unhealthy(fb_provider, fb_base_url):
- _log_skip_unhealthy(fb_provider, task, base_url=fb_base_url)
- tried.append(f"fallback_chain[{i}]({fb_provider}) (unhealthy)")
- continue
- fb_model = fb_model_raw or None
- label = f"fallback_chain[{i}]({fb_provider})"
- try:
- fb_client, resolved_model = _resolve_fallback_entry(entry)
- except Exception:
- fb_client, resolved_model = None, None
- if fb_client is not None:
- too_small = _context_too_small(
- entry, fb_provider, resolved_model, min_ctx, task=task, label=label, name_model=True,
- ) if resolved_model else None
- if too_small:
- tried.append(too_small)
- continue
- logger.info("Auxiliary %s: %s on %s — configured fallback to %s (%s)",
- task, reason, failed_provider, label, resolved_model or fb_model or "default")
- return fb_client, resolved_model or fb_model, label
- tried.append(label)
- if tried:
- logger.debug("Auxiliary %s: configured fallback_chain exhausted (tried: %s)", task, ", ".join(tried))
- return None, None, ""
- def _try_configured_fallback_for_unavailable_client(
- task: Optional[str], failed_provider: str
- ) -> tuple[Optional[Any], Optional[str], str]:
- """Task fallback_chain when an explicit aux provider cannot build a client (no key/OAuth/pool creds);
- stops at the per-task chain — the main-agent model stays the runtime last resort."""
- explicit = (failed_provider or "").strip().lower()
- if not task or not explicit or explicit in {"auto"}:
- return None, None, ""
- return _try_configured_fallback_chain(task, explicit, reason="provider unavailable")
- def _fallback_entry_api_key(entry: dict[str, Any]) -> Optional[str]:
- """Resolve inline or env-backed API key via the secret-scope-aware resolver (no raw os.getenv under multiplexing)."""
- from hermes_cli.fallback_config import resolve_entry_api_key
- return resolve_entry_api_key(entry)
- def _resolve_fallback_entry(entry: dict[str, Any]) -> tuple[Optional[Any], Optional[str]]:
- """Resolve one fallback entry through the central provider router."""
- provider = str(entry.get("provider") or "").strip()
- model = str(entry.get("model") or "").strip() or None
- if not provider or not model:
- return None, None
- client, resolved_model = resolve_provider_client(
- provider, model=model, explicit_base_url=str(entry.get("base_url") or "").strip() or None,
- explicit_api_key=_fallback_entry_api_key(entry),
- api_mode=str(entry.get("api_mode") or entry.get("transport") or "").strip() or None,
- )
- if client is not None:
- with contextlib.suppress(Exception):
- client._hermes_fallback_destination = _fallback_destination_from_entry(entry, client, resolved_model)
- return client, resolved_model
- def _try_main_fallback_chain(
- task: Optional[str], failed_provider: str = "", reason: str = "error", *,
- failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
- ) -> tuple[Optional[Any], Optional[str], str]:
- """Top-level main-agent fallback chain for a ``provider: auto`` auxiliary call: auto tasks honour the
- user's main fallback policy before the built-in discovery chain; read via ``get_fallback_chain`` so
- ``fallback_providers`` and legacy ``fallback_model`` keep the main agent's order."""
- try:
- from hermes_cli.config import load_config_readonly
- from hermes_cli.fallback_config import get_fallback_chain
- chain = get_fallback_chain(load_config_readonly())
- except Exception as exc:
- logger.debug("Auxiliary %s: could not load main fallback chain: %s", task or "call", exc)
- return None, None, ""
- if not chain:
- return None, None, ""
- skip = _failed_backend_skip(
- failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
- tried: list[str] = []
- min_ctx = _task_minimum_context_length(task)
- for i, entry in enumerate(chain):
- if not isinstance(entry, dict):
- continue
- fb_provider = str(entry.get("provider") or "").strip()
- fb_model = str(entry.get("model") or "").strip()
- if not fb_provider or not fb_model:
- continue
- fb_norm = fb_provider.lower()
- label = f"fallback_providers[{i}]({fb_provider})"
- fb_base_url = _custom_health_base_url(fb_provider, entry.get("base_url"))
- if fb_norm == "auto" or skip(fb_provider, fb_model, fb_base_url):
- tried.append(f"{label} (skipped)")
- continue
- if _is_provider_unhealthy(fb_norm, fb_base_url):
- _log_skip_unhealthy(fb_norm, task, base_url=fb_base_url)
- tried.append(f"{label} (unhealthy)")
- continue
- try:
- fb_client, resolved_model = _resolve_fallback_entry(entry)
- except Exception as exc:
- logger.debug("Auxiliary %s: main fallback %s failed to resolve: %s", task or "call", label, exc)
- fb_client, resolved_model = None, None
- if fb_client is not None:
- too_small = _context_too_small(
- entry, fb_provider, resolved_model or fb_model, min_ctx, task=task, label=label,
- )
- if too_small:
- tried.append(too_small)
- continue
- logger.info("Auxiliary %s: %s on %s — main fallback chain to %s (%s)",
- task or "call", reason, failed_provider or "auto", label, resolved_model or fb_model)
- return fb_client, resolved_model or fb_model, fb_provider
- tried.append(label)
- if tried:
- logger.debug("Auxiliary %s: main fallback chain exhausted (tried: %s)", task or "call", ", ".join(tried))
- return None, None, ""
- def _warn_stale_openai_base_url(runtime_provider: str) -> None:
- """Warn once when OPENAI_BASE_URL is set but config.yaml names a non-custom provider (a stale
- ~/.hermes/.env value after `hermes model` poisons routing)."""
- global _stale_base_url_warned
- if _stale_base_url_warned:
- return
- _env_base = os.getenv("OPENAI_BASE_URL", "").strip()
- _cfg_provider = runtime_provider or _read_main_provider()
- if (_env_base and _cfg_provider and _cfg_provider != "custom" and not _cfg_provider.startswith("custom:")):
- logger.warning(
- "OPENAI_BASE_URL is set (%s) but model.provider is '%s'. "
- "Auxiliary clients may route to the wrong endpoint. "
- "Run: hermes model to reconfigure, or remove "
- "OPENAI_BASE_URL from ~/.hermes/.env",
- _env_base, _cfg_provider,
- )
- _stale_base_url_warned = True
- def _main_route_target(runtime: dict[str, Any], task: Optional[str]) -> tuple[str, str, str, Any, str]:
- """Step-1 target: (provider, model, base_url, api_key, api_mode) of the main runtime, after the
- fast-model opt-in and the MoA aggregator substitution."""
- main_provider = str(runtime.get("provider", "") or _read_main_provider() or "")
- main_model = str(runtime.get("model") or _read_main_model() or "")
- runtime_base_url = str(runtime.get("base_url") or "")
- runtime_api_key = runtime.get("api_key", "")
- runtime_api_mode = str(runtime.get("api_mode") or "")
- # Latency-critical tasks (titling only) opt in to the provider's fast model. Opt-in only:
- # every settings surface defines "auto" as the main model.
- if _task_prefers_fast_model(task) and main_provider and main_provider not in {"auto", ""}:
- fast_model = _get_aux_model_for_provider(main_provider, prefer_fast=True)
- if fast_model and fast_model != main_model:
- logger.debug("Auxiliary task %s: preferring fast model %s over main model %s",
- task, fast_model, main_model)
- main_model = fast_model
- # MoA virtual provider: the preset name is not a wire model; run aux on the aggregator and drop
- # the facade's "moa://local" base_url / placeholder key so it uses its own credentials.
- if main_provider == "moa":
- _agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
- if _agg_provider and _agg_model:
- main_provider, main_model = _agg_provider, _agg_model
- runtime_base_url = runtime_api_key = runtime_api_mode = ""
- return main_provider, main_model, runtime_base_url, runtime_api_key, runtime_api_mode
- def _try_main_provider_route(
- main_provider: str, main_model: str, runtime_base_url: str, runtime_api_key: Any, runtime_api_mode: str,
- ) -> Optional[tuple[Any, str, str]]:
- """Step 1: route aux onto the main provider + main model; None if unusable."""
- if not (main_provider and main_model and main_provider not in {"auto", ""}):
- return None
- resolved_provider = main_provider
- explicit_base_url = runtime_base_url or None
- health_base_url = _custom_health_base_url(main_provider, explicit_base_url)
- explicit_api_key = None
- if runtime_base_url and main_provider == "custom":
- # Anonymous custom endpoint — pass through explicit base_url + api_key.
- explicit_api_key = runtime_api_key or None
- elif main_provider.startswith("custom:"):
- # Named custom provider (custom_providers / providers dict entry).
- _has_named_entry = False
- with contextlib.suppress(ImportError):
- from hermes_cli.runtime_provider import _get_named_custom_provider
- _has_named_entry = _get_named_custom_provider(main_provider) is not None
- if _has_named_entry:
- # KEEP the full ``custom:<name>`` so the named arm honours the entry's api_mode
- # (collapsing to "custom" strips /anthropic → 404s). base_url/api_key come from the entry.
- explicit_base_url = None
- elif runtime_base_url:
- # Config-less named custom provider (live runtime only): anonymous custom arm + runtime key.
- # See #34777.
- resolved_provider = "custom"
- explicit_api_key = runtime_api_key or None
- elif runtime_api_key:
- explicit_api_key = runtime_api_key
- elif runtime_api_key:
- # Pin aux to the main session's working key, not a re-selected (maybe exhausted) pool key.
- explicit_api_key = runtime_api_key
- # Skip if the main provider was recently 402'd (unhealthy TTL bounds the bypass).
- main_chain_label = _normalize_chain_label(resolved_provider)
- if main_chain_label and _is_provider_unhealthy(main_chain_label, health_base_url):
- _log_skip_unhealthy(main_chain_label, base_url=health_base_url)
- return None
- client, resolved = resolve_provider_client(
- resolved_provider, main_model, explicit_base_url=explicit_base_url,
- explicit_api_key=explicit_api_key, api_mode=runtime_api_mode or None,
- )
- if client is None:
- return None
- logger.info("Auxiliary auto-detect: using main provider %s (%s)", main_provider, resolved or main_model)
- return client, resolved or main_model, resolved_provider
- def _discovery_chain_allowed(main_provider: str, task: Optional[str] = None) -> bool:
- """The built-in discovery chain is a convenience for installs with NO selected main provider.
- Once the user picked one, every auxiliary route must be a provider they configured (main,
- ``auxiliary.<task>``, ``fallback_providers``); guessing "whatever else is logged in" bills an
- account they never pointed this session at (xAI OAuth session with a dead token → every
- compression silently charged to a Nous Portal balance)."""
- if (main_provider or "").strip().lower() in {"", "auto"}:
- return True
- logger.warning(
- "Auxiliary %s: main provider %s is unavailable and no fallback_chain / fallback_providers is "
- "configured — refusing to guess another logged-in provider. Re-authenticate (`hermes model`) "
- "or declare a fallback.", task or "call", main_provider)
- return False
- def _try_discovery_chain() -> tuple[Optional[OpenAI], Optional[str], str]:
- """Step 3: hardcoded aggregator/fallback chain, skipping unhealthy providers."""
- tried = []
- for label, try_fn in _get_provider_chain():
- candidate_base_url = _custom_health_base_url(label)
- if _is_provider_unhealthy(label, candidate_base_url):
- _log_skip_unhealthy(label, base_url=candidate_base_url)
- tried.append(f"{label} (unhealthy)")
- continue
- client, model = try_fn()
- if client is not None:
- if tried:
- logger.info("Auxiliary auto-detect: using %s (%s) — skipped: %s",
- label, model or "default", ", ".join(tried))
- else:
- logger.info("Auxiliary auto-detect: using %s (%s)", label, model or "default")
- return client, model, label
- tried.append(label)
- logger.warning("Auxiliary auto-detect: no provider available (tried: %s). "
- "Compression, summarization, and memory flush will not work. "
- "Set OPENROUTER_API_KEY or configure a local model in config.yaml.", ", ".join(tried))
- return None, None, ""
- def _resolve_auto_route(
- main_runtime: Optional[dict[str, Any]] = None, task: Optional[str] = None
- ) -> tuple[Optional[OpenAI], Optional[str], str]:
- """Full auto-detection chain, including the selected provider identity. Priority: (1) main provider +
- main model, regardless of provider type ("auto" means "my main model for side tasks too"; explicit
- per-task overrides still win); (2) configured fallback policy — task chain, then the main agent's
- top-level chain; (3) OpenRouter → Nous → custom → Codex → API-key providers, only with no policy
- and no working main client."""
- runtime = _normalize_main_runtime(main_runtime)
- _warn_stale_openai_base_url(runtime.get("provider", ""))
- main_provider, main_model, base_url, api_key, api_mode = _main_route_target(runtime, task)
- routed = _try_main_provider_route(main_provider, main_model, base_url, api_key, api_mode)
- if routed is not None:
- return routed
- if task:
- fb_client, fb_model, fb_label = _try_configured_fallback_chain(
- task, main_provider or "auto", reason="main provider unavailable")
- if fb_client is not None:
- return fb_client, fb_model, _fallback_provider_from_label(fb_label)
- fb_client, fb_model, fb_label = _try_main_fallback_chain(
- task, main_provider or "auto", reason="main provider unavailable")
- if fb_client is not None:
- return fb_client, fb_model, fb_label
- if not _discovery_chain_allowed(main_provider, task):
- return None, None, ""
- return _try_discovery_chain()
- def _effective_provider_for_client(client: Any, fallback: str) -> str:
- """Return the concrete provider selected for an auto-routed client."""
- effective_provider = getattr(client, "_hermes_aux_effective_provider", "")
- if isinstance(effective_provider, str) and effective_provider:
- return effective_provider
- return str(fallback or "")
- # Centralized Provider Router: resolve_provider_client() is the single entry point for building a configured
- # client (auth, base URL, headers, API format) from (provider, model). Never read auth env vars ad-hoc.
- def _to_async_client(sync_client, model: str, is_vision: bool = False):
- """Sync client → async counterpart, preserving Codex routing (``is_vision`` keeps the endpoint's vision headers)."""
- from openai import AsyncOpenAI
- if isinstance(sync_client, _AuxProbeClientStub):
- return sync_client, model
- if isinstance(sync_client, CodexAuxiliaryClient):
- return AsyncCodexAuxiliaryClient(sync_client), model
- if isinstance(sync_client, AnthropicAuxiliaryClient):
- return AsyncAnthropicAuxiliaryClient(sync_client), model
- if isinstance(sync_client, BedrockAuxiliaryClient):
- return AsyncBedrockAuxiliaryClient(sync_client), model
- with contextlib.suppress(ImportError):
- from agent.gemini_native_adapter import GeminiNativeClient, AsyncGeminiNativeClient
- if isinstance(sync_client, GeminiNativeClient):
- return AsyncGeminiNativeClient(sync_client), model
- # ACP shims (subprocess, not an HTTP pool) are already async-safe and opt out of the wrapper.
- if _client_declares(sync_client, "HERMES_SKIP_ASYNC_WRAP"):
- return sync_client, model
- sync_base_url = str(sync_client.base_url)
- # A key_cmd/Entra client keeps its credential in the SDK's per-request provider slot, not in
- # ``.api_key`` (which stays ""); rebuilding from the snapshot alone ships NO Authorization
- # header. Configured default_headers (a named entry's extra_headers) are likewise carried over,
- # merged last exactly as the SDK merges them on the sync client. See #109595.
- from agent.auxiliary_async_rebuild import async_api_key, configured_default_headers
- async_kwargs = {"api_key": async_api_key(sync_client), "base_url": sync_base_url}
- if base_url_host_matches(sync_base_url, "openrouter.ai"):
- headers = _apply_user_default_headers(build_or_headers())
- elif _is_official_codex_base_url(sync_base_url):
- headers = _apply_user_default_headers(_codex_cloudflare_headers(sync_client.api_key, base_url=sync_base_url))
- else:
- # Provider for the profile-header fallback is inferred from the hostname.
- try:
- from agent.model_metadata import _infer_provider_from_url
- inferred = _infer_provider_from_url(sync_base_url) or ""
- except Exception:
- inferred = ""
- headers = _endpoint_default_headers(sync_base_url, inferred, is_vision=is_vision, xai=True)
- headers = {**(headers or {}), **configured_default_headers(sync_client)}
- if headers:
- async_kwargs["default_headers"] = headers
- _apply_required_codex_headers(async_kwargs, access_token=sync_client.api_key, base_url=sync_base_url)
- async_kwargs = {**_openai_http_client_kwargs(sync_base_url, async_mode=True), **async_kwargs}
- # Hermes owns the auxiliary retry/timeout budget; disable SDK-internal retries.
- # See #54465.
- async_kwargs.setdefault("max_retries", 0)
- return AsyncOpenAI(**async_kwargs), model
- def _normalize_resolved_model(model_name: Optional[str], provider: str) -> Optional[str]:
- """Normalize a resolved model for the provider that will receive it."""
- if not model_name:
- return model_name
- try:
- from hermes_cli.model_normalize import normalize_model_for_provider
- return normalize_model_for_provider(model_name, provider)
- except Exception:
- return model_name
- def _named_custom_api_key(custom_entry: dict[str, Any], provider: str, custom_base: str) -> Any:
- """Credential for a named custom provider: inline api_key → key_env → key_cmd → credential pool → placeholder.
- Aux resolves named custom providers here, not via _resolve_named_custom_runtime, so key_cmd must be
- honoured at the same precedence or every aux call 401s."""
- custom_key: Any = (custom_entry.get("api_key") or "").strip()
- custom_key_env = (custom_entry.get("key_env") or custom_entry.get("api_key_env") or "").strip()
- if not custom_key and custom_key_env:
- custom_key = _scoped_key_env(custom_key_env)
- custom_key_cmd = str(custom_entry.get("key_cmd", "") or "").strip()
- if custom_key_cmd:
- from agent.command_token_source import build_command_token_provider
- custom_key = build_command_token_provider(custom_key_cmd, custom_entry.get("name") or provider) or custom_key
- if not custom_key:
- with contextlib.suppress(Exception):
- from agent.credential_pool import custom_provider_pool_key_candidates
- pool_name = custom_entry.get("provider_key") or custom_entry.get("name") or provider
- for pool_key in custom_provider_pool_key_candidates(custom_base, pool_name):
- try:
- pool = load_pool(pool_key)
- except Exception:
- continue
- if (pool := entry_scoped_pool(pool, pool_key, custom_entry)) is None or not pool.has_credentials():
- continue
- if (pool_entry := pool.select()) is None:
- continue
- pool_api_key = getattr(pool_entry, "runtime_api_key", None) or getattr(pool_entry, "access_token", "") or ""
- if str(pool_api_key).strip():
- custom_key = str(pool_api_key).strip()
- break
- return custom_key or "no-key-required"
- def _build_bedrock_client(provider: str, model: Optional[str], *, raw_codex: bool) -> tuple[Optional[Any], Optional[str]]:
- """AWS Bedrock: Claude → Anthropic Bedrock SDK (prompt caching, thinking); bare in-Region OpenAI IDs
- → Mantle Responses; everything else, incl. OpenAI ``us.``/``global.`` profiles, → Converse API."""
- try:
- from agent.bedrock_adapter import (
- has_aws_credentials, is_anthropic_bedrock_model, resolve_bedrock_runtime_region,
- bedrock_openai_uses_mantle, bedrock_openai_base_url, resolve_bedrock_bearer_token,
- configure_bedrock_openai_client_kwargs,
- )
- from agent.anthropic_adapter import build_anthropic_bedrock_client
- except ImportError:
- logger.warning("resolve_provider_client: bedrock requested but boto3, httpx/openai, or anthropic SDK not installed")
- return None, None
- if not has_aws_credentials():
- logger.debug("resolve_provider_client: bedrock requested but no AWS credentials found")
- return None, None
- # Region must match the main runtime's resolution (bedrock.region in config first, then
- # env/profile) so aux calls never leave the primary runtime's configured region.
- # See #53880, #65076.
- region = resolve_bedrock_runtime_region()
- default_model = "anthropic.claude-haiku-4-5-20251001-v1:0"
- final_model = _normalize_resolved_model(model or default_model, provider) or default_model
- if bedrock_openai_uses_mantle(final_model):
- # Module-level lazy ``OpenAI`` proxy on purpose so tests can patch("agent.auxiliary_client.OpenAI").
- client_kwargs: dict[str, Any] = {
- "api_key": resolve_bedrock_bearer_token() or "aws-sdk",
- "base_url": bedrock_openai_base_url(region),
- }
- configure_bedrock_openai_client_kwargs(client_kwargs)
- client = OpenAI(**client_kwargs)
- logger.debug("resolve_provider_client: bedrock-openai (%s, %s)", final_model, region)
- return (client if raw_codex else CodexAuxiliaryClient(client, final_model)), final_model
- base_url = f"https://bedrock-runtime.{region}.amazonaws.com"
- if is_anthropic_bedrock_model(final_model):
- try:
- real_client = build_anthropic_bedrock_client(region)
- except ImportError as exc:
- logger.warning("resolve_provider_client: cannot create Bedrock client: %s", exc)
- return None, None
- client = AnthropicAuxiliaryClient(real_client, final_model, api_key="aws-sdk", base_url=base_url)
- logger.debug("resolve_provider_client: bedrock anthropic (%s, %s)", final_model, region)
- else:
- client = BedrockAuxiliaryClient(region, final_model)
- logger.debug("resolve_provider_client: bedrock converse (%s, %s)", final_model, region)
- return client, final_model
- def _build_vertex_client(provider: str, model: Optional[str]) -> tuple[Optional[Any], Optional[str]]:
- """Google Vertex AI: Gemini via the OpenAI-compatible endpoint with an OAuth2 bearer (standard OpenAI client)."""
- try:
- from agent.vertex_adapter import get_vertex_config, has_vertex_credentials
- except ImportError:
- logger.warning("resolve_provider_client: vertex requested but google-auth not installed")
- return None, None
- if not has_vertex_credentials():
- logger.debug("resolve_provider_client: vertex requested but no GCP credentials found")
- return None, None
- token, base_url = get_vertex_config()
- if not token or not base_url:
- logger.warning("resolve_provider_client: vertex requested but could not mint token / resolve project")
- return None, None
- final_model = _normalize_resolved_model(model or "google/gemini-3-flash-preview", provider)
- try:
- # Aliased import: a bare `from openai import OpenAI` would shadow the module-level lazy proxy.
- from openai import OpenAI as _VertexOpenAI
- client = _VertexOpenAI(api_key=token, base_url=base_url)
- except Exception as exc:
- logger.warning("resolve_provider_client: cannot create Vertex client: %s", exc)
- return None, None
- logger.debug("resolve_provider_client: vertex (%s)", final_model)
- return client, final_model
- class _ResolveRequest(NamedTuple):
- """Normalized resolve_provider_client() arguments shared by the per-provider branch helpers."""
- provider: str
- original_provider: str
- model: Optional[str]
- async_mode: bool
- raw_codex: bool
- explicit_base_url: Optional[str]
- explicit_api_key: Optional[str | Callable[[], str]]
- api_mode: Optional[str]
- main_runtime: Optional[dict[str, Any]]
- is_vision: bool
- task: Optional[str]
- _ResolveResult = tuple[Optional[Any], Optional[str]]
- def _normalize_api_key(raw: Any) -> str | Callable[[], str]:
- """A key_cmd/Entra callable passes through uncalled; strings are stripped; anything else is ''."""
- if callable(raw) and not isinstance(raw, str):
- return raw
- return raw.strip() if isinstance(raw, str) else ""
- def _log_once_debug(seen: set, key: Any, msg: str, *args: Any) -> None:
- """Debug-log ``msg`` the first time ``key`` is seen so per-call retries stay silent."""
- if key not in seen:
- seen.add(key)
- logger.debug(msg, *args)
- def _is_actual_auxiliary_route(req: _ResolveRequest, base_url: str) -> bool:
- from hermes_cli.auth import normalize_actual_base_url
- from hermes_cli.providers import is_actual_route
- from hermes_cli.route_identity import normalize_route_base_url
- if is_actual_route(req.provider, base_url):
- return True
- runtime = _normalize_main_runtime(req.main_runtime)
- return bool(
- base_url
- and is_actual_route(runtime.get("provider", ""), runtime.get("base_url", ""))
- and normalize_route_base_url(normalize_actual_base_url(base_url))
- == normalize_route_base_url(
- normalize_actual_base_url(runtime.get("base_url", ""))
- )
- )
- def _wrap_transport(req: _ResolveRequest, client_obj: Any, final_model_str: str,
- base_url_str: str = "", api_key_str: str = ""):
- """Wrap a plain OpenAI client in the right transport adapter; specialized wrappers pass through.
- Codex (Responses API): explicit ``api_mode=codex_responses``, else — with no
- explicit api_mode — api.openai.com + codex model. Anthropic (Messages): ``api_mode=anthropic_messages``,
- any ``/anthropic`` suffix, ``api.kimi.com/coding``, or ``api.anthropic.com``."""
- if _is_actual_auxiliary_route(req, base_url_str):
- client = (
- client_obj._real_client
- if isinstance(client_obj, CodexAuxiliaryClient)
- else client_obj
- )
- client._hermes_aux_effective_provider = "actual"
- return client
- # OpenCode relay targets pick the wire per model; a task/provider-level api_mode is stale for
- # every other model (#98799), so it is re-derived here like the main runtime does.
- from agent.opencode_affinity import opencode_transport
- _oc_mode, _oc_base = opencode_transport(req.provider, final_model_str, base_url_str)
- if _oc_mode:
- req, base_url_str = req._replace(api_mode=_oc_mode), _oc_base
- needs_codex = not (
- isinstance(client_obj, CodexAuxiliaryClient) or req.raw_codex
- ) and (
- req.api_mode == "codex_responses"
- or (
- not req.api_mode
- and base_url_hostname(base_url_str) == "api.openai.com"
- and "codex" in (final_model_str or "").lower()
- )
- )
- if needs_codex:
- logger.debug("resolve_provider_client: wrapping client in CodexAuxiliaryClient "
- "(api_mode=%s, model=%s, base_url=%s)",
- req.api_mode or "auto-detected", final_model_str, base_url_str[:60] if base_url_str else "")
- return CodexAuxiliaryClient(client_obj, final_model_str)
- # A profile that declares the Messages wire (commandcode-anthropic) is on it whatever the URL
- # looks like; the same declaration gates ``_reasoning_config`` in _build_call_kwargs.
- api_mode = req.api_mode or _profile_declared_messages_wire(req.provider)
- return _maybe_wrap_anthropic(client_obj, final_model_str, api_key_str, base_url_str, api_mode)
- def _profile_declared_messages_wire(provider: str) -> Optional[str]:
- """``"anthropic_messages"`` when the registered profile declares that api_mode, else None."""
- from providers import get_provider_profile
- profile = get_provider_profile(str(provider or "").strip().lower())
- return "anthropic_messages" if profile is not None and profile.api_mode == "anthropic_messages" else None
- def _route_client(req: _ResolveRequest, client_obj: Any, final_model_str: Optional[str]) -> _ResolveResult:
- """Return (client, model), converting to the async wrapper when ``req.async_mode``."""
- if req.async_mode:
- return _to_async_client(client_obj, final_model_str, is_vision=req.is_vision)
- return client_obj, final_model_str
- def _route_or_warn(req: _ResolveRequest, client: Any, default: Optional[str], unavailable_msg: str, *args: Any) -> _ResolveResult:
- """Route ``client`` on ``req.model or default``; warn and return (None, None) when the provider produced no client."""
- if client is None:
- logger.warning(unavailable_msg, *args)
- return None, None
- return _route_client(req, client, _normalize_resolved_model(req.model or default, req.provider))
- def _resolve_auto_branch(req: _ResolveRequest) -> _ResolveResult:
- """Auto: try all providers in priority order; tag the client with the effective provider (survives cache reuse)."""
- client, resolved, effective_provider = _resolve_auto_route(main_runtime=req.main_runtime, task=req.task)
- if client is None:
- return None, None
- model = req.model
- # An OpenRouter-format model override won't work on a non-OpenRouter provider (e.g. local
- # server); drop it for the provider's default.
- if model and "/" in model and resolved and "/" not in resolved:
- logger.debug("Dropping OpenRouter-format model %r for non-OpenRouter "
- "auxiliary provider (using %r instead)", model, resolved)
- model = None
- routed_client, routed_model = _route_client(req, client, model or resolved)
- if routed_client is not None and effective_provider:
- try:
- routed_client._hermes_aux_effective_provider = effective_provider
- except (AttributeError, TypeError):
- logger.debug("Auxiliary client %s cannot retain effective provider %s",
- type(routed_client).__name__, effective_provider)
- return routed_client, routed_model
- def _resolve_openrouter_branch(req: _ResolveRequest) -> _ResolveResult:
- """OpenRouter."""
- client, default = _try_openrouter(explicit_api_key=req.explicit_api_key, model=req.model,
- explicit_base_url=req.explicit_base_url)
- if client is None:
- logger.warning("resolve_provider_client: openrouter requested but %s",
- _describe_openrouter_unavailable(model=req.model))
- return None, None
- return _route_client(req, client, _normalize_resolved_model(req.model or default, req.provider))
- def _resolve_nous_branch(req: _ResolveRequest) -> _ResolveResult:
- """Nous Portal (OAuth)."""
- model = req.model
- # Vision: caller flag, _PROVIDER_VISION_MODELS override, or a known vision id.
- client, default = _try_nous(vision=(req.is_vision or model in _PROVIDER_VISION_MODELS.values()
- or (model or "").strip().lower() == "mimo-v2-omni"))
- if client is None:
- logger.warning("resolve_provider_client: nous requested but Nous Portal not configured (run: hermes auth)")
- return None, None
- final_model = _normalize_resolved_model(model or default, req.provider)
- # Dual-wire: anthropic/* → /v1/messages, else /chat/completions. Derive from the catalog id
- # (not a stale api_mode) so aux matches the main agent.
- from hermes_cli.providers import nous_api_mode
- client = _maybe_wrap_anthropic(
- client, final_model, str(getattr(client, "api_key", "") or ""),
- str(getattr(client, "base_url", "") or ""), nous_api_mode(final_model),
- )
- return _route_client(req, client, final_model)
- def _resolve_openai_codex_branch(req: _ResolveRequest) -> _ResolveResult:
- """OpenAI Codex (OAuth → Responses API)."""
- model = req.model
- if not model:
- logger.warning("resolve_provider_client: openai-codex requested without a "
- "model; pass model explicitly (e.g. model.model in config.yaml "
- "or auxiliary.<task>.model for per-task aux routing).")
- return None, None
- no_token_msg = "resolve_provider_client: openai-codex requested but no Codex OAuth token found (run: hermes model)"
- if req.raw_codex:
- # Raw OpenAI client for callers needing responses.stream() (main agent loop).
- codex_token, base_url = _resolve_codex_credential_and_base()
- if not codex_token:
- logger.warning(no_token_msg)
- return None, None
- raw_client = _create_openai_client(api_key=codex_token, base_url=base_url,
- default_headers=_codex_cloudflare_headers(codex_token, base_url=base_url))
- return raw_client, _normalize_resolved_model(model, req.provider)
- client, default = _build_codex_client(model)
- return _route_or_warn(req, client, default, no_token_msg)
- def _resolve_xai_oauth_branch(req: _ResolveRequest) -> _ResolveResult:
- """xAI Grok OAuth (device code → Responses API). Without this branch xai-oauth falls to the generic
- oauth_external arm, returns (None, None), and silently re-routes every aux task to the Step-2 fallback."""
- client, default = _build_xai_oauth_aux_client(req.model)
- return _route_or_warn(req, client, default,
- "resolve_provider_client: xai-oauth requested but no xAI "
- "OAuth token found (run: hermes model -> xAI Grok OAuth — SuperGrok / Premium+)")
- def _resolve_custom_branch(req: _ResolveRequest) -> _ResolveResult:
- """Custom endpoint (OPENAI_BASE_URL + OPENAI_API_KEY)."""
- provider, model, main_runtime = req.provider, req.model, req.main_runtime
- # wrap_base: base for the Anthropic-wrap decision. anthropic_messages must keep the raw
- # /anthropic base while the plain OpenAI client uses the /v1-rewritten custom_base (never
- # /anthropic/chat/completions). Empty means "use custom_base".
- custom_base = custom_key = wrap_base = ""
- if req.explicit_base_url:
- custom_base = _to_openai_base_url(req.explicit_base_url).strip()
- # Ollama/vLLM/llama.cpp serve the OpenAI surface under /v1; a bare host posts to the
- # native API and 404s (#106010). Only the alias group gets the tail — ``custom`` is verbatim.
- if req.original_provider in _LOCAL_SERVER_ALIASES and not urlparse(custom_base).path.strip("/"):
- custom_base = custom_base.rstrip("/") + "/v1"
- if req.api_mode == "anthropic_messages":
- wrap_base = (req.explicit_base_url or "").strip().rstrip("/")
- if req.original_provider in _LOCAL_SERVER_ALIASES:
- # SECURITY: a local-server alias never borrows OPENAI_API_KEY or the main key —
- # the alias means "this is my own server"; sending an OpenAI secret to whatever
- # host base_url names is never intended. Explicit api_key or the placeholder only.
- custom_key = _normalize_api_key(req.explicit_api_key) or "no-key-required"
- else:
- custom_key = (
- _normalize_api_key(req.explicit_api_key)
- or _scoped_key_env("OPENAI_API_KEY")
- or _read_main_api_key_if_same_origin(custom_base)
- or "no-key-required" # local servers don't need auth
- )
- if not custom_base:
- logger.warning("resolve_provider_client: explicit custom endpoint requested but base_url is empty")
- return None, None
- elif main_runtime:
- # Reuse main_runtime's concrete base_url + api_key for a named custom provider;
- # re-resolving from bare "custom" loses the name and lands on the wrong provider.
- # Re-resolution loses the provider name and falls back to OpenRouter or a wrong API-key provider —
- # the main agent already solved this, we just need to reuse its answer. (#45472)
- _main_base = str(main_runtime.get("base_url") or "").strip().rstrip("/")
- _main_key = _normalize_api_key(main_runtime.get("api_key"))
- if _main_base and _main_key:
- custom_base, custom_key = _main_base, _main_key
- if custom_base and custom_key:
- if _is_actual_auxiliary_route(req, custom_base):
- from hermes_cli.auth import normalize_actual_base_url
- custom_base = normalize_actual_base_url(custom_base)
- final_model = _normalize_resolved_model(
- model or (main_runtime.get("model") if main_runtime else None) or "gpt-4o-mini", provider,
- )
- _clean_base, _dq = _extract_url_query_params(custom_base)
- extra = {"default_query": _dq} if _dq else {}
- from hermes_cli.config import get_custom_provider_extra_headers # this endpoint's entry, as agent_init
- _custom_headers = {**(_endpoint_default_headers(custom_base, provider, is_vision=req.is_vision) or {}),
- **get_custom_provider_extra_headers(custom_base)}
- if _custom_headers:
- extra["default_headers"] = _custom_headers
- client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **extra)
- client = _wrap_transport(req, client, final_model, wrap_base or custom_base, custom_key)
- return _route_client(req, client, final_model)
- # Try custom first, then API-key providers (Codex excluded here:
- # falling through to Codex with no model is a stale-constant trap).
- for try_fn in (_try_custom_endpoint, _resolve_api_key_provider):
- client, default = try_fn()
- if client is not None:
- final_model = _normalize_resolved_model(model or default, provider)
- # ``client.api_key`` may be a callable (Azure Entra bearer provider);
- # wrapping decisions only need base_url + api_mode.
- _raw_ckey = getattr(client, "api_key", "")
- _ckey = "" if (callable(_raw_ckey) and not isinstance(_raw_ckey, str)) else str(_raw_ckey or "")
- client = _wrap_transport(req, client, final_model, str(getattr(client, "base_url", "") or ""), _ckey)
- return _route_client(req, client, final_model)
- logger.warning("resolve_provider_client: custom/main requested but no endpoint credentials found")
- return None, None
- def _named_custom_openai_wire_client(custom_base: str, custom_key: Any, extra_headers: Optional[dict[str, str]] = None):
- """Plain OpenAI client on the /v1 equivalent of a named custom entry's base URL.
- ``extra_headers`` is the entry's own header block (gateway routing tags, proxy auth): the main
- runtime lifts it onto every request, so aux must too (#109595). SECURITY: never log the values."""
- _clean_base, _dq = _extract_url_query_params(_to_openai_base_url(custom_base))
- _extra = {"default_query": _dq} if _dq else {}
- _headers = _apply_user_default_headers(None)
- if extra_headers:
- _headers = {**(_headers or {}), **extra_headers}
- if _headers:
- _extra["default_headers"] = _headers
- return _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
- def _resolve_named_custom_branch(req: _ResolveRequest) -> Optional[_ResolveResult]:
- """Named custom provider (config.yaml providers dict / custom_providers list); None if no entry matches."""
- from hermes_cli.runtime_provider import _get_named_custom_provider
- provider = req.provider
- # If the raw name is an alias (``kimi`` → ``kimi-coding``) and a custom_providers entry exists
- # under it, the custom entry wins over alias rewriting. Only for aliases, so entries matching a
- # canonical name (e.g. ``nous``) still defer to the built-in.
- custom_entry = None
- if req.original_provider and req.original_provider != provider:
- custom_entry = _get_named_custom_provider(req.original_provider)
- if custom_entry is None:
- custom_entry = _get_named_custom_provider(provider)
- if not custom_entry:
- return None
- # A per-task/explicit base_url or api_key composes OVER the named entry's defaults: the entry supplies
- # whatever the caller left blank, never replaces what the caller set (compression prompts carry
- # conversation history, so a silently swapped destination is a data-routing bug, not a nuisance).
- custom_base = (req.explicit_base_url or custom_entry.get("base_url") or "").strip()
- custom_key = _normalize_api_key(req.explicit_api_key) or _named_custom_api_key(custom_entry, provider, custom_base)
- if custom_key == "no-key-required":
- logger.warning("resolve_provider_client: named custom provider %r has no resolvable "
- "api_key — request will be sent with placeholder no-key-required "
- "and will 401 on auth-required endpoints", custom_entry.get("name") or provider)
- # Actual's wire protocol takes precedence over persisted task/provider modes.
- entry_api_mode = (req.api_mode or custom_entry.get("api_mode") or "").strip()
- if _is_actual_auxiliary_route(req, custom_base):
- from hermes_cli.auth import normalize_actual_base_url
- custom_base = normalize_actual_base_url(custom_base)
- entry_api_mode = "chat_completions"
- if not custom_base:
- logger.warning("resolve_provider_client: named custom provider %r has no base_url", provider)
- return None, None
- from hermes_cli.config import normalize_extra_headers
- entry_headers = normalize_extra_headers(custom_entry.get("extra_headers"))
- final_model = _normalize_resolved_model(
- req.model
- or custom_entry.get("model")
- or (req.main_runtime.get("model") if req.main_runtime else None)
- or _read_main_model_for_aux()
- or "gpt-4o-mini",
- provider,
- )
- # An OpenCode-family entry (``opencode-go-bridge``, #85589) persisted the api_mode of whichever
- # model was selected at save time; the relay picks the wire per model (#98799).
- from agent.opencode_affinity import opencode_transport
- _oc_mode, _oc_base = opencode_transport(provider, final_model, custom_base)
- if _oc_mode:
- entry_api_mode, custom_base = _oc_mode, _oc_base
- logger.debug("resolve_provider_client: named custom provider %r (%s, api_mode=%s)",
- provider, final_model, entry_api_mode or "chat_completions")
- # anthropic_messages: route via AnthropicAuxiliaryClient (mirrors _try_custom_endpoint);
- # the Anthropic SDK sees the original (un-rewritten) URL.
- # Mirrors the anonymous-custom branch in _try_custom_endpoint(). See #15033.
- if entry_api_mode == "anthropic_messages":
- try:
- from agent.anthropic_adapter import build_anthropic_client
- from agent.anthropic_credentials import anthropic_route_is_oauth
- real_client = build_anthropic_client(custom_key, custom_base)
- if entry_headers:
- # Same entry headers as the two OpenAI-wire arms; ``with_options`` merges onto the
- # beta/credential-Omit headers the builder installed (#109595).
- real_client = real_client.with_options(default_headers=entry_headers)
- except ImportError:
- logger.warning("Named custom provider %r declares api_mode=anthropic_messages but the anthropic SDK "
- "is not installed — falling back to OpenAI-wire.", provider)
- return _route_client(req, _named_custom_openai_wire_client(custom_base, custom_key, entry_headers), final_model)
- return _route_client(
- req, AnthropicAuxiliaryClient(real_client, final_model, custom_key, custom_base,
- is_oauth=anthropic_route_is_oauth(custom_base, custom_key)), final_model)
- client = _named_custom_openai_wire_client(custom_base, custom_key, entry_headers)
- # codex_responses, or auto-detect via _wrap_transport (which reads the task-level api_mode).
- if entry_api_mode == "codex_responses":
- client = CodexAuxiliaryClient(client, final_model)
- else:
- client = _wrap_transport(req, client, final_model, custom_base, custom_key)
- return _route_client(req, client, final_model)
- def _resolve_azure_foundry_branch(req: _ResolveRequest) -> _ResolveResult:
- """Azure Foundry via the runtime resolver: the generic PROVIDER_REGISTRY path only knows the static
- AZURE_FOUNDRY_API_KEY env var, missing ``auth_mode: entra_id`` (callable bearer) and config base_url overrides."""
- client, default_model = _try_azure_foundry(model=req.model, explicit_api_key=req.explicit_api_key,
- explicit_base_url=req.explicit_base_url, api_mode=req.api_mode)
- return _route_or_warn(req, client, default_model,
- "resolve_provider_client: azure-foundry requested but "
- "runtime resolution failed (run: hermes doctor for diagnostics)")
- def _api_key_profile_supplied_client(provider: str, **client_kwargs: Any) -> Any | None:
- """Registered profile's own client for an ``api_key`` aux route, or ``None``.
- Same registration seam as ``agent_runtime_helpers._provider_supplied_client`` (main agent)
- and the ``external_process`` branch below: a profile whose wire protocol is not
- OpenAI-over-HTTP overrides ``ProviderProfile.create_client()`` to supply its transport.
- A profile that raises is logged and skipped — a third-party plugin can only fail to
- provide a client, never take the auxiliary resolution down."""
- try:
- from providers import get_provider_profile
- profile = get_provider_profile(provider)
- except Exception:
- return None
- if profile is None:
- return None
- try:
- return profile.create_client(**client_kwargs)
- except Exception:
- logger.warning("resolve_provider_client: provider profile %r failed to create an "
- "auxiliary client; falling back to the standard client path",
- provider, exc_info=True)
- return None
- def _resolve_api_key_branch(req: _ResolveRequest, pconfig: Any, resolve_creds: Callable) -> _ResolveResult:
- """PROVIDER_REGISTRY ``api_key`` providers (Anthropic via its own resolver), honouring explicit overrides."""
- provider = req.provider
- if provider == "anthropic":
- client, default_model = _try_anthropic(explicit_api_key=req.explicit_api_key,
- explicit_base_url=req.explicit_base_url)
- return _route_or_warn(req, client, default_model,
- "resolve_provider_client: anthropic requested but no Anthropic credentials found")
- creds = resolve_creds(provider)
- api_key = str(creds.get("api_key", "")).strip()
- # Explicit api_key override (fallback_model / custom_providers entry) lets callers
- # authenticate where no built-in credential is registered for this alias.
- api_key = _normalize_api_key(req.explicit_api_key) or api_key
- raw_base_url = str(creds.get("base_url", "")).strip().rstrip("/") or pconfig.inference_base_url
- if req.explicit_base_url:
- raw_base_url = req.explicit_base_url.strip().rstrip("/")
- if provider == "actual":
- with contextlib.suppress(Exception):
- from hermes_cli.auth import (
- ACTUAL_LOCAL_NOAUTH_PLACEHOLDER, is_actual_local_base_url, normalize_actual_base_url
- )
- raw_base_url = normalize_actual_base_url(raw_base_url)
- if not api_key and is_actual_local_base_url(raw_base_url):
- api_key = ACTUAL_LOCAL_NOAUTH_PLACEHOLDER
- if not api_key:
- logger.debug("resolve_provider_client: provider %s has no API key configured (tried: %s)",
- provider, ", ".join(pconfig.api_key_env_vars))
- return None, None
- base_url = _to_openai_base_url(raw_base_url)
- # Explicit base_url override: a fallback_model/custom_providers entry pointing a built-in name elsewhere.
- if req.explicit_base_url and provider != "actual":
- base_url = _to_openai_base_url(req.explicit_base_url.strip().rstrip("/"))
- final_model = _normalize_resolved_model(req.model or _get_aux_model_for_provider(provider), provider)
- # Consulted before the built-in gemini/OpenAI ladder so a registered native transport wins (#112384).
- profile_client = _api_key_profile_supplied_client(provider, api_key=api_key, base_url=base_url)
- if profile_client is not None:
- logger.debug("resolve_provider_client: %s native client from provider profile (%s)", provider, final_model)
- return _route_client(req, profile_client, final_model)
- if provider == "gemini":
- from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url
- if is_native_gemini_base_url(base_url):
- client = GeminiNativeClient(api_key=api_key, base_url=base_url)
- logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
- return _route_client(req, client, final_model)
- headers = _endpoint_default_headers(base_url, provider, is_vision=req.is_vision, xai=True)
- client = _create_openai_client(api_key=api_key, base_url=base_url, **({"default_headers": headers} if headers else {}))
- from agent.provider_route_hooks import model_api_mode # a Responses-only model: call_llm() streams responses
- from providers import get_provider_profile
- profile = get_provider_profile(provider) if final_model and not req.raw_codex else None
- if profile is not None and model_api_mode(profile, final_model) == "codex_responses":
- logger.debug("resolve_provider_client: %s model %s needs Responses API — wrapping", provider, final_model)
- client = CodexAuxiliaryClient(client, final_model)
- # api_mode handling for any API-key provider (direct OpenAI + codex model) and Anthropic-wire
- # endpoints (api.kimi.com/coding, /anthropic gateways) without per-provider branches.
- client = _wrap_transport(req, client, final_model, raw_base_url, api_key)
- logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
- return _route_client(req, client, final_model)
- def _resolve_external_process_branch(req: _ResolveRequest, creds: dict[str, Any]) -> _ResolveResult:
- """PROVIDER_REGISTRY ``external_process`` providers, served via their registered profile."""
- provider = req.provider
- final_model = _normalize_resolved_model(
- req.model or (req.main_runtime.get("model") if req.main_runtime else None) or _read_main_model_for_aux(),
- provider,
- )
- # Keyed on the registered profile, not a provider name, so an out-of-tree ACP provider reaches
- # the auxiliary path (compression, vision, background review) exactly like the in-tree one.
- try:
- from providers import get_provider_profile as _get_provider_profile
- _extproc_profile = _get_provider_profile(provider)
- except Exception:
- _extproc_profile = None
- if _extproc_profile is not None:
- api_key = str(creds.get("api_key", "")).strip()
- base_url = str(creds.get("base_url", "")).strip()
- if not final_model:
- logger.warning("resolve_provider_client: %s requested but no model was provided or configured", provider)
- return None, None
- if not api_key or not base_url:
- logger.warning("resolve_provider_client: %s requested but external process credentials are incomplete", provider)
- return None, None
- try:
- client = _extproc_profile.create_client(
- api_key=api_key, base_url=base_url,
- command=str(creds.get("command", "")).strip() or None, args=list(creds.get("args") or []))
- except Exception:
- logger.warning("resolve_provider_client: profile %r failed to create an external-process client",
- provider, exc_info=True)
- client = None
- if client is not None:
- logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
- return _route_client(req, client, final_model)
- _log_once_debug(_LOGGED_UNSUPPORTED_EXTPROC_KEYS, provider,
- "resolve_provider_client: external-process provider %s not "
- "directly supported", provider)
- return None, None
- def _resolve_registry_branch(req: _ResolveRequest) -> _ResolveResult:
- """PROVIDER_REGISTRY providers, dispatched on ``auth_type``; unknown providers log once."""
- provider = req.provider
- try:
- from hermes_cli.auth import (
- PROVIDER_REGISTRY, resolve_api_key_provider_credentials,
- resolve_external_process_provider_credentials,
- )
- except ImportError:
- logger.debug("hermes_cli.auth not available for provider %s", provider)
- return None, None
- pconfig = PROVIDER_REGISTRY.get(provider)
- if pconfig is None:
- _log_once_debug(_LOGGED_UNKNOWN_PROVIDER_KEYS, provider,
- "resolve_provider_client: unknown provider %r", provider)
- return None, None
- auth_type = pconfig.auth_type
- if auth_type == "api_key":
- return _resolve_api_key_branch(req, pconfig, resolve_api_key_provider_credentials)
- if auth_type == "external_process":
- return _resolve_external_process_branch(req, resolve_external_process_provider_credentials(provider))
- from agent.auxiliary_client_registry import REGISTRY_AUTHTYPE_ARMS
- arm = REGISTRY_AUTHTYPE_ARMS.get(auth_type)
- if arm is not None:
- return arm(req)
- # The first occurrence surfaces a real schema-drift bug; per-call retries stay silent.
- _log_once_debug(_LOGGED_UNHANDLED_AUTHTYPE_KEYS, (auth_type, provider),
- "resolve_provider_client: unhandled auth_type %s for %s",
- auth_type, provider)
- return None, None
- # Explicit providers with a dedicated branch; anything else falls through to named custom
- # providers → azure-foundry → PROVIDER_REGISTRY (order preserved from the original if-chain).
- _EXPLICIT_PROVIDER_BRANCHES: dict[str, Callable[[_ResolveRequest], _ResolveResult]] = {
- "auto": _resolve_auto_branch,
- "openrouter": _resolve_openrouter_branch,
- "nous": _resolve_nous_branch,
- "openai-codex": _resolve_openai_codex_branch,
- "xai-oauth": _resolve_xai_oauth_branch,
- "custom": _resolve_custom_branch,
- }
- def resolve_provider_client(
- provider: str, model: str | None = None, async_mode: bool = False, raw_codex: bool = False,
- explicit_base_url: str | None = None, explicit_api_key: Optional[str | Callable[[], str]] = None,
- api_mode: str | None = None, main_runtime: Optional[dict[str, Any]] = None, is_vision: bool = False,
- task: Optional[str] = None,
- ) -> tuple[Optional[Any], Optional[str]]:
- """Central router: return a configured client (auth, base URL, API format) for a provider + optional model.
- The client always exposes ``.chat.completions.create()``; Codex/Responses providers get an adapter.
- ``provider``: built-in name, ``custom:<name>``, "custom" (OPENAI_BASE_URL + OPENAI_API_KEY) or "auto"
- (full auto-detection chain). ``model=None`` → provider's default aux model. ``raw_codex`` → bare OpenAI
- client for ``responses.stream()`` callers. ``api_mode`` forces "codex_responses"/"chat_completions"/
- "anthropic_messages" instead of auto-detect. Returns (client, resolved_model) or (None, None)."""
- _validate_proxy_env_urls()
- # Keep the pre-alias name so a custom_providers entry named like a built-in alias
- # (e.g. "kimi" → "kimi-coding") is still reachable via the named-custom branch.
- original_provider = (provider or "").strip().lower()
- provider = _normalize_aux_provider(provider)
- api_mode = _canonical_api_mode(str(api_mode or "")).lower() or None
- # MoA chokepoint: "moa" is not an HTTP provider; resolve to the aggregator so direct callers don't
- # dead-end in unknown-provider. Unresolvable preset → leave untouched for the normal diagnostic.
- if provider == "moa":
- _agg_provider, _agg_model = _resolve_moa_aggregator(model)
- if _agg_provider and _agg_model:
- original_provider = _agg_provider.strip().lower()
- provider = _normalize_aux_provider(_agg_provider)
- model = _agg_model
- # The moa:// facade endpoint/key belong to the virtual runtime, not the aggregator.
- if explicit_base_url and str(explicit_base_url).lower().startswith("moa://"):
- explicit_base_url = None
- explicit_api_key = None
- from agent.auxiliary_local_runtime import bare_llamacpp_endpoint
- local = bare_llamacpp_endpoint(original_provider, explicit_base_url, explicit_api_key)
- if local is not None:
- if not local[0]:
- logger.warning("resolve_provider_client: %s requested but no local llama.cpp server is running", original_provider)
- return None, None
- explicit_base_url, explicit_api_key = local
- # Model for concrete providers: caller ``model`` → catalog default (empty for OAuth-gated providers whose
- # lists drift) → configured main model (MoA → aggregator), keeping OAuth aux tasks off the Step-2 fallback.
- # Excluded: ``auto`` (a stale main slug could pair with any picked provider) and Nous + vision (the
- # Portal's tier-aware vision recommendation must win over a text-only model).
- if not model and provider != "auto" and not (provider == "nous" and is_vision):
- # ``auto`` is intentionally excluded: `_resolve_auto_route(main_runtime=...)` returns the model paired
- # with the provider it actually selected. Pre-filling an auto call from `_read_main_model()` can
- # leak a stale process-global runtime into a different provider (for example Claude model slug on
- # Codex OAuth) and override that correctly resolved model. 1. ``model`` argument (caller knew what
- # they wanted) 2. Provider's catalog default — cheap/fast model the provider registered via
- # ``ProviderProfile.default_aux_model`` or the legacy ``_API_KEY_PROVIDER_AUX_MODELS_FALLBACK``
- # dict. 3. User's main model from ``model.model`` in config.yaml. This is the load-bearing step for
- # OAuth providers: an xai-oauth user with grok-4.3 configured gets grok-4.3 for title generation
- # instead of silently dropping to whatever Step-2 fallback (#31845). When the main provider is MoA,
- # ``_read_main_model_for_aux()`` substitutes the preset's aggregator model — the preset NAME is
- # never a valid wire model id, so unset aux models default to the preset's acting model instead.
- # Each provider branch below sees a non-empty ``model`` whenever the user has *anything* configured
- # — no provider-specific empty-model guards needed. When the user has NOTHING configured (fresh
- # install, main_model also empty), the branches still hit their own missing-credentials returns and
- # ``_resolve_auto_route`` falls through to the Step-2 chain as before. Do NOT pre-fill a blank ``auto``
- # request from the config/main default here. Claude model sent to Codex after the main lane fell
- # back to gpt-5.5). Let _resolve_auto_route() return the actual current runtime model when the caller did
- # not explicitly request one. (# compression-current-model) Nous + vision is the one carve-out: the
- # branch below resolves its model from the Portal's tier-aware vision recommendation
- # (``_try_nous(vision= True)``), and ``final_model = model or default`` means anything pre-filled
- # here wins over that. The main chat model is routinely text-only (e.g. a ``:free`` chat SKU), so
- # pre-filling it sends the image to a model that cannot accept one and the Portal 404s. Leave
- # ``model`` unset and let the Portal slot through; only an explicit caller model may override it.
- model = _get_aux_model_for_provider(provider) or _read_main_model_for_aux() or model
- req = _ResolveRequest(
- provider, original_provider, model, async_mode, raw_codex,
- explicit_base_url, explicit_api_key, api_mode, main_runtime, is_vision, task,
- )
- branch = _EXPLICIT_PROVIDER_BRANCHES.get(provider)
- alias_identity = original_provider.removeprefix("custom:")
- # A configured provider whose name collides with a local-server alias is still a named
- # provider. Resolve it before the alias's generic ``custom`` branch, which otherwise loses
- # the configured base_url and can fall through to an unrelated credentialed provider.
- if branch is None or alias_identity in _LOCAL_SERVER_ALIASES:
- try:
- result = _resolve_named_custom_branch(req)
- except ImportError:
- result = None
- if result is not None:
- return result
- if branch is not None:
- return branch(req)
- if provider == "azure-foundry":
- return _resolve_azure_foundry_branch(req)
- return _resolve_registry_branch(req)
- # ── Public API ──────────────────────────────────────────────────────────────
- def get_text_auxiliary_client(task: str = "", *, main_runtime: Optional[dict[str, Any]] = None) -> tuple[Optional[OpenAI], Optional[str]]:
- """Return (client, default_model_slug) for text-only aux tasks; ``task`` selects auxiliary.<task> overrides."""
- provider, model, base_url, api_key, api_mode = _resolve_task_provider_model(task or None)
- return resolve_provider_client(
- provider, model=model, explicit_base_url=base_url, explicit_api_key=api_key,
- api_mode=api_mode, main_runtime=main_runtime,
- )
- _VISION_AUTO_PROVIDER_ORDER = ("openrouter", "nous", "deepinfra")
- def _main_model_supports_vision(provider: str, model: Optional[str]) -> bool:
- """True when ``provider``/``model`` is known to accept image input; unknown capability → True (attempt the call)."""
- try:
- from agent.image_routing import _lookup_supports_vision
- from hermes_cli.config import load_config_readonly
- except ImportError:
- return True
- try:
- supports = _lookup_supports_vision(provider, model, load_config_readonly())
- except Exception: # pragma: no cover - defensive
- return True
- return True if supports is None else bool(supports)
- def _normalize_vision_provider(provider: Optional[str]) -> str:
- return _normalize_aux_provider(provider)
- def _deepinfra_strict_vision_backend(model: Optional[str]) -> tuple[Optional[Any], Optional[str]]:
- """DeepInfra vision: default model is discovered live via default_vision_model() so no hardcoded id can rot."""
- vision_model = model or _resolve_provider_vision_default("deepinfra")
- if not vision_model:
- logger.debug("Vision auto-detect: deepinfra catalog unreachable or returned no vision-tagged models — skipping")
- return None, None
- return resolve_provider_client("deepinfra", vision_model, is_vision=True)
- # Strict (explicitly requested) vision backends by normalized provider name. nous MUST go
- # through resolve_provider_client so anthropic/* picks wrap onto /v1/messages (a bare _try_nous
- # client 404s). openai-codex has no safe default model; callers set auxiliary.<task>.model.
- _STRICT_VISION_BACKENDS: dict[str, Callable[[Optional[str]], tuple[Optional[Any], Optional[str]]]] = {
- "openrouter": lambda model: _try_openrouter(model=model),
- "nous": lambda model: resolve_provider_client("nous", model, is_vision=True),
- "openai-codex": lambda model: resolve_provider_client("openai-codex", model, is_vision=True),
- "anthropic": lambda model: _try_anthropic(),
- "deepinfra": _deepinfra_strict_vision_backend,
- "custom": lambda model: _try_custom_endpoint(),
- }
- def _resolve_strict_vision_backend(provider: str, model: Optional[str] = None) -> tuple[Optional[Any], Optional[str]]:
- backend = _STRICT_VISION_BACKENDS.get(_normalize_vision_provider(provider))
- return backend(model) if backend is not None else (None, None)
- def get_available_vision_backends() -> list[str]:
- """Available vision backends in auto-selection order (active provider → OpenRouter → Nous → DeepInfra).
- Single source of truth for setup, tool gating, and runtime auto-routing.
- """
- available: list[str] = []
- main_provider = _read_main_provider()
- if main_provider and main_provider not in {"auto", ""}:
- if main_provider in _VISION_AUTO_PROVIDER_ORDER:
- main_ok = _resolve_strict_vision_backend(main_provider)[0] is not None
- else:
- main_ok = resolve_provider_client(main_provider, _read_main_model())[0] is not None
- if main_ok:
- available.append(main_provider)
- for p in _VISION_AUTO_PROVIDER_ORDER: # skip if already covered by main provider
- if p not in available and _resolve_strict_vision_backend(p)[0] is not None:
- available.append(p)
- return available
- def _finalize_vision_client(
- resolved_provider: str, sync_client: Any, default_model: Optional[str],
- resolved_model: Optional[str], async_mode: bool,
- ) -> tuple[Optional[str], Optional[Any], Optional[str]]:
- """Apply the explicit model override (and async wrapping) to a resolved vision client."""
- if sync_client is None:
- return resolved_provider, None, None
- final_model = resolved_model or default_model
- if async_mode:
- async_client, async_model = _to_async_client(sync_client, final_model, is_vision=True)
- return resolved_provider, async_client, async_model
- return resolved_provider, sync_client, final_model
- def _vision_main_provider_client(
- main_provider: str, main_model: str, runtime: dict[str, Any], resolved_model: Optional[str],
- resolved_api_mode: Optional[str],
- ) -> tuple[Optional[Any], Optional[str]]:
- """Auto-detect step 1: try the main provider; (None, None) falls through to the aggregator chain."""
- # A provider vision default (static override or catalog discovery) is a *known* multimodal
- # model; the pinned chat model usually isn't, so only fall back to it when no default exists.
- provider_vision_default = _resolve_provider_vision_default(main_provider)
- vision_model = provider_vision_default or main_model
- if main_provider == "nous":
- # Nous picks its vision model from Portal tier-aware slots inside _try_nous(vision=True);
- # passing the chat model would override that and 404. Only auxiliary.vision.model may.
- sync_client, default_model = _resolve_strict_vision_backend(main_provider, resolved_model or provider_vision_default)
- if sync_client is None:
- return None, None
- logger.info("Vision auto-detect: using main provider %s (%s)", main_provider, default_model or resolved_model or main_model)
- return sync_client, default_model
- if main_provider in _PROVIDERS_WITHOUT_VISION: # endpoint rejects image input entirely
- logger.debug("Vision auto-detect: skipping main provider %s (no vision support) — falling through to aggregator chain", main_provider)
- return None, None
- if not _main_model_supports_vision(main_provider, vision_model):
- # Known text-only model. Log only the provider name (CodeQL clear-text-logging FPs).
- logger.debug(
- "Vision auto-detect: skipping main provider %s (reports no vision capability) — falling through to aggregator chain",
- main_provider,
- )
- return None, None
- # Custom endpoints carry no built-in base_url/api_key: recover the live main endpoint from
- # set_runtime_main() or, with no live runtime recorded, the configured custom endpoint.
- rpc_base_url = rpc_api_key = None
- rpc_api_mode = resolved_api_mode
- if main_provider == "custom" or main_provider.startswith("custom:"):
- if runtime.get("base_url"):
- custom_base, custom_key, custom_mode = runtime.get("base_url"), runtime.get("api_key") or None, runtime.get("api_mode")
- else:
- custom_base, custom_key, custom_mode = _resolve_custom_runtime()
- if custom_base:
- rpc_base_url, rpc_api_key = custom_base, custom_key
- rpc_api_mode = resolved_api_mode or custom_mode or None
- rpc_client, rpc_model = resolve_provider_client(
- main_provider, vision_model, api_mode=rpc_api_mode, explicit_base_url=rpc_base_url,
- explicit_api_key=rpc_api_key, main_runtime=runtime, is_vision=True)
- if rpc_client is None:
- return None, None
- logger.info("Vision auto-detect: using main provider %s (%s)", main_provider, rpc_model or vision_model)
- return rpc_client, rpc_model or vision_model
- def _vision_auto_route(
- runtime: dict[str, Any], resolved_model: Optional[str], resolved_api_mode: Optional[str],
- async_mode: bool,
- ) -> tuple[Optional[str], Optional[Any], Optional[str]]:
- """Auto-detect order: 1. main provider + model, 2. OpenRouter, 3. Nous Portal, 4. DeepInfra, 5. stop."""
- main_provider = str(runtime.get("provider") or _read_main_provider())
- main_model = str(runtime.get("model") or _read_main_model())
- if main_provider.strip().lower() == "moa":
- # MoA main_model is a preset NAME, not a wire model — unwrap to the preset's aggregator
- # slot. The moa:// facade endpoint belongs to the virtual provider, not the real one.
- _agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
- if _agg_provider and _agg_model:
- main_provider, main_model = _agg_provider, _agg_model
- runtime = dict(runtime, base_url="", api_key="", api_mode="")
- if main_provider and main_provider not in {"auto", "", "moa"}:
- client, default_model = _vision_main_provider_client(main_provider, main_model, runtime, resolved_model, resolved_api_mode)
- if client is not None:
- return _finalize_vision_client(main_provider, client, default_model, resolved_model, async_mode)
- # Aggregators use their dedicated vision model, not the user's main model.
- for candidate in _VISION_AUTO_PROVIDER_ORDER:
- if candidate == main_provider:
- continue # already tried above
- sync_client, default_model = _resolve_strict_vision_backend(candidate)
- if sync_client is not None:
- return _finalize_vision_client(candidate, sync_client, default_model, resolved_model, async_mode)
- logger.debug("Auxiliary vision client: none available")
- return None, None, None
- # ZAI vision must use the OpenAI-compatible endpoint: the Anthropic wire rejects max_tokens on
- # multimodal calls (error 1210).
- _ZAI_OPENAI_VISION_URLS = ("https://open.bigmodel.cn/api/paas/v4", "https://api.z.ai/api/paas/v4")
- def resolve_vision_provider_client(
- provider: Optional[str] = None, model: Optional[str] = None, *, base_url: Optional[str] = None,
- api_key: Optional[str] = None, async_mode: bool = False,
- main_runtime: Optional[dict[str, Any]] = None,
- ) -> tuple[Optional[str], Optional[Any], Optional[str]]:
- """Resolve the client actually used for vision tasks.
- Direct endpoint overrides beat provider selection; explicit providers may force
- experimental backends; auto mode only tries backends known to work.
- """
- runtime = _normalize_main_runtime(main_runtime)
- requested, resolved_model, resolved_base_url, resolved_api_key, resolved_api_mode = _resolve_task_provider_model(
- "vision", provider, model, base_url, api_key
- )
- # The raw name keeps a llama.cpp alias distinguishable from bare ``custom`` for the last resolve.
- raw_requested, requested = requested, _normalize_vision_provider(requested)
- if resolved_base_url:
- provider_for_base_override = requested if requested and requested not in {"", "auto"} else "custom"
- client, final_model = resolve_provider_client(
- provider_for_base_override, model=resolved_model, async_mode=async_mode,
- explicit_base_url=resolved_base_url, explicit_api_key=resolved_api_key,
- api_mode=resolved_api_mode, main_runtime=runtime,
- )
- return provider_for_base_override, client, (final_model if client is not None else None)
- if requested == "auto":
- return _vision_auto_route(runtime, resolved_model, resolved_api_mode, async_mode)
- if requested in _VISION_AUTO_PROVIDER_ORDER:
- sync_client, default_model = _resolve_strict_vision_backend(requested, resolved_model)
- return _finalize_vision_client(requested, sync_client, default_model, resolved_model, async_mode)
- if requested == "zai":
- for _zai_url in _ZAI_OPENAI_VISION_URLS:
- client, final_model = _get_cached_client(
- requested, resolved_model, async_mode, base_url=_zai_url,
- api_key=resolved_api_key or None, api_mode="chat_completions", main_runtime=runtime,
- is_vision=True,
- )
- if client is not None:
- return _finalize_vision_client(requested, client, final_model, resolved_model, async_mode)
- # Fallback: try without explicit base_url (old behavior)
- client, final_model = _get_cached_client(
- raw_requested, resolved_model, async_mode, api_mode=resolved_api_mode, main_runtime=runtime, is_vision=True,
- )
- return requested, client, (final_model if client is not None else None)
- def auxiliary_max_tokens_param(value: int, *, model: Optional[str] = None) -> dict:
- """Max-tokens kwarg for the auxiliary provider: direct OpenAI, a custom endpoint whose profile declares
- it, and newer OpenAI-family models (by ``model`` name: custom endpoints fronting gpt-5.x) need max_completion_tokens."""
- from agent.provider_route_hooks import route_answer
- _custom_base = _current_custom_base_url()
- direct_openai_family = not _scoped_key_env("OPENROUTER_API_KEY") and _read_nous_auth() is None and (
- base_url_hostname(_custom_base) == "api.openai.com"
- or route_answer("max_tokens_param", base_url=_custom_base, model=model or "") == "max_completion_tokens")
- if direct_openai_family or model_forces_max_completion_tokens(model):
- return {"max_completion_tokens": value}
- return {"max_tokens": value}
- # ── Centralized LLM Call API: call_llm()/async_call_llm() own resolve → cached client → shape
- # request → call → return. Every auxiliary LLM consumer should use these.
- # Client cache: (provider, async_mode, base_url, api_key, api_mode, runtime_key) -> (client, default_model, loop)
- # Loop identity is NOT part of the key: stale-loop entries are replaced in place on async hits,
- # bounding growth to one entry per provider config (avoids fd accumulation in gateways).
- # This bounds cache growth to one entry per unique provider config rather than one per (config ×
- # event-loop), which previously caused unbounded fd accumulation in long-running gateway processes (#10200).
- _client_cache: dict[tuple, tuple] = {}
- _client_cache_lock = threading.Lock()
- _CLIENT_CACHE_MAX_SIZE = 64 # safety belt — evict oldest when exceeded
- class _CallableCacheDiscriminator:
- """Hash a credential callback by identity without exposing its state."""
- __slots__ = ("_callback",)
- def __init__(self, callback: Any) -> None:
- self._callback = callback # retained so its id cannot be reused while cached
- def __hash__(self) -> int:
- return id(self._callback)
- def __eq__(self, other: object) -> bool:
- return isinstance(other, _CallableCacheDiscriminator) and self._callback is other._callback
- def __repr__(self) -> str:
- return "<callable-api-key>"
- def _runtime_cache_discriminator(field: str, value: Any) -> Any:
- """Return a hashable, secret-safe runtime cache-key component."""
- if field == "api_key" and callable(value):
- return _CallableCacheDiscriminator(value)
- if field == "api_key" and isinstance(value, str) and value:
- return ("api-key-digest", hashlib.blake2b(value.encode("utf-8"), digest_size=16).digest())
- return value
- def _client_cache_key(
- provider: str, *, async_mode: bool, base_url: Optional[str] = None,
- api_key: Optional[str] = None, api_mode: Optional[str] = None,
- main_runtime: Optional[dict[str, Any]] = None, is_vision: bool = False,
- task: Optional[str] = None, model: Optional[str] = None,
- ) -> tuple:
- runtime = _normalize_main_runtime(main_runtime)
- # `auto` resolves through the main runtime and task-specific policy, so both join the key.
- runtime_key = tuple(_runtime_cache_discriminator(f, runtime.get(f, "")) for f in _MAIN_RUNTIME_FIELDS) if provider == "auto" else ()
- task_key = (task or "", _task_prefers_fast_model(task)) if provider == "auto" else ""
- pool_hint = _pool_cache_hint(provider, main_runtime=main_runtime)
- # Model MUST be in the key: concurrent calls to the same endpoint with different models would
- # share an entry, and the second builder's _store_cached_client would close the first's client.
- model_key = model or runtime.get("model", "")
- api_key_key = _runtime_cache_discriminator("api_key", api_key or "")
- # Profile home leads the key: callers that omit api_key (pool / Nous auth.json paths) would
- # otherwise share one client across multiplex profiles holding different credentials.
- return (hermes_home_key(), provider, async_mode, base_url or "", api_key_key, api_mode or "", runtime_key, is_vision, task_key, pool_hint, model_key,
- _borrowed_main_credential_key(provider, base_url, api_key, runtime))
- def _borrowed_main_credential_key(provider: str, base_url: Optional[str], api_key: Any, runtime: dict[str, Any]) -> tuple:
- """What a keyless ``custom`` route borrows from the main runtime when its client is built.
- The client keeps that credential for its lifetime, so it joins the cache key: otherwise a
- later runtime (another session, a ``/model`` switch) is served the earlier one's key.
- """
- if _normalize_aux_provider(provider) != "custom":
- return ()
- if not base_url:
- # This shape takes the runtime's endpoint and key even when an explicit key was passed.
- return (runtime.get("base_url", ""), _runtime_cache_discriminator("api_key", runtime.get("api_key", "")))
- # Same normalization as the client build, which treats a blank explicit key as keyless.
- if _normalize_api_key(api_key):
- return ()
- borrowed = _read_main_api_key_if_same_origin(_to_openai_base_url(base_url).strip())
- return (_runtime_cache_discriminator("api_key", borrowed),)
- def _current_event_loop() -> Any:
- """``asyncio.get_event_loop()`` or None when no loop can be obtained (async cache-key binding)."""
- try:
- import asyncio as _aio
- return _aio.get_event_loop()
- except RuntimeError:
- return None
- def _store_cached_client(cache_key: tuple, client: Any, default_model: Optional[str], *, bound_loop: Any = None) -> None:
- if isinstance(client, _AuxProbeClientStub):
- return # probe stubs must never be cached — the next hit would get a dud client
- with _client_cache_lock:
- old_entry = _client_cache.get(cache_key)
- if old_entry is not None and old_entry[0] is not client:
- _close_cached_client(old_entry[0])
- _client_cache[cache_key] = (client, default_model, bound_loop)
- def _refresh_nous_auxiliary_client(
- *, cache_provider: str, model: Optional[str], async_mode: bool, base_url: Optional[str] = None,
- api_key: Optional[str] = None, api_mode: Optional[str] = None,
- main_runtime: Optional[dict[str, Any]] = None, is_vision: bool = False,
- lookup_model: Optional[str] = None, lookup_task: Optional[str] = None,
- ) -> tuple[Optional[Any], Optional[str]]:
- """Refresh Nous runtime creds, rebuild the client, and replace the cache entry.
- ``model`` is the resolved wire model stored as the entry's usable model and returned. The
- cache KEY MUST be built from ``lookup_model``/``lookup_task`` — the model and task as passed
- to ``_get_cached_client`` when the stale client was acquired — so the fresh client overwrites
- the exact entry the stale one is served from. Keying on the resolved model or an empty task
- would leave the expired client immortal and every auxiliary call 401ing forever.
- See #56889.
- For ``provider == "auto"`` the task participates in the cache key (task-specific fallback policy), so it
- MUST be carried into the key here for the same reason as ``lookup_model``; otherwise an auto-provider
- client refreshed on a 401 lands under the ``task=""`` key while the stale entry survives under the
- task-scoped key (#58894).
- """
- runtime = _resolve_nous_runtime_api(force_refresh=True, stale_access_token=api_key)
- if runtime is None:
- return None, model
- fresh_key, fresh_base_url = runtime
- sync_client = _create_openai_client(api_key=fresh_key, base_url=fresh_base_url)
- current_loop = _current_event_loop() if async_mode else None
- if async_mode:
- client, final_model = _to_async_client(sync_client, model or "", is_vision=is_vision)
- else:
- client, final_model = sync_client, model
- cache_key = _client_cache_key(
- cache_provider, async_mode=async_mode, base_url=base_url, api_key=api_key,
- api_mode=api_mode, main_runtime=main_runtime, is_vision=is_vision, task=lookup_task,
- model=lookup_model,
- )
- _store_cached_client(cache_key, client, final_model, bound_loop=current_loop)
- return client, final_model
- def neuter_async_httpx_del() -> None:
- """Monkey-patch ``AsyncHttpxClientWrapper.__del__`` to be a no-op.
- The SDK's ``__del__`` schedules ``aclose()`` on the *running* loop, but the transport is
- bound to the loop the client was created on; when that loop is dead this raises "Event loop
- is closed" into prompt_toolkit's loop. Safe because cached clients are closed explicitly and
- the OS reaps the rest. Call once at CLI startup, before any ``AsyncOpenAI`` is created.
- """
- try:
- from openai._base_client import AsyncHttpxClientWrapper
- AsyncHttpxClientWrapper.__del__ = lambda self: None # type: ignore[assignment]
- except (ImportError, AttributeError):
- pass # Graceful degradation if the SDK changes its internals
- def _force_close_async_httpx(client: Any) -> None:
- """Mark the httpx AsyncClient inside an AsyncOpenAI client as closed so ``__del__`` won't
- schedule ``aclose()`` on a dead loop. Skips the full async close — the OS drops connections."""
- with contextlib.suppress(Exception):
- from httpx._client import ClientState
- inner = getattr(client, "_client", None)
- if inner is not None and not getattr(inner, "is_closed", True):
- inner._state = ClientState.CLOSED
- def _schedule_async_close(close_result: Any, client: Any) -> None:
- """Finish an async close without leaking an unawaited coroutine."""
- async def _await_close() -> None:
- try:
- await close_result
- except Exception:
- pass
- finally:
- _force_close_async_httpx(client)
- runner = _await_close()
- try:
- import asyncio as _aio
- try:
- loop = _aio.get_running_loop()
- except RuntimeError:
- _aio.run(runner)
- else:
- task = loop.create_task(runner)
- def _consume(completed_task) -> None:
- with contextlib.suppress(BaseException):
- completed_task.exception()
- task.add_done_callback(_consume)
- runner = None
- except Exception:
- if runner is not None:
- with contextlib.suppress(Exception):
- runner.close()
- _force_close_async_httpx(client)
- def _close_cached_client(client: Any, *, close_async: bool = False) -> None:
- """Close one cached client, awaiting async transports only when safe."""
- if client is None:
- return
- close_fn = getattr(client, "close", None)
- if not callable(close_fn):
- _force_close_async_httpx(client)
- return
- try:
- close_result = close_fn()
- except Exception:
- _force_close_async_httpx(client)
- return
- if inspect.isawaitable(close_result):
- if close_async:
- _schedule_async_close(close_result, client)
- else:
- # Never await a client owned by another live loop; close the coroutine (no
- # unawaited warning) and neuter the transport.
- with contextlib.suppress(Exception):
- close_result.close()
- _force_close_async_httpx(client)
- return
- _force_close_async_httpx(client)
- def shutdown_cached_clients() -> None:
- """Close all cached clients; call at CLI shutdown *before* the loop closes.
- Snapshot+clear under the lock, close outside it: async teardown can block while an owner
- loop drains, and holding the lock would convoy every caller.
- """
- with _client_cache_lock:
- clients = [(entry[0], entry[2]) for entry in _client_cache.values() if entry[0] is not None]
- _client_cache.clear()
- try:
- import asyncio as _aio
- running_loop = _aio.get_running_loop()
- except RuntimeError:
- running_loop = None
- for client, owner_loop in clients:
- # A live foreign loop owns its transport — neuter only and let it finish teardown.
- # Closed loops and the current loop are safe to drain here.
- close_async = owner_loop is not None and (owner_loop.is_closed() or owner_loop is running_loop)
- _close_cached_client(client, close_async=close_async)
- def cleanup_stale_async_clients() -> None:
- """Force-close cached async clients whose loop is closed; call after each agent turn
- (defense-in-depth behind ``neuter_async_httpx_del``)."""
- with _client_cache_lock:
- stale = [(key, entry[0]) for key, entry in _client_cache.items() if entry[2] is not None and entry[2].is_closed()]
- for key, _client in stale:
- del _client_cache[key]
- for _key, client in stale:
- _close_cached_client(client, close_async=True)
- def _compat_model(client: Any, model: Optional[str], cached_default: Optional[str]) -> Optional[str]:
- """Keep slash-bearing model IDs only for cached clients that accept ``vendor/model`` (OpenRouter
- or a slash-bearing default). Mirrors the resolve_provider_client() guard, which cache hits skip."""
- if model and "/" in model:
- accepts_slash = any(
- obj and base_url_host_matches(str(getattr(obj, "base_url", "") or ""), "openrouter.ai")
- for obj in (client, getattr(client, "_client", None), getattr(client, "client", None))
- ) or bool(cached_default and "/" in cached_default)
- if not accepts_slash:
- return cached_default
- return model or cached_default
- def _get_cached_client(
- provider: str, model: str | None = None, async_mode: bool = False, base_url: str | None = None,
- api_key: str | None = None, api_mode: str | None = None, main_runtime: Optional[dict[str, Any]] = None,
- is_vision: bool = False, task: Optional[str] = None,
- ) -> tuple[Optional[Any], Optional[str]]:
- """Get or create a cached client for the given provider.
- Async clients bind to the loop they were created on, so every async hit validates the cached
- loop is the current, open loop; stale entries are replaced in place (bounded, no cross-loop reuse).
- This keeps cache size bounded to one entry per unique provider config, preventing the fd-exhaustion that
- previously occurred in long-running gateways where recycled worker threads created unbounded entries
- (#10200).
- """
- # A bare llama.cpp alias keys on the live local endpoint: a restarted server (new port/key)
- # must not be served the old client, and a stopped one resolves to nothing, not a cloud client.
- from agent.auxiliary_local_runtime import bare_llamacpp_endpoint
- local = bare_llamacpp_endpoint(provider, base_url, api_key)
- if local is not None:
- if not local[0]:
- return None, None
- base_url, api_key = local
- current_loop = _current_event_loop() if async_mode else None
- runtime = _normalize_main_runtime(main_runtime)
- cache_key = _client_cache_key(
- provider, async_mode=async_mode, base_url=base_url, api_key=api_key, api_mode=api_mode,
- main_runtime=main_runtime, is_vision=is_vision, task=task, model=model,
- )
- with _client_cache_lock:
- if cache_key in _client_cache:
- cached_client, cached_default, cached_loop = _client_cache[cache_key]
- loop_ok = not async_mode or (
- cached_loop is not None and cached_loop is current_loop and not cached_loop.is_closed()
- )
- if loop_ok:
- return cached_client, _compat_model(cached_client, model, cached_default)
- # Stale async entry — evict. Only a closed owner loop may be awaited here; a live
- # foreign loop stays force-neutered.
- _close_cached_client(cached_client, close_async=cached_loop is not None and cached_loop.is_closed())
- del _client_cache[cache_key]
- # Build outside the lock. For pool-backed providers derive the key from the pool entry:
- # resolve_api_key_provider_credentials prefers env vars, which would bypass pool rotation
- # and retry an exhausted key. A pinned ``custom:<name>`` peeks only its entry's rows.
- effective_api_key = api_key
- if not effective_api_key:
- _pool = pool_for_aux_provider(provider, _normalize_aux_provider(provider), lambda k: _load_pool_with_credentials(k, " (peek)"))
- if _pool is not None and (_pe := _peek_pool_entry(provider, _pool)) is not None:
- effective_api_key = _pool_runtime_api_key(_pe) or api_key
- client, default_model = resolve_provider_client(
- provider, model, async_mode, explicit_base_url=base_url, explicit_api_key=effective_api_key,
- api_mode=api_mode, main_runtime=runtime, is_vision=is_vision, task=task,
- )
- if client is not None and _aux_probe_active():
- # Availability probes answer "resolvable?" and must leave the cache untouched: the
- # probe stub (bare, or wrapped in a Codex/Anthropic adapter whose leaf is the stub)
- # shares the runtime key, and a cached one is served to every later caller — the
- # next probe dies in _compat_model() on stub attribute access, so check_fns flip to
- # False and vision tools vanish for the process lifetime (#87654).
- return client, model or default_model
- if client is not None:
- with _client_cache_lock:
- if cache_key not in _client_cache:
- # FIFO safety-belt eviction. Do NOT close evicted clients: another caller may be
- # mid-request on one; refcount/GC handles it.
- while len(_client_cache) >= _CLIENT_CACHE_MAX_SIZE:
- del _client_cache[next(iter(_client_cache))]
- _client_cache[cache_key] = (client, default_model, current_loop)
- else:
- built_client = client
- client, default_model, _ = _client_cache[cache_key]
- # Race loser was never exposed to a caller — safe to close now.
- _close_cached_client(built_client, close_async=async_mode)
- return client, _compat_model(client, model, default_model)
- # MoA virtual provider: an *explicit* `provider: moa` override (either the caller-passed `provider` arg or
- # `auxiliary.<task>.provider` in config.yaml) reaches this function directly — it never goes through
- # _resolve_auto_route(), which only unwraps the *implicit* "main provider is moa" case (#53827). Left as-is, "moa"
- # is returned verbatim and resolve_provider_client() looks it up in PROVIDER_REGISTRY (which has no "moa"
- # entry — it's not a real HTTP provider), falls to the unknown-provider dead end, and call_llm surfaces a
- # nonsensical "MOA_API_KEY environment variable" error for a provider that was never meant to be reached
- # over the wire. Auxiliary tasks don't need the reference fan-out — resolve to the preset's aggregator slot
- # instead, exactly like the implicit path does (shared helper: _resolve_moa_aggregator).
- def _unwrap_moa_provider(prov: str, mdl: Optional[str]) -> tuple[str, Optional[str]]:
- """Resolve an *explicit* ``provider: moa`` to its preset's aggregator slot (_resolve_auto_route()
- only unwraps the implicit case; "moa" isn't in PROVIDER_REGISTRY and would dead-end)."""
- if prov.strip().lower() != "moa":
- return prov, mdl
- agg_provider, agg_model = _resolve_moa_aggregator(mdl)
- if agg_provider and agg_model:
- return agg_provider, agg_model
- return prov, mdl
- def _preserve_provider_with_base_url(prov: Optional[str]) -> bool:
- """True when a first-class provider keeps its identity alongside an explicit base_url."""
- normalized = str(prov or "").strip().lower()
- if normalized in {"", "auto", "custom"} or normalized.startswith("custom:"):
- return False
- if normalized in _LOCAL_SERVER_ALIASES:
- return True # the custom branch applies the /v1 tail only when it still sees the alias
- # #76602 — two independent lookups, each guarded by its own try/except so a partial
- # catalog-load failure in either path doesn't suppress the other. A user-defined
- # ``providers:`` entry keeps its name alongside an explicit base_url so the named-custom
- # branch resolves the entry's key/transport instead of the anonymous ``custom`` downgrade
- # (which sends ``no-key-required`` and 401s on auth-required endpoints).
- if _builtin_provider_present(normalized):
- return True
- if _named_custom_provider_present(normalized):
- return True
- return False
- def _builtin_provider_present(name: str) -> bool:
- """*name* is a built-in provider. Never raises, so a failing catalog cannot suppress the parallel
- user-defined lookup in ``_preserve_provider_with_base_url`` (#76602)."""
- try:
- from hermes_cli.providers import get_provider
- return get_provider(name) is not None
- except Exception:
- # Catalog unavailable (early import/test paths): still keep the high-risk provider-backed routes.
- return (name in {"anthropic", "minimax-oauth", "nous", "openai-codex", "qwen-oauth", "xai-oauth"}
- or _profile_credentials_bypass_custom(name))
- def _profile_credentials_bypass_custom(name: str) -> bool:
- """A profile (canonical name) whose credentials ``custom`` cannot carry: ambient, or an external process's."""
- try:
- from providers import get_provider_profile
- profile = get_provider_profile(name)
- except Exception:
- logger.debug("provider profile lookup for %s failed", name, exc_info=True)
- return False
- return profile is not None and profile.name == name and (
- profile.ambient_credentials or profile.auth_type == "external_process")
- def _named_custom_provider_present(name: str) -> bool:
- """Look up *name* in the user-defined ``providers:`` section of
- config.yaml, returning False when the config is unavailable or
- fails to load.
- Used by ``_preserve_provider_with_base_url`` so a user-defined
- provider remains preserved even when the built-in registry raises
- (parallel lookup; each side fails independently — #76602).
- """
- try:
- from hermes_cli.runtime_provider import _get_named_custom_provider
- return _get_named_custom_provider(name) is not None
- except Exception:
- # Config not loaded yet (early import paths, tests) — fail closed:
- # never widen True just because the import / load failed.
- return False
- def _resolve_task_provider_model(
- task: str | None = None, provider: str | None = None, model: str | None = None, base_url: Optional[str] = None,
- api_key: Optional[str] = None,
- ) -> tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]]:
- """Determine (provider, model, base_url, api_key, api_mode) for a call.
- Priority: explicit args > config auxiliary.{task}.* > "auto". A bare base_url means custom,
- but a first-class provider + base_url keeps the provider identity so its auth/transport
- shaping still applies. api_mode is "chat_completions", "codex_responses", or None (auto).
- """
- cfg_provider = cfg_model = cfg_base_url = cfg_api_key = resolved_api_mode = None
- if task:
- task_config = _get_auxiliary_task_config(task)
- cfg_provider = str(task_config.get("provider", "")).strip() or None
- cfg_model = str(task_config.get("model", "")).strip() or None
- cfg_base_url = str(task_config.get("base_url", "")).strip() or None
- cfg_api_key = str(task_config.get("api_key", "")).strip() or None
- if not cfg_api_key: # key_env → env var when api_key is not set directly
- cfg_key_env = str(task_config.get("key_env") or task_config.get("api_key_env") or "").strip()
- if cfg_key_env:
- cfg_api_key = _scoped_key_env(cfg_key_env) or None
- # User-facing spellings (``responses``, ``anthropic``, …) canonicalize here so every
- # branch downstream compares against the transport names only (#39750).
- resolved_api_mode = _canonical_api_mode(str(task_config.get("api_mode") or "")).lower() or None
- # 'auto' is a sentinel ("inherit / auto-detect"), not a model id — leaking it to the wire
- # yields a 200 with an error-text body that consumers accept as output. The explicit `model`
- # kwarg needs the same normalization: MoA slots forward preset `model:` fields through it.
- if model and model.lower() == "auto":
- model = None
- if cfg_model and cfg_model.lower() == "auto":
- cfg_model = None
- resolved_model = model or cfg_model
- # Any moa:// facade endpoint belongs to the facade, not the aggregator's real provider —
- # drop it (mirrors _resolve_auto_route()).
- if provider and str(provider).strip().lower() == "moa":
- provider, resolved_model = _unwrap_moa_provider(provider, resolved_model)
- if provider and provider.lower() != "moa":
- base_url = None
- api_key = None
- elif cfg_provider and str(cfg_provider).strip().lower() == "moa":
- cfg_provider, cfg_model = _unwrap_moa_provider(cfg_provider, resolved_model)
- if cfg_provider and cfg_provider.lower() != "moa":
- resolved_model = cfg_model
- cfg_base_url = None
- cfg_api_key = None
- # One shared alias table with resolve_runtime_provider(): ``provider: openai`` routes the same
- # way here (compression/vision/title) and on the runtime path (background review, curator, MoA).
- from hermes_cli.runtime_provider_custom import expand_direct_api_alias
- if provider:
- provider, base_url = expand_direct_api_alias(provider, base_url)
- if cfg_provider:
- cfg_provider, cfg_base_url = expand_direct_api_alias(cfg_provider, cfg_base_url)
- # An explicit provider without base_url adopts the task's configured endpoint (same or
- # unnamed provider) so the early return below carries it. Explicit "auto" is excluded — it
- # must keep flowing through auto-resolution.
- # See #58515.
- if provider and provider != "auto" and not base_url and cfg_base_url and cfg_provider in (None, provider):
- base_url = cfg_base_url
- if not api_key:
- api_key = cfg_api_key
- if base_url:
- kept = provider if _preserve_provider_with_base_url(provider) else "custom"
- return kept, resolved_model, base_url, api_key, resolved_api_mode
- if provider:
- return provider, resolved_model, base_url, api_key, resolved_api_mode
- if cfg_base_url and cfg_api_key:
- kept = cfg_provider if str(cfg_provider or "").strip().lower() in _LOCAL_SERVER_ALIASES else "custom"
- return kept, resolved_model, cfg_base_url, cfg_api_key, resolved_api_mode
- if cfg_base_url and cfg_provider and cfg_provider != "auto":
- # base_url without api_key: keep the provider so it can resolve credentials from env
- # vars instead of locking into "custom".
- return cfg_provider, resolved_model, cfg_base_url, None, resolved_api_mode
- if cfg_provider and cfg_provider != "auto":
- return cfg_provider, resolved_model, cfg_base_url, cfg_api_key, resolved_api_mode
- return "auto", resolved_model, None, None, resolved_api_mode
- _DEFAULT_AUX_TIMEOUT = 30.0
- # Reasoning compression models can exceed the default 120 s config timeout, falling back to the
- # deterministic marker. Bounded *floor* for config-derived compression timeouts only; never
- # overrides an explicit per-call timeout.
- # Compression summarises large conversation histories; a reasoning auxiliary model (e.g. Codex / GPT-5.5)
- # can legitimately take longer than the default ``auxiliary.compression.timeout`` (120 s), causing the
- # stream to time out and the compressor to fall back to the deterministic context marker (#54915). A floor
- # is harmless for fast compression models (they finish before the deadline) and is a minimum, so a higher
- # config value is kept unchanged.
- _COMPRESSION_TIMEOUT_FLOOR_SECONDS = 300.0
- # Read-time resolution of auxiliary.<task> (plugin defaults, inherit_from) lives in its own module;
- # re-exported here because callers and tests reach it as agent.auxiliary_client._get_auxiliary_task_config.
- from agent.auxiliary_task_config import _get_auxiliary_task_config
- class CompressionFastLane(NamedTuple):
- """Explicit, non-reasoning compression route."""
- certified_non_reasoning: bool
- reasoning_config: Optional[dict[str, Any]]
- def _fast_lane_config_fields(config: dict[str, Any]) -> tuple[str, str, bool]:
- """Only explicit reasoning disablement certifies a non-reasoning route."""
- from hermes_constants import parse_reasoning_effort
- provider = str(config.get("provider") or "").strip().lower()
- model = str(config.get("model") or "").strip()
- parsed_effort = parse_reasoning_effort(config.get("reasoning_effort"))
- non_reasoning = parsed_effort is not None and parsed_effort.get("enabled") is False
- return provider, model, non_reasoning
- def resolve_compression_fast_lane(
- actual_provider: str, actual_model: Optional[str], *, requested_provider: Optional[str] = None,
- requested_model: Optional[str] = None, route_config: Optional[dict[str, Any]] = None,
- ) -> CompressionFastLane:
- """Certify explicit non-reasoning settings only on the matching destination."""
- config = route_config if route_config is not None else _get_auxiliary_task_config("compression")
- cfg_provider, cfg_model, non_reasoning = _fast_lane_config_fields(config)
- provider = str(requested_provider or "").strip().lower() or cfg_provider
- model = str(requested_model or "").strip() or cfg_model
- explicit_route = provider not in {"", "auto"} and model.lower() not in {"", "auto"}
- actual_norm = _normalize_aux_provider(_fallback_provider_from_label(str(actual_provider or "")))
- provider_matches = actual_norm == _normalize_aux_provider(provider)
- model_matches = str(actual_model or "").strip().lower() == model.lower()
- if explicit_route and provider_matches and model_matches and non_reasoning:
- return CompressionFastLane(True, {"enabled": False, "effort": "none"})
- return CompressionFastLane(False, None)
- def _compression_config_claims_fast_lane(config: dict[str, Any]) -> bool:
- """Whether task config declares fast-only controls that cannot leak."""
- provider, model, non_reasoning = _fast_lane_config_fields(config)
- return provider not in {"", "auto"} and model.lower() not in {"", "auto"} and non_reasoning
- def _compression_fast_lane_controls(
- task: str | None, *, actual_provider: str, actual_model: str | None,
- requested_provider: str | None, requested_model: str | None, route_config: dict[str, Any],
- leak_guard_config: dict[str, Any], max_tokens: int | None, extra_body: dict[str, Any],
- ) -> tuple[int | None, dict[str, Any]]:
- """Apply the certified compression controls to one resolved route."""
- if task != "compression" or max_tokens is not None:
- return max_tokens, extra_body
- body = dict(extra_body)
- lane = resolve_compression_fast_lane(
- actual_provider, actual_model, requested_provider=requested_provider, requested_model=requested_model, route_config=route_config,
- )
- if lane.reasoning_config is not None:
- if "reasoning" not in body:
- body["reasoning"] = lane.reasoning_config
- elif _compression_config_claims_fast_lane(leak_guard_config):
- body.pop("reasoning", None)
- return max_tokens, body
- def _get_task_no_progress_timeout(task: str) -> Optional[float]:
- """``auxiliary.<task>.no_progress_timeout`` from config, or None when unset/invalid
- (the Codex and chat-stream watchdogs then keep the built-in ``_AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS``
- default). Lets an operator widen the substantive-progress window independently of the
- overall request timeout — see #108104."""
- if not task:
- return None
- raw = _get_auxiliary_task_config(task).get("no_progress_timeout")
- if raw is None:
- return None
- try:
- value = float(raw)
- except (ValueError, TypeError):
- value = 0.0
- if isinstance(raw, bool) or value <= 0:
- # Fail clearly: a typo here silently leaving the 60s default is exactly the
- # "why did my 600s request abort after 60s" confusion the key exists to remove.
- logger.warning(
- "auxiliary.%s.no_progress_timeout=%r is not a positive number of seconds; "
- "using the built-in %.0fs default", task, raw, _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS)
- return None
- return value
- def _get_task_timeout(task: str, default: float = _DEFAULT_AUX_TIMEOUT) -> float:
- """``auxiliary.<task>.timeout`` from config, else *default*."""
- if not task:
- return default
- raw = _get_auxiliary_task_config(task).get("timeout")
- if raw is not None:
- with contextlib.suppress(ValueError, TypeError):
- return float(raw)
- return default
- def _effective_aux_timeout(task: str, timeout: Optional[float]) -> float:
- """Explicit ``timeout`` wins, else config; compression gets a floor so a reasoning model
- summarising a large context isn't cut off."""
- if timeout is not None:
- return timeout
- effective = _get_task_timeout(task)
- return max(effective, _COMPRESSION_TIMEOUT_FLOOR_SECONDS) if task == "compression" else effective
- def _with_custom_endpoint_extra_body(
- extra_body: Optional[dict], provider: str, model: Optional[str], base_url: Optional[str],
- ) -> Optional[dict]:
- """Layer the destination custom provider's ``extra_body`` UNDER the task/caller body.
- The main agent merges a ``custom_providers`` / ``providers:`` entry's ``extra_body`` into every
- request to that endpoint (``agent_init._merge_custom_provider_extra_body``); an aux request routed
- to the same entry must carry it too, or a proxy that 400s without e.g. a ``user`` field breaks
- smart approval, titles and compression (#103738). Resolved per destination with the agent's own
- matcher, so a fallback to another provider never inherits it; ``auxiliary.<task>.extra_body`` and
- caller keys win on conflict, as request_overrides win over the entry on the main path."""
- if not base_url:
- return extra_body
- try:
- from agent.agent_init import _custom_provider_extra_body_for_agent
- from hermes_cli.config import get_compatible_custom_providers, load_config_readonly
- inherited = _custom_provider_extra_body_for_agent(
- provider=provider or "", model=model or "", base_url=str(base_url),
- custom_providers=get_compatible_custom_providers(load_config_readonly()),
- )
- except Exception:
- logger.debug("custom provider extra_body lookup failed for aux request", exc_info=True)
- return extra_body
- if not inherited:
- return extra_body
- return {**inherited, **(extra_body or {})}
- def _get_task_extra_body(task: str) -> dict[str, Any]:
- """Shallow copy of ``auxiliary.<task>.extra_body`` with ``reasoning_effort`` folded into
- ``reasoning`` unless one is configured (more specific wins). MoA tasks are excluded: their
- reasoning depth is per-slot in the preset."""
- task_config = _get_auxiliary_task_config(task)
- raw = task_config.get("extra_body")
- result = dict(raw) if isinstance(raw, dict) else {}
- if "reasoning" in result:
- return result
- effort = task_config.get("reasoning_effort")
- if effort is None or effort == "":
- return result
- if task in ("moa_reference", "moa_aggregator"):
- logger.warning(
- "auxiliary.%s.reasoning_effort is not supported — MoA reasoning depth is per-slot: set reasoning_effort "
- "on the preset's reference_models entries / aggregator instead (moa.presets.<name>...). Ignoring.",
- task,
- )
- return result
- from hermes_constants import parse_reasoning_effort
- parsed = parse_reasoning_effort(effort)
- if parsed is not None:
- result["reasoning"] = parsed
- else:
- logger.warning(
- "auxiliary.%s.reasoning_effort %r is not a valid level (none, minimal, low, medium, high, xhigh, max, ultra) — ignoring",
- task, effort,
- )
- return result
- # Per-task concurrency limiting: many sessions can spawn background aux calls, and during provider incidents
- # each retries / fans out across the fallback chain, multiplying request volume on already-degraded endpoints.
- # A per-task semaphore caps in-flight calls so retry amplification stays bounded. See #23324.
- # Keyed by profile home as well: the limit is the profile's ``auxiliary.<task>.max_concurrency``, and two
- # multiplexed profiles with different limits would otherwise rebuild (and reset) one shared semaphore.
- _aux_sync_semaphores: dict[tuple[str, str], tuple[int, threading.BoundedSemaphore]] = {}
- _aux_async_semaphores: dict[Any, dict] = {} # loop -> {(home, task): (limit, sem)} until closed: cache_bounds.live_loop_table
- _aux_sem_lock = threading.Lock()
- def _get_task_max_concurrency(task: Optional[str]) -> Optional[int]:
- """``auxiliary.<task>.max_concurrency`` as a positive int, or None. Vision uses this key for
- its encode/resize CPU pool; its LLM calls stay concurrent."""
- if not task or task == "vision":
- return None
- try:
- value = int(_get_auxiliary_task_config(task).get("max_concurrency"))
- except (TypeError, ValueError): # missing (None) or malformed
- return None
- return value if value > 0 else None
- def _cached_semaphore(store: dict, key: Any, limit: int, factory: Callable[[int], Any]) -> Any:
- """Return the cached semaphore for ``key``, rebuilding it when the limit changed."""
- with _aux_sem_lock:
- entry = store.get(key)
- if entry is None or entry[0] != limit:
- store[key] = entry = (limit, factory(limit))
- return entry[1]
- def _acquire_sync_aux_semaphore(task: Optional[str]) -> Optional[threading.BoundedSemaphore]:
- """Get a per-task sync semaphore, rebuilding it after a config change."""
- limit = _get_task_max_concurrency(task)
- if limit is None:
- return None
- from hermes_constants import hermes_home_key
- return _cached_semaphore(_aux_sync_semaphores, (hermes_home_key(), task), limit, threading.BoundedSemaphore)
- def _acquire_async_aux_semaphore(task: Optional[str]):
- """Get a per-task, per-event-loop async semaphore after config lookup."""
- limit = _get_task_max_concurrency(task)
- if limit is None:
- return None
- import asyncio
- try:
- loop = asyncio.get_running_loop()
- except RuntimeError:
- return None
- from agent.cache_bounds import live_loop_table
- from hermes_constants import hermes_home_key
- tables = live_loop_table(_aux_async_semaphores, loop, _aux_sem_lock)
- return _cached_semaphore(tables, (hermes_home_key(), task), limit, asyncio.Semaphore)
- def _reset_aux_semaphores() -> None:
- """Drop cached semaphores (test helper)."""
- with _aux_sem_lock:
- _aux_sync_semaphores.clear()
- _aux_async_semaphores.clear()
- # Anthropic-compatible endpoints reached via the OpenAI SDK wrapper; their image content blocks
- # must use Anthropic format.
- _ANTHROPIC_COMPAT_PROVIDERS = frozenset({"minimax", "minimax-oauth", "minimax-cn"})
- def _is_anthropic_compat_endpoint(provider: str, base_url: str) -> bool:
- """True for known Anthropic-compatible providers or any ``/anthropic`` URL path."""
- return provider in _ANTHROPIC_COMPAT_PROVIDERS or "/anthropic" in (base_url or "").lower()
- # OpenAI block type → (Anthropic block type, default media type for data: URLs). MiniMax's
- # Anthropic-compatible endpoint wants type="video" (not "video_url"/"input_video") with the same
- # ``source`` shape as "image".
- _ANTHROPIC_MEDIA_BLOCKS = {"image_url": ("image", "image/png"), "video_url": ("video", "video/mp4")}
- def _convert_openai_images_to_anthropic(messages: list) -> list:
- """Convert OpenAI ``image_url``/``video_url`` blocks to Anthropic ``image``/``video``;
- only list-content messages with such blocks change."""
- converted = []
- for msg in messages:
- content = msg.get("content")
- if not isinstance(content, list):
- converted.append(msg)
- continue
- new_content = []
- changed = False
- for block in content:
- block_type = block.get("type")
- if block_type not in _ANTHROPIC_MEDIA_BLOCKS:
- new_content.append(block)
- continue
- url = (block.get(block_type) or {}).get("url", "")
- anth_type, media_type = _ANTHROPIC_MEDIA_BLOCKS[block_type]
- if url.startswith("data:"):
- header, _, b64data = url.partition(",")
- if ":" in header and ";" in header:
- media_type = header.split(":", 1)[1].split(";", 1)[0]
- source = {"type": "base64", "media_type": media_type, "data": b64data}
- else:
- source = {"type": "url", "url": url}
- new_content.append({"type": anth_type, "source": source})
- changed = True
- converted.append({**msg, "content": new_content} if changed else msg)
- return converted
- _PROFILE_REASONING_KEYS = {
- "reasoning", "reasoning_effort", "thinking", "thinking_config", "thinkingconfig",
- "thinking_budget", "thinkingbudget", "enable_thinking", "think", "verbosity",
- }
- def _contains_profile_reasoning_fields(value: Any) -> bool:
- """Return whether a profile payload contains a reasoning wire control (recursive)."""
- if not isinstance(value, dict):
- return False
- return any(
- str(key).strip().lower() in _PROFILE_REASONING_KEYS or _contains_profile_reasoning_fields(nested)
- for key, nested in value.items()
- )
- _NOUS_PROVIDER_NAMES = frozenset({"nous", "nous-portal", "nousresearch"})
- def _nous_on_messages_wire(provider_norm: str, model: str) -> bool:
- """True when a Nous Portal route serves ``model`` over /v1/messages (dual-wire catalog)."""
- if provider_norm not in _NOUS_PROVIDER_NAMES:
- return False
- from hermes_cli.providers import nous_api_mode
- return nous_api_mode(model) == "anthropic_messages"
- _NVIDIA_PROVIDER_NAMES = {"nvidia", "nvidia-nim", "nim", "build-nvidia", "nemotron"}
- _GEMINI_NATIVE_PROVIDER_NAMES = {"gemini", "google", "google-gemini", "google-ai-studio"}
- def _is_gemini_native_route(provider_norm: str, effective_base: str) -> bool:
- """Gemini native by provider name, else (best-effort) by base URL shape."""
- if provider_norm in _GEMINI_NATIVE_PROVIDER_NAMES:
- return True
- if not effective_base:
- return False
- try:
- from agent.gemini_native_adapter import is_native_gemini_base_url
- return is_native_gemini_base_url(effective_base)
- except Exception:
- return False
- def _forwards_max_tokens(provider: str, provider_norm: str, model: str, effective_base: str, task: Optional[str]) -> bool:
- """Whether an explicit max_tokens is forwarded on this route.
- No default cap elsewhere (omitted = provider default; avoids max_completion_tokens / ZAI-vision
- quirks). Forward only where mandatory or honored: Anthropic Messages wire (400 without it);
- NVIDIA NIM (empty choices[] when omitted); MoA reference slots; Gemini native (fixed 65,535
- ceiling otherwise); OpenRouter (budgets the FULL window when omitted → 402 on low credit);
- managed local llama-server (uncapped decode with no EOS burns the GPU to the context window).
- """
- return (
- _is_anthropic_compat_endpoint(provider, effective_base)
- or _nous_on_messages_wire(provider_norm, model)
- or provider_norm in _NVIDIA_PROVIDER_NAMES
- or base_url_host_matches(effective_base, "integrate.api.nvidia.com")
- or str(task) == "moa_reference"
- or _is_gemini_native_route(provider_norm, effective_base)
- or provider_norm == "openrouter"
- or base_url_host_matches(effective_base, "openrouter.ai")
- or _is_managed_local_endpoint(effective_base)
- )
- def _dedupe_tool_names(tools: list, provider: str, model: str) -> list:
- """Drop duplicate tool names (Vertex/Azure/Bedrock 400 on them) with a warning."""
- seen: set = set()
- deduped: list = []
- for tool in tools:
- name = (tool.get("function") or {}).get("name", "")
- if name and name in seen:
- logger.warning("_build_call_kwargs: duplicate tool name '%s' removed (provider=%s model=%s)", name, provider, model)
- continue
- if name:
- seen.add(name)
- deduped.append(tool)
- return deduped
- class _ProfileProjection(NamedTuple):
- body: dict[str, Any]
- reasoning_extra: dict[str, Any]
- top_level: dict[str, Any]
- handles_reasoning: bool
- messages_wire: bool = False
- def _routes_to_custom_endpoint(provider_norm: str) -> bool:
- """True when a profile-less provider name is really a configured OpenAI-compatible custom endpoint.
- Keyed ``providers:`` / ``custom_providers`` entries (by bare key, or ``main``/``auto`` resolving to
- one). An unconfigured name with only an explicit base_url keeps the generic nested fallback: the
- operator declared nothing about that endpoint's wire, and the fireworks control contract pins it.
- """
- name = _normalize_aux_provider(provider_norm)
- if name == "custom":
- return True
- from hermes_cli.runtime_provider import _get_named_custom_provider
- return _get_named_custom_provider(name) is not None
- def _project_provider_profile(
- provider: str, provider_norm: str, model: str, effective_base: str, reasoning_config: Optional[dict], data_use: frozenset[str],
- ) -> _ProfileProjection:
- """Provider profile's extra_body / kwargs projection; partial on failure."""
- body: dict[str, Any] = {}
- reasoning_extra: dict[str, Any] = {}
- top_level: dict[str, Any] = {}
- handles_reasoning = False
- messages_wire = False
- try:
- from providers import get_provider_profile
- from providers.base import ProviderProfile
- profile = get_provider_profile(provider_norm)
- if profile is None and _routes_to_custom_endpoint(provider_norm):
- # A keyed ``providers:`` entry referenced by its bare key (or via ``main``/``auto``) is
- # the same OpenAI-compatible custom endpoint the main path already projects with the
- # ``custom`` profile (``custom:<key>`` falls back inside get_provider_profile). Without
- # it the generic nested ``extra_body.reasoning`` fallback below ships to a strict
- # gateway that only accepts top-level ``reasoning_effort`` -> 400 (#75089).
- profile = get_provider_profile("custom")
- if profile is not None:
- messages_wire = profile.api_mode == "anthropic_messages"
- body = profile.build_extra_body(model=model, base_url=effective_base, reasoning_config=reasoning_config, data_use=data_use) or {}
- reasoning_extra, top_level = profile.build_api_kwargs_extras(
- reasoning_config=reasoning_config, supports_reasoning=reasoning_config is not None,
- model=model, base_url=effective_base,
- )
- reasoning_extra = reasoning_extra or {}
- top_level = top_level or {}
- handles_reasoning = (
- type(profile).build_api_kwargs_extras is not ProviderProfile.build_api_kwargs_extras
- or _contains_profile_reasoning_fields(body)
- or _contains_profile_reasoning_fields(reasoning_extra)
- or _contains_profile_reasoning_fields(top_level)
- )
- except Exception as exc:
- logger.debug("_build_call_kwargs: provider profile projection failed for %s: %s", provider, exc)
- return _ProfileProjection(body, reasoning_extra, top_level, handles_reasoning, messages_wire)
- def _merge_aux_extra_body(
- extra_body: Optional[dict], projection: _ProfileProjection, reasoning_config: Optional[dict], provider_norm: str,
- ) -> dict[str, Any]:
- """Caller extra_body + profile body/reasoning + generic reasoning fallback + Nous tags."""
- merged_extra = dict(extra_body or {})
- caller_reasoning_fields = {
- key: value for key, value in merged_extra.items()
- if str(key).strip().lower() in _PROFILE_REASONING_KEYS and str(key).strip().lower() != "reasoning"
- }
- caller_disabled = isinstance(reasoning_config, dict) and reasoning_config.get("enabled") is False
- if caller_disabled:
- # The caller's thinking-off beats ``auxiliary.<task>.reasoning_effort`` (folded into
- # ``extra_body.reasoning`` by _get_task_extra_body). Dropped BEFORE the profile merge so a
- # profile that projects disabled reasoning onto its own wire (custom: top-level
- # ``reasoning_effort=none``) never ships beside a task-level ``reasoning.effort`` — strict
- # gateways 400 on the contradiction (#114020) — while a profile whose disabled shape IS
- # ``extra_body.reasoning`` (OpenRouter) still lands it below.
- merged_extra.pop("reasoning", None)
- merged_extra.update(projection.body)
- merged_extra.update(projection.reasoning_extra)
- # Profiles supply route defaults, but an explicit vendor wire control in the task/call config
- # is already provider-specific and must not be replaced by that default.
- merged_extra.update(caller_reasoning_fields)
- if not projection.handles_reasoning:
- # ``reasoning_config`` is already clamped to the OpenAI-compat wire by _build_call_kwargs.
- from agent.reasoning_effort import generic_nested_reasoning
- merged_extra.update(generic_nested_reasoning(reasoning_config))
- # Caller/task ``extra_body.reasoning`` (``auxiliary.<task>.reasoning_effort`` folds in here via
- # _get_task_extra_body) takes the same wire clamp: Hermes-only ``ultra`` never reaches the
- # OpenAI-compat wire from any aux task (#112010).
- if isinstance(merged_extra.get("reasoning"), dict):
- from agent.reasoning_effort import clamp_reasoning_config
- merged_extra["reasoning"] = clamp_reasoning_config(merged_extra["reasoning"])
- # Portal tags + sticky session_id fallback when the profile didn't supply them; session_id
- # keeps aux calls on the main turn's upstream instance (cache warmth) — tags alone are not
- # enough on /v1/messages.
- if provider_norm in _NOUS_PROVIDER_NAMES:
- if "tags" not in merged_extra:
- merged_extra["tags"] = _nous_portal_tags()
- if "session_id" not in merged_extra:
- try:
- from agent.portal_tags import get_conversation_context
- sticky_key = get_conversation_context()
- except Exception:
- sticky_key = None
- if sticky_key:
- merged_extra["session_id"] = sticky_key
- return merged_extra
- def _build_call_kwargs(
- provider: str, model: str, messages: list, temperature: Optional[float] = None,
- max_tokens: Optional[int] = None, tools: Optional[list] = None, timeout: float = 30.0,
- extra_body: Optional[dict] = None, reasoning_config: Optional[dict] = None,
- base_url: Optional[str] = None, task: Optional[str] = None,
- no_progress_timeout: Optional[float] = None,
- ) -> dict:
- """Build kwargs for .chat.completions.create() with model/provider adjustments.
- ``no_progress_timeout`` is a Codex-Responses-only extra (consumed by
- ``_CodexCompletionsAdapter.create``'s ``**kwargs`` catch-all); callers must only pass it
- when the resolved client is a ``CodexAuxiliaryClient`` — real SDK clients don't accept it."""
- kwargs: dict[str, Any] = {"model": model, "messages": messages, "timeout": timeout}
- if no_progress_timeout is not None:
- kwargs["no_progress_timeout"] = no_progress_timeout
- effective_base = base_url or (_current_custom_base_url() if provider == "custom" else "")
- extra_body = _with_custom_endpoint_extra_body(extra_body, provider, model, effective_base)
- # Per-model fixed/omitted temperature, then Opus 4.7+ sampling bans: it rejects any
- # non-default temperature/top_p/top_k, so drop silently rather than 400 when the aux model flips.
- fixed_temperature = _fixed_temperature_for_model(model, effective_base, provider)
- if fixed_temperature is OMIT_TEMPERATURE:
- temperature = None # strip — let server choose
- elif fixed_temperature is not None:
- temperature = fixed_temperature
- if temperature is not None:
- from agent.anthropic_adapter import _forbids_sampling_params
- if not _forbids_sampling_params(model):
- kwargs["temperature"] = temperature
- provider_norm = str(provider or "").strip().lower()
- if max_tokens is not None and _forwards_max_tokens(provider, provider_norm, model, effective_base, task):
- kwargs.update(auxiliary_max_tokens_param(max_tokens, model=model)) # picks max_completion_tokens where needed
- if tools:
- kwargs["tools"] = _dedupe_tool_names(tools, provider, model)
- # Provider profiles are the source of truth for reasoning wire shapes (top-level, nested body,
- # or extra_body.reasoning); providers without a reasoning-aware profile keep the generic
- # ``extra_body.reasoning`` fallback. Clamp Hermes-internal levels (``ultra``) to the
- # OpenAI-compat wire ONCE here, before either path sees the config — the same entry clamp the
- # main transport applies (#89503); MoA aggregator/reference and aux calls 400'd without it (#112010).
- from agent.reasoning_effort import clamp_reasoning_config
- from agent.auxiliary_reasoning_floor import known_reasoning_floor
- if isinstance(extra_body, dict):
- task_reasoning = extra_body.get("reasoning")
- if isinstance(task_reasoning, dict) and "enabled" in task_reasoning:
- extra_body = dict(extra_body)
- extra_body.pop("reasoning")
- if reasoning_config is None:
- reasoning_config = task_reasoning
- reasoning_config = clamp_reasoning_config(known_reasoning_floor(reasoning_config, provider_norm, effective_base, model, task))
- projection = _project_provider_profile(provider, provider_norm, model, effective_base, reasoning_config, data_use_requirement(messages))
- kwargs.update(projection.top_level)
- merged_extra = _merge_aux_extra_body(extra_body, projection, reasoning_config, provider_norm)
- if "response_format" in merged_extra:
- from agent.auxiliary_structured_output import without_unsupported_response_format
- merged_extra = without_unsupported_response_format(merged_extra, provider_norm, effective_base, model, task)
- if merged_extra:
- kwargs["extra_body"] = merged_extra
- # Anthropic Messages adapters take reasoning via a private kwarg that plain OpenAI SDK clients
- # would reject; Portal Claude is dual-wire, so include it only when the catalog id selects
- # /v1/messages. A profile declaring api_mode=anthropic_messages (commandcode-anthropic) is on
- # that wire regardless of URL shape — once it overrides build_api_kwargs_extras the generic
- # ``extra_body.reasoning`` fallback the adapter used to read is gone, so this is the adapter's
- # only path. _wrap_transport wraps such providers on the same declaration.
- if reasoning_config and isinstance(reasoning_config, dict):
- raw_base = base_url or ""
- if (
- provider_norm == "anthropic" or projection.messages_wire or _nous_on_messages_wire(provider_norm, model)
- or _endpoint_speaks_anthropic_messages(raw_base) or _is_anthropic_compat_endpoint(provider_norm, raw_base)
- ):
- kwargs["_reasoning_config"] = dict(reasoning_config)
- # Conversation affinity (OpenCode relay, opt-in custom-provider header) — same key as the main
- # turn so compression/title/vision calls stay on the conversation's warm backend.
- from agent.opencode_affinity import merge_session_affinity_headers
- return merge_session_affinity_headers(kwargs, provider, base_url, _runtime_main_value("session_id") or None)
- def _validate_llm_response(
- response: Any, task: Optional[str] = None, provider: Optional[str] = None, base_url: Optional[str] = None,
- ) -> Any:
- """Validate the .choices[0].message shape (fail fast, not a downstream AttributeError).
- Also the single aux-usage accounting chokepoint: every successful non-streaming response
- passes here exactly once; *provider*/*base_url* are optional hints. Async callers
- await this off-loop because accounting may commit through synchronous worker RPC.
- See #7264.
- Recording is best-effort and never affects validation. *provider*/*base_url* are optional accounting
- hints — fallback-path calls omit them and the row keeps the model (read from the response itself) with
- an empty route. See #23270.
- """
- if response is None:
- raise RuntimeError(
- f"Auxiliary {task or 'call'}: LLM returned None response"
- )
- response = _unwrap_data_envelope(response, task)
- from agent.aux_accounting import record_aux_usage
- record_aux_usage(response, task, provider=provider, base_url=base_url)
- # Adapter SimpleNamespace responses are fine — they have .choices[0].message.
- try:
- choices = response.choices
- if not choices or not hasattr(choices[0], "message"):
- raise AttributeError("missing choices[0].message")
- except (AttributeError, TypeError, IndexError) as exc:
- recovered = _recover_aux_response_message(response)
- if recovered is None:
- invalid = RuntimeError(
- f"Auxiliary {task or 'call'}: LLM returned invalid response (type={type(response).__name__}): "
- f"{str(response)[:120]!r}. Expected object with .choices[0].message — check provider "
- f"adapter or custom endpoint compatibility."
- )
- # An HTTP-200 body carrying a provider ``error`` (aggregators relay upstream failures
- # this way): keep it for the classifier without changing how the ladder routes this.
- from agent import auxiliary_call_outcome
- setattr(invalid, auxiliary_call_outcome.PROVIDER_ERROR_ATTR,
- auxiliary_call_outcome.embedded_provider_error(response))
- _note_relay_auxiliary_error(invalid)
- raise invalid from exc
- response = recovered
- from agent.transports.chat_completions import is_router_timeout_shim
- if is_router_timeout_shim(response):
- # HTTP-200 router failure shim (#68396): invalid like a malformed shape so the
- # auxiliary fallback chain moves to the next candidate instead of titling with it.
- shim = RuntimeError(f"Auxiliary {task or 'call'}: provider returned a timeout shim instead of a completion")
- _note_relay_auxiliary_error(shim)
- raise shim
- # Retain the provider-reported model for terminal relay route attribution.
- context = _RELAY_AUX_CALL_CONTEXT.get()
- if context is not None:
- model = _field(response, "model")
- if isinstance(model, str) and model.strip():
- context["response_model"] = model
- _complete_relay_auxiliary_call()
- return response
- def _complete_relay_auxiliary_call(*, outcome: str = "success", error_class: Optional[str] = None) -> None:
- """Close one auxiliary logical call after acceptance or terminal failure. A success
- reports the last attempt error it recovered from (``none`` when the first attempt held)."""
- context = _RELAY_AUX_CALL_CONTEXT.get()
- if context is None:
- return
- from agent import relay_llm
- relay_llm.complete_logical_call(
- str(context.get("request_id") or ""), outcome=outcome,
- model_name=str(context.get("model") or "unknown"),
- provider_name=str(context.get("provider") or "auxiliary"),
- response_model_name=context.get("response_model"),
- error_class=error_class or context.get("error_class") or "none",
- )
- def _note_relay_auxiliary_error(exc: BaseException) -> None:
- """Remember one failed attempt's classified reason on the logical call."""
- context = _RELAY_AUX_CALL_CONTEXT.get()
- if context is not None and isinstance(exc, Exception):
- from agent import auxiliary_call_outcome
- context["error_class"] = auxiliary_call_outcome.error_class(
- exc, provider=str(context.get("provider") or ""), model=str(context.get("model") or ""))
- def _fail_relay_auxiliary_call(exc: BaseException) -> None:
- """Close a call that ended in ``exc`` without replacing it: a Hermes abort is ``cancelled``,
- anything else ``failed`` with the classifier's reason for the error that ended it."""
- try:
- from agent import auxiliary_call_outcome
- if auxiliary_call_outcome.is_cancellation(exc):
- _complete_relay_auxiliary_call(outcome="cancelled", error_class="none")
- return
- _note_relay_auxiliary_error(exc)
- _complete_relay_auxiliary_call(outcome="failed")
- except Exception:
- logger.warning("Relay auxiliary failure finalization failed", exc_info=True)
- def _unwrap_data_envelope(response: Any, task: str | None = None) -> Any:
- """Unwrap gateway envelopes like {"data": {<chat completion>}, "success": true}.
- Some OpenAI-compatible gateways (e.g. api.cline.bot) wrap non-streaming
- JSON bodies in a data/success envelope. The SDK leniently parses this
- into a ChatCompletion with choices=None and keeps the envelope keys as
- extra fields, so the real completion is reachable at ``response.data``.
- Error envelopes ({"success": false, "data": {"error": ...}}) raise the
- provider's actual error instead of a generic invalid-response error.
- """
- if _field(response, "choices"):
- return response
- data = _field(response, "data")
- if not isinstance(data, dict):
- return response
- if _field(response, "success") is False:
- err = data.get("error") or data
- msg = err.get("message") if isinstance(err, dict) else None
- raise RuntimeError(
- f"Auxiliary {task or 'call'}: provider returned error envelope: "
- f"{msg or err}"
- )
- if not data.get("choices"):
- return response
- try:
- return type(response).model_validate(data)
- except (AttributeError, TypeError, ValueError):
- # No model_validate (SimpleNamespace) or the inner payload fails
- # strict validation (pydantic ValidationError is a ValueError).
- pass
- choices = []
- for ch in data["choices"]:
- msg = ch.get("message") if isinstance(ch, dict) else None
- if isinstance(msg, dict):
- choices.append(SimpleNamespace(
- message=SimpleNamespace(**msg),
- finish_reason=ch.get("finish_reason") or "stop",
- ))
- if not choices:
- return response
- return SimpleNamespace(
- id=data.get("id", ""),
- model=data.get("model", ""),
- object=data.get("object", "chat.completion"),
- choices=choices,
- usage=data.get("usage"),
- )
- def _recover_aux_response_message(response: Any) -> Optional[Any]:
- """Synthesize chat-completions shape from Responses-style text (``output_text``,
- ``output`` items) that some compatible endpoints return outside ``choices``."""
- text = _extract_aux_response_text(response)
- if not text:
- return None
- choice = SimpleNamespace(message=SimpleNamespace(content=text), finish_reason=getattr(response, "finish_reason", None) or "stop")
- try:
- response.choices = [choice]
- return response
- except Exception:
- return SimpleNamespace(
- id=getattr(response, "id", ""), model=getattr(response, "model", ""),
- object=getattr(response, "object", "chat.completion"), choices=[choice],
- usage=getattr(response, "usage", None),
- )
- def _extract_aux_response_text(response: Any) -> str:
- """Text from Responses-style ``output_text`` or ``output[].content[].text``."""
- output_text = _field(response, "output_text")
- if isinstance(output_text, str) and output_text.strip():
- return output_text.strip()
- output = _field(response, "output")
- if not isinstance(output, list):
- return ""
- parts: list[str] = []
- for item in output:
- item_type = _field(item, "type")
- if item_type and item_type != "message":
- continue
- for part in (_field(item, "content") or []):
- if _field(part, "type") in {"output_text", "text", None}:
- text = _field(part, "text")
- if isinstance(text, str) and text.strip():
- parts.append(text.strip())
- return "\n".join(parts).strip()
- # Streamed aggregation for progress-hooked aux calls: ``timeout`` becomes an inter-chunk idle
- # timeout (httpx read timeout is per read), each chunk ticks outer watchdogs; the total ceiling
- # bounds trickles.
- _AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0
- _AUX_STREAM_CEILING_MULTIPLIER = 4.0
- def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float:
- """Absolute wall-clock bound for a streamed aux call; generous by design (the idle
- timeout is the real guard — this only stops a one-token-per-idle-window trickle)."""
- try:
- timeout = float(effective_timeout) if effective_timeout is not None else 0.0
- except (TypeError, ValueError):
- timeout = 0.0
- return max(_AUX_STREAM_CEILING_FLOOR_SECONDS, _AUX_STREAM_CEILING_MULTIPLIER * timeout)
- def _client_streams_internally(client: Any) -> bool:
- """Adapters that stream inside .create() tick the hook themselves (Codex, Anthropic) or
- cannot stream (Bedrock); none accept ``stream=True`` from us."""
- return isinstance(client, (CodexAuxiliaryClient, AnthropicAuxiliaryClient, BedrockAuxiliaryClient))
- def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool:
- """Providers that only accept streaming (non-stream = 400): Tencent Copilot, any
- ``auxiliary.stream_only_base_urls`` substring, and the managed local llama-server
- (streamed for cancellation — it only notices a dead client on socket write)."""
- _url = str(base_url or "").lower()
- if not _url:
- return False
- if base_url_host_matches(_url, "copilot.tencent.com") or _is_managed_local_endpoint(_url):
- return True
- try:
- from hermes_cli.config import load_config
- markers = (load_config() or {}).get("auxiliary", {}).get("stream_only_base_urls") or []
- if isinstance(markers, (list, tuple)):
- return any(
- isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url
- for marker in markers)
- except Exception:
- pass # Config read is best-effort; never break an aux call over it.
- return False
- _AFFORDABLE_TOKENS_RE = re.compile(r"can only afford\s+([0-9][0-9,]*)", re.IGNORECASE)
- # Below the floor the affordable budget can't fit a useful aux output — treat as exhaustion;
- # the margin keeps provider-side token-count rounding from 402-ing the retry.
- _AFFORDABLE_RETRY_FLOOR_TOKENS = 512
- # See #49785.
- _AFFORDABLE_RETRY_MARGIN_TOKENS = 64
- def _affordable_max_tokens_from_error(exc: Exception) -> Optional[int]:
- """Affordable output budget (minus margin) from an OpenRouter credit-limited 402
- ("...but can only afford 7117": credit exists, the cap was too large); ``None``
- when no count is present or the budget is too small to be useful."""
- if not _is_payment_error(exc):
- return None
- match = _AFFORDABLE_TOKENS_RE.search(str(exc))
- if not match:
- return None
- try:
- affordable = int(match.group(1).replace(",", ""))
- except (TypeError, ValueError):
- return None
- capped = affordable - _AFFORDABLE_RETRY_MARGIN_TOKENS
- return capped if capped >= _AFFORDABLE_RETRY_FLOOR_TOKENS else None
- def _create_with_progress(
- client: Any, kwargs: dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
- ) -> Any:
- """Credit-aware :func:`_create_with_progress_once`: a 402 naming an affordable
- budget retries ONCE with that cap (only ever lowering); anything else re-raises."""
- try:
- return _create_with_progress_once(client, kwargs, task, force_stream=force_stream)
- except Exception as exc:
- affordable = _affordable_max_tokens_from_error(exc)
- if affordable is None:
- raise
- existing_cap = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
- if isinstance(existing_cap, (int, float)) and 0 < existing_cap <= affordable:
- raise # Already within budget — the error is something else; don't spin.
- retry_kwargs = dict(kwargs)
- retry_kwargs.pop("max_tokens", None)
- retry_kwargs.pop("max_completion_tokens", None)
- retry_kwargs.update(
- auxiliary_max_tokens_param(affordable, model=str(kwargs.get("model") or "") or None))
- logger.info("Auxiliary %s: credit-limited 402 (affordable=%d tokens); "
- "retrying once with a clamped output cap instead of failing: %s",
- task or "call", affordable, exc)
- return _create_with_progress_once(client, retry_kwargs, task, force_stream=force_stream)
- def _stream_request_plan(kwargs: dict[str, Any], client: Any = None) -> tuple[dict[str, Any], str, float]:
- """(stream kwargs, model name, total ceiling) for a streamed re-aggregation. Requests to the
- managed llama-server also ask for prompt-reading progress, which keeps the watchdogs fed
- through a long prefill."""
- stream_kwargs = dict(kwargs)
- stream_kwargs["stream"] = True
- stream_kwargs["stream_options"] = {"include_usage": True}
- if _is_managed_local_endpoint(str(getattr(client, "base_url", "") or "")):
- stream_kwargs = with_prompt_progress(stream_kwargs)
- return (stream_kwargs, str(kwargs.get("model") or ""),
- _aux_stream_total_ceiling(kwargs.get("timeout")))
- def _create_with_progress_once(
- client: Any, kwargs: dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
- ) -> Any:
- """create() that streams (and re-aggregates, ticking the hook per substantive chunk) when a
- progress hook is active or the provider is stream-only; plain ``create(**kwargs)`` otherwise
- or when the adapter streams internally. Streaming rejections fall back to a plain call —
- except under ``force_stream``.
- Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when neither trigger applies (every
- existing caller/task) or when the client's wire adapter streams internally. With a hook + a
- chunk-capable client, the request is sent with ``stream=True`` and aggregated, ticking the hook only for
- substantive chunks. The configured ``timeout`` acts per stream read (idle) rather than as a total
- budget, and outer liveness watchdogs see tokens moving. ``force_stream=True`` (stream-only providers
- such as Tencent Copilot — credit @kudi88, PR #60686) takes the same streamed path even without a hook.
- Providers that reject the streamed request fall back to the plain non-streaming call — except under
- ``force_stream``, where a stream-only provider rejects the plain call by definition, so the original
- error is surfaced to the normal recovery chains instead.
- """
- kwargs = bypass_chat_sdk_request_transform(kwargs, client)
- _notify_aux_dispatch()
- # Dispatch alone is not forward progress: a 401/retry/fallback dispatch must not
- # reset the compression inactivity fence, or a zero-output attempt runs to the
- # total ceiling instead of idling out (#114938). Progress ticks only for
- # substantive stream payloads or a completed usable response.
- if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client):
- response = client.chat.completions.create(**kwargs)
- if not _client_streams_internally(client):
- _notify_aux_provider_response()
- return response
- stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs, client)
- try:
- chunks = client.chat.completions.create(**stream_kwargs)
- except Exception as exc:
- # Genuine provider failures aren't streaming's fault — surface unchanged so the
- # recovery chains see the same error as a plain call.
- if (force_stream or _is_transient_transport_error(exc) or _is_auth_error(exc)
- or _is_payment_error(exc) or _is_rate_limit_error(exc)):
- raise
- # Possibly a streaming-specific rejection: retry non-streaming once; a genuinely bad
- # request reproduces the real error for the except-chains.
- logger.debug("Auxiliary %s: streamed request failed (%s); retrying non-streaming",
- task or "call", exc)
- _notify_aux_dispatch()
- response = client.chat.completions.create(**kwargs)
- _notify_aux_provider_response()
- return response
- # Some shims (MoA quiet mode, defensive adapters) return a complete response despite
- # stream=True; it counts as provider response + forward progress.
- if hasattr(chunks, "choices"):
- _notify_aux_provider_response()
- return chunks
- from agent.auxiliary_stream_watchdog import chat_stream_windows
- return _aggregate_chat_stream(
- chunks, model=model, total_ceiling=total_ceiling, no_progress=chat_stream_windows(client, kwargs, task))
- def _aggregate_chat_stream(
- chunks: Any, *, model: str = "", total_ceiling: Optional[float] = None,
- no_progress: Optional[tuple[float, Optional[float]]] = None,
- ) -> Any:
- """Consume a chunk stream into a complete response; TimeoutError (phrased "timed out" so
- ``_is_timeout_error`` matches) past *total_ceiling* or the *no_progress* windows (#100501)."""
- from agent.auxiliary_stream_watchdog import _close_chunk_stream, consume_chat_stream
- acc = _ChatStreamAccumulator(
- model=model, total_ceiling=total_ceiling, host_deadline=_current_aux_stream_deadline)
- try:
- consume_chat_stream(chunks, acc, no_progress)
- finally:
- _close_chunk_stream(chunks)
- return acc.finish()
- # Reasoning-detail fields whose non-empty text counts as forward progress.
- _REASONING_DETAIL_TEXT_FIELDS = ("summary", "thinking", "content", "text")
- class _ChatStreamAccumulator:
- """Shared per-chunk accumulation so sync and async aggregation cannot drift."""
- def __init__(self, model: str = "", total_ceiling: Optional[float] = None,
- host_deadline: float | Callable[[], float | None] | None = None):
- self._started = time.monotonic()
- # llama-server prompt reading doesn't count toward the ceiling (``local_prompt_progress``).
- self._prompt_reading = PromptReadingClock(self._started)
- self._total_ceiling = total_ceiling
- # Absolute instant the waiting host gives up, checked as well as (not instead of) the ceiling, which
- # still bounds callers with no host deadline; unaffected by pre-construction dispatch/TTFT (#99692).
- self._host_deadline = host_deadline
- self.content_parts: list[str] = []
- self.reasoning_parts: list[str] = []
- self.reasoning_details: list[Any] = []
- self.tool_calls_acc: dict[int, dict[str, Any]] = {}
- self.finish_reason = self.usage = None
- self.resp_id = ""
- self.resp_model = model or ""
- def _check_deadlines(self) -> None:
- """Raise TimeoutError past the total ceiling or the host deadline; a host hard cancel (/stop, session
- boundary) stops the stream at its next chunk instead of letting the daemon bill on to the deadline."""
- if _aux_interrupt_cancel_requested():
- raise AuxiliaryExplicitCancellation()
- now = time.monotonic()
- if self._total_ceiling is not None and (now - self._started - self._prompt_reading.seconds) >= self._total_ceiling:
- raise TimeoutError(f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s "
- "total ceiling (stream still open but over budget)")
- host_deadline = self._host_deadline() if callable(self._host_deadline) else self._host_deadline
- if host_deadline is not None and now >= host_deadline:
- raise TimeoutError("Auxiliary streamed call timed out at the host compression "
- f"deadline after {time.monotonic() - self._started:.0f}s "
- "(the caller already stopped waiting; streaming on would only "
- "pin its session lease)")
- def _feed_reasoning_details(self, delta: Any) -> bool:
- """Collect ``reasoning_details`` (OpenRouter-style thinking); True only when a detail
- carries text, so structural/signed envelopes can't keep a stall alive."""
- reasoning_details = getattr(delta, "reasoning_details", None)
- if reasoning_details is None:
- model_extra = getattr(delta, "model_extra", None)
- if isinstance(model_extra, dict):
- reasoning_details = model_extra.get("reasoning_details")
- if not isinstance(reasoning_details, list):
- return False
- made_progress = False
- for detail in reasoning_details:
- self.reasoning_details.append(detail)
- if isinstance(detail, dict) and any(
- isinstance(detail.get(f), str) and detail[f] for f in _REASONING_DETAIL_TEXT_FIELDS):
- made_progress = True
- return made_progress
- def _feed_tool_calls(self, delta: Any) -> bool:
- """Merge tool-call fragments by index; True when any fragment carried data."""
- made_progress = False
- for tc in (getattr(delta, "tool_calls", None) or []):
- idx = getattr(tc, "index", 0) or 0
- acc = self.tool_calls_acc.setdefault(idx, {"id": "", "name": "", "arguments": []})
- if getattr(tc, "id", None):
- acc["id"] = tc.id
- made_progress = True
- fn = getattr(tc, "function", None)
- if fn is not None:
- if getattr(fn, "name", None):
- acc["name"] = fn.name
- made_progress = True
- if getattr(fn, "arguments", None):
- acc["arguments"].append(fn.arguments)
- made_progress = True
- return made_progress
- def feed(self, chunk: Any) -> bool:
- # Every frame records transport timing (TTFP); only a substantive payload ticks the
- # forward-progress hook that keeps compression alive (and is reported as True).
- _notify_aux_timing_response()
- progress = prompt_progress(chunk)
- # Count the reading before the deadline check it extends.
- read_more = progress is not None and self._prompt_reading.tick(progress)
- self._check_deadlines()
- self.resp_id = getattr(chunk, "id", None) or self.resp_id
- self.resp_model = getattr(chunk, "model", None) or self.resp_model
- chunk_usage = getattr(chunk, "usage", None)
- if chunk_usage:
- self.usage = chunk_usage
- if progress is not None:
- # llama-server is still reading the prompt: new tokens read keep the compression fence
- # alive. It is no first token, so the stream watchdog keeps its pre-token patience: a
- # busy server reports a batch every minute or more.
- if read_more:
- _notify_aux_progress()
- return False
- choices = getattr(chunk, "choices", None) or []
- if not choices:
- return False
- choice = choices[0]
- self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason
- delta = getattr(choice, "delta", None)
- if delta is None:
- return False
- made_progress = False
- from agent.message_content import flatten_message_text
- piece = flatten_message_text(getattr(delta, "content", None), sep="")
- if piece:
- self.content_parts.append(piece)
- made_progress = True
- reasoning_piece = getattr(delta, "reasoning", None) or getattr(delta, "reasoning_content", None)
- if reasoning_piece is None and isinstance(getattr(delta, "model_extra", None), dict):
- reasoning_piece = delta.model_extra.get("reasoning") or delta.model_extra.get("reasoning_content")
- reasoning_piece = flatten_message_text(reasoning_piece, sep="")
- if reasoning_piece:
- self.reasoning_parts.append(reasoning_piece)
- made_progress = True
- # Evaluate both unconditionally: they accumulate state, not just progress.
- made_progress |= self._feed_reasoning_details(delta)
- made_progress |= self._feed_tool_calls(delta)
- if made_progress:
- _notify_aux_progress()
- return made_progress
- def finish(self) -> Any:
- tool_calls = None
- if self.tool_calls_acc:
- tool_calls = [
- SimpleNamespace(id=acc["id"], type="function", function=SimpleNamespace(
- name=acc["name"], arguments="".join(acc["arguments"])))
- for _idx, acc in sorted(self.tool_calls_acc.items())]
- message = SimpleNamespace(
- role="assistant", content="".join(self.content_parts), tool_calls=tool_calls,
- reasoning="".join(self.reasoning_parts) or None,
- reasoning_details=self.reasoning_details or None,
- )
- choice = SimpleNamespace(index=0, message=message, finish_reason=self.finish_reason or "stop")
- return SimpleNamespace(id=self.resp_id, model=self.resp_model, object="chat.completion",
- choices=[choice], usage=self.usage)
- async def _aggregate_chat_stream_async(
- chunks: Any, *, model: str = "", total_ceiling: Optional[float] = None
- ) -> Any:
- """Async mirror of :func:`_aggregate_chat_stream` (AsyncOpenAI streams need ``async for``)."""
- acc = _ChatStreamAccumulator(
- model=model, total_ceiling=total_ceiling, host_deadline=_current_aux_stream_deadline)
- try:
- async for chunk in chunks:
- acc.feed(chunk)
- finally:
- from agent.auxiliary_stream_watchdog import _close_chunk_stream
- pending = _close_chunk_stream(chunks, allow_aclose=True)
- if pending is not None:
- with contextlib.suppress(Exception):
- await pending
- return acc.finish()
- async def _acreate_with_stream(client: Any, kwargs: dict[str, Any], task: Optional[str] = None) -> Any:
- """Async create() for stream-only providers: ``stream=True`` + aggregate the async chunks."""
- stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs, client)
- chunks = await client.chat.completions.create(**stream_kwargs)
- if hasattr(chunks, "choices"): # shims may hand back a complete response despite stream=True
- return chunks
- return await _aggregate_chat_stream_async(chunks, model=model, total_ceiling=total_ceiling)
- def _async_client_streams_internally(client: Any) -> bool:
- """Async twin of :func:`_client_streams_internally` (the async adapters are separate classes)."""
- return isinstance(client, (AsyncCodexAuxiliaryClient, AsyncAnthropicAuxiliaryClient, AsyncBedrockAuxiliaryClient))
- async def _acreate_with_progress(
- client: Any, kwargs: dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
- ) -> Any:
- """Async :func:`_create_with_progress`: stream + re-aggregate (ticking the hook per substantive
- chunk) when a progress hook is active or the provider is stream-only; plain create otherwise."""
- kwargs = bypass_chat_sdk_request_transform(kwargs, client)
- _notify_aux_dispatch()
- # Same contract as the sync twin (#114938): dispatch alone is not progress.
- if (not _aux_progress_active() and not force_stream) or _async_client_streams_internally(client):
- response = await client.chat.completions.create(**kwargs)
- if not _async_client_streams_internally(client):
- _notify_aux_provider_response()
- return response
- stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs, client)
- try:
- chunks = await client.chat.completions.create(**stream_kwargs)
- except Exception as exc:
- # Only a rejected stream NEGOTIATION falls back to a plain call (mirrors the sync wrapper); a
- # failure mid-consumption below reaches the classified recovery ladder instead of silently
- # re-sending the whole prompt non-streaming.
- if (force_stream or _is_transient_transport_error(exc) or _is_auth_error(exc)
- or _is_payment_error(exc) or _is_rate_limit_error(exc)):
- raise
- logger.debug("Auxiliary %s: streamed async request failed (%s); retrying non-streaming",
- task or "call", exc)
- _notify_aux_dispatch()
- response = await client.chat.completions.create(**kwargs)
- _notify_aux_provider_response()
- return response
- if hasattr(chunks, "choices"): # shims may hand back a complete response despite stream=True
- _notify_aux_provider_response()
- return chunks
- return await _aggregate_chat_stream_async(chunks, model=model, total_ceiling=total_ceiling)
- # Shared request head + recovery ladder for call_llm / async_call_llm: the entry points differ
- # only in how a request is awaited, so route resolution and the ordered recovery ladder are
- # written once. The ladder is a generator yielding ``_LadderStep`` requests and receiving the
- # response (or thrown exception), so rung ORDER and accept/re-raise contracts match on both wires.
- class _ResolvedAuxRoute(NamedTuple):
- client: Any
- final_model: Optional[str]
- resolved_provider: str
- effective_provider: str
- def _resolve_call_client(
- task: Optional[str], *, provider: Optional[str], model: Optional[str], base_url: Optional[str],
- api_key: Optional[str], resolved_provider: str, resolved_model: Optional[str],
- resolved_base_url: Optional[str], resolved_api_key: Optional[str],
- resolved_api_mode: Optional[str], main_runtime: Optional[dict[str, Any]], async_mode: bool,
- ) -> _ResolvedAuxRoute:
- """Resolve the client for one aux call: vision chain, or cached text client with the
- explicit-provider fallback_chain / auto-chain rescue; RuntimeError when nothing is configured."""
- effective_provider = resolved_provider
- if task == "vision":
- effective_provider, client, final_model = resolve_vision_provider_client(
- provider=resolved_provider if resolved_provider != "auto" else provider,
- model=resolved_model or model, base_url=resolved_base_url or base_url,
- api_key=resolved_api_key or api_key, async_mode=async_mode, main_runtime=main_runtime,
- )
- if client is None and resolved_provider != "auto" and not resolved_base_url:
- logger.warning("Vision provider %s unavailable, falling back to auto vision backends",
- resolved_provider)
- effective_provider, client, final_model = resolve_vision_provider_client(
- provider="auto", model=resolved_model, async_mode=async_mode,
- main_runtime=main_runtime)
- if client is not None:
- resolved_provider = effective_provider or resolved_provider
- else:
- client, final_model = _get_cached_client(
- resolved_provider, resolved_model, async_mode=async_mode, base_url=resolved_base_url,
- api_key=resolved_api_key, api_mode=resolved_api_mode, main_runtime=main_runtime,
- task=task)
- effective_provider = _effective_provider_for_client(client, resolved_provider)
- if client is None:
- # Explicit provider with no credentials: honor the task fallback_chain before
- # raising (fallback entries may use OAuth / credential-pool auth).
- _explicit = (resolved_provider or "").strip().lower()
- if _explicit and _explicit not in {"auto", "openrouter", "custom"}:
- fb_client, fb_model, fb_label = _try_configured_fallback_for_unavailable_client(
- task, _explicit)
- if fb_client is None:
- nous_detail = nous_credential_failure_detail() if _explicit == "nous" else None
- raise AuxiliaryClientUnavailable(
- nous_detail or missing_provider_credentials_message(_explicit))
- client, final_model = fb_client, fb_model
- if async_mode:
- client, final_model = _to_async_client(
- fb_client, fb_model or "", is_vision=(task == "vision"))
- resolved_provider = fb_label or resolved_provider
- effective_provider = resolved_provider
- # Auto/custom with no credentials: walk the full auto chain (not just OpenRouter).
- # model=None so each provider uses its own default.
- if client is None and not resolved_base_url:
- logger.info("Auxiliary %s: provider %s unavailable, trying auto-detection chain",
- task or "call", resolved_provider)
- client, final_model = _get_cached_client(
- "auto", async_mode=async_mode, main_runtime=main_runtime, task=task)
- effective_provider = _effective_provider_for_client(client, "auto")
- if client is None:
- raise AuxiliaryClientUnavailable(f"No LLM provider configured for task={task} "
- f"provider={resolved_provider}. Run: hermes setup")
- return _ResolvedAuxRoute(client, final_model, resolved_provider, effective_provider)
- class _PreparedAuxRequest(NamedTuple):
- client: Any
- final_model: Optional[str]
- kwargs: dict[str, Any]
- resolved_provider: str
- request_provider: str
- resolved_model: Optional[str]
- resolved_base_url: Optional[str]
- resolved_api_key: Optional[str]
- resolved_api_mode: Optional[str]
- effective_timeout: float
- effective_extra_body: dict[str, Any]
- base_info: str
- def _prepare_aux_request(
- task: Optional[str], *, provider: Optional[str], model: Optional[str], base_url: Optional[str],
- api_key: Optional[str], main_runtime: dict[str, Any], messages: list,
- temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
- timeout: Optional[float], extra_body: Optional[dict], reasoning_config: Optional[dict],
- extra_headers: Optional[dict[str, str]], api_mode: Optional[str],
- route_info: Optional[dict[str, str]], async_mode: bool,
- ) -> _PreparedAuxRequest:
- """Shared head of call_llm/async_call_llm: resolve route + client, publish it, build request kwargs.
- Sync-only: compression fast lane, per-request ``extra_headers``, and ``base_info`` falling
- back to the resolved base_url when the client exposes none."""
- resolved_provider, resolved_model, resolved_base_url, resolved_api_key, resolved_api_mode = _resolve_task_provider_model(
- task, provider, model, base_url, api_key)
- if api_mode:
- resolved_api_mode = api_mode
- effective_extra_body = _get_task_extra_body(task)
- effective_extra_body.update(extra_body or {})
- client, final_model, resolved_provider, effective_provider = _resolve_call_client(
- task, provider=provider, model=model, base_url=base_url, api_key=api_key,
- resolved_provider=resolved_provider, resolved_model=resolved_model,
- resolved_base_url=resolved_base_url, resolved_api_key=resolved_api_key,
- resolved_api_mode=resolved_api_mode, main_runtime=main_runtime, async_mode=async_mode,
- )
- effective_timeout = _effective_aux_timeout(task, timeout)
- # Codex-Responses-only: real SDK clients reject an unrecognized ``no_progress_timeout``
- # kwarg, so only resolve/forward it when the route is actually a Codex stream (#108104).
- no_progress_timeout = (
- _get_task_no_progress_timeout(task)
- if isinstance(client, (CodexAuxiliaryClient, AsyncCodexAuxiliaryClient)) else None
- )
- request_provider = effective_provider or resolved_provider
- if not async_mode:
- compression_config = _get_auxiliary_task_config(task) if task == "compression" else {} # authoritative: no aux route
- _, effective_extra_body = _compression_fast_lane_controls(
- task, actual_provider=request_provider, actual_model=final_model,
- requested_provider=provider, requested_model=model, route_config=compression_config,
- leak_guard_config=compression_config, max_tokens=max_tokens,
- extra_body=effective_extra_body,
- )
- _set_relay_auxiliary_route(request_provider, final_model, resolved_api_mode)
- _record_route_info(route_info, _fallback_provider_from_label(request_provider), final_model)
- if async_mode:
- base_info = str(getattr(client, "base_url", "") or "")
- else:
- base_info = str(getattr(client, "base_url", resolved_base_url) or "")
- if task:
- logger.info("Auxiliary %s: using %s (%s)%s",
- task, request_provider or "auto", final_model or "default",
- f" at {base_info}" if base_info and "openrouter" not in base_info else "")
- # Client's actual base_url so endpoint-specific temperature overrides work on
- # auto-detected routes (api.moonshot.ai vs api.kimi.com/coding).
- kwargs = _build_call_kwargs(
- request_provider, final_model, messages, temperature=temperature, max_tokens=max_tokens,
- tools=tools, timeout=effective_timeout, extra_body=effective_extra_body,
- reasoning_config=reasoning_config, base_url=base_info or resolved_base_url, task=task,
- no_progress_timeout=no_progress_timeout)
- if extra_headers:
- kwargs["extra_headers"] = dict(extra_headers)
- # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)
- client_base = str(getattr(client, "base_url", "") or "")
- if _is_anthropic_compat_endpoint(request_provider, client_base):
- kwargs["messages"] = _convert_openai_images_to_anthropic(kwargs["messages"])
- return _PreparedAuxRequest(
- client, final_model, kwargs, resolved_provider, request_provider, resolved_model,
- resolved_base_url, resolved_api_key, resolved_api_mode, effective_timeout,
- effective_extra_body, base_info)
- class _LadderStep(NamedTuple):
- """A provider request the ladder asks its driver to perform. kind: "call" (client, kwargs) |
- "retry_same_provider" (provider, model) | "fallback" (fb_client, fb_model, fb_label)."""
- kind: str
- args: tuple
- # Ordered (predicate, reason) pairs for the provider-fallback rung: first match
- # wins, so a payment-flavoured 429 reads as "payment error", not "rate limit".
- _FALLBACK_REASONS: tuple[tuple[Callable[[Exception], bool], str], ...] = (
- (_is_auth_error, "auth error"), (_is_payment_error, "payment error"),
- (_is_rate_limit_error, "rate limit"), (_is_model_incompatible_error, "model incompatible with route"),
- (_is_invalid_aux_response_error, "invalid provider response"),
- # A status-less in-stream ``error`` event (SSE committed 200) is a route failure (#101538).
- (_is_statusless_structured_provider_error, "structured provider error"),
- # Before the connection-error rung (its superset): a full-budget timeout must be named as one, or
- # a slow local model reads as an unreachable endpoint (#89445).
- (_is_timeout_error, "request timed out"), (_is_connection_error, "connection error"),
- )
- def _rung(step: _LadderStep, accept: Callable[[Exception], bool]):
- """One ladder rung: perform ``step``; yields ``(response, None)`` on success,
- ``(None, exc)`` when ``accept(exc)`` lets the next rung handle it, else re-raises."""
- try:
- result = yield step
- except Exception as exc:
- if not accept(exc):
- raise
- return None, exc
- return result, None
- def _param_rung_accepts(exc: Exception) -> bool:
- """After a parameter-strip retry: fall through to the max_tokens/payment/auth
- chains with the stripped kwargs; re-raise anything those chains won't handle.
- A 429 on the retry is the credential/provider-fallback rungs' job, so it falls
- through too (the pre-ladder max_tokens rung accepted rate limits)."""
- return (_is_payment_error(exc) or _is_connection_error(exc) or _is_auth_error(exc)
- or _is_rate_limit_error(exc) or _is_statusless_structured_provider_error(exc)
- or "max_tokens" in str(exc) or "unsupported_parameter" in str(exc)
- # Parameter rungs chain in any order (a reasoning-strip retry can 400 on temperature,
- # a temperature-strip retry on max_tokens), and a route-gating 400 after a strip still
- # reaches the provider-fallback rung.
- or _is_unsupported_parameter_error(exc, "temperature")
- or _is_reasoning_field_rejection(exc) or _is_reasoning_required_rejection(exc)
- or _is_structured_output_rejection(exc)
- or _is_model_incompatible_error(exc))
- def _credential_rung_accepts(exc: Exception) -> bool:
- return _is_auth_error(exc) or _is_payment_error(exc) or _is_rate_limit_error(exc)
- # Immutable route context shared by the recovery rungs.
- class _LadderRoute(NamedTuple):
- client: Any
- task: Optional[str]
- tag: str
- async_mode: bool
- base_info: str
- resolved_provider: str
- resolved_model: Optional[str]
- resolved_base_url: Optional[str]
- resolved_api_key: Optional[str]
- resolved_api_mode: Optional[str]
- final_model: Optional[str]
- main_runtime: Optional[dict[str, Any]]
- route_info: Optional[dict[str, str]]
- timeout: Optional[float]
- def _without_temperature(kwargs: dict) -> Optional[dict]:
- """Copy *kwargs* without ``temperature``; None when it was not sent."""
- return {k: v for k, v in kwargs.items() if k != "temperature"} if "temperature" in kwargs else None
- def _without_max_tokens(kwargs: dict) -> dict:
- """Copy *kwargs* without either output cap. Unlike the other strips this never returns None: a
- route can translate the caller's cap into a field the wire kwargs no longer show (Codex
- Responses), and the provider's own gateway may inject one — the 400 still names ``max_tokens``
- and the identical request completes on retry (registry class #89897/#90257)."""
- return {k: v for k, v in kwargs.items() if k not in ("max_tokens", "max_completion_tokens")}
- def _is_max_tokens_rejection(exc: Exception, client: Any) -> bool:
- err_str = str(exc)
- # ZAI vision models reject max_tokens with code 1210 and a message that never
- # mentions "max_tokens", so detect it explicitly.
- is_zai_param_error = "1210" in err_str and "bigmodel" in str(getattr(client, "base_url", ""))
- return ("max_tokens" in err_str or "unsupported_parameter" in err_str
- or _is_unsupported_parameter_error(exc, "max_tokens") or is_zai_param_error)
- def _parameter_rungs(client: Any, max_tokens: Optional[int]) -> tuple:
- """Ordered ``(matches, strip, log message, remember)`` parameter rungs; ``strip`` returns None
- when the field was not on the wire, so an unchanged request is never re-sent; ``remember``
- (optional) records the rejection per route so the next call omits the field up front."""
- return (
- (lambda exc: _is_unsupported_parameter_error(exc, "temperature"), _without_temperature,
- "provider rejected temperature; retrying without it", remember_temperature_rejection),
- (_is_structured_output_rejection, _without_structured_output_format,
- "provider rejected the structured-output format field; retrying without it "
- "(schema enforcement degrades to prompt compliance)", remember_structured_output_rejection),
- # A chat-only model on an OpenAI-compatible relay rejects the profile's thinking-off encoding
- # (top-level ``reasoning_effort: none``), and strict-schema gateways reject the generic
- # ``extra_body.reasoning`` fallback outright (#109774); the caller only wanted "no thinking",
- # so retry with every reasoning field omitted and let the route default apply (#112781).
- # The endpoint refuses the *disable* rather than the field (Nous Portal gpt-6-astra: "Reasoning is
- # mandatory ... cannot be disabled"): step the effort up to the floor and remember the route so
- # the next thinking-off aux call starts there. Ordered before the strip so a floor that still
- # 400s falls through to it.
- (_is_reasoning_required_rejection, with_reasoning_floor,
- "provider requires reasoning; retrying at the floor effort", remember_reasoning_floor),
- (_is_reasoning_field_rejection, _without_reasoning_fields,
- "provider rejected the reasoning field; retrying without it (route default applies)", None),
- (lambda exc: max_tokens is not None and _is_max_tokens_rejection(exc, client), _without_max_tokens,
- "provider rejected the output cap; retrying without it", None),
- )
- def _ladder_parameter_rungs(
- first_err: Exception, route: _LadderRoute, kwargs: dict[str, Any], max_tokens: Optional[int],
- ):
- """Parameter rungs: retry without temperature / structured-output format / reasoning field /
- max_tokens. Rungs chain in whichever order the provider raises them (reasoning models reject
- temperature AND max_tokens; a reasoning-strip retry can then trip temperature, #78273), each
- field stripped at most once, so a request with N rejected fields recovers in N retries.
- Returns ``(response, None, kwargs)`` or ``(None, narrowed_err, stripped_kwargs)``."""
- client, task, tag = route.client, route.task, route.tag
- rungs = list(_parameter_rungs(client, max_tokens))
- while rungs:
- hit = next((rung for rung in rungs
- if rung[0](first_err) and rung[1](kwargs) is not None), None)
- if hit is None:
- break
- rungs.remove(hit)
- _matches, strip, message, remember = hit
- retry_kwargs = strip(kwargs)
- logger.info("Auxiliary %s%s: %s: %s", task or "call", tag, message, first_err)
- rejection = first_err
- resp, first_err = yield from _rung(
- _LadderStep("call", (client, retry_kwargs)), _param_rung_accepts)
- if first_err is None:
- if remember is not None:
- # Same key _build_call_kwargs looks up (base_info or resolved_base_url), so the
- # memory hits when the client exposes no base_url but the task resolved one.
- remember(route.resolved_provider, route.base_info or route.resolved_base_url,
- kwargs, rejection)
- return resp, None, retry_kwargs
- kwargs = retry_kwargs
- return None, first_err, kwargs
- def _refreshed_nous_step(route: _LadderRoute, kwargs: dict[str, Any], message: str) -> Optional[_LadderStep]:
- """Rebuild the Nous client after a credential event; None when nothing refreshed."""
- refreshed_client, refreshed_model = _refresh_nous_auxiliary_client(
- cache_provider=route.resolved_provider or "nous", model=route.final_model,
- lookup_model=route.resolved_model, lookup_task=route.task, async_mode=route.async_mode,
- base_url=route.resolved_base_url, api_key=route.resolved_api_key,
- api_mode=route.resolved_api_mode, main_runtime=route.main_runtime,
- is_vision=(route.task == "vision"),
- )
- if refreshed_client is None:
- return None
- logger.info(message, route.task or "call", route.tag)
- if refreshed_model and refreshed_model != kwargs.get("model"):
- kwargs["model"] = refreshed_model
- return _LadderStep("call", (refreshed_client, kwargs))
- def _ladder_nous_rungs(
- first_err: Exception, route: _LadderRoute, kwargs: dict[str, Any], client_is_nous: bool,
- ):
- """Nous-only rungs: stale-model self-heal, paid-account refresh, 401 refresh.
- Returns ``(response, None)`` or ``(None, first_err)`` to fall through."""
- client, task, tag = route.client, route.task, route.tag
- # A long-lived process can pin a Portal model since dropped from the catalog (every call
- # 404s); force a fresh Portal fetch and retry once.
- if _is_model_not_found_error(first_err) and client_is_nous:
- healed_model = _refresh_nous_recommended_model(
- vision=(task == "vision"), stale_model=kwargs.get("model"))
- if healed_model and healed_model != kwargs.get("model"):
- logger.warning("Auxiliary %s%s: model %r no longer in Nous catalog; "
- "retrying with refreshed recommendation %r",
- task or "call", tag, kwargs.get("model"), healed_model)
- kwargs["model"] = healed_model
- resp, first_err = yield from _rung(_LadderStep("call", (client, kwargs)), lambda exc: True)
- if first_err is None:
- return resp, None
- # Auth refresh parity with the main agent.
- if _is_payment_error(first_err) and client_is_nous and _nous_portal_account_has_fresh_paid_access():
- step = _refreshed_nous_step(
- route, kwargs,
- "Auxiliary %s%s: refreshed Nous runtime credentials after paid account check, retrying")
- if step is not None:
- resp, first_err = yield from _rung(
- step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
- if first_err is None:
- return resp, None
- if _is_auth_error(first_err) and client_is_nous:
- step = _refreshed_nous_step(
- route, kwargs, "Auxiliary %s%s: refreshed Nous runtime credentials after 401, retrying")
- if step is not None:
- resp, first_err = yield from _rung(
- step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
- if first_err is None:
- return resp, None
- return None, first_err
- def _ladder_credential_rungs(
- first_err: Exception, route: _LadderRoute, kwargs: dict[str, Any], client_is_nous: bool,
- ):
- """OAuth credential refresh + same-provider retry, then credential-pool rotation.
- Returns ``(response, None)`` or ``(None, first_err)`` to fall through."""
- client, task, tag, resolved_provider = route.client, route.task, route.tag, route.resolved_provider
- auth_refresh_provider = _auth_refresh_provider_for_route(
- resolved_provider, route.base_info, _effective_provider_for_client(client, ""))
- if (_is_auth_error(first_err) and auth_refresh_provider not in {"auto", "", None}
- and not client_is_nous):
- refresh_kwargs = ({"failed_api_key": getattr(client, "api_key", "")}
- if auth_refresh_provider == "anthropic" else {})
- if _refresh_provider_credentials(auth_refresh_provider, **refresh_kwargs):
- if auth_refresh_provider != _normalize_aux_provider(resolved_provider):
- # The stale client is cached under the route label (e.g. "auto"), not the
- # concrete backend we refreshed.
- _evict_cached_clients(resolved_provider)
- logger.info("Auxiliary %s%s: refreshed %s credentials after auth error, retrying",
- task or "call", tag, auth_refresh_provider)
- step = _LadderStep(
- "retry_same_provider",
- (auth_refresh_provider, route.resolved_model or route.final_model))
- resp, first_err = yield from _rung(
- step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
- if first_err is None:
- return resp, None
- # ``first_err`` is now the retry's own failure, not the original auth error: the
- # pool gate below and the ladder tail's eviction check both read this narrowed
- # value. An unclaimed failure (e.g. a 500) re-raised out of ``_rung`` above
- # instead, since the provider-fallback rung only acts on ``_FALLBACK_REASONS``.
- pool_provider = _recoverable_pool_provider(resolved_provider, client, main_runtime=route.main_runtime)
- # Capture the exact key used so recovery finds the right pool entry even if another
- # process rotated the pool meanwhile (current() would be None).
- _client_api_key = str(getattr(client, "api_key", "") or "")
- # Gate on the narrowed error: a connection failure from the retry above arrives here
- # unaccepted on purpose (a fresh key cannot fix an unreachable endpoint), so rotation
- # is skipped and ``first_err`` is handed to the provider-fallback chain as-is.
- if pool_provider and _credential_rung_accepts(first_err):
- recovery_err = first_err
- # No backoff retry for a payment/quota error: the same exhausted key stays refused.
- if _is_rate_limit_error(first_err) and not _is_payment_error(first_err):
- from agent.auxiliary_rate_limit_backoff import rate_limit_retry_rungs
- resp, recovery_err = yield from rate_limit_retry_rungs(first_err, client, kwargs, f"{task or 'call'}{tag}")
- if recovery_err is None:
- return resp, None
- if _recover_provider_pool(pool_provider, recovery_err, failed_api_key=_client_api_key):
- logger.info("Auxiliary %s%s: recovered %s via credential-pool rotation after %s",
- task or "call", tag, pool_provider, type(recovery_err).__name__)
- try:
- return (yield _LadderStep(
- "retry_same_provider", (resolved_provider, route.resolved_model))), None
- except Exception as retry2_err:
- # Rotated key also hit a wall: mark it now so concurrent processes skip it,
- # then fall through to the provider fallback.
- if (_is_payment_error(retry2_err) or _is_auth_error(retry2_err)
- or _is_rate_limit_error(retry2_err)):
- _recover_provider_pool(pool_provider, retry2_err)
- first_err = retry2_err
- else:
- raise
- return None, first_err
- def _next_fallback_after_quarantine(
- task: Optional[str], resolved_provider: str, is_auto: bool, route: _LadderRoute,
- failed_model: Optional[str], failure_scope: Any, *, task_chain_only: bool = False,
- ) -> tuple[Optional[Any], Optional[str], str]:
- """Next candidate after a fallback entry was quarantined mid-request (dead credential or a
- capacity error): remaining configured entries (task chain, then main chain on auto) before the
- discovery chain. ``task_chain_only`` (explicit-provider auth error) stops at the task chain —
- the user never opted that task into discovery or the main model."""
- reason = "fallback candidate unavailable"
- fb = _try_configured_fallback_chain(
- task, resolved_provider or "auto", reason=reason, failed_model=failed_model,
- failed_base_url=route.base_info, failure_scope=failure_scope)
- if task_chain_only:
- return fb
- if fb[0] is None and is_auto:
- fb = _try_main_fallback_chain(
- task, resolved_provider or "auto", reason=reason, failed_model=failed_model,
- failed_base_url=route.base_info, failure_scope=failure_scope)
- if fb[0] is None:
- fb = _try_payment_fallback(
- resolved_provider, task, reason=reason, failed_base_url=route.base_info,
- failure_scope=failure_scope, main_runtime=route.main_runtime)
- return fb
- def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):
- """Last rung: other providers (per-task chain; then auto: main fallback chain + discovery
- chain, explicit: main-agent-model net). Returns the response or None.
- Capacity errors (payment/quota, connection, exhausted 429, model incompatible, malformed
- response) bypass the explicit-provider gate — the provider cannot serve this request
- regardless of user intent. Auth errors from an explicit provider may only use the task's
- own configured fallback_chain; they never imply an unconfigured provider hop."""
- task, tag, resolved_provider = route.task, route.tag, route.resolved_provider
- # Respect explicit provider choice for transient errors (auth, request validation, etc.) but fall back when
- # the provider cannot serve due to capacity: payment/quota exhaustion (a daily-token 429 like a 402, #26803),
- # connection failures, and a 429 after retries are exhausted (#52228) are not request constraints.
- is_auto = resolved_provider in {"auto", "", None}
- reason = next((label for predicate, label in _FALLBACK_REASONS if predicate(first_err)), None)
- is_capacity_error = any(
- predicate(first_err) for predicate, label in _FALLBACK_REASONS if label != "auth error")
- task_chain = _get_auxiliary_task_config(task).get("fallback_chain") if task else None
- has_task_fallback_chain = isinstance(task_chain, list) and bool(task_chain)
- explicit_auth_with_task_chain = (
- reason == "auth error" and not is_auto and has_task_fallback_chain
- )
- if reason is None or not (is_auto or is_capacity_error or explicit_auth_with_task_chain):
- return None
- if reason == "payment error":
- # Mark the concrete backend (not the "auto" label) unhealthy so later aux calls skip
- # it instead of paying another doomed RTT.
- _mark_provider_unhealthy(
- _recoverable_pool_provider(resolved_provider, route.client, main_runtime=route.main_runtime)
- or resolved_provider, base_url=route.base_info)
- if reason == "request timed out":
- # WARNING, naming the endpoint, the budget and the knob: the only other trace of a slow
- # local model is the fallback provider's complaint about a model it never had (#89445).
- stalled = "Auxiliary chat stream" in str(first_err)
- logger.warning("Auxiliary %s%s: request to %s %s (raise auxiliary.%s.%s for slow or reasoning "
- "models) on %s, trying fallback", task or "call", tag, route.base_info or resolved_provider,
- first_err if stalled else f"timed out after {route.timeout}s", task or "call",
- "no_progress_timeout" if stalled else "timeout", resolved_provider)
- else:
- logger.info("Auxiliary %s%s: %s on %s (%s), trying fallback",
- task or "call", tag, reason, resolved_provider, first_err)
- # Skip only the failed model for model-specific failures; 401/402 are provider-wide, so
- # auth keeps skipping the credential surface, while billing is scoped to the endpoint:
- # separate custom URLs can carry separate credentials (or no billing relationship at all).
- _chain_failed_model = None if reason in ("auth error", "payment error") else route.final_model
- from agent.backend_identity import FailureScope
- _chain_failure_scope = (
- FailureScope.ENDPOINT
- if reason == "payment error" and _custom_health_base_url(resolved_provider, route.base_info)
- else None
- )
- fb_client, fb_model, fb_label = _try_configured_fallback_chain(
- task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
- failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
- if fb_client is None and is_auto:
- fb_client, fb_model, fb_label = _try_main_fallback_chain(
- task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
- failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
- if fb_client is None:
- fb_client, fb_model, fb_label = _try_payment_fallback(
- resolved_provider, task, reason=reason, failed_base_url=route.base_info,
- failure_scope=_chain_failure_scope, main_runtime=route.main_runtime)
- elif fb_client is None and not explicit_auth_with_task_chain:
- fb_client, fb_model, fb_label = _try_main_agent_model_fallback(
- resolved_provider, task, reason=reason, failed_model=_chain_failed_model,
- failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
- # Ordered walk: a candidate that returns None was quarantined (dead credential or a capacity
- # error such as a quota 429) and is now unhealthy, so re-walking the CONFIGURED chains first
- # lands on the next entry, then discovery where the selection policy allows it (#106367).
- # Bounded: each pass quarantines its candidate and a re-selected lane ends the walk (≤1 try each).
- # Candidates overwrite route_info; on exhaustion the PRIMARY error is raised, so restore its route.
- tried_lanes: set = set()
- primary_route = dict(route.route_info) if route.route_info is not None else None
- while fb_client is not None:
- lane = (fb_label, fb_model, str(getattr(fb_client, "base_url", "") or ""))
- if lane in tried_lanes:
- break
- tried_lanes.add(lane)
- _record_route_info(route.route_info, _fallback_provider_from_label(fb_label), fb_model)
- fb_resp = yield _LadderStep("fallback", (fb_client, fb_model, fb_label))
- if fb_resp is not None:
- if route.route_info is not None: # tell the caller its own route failed, not only where it landed
- route.route_info.update(failed_model=route.final_model or "", failure=f"{reason}: {first_err}",
- fallback_label=fb_label)
- return fb_resp
- fb_client, fb_model, fb_label = _next_fallback_after_quarantine(
- task, resolved_provider, is_auto, route, _chain_failed_model, _chain_failure_scope,
- task_chain_only=explicit_auth_with_task_chain)
- # All fallback layers exhausted: one user-visible warning (#26882), then the caller re-raises.
- logger.warning("Auxiliary %s%s: %s on %s and all fallbacks exhausted "
- "(fallback_chain + main agent model). Raising the primary error.",
- task or "call", tag, reason, resolved_provider)
- if primary_route is not None and route.route_info is not None:
- route.route_info.update(primary_route)
- return None
- def _aux_recovery_ladder(
- first_err: Exception, *, client: Any, kwargs: dict[str, Any], task: Optional[str],
- async_mode: bool, base_info: str, resolved_provider: str, resolved_model: Optional[str],
- resolved_base_url: Optional[str], resolved_api_key: Optional[str],
- resolved_api_mode: Optional[str], final_model: Optional[str], max_tokens: Optional[int],
- main_runtime: Optional[dict[str, Any]], route_info: Optional[dict[str, str]],
- ):
- """Ordered recovery rungs after the primary request failed (generator): parameter
- strips → Nous heal/refresh → credential refresh/pool rotation → provider fallback.
- Each rung returns a response, narrows ``first_err`` and falls through, or re-raises.
- Raises the narrowed ``first_err`` when exhausted (after evicting a connection-poisoned client)."""
- tag = " (async)" if async_mode else ""
- route = _LadderRoute(
- client, task, tag, async_mode, base_info, resolved_provider, resolved_model,
- resolved_base_url, resolved_api_key, resolved_api_mode, final_model, main_runtime, route_info,
- kwargs.get("timeout"))
- resp, first_err, kwargs = yield from _ladder_parameter_rungs(first_err, route, kwargs, max_tokens)
- if first_err is None:
- return resp
- client_is_nous = (resolved_provider == "nous"
- or base_url_host_matches(base_info, "inference-api.nousresearch.com"))
- resp, first_err = yield from _ladder_nous_rungs(first_err, route, kwargs, client_is_nous)
- if first_err is None:
- return resp
- resp, first_err = yield from _ladder_credential_rungs(first_err, route, kwargs, client_is_nous)
- if first_err is None:
- return resp
- resp = yield from _ladder_provider_fallback(first_err, route)
- if resp is not None:
- return resp
- # Connection/timeout errors poison the cached client (closed transport, half-read
- # stream); evict so the next aux call rebuilds a fresh one.
- # Reached only when no fallback answered, so the next auxiliary call rebuilds a fresh
- # client instead of reusing the dead one. ``first_err`` is the narrowed error from the
- # rungs above, not necessarily the original one. See issue #23432.
- # Mirror the sync path: drop poisoned clients on connection/timeout so the next aux call rebuilds. See
- # issue #23432.
- if _is_connection_error(first_err):
- try:
- _evict_cached_client_instance(client)
- except Exception:
- logger.debug("Auxiliary%s: cache eviction after connection error failed",
- tag, exc_info=True)
- # The narrowed error is the actionable one (e.g. a 404 "requires credits" from the
- # retry after a healed 401), so surface it rather than the original.
- raise first_err
- def _drive_ladder(ladder, perform: Callable[[_LadderStep], Any]) -> Any:
- """Run a ladder generator, feeding each step's result (or exception) back in; a ``wait`` step sleeps."""
- try:
- step = next(ladder)
- while True:
- try:
- result = time.sleep(step.args[0]) if step.kind == "wait" else perform(step)
- except Exception as exc:
- step = ladder.throw(exc)
- else:
- step = ladder.send(result)
- except StopIteration as stop:
- return stop.value
- async def _drive_ladder_async(ladder, perform: Callable[[_LadderStep], Any]) -> Any:
- """Async twin of :func:`_drive_ladder` (``perform`` is awaited)."""
- import asyncio
- try:
- step = next(ladder)
- while True:
- try:
- result = await (asyncio.sleep(step.args[0]) if step.kind == "wait" else perform(step))
- except Exception as exc:
- step = ladder.throw(exc)
- else:
- step = ladder.send(result)
- except StopIteration as stop:
- return stop.value
- def _elapsed_ms(started_at: float, now: Optional[float] = None) -> int:
- """Whole milliseconds since ``started_at`` (clamped at 0)."""
- return max(0, int(((time.monotonic() if now is None else now) - started_at) * 1000))
- def _stamp_latency_once(latency_info: Optional[dict[str, int]], key: str, started_at: float) -> None:
- """Record ``key`` in ``latency_info`` the first time it fires."""
- if latency_info is not None and key not in latency_info:
- latency_info[key] = _elapsed_ms(started_at)
- @_relay_auxiliary_call
- def call_llm(
- task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
- api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
- temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
- timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
- extra_headers: Optional[dict[str, str]] = None, api_mode: str | None = None, stream: bool = False,
- stream_options: dict | None = None, route_info: Optional[dict[str, str]] = None,
- latency_info: Optional[dict[str, int]] = None,
- ) -> Any:
- """Run an auxiliary LLM request, applying the configured task limit."""
- queue_started_at = time.monotonic()
- semaphore = _acquire_sync_aux_semaphore(task)
- if semaphore is not None:
- semaphore.acquire()
- request_started_at = time.monotonic()
- if latency_info is not None:
- latency_info["queue_wait_ms"] = _elapsed_ms(queue_started_at, request_started_at)
- prior_progress_hook = getattr(_aux_progress, "hook", None)
- try:
- with (
- scoped_runtime_main(main_runtime),
- aux_progress_hook(
- prior_progress_hook
- if callable(prior_progress_hook)
- else ((lambda: None) if latency_info is not None else None)
- ),
- _aux_thread_local_hook(_aux_dispatch, functools.partial(
- _stamp_latency_once, latency_info, "provider_dispatch_ms", request_started_at)),
- _aux_thread_local_hook(_aux_provider_response, functools.partial(
- _stamp_latency_once, latency_info, "time_to_first_progress_ms", request_started_at)),
- ):
- response = _call_llm_impl(
- task=task, provider=provider, model=model, base_url=base_url, api_key=api_key,
- main_runtime=main_runtime, messages=messages, temperature=temperature,
- max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
- reasoning_config=reasoning_config, extra_headers=extra_headers, api_mode=api_mode,
- stream=stream, stream_options=stream_options, route_info=route_info,
- )
- if stream and semaphore is not None:
- stream_semaphore = semaphore
- semaphore = None
- return _release_sync_semaphore_after_stream(response, stream_semaphore)
- return response
- finally:
- if latency_info is not None:
- latency_info["summary_generation_ms"] = _elapsed_ms(request_started_at)
- if semaphore is not None:
- semaphore.release()
- def _release_sync_semaphore_after_stream(stream: Any, semaphore: threading.BoundedSemaphore):
- """Release a permit only after a streaming response is consumed or closed."""
- try:
- yield from stream
- finally:
- try:
- close = getattr(stream, "close", None)
- if callable(close):
- close()
- finally:
- semaphore.release()
- def _plan_aux_call(
- task: Optional[str], *, async_mode: bool, provider: Optional[str], model: Optional[str],
- base_url: Optional[str], api_key: Optional[str], main_runtime: Optional[dict[str, Any]],
- messages: list, temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
- timeout: Optional[float], extra_body: Optional[dict], reasoning_config: Optional[dict],
- extra_headers: Optional[dict[str, str]], api_mode: Optional[str],
- route_info: Optional[dict[str, str]],
- ) -> tuple[_PreparedAuxRequest, dict[str, Any], dict[str, Any]]:
- """Shared head of both call impls: prepare the request and bundle the kwargs the recovery
- drivers pass to ``_retry_same_provider_*`` / ``_call_fallback_candidate_*``. One immutable
- runtime snapshot for keying/resolution/retries/fallbacks, so a concurrent /model switch
- can't mix key and client from different runtimes."""
- main_runtime = _normalize_main_runtime(main_runtime)
- req = _prepare_aux_request(
- task, provider=provider, model=model, base_url=base_url, api_key=api_key,
- main_runtime=main_runtime, messages=messages, temperature=temperature,
- max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
- reasoning_config=reasoning_config, extra_headers=extra_headers,
- api_mode=api_mode, route_info=route_info, async_mode=async_mode,
- )
- candidate_kwargs = dict(
- task=task, messages=messages, temperature=temperature, max_tokens=max_tokens,
- tools=tools, effective_timeout=req.effective_timeout,
- effective_extra_body=req.effective_extra_body, reasoning_config=reasoning_config,
- )
- retry_kwargs = dict(
- candidate_kwargs, resolved_base_url=req.resolved_base_url,
- resolved_api_key=req.resolved_api_key, resolved_api_mode=req.resolved_api_mode,
- main_runtime=main_runtime, final_model=req.final_model, extra_headers=extra_headers,
- )
- return req, retry_kwargs, candidate_kwargs
- def _should_retry_same_provider(task: Optional[str], exc: Exception, tag: str) -> bool:
- """True when ``exc`` is a transient transport blip worth a same-provider retry; critical-path
- tasks skip it on a full-budget timeout (``_should_skip_same_provider_retry``) and go straight
- to fallback."""
- if not _is_transient_transport_error(exc):
- return False
- if _should_skip_same_provider_retry(task, exc):
- logger.info("Auxiliary %s%s: timeout on the critical path; "
- "skipping same-provider retry and falling back: %s", task, tag, exc)
- return False
- return True
- def _ladder_step_call(
- step: _LadderStep, req: _PreparedAuxRequest, retry_kwargs: dict[str, Any], candidate_kwargs: dict[str, Any],
- ) -> tuple[str, tuple, dict[str, Any]]:
- """Resolve a ladder step into ``(kind, args, kwargs)`` for the sync/async performer."""
- if step.kind == "call":
- return "call", step.args, dict(provider=req.resolved_provider, api_mode=req.resolved_api_mode)
- if step.kind == "retry_same_provider":
- retry_provider, retry_model = step.args
- return "retry", (), dict(retry_kwargs, resolved_provider=retry_provider, resolved_model=retry_model)
- return "fallback", step.args, candidate_kwargs
- def _start_recovery_ladder(
- first_err: Exception, req: _PreparedAuxRequest, retry_kwargs: dict[str, Any], *,
- task: Optional[str], async_mode: bool, route_info: Optional[dict[str, str]],
- ):
- """Build the recovery-ladder generator for a failed primary request."""
- return _aux_recovery_ladder(
- first_err, client=req.client, kwargs=req.kwargs, task=task, async_mode=async_mode,
- base_info=req.base_info, resolved_provider=req.resolved_provider,
- resolved_model=req.resolved_model, resolved_base_url=req.resolved_base_url,
- resolved_api_key=req.resolved_api_key, resolved_api_mode=req.resolved_api_mode,
- final_model=req.final_model, max_tokens=retry_kwargs["max_tokens"],
- main_runtime=retry_kwargs["main_runtime"], route_info=route_info)
- def _call_llm_impl(
- task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
- api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
- temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
- timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
- extra_headers: Optional[dict[str, str]] = None, api_mode: str | None = None, stream: bool = False,
- stream_options: dict | None = None, route_info: Optional[dict[str, str]] = None,
- ) -> Any:
- """Centralized synchronous LLM call: resolve provider/model, auth, kwargs, fallbacks.
- task: aux task whose provider:model comes from config (ignored if provider set); api_mode
- overrides task config; timeout=None reads auxiliary.{task}.timeout; extra_headers override
- client defaults. stream=True returns the raw SDK stream (caller consumes/falls back)
- instead of a validated response. RuntimeError if no provider is configured."""
- req, retry_kwargs, candidate_kwargs = _plan_aux_call(
- task, async_mode=False, provider=provider, model=model, base_url=base_url,
- api_key=api_key, main_runtime=main_runtime, messages=messages,
- temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=timeout,
- extra_body=extra_body, reasoning_config=reasoning_config,
- extra_headers=extra_headers, api_mode=api_mode, route_info=route_info,
- )
- client, kwargs, request_provider = req.client, req.kwargs, req.request_provider
- # Streaming path (MoA aggregator): return the raw SDK stream, skipping validation and
- # the fallback chain (they assume a complete response); the caller owns reassembly/fallback.
- if stream:
- kwargs["stream"] = True
- if stream_options:
- kwargs["stream_options"] = stream_options
- if task == "moa_aggregator" and isinstance(client, CodexAuxiliaryClient):
- # Responses-shim clients consume the stream internally and return a completed
- # object Relay's managed stream would iterate; the MoA facade wraps it as one chunk.
- return client.chat.completions.create(**kwargs)
- return _relay_sync_stream(client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode)
- def _primary(**validate_kw: Any) -> Any:
- # Retry on the same provider for a transient transport blip (connection reset / streaming-close /
- # incomplete chunked read / 5xx / 408) before the except-chain below escalates to provider/model
- # fallback. A dropped connection shouldn't abandon an otherwise-healthy provider — this especially
- # matters for pinned auxiliary calls like MoA reference advisors, where "fallback to another
- # provider" is not a meaningful recovery (the advisor is a specific model), so a transient blip that
- # isn't retried simply loses that advisor for the turn (root of the run2 double-advisor "Connection
- # error" collapse — a genuine upstream blip hitting both parallel advisors at once). Attempts are
- # bounded and use exponential backoff. Count is configurable via auxiliary.transient_retries
- # (default 2 retries → 3 total attempts); a second/third failure or any non-transient error falls
- # through to ``first_err`` and the existing fallback handling unchanged. Unified home for the
- # transient retry every auxiliary task shares. (PR #16587)
- return _validate_llm_response(
- _relay_sync_completion(
- client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode,
- create=lambda request: _create_with_progress(
- client, request, task,
- force_stream=_provider_requires_stream(
- request_provider, req.base_info or req.resolved_base_url),
- ),
- ),
- task, **validate_kw,
- )
- try:
- # Bounded same-provider retry (exponential backoff, auxiliary.transient_retries) for
- # transient blips before escalating to fallback — a dropped connection shouldn't
- # abandon a healthy provider (matters for pinned MoA advisors).
- try:
- return _primary(provider=request_provider, base_url=req.base_info)
- except Exception as transient_err:
- if not _should_retry_same_provider(task, transient_err, ""):
- raise
- _max_transient_retries = _transient_retry_count()
- _last_transient = transient_err
- for _attempt in range(1, _max_transient_retries + 1):
- _backoff = min(_TRANSIENT_RETRY_BACKOFF_BASE * (2.0 ** (_attempt - 1)), 8.0)
- logger.info("Auxiliary %s: transient transport error (attempt %d/%d); "
- "retrying same provider after %.1fs before fallback: %s",
- task or "call", _attempt, _max_transient_retries, _backoff, _last_transient)
- time.sleep(_backoff)
- try:
- return _primary()
- except Exception as retry_transient:
- if not _is_transient_transport_error(retry_transient):
- raise
- _last_transient = retry_transient
- raise _last_transient
- except Exception as first_err:
- def _perform(step: _LadderStep) -> Any:
- kind, args, kw = _ladder_step_call(step, req, retry_kwargs, candidate_kwargs)
- if kind == "call":
- return _validate_llm_response(_relay_sync_completion(*args, **kw), task)
- if kind == "retry":
- return _retry_same_provider_sync(**kw)
- return _call_fallback_candidate_sync(*args, **kw)
- return _drive_ladder(
- _start_recovery_ladder(first_err, req, retry_kwargs, task=task, async_mode=False, route_info=route_info),
- _perform)
- def _coerce_llm_message(response):
- """Pull a message (dict, object, or str) out of a response-or-message value: dict-shaped
- responses/bare messages (compression, proxies) and ChatCompletion objects; MagicMock
- ``reasoning_*`` attrs are deliberately not strings."""
- if response is None or isinstance(response, str):
- return response
- if isinstance(response, dict):
- if "choices" not in response:
- return response
- choices = response.get("choices") or []
- else:
- choices = getattr(response, "choices", None)
- if not choices:
- return response
- return _message_field(choices[0], "message") if choices else None
- def _message_field(msg, name):
- return msg.get(name) if isinstance(msg, dict) else getattr(msg, name, None)
- def extract_content_or_reasoning(response, *, max_reasoning_chars: int | None = None) -> str:
- """Extract content from an LLM response, falling back to reasoning fields.
- Order: ``content`` (inline think blocks stripped) → ``reasoning``/``reasoning_content`` →
- ``reasoning_details`` (OpenRouter array). Accepts a response or bare message;
- ``max_reasoning_chars`` bounds a reasoning fallback so unbounded chain-of-thought can't
- become the compaction summary. Returns ``""`` if nothing found."""
- msg = _coerce_llm_message(response)
- if msg is None:
- return ""
- if isinstance(msg, str):
- return msg.strip()
- raw = _message_field(msg, "content")
- if not isinstance(raw, str):
- raw = str(raw) if raw else ""
- content = raw.strip()
- if content:
- # Same precompiled closed-pair patterns as strip_think_blocks.
- from agent.agent_runtime_helpers import _REASONING_BLOCK_PATTERNS
- cleaned = content
- for pattern in _REASONING_BLOCK_PATTERNS:
- cleaned = pattern.sub("", cleaned)
- cleaned = cleaned.strip()
- if cleaned:
- return cleaned
- # Content is empty or reasoning-only — try structured reasoning fields
- reasoning_parts: list[str] = []
- for field in ("reasoning", "reasoning_content"):
- val = _message_field(msg, field)
- if val and isinstance(val, str) and val.strip() and val not in reasoning_parts:
- reasoning_parts.append(val.strip())
- details = _message_field(msg, "reasoning_details")
- if details and isinstance(details, list):
- for detail in details:
- if isinstance(detail, dict):
- summary = detail.get("summary") or detail.get("content") or detail.get("text")
- if summary and summary not in reasoning_parts:
- reasoning_parts.append(summary.strip() if isinstance(summary, str) else str(summary))
- if not reasoning_parts:
- return ""
- text = "\n\n".join(reasoning_parts)
- if max_reasoning_chars is not None and len(text) > max_reasoning_chars:
- logger.warning("fell back to reasoning fields (%d chars); truncating to %d",
- len(text), max_reasoning_chars)
- return text[:max_reasoning_chars]
- return text
- @_relay_auxiliary_call_async
- async def async_call_llm(
- task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
- api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
- temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
- timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
- route_info: Optional[dict[str, str]] = None,
- ) -> Any:
- """Run an asynchronous auxiliary LLM request under the configured limit."""
- semaphore = _acquire_async_aux_semaphore(task)
- if semaphore is not None:
- await semaphore.acquire()
- try:
- with scoped_runtime_main(main_runtime):
- return await _async_call_llm_impl(
- task=task, provider=provider, model=model, base_url=base_url, api_key=api_key,
- main_runtime=main_runtime, messages=messages, temperature=temperature,
- max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
- reasoning_config=reasoning_config, route_info=route_info,
- )
- finally:
- if semaphore is not None:
- semaphore.release()
- async def _async_call_llm_impl(
- task: str | None = None, *, provider: str | None = None, model: str | None = None, base_url: str | None = None,
- api_key: str | None = None, main_runtime: Optional[dict[str, Any]] = None, messages: list,
- temperature: Optional[float] = None, max_tokens: int | None = None, tools: list | None = None,
- timeout: float | None = None, extra_body: dict | None = None, reasoning_config: Optional[dict] = None,
- route_info: Optional[dict[str, str]] = None,
- ) -> Any:
- """Centralized asynchronous LLM call; see call_llm() for full documentation.
- No per-request header / api_mode override on the async entry point."""
- req, retry_kwargs, candidate_kwargs = _plan_aux_call(
- task, async_mode=True, provider=provider, model=model, base_url=base_url,
- api_key=api_key, main_runtime=main_runtime, messages=messages,
- temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=timeout,
- extra_body=extra_body, reasoning_config=reasoning_config,
- extra_headers=None, api_mode=None, route_info=route_info,
- )
- client, kwargs, request_provider = req.client, req.kwargs, req.request_provider
- try:
- # Retry ONCE on the same provider for a transient blip before fallback (see call_llm()).
- # (PR #16587)
- _force_stream_async = _provider_requires_stream(request_provider, req.base_info or req.resolved_base_url)
- async def _acreate(_kwargs: dict[str, Any]) -> Any:
- return await _acreate_with_progress(client, _kwargs, task, force_stream=_force_stream_async)
- async def _primary(**validate_kw: Any) -> Any:
- import asyncio
- return await asyncio.to_thread(
- _validate_llm_response,
- await _relay_async_completion(
- client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode,
- create=_acreate),
- task, **validate_kw)
- try:
- return await _primary(provider=request_provider, base_url=req.base_info)
- except Exception as transient_err:
- # The async Codex adapter wraps the sync stream via to_thread: same TimeoutError here.
- if not _should_retry_same_provider(task, transient_err, " (async)"):
- raise
- logger.info("Auxiliary %s (async): transient transport error; retrying "
- "once on the same provider before fallback: %s", task or "call", transient_err)
- return await _primary()
- except Exception as first_err:
- async def _perform(step: _LadderStep) -> Any:
- kind, args, kw = _ladder_step_call(step, req, retry_kwargs, candidate_kwargs)
- if kind == "call":
- import asyncio
- return await asyncio.to_thread(_validate_llm_response, await _relay_async_completion(*args, **kw), task)
- if kind == "retry":
- return await _retry_same_provider_async(**kw)
- fb_client, fb_model, fb_label = args
- fb_client, _ = _to_async_client(fb_client, fb_model or "", is_vision=(task == "vision"))
- return await _call_fallback_candidate_async(fb_client, fb_model, fb_label, **kw)
- return await _drive_ladder_async(
- _start_recovery_ladder(first_err, req, retry_kwargs, task=task, async_mode=True, route_info=route_info),
- _perform)
|