adapter.py 433 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385
  1. # health: allow FILE_LINES -- platform-tables refactor: core's PLATFORM_HINTS/_PLATFORM_DEFAULTS rows for this platform moved onto its register() call (hint text lives in adapter_hint.py); the growth is only those kwargs
  2. """Telegram platform adapter (python-telegram-bot): inbound messages/media/commands, outbound replies."""
  3. import asyncio
  4. import contextlib
  5. import dataclasses
  6. import hashlib
  7. import inspect
  8. import json
  9. import logging
  10. import os
  11. import html as _html
  12. import re
  13. import time
  14. from contextvars import ContextVar
  15. from datetime import datetime, timezone, UTC
  16. from typing import Any, Awaitable, Callable, Dict, Iterator, List, Optional, Set
  17. from hermes_cli import setup_platforms
  18. logger = logging.getLogger(__name__)
  19. from agent.deadline import run_bounded_async
  20. from agent.ssl_verify import platform_ssl_context
  21. from gateway.platforms._shared import (
  22. decode_json_list_literal as _decode_json_list_literal,
  23. extra_or_secret as _extra_or_secret, get_scoped_secret as _get_scoped_secret,
  24. platform_gate_env as _scoped_gate_env,
  25. )
  26. def _redact_telegram_error_text(error: object) -> str:
  27. """Redact secrets from Telegram transport errors before logging or returning them."""
  28. text = "" if error is None else str(error)
  29. if not text:
  30. # httpx timeout exceptions (ConnectTimeout, ReadTimeout, ...) stringify to "" — keep the
  31. # class name so failure lines never log an empty reason (#111211).
  32. return f"<{type(error).__name__}>" if error is not None else text
  33. try:
  34. from agent.redact import redact_sensitive_text
  35. return redact_sensitive_text(text, force=True)
  36. except Exception:
  37. return "<telegram error redacted>"
  38. def _consume_abandoned_task(task: asyncio.Task) -> None:
  39. """Observe a detached task's terminal exception to avoid noisy loop logs."""
  40. try:
  41. task.exception()
  42. except asyncio.CancelledError:
  43. pass
  44. except Exception:
  45. logger.debug("Abandoned Telegram init task failed after timeout", exc_info=True)
  46. async def _await_with_thread_deadline(
  47. awaitable, timeout: float, *, on_abandon=None, label: str = "telegram", dump_on_blocked_loop: bool = True,
  48. ):
  49. """Wall-clock deadline that survives a blocked loop / cancellation-shielded PTB+httpcore init.
  50. ``on_abandon`` runs detached so an abandoned initialize() can't leak an httpx pool. Raises
  51. ``asyncio.TimeoutError`` on expiry (feeds the PTB retry ladder). Send/media call sites pass
  52. ``dump_on_blocked_loop=False``: the bug they bound is a shielded socket on a LIVE loop, and the init
  53. wrapper already reports a blocked loop, so N in-flight sends must not each arm a stack-dump timer.
  54. Thin wrapper over :func:`agent.deadline.run_bounded_async` (#85125 Phase 2f) — this adapter's private
  55. implementation was the ancestor of that primitive and is now consolidated onto it. The unified layer
  56. keeps every property the 9 call sites here rely on: thread-timer deadline that survives a blocked event
  57. loop (#63309), abandonment of cancellation-shielded tasks (PTB/httpcore init inside anyio scopes),
  58. detached best-effort ``on_abandon`` cleanup so an abandoned initialize() can't leak an httpx pool per
  59. retry attempt, and off-loop stack-dump diagnostics when the loop never processes the expiry.
  60. """
  61. result = await run_bounded_async(
  62. awaitable, timeout, label=label, on_abandon=on_abandon, dump_on_blocked_loop=dump_on_blocked_loop)
  63. if result.timed_out:
  64. raise TimeoutError(f"timed out after {timeout:.0f}s ({label})")
  65. return result.value
  66. def _iter_exception_graph(error: BaseException) -> Iterator[BaseException]:
  67. """Yield ``error`` and every ``__cause__``/``__context__`` ancestor (DFS, cycle-safe) —
  68. PTB wraps httpx errors, so classifiers must inspect the whole graph."""
  69. seen: set[int] = set()
  70. stack: list[BaseException] = [error]
  71. while stack:
  72. cur = stack.pop()
  73. ident = id(cur)
  74. if ident in seen:
  75. continue
  76. seen.add(ident)
  77. yield cur
  78. stack.extend(x for x in (getattr(cur, "__cause__", None), getattr(cur, "__context__", None)) if x is not None)
  79. async def _shutdown_abandoned_app(app) -> None:
  80. """Release a half-built PTB app's httpx transports after an abandoned init: ``app.shutdown()``
  81. no-ops when ``_initialized`` was never set, so the request transports are closed directly."""
  82. if app is None:
  83. return
  84. try:
  85. await app.shutdown()
  86. except Exception:
  87. logger.debug("Abandoned Telegram app.shutdown() failed", exc_info=True)
  88. bot = getattr(app, "bot", None)
  89. for request in (getattr(bot, "_request", None) if bot is not None else None) or ():
  90. shutdown = getattr(request, "shutdown", None)
  91. if shutdown is None:
  92. continue
  93. try:
  94. result = shutdown()
  95. if asyncio.iscoroutine(result) or asyncio.isfuture(result):
  96. await result
  97. except Exception:
  98. logger.debug("Abandoned Telegram request shutdown failed", exc_info=True)
  99. try:
  100. from telegram import Update, Bot, Message, InlineKeyboardButton, InlineKeyboardMarkup
  101. try:
  102. from telegram import LinkPreviewOptions
  103. except ImportError:
  104. LinkPreviewOptions = None
  105. from telegram.ext import (
  106. Application, CommandHandler, CallbackQueryHandler, InlineQueryHandler, MessageHandler as TelegramMessageHandler,
  107. ContextTypes, TypeHandler, filters)
  108. from telegram.constants import ParseMode, ChatType
  109. from telegram.request import HTTPXRequest
  110. TELEGRAM_AVAILABLE = True
  111. except ImportError:
  112. TELEGRAM_AVAILABLE = False
  113. Update = Bot = Message = InlineKeyboardButton = InlineKeyboardMarkup = Application = Any
  114. CommandHandler = CallbackQueryHandler = InlineQueryHandler = TypeHandler = TelegramMessageHandler = HTTPXRequest = Any
  115. LinkPreviewOptions = filters = ParseMode = ChatType = None
  116. # Mock so ContextTypes.DEFAULT_TYPE annotations don't crash class definition without the lib.
  117. class _MockContextTypes:
  118. DEFAULT_TYPE = Any
  119. ContextTypes = _MockContextTypes
  120. import sys
  121. from pathlib import Path as _Path
  122. sys.path.insert(0, str(_Path(__file__).resolve().parents[3]))
  123. from gateway.authz_mixin import _coerce_allow_set
  124. from agent.i18n import get_language, t
  125. from gateway.config import Platform, PlatformConfig
  126. from gateway.platforms.base import (
  127. BasePlatformAdapter, ExecApprovalPrompt, SendResult, classify_send_error, unauthorized_action_notice,
  128. cache_image_from_bytes_async, cache_audio_from_bytes_async, cache_video_from_bytes_async, resolve_proxy_url, SUPPORTED_VIDEO_TYPES,
  129. SUPPORTED_DOCUMENT_TYPES, SUPPORTED_IMAGE_DOCUMENT_TYPES, _TEXT_INJECT_EXTENSIONS, utf16_len,
  130. )
  131. # Telegram truncates ``answerCallbackQuery`` text at 200 chars; ``BotCommand`` descriptions at 256.
  132. _TOAST_LIMIT = 200
  133. _BOT_COMMAND_DESCRIPTION_LIMIT = 256
  134. def _unauthorized() -> str:
  135. """Every refused button tap answers with the same sentence — resolved per tap so the active
  136. language applies (never bound at import)."""
  137. return unauthorized_action_notice(Platform.TELEGRAM)[:_TOAST_LIMIT]
  138. def _toast(key: str, **kwargs: Any) -> str:
  139. """``t()`` for ``query.answer(text=...)`` payloads, cut at Telegram's 200-char toast cap."""
  140. return t(key, **kwargs)[:_TOAST_LIMIT]
  141. def _bold_label_html(line: str) -> str:
  142. """HTML-bold the ``Label:`` prefix of a translated line (whole line when it has no colon)."""
  143. label, sep, rest = line.partition(":")
  144. if not sep:
  145. return f"<b>{_html.escape(line)}</b>"
  146. return f"<b>{_html.escape(label)}:</b>{_html.escape(rest)}"
  147. # Inbound-media ``kind`` labels are dual-use: the English word stays in the agent-visible observed note,
  148. # the chat reply shows the catalog translation.
  149. _MEDIA_KIND_KEYS = {
  150. "attachment": "platform.telegram.media.kind_attachment", "photo": "platform.telegram.media.kind_photo",
  151. "voice message": "platform.telegram.media.kind_voice", "audio file": "platform.telegram.media.kind_audio",
  152. "video file": "platform.telegram.media.kind_video", "video note": "platform.telegram.media.kind_video"}
  153. from gateway.platforms.event import MessageEvent, MessageType, ProcessingOutcome
  154. from plugins.platforms.telegram.telegram_entities import expand_link_entities
  155. from plugins.platforms.telegram.telegram_held_inbound import TelegramHeldInboundMixin
  156. from plugins.platforms.telegram.telegram_ids import normalize_telegram_chat_id, parse_send_target, parse_telegram_message_id
  157. from plugins.platforms.telegram.telegram_resolved_prompt import edit_keeping_prompt
  158. from plugins.platforms.telegram.telegram_topic_transfer import TopicTransferFn, apply_root_dm_topic_transfer
  159. from plugins.platforms.telegram.telegram_network import (
  160. SEED_FALLBACK_IPS, TelegramFallbackTransport, discover_fallback_ips, parse_fallback_ip_env, tcp_keepalive_socket_options)
  161. from plugins.platforms.telegram.telegram_topic_routing import TelegramTopicRoutingMixin
  162. from utils import env_float, env_int
  163. _TELEGRAM_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".gif"}
  164. # Max seconds a send/edit may sleep inline on a flood-control RetryAfter; longer penalties fail
  165. # closed with ``flood_control:{wait}`` so the caller's retry machinery owns the wait.
  166. # Longer server penalties fail closed with a ``flood_control:{wait}`` SendResult so the caller's retry
  167. # machinery (delivery ledger, streaming fallback) owns the wait instead of the coroutine pinning its worker
  168. # — a 97-minute penalty on the boot path froze inbound on every platform (#91969).
  169. _FLOOD_INLINE_WAIT_CAP_SECS = 5.0
  170. # Shared per-chat outbound budget (#116312): Telegram counts an editMessageText against the
  171. # SAME per-chat allowance as a sendMessage, but streaming previews used to pace only edits at
  172. # DEFAULT_STREAMING_EDIT_INTERVAL = 0.8s (1.25 msg/s into one chat before any reply was sent)
  173. # — that was 83% of measured flood penalties. One shared slot per chat: a SEND waits for its
  174. # slot (skipping a send would drop a message), an INTERIM edit is skipped (the next tick shows
  175. # the same text anyway), and the FINAL edit is never gated (the answer itself is never
  176. # withheld). Tunable: validated in production by the issue reporter at 0 flood events.
  177. _TELEGRAM_CHAT_OUTBOUND_BUDGET_SECS = 1.0
  178. def _flood_cap_result(wait: float) -> SendResult:
  179. """The shared fail-closed SendResult for an over-cap flood wait."""
  180. return SendResult(success=False, error=f"flood_control:{wait}", retry_after=float(wait))
  181. _TELEGRAM_IMAGE_MIME_TO_EXT = {"image/png": ".png", "image/jpeg": ".jpg", "image/jpg": ".jpg", "image/webp": ".webp", "image/gif": ".gif"}
  182. _TELEGRAM_IMAGE_EXT_TO_MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif"}
  183. def _coerce_duration_seconds(value: Any) -> Optional[int]:
  184. """Round a raw length to whole positive seconds, or None if unusable."""
  185. try:
  186. secs = round(float(value))
  187. except (TypeError, ValueError):
  188. return None
  189. return secs if secs > 0 else None
  190. def _probe_voice_duration_seconds(path: str) -> Optional[int]:
  191. """Best-effort whole-second audio length (wave → mutagen → ffprobe; None if unreadable).
  192. Telegram renders long clips as 0:00 without an explicit duration. Blocking: use ``to_thread``."""
  193. if os.path.splitext(path)[1].lower() == ".wav":
  194. try:
  195. import wave
  196. with wave.open(path, "rb") as wf:
  197. rate = wf.getframerate() or 0
  198. secs = _coerce_duration_seconds(wf.getnframes() / float(rate)) if rate else None
  199. if secs is not None:
  200. return secs
  201. except Exception:
  202. pass
  203. try:
  204. import mutagen
  205. secs = _coerce_duration_seconds(getattr(getattr(mutagen.File(path), "info", None), "length", None))
  206. if secs is not None:
  207. return secs
  208. except Exception:
  209. pass
  210. try:
  211. import shutil
  212. import subprocess
  213. if shutil.which("ffprobe"):
  214. proc = subprocess.run(
  215. ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", path],
  216. stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=5, check=False)
  217. if proc.returncode == 0:
  218. return _coerce_duration_seconds(proc.stdout.strip())
  219. except Exception:
  220. pass
  221. return None
  222. def _probe_video_geometry(path: str) -> dict[str, int]:
  223. """``{"width", "height", "duration"}`` for a local video; ``{}`` when ffprobe can't read it.
  224. Telegram runs its own video processing only for uploads under roughly 10 MB; above that it
  225. stores the file as an unprocessed ``320x320`` video with ``duration=0``, so the message must
  226. carry the real geometry or clients draw a square tile for any aspect ratio.
  227. """
  228. try:
  229. import shutil
  230. import subprocess
  231. if not shutil.which("ffprobe"):
  232. return {}
  233. proc = subprocess.run(
  234. ["ffprobe", "-v", "error", "-select_streams", "v:0",
  235. "-show_entries", "stream=width,height", "-show_entries", "format=duration",
  236. "-of", "json", path],
  237. stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=20, check=False)
  238. if proc.returncode != 0:
  239. return {}
  240. blob = json.loads(proc.stdout or "{}")
  241. streams = blob.get("streams") or []
  242. if not streams:
  243. return {}
  244. geometry = {"width": int(streams[0]["width"]), "height": int(streams[0]["height"])}
  245. duration = _coerce_duration_seconds((blob.get("format") or {}).get("duration"))
  246. if duration:
  247. geometry["duration"] = duration
  248. return geometry
  249. except Exception:
  250. logger.debug("[Telegram] video geometry probe failed for %s", path, exc_info=True)
  251. return {}
  252. def _video_thumbnail_jpeg(path: str, duration: Optional[int]) -> Optional[str]:
  253. """Write a 320px-wide JPEG frame for Telegram's ``thumbnail`` field; None on failure.
  254. Telegram keeps a supplied thumbnail for the uploads it did not process itself — without one the
  255. chat shows a square placeholder tile until the video is opened.
  256. """
  257. out = None
  258. try:
  259. import shutil
  260. import subprocess
  261. import tempfile
  262. if not shutil.which("ffmpeg"):
  263. return None
  264. seek = max(1, int((duration or 3) * 0.25))
  265. fd, out = tempfile.mkstemp(suffix=".jpg", prefix="hermes-tg-thumb-")
  266. os.close(fd)
  267. proc = subprocess.run(
  268. ["ffmpeg", "-y", "-ss", str(seek), "-i", path, "-frames:v", "1",
  269. "-vf", "scale=320:-2", "-q:v", "6", out],
  270. stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, check=False)
  271. if proc.returncode != 0 or not os.path.getsize(out):
  272. with contextlib.suppress(OSError):
  273. os.remove(out)
  274. return None
  275. return out
  276. except Exception:
  277. logger.debug("[Telegram] video thumbnail extraction failed for %s", path, exc_info=True)
  278. if out:
  279. with contextlib.suppress(OSError):
  280. os.remove(out)
  281. return None
  282. def telegram_deps_present() -> bool:
  283. """PASSIVE registry ``check_fn``: is python-telegram-bot importable? Never installs
  284. (``check_telegram_requirements`` is the active ``ensure_deps_fn``).
  285. Registry ``check_fn`` — called from status displays and config loading, so it must never install
  286. anything. The ACTIVE lazy-installer (``check_telegram_requirements``) is registered as
  287. ``ensure_deps_fn`` and runs from ``create_adapter()`` when this returns False (#79812).
  288. """
  289. return TELEGRAM_AVAILABLE
  290. def check_telegram_requirements() -> bool:
  291. """Check if Telegram dependencies are available.
  292. If python-telegram-bot is missing, attempts to lazy-install it via
  293. ``pm.ensure_import("telegram")``. After a successful
  294. install, re-imports the SDK and flips ``TELEGRAM_AVAILABLE`` to True
  295. so the adapter's class-level type aliases get rebound.
  296. """
  297. global TELEGRAM_AVAILABLE, Update, Bot, Message, InlineKeyboardButton
  298. global InlineKeyboardMarkup, LinkPreviewOptions, Application
  299. global CommandHandler, CallbackQueryHandler, InlineQueryHandler, TelegramMessageHandler
  300. global ContextTypes, filters, ParseMode, ChatType, HTTPXRequest, TypeHandler
  301. if TELEGRAM_AVAILABLE:
  302. return True
  303. try:
  304. from pm import ensure_import as _lazy_ensure
  305. _lazy_ensure("telegram")
  306. except Exception:
  307. return False
  308. try:
  309. import importlib
  310. _tg, _ext, _const, _req = (
  311. importlib.import_module(m) for m in ("telegram", "telegram.ext", "telegram.constants", "telegram.request"))
  312. Update, Bot, Message, InlineKeyboardButton, InlineKeyboardMarkup = (
  313. getattr(_tg, n) for n in ("Update", "Bot", "Message", "InlineKeyboardButton", "InlineKeyboardMarkup"))
  314. LinkPreviewOptions = getattr(_tg, "LinkPreviewOptions", None)
  315. Application, CommandHandler, CallbackQueryHandler, InlineQueryHandler, TelegramMessageHandler = (
  316. getattr(_ext, n) for n in ("Application", "CommandHandler", "CallbackQueryHandler", "InlineQueryHandler", "MessageHandler"))
  317. ContextTypes, filters, TypeHandler = _ext.ContextTypes, _ext.filters, _ext.TypeHandler
  318. ParseMode, ChatType = _const.ParseMode, _const.ChatType
  319. HTTPXRequest = _req.HTTPXRequest
  320. except (ImportError, AttributeError):
  321. return False
  322. TELEGRAM_AVAILABLE = True
  323. return True
  324. # Every char MarkdownV2 requires backslash-escaped outside code spans/fences.
  325. _MDV2_ESCAPE_RE = re.compile(r'([_*\[\]()~`>#\+\-=|{}.!\\])')
  326. def _escape_mdv2(text: str) -> str:
  327. """Escape Telegram MarkdownV2 special characters with a preceding backslash."""
  328. return _MDV2_ESCAPE_RE.sub(r'\\\1', text)
  329. def _strip_mdv2(text: str) -> str:
  330. """Strip MarkdownV2 escapes and formatting markers for the plain-text fallback."""
  331. cleaned = re.sub(r'\\([_*\[\]()~`>#\+\-=|{}.!\\])', r'\1', text) # escape backslashes
  332. cleaned = re.sub(r'\*\*([^*]+)\*\*', r'\1', cleaned) # **bold** BEFORE MarkdownV2 *bold*
  333. cleaned = re.sub(r'\*([^*]+)\*', r'\1', cleaned)
  334. cleaned = re.sub(r'(?<!\w)_([^_]+)_(?!\w)', r'\1', cleaned) # italic; word-bounded so snake_case survives
  335. cleaned = re.sub(r'~([^~]+)~', r'\1', cleaned) # strikethrough
  336. cleaned = re.sub(r'\|\|([^|]+)\|\|', r'\1', cleaned) # spoiler
  337. return cleaned
  338. _CHUNK_INDICATOR_ON_FENCE_RE = re.compile(r'(?m)^``` (?P<indicator>(?:\\)?\(\d+/\d+(?:\\)?\))$')
  339. def _separate_chunk_indicator_from_fence(text: str) -> str:
  340. """Move a ``(N/M)`` chunk marker that ``truncate_message()`` appended to a synthesized closing
  341. fence onto its own line — Telegram rejects ````` \\(1/2\\)`` as a fence."""
  342. return _CHUNK_INDICATOR_ON_FENCE_RE.sub(r'```\n\g<indicator>', text)
  343. # MarkdownV2 has no table syntax, so pipe tables become bullet groups via convert_table_to_bullets().
  344. from gateway.platforms.helpers import (
  345. TABLE_SEPARATOR_RE as _TABLE_SEPARATOR_RE, compile_mention_patterns, convert_table_to_bullets as _wrap_markdown_tables)
  346. # Rich-message regions whose internal newlines must stay bare (Telegram renders them natively):
  347. # fenced code blocks OR GFM pipe-table blocks (header row, delimiter row, data rows).
  348. _RICH_PROTECTED_REGION_RE = re.compile(
  349. r'(?:```[^\n]*\n[\s\S]*?```)' # fenced code block
  350. r'|(?:^[^\n]*\|[^\n]*\n' # table header row (has a pipe)
  351. r'[ \t]*\|?[ \t]*:?-+:?[ \t]*(?:\|[ \t]*:?-+:?[ \t]*)+\|?[ \t]*' # delimiter
  352. r'(?:\n[^\n]*\|[^\n]*)*)', # data rows (newline-led, trailing \n left for prose)
  353. re.MULTILINE)
  354. def _rich_normalize_linebreaks(text: str) -> str:
  355. """Convert lone ``\\n`` (a Markdown soft break) to hard breaks for sendRichMessage; ``\\n\\n``,
  356. fenced code and pipe tables are left untouched."""
  357. if not text or '\n' not in text:
  358. return text
  359. out: list[str] = []
  360. pos = 0
  361. for m in _RICH_PROTECTED_REGION_RE.finditer(text):
  362. out.append(re.sub(r'(?<!\n)\n(?!\n)', ' \n', text[pos:m.start()]))
  363. out.append(m.group(0)) # protected region kept verbatim
  364. pos = m.end()
  365. out.append(re.sub(r'(?<!\n)\n(?!\n)', ' \n', text[pos:]))
  366. return ''.join(out)
  367. # Internal safety bounds (not user knobs): no reconnect/teardown path may hang on a dead CLOSE-WAIT
  368. # socket PTB's polling task is blocked on in epoll.
  369. _UPDATER_STOP_TIMEOUT = 15.0 # `await updater.stop()`, applied identically at every site
  370. _DISCONNECT_STEP_TIMEOUT = 2.0 # other disconnect() steps: short, so a swallowed cancel can't burn the fatal budget
  371. _UPDATER_START_TIMEOUT = 30.0 # start_polling() can hang on a degraded pool after a drain
  372. # Initial connect is unhealthy until getUpdates completes one round trip; bootstrap fails closed so
  373. # GatewayRunner disposes the adapter and retries fresh.
  374. # Per-step bound for disconnect() awaits that are not updater.stop() itself. Kept short so a
  375. # cancellation-swallowing lifecycle/PTB close cannot burn the gateway's whole fatal-handler budget before
  376. # the reconnect queue is useful (#80598). updater.stop() keeps the longer _UPDATER_STOP_TIMEOUT.
  377. # start_polling() can also hang when the connection pool is in a degraded state after
  378. # _drain_polling_connections(), particularly when both primary and fallback Telegram endpoints are
  379. # unreachable. Bounding start_polling() prevents the reconnect ladder from stalling indefinitely and allows
  380. # the heartbeat loop to trigger its own recovery path. Refs: NousResearch/hermes-agent#59614
  381. _INITIAL_POLLING_PROGRESS_TIMEOUT = 60.0
  382. # Bounded drain (shutdown()/initialize() of the getUpdates request) so a wedged socket can't freeze
  383. # _polling_error_task and gate every escalation path behind its in-flight guard.
  384. # shutdown()/initialize() on the getUpdates httpx request close and rebuild the connection pool. When a
  385. # connection is wedged on a stale CLOSE-WAIT socket that close can block forever, hanging
  386. # _drain_polling_connections() and freezing the whole reconnect ladder (the tracked _polling_error_task
  387. # never completes, so every escalation path stays gated behind its in-flight guard). Bound the drain so the
  388. # ladder always advances toward the fatal-restart escalation. Matches _UPDATER_STOP_TIMEOUT. Refs:
  389. # NousResearch/hermes-agent#66377
  390. _DRAIN_TIMEOUT = 15.0
  391. # Wedged-recovery watchdog: healthy worst case is stop + 2x drain + start + 60s backoff ≈ 135s, so
  392. # 300s in flight is unambiguously stuck and the heartbeat force-escalates.
  393. # Every recovery path (the reconnect ladder's re-entry, the pending-update probe, PTB's error callback)
  394. # gates new recovery on ``_polling_error_task.done()``; if that task ever wedges on a hung await that no
  395. # local bound covers, the whole gateway goes silently deaf with nothing retrying. The heartbeat loop
  396. # force-escalates a recovery task that stays in-flight far longer than any healthy ladder attempt could take
  397. # — stop (_UPDATER_STOP_TIMEOUT) + drain (2x_DRAIN_TIMEOUT) + start (_UPDATER_START_TIMEOUT) + max backoff
  398. # (60s) is ~135s, so 300s is unambiguously stuck. See #66377.
  399. _POLLING_ERROR_TASK_STUCK_TIMEOUT = 300.0
  400. _POLLING_PROGRESS_TIMEOUT = 60.0 # generation unhealthy until getUpdates returns; exceeds one idle long-poll
  401. # Telegram answers a long-poll within ~50s; no round-trip for ~3x that while get_me() is healthy and
  402. # nothing is queued means a consumer wedged on a socket that never raises (CLOSE-WAIT behind a route flip).
  403. # Telegram holds a long-poll open for at most ~50s before answering (empty or not), so a healthy idle poller
  404. # completes a getUpdates round-trip well inside this window. If no round-trip has completed for longer than
  405. # this — while get_me() on the general request path stays healthy and no updates are queued server-side —
  406. # the long-poll consumer is wedged on a socket that never raises (CLOSE-WAIT behind a TUN/proxy route flip,
  407. # #92991) and no other probe can see it. ~3x the worst-case poll window leaves ample margin against false
  408. # positives while still recovering within a few heartbeat intervals.
  409. _POLLING_STALL_TIMEOUT = 150.0
  410. # Ingress dispatch stall (#102260, #130407): the transport probes prove getUpdates round-trips complete, not
  411. # that PTB's dispatcher ever handed the fetched updates to a handler. A backlog with no dispatch progress for
  412. # four heartbeats (360s) hands the adapter to the supervisor for a rebuild (an in-place restart keeps the
  413. # wedged dispatcher). Sized past _POLLING_ERROR_TASK_STUCK_TIMEOUT (300s), the bound on a slow handler: updates
  414. # of one chat still dispatch in order (PerChatUpdateProcessor), so e.g. a sticker vision_analyze (120s default)
  415. # must not trip it. Re-arms on progress.
  416. _INGRESS_DISPATCH_STALL_HEARTBEATS = 4
  417. # sendVideo transcodes before answering, outlasting the 20s read timeout; also how long a user waits
  418. # to hear the attachment failed, so kept modest.
  419. _MEDIA_SEND_READ_TIMEOUT = 60.0
  420. # Text send used to hang forever on a shielded httpcore socket (NordVPN/Telegram sticky IP).
  421. # A wedged send froze getUpdates on the same loop. Bound every text send/edit and media upload.
  422. _TEXT_SEND_DEADLINE = 30.0
  423. # Wall-clock cap on one media upload (whole request: pool wait + connect + body upload + server
  424. # processing). NOT `_MEDIA_SEND_READ_TIMEOUT`: that is httpx's per-phase stall budget (time-to-first-byte
  425. # after the body is sent), whereas this bounds the entire call, so it must leave room for bandwidth. The
  426. # Bot API upload cap is 50 MB; at ~2 Mbit/s that is ~200 s, plus connect (10 s) and sendVideo transcoding
  427. # (up to the 60 s read timeout) — 300 s covers it. It is also >2x the sum of the per-phase httpx budgets
  428. # (pool 8 + connect 10 + media_write 60 + read 60 = 138 s), so it only fires when a shielded socket has
  429. # stopped raising at all, never on a merely slow link.
  430. # On expiry `run_bounded_async` cancels and then abandons the upload task (never awaited), inside
  431. # `_chat_send_lock`: the lock is released while the abandoned task drains. Awaiting the cancel with a grace
  432. # period would re-hang the lock on exactly the wedged socket this bounds, so the rare late landing is
  433. # accepted; httpx's own timeouts free the pool slot.
  434. _MEDIA_SEND_DEADLINE = 300.0
  435. _POLLING_GENERATION_CONTEXT: ContextVar[Optional[int]] = ContextVar("telegram_polling_generation", default=None)
  436. class _PollingLifecycleAbort(RuntimeError):
  437. """Internal control flow for polling startup fenced by teardown."""
  438. class _PollingStallError(RuntimeError):
  439. """A confirmed getUpdates stall (watchdog or post-reconnect verifier), as opposed to a transport drop.
  440. Typed so the recovery ladder can hand the adapter to the supervisor instead of classifying log text:
  441. restarting the same Updater cannot heal a wedged long-poll consumer whose stop() did not quiesce (#113618).
  442. """
  443. class TelegramAdapter(TelegramHeldInboundMixin, TelegramTopicRoutingMixin, BasePlatformAdapter):
  444. """Telegram bot adapter: users/groups, MarkdownV2 replies, forum topics, media."""
  445. # Bound for the per-(chat_id, status_key) status-message cache; FIFO half-trim on overflow.
  446. _STATUS_MESSAGE_IDS_MAX = 2000
  447. MAX_MESSAGE_LENGTH = 4096
  448. supports_code_blocks = True # MarkdownV2 renders fenced code blocks
  449. splits_long_messages = True # send() chunks via truncate_message(MAX_MESSAGE_LENGTH)
  450. RICH_MESSAGE_MAX_CHARS = 32768 # Bot API 10.1 rich cap; above it use legacy chunking
  451. _SPLIT_THRESHOLD = 4000 # chunk near this length ⇒ a client-side split continuation is almost certain
  452. MEDIA_GROUP_WAIT_SECONDS = 0.8
  453. HELD_INBOUND_MAX = 64 # inbound events held across a disconnect window; oldest dropped first
  454. _GENERAL_TOPIC_THREAD_ID = "1"
  455. # send() can race a disconnect blip; failing "Not connected" (retryable=False) parks the answer in the
  456. # delivery ledger until next boot, so wait briefly for _bot (or a replacement adapter) instead.
  457. _RECONNECT_WAIT_SECONDS = 15.0
  458. _RECONNECT_POLL_INTERVAL = 0.5
  459. # Large-image compression for Telegram photo sends. Behind an HTTP proxy the PTB
  460. # media_write_timeout is easily exceeded by raw PNGs > 1-2MB; pre-compressing to
  461. # progressive JPEG keeps the upload well under the timeout and reduces bandwidth.
  462. _IMG_JPEG_QUALITY = 85
  463. _IMG_MAX_DIMENSION = 1600 # above this, resize before JPEG
  464. _IMG_COMPRESS_THRESHOLD_BYTES = 1_048_576 # 1MB
  465. # edit_message applies MarkdownV2 only on finalize=True; without this flag stream_consumer skips
  466. # the final edit when raw text is unchanged.
  467. # Fixes #25710.
  468. REQUIRES_EDIT_FINALIZE: bool = True
  469. FALLBACK_ON_FINAL_EDIT_FLOOD: bool = True # retrying a final edit burns the same flood budget
  470. RESEND_FINAL_ON_EMPTY_STREAM_FALLBACK: bool = True # a failed final edit may leave a partial preview
  471. # Adaptive text-batch ingress ("feels instant"): ≤320 codepoints settle in ~180ms, ≤1024 in ~240ms,
  472. # longer waits the configured cap; always clamped to ``_text_batch_delay_seconds``.
  473. _TEXT_BATCH_FAST_LEN = 320
  474. _TEXT_BATCH_FAST_DELAY_S = 0.18
  475. _TEXT_BATCH_SHORT_LEN = 1024
  476. _TEXT_BATCH_SHORT_DELAY_S = 0.24
  477. @staticmethod
  478. def _env_float_clamped(name: str, default: float, *, min_value: Optional[float] = None, max_value: Optional[float] = None) -> float:
  479. """Read a float env var; non-finite → default; clamp to bounds (safe for asyncio.sleep)."""
  480. import math
  481. raw = os.getenv(name)
  482. try:
  483. value = float(raw) if raw is not None else float(default)
  484. except (TypeError, ValueError):
  485. value = float(default)
  486. if not math.isfinite(value):
  487. value = float(default)
  488. if min_value is not None:
  489. value = max(value, min_value)
  490. if max_value is not None:
  491. value = min(value, max_value)
  492. return value
  493. @property
  494. def _teardown_started(self) -> bool:
  495. """True once disconnect() fenced polling (tolerates object.__new__ test adapters)."""
  496. return getattr(self, "_polling_teardown_started", False)
  497. @property
  498. def message_len_fn(self):
  499. """Telegram measures message length in UTF-16 code units."""
  500. return utf16_len
  501. def __init__(self, config: PlatformConfig):
  502. super().__init__(config, Platform.TELEGRAM)
  503. extra = self.config.extra
  504. self._app: Optional[Application] = None
  505. self._seen_update_ids: dict = {}
  506. self._inflight_update_ids: dict = {}
  507. self._update_admission = None
  508. # Completed update IDs survive adapter replacement and restarts (update_admission.py).
  509. # Resolved now: secondary profiles construct adapters inside their own home scope.
  510. from hermes_constants import get_hermes_home
  511. self._update_receipt_dir = get_hermes_home()
  512. self._update_receipts_loaded: set = set()
  513. self._update_receipts_dirty: set = set()
  514. self._update_receipt_flush: Optional[asyncio.Task] = None
  515. self._bot: Optional[Bot] = None
  516. self._webhook_mode: bool = False
  517. self._mention_patterns = self._compile_mention_patterns()
  518. self._reply_to_mode: str = getattr(config, 'reply_to_mode', 'first') or 'first'
  519. self._disable_link_previews: bool = self._coerce_bool_extra("disable_link_previews", False)
  520. # Bot API 10.1 Rich Messages render what MarkdownV2 degrades (tables, task lists, <details>, block
  521. # math). Opt-in: current clients make rich messages hard to copy as plain text. rich_drafts is a
  522. # separate opt-in (Desktop can leave rich draft frames overlaid): off keeps native draft transport
  523. # but skips rich draft rendering; the final reply still lands via sendRichMessage.
  524. self._rich_messages_enabled: bool = self._coerce_bool_extra("rich_messages", False)
  525. # CJK stays on legacy MarkdownV2 by default (Desktop/macOS garble, #47653); opt-in for unaffected clients.
  526. self._allow_cjk_rich_messages: bool = self._coerce_bool_extra("allow_cjk_rich_messages", False)
  527. self._rich_drafts_enabled: bool = self._coerce_bool_extra("rich_drafts", False)
  528. self._rich_send_disabled = self._rich_draft_disabled = False # latched after a capability failure
  529. # Transient sendChatAction failures recur on every keep-typing tick; back off per chat.
  530. self._telegram_typing_cooldown_until: dict[str, float] = {}
  531. self._telegram_typing_cooldown_seconds: float = self._coerce_float_extra(
  532. "typing_cooldown_seconds", 30.0, min_value=1.0, max_value=300.0)
  533. # Post-send typing re-arm: scheduled, deduped and rate-limited per chat. Awaiting a
  534. # sendChatAction round-trip on the send path shares the loop with the getUpdates long-polls,
  535. # and under concurrent streaming it starved them until they rotted into CLOSE-WAIT (#111727).
  536. self._telegram_typing_retrigger_tasks: dict[str, asyncio.Task] = {}
  537. self._telegram_typing_retrigger_at: dict[str, float] = {}
  538. # Telegram's bubble lasts ~5s and _keep_typing already refreshes every 2s, so the re-arm only
  539. # has to cover the gap left by a landed message. 0 restores a call per intermediate send.
  540. self._telegram_typing_retrigger_interval: float = self._coerce_float_extra(
  541. "typing_retrigger_min_interval_seconds", 2.0, min_value=0.0, max_value=30.0)
  542. # Buffer album/photo bursts into a single MessageEvent instead of self-interrupting turns.
  543. self._media_batch_delay_seconds = env_float("HERMES_TELEGRAM_MEDIA_BATCH_DELAY_SECONDS", 0.8)
  544. self._pending_photo_batches: dict[str, MessageEvent] = {}
  545. self._pending_photo_batch_tasks: dict[str, asyncio.Task] = {}
  546. self._media_group_events: dict[str, MessageEvent] = {}
  547. self._media_group_tasks: dict[str, asyncio.Task] = {}
  548. # Flushes past their pop, by (family, key): a newer chunk must not cancel them (teardown still does).
  549. self._committed_flush_tasks: dict[tuple[str, str], asyncio.Task] = {}
  550. # Aggregate client-side splits of long messages into one MessageEvent; bounds are conservative
  551. # for Telegram's ~1 edit/s flood envelope.
  552. self._text_batch_delay_seconds = self._env_float_clamped(
  553. "HERMES_TELEGRAM_TEXT_BATCH_DELAY_SECONDS", self._TEXT_BATCH_DEFAULT_DELAY_S,
  554. min_value=0.08, max_value=self._TEXT_BATCH_MAX_DELAY_S)
  555. self._text_batch_split_delay_seconds = self._env_float_clamped(
  556. "HERMES_TELEGRAM_TEXT_BATCH_SPLIT_DELAY_SECONDS", self._TEXT_BATCH_DEFAULT_SPLIT_DELAY_S,
  557. min_value=self._text_batch_delay_seconds, max_value=self._TEXT_BATCH_MAX_SPLIT_DELAY_S)
  558. self._drop_delayed_deliveries = False
  559. # Held across disconnect: PTB advances the offset before our drop-guard runs, so Telegram won't
  560. # redeliver — dropping is permanent loss (see _hold_inbound_event).
  561. self._held_inbound_events: list[MessageEvent] = []
  562. self._held_inbound_redispatch_task: Optional[asyncio.Task] = None
  563. self._polling_error_task: Optional[asyncio.Task] = None
  564. self._polling_progress_verifier_task: Optional[asyncio.Task] = None
  565. self._polling_heartbeat_task: Optional[asyncio.Task] = None
  566. self._bot_identity_refresh_task: Optional[asyncio.Task] = None
  567. self._post_connect_task: Optional[asyncio.Task] = None # command menu + DM topics, off the connect path
  568. self._polling_conflict_count = self._polling_network_error_count = self._polling_generation = 0
  569. self._polling_conflict_recovery_generation: Optional[int] = None
  570. self._polling_progress_event = asyncio.Event()
  571. self._polling_progress_accepting = self._polling_teardown_started = False
  572. self._polling_error_callback_ref = None
  573. # Stall watchdog: generation start and last successful getUpdates (None = unknown).
  574. # Monotonic timestamps for the polling stall watchdog (#92991): when the current polling generation
  575. # began, and when the last successful getUpdates round-trip completed.
  576. self._polling_generation_started_monotonic: Optional[float] = None
  577. self._polling_last_progress_monotonic: Optional[float] = None
  578. # Ingress accounting (#102260): received (getUpdates wire) vs dispatched (PTB admission).
  579. self._updates_received_total: int = 0
  580. self._updates_dispatched_total: int = 0
  581. self._ingress_dispatched_seen: int = 0
  582. self._ingress_stalled_heartbeats: int = 0
  583. # Live @username: PTB caches getMe() at initialize() and only rewrites it inside get_me(), so a
  584. # BotFather rename leaves self._bot.username stale; routing reads _current_bot_username().
  585. self._bot_username_observed: Optional[str] = None
  586. # None = never checked. Must NOT be 0.0: compared against time.monotonic(), which on a fresh host
  587. # starts near zero, so 0.0 would suppress the first refresh for a TTL.
  588. self._bot_identity_checked_at: Optional[float] = None
  589. # Consecutive heartbeat probes seeing queued updates the poller isn't consuming (get_me() can't
  590. # see a wedged getUpdates) / finding the updater stopped with no reconnect in flight; escalate after two.
  591. self._polling_pending_stuck_count = self._polling_not_running_count = 0
  592. # Degraded until getUpdates makes progress; while True, send() short-circuits to failure so callers
  593. # (cron live-adapter branch) fall through to standalone delivery.
  594. # Consecutive heartbeat probes that saw queued updates the running poller is not consuming. get_me()
  595. # can't see this — the send path is healthy while the getUpdates consumer is wedged — so the
  596. # heartbeat also probes get_webhook_info().pending_update_count and escalates to recovery after two
  597. # consecutive stuck probes (#42909).
  598. # Consecutive heartbeat probes that found the updater stopped entirely (running=False) while we are
  599. # in polling mode with no reconnect in flight. Distinct from the wedged-but-running case above: the
  600. # long-poll task is simply gone, so neither the connectivity probe nor PTB's error_callback ever
  601. # fires and the gateway silently stops receiving messages with the process still alive (#55769).
  602. self._send_path_degraded: bool = False
  603. self._general_request_drain_lock = asyncio.Lock()
  604. self._dm_topics: dict[str, int] = {} # topic_name -> message_thread_id
  605. # (chat_id, sender_id) -> (new topic id, last Message.date, monotonic record time) for client-created
  606. # DM topics whose thread-less opener may still arrive (#136567); see TelegramTopicRoutingMixin.
  607. self._pending_created_topics: dict[tuple[str, str], tuple[str, float, float]] = {}
  608. self._forum_command_registered: set[int] = set() # forum chats with commands registered
  609. self._command_menu_fingerprint: str = "" # language + menu payload last pushed via set_my_commands
  610. self._forum_lock = asyncio.Lock()
  611. # Status indicator: bot short description "Online"/"Offline" on connect/clean disconnect. Off by
  612. # default because it mutates the GLOBAL profile; opt in via extra.status_indicator.
  613. self._status_indicator_enabled: bool = bool(extra.get("status_indicator", False))
  614. self._status_online_text: str = str(extra.get("status_online", "Online"))
  615. self._status_offline_text: str = str(extra.get("status_offline", "Offline"))
  616. # Cold-boot queue: drop server-side pending updates on first boot (default True,
  617. # preserves historical behaviour). Set extra.drop_pending_on_cold_boot: false to
  618. # receive messages sent while the gateway was offline (e.g. nightly-off hosts).
  619. # Watcher reconnects always preserve the queue regardless of this setting.
  620. self._drop_pending_on_cold_boot: bool = self._coerce_bool_extra("drop_pending_on_cold_boot", True)
  621. self._dm_topics_config: list[dict[str, Any]] = extra.get("dm_topics", [])
  622. # chat_ids with DM topics configured (O(1) root-DM ignore check)
  623. self._dm_topic_chat_ids: set[str] = {str(e["chat_id"]) for e in self._dm_topics_config if "chat_id" in e}
  624. # getFile cap: 20MB on the public Bot API, 2GB on a local telegram-bot-api (base_url).
  625. self._max_doc_bytes: int = 2 * 1024 * 1024 * 1024 if extra.get("base_url") else 20 * 1024 * 1024
  626. self._model_picker_state: dict[str, dict] = {} # per-chat interactive picker state
  627. self._choice_picker_state: dict[str, dict] = {}
  628. self._approval_state: dict[int, str] = {} # message_id → session_key
  629. self._approval_request_ids: dict[int, str] = {} # message_id → tools.approval request id
  630. self._slash_confirm_state: dict[str, str] = {} # confirm_id → session_key
  631. self._clarify_state: dict[str, str] = {} # clarify_id → session_key
  632. # "important" (default): only final responses, approvals and slash confirmations notify;
  633. # "all": every message notifies (display.platforms.telegram.notifications).
  634. self._notifications_mode: str = "important"
  635. # send_or_update_status(): {(chat_id, status_key) -> message_id} so repeat calls edit in place.
  636. # send_or_update_status() bookkeeping: {(chat_id, status_key) -> bot message_id} Tracks status
  637. # bubbles owned by this adapter so subsequent calls with the same key edit the same message instead
  638. # of appending new ones (#30045).
  639. self._status_message_ids: dict[tuple, str] = {}
  640. # Last truncated mid-stream preview per (chat_id, message_id): past the 4096 cap every edit
  641. # truncates to the SAME text, and resending burns flood budget. Dropped on finalize.
  642. self._last_overflow_preview: dict[tuple, str] = {}
  643. @property
  644. def send_path_degraded(self) -> bool:
  645. # True from polling-generation start until the first getUpdates
  646. # round-trip is proven (_record_polling_progress), and again at every
  647. # polling-death site. getattr: tests build adapters via object.__new__().
  648. return bool(getattr(self, "_send_path_degraded", False))
  649. def _mark_connected(self) -> None:
  650. self._drop_delayed_deliveries = False
  651. super()._mark_connected()
  652. self._schedule_held_inbound_redispatch() # PTB will not redeliver these events
  653. def _mark_disconnected(self) -> None:
  654. self._drop_delayed_deliveries = True
  655. super()._mark_disconnected()
  656. def _set_fatal_error(self, code: str, message: str, *, retryable: bool) -> None:
  657. self._drop_delayed_deliveries = True
  658. super()._set_fatal_error(code, message, retryable=retryable)
  659. # Permanent fatal: no reconnect will drain, so discard the hold queue (later holds are refused).
  660. # Discard the hold queue now and refuse further holds (teardown salvage / late enqueue must not
  661. # re-populate a queue that can never drain — review #83878).
  662. if not retryable:
  663. held = getattr(self, "_held_inbound_events", None)
  664. n = len(held) if held else 0
  665. if held:
  666. held.clear()
  667. if n:
  668. logger.warning("[Telegram] Non-retryable fatal (%s); discarding %d held inbound message(s)", code, n)
  669. def _is_permanent_fatal(self) -> bool:
  670. """True after non-retryable fatal — holds must discard, not queue."""
  671. if not getattr(self, "_fatal_error_code", None):
  672. return False
  673. return not bool(getattr(self, "_fatal_error_retryable", True))
  674. def _replacement_telegram_adapter(self) -> "Optional[TelegramAdapter]":
  675. """Live adapter if the reconnect watcher replaced us in ``runner.adapters`` (an in-flight
  676. ``send()`` still holds the old instance whose ``_bot`` stays None)."""
  677. runner = getattr(self, "gateway_runner", None)
  678. adapters = getattr(runner, "adapters", None) or {}
  679. live = adapters.get(self.platform)
  680. if live is not None and live is not self and getattr(live, "_bot", None):
  681. return live
  682. return None
  683. async def _wait_for_reconnection(self) -> bool:
  684. """Wait for ``_bot`` or a replacement adapter; False on expiry or permanent fatal."""
  685. if self._bot or self._replacement_telegram_adapter() is not None:
  686. return True
  687. if self._is_permanent_fatal():
  688. return False
  689. wait_s = float(getattr(self, "_RECONNECT_WAIT_SECONDS", 15.0))
  690. poll_s = float(getattr(self, "_RECONNECT_POLL_INTERVAL", 0.5))
  691. logger.info("[%s] Not connected — waiting for reconnection (up to %.0fs)", self.name, wait_s)
  692. waited = 0.0
  693. while waited < wait_s:
  694. await asyncio.sleep(poll_s)
  695. waited += poll_s
  696. if self._is_permanent_fatal():
  697. return False
  698. if self._bot or self._replacement_telegram_adapter() is not None:
  699. logger.info("[%s] Reconnected after %.1fs", self.name, waited)
  700. return True
  701. logger.warning("[%s] Still not connected after %.0fs", self.name, wait_s)
  702. return False
  703. def _should_drop_delayed_delivery(self) -> bool:
  704. """True once teardown/fatal started: delayed flushes must not dispatch onto a torn-down session.
  705. Callers must NOT destroy the event (PTB already advanced the offset) — hold and redispatch."""
  706. return bool(getattr(self, "_drop_delayed_deliveries", False))
  707. def _notification_kwargs(self, metadata: Optional[dict[str, Any]]) -> dict[str, Any]:
  708. """In "important" mode return disable_notification=True unless ``notify`` or ``is_approval_prompt`` (#132516)."""
  709. if getattr(self, "_notifications_mode", "important") != "important" or (metadata or {}).get("notify") \
  710. or (metadata or {}).get("is_approval_prompt"):
  711. return {}
  712. return {"disable_notification": True}
  713. @staticmethod
  714. def _normalize_chat_type(chat_type: Any, *, is_forum: bool) -> str:
  715. """Telegram chat type → gateway chat type (``private``→``dm``, ``supergroup``→forum/group)."""
  716. normalized = str(chat_type or "dm").strip().lower() or "dm"
  717. if normalized == "private":
  718. return "dm"
  719. if normalized == "supergroup":
  720. return "forum" if is_forum else "group"
  721. return normalized
  722. def _legacy_runner_auth_fn(self):
  723. """``runner._is_user_authorized`` resolved off the bound handler (bare-adapter tests, direct
  724. embedding); None under multiplex where the handler is a profile closure."""
  725. # Resolve through the runner's full auth chain (platform + group allowlists, pairing store,
  726. # allow-all flags). Prefer the platform-bound callback registered via set_authorization_check: it
  727. # routes to GatewayRunner._is_user_authorized AND survives multiplex handler wrapping, whereas the
  728. # bound-handler __self__ lookup is None when the primary handler is a profile closure — which
  729. # silently dropped the chat allowlist and default-denied allowlisted group members under
  730. # multiplex_profiles (#87132). Fall back to the bound handler for setups without a registered
  731. # callback.
  732. runner = getattr(getattr(self, "_message_handler", None), "__self__", None)
  733. auth_fn = getattr(runner, "_is_user_authorized", None)
  734. return auth_fn if callable(auth_fn) else None
  735. @staticmethod
  736. def _env_allowlist_decision(user_id: str) -> Optional[bool]:
  737. """TELEGRAM_ALLOWED_USERS decision; None when no allowlist is configured."""
  738. allowed_csv = _scoped_gate_env("TELEGRAM_ALLOWED_USERS").strip()
  739. if not allowed_csv:
  740. return None
  741. allowed_ids = {uid.strip() for uid in allowed_csv.split(",") if uid.strip()}
  742. return "*" in allowed_ids or user_id in allowed_ids
  743. def _is_callback_user_authorized(
  744. self, user_id: str, *, chat_id: Optional[str] = None, chat_type: Optional[str] = None,
  745. thread_id: Optional[str] = None, user_name: Optional[str] = None) -> bool:
  746. """Return whether a Telegram inline-button caller may perform gated actions."""
  747. normalized_user_id = str(user_id or "").strip()
  748. if not normalized_user_id:
  749. return False
  750. normalized_chat_type = self._normalize_chat_type(chat_type, is_forum=thread_id is not None)
  751. # Preferred: the auth callback GatewayRunner injects (set_authorization_check) → full
  752. # _is_user_authorized chain; also works for a multiplexed adapter whose _message_handler is a
  753. # profile closure. getattr tolerates partially-constructed adapters (object.__new__ in tests).
  754. if getattr(self, "_authorization_check", None) is not None:
  755. injected = self._is_sender_authorized(
  756. normalized_user_id, chat_type=normalized_chat_type, chat_id=str(chat_id or normalized_user_id),
  757. thread_id=str(thread_id) if thread_id is not None else None)
  758. if injected is not None:
  759. return injected
  760. auth_fn = self._legacy_runner_auth_fn()
  761. if auth_fn is not None:
  762. try:
  763. from gateway.session import SessionSource
  764. source = SessionSource(
  765. platform=Platform.TELEGRAM, chat_id=str(chat_id or normalized_user_id), chat_type=normalized_chat_type,
  766. user_id=normalized_user_id, user_name=str(user_name).strip() if user_name else None,
  767. thread_id=str(thread_id) if thread_id is not None else None)
  768. return bool(auth_fn(source))
  769. except Exception:
  770. logger.debug(
  771. "[Telegram] Falling back to env-only callback auth for user %s", normalized_user_id, exc_info=True)
  772. decision = self._env_allowlist_decision(normalized_user_id)
  773. if decision is None:
  774. # Fail-closed: no allowlist means deny unless GATEWAY_ALLOW_ALL_USERS is set.
  775. # The runner auth path in _is_user_authorized() handles GATEWAY_ALLOW_ALL_USERS; this fallback
  776. # must not silently allow everyone (fixes #24457).
  777. return _scoped_gate_env("GATEWAY_ALLOW_ALL_USERS").lower() in {"true", "1", "yes"}
  778. return decision
  779. def _source_from_message_for_auth(self, message: Message):
  780. """Build the SessionSource the gateway auth path expects; identity comes from ``from_user``,
  781. falling back to ``sender_chat`` for channel posts so an unauthorized channel can't inject."""
  782. from gateway.session import SessionSource
  783. user = getattr(message, "from_user", None)
  784. chat = getattr(message, "chat", None)
  785. user_id = str(getattr(user, "id", "")).strip() or None
  786. # Carry is_bot so the runner's ``*_ALLOW_BOTS`` branch is reachable, as in build_source.
  787. is_bot = bool(getattr(user, "is_bot", False)) if user is not None else False
  788. user_name = str(getattr(user, "username", "") or getattr(user, "full_name", "") or "").strip() or None
  789. if not user_id: # channel post — authorize the sender chat instead
  790. sender_chat = getattr(message, "sender_chat", None)
  791. if sender_chat is not None:
  792. user_id = str(getattr(sender_chat, "id", "")).strip() or None
  793. if not user_name:
  794. user_name = str(getattr(sender_chat, "title", "") or "").strip() or None
  795. chat_id = str(getattr(chat, "id", "")).strip() or user_id
  796. thread_id_raw = getattr(message, "message_thread_id", None)
  797. is_topic_message = bool(getattr(message, "is_topic_message", False))
  798. is_forum_group = getattr(chat, "is_forum", False) is True
  799. chat_type = self._normalize_chat_type(
  800. getattr(chat, "type", "dm"), is_forum=thread_id_raw is not None and (is_topic_message or is_forum_group))
  801. thread_id = None
  802. if thread_id_raw is not None and (
  803. (chat_type == "forum" and (is_topic_message or is_forum_group)) or (chat_type == "dm" and is_topic_message)):
  804. thread_id = str(thread_id_raw)
  805. return SessionSource(
  806. platform=Platform.TELEGRAM, chat_id=chat_id or "", chat_type=chat_type, user_id=user_id,
  807. user_name=user_name, thread_id=thread_id, is_bot=is_bot)
  808. def _source_from_reaction_for_auth(self, update):
  809. """SessionSource for a ``message_reaction`` update's actor (``user`` or ``actor_chat``).
  810. Raises ``ValueError`` when actor, chat or message identity is absent so the post-auth boundary fails closed."""
  811. mr = getattr(update, "message_reaction", None)
  812. if mr is None:
  813. raise ValueError("gateway_platform_event source extraction requires a message_reaction update")
  814. user = getattr(mr, "user", None) or getattr(mr, "actor_chat", None)
  815. chat = getattr(mr, "chat", None)
  816. user_id = str(getattr(user, "id", "")).strip() or None
  817. user_name = str(getattr(user, "username", "") or getattr(user, "full_name", "") or getattr(user, "title", "")).strip() or None
  818. chat_id = str(getattr(chat, "id", "")).strip() or None
  819. message_id = getattr(mr, "message_id", None)
  820. if not user_id or not chat_id or message_id is None or not str(message_id).strip():
  821. raise ValueError("gateway_platform_event reaction requires actor, chat, and message identities")
  822. # Reactions carry no message_thread_id; is_forum is the only forum signal.
  823. chat_type = self._normalize_chat_type(getattr(chat, "type", "dm"), is_forum=getattr(chat, "is_forum", False) is True)
  824. return self.build_source(
  825. chat_id=chat_id, chat_type=chat_type, user_id=user_id, user_name=user_name, thread_id=None, message_id=str(message_id))
  826. def _telegram_auth_env_configured(self) -> bool:
  827. """Return True when Telegram auth env vars make an early decision safe."""
  828. keys = (
  829. "TELEGRAM_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_CHATS",
  830. "TELEGRAM_ALLOW_ALL_USERS", "GATEWAY_ALLOWED_USERS", "GATEWAY_ALLOW_ALL_USERS")
  831. return any(_scoped_gate_env(key).strip() for key in keys)
  832. def _should_pass_unauthorized_dm_for_pairing(self, source) -> bool:
  833. """True when an unauthorized DM must still reach the gateway for an outbound reply
  834. (``unauthorized_dm_behavior`` resolves to anything but ``ignore`` — a pairing code or a
  835. one-time decline — incl. an allowlist plus an explicit platform override)."""
  836. if source.chat_type != "dm":
  837. return False
  838. # Bound-handler ``__self__`` is None under multiplex; ``gateway_runner`` survives that wrapping.
  839. runner = getattr(getattr(self, "_message_handler", None), "__self__", None) or getattr(self, "gateway_runner", None)
  840. behavior_fn = getattr(runner, "_get_unauthorized_dm_behavior", None)
  841. if callable(behavior_fn):
  842. try:
  843. profile = getattr(source, "profile", None) or getattr(self, "_owner_profile", None)
  844. return behavior_fn(Platform.TELEGRAM, profile=profile) != "ignore"
  845. except Exception:
  846. logger.debug("[Telegram] Failed to resolve unauthorized DM behavior; falling back to adapter-local override", exc_info=True)
  847. extra = getattr(getattr(self, "config", None), "extra", None) or {}
  848. return str(extra.get("unauthorized_dm_behavior", "")).strip().lower() in ("pair", "decline")
  849. def _is_user_authorized_from_message(self, message: Message) -> bool:
  850. """Intake auth prefilter, run BEFORE batching/event construction/group observation.
  851. Only rejects when it can make the same context-aware decision the runner would; unknown DMs pass through when
  852. there is no allowlist or pairing is the unauthorized-DM behavior."""
  853. source = self._source_from_message_for_auth(message)
  854. user_id = source.user_id
  855. # No identity → service message or channel post without sender_chat; defer to message gating.
  856. if not user_id:
  857. return True
  858. authorized: Optional[bool] = None
  859. # Adapter-level allow_from (DMs) / group_allow_from (groups) are the sole authority if set.
  860. adapter_allow_from = self.config.extra.get(
  861. "group_allow_from" if (source.chat_type or "") in ("group", "forum", "channel") else "allow_from")
  862. if adapter_allow_from is not None:
  863. allowed = _coerce_allow_set(adapter_allow_from)
  864. authorized = user_id in allowed or "*" in allowed
  865. # Instance-level override only (tests): the class method _is_callback_user_authorized is for
  866. # inline buttons and must not become a user-id-only shortcut for real messages.
  867. if authorized is None:
  868. callback_auth = self.__dict__.get("_is_callback_user_authorized")
  869. if callable(callback_auth):
  870. with contextlib.suppress(Exception):
  871. authorized = bool(callback_auth(
  872. user_id, chat_id=source.chat_id, chat_type=source.chat_type, thread_id=source.thread_id,
  873. user_name=source.user_name))
  874. if authorized is None:
  875. # Runner's full auth chain; prefer the set_authorization_check callback (survives multiplex
  876. # handler wrapping, unlike bound-handler __self__).
  877. auth_fn = self._legacy_runner_auth_fn()
  878. has_callback = getattr(self, "_authorization_check", None) is not None
  879. if has_callback or auth_fn is not None:
  880. # No allowlist → unknown DMs must reach pairing, not be default-denied here.
  881. if not self._telegram_auth_env_configured():
  882. return True
  883. decision = self._is_sender_authorized(
  884. user_id, chat_type=source.chat_type, chat_id=source.chat_id, is_bot=source.is_bot,
  885. thread_id=source.thread_id) if has_callback else None
  886. if decision is not None:
  887. authorized = decision
  888. elif auth_fn is not None:
  889. try:
  890. authorized = bool(auth_fn(source))
  891. except Exception:
  892. logger.debug("[Telegram] Falling back to env-only auth for user %s", user_id, exc_info=True)
  893. if authorized is None:
  894. authorized = self._env_allowlist_decision(user_id)
  895. if authorized is None:
  896. return True
  897. if authorized:
  898. return True
  899. # Unauthorized DM the gateway would pair: forward so pairing can run.
  900. return self._should_pass_unauthorized_dm_for_pairing(source)
  901. @classmethod
  902. def _metadata_thread_id(cls, metadata: Optional[dict[str, Any]]) -> Optional[str]:
  903. thread_id = (metadata or {}).get("thread_id") or (metadata or {}).get("message_thread_id")
  904. return str(thread_id) if thread_id is not None else None
  905. @classmethod
  906. def _metadata_direct_messages_topic_id(cls, metadata: Optional[dict[str, Any]]) -> Optional[str]:
  907. topic_id = (metadata or {}).get("direct_messages_topic_id") or (metadata or {}).get("telegram_direct_messages_topic_id")
  908. return str(topic_id) if topic_id is not None else None
  909. @classmethod
  910. def _metadata_reply_to_message_id(cls, metadata: Optional[dict[str, Any]]) -> Optional[int]:
  911. return parse_telegram_message_id((metadata or {}).get("telegram_reply_to_message_id"))
  912. @staticmethod
  913. def _dm_topic_fallback(metadata: Optional[dict[str, Any]]) -> bool:
  914. """True for Hermes private-chat topic lanes (``telegram_dm_topic_reply_fallback``)."""
  915. return bool(metadata and metadata.get("telegram_dm_topic_reply_fallback"))
  916. @classmethod
  917. def _is_private_dm_topic_send(cls, chat_id: str, thread_id: Optional[str], metadata: Optional[dict[str, Any]]) -> bool:
  918. if cls._metadata_direct_messages_topic_id(metadata) is not None:
  919. return cls._dm_topic_fallback(metadata) and cls._metadata_reply_to_message_id(metadata) is not None
  920. if metadata and metadata.get("telegram_dm_topic_created_for_send"):
  921. return False
  922. return bool(thread_id) and cls._dm_topic_fallback(metadata)
  923. @staticmethod
  924. def _dm_topic_missing_anchor_error() -> str:
  925. return "Telegram DM topic delivery requires a reply anchor; refusing to send outside the requested topic"
  926. @classmethod
  927. def _reply_to_message_id_for_send(
  928. cls, reply_to: Optional[str], metadata: Optional[dict[str, Any]] = None, reply_to_mode: Optional[str] = None) -> Optional[int]:
  929. if (anchor := parse_telegram_message_id(reply_to)) is not None:
  930. return anchor
  931. if cls._dm_topic_fallback(metadata) and reply_to_mode != "off":
  932. return cls._metadata_reply_to_message_id(metadata)
  933. return None
  934. @classmethod
  935. def _thread_kwargs_for_send(
  936. cls, chat_id: str, thread_id: Optional[str], metadata: Optional[dict[str, Any]] = None,
  937. reply_to_message_id: Optional[int] = None, reply_to_mode: Optional[str] = None) -> dict[str, Any]:
  938. """Telegram send kwargs for forum and direct-message topic routing.
  939. Forum topics use ``message_thread_id``; native Bot API DM topics opt in via explicit ``direct_messages_topic_id``
  940. metadata; Hermes private-chat topic lanes are marked ``telegram_dm_topic_reply_fallback``. Anchor-less synthetic sends
  941. prefer the Hermes topic's ``message_thread_id`` (the native DM-topic id renders in a different chat lane).
  942. ``reply_to_mode="off"`` suppresses the anchor but keeps ``message_thread_id``.
  943. Live replies send the private topic thread id together with a reply anchor. Synthetic/resumed sends
  944. without an anchor (loop wakeups, background-process notifications, queued follow-ups after a gateway
  945. restart) prefer the Hermes topic's ``message_thread_id`` so they stay in the active topic lane
  946. (#87051); ``direct_messages_topic_id`` is only used when no topic thread resolves, since the native
  947. DM-topic id does not match the Hermes topic lane and can render the message in a different chat
  948. lane.
  949. """
  950. fallback = cls._dm_topic_fallback(metadata)
  951. if fallback and reply_to_mode != "off":
  952. if reply_to_message_id is None:
  953. reply_to_message_id = cls._metadata_reply_to_message_id(metadata)
  954. if reply_to_message_id is None:
  955. # Anchor-less synthetic send: prefer the Hermes topic thread id (see docstring).
  956. # Anchor-less synthetic sends (loop wakeups, watch notifications, restart-resumed
  957. # follow-ups) must stay in the active topic lane: prefer the Hermes topic thread id when it
  958. # resolves (#87051). Routing via direct_messages_topic_id here sent these to a different
  959. # lane than the topic the session runs in.
  960. thread_message_id = cls._message_thread_id_for_send(thread_id)
  961. if thread_message_id is not None:
  962. return {"message_thread_id": thread_message_id}
  963. return cls._direct_topic_kwargs(metadata) or {}
  964. elif not fallback:
  965. direct_kwargs = cls._direct_topic_kwargs(metadata)
  966. if direct_kwargs is not None:
  967. return direct_kwargs
  968. return {"message_thread_id": cls._message_thread_id_for_send(thread_id)}
  969. @classmethod
  970. def _direct_topic_kwargs(cls, metadata: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
  971. """Native Bot API DM-topic routing kwargs, or None when no ``direct_messages_topic_id``."""
  972. direct_topic_id = cls._metadata_direct_messages_topic_id(metadata)
  973. if direct_topic_id is None:
  974. return None
  975. return {"message_thread_id": None, "direct_messages_topic_id": int(direct_topic_id)}
  976. def _thread_kwargs_for_draft(self, chat_id: str, metadata: Optional[dict[str, Any]]) -> dict[str, Any]:
  977. """Routing kwargs for ``sendMessageDraft`` / ``sendRichMessageDraft`` (integer
  978. ``message_thread_id`` for DM topics — Telegram rejects the raw string ``thread_id``)."""
  979. kwargs = self._thread_kwargs_for_send(
  980. chat_id, self._metadata_thread_id(metadata), metadata, reply_to_message_id=self._reply_to_message_id_for_send(None, metadata),
  981. reply_to_mode=getattr(self, "_reply_to_mode", None))
  982. return {k: v for k, v in kwargs.items() if v is not None}
  983. @classmethod
  984. def _message_thread_id_for_send(cls, thread_id: Optional[str]) -> Optional[int]:
  985. if not thread_id or str(thread_id) == cls._GENERAL_TOPIC_THREAD_ID:
  986. return None
  987. return int(thread_id)
  988. @classmethod
  989. def _message_thread_id_for_typing(cls, thread_id: Optional[str]) -> Optional[int]:
  990. # Deliberately asymmetric with _message_thread_id_for_send: sendMessage rejects message_thread_id=1
  991. # (forum General), but sendChatAction NEEDS it to place the typing bubble in General.
  992. return int(thread_id) if thread_id else None
  993. @staticmethod
  994. def _is_thread_not_found_error(error: Exception) -> bool:
  995. return "thread not found" in str(error).lower()
  996. def _prune_stale_dm_topic_binding(self, chat_id: Any, thread_id: Any, *, metadata: Optional[dict[str, Any]] = None) -> None:
  997. """Drop the stale ``telegram_dm_topic_bindings`` row for a topic Telegram confirmed deleted, else
  998. ``_recover_telegram_topic_thread_id`` keeps steering inbound to the dead thread. Best-effort.
  999. Rows are namespaced by profile: the send's ``hermes_profile`` wins over the adapter's stamp.
  1000. Without this prune the recovery logic in ``gateway.run._recover_telegram_topic_thread_id`` keeps
  1001. steering future inbound messages to the dead thread (the bug behind #31501 — tool progress,
  1002. approvals, replies all end up in the wrong place even though the user has moved on to a fresh
  1003. topic). Best-effort: we never raise from a send-fallback path — a failed cleanup must not turn into
  1004. a failed user-facing send.
  1005. Under ``gateway.profile_routes`` the transport adapter may not be the profile that wrote the
  1006. binding, so the send's ``hermes_profile`` metadata wins over the adapter's own profile stamp;
  1007. single-profile bots fall back to ``"default"``. See #76423.
  1008. """
  1009. if chat_id is None or thread_id is None:
  1010. return
  1011. db = getattr(getattr(self, "_session_store", None), "_db", None)
  1012. if db is None or not hasattr(db, "delete_telegram_topic_binding"):
  1013. return
  1014. try:
  1015. profile_name = (metadata or {}).get("hermes_profile") or getattr(self, "_hermes_profile_name", None) or "default"
  1016. removed = db.delete_telegram_topic_binding(chat_id=str(chat_id), thread_id=str(thread_id), profile_name=profile_name)
  1017. except Exception:
  1018. logger.debug(
  1019. "[%s] delete_telegram_topic_binding failed for chat=%s thread=%s — skipping prune",
  1020. self.name, chat_id, thread_id, exc_info=True)
  1021. return
  1022. if removed:
  1023. logger.info(
  1024. "[%s] Pruned stale Telegram DM topic binding chat=%s thread=%s (Bot API: thread not found)", self.name, chat_id, thread_id)
  1025. @staticmethod
  1026. def _is_bad_request_error(error: Exception) -> bool:
  1027. name = error.__class__.__name__.lower()
  1028. if name == "badrequest" or name.endswith("badrequest"):
  1029. return True
  1030. try:
  1031. from telegram.error import BadRequest
  1032. return isinstance(error, BadRequest)
  1033. except ImportError:
  1034. return False
  1035. @classmethod
  1036. def _should_retry_without_dm_topic_reply_anchor(
  1037. cls, error: Exception, metadata: Optional[dict[str, Any]], reply_to_message_id: Optional[int]) -> bool:
  1038. """True when a DM-topic send should be retried with routing stripped: (1) stale anchor — reply
  1039. target deleted; (2) anchor-less synthetic send whose ``direct_messages_topic_id`` Bot API rejects.
  1040. 2. The synthetic-event case (added when #27937 introduced ``direct_messages_topic_id`` fallback for
  1041. sends without an anchor): if Bot API rejects the topic id itself with any BadRequest that mentions
  1042. topic/thread routing, we retry without routing rather than dropping the message.
  1043. """
  1044. if not cls._dm_topic_fallback(metadata) or not cls._is_bad_request_error(error):
  1045. return False
  1046. err_lower = str(error).lower()
  1047. if reply_to_message_id is not None and "message to be replied not found" in err_lower:
  1048. return True
  1049. if not metadata.get("direct_messages_topic_id"): # topic id rejected → plain DM send
  1050. return False
  1051. topic_markers = (
  1052. "direct_messages_topic", "message thread not found", "thread not found", "topic_closed", "topic_deleted", "topic not found")
  1053. return any(marker in err_lower for marker in topic_markers)
  1054. async def _send_with_dm_topic_reply_anchor_retry(
  1055. self, send_fn: Any, send_kwargs: dict[str, Any], metadata: Optional[dict[str, Any]],
  1056. reply_to_message_id: Optional[int], media_label: str, reset_media: Optional[Any] = None) -> Any:
  1057. """Retry stale private-topic media replies once without the topic anchor. Serialized per chat with
  1058. ``send()`` so a file upload cannot land between two chunks of the text it accompanies."""
  1059. async with self._chat_send_lock(send_kwargs.get("chat_id")):
  1060. try:
  1061. return await _await_with_thread_deadline(
  1062. send_fn(**send_kwargs), timeout=_MEDIA_SEND_DEADLINE, label="telegram-media-send", dump_on_blocked_loop=False)
  1063. except Exception as send_err:
  1064. if not self._should_retry_without_dm_topic_reply_anchor(send_err, metadata, reply_to_message_id):
  1065. raise
  1066. logger.warning(
  1067. "[%s] Reply target deleted for Telegram %s, retrying without reply/topic anchor: %s",
  1068. self.name, media_label, _redact_telegram_error_text(send_err))
  1069. if reset_media is not None:
  1070. reset_media()
  1071. retry_kwargs = dict(send_kwargs)
  1072. retry_kwargs["reply_to_message_id"] = None
  1073. retry_kwargs.pop("message_thread_id", None)
  1074. retry_kwargs.pop("direct_messages_topic_id", None)
  1075. return await _await_with_thread_deadline(
  1076. send_fn(**retry_kwargs), timeout=_MEDIA_SEND_DEADLINE, label="telegram-media-send", dump_on_blocked_loop=False)
  1077. def _fallback_ips(self) -> list[str]:
  1078. """Return validated fallback IPs from config (populated by _apply_env_overrides)."""
  1079. configured = self.config.extra.get("fallback_ips", []) if getattr(self.config, "extra", None) else []
  1080. if isinstance(configured, str):
  1081. configured = configured.split(",")
  1082. return parse_fallback_ip_env(",".join(str(v) for v in configured) if configured else None)
  1083. @staticmethod
  1084. def _looks_like_polling_conflict(error: Exception) -> bool:
  1085. text = str(error).lower()
  1086. return (
  1087. error.__class__.__name__.lower() == "conflict"
  1088. or "terminated by other getupdates request" in text
  1089. or "another bot instance is running" in text)
  1090. @staticmethod
  1091. def _looks_like_auth_error(error: Exception) -> bool:
  1092. """True for terminal credential failures (InvalidToken, Forbidden) → retryable=False. Type-based
  1093. only, never message text; BadRequest/RetryAfter are transient at connect time."""
  1094. if error.__class__.__name__.lower() in {"invalidtoken", "forbidden"}:
  1095. return True
  1096. try:
  1097. from telegram.error import Forbidden, InvalidToken
  1098. return isinstance(error, (InvalidToken, Forbidden))
  1099. except ImportError:
  1100. return False
  1101. @staticmethod
  1102. def _looks_like_network_error(error: Exception) -> bool:
  1103. """Return True for transient transport failures that warrant reconnect."""
  1104. name = error.__class__.__name__.lower()
  1105. if name in {"badrequest", "invalidtoken", "forbidden", "retryafter"}:
  1106. return False
  1107. if name in {"networkerror", "timedout", "connectionerror"}:
  1108. return True
  1109. try:
  1110. from telegram.error import BadRequest, Forbidden, InvalidToken, NetworkError, RetryAfter, TimedOut
  1111. if isinstance(error, (BadRequest, InvalidToken, Forbidden, RetryAfter)):
  1112. return False
  1113. if isinstance(error, (NetworkError, TimedOut)):
  1114. return True
  1115. except ImportError:
  1116. pass
  1117. return isinstance(error, OSError)
  1118. @staticmethod
  1119. def _exception_graph_matches(error: Exception, name_marker: str, *text_markers: str) -> bool:
  1120. """True when any exception in ``error``'s cause/context graph matches by class name or text."""
  1121. for cur in _iter_exception_graph(error):
  1122. text = str(cur).lower()
  1123. if name_marker in cur.__class__.__name__.lower() or any(m in text for m in text_markers):
  1124. return True
  1125. return False
  1126. @classmethod
  1127. def _looks_like_connect_timeout(cls, error: Exception) -> bool:
  1128. """True when a TimedOut wraps a ConnectTimeout: TCP never connected, so re-sending is safe
  1129. (a plain TimedOut may have reached Telegram and must not be re-sent)."""
  1130. return cls._exception_graph_matches(error, "connecttimeout", "connect timeout", "connect timed out")
  1131. @staticmethod
  1132. def _looks_like_pool_timeout(error: Exception) -> bool:
  1133. """True when a TimedOut wraps ``httpx.PoolTimeout``: PTB says "Request was *not* sent", so
  1134. re-sending cannot duplicate. Matches class AND text to survive rewording."""
  1135. for cur in _iter_exception_graph(error):
  1136. name = cur.__class__.__name__.lower()
  1137. text = str(cur).lower()
  1138. if "pooltimeout" in name or "pool timeout" in text or ("connection pool" in text and "occupied" in text):
  1139. return True
  1140. return False
  1141. def _coerce_bool_extra(self, key: str, default: bool = False) -> bool:
  1142. value = self.config.extra.get(key) if getattr(self.config, "extra", None) else None
  1143. if value is None:
  1144. return default
  1145. if isinstance(value, str):
  1146. lowered = value.strip().lower()
  1147. if lowered in {"true", "1", "yes", "on"}:
  1148. return True
  1149. if lowered in {"false", "0", "no", "off"}:
  1150. return False
  1151. return default
  1152. return bool(value)
  1153. def _link_preview_kwargs(self) -> dict[str, Any]:
  1154. if not getattr(self, "_disable_link_previews", False):
  1155. return {}
  1156. if LinkPreviewOptions is not None:
  1157. return {"link_preview_options": LinkPreviewOptions(is_disabled=True)}
  1158. return {"disable_web_page_preview": True}
  1159. # --- Bot API 10.1 Rich Messages (sendRichMessage): final/new-message replies opportunistically send
  1160. # RAW agent markdown so tables, task lists, <details>, math render natively; legacy MarkdownV2 send()
  1161. # is the fallback. Streaming edits stay on the MarkdownV2 edit path.
  1162. def _content_fits_rich_limits(self, content: str) -> bool:
  1163. """Pre-check the 32,768-char cap only; other rich limits surface as BadRequest (permanent)."""
  1164. return len(content) <= self.RICH_MESSAGE_MAX_CHARS
  1165. def _bot_supports_rich(self) -> bool:
  1166. """True when ``do_api_request`` is an *async* callable (real Bot or AsyncMock); plain MagicMock
  1167. and SimpleNamespace bots resolve False → legacy path."""
  1168. return inspect.iscoroutinefunction(getattr(self._bot, "do_api_request", None))
  1169. _RICH_DETAILS_RE = re.compile(r"<details\b[^>]*>.*?</details>", re.IGNORECASE | re.DOTALL)
  1170. _RICH_MATH_IN_DETAILS_RE = re.compile(
  1171. r"(\$\$.*?\$\$|\\\[.*?\\\]|\\\(.*?\\\)|"
  1172. r"\\(?:sum|frac|alpha|beta|gamma|delta|theta|lambda|mu|pi|sigma|"
  1173. r"int|prod|sqrt|lim|infty|begin\{(?:equation|align|matrix|cases)\}))",
  1174. re.IGNORECASE | re.DOTALL)
  1175. # Hiragana/Katakana, CJK Ext A, CJK Unified, Hangul, CJK Compatibility, CJK ext/compat supplement.
  1176. _RICH_CJK_RE = re.compile("[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff\U00020000-\U000323af]")
  1177. def _has_telegram_desktop_details_math_crash_shape(self, content: str) -> bool:
  1178. """Math inside <details> crashes Telegram Desktop 6.9.1 (tdesktop#30808); the Bot API accepts
  1179. the payload, so rich delivery must be skipped up front."""
  1180. if not content:
  1181. return False
  1182. return any(self._RICH_MATH_IN_DETAILS_RE.search(block) for block in self._RICH_DETAILS_RE.findall(content))
  1183. def _has_telegram_desktop_cjk_rich_garble_shape(self, content: str) -> bool:
  1184. """True for CJK content: Telegram Mac/Desktop rich rendering leaves overlapping glyphs.
  1185. Telegram Mac/Desktop Bot API 10.1 rich-message rendering currently leaves overlapping draft/overlay
  1186. glyph artifacts for CJK text (#47653). The legacy MarkdownV2 path renders the same text cleanly, so
  1187. skip rich delivery up front until affected clients age out.
  1188. """
  1189. return bool(content and self._RICH_CJK_RE.search(content))
  1190. def _needs_rich_rendering(self, content: str) -> bool:
  1191. """True for constructs MarkdownV2 degrades: pipe tables, task lists, <details>, block math.
  1192. Ordinary replies stay on MarkdownV2 so clients render consistent font weight/spacing.
  1193. The rich endpoint is reserved for constructs where raw markdown materially improves output: pipe
  1194. tables (MarkdownV2 has no table syntax and rewrites them into bullet lists), GFM task lists,
  1195. collapsible ``<details>`` blocks, and block math. Adapted from #45995 (@YonganZhang).
  1196. """
  1197. if not content:
  1198. return False
  1199. if any(_TABLE_SEPARATOR_RE.match(line) for line in content.splitlines()):
  1200. return True
  1201. if re.search(r"(?m)^\s*[-*]\s+\[[ xX]\]\s+", content):
  1202. return True
  1203. if re.search(r"(?m)^<details\b|^</details>|^<summary\b|^</summary>", content):
  1204. return True
  1205. return "$$" in content
  1206. def _rich_delivery_enabled(self) -> bool:
  1207. """Whether rich delivery is allowed (``rich_messages`` opt-in)."""
  1208. return bool(getattr(self, "_rich_messages_enabled", True))
  1209. def _rich_content_ok(self, content: str) -> bool:
  1210. """Shape checks shared by rich sends and rich drafts (non-blank, no Desktop crash/garble
  1211. shapes, under the cap, async-capable bot)."""
  1212. return bool(
  1213. content and content.strip()
  1214. and not self._has_telegram_desktop_details_math_crash_shape(content)
  1215. and (
  1216. getattr(self, "_allow_cjk_rich_messages", False)
  1217. or not self._has_telegram_desktop_cjk_rich_garble_shape(content)
  1218. )
  1219. and self._content_fits_rich_limits(content)
  1220. and self._bot_supports_rich())
  1221. def _rich_eligible(self, content: str) -> bool:
  1222. """Rich eligibility ignoring ``expect_edits`` (a streamed preview's FINAL edit still upgrades)."""
  1223. return bool(
  1224. self._rich_delivery_enabled()
  1225. and not getattr(self, "_rich_send_disabled", False)
  1226. and content and content.strip()
  1227. and self._needs_rich_rendering(content)
  1228. and self._rich_content_ok(content))
  1229. def _should_attempt_rich(self, content: str, metadata: Optional[dict[str, Any]] = None) -> bool:
  1230. return bool(not (metadata or {}).get("expect_edits") and self._rich_eligible(content))
  1231. def prefers_fresh_final_streaming(self, content: str, metadata: Optional[dict[str, Any]] = None) -> bool:
  1232. """Replace a streamed preview with a fresh rich final — DM topics only. Root DMs stay off (a live
  1233. draft has no preview id); DM *topics* degrade to edit-in-place whose MarkdownV2 preview Telegram
  1234. refuses to rich-edit, so a fresh sendRichMessage + delete is the only way to keep native tables.
  1235. Root DMs keep this off (#46206 / #47048): successful draft streaming has no preview ``message_id``,
  1236. so the hook is not consulted, and in-place ``editMessageText.rich_message`` would duplicate a live
  1237. draft turn. Private DM *topics* often reject ``sendMessageDraft``; the consumer then degrades to
  1238. edit-in-place. Telegram rejects a rich edit of that plain MarkdownV2 preview, and the fallback
  1239. formatter permanently turns pipe tables into bullet lists.
  1240. """
  1241. metadata = metadata or {}
  1242. if not (metadata.get("telegram_dm_topic_reply_fallback") or self._metadata_direct_messages_topic_id(metadata)):
  1243. return False
  1244. return self._rich_eligible(content)
  1245. def _rich_transport_available(self) -> bool:
  1246. return bool(
  1247. getattr(self, "_rich_messages_enabled", True) and not getattr(self, "_rich_send_disabled", False) and self._bot_supports_rich())
  1248. def streaming_overflow_limit(self) -> Optional[int]:
  1249. """Let the stream consumer accumulate up to the rich cap so a reply that fits one sendRichMessage
  1250. isn't fragmented at 4,096. None (→ legacy limit) if rich is unavailable."""
  1251. return self.RICH_MESSAGE_MAX_CHARS if self._rich_transport_available() else None
  1252. def _rich_message_payload(self, content: str, *, skip_entity_detection: bool = False) -> dict[str, Any]:
  1253. """``InputRichMessage`` from RAW markdown — never ``format_message(content)``, whose MarkdownV2
  1254. escaping destroys table pipes."""
  1255. payload: dict[str, Any] = {"markdown": _rich_normalize_linebreaks(content)}
  1256. if skip_entity_detection:
  1257. payload["skip_entity_detection"] = True
  1258. return payload
  1259. def _is_rich_capability_error(self, exc: Exception) -> bool:
  1260. """True ⇒ the rich endpoint itself is unavailable (old PTB/server); latches rich off.
  1261. Per-message BadRequests (parser/limit) are NOT capability errors."""
  1262. if exc.__class__.__name__.lower() in {"endpointnotfound", "invalidtoken"}:
  1263. return True
  1264. if isinstance(exc, (AttributeError, TypeError, NotImplementedError)) or getattr(exc, "error_code", None) == 404:
  1265. return True
  1266. s = str(exc).lower()
  1267. if ("method" in s or "endpoint" in s) and ("not found" in s or "does not exist" in s):
  1268. return True
  1269. return "no such method" in s
  1270. def _is_rich_fallback_error(self, exc: Exception) -> bool:
  1271. """True ⇒ permanent/capability error ⇒ safe to fall back to legacy. Conservative: anything not
  1272. clearly permanent is transient — the rich request may have reached Telegram (duplicate risk)."""
  1273. if self._is_bad_request_error(exc) or self._is_rich_capability_error(exc):
  1274. return True
  1275. s = str(exc).lower()
  1276. return "unsupported" in s or "not implemented" in s
  1277. def _chunk_reply_routing(
  1278. self, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]], thread_id: Optional[str], index: int) -> tuple:
  1279. """Reply-anchor routing for chunk ``index``: ``(private_dm_topic_send, anchor_off, reply_to_id)``.
  1280. ``anchor_off``: reply_to_mode="off" on the DM-topic fallback path opts into "message_thread_id
  1281. alone is enough" — don't fail loud because the anchor was suppressed by config."""
  1282. metadata_reply_to = self._metadata_reply_to_message_id(metadata)
  1283. private_dm_topic_send = self._is_private_dm_topic_send(chat_id, thread_id, metadata)
  1284. dm_topic_reply_to_off = private_dm_topic_send and self._reply_to_mode == "off" and self._dm_topic_fallback(metadata)
  1285. reply_to_source = next((str(i) for i in (parse_telegram_message_id(reply_to), metadata_reply_to if private_dm_topic_send else None) if i is not None), None)
  1286. if private_dm_topic_send:
  1287. should_thread = reply_to_source is not None and self._reply_to_mode != "off"
  1288. else:
  1289. should_thread = self._should_thread_reply(reply_to_source, index)
  1290. reply_to_id = int(reply_to_source) if should_thread and reply_to_source else None
  1291. return private_dm_topic_send, dm_topic_reply_to_off, reply_to_id
  1292. def _compute_single_send_routing(
  1293. self, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]], thread_id: Optional[str]) -> Optional[tuple]:
  1294. """Routing for a single (rich) send — mirrors send()'s index-0 block. Returns ``(reply_to_id,
  1295. thread_kwargs)`` or ``None`` = skip rich, legacy owns the DM-topic fail-loud SendResult."""
  1296. private_dm_topic_send, dm_topic_reply_to_off, reply_to_id = self._chunk_reply_routing(chat_id, reply_to, metadata, thread_id, 0)
  1297. thread_kwargs = self._thread_kwargs_for_send(
  1298. chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
  1299. # Synthetic/resumed sends via direct_messages_topic_id need no reply anchor.
  1300. if (
  1301. private_dm_topic_send and reply_to_id is None and not dm_topic_reply_to_off
  1302. and not thread_kwargs.get("direct_messages_topic_id")):
  1303. return None
  1304. return reply_to_id, thread_kwargs
  1305. @staticmethod
  1306. def _is_timed_out(exc: Exception) -> bool:
  1307. """PTB ``TimedOut`` (when importable) or a "timed out" message."""
  1308. try:
  1309. from telegram.error import TimedOut as _TimedOut
  1310. except (ImportError, AttributeError):
  1311. _TimedOut = None
  1312. return bool((_TimedOut and isinstance(exc, _TimedOut)) or "timed out" in str(exc).lower())
  1313. def _rich_transient_result(self, exc: Exception, what: str, *, retry_after: Any = None) -> SendResult:
  1314. """SendResult for a transient/unknown rich-API failure (request may have reached Telegram, so the
  1315. caller must NOT legacy-resend); retry semantics mirror legacy send()."""
  1316. safe_error = _redact_telegram_error_text(exc)
  1317. logger.warning("[%s] %s transient failure (no legacy resend): %s", self.name, what, safe_error)
  1318. return SendResult(
  1319. success=False, error=safe_error,
  1320. retryable=(self._looks_like_connect_timeout(exc) or not self._is_timed_out(exc)), retry_after=retry_after)
  1321. @staticmethod
  1322. async def _record_rich_sent(chat_id: Any, message_id: Any, content: str) -> None:
  1323. """Index rich content we sent: Telegram won't echo it back in reply_to_message.
  1324. Awaited so the store's read-modify-write + ``os.replace`` runs off the loop."""
  1325. try:
  1326. from gateway import rich_sent_store
  1327. await rich_sent_store.record_async(str(chat_id), str(message_id), content)
  1328. except Exception:
  1329. pass
  1330. async def _try_send_rich(
  1331. self, chat_id: str, content: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> Optional[SendResult]:
  1332. """Attempt a single ``sendRichMessage``. Returns a SendResult (success, or a transient failure the
  1333. caller must NOT legacy-resend), or ``None`` = fall back to legacy MarkdownV2."""
  1334. thread_id = self._metadata_thread_id(metadata)
  1335. routing = self._compute_single_send_routing(chat_id, reply_to, metadata, thread_id)
  1336. if routing is None:
  1337. return None
  1338. reply_to_id, thread_kwargs = routing
  1339. payload = self._rich_payload_base(chat_id, content)
  1340. # Only non-None routing keys: direct_messages_topic_id is paired with message_thread_id=None.
  1341. payload.update({k: v for k, v in thread_kwargs.items() if v is not None})
  1342. payload.update(self._notification_kwargs(metadata))
  1343. if reply_to_id is not None:
  1344. # sendRichMessage takes reply_parameters, NOT reply_to_message_id (silently ignored → anchor dropped).
  1345. payload["reply_parameters"] = {"message_id": reply_to_id}
  1346. try:
  1347. # Raw Bot API result: return_type=Message would make PTB deserialize a 10.1 shape it doesn't
  1348. # fully model; a post-delivery parse error ≠ send failure.
  1349. msg = await _await_with_thread_deadline(
  1350. self._bot.do_api_request("sendRichMessage", api_kwargs=payload),
  1351. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  1352. except Exception as exc:
  1353. if self._rich_rejected(exc, "sendRichMessage", "MarkdownV2"):
  1354. return None
  1355. # Honor Telegram's flood-control retry_after over the base retry schedule.
  1356. _retry_after = getattr(exc, "retry_after", None)
  1357. if _retry_after is None:
  1358. _m = re.search(r"retry\s+(?:in\s+)?(\d+)", str(exc).lower(), re.IGNORECASE)
  1359. if _m:
  1360. _retry_after = float(_m.group(1))
  1361. return self._rich_transient_result(exc, "sendRichMessage", retry_after=_retry_after)
  1362. if isinstance(msg, dict):
  1363. message_id = msg.get("message_id")
  1364. if message_id is None:
  1365. message_id = (msg.get("result") or {}).get("message_id")
  1366. else:
  1367. message_id = getattr(msg, "message_id", None)
  1368. if message_id is not None:
  1369. await self._record_rich_sent(chat_id, message_id, content)
  1370. return SendResult(success=True, message_id=str(message_id) if message_id is not None else None)
  1371. def _rich_payload_base(self, chat_id: str, content: str) -> dict[str, Any]:
  1372. payload: dict[str, Any] = {"chat_id": normalize_telegram_chat_id(chat_id), "rich_message": self._rich_message_payload(content)}
  1373. if getattr(self, "_disable_link_previews", False):
  1374. payload["link_preview_options"] = {"is_disabled": True}
  1375. return payload
  1376. def _rich_rejected(self, exc: Exception, what: str, fallback: str) -> bool:
  1377. """True for a permanent/capability rich-API failure (caller falls back to legacy); capability
  1378. errors latch rich off so no doomed roundtrip repeats per send."""
  1379. if not self._is_rich_fallback_error(exc):
  1380. return False
  1381. if self._is_rich_capability_error(exc):
  1382. self._rich_send_disabled = True
  1383. logger.debug("[%s] %s rejected (%s) — falling back to %s", self.name, what, _redact_telegram_error_text(exc), fallback)
  1384. return True
  1385. async def _try_edit_rich(
  1386. self, chat_id: str, message_id: str, content: str, metadata: Optional[dict[str, Any]] = None) -> Optional[SendResult]:
  1387. """Edit a message in place as rich (``editMessageText`` + ``rich_message``) so a streamed preview
  1388. finalizes without send+delete. Same contract as :meth:`_try_send_rich`."""
  1389. # No topic routing on edits: message_thread_id/direct_messages_topic_id make Telegram reject it.
  1390. payload = {**self._rich_payload_base(chat_id, content), "message_id": int(message_id)}
  1391. try:
  1392. await _await_with_thread_deadline(
  1393. self._bot.do_api_request("editMessageText", api_kwargs=payload),
  1394. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  1395. except Exception as exc:
  1396. # "Message is not modified" = successful no-op; skip the redundant legacy edit.
  1397. if "not modified" in str(exc).lower():
  1398. if self._is_rich_fallback_error(exc) and self._is_rich_capability_error(exc):
  1399. self._rich_send_disabled = True
  1400. return SendResult(success=True, message_id=message_id)
  1401. if self._rich_rejected(exc, "rich editMessageText", "MarkdownV2 edit"):
  1402. return None
  1403. return self._rich_transient_result(exc, "rich editMessageText")
  1404. # Mirror the fresh-send index: a streamed final finalized via edit is otherwise never recorded.
  1405. await self._record_rich_sent(chat_id, message_id, content)
  1406. return SendResult(success=True, message_id=message_id)
  1407. def _should_attempt_rich_draft(self, content: str) -> bool:
  1408. return bool(
  1409. getattr(self, "_rich_messages_enabled", True)
  1410. and getattr(self, "_rich_drafts_enabled", False)
  1411. and not getattr(self, "_rich_send_disabled", False)
  1412. and not getattr(self, "_rich_draft_disabled", False)
  1413. and self._rich_content_ok(content))
  1414. async def _try_send_rich_draft(self, chat_id: str, draft_id: int, content: str, metadata: Optional[dict[str, Any]]) -> bool:
  1415. """Emit one ``sendRichMessageDraft`` frame; True on success. Frames are ephemeral, so any failure
  1416. returns False and the caller renders the legacy draft; capability failures latch off."""
  1417. payload: dict[str, Any] = {
  1418. "chat_id": normalize_telegram_chat_id(chat_id), "draft_id": int(draft_id), "rich_message": self._rich_message_payload(content)}
  1419. payload.update(self._thread_kwargs_for_draft(chat_id, metadata))
  1420. try:
  1421. return bool(await _await_with_thread_deadline(
  1422. self._bot.do_api_request("sendRichMessageDraft", api_kwargs=payload),
  1423. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False))
  1424. except Exception as exc:
  1425. if self._is_rich_capability_error(exc):
  1426. self._rich_draft_disabled = True
  1427. logger.debug(
  1428. "[%s] sendRichMessageDraft unsupported (%s) — using legacy drafts", self.name, _redact_telegram_error_text(exc))
  1429. else:
  1430. logger.debug(
  1431. "[%s] sendRichMessageDraft transient failure (%s) — legacy draft this frame", self.name,
  1432. _redact_telegram_error_text(exc))
  1433. return False
  1434. async def _drain_polling_connections(self) -> None:
  1435. """Reset the httpx pool used for getUpdates polling before a reconnect.
  1436. Half-closed connections (esp. via proxies) occupy pool slots until "Pool timeout: All connections in the connection pool
  1437. are occupied". Only ``_request[0]`` (getUpdates) is reset; the general request stays untouched so concurrent sends are
  1438. never interrupted. Relies on PTB 22.x's private ``(get_updates, general)`` tuple — review on PTB 23+."""
  1439. if not (self._app and self._app.bot):
  1440. return
  1441. try:
  1442. polling_req = self._app.bot._request[0]
  1443. except Exception:
  1444. return
  1445. # Bounded wall-clock deadline (not asyncio.wait_for): httpcore's pool close runs under
  1446. # AsyncShieldCancellation and a wedged CLOSE-WAIT socket can hang it forever.
  1447. if not await self._bounded_request_step(polling_req.shutdown(), "Polling request shutdown failed/timed out (non-fatal)"):
  1448. # initialize() only rebuilds the client when ``client.is_closed``; an abandoned aclose()
  1449. # leaves it false, so start_polling would reuse the CLOSE-WAIT socket (alive but deaf).
  1450. # Swap in a fresh client before initialize(). See #87057.
  1451. self._orphan_and_rebuild_polling_client(polling_req)
  1452. if await self._bounded_request_step(polling_req.initialize(), "Polling request re-initialize failed/timed out (non-fatal)"):
  1453. logger.debug("[%s] Polling request pool drained before reconnect", self.name)
  1454. else:
  1455. self._orphan_and_rebuild_polling_client(polling_req)
  1456. async def _bounded_request_step(self, awaitable, failure_msg: str) -> bool:
  1457. """Await a request shutdown()/initialize() under ``_DRAIN_TIMEOUT``; False (debug-logged) on failure."""
  1458. try:
  1459. await _await_with_thread_deadline(awaitable, timeout=_DRAIN_TIMEOUT)
  1460. return True
  1461. except Exception:
  1462. logger.debug("[%s] " + failure_msg, self.name, exc_info=True)
  1463. return False
  1464. def _orphan_and_rebuild_polling_client(self, polling_req) -> None:
  1465. """Replace a wedged HTTPXRequest client after a hung aclose(): swap in a fresh client and close
  1466. the old one in a detached, bounded task so it can't block the reconnect ladder.
  1467. PTB's ``HTTPXRequest.initialize()`` only calls ``_build_client()`` when the current client reports
  1468. ``is_closed``. If ``shutdown()`` was abandoned on a CLOSE-WAIT socket, that flag stays false and the
  1469. next ``start_polling()`` reuses the dead getUpdates connection (#87057).
  1470. """
  1471. old = getattr(polling_req, "_client", None)
  1472. build = getattr(polling_req, "_build_client", None)
  1473. if old is None or not callable(build) or getattr(old, "is_closed", True):
  1474. return
  1475. try:
  1476. polling_req._client = build()
  1477. except Exception:
  1478. logger.debug("[%s] Failed to rebuild polling HTTP client after hung drain", self.name, exc_info=True)
  1479. return
  1480. logger.warning("[%s] Replaced wedged getUpdates HTTP client after drain timeout (likely CLOSE-WAIT socket)", self.name)
  1481. async def _orphan_aclose() -> None:
  1482. try:
  1483. aclose = getattr(old, "aclose", None)
  1484. if not callable(aclose):
  1485. return
  1486. # Same cancellation-swallowing httpcore scope as shutdown(): wall-clock deadline.
  1487. await _await_with_thread_deadline(aclose(), timeout=_DRAIN_TIMEOUT)
  1488. except Exception:
  1489. logger.debug("[%s] Orphan polling client aclose failed (non-fatal)", self.name, exc_info=True)
  1490. try:
  1491. task = asyncio.ensure_future(_orphan_aclose())
  1492. self._background_tasks.add(task)
  1493. task.add_done_callback(self._background_tasks.discard)
  1494. task.add_done_callback(_consume_abandoned_task)
  1495. except Exception:
  1496. pass
  1497. def _fence_polling(self) -> None:
  1498. """Mark polling closed: no progress accepted, send path degraded."""
  1499. self._polling_progress_accepting = False
  1500. self._send_path_degraded = True
  1501. def _begin_polling_generation(self) -> tuple[int, asyncio.Event]:
  1502. """Start accepting progress for a new getUpdates polling generation."""
  1503. if self._teardown_started:
  1504. self._fence_polling()
  1505. progress = getattr(self, "_polling_progress_event", None)
  1506. if progress is None:
  1507. progress = self._polling_progress_event = asyncio.Event()
  1508. return getattr(self, "_polling_generation", 0), progress
  1509. verifier = getattr(self, "_polling_progress_verifier_task", None)
  1510. if verifier is not None and not verifier.done():
  1511. verifier.cancel()
  1512. self._polling_progress_verifier_task = None
  1513. self._polling_generation = getattr(self, "_polling_generation", 0) + 1
  1514. self._polling_progress_event = asyncio.Event()
  1515. self._polling_progress_accepting = True
  1516. self._send_path_degraded = True
  1517. # Reset stall-watchdog timestamps: no proven progress yet, age measured from here.
  1518. # See #92991.
  1519. self._polling_generation_started_monotonic = time.monotonic()
  1520. self._polling_last_progress_monotonic = None
  1521. # Re-base the backlog per generation. On an in-place updater restart PTB keeps the old
  1522. # update_queue, so old dispatches can briefly exceed received; the check treats that as no backlog.
  1523. self._updates_received_total = self._updates_dispatched_total = 0
  1524. self._ingress_dispatched_seen = self._ingress_stalled_heartbeats = 0
  1525. return self._polling_generation, self._polling_progress_event
  1526. def _record_polling_progress(self, generation: int) -> bool:
  1527. """Record successful getUpdates I/O for the current generation only; True when accepted."""
  1528. if self._teardown_started or not self._polling_progress_accepting or generation != self._polling_generation:
  1529. return False
  1530. if not self._polling_progress_event.is_set():
  1531. # First confirmed round-trip resolves the "health pending" line both reconnect paths end on.
  1532. # After network-error WARNINGs the line must read as the matching recovery event (#111211).
  1533. state = "recovered" if self._polling_network_error_count else "confirmed healthy"
  1534. logger.info("[%s] Telegram polling %s: getUpdates progressing (generation %d)", self.name, state, generation)
  1535. self._polling_progress_event.set()
  1536. self._polling_last_progress_monotonic = time.monotonic()
  1537. self._polling_network_error_count = 0
  1538. if generation == self._polling_conflict_recovery_generation:
  1539. self._polling_conflict_recovery_generation = None
  1540. else:
  1541. self._polling_conflict_count = 0
  1542. # First proof getUpdates is flowing for this generation: flip a
  1543. # published "retrying" (degraded connect, reconnect stamp, or the
  1544. # mid-session recovery below) back to "connected" (#101391).
  1545. if self._send_path_degraded and getattr(self, "_running", False) and not self.has_fatal_error:
  1546. self._write_runtime_status_safe(
  1547. "connected", platform_state="connected", error_code=None, error_message=None,
  1548. )
  1549. self._send_path_degraded = False
  1550. return True
  1551. def _observe_polling_request_result(self, request, generation, result):
  1552. """Record getUpdates progress from an observed do_request result (purely observational: PTB still
  1553. parses the untouched payload and owns any resulting exception)."""
  1554. status_code, payload = result
  1555. if generation is None or not (200 <= status_code < 300):
  1556. return
  1557. try:
  1558. # The request's own parser keeps health observation in agreement with PTB.
  1559. envelope = request.parse_json_payload(payload)
  1560. except Exception:
  1561. return
  1562. if isinstance(envelope, dict) and envelope.get("ok") is True and "result" in envelope:
  1563. if self._record_polling_progress(generation):
  1564. self._record_updates_received(envelope.get("result"))
  1565. def _record_updates_received(self, result) -> None:
  1566. """Count updates Telegram handed us on the getUpdates wire (#102260). Only reached for the
  1567. accepted generation, so a late response from a fenced poll cannot inflate the backlog."""
  1568. if isinstance(result, list) and result:
  1569. self._updates_received_total += len(result)
  1570. def _instrument_polling_request(self, request):
  1571. """Instrument one dedicated PTB getUpdates request with progress tracking.
  1572. PTB request classes use ``__slots__`` (no ``__dict__`` on 3.13), so re-tag the instance to a thin ``__slots__ = ()``
  1573. subclass overriding ``do_request`` — identical layout makes the ``__class__`` swap legal; works for test doubles too.
  1574. On Python 3.13 their instances no longer carry a ``__dict__`` (the ``AbstractAsyncContextManager``
  1575. MRO stopped yielding one), so ``request.do_request = wrapper`` raises ``AttributeError:
  1576. 'HTTPXRequest' object attribute 'do_request' is read-only`` and the whole Telegram connect fails
  1577. (#64482). It only appeared to work on Python 3.12, where those instances still had a ``__dict__``.
  1578. """
  1579. adapter = self
  1580. class _InstrumentedPollingRequest(type(request)):
  1581. __slots__ = ()
  1582. async def do_request(self, *args, **kwargs):
  1583. generation = _POLLING_GENERATION_CONTEXT.get()
  1584. result = await super().do_request(*args, **kwargs)
  1585. adapter._observe_polling_request_result(self, generation, result)
  1586. return result
  1587. request.__class__ = _InstrumentedPollingRequest
  1588. return request
  1589. async def _start_polling_once(
  1590. self, app, *, drop_pending_updates: bool, error_callback, abandon_app_on_timeout: bool = False,
  1591. schedule_verifier: bool = True) -> tuple[int, asyncio.Event]:
  1592. """Start one generation and verify real getUpdates progress. Returns this generation's
  1593. ``(generation, progress_event)`` so readiness-gating callers bind to exactly it."""
  1594. if self._teardown_started:
  1595. raise _PollingLifecycleAbort("Telegram polling teardown started")
  1596. generation, progress = self._begin_polling_generation()
  1597. if not self._polling_progress_accepting:
  1598. raise _PollingLifecycleAbort("Telegram polling teardown started")
  1599. def _generation_error_callback(error: Exception) -> None:
  1600. if self._teardown_started or generation != self._polling_generation or error_callback is None:
  1601. return
  1602. callback_context_token = _POLLING_GENERATION_CONTEXT.set(None)
  1603. try:
  1604. error_callback(error)
  1605. finally:
  1606. _POLLING_GENERATION_CONTEXT.reset(callback_context_token)
  1607. context_token = _POLLING_GENERATION_CONTEXT.set(generation)
  1608. try:
  1609. # asyncio.wait_for can wait forever on httpcore/AnyIO shielded scopes; use the wall-deadline
  1610. # helper and abandon the partial updater (caller rebuilds).
  1611. await _await_with_thread_deadline(
  1612. app.updater.start_polling(
  1613. allowed_updates=Update.ALL_TYPES, drop_pending_updates=drop_pending_updates, error_callback=_generation_error_callback),
  1614. timeout=_UPDATER_START_TIMEOUT,
  1615. on_abandon=((lambda app=app: _shutdown_abandoned_app(app)) if abandon_app_on_timeout else None))
  1616. finally:
  1617. _POLLING_GENERATION_CONTEXT.reset(context_token)
  1618. if self._teardown_started:
  1619. self._fence_polling()
  1620. raise _PollingLifecycleAbort("Telegram polling teardown started")
  1621. if schedule_verifier:
  1622. self._schedule_polling_progress_verifier(generation, progress)
  1623. return generation, progress
  1624. def _schedule_polling_progress_verifier(self, generation: int, progress: asyncio.Event) -> None:
  1625. """Own exactly one tracked verifier for the current generation."""
  1626. if self._teardown_started:
  1627. self._fence_polling()
  1628. return
  1629. previous = getattr(self, "_polling_progress_verifier_task", None)
  1630. if previous is not None and not previous.done():
  1631. previous.cancel()
  1632. task = asyncio.get_running_loop().create_task(self._verify_polling_after_reconnect(generation, progress))
  1633. self._polling_progress_verifier_task = task
  1634. self._background_tasks.add(task)
  1635. def _clear_finished_verifier(finished: asyncio.Task) -> None:
  1636. self._background_tasks.discard(finished)
  1637. if self._polling_progress_verifier_task is finished:
  1638. self._polling_progress_verifier_task = None
  1639. task.add_done_callback(_clear_finished_verifier)
  1640. def _get_general_request_drain_lock(self) -> asyncio.Lock:
  1641. lock = getattr(self, "_general_request_drain_lock", None)
  1642. if lock is None:
  1643. lock = self._general_request_drain_lock = asyncio.Lock()
  1644. return lock
  1645. async def _drain_general_connections_after_pool_timeout(self) -> None:
  1646. """Reset the general Bot API pool (``_request[1]``) after a confirmed send pool timeout — PTB
  1647. guarantees the request was not sent, so resetting before retrying is safe."""
  1648. bot = getattr(getattr(self, "_app", None), "bot", None)
  1649. if bot is None:
  1650. bot = getattr(self, "_bot", None)
  1651. if bot is None:
  1652. return
  1653. try:
  1654. general_req = bot._request[1]
  1655. except Exception:
  1656. return
  1657. async with self._get_general_request_drain_lock():
  1658. await self._bounded_request_step(
  1659. general_req.shutdown(), "General request shutdown failed/timed out after pool timeout (non-fatal)")
  1660. if await self._bounded_request_step(
  1661. general_req.initialize(), "General request re-initialize failed/timed out after pool timeout (non-fatal)"):
  1662. logger.warning("[%s] General request pool drained after Telegram pool timeout", self.name)
  1663. def _spawn_polling_recovery(self, loop, coro) -> None:
  1664. """Start ``coro`` as the tracked in-flight recovery task (reentrancy guard)."""
  1665. self._polling_error_task = loop.create_task(coro)
  1666. self._background_tasks.add(self._polling_error_task)
  1667. self._polling_error_task.add_done_callback(self._background_tasks.discard)
  1668. def _recovery_in_flight(self) -> bool:
  1669. return bool(self._polling_error_task and not self._polling_error_task.done())
  1670. def _schedule_polling_recovery(self, error: Exception, *, reason: str) -> None:
  1671. """Schedule background polling recovery without failing gateway startup: a transient bootstrap
  1672. failure degrades only this adapter; the reconnect ladder recovers in the background."""
  1673. if self._teardown_started or self.has_fatal_error:
  1674. return
  1675. if self._recovery_in_flight():
  1676. logger.debug(
  1677. "[%s] Telegram polling recovery already scheduled; ignoring %s: %s", self.name, reason, _redact_telegram_error_text(error))
  1678. return
  1679. self._send_path_degraded = True
  1680. # Polling died mid-session on an adapter that published "connected"
  1681. # at connect time. Without this, gateway_state.json keeps saying
  1682. # connected for as long as the recovery ladder runs (#101391: 11 h).
  1683. if getattr(self, "_running", False):
  1684. self._mark_degraded()
  1685. if isinstance(error, _PollingStallError):
  1686. # Not a retry promise: the recovery path hands a confirmed stall straight to the supervisor
  1687. # (``_go_fatal_network`` logs the single error-level line for it).
  1688. logger.warning(
  1689. "[%s] Telegram polling stall confirmed (%s); handing off to the supervisor for an adapter rebuild. "
  1690. "Error: %s", self.name, reason, _redact_telegram_error_text(error))
  1691. else:
  1692. logger.warning(
  1693. "[%s] Telegram polling degraded (%s); gateway stays alive and will retry. Error: %s", self.name, reason,
  1694. _redact_telegram_error_text(error))
  1695. self._spawn_polling_recovery(asyncio.get_running_loop(), self._handle_polling_network_error(error))
  1696. async def _delete_webhook_best_effort(self, *, require_success: bool = False) -> bool:
  1697. """Clear a stale webhook; ``require_success`` (cold start) raises so GatewayRunner disposes the
  1698. partial adapter, while reconnects recover transient errors in background."""
  1699. if not self._bot:
  1700. return False
  1701. delete_webhook = getattr(self._bot, "delete_webhook", None)
  1702. if not callable(delete_webhook):
  1703. return True
  1704. try:
  1705. # Same shielded-cancellation class as initialize/start_polling: never let it pin connect.
  1706. await _await_with_thread_deadline(delete_webhook(drop_pending_updates=False), timeout=_UPDATER_START_TIMEOUT)
  1707. return True
  1708. except Exception as err:
  1709. if not self._looks_like_network_error(err):
  1710. raise
  1711. if require_success:
  1712. raise OSError("Telegram deleteWebhook did not complete during initial connect") from err
  1713. logger.warning(
  1714. "[%s] deleteWebhook failed with a recoverable network error; continuing to polling so getUpdates/retry can recover: %s",
  1715. self.name, _redact_telegram_error_text(err))
  1716. self._send_path_degraded = True
  1717. return False
  1718. async def _await_cold_start_readiness(self, progress: asyncio.Event, strict_error_event: asyncio.Event, strict_error: list) -> None:
  1719. """Cold start: wait for THIS generation's first getUpdates success or the first polling error;
  1720. raises OSError so GatewayRunner disposes the partial adapter and retries fresh."""
  1721. progress_wait = asyncio.ensure_future(progress.wait())
  1722. error_wait = asyncio.ensure_future(strict_error_event.wait())
  1723. try:
  1724. # Losers are NOT cancelled here; the finally below does it.
  1725. await _await_with_thread_deadline(
  1726. asyncio.wait({progress_wait, error_wait}, return_when=asyncio.FIRST_COMPLETED), timeout=_INITIAL_POLLING_PROGRESS_TIMEOUT)
  1727. except TimeoutError as exc:
  1728. raise OSError(
  1729. "Telegram getUpdates made no progress within "
  1730. f"{_INITIAL_POLLING_PROGRESS_TIMEOUT:.0f}s during initial "
  1731. "connect — failing startup so the gateway retries with a fresh adapter (#67498)"
  1732. ) from exc
  1733. finally:
  1734. for fut in (progress_wait, error_wait):
  1735. if not fut.done():
  1736. fut.cancel()
  1737. await asyncio.gather(progress_wait, error_wait, return_exceptions=True)
  1738. if strict_error and not progress.is_set():
  1739. raise OSError(
  1740. "Telegram polling errored before first getUpdates success during initial connect: "
  1741. f"{_redact_telegram_error_text(strict_error[0])}"
  1742. ) from strict_error[0]
  1743. if not progress.is_set():
  1744. raise OSError("Telegram getUpdates did not become ready during initial connect")
  1745. async def _start_polling_resilient(self, *, drop_pending_updates: bool, error_callback, require_progress: bool = False) -> bool:
  1746. """Start PTB polling; ``require_progress`` (initial connect) demands real readiness. Reconnects
  1747. may recover in background; on cold start a bootstrap failure raises (see _await_cold_start_readiness)."""
  1748. if self._teardown_started:
  1749. return False
  1750. if not (self._app and self._app.updater):
  1751. raise RuntimeError("Telegram application/updater not initialized")
  1752. # Strict cold start: background recovery must not run while the readiness gate waits, else a G1
  1753. # error starts G2 on the same partial app and GatewayRunner never disposes it.
  1754. strict_error: list[BaseException] = []
  1755. strict_error_event = asyncio.Event()
  1756. strict_gate_open = True
  1757. effective_callback = error_callback
  1758. if require_progress:
  1759. loop = asyncio.get_running_loop()
  1760. def _strict_error_callback(error: Exception) -> None:
  1761. # Once the gate closes, delegate so later errors still reach background recovery.
  1762. if not strict_gate_open:
  1763. if error_callback is not None:
  1764. error_callback(error)
  1765. return
  1766. if not strict_error:
  1767. strict_error.append(error)
  1768. # Called from the polling task; set on the loop to wake the strict waiter.
  1769. loop.call_soon_threadsafe(strict_error_event.set)
  1770. effective_callback = _strict_error_callback
  1771. try:
  1772. # Same watchdog bound as the reconnect ladders; the TimeoutError is an OSError subclass, so
  1773. # the except below classifies it as a network error → background recovery.
  1774. # Same watchdog bound as the reconnect ladders: a wedged httpx connection pool can hang
  1775. # start_polling() forever at bootstrap too (#59614).
  1776. _generation, progress = await self._start_polling_once(
  1777. self._app, drop_pending_updates=drop_pending_updates, error_callback=effective_callback,
  1778. abandon_app_on_timeout=require_progress,
  1779. # The strict gate IS the cold-start verifier; a background one would race it.
  1780. schedule_verifier=not require_progress)
  1781. if require_progress:
  1782. await self._await_cold_start_readiness(progress, strict_error_event, strict_error)
  1783. # Readiness proven — close the gate so later errors reach background recovery.
  1784. strict_gate_open = False
  1785. self._polling_error_callback_ref = error_callback
  1786. return True
  1787. except _PollingLifecycleAbort:
  1788. return False
  1789. except Exception as err:
  1790. if self._teardown_started:
  1791. return False
  1792. if require_progress:
  1793. raise
  1794. if self._looks_like_polling_conflict(err):
  1795. logger.warning(
  1796. "[%s] Telegram polling bootstrap conflict; gateway stays alive while conflict retry runs: %s",
  1797. self.name, _redact_telegram_error_text(err))
  1798. self._spawn_polling_recovery(asyncio.get_running_loop(), self._handle_polling_conflict(err))
  1799. return False
  1800. if self._looks_like_network_error(err):
  1801. self._schedule_polling_recovery(err, reason="polling bootstrap")
  1802. return False
  1803. raise
  1804. async def _go_fatal_network(self, message: str, log_message: str, *log_args) -> None:
  1805. """Retryable ``telegram_network_error`` fatal + runner handoff (supervisor rebuilds the adapter)."""
  1806. logger.error(log_message, *log_args)
  1807. self._set_fatal_error("telegram_network_error", message, retryable=True)
  1808. await self._handoff_polling_fatal_error()
  1809. async def _stop_updater_or_go_fatal(self, app, what: str) -> bool:
  1810. """Bounded ``updater.stop()`` before a recovery restart; False = went fatal, caller returns.
  1811. Wall-clock deadline, not asyncio.wait_for: a CLOSE-WAIT socket wedges stop() on epoll and PTB/AnyIO shielded cleanup
  1812. hangs wait_for. On timeout the Updater's lifecycle lock may still be held, so rebuild the adapter instead."""
  1813. try:
  1814. if app and app.updater and app.updater.running:
  1815. try:
  1816. await _await_with_thread_deadline(app.updater.stop(), timeout=_UPDATER_STOP_TIMEOUT)
  1817. except TimeoutError:
  1818. message = (
  1819. f"Telegram updater.stop() did not finish before the {what} deadline; "
  1820. "rebuilding the adapter instead of reusing an Updater whose lifecycle lock may still be held.")
  1821. await self._go_fatal_network(message, "[%s] %s (likely CLOSE-WAIT socket)", self.name, message)
  1822. return False
  1823. except Exception:
  1824. pass
  1825. return True
  1826. def _restart_polling_in_task(self, coro) -> None:
  1827. """Run a recovery coroutine as the tracked in-flight ``_polling_error_task``."""
  1828. self._polling_error_task = asyncio.get_running_loop().create_task(coro)
  1829. async def _handle_polling_network_error(self, error: Exception) -> None:
  1830. """Reconnect polling after a transient network interruption (NetworkError/TimedOut).
  1831. Host connectivity loss (sleep, WiFi switch, VPN) kills the long-poll silently. Exponential back-off (5s→60s
  1832. cap) up to MAX_NETWORK_RETRIES, then retryable-fatal so the supervisor restarts the gateway.
  1833. A confirmed polling stall (``_PollingStallError``) skips the ladder entirely: the Updater's long-poll
  1834. action never quiesced, so it is handed to the supervisor for a rebuild before any backoff."""
  1835. if self._teardown_started or self.has_fatal_error:
  1836. return
  1837. if isinstance(error, _PollingStallError):
  1838. # Not a retry: no counter bump, no backoff, no in-place stop/drain. The supervisor's rebuild
  1839. # runs disconnect(), which performs the same bounded updater.stop() and app.shutdown().
  1840. message = (
  1841. f"Telegram polling stall confirmed ({_redact_telegram_error_text(error)}); rebuilding the adapter instead of reusing "
  1842. "a wedged Updater/dispatcher in place."
  1843. )
  1844. await self._go_fatal_network(message, "[%s] %s (rebuilding adapter via supervisor)", self.name, message)
  1845. return
  1846. MAX_NETWORK_RETRIES = 10
  1847. BASE_DELAY = 5
  1848. MAX_DELAY = 60
  1849. self._polling_network_error_count += 1
  1850. self._send_path_degraded = True
  1851. attempt = self._polling_network_error_count
  1852. if attempt > MAX_NETWORK_RETRIES:
  1853. message = (
  1854. f'Telegram polling could not reconnect after {MAX_NETWORK_RETRIES:d} network error retries. Escalating to gateway recovery.')
  1855. await self._go_fatal_network(message, "[%s] %s Last error: %s", self.name, message, _redact_telegram_error_text(error))
  1856. return
  1857. delay = min(BASE_DELAY * (2 ** (attempt - 1)), MAX_DELAY)
  1858. logger.warning(
  1859. "[%s] Telegram network error (attempt %d/%d), reconnecting in %ds. Error: %s", self.name, attempt,
  1860. MAX_NETWORK_RETRIES, delay, _redact_telegram_error_text(error))
  1861. await asyncio.sleep(delay)
  1862. if self._teardown_started:
  1863. return
  1864. # Stable local ref: a concurrent disconnect() may set self._app = None while we await.
  1865. app = self._app
  1866. # Unguarded stop() on a CLOSE-WAIT socket would leave _polling_error_task perpetually
  1867. # "in-flight" so every probe skips reconnect for hours.
  1868. if not await self._stop_updater_or_go_fatal(app, "network-recovery") or self._teardown_started:
  1869. return
  1870. # start_polling() bootstraps through the *general* pool before getUpdates; a confirmed pool timeout means the request
  1871. # was never sent, so rebuilding that pool is safe. Generic network errors stay polling-only (sends untouched).
  1872. if self._looks_like_pool_timeout(error):
  1873. await self._drain_general_connections_after_pool_timeout()
  1874. if self._teardown_started:
  1875. return
  1876. await self._drain_polling_connections()
  1877. if self._teardown_started:
  1878. return
  1879. try:
  1880. if not app:
  1881. raise RuntimeError("Telegram application was torn down during reconnect")
  1882. await self._start_polling_once(app, drop_pending_updates=False, error_callback=self._polling_error_callback_ref)
  1883. logger.info(
  1884. "[%s] Telegram polling restarted after network error (attempt %d); health pending getUpdates progress", self.name, attempt)
  1885. except _PollingLifecycleAbort:
  1886. return
  1887. except Exception as retry_err:
  1888. if self._teardown_started:
  1889. return
  1890. logger.warning("[%s] Telegram polling reconnect failed: %s", self.name, _redact_telegram_error_text(retry_err))
  1891. # Polling is dead and no more error callbacks will fire — chain the retry ourselves.
  1892. if not self.has_fatal_error and not self._teardown_started:
  1893. task = asyncio.ensure_future(self._handle_polling_network_error(retry_err))
  1894. self._background_tasks.add(task)
  1895. task.add_done_callback(self._background_tasks.discard)
  1896. # The chained retry IS the in-flight recovery: it must replace the reentrancy guard.
  1897. self._polling_error_task = task
  1898. async def _polling_heartbeat_loop(self) -> None:
  1899. """Detect dead Telegram TCP sockets (CLOSE-WAIT) by periodic probing.
  1900. In CLOSE-WAIT epoll still reports the long-poll socket readable and nothing raises, so PTB's
  1901. ``error_callback`` never fires. Probe ``get_me()`` on the *general* path (never the getUpdates pool);
  1902. connect-level failures feed ``_handle_polling_network_error``. Runs for the connection's lifetime, catching
  1903. steady-state wedges the one-shot verifier can't."""
  1904. HEARTBEAT_INTERVAL = 90 # seconds between probes
  1905. PROBE_TIMEOUT = 15 # seconds before declaring the path dead
  1906. # Wedged-recovery watchdog: note when a recovery task is first seen in-flight and force-escalate
  1907. # if the *same* task object still runs past the stuck timeout.
  1908. # Tracked locally so no _polling_error_task assignment site needs to stamp a timestamp: the
  1909. # heartbeat notes when it first observes a given recovery task still in-flight, and force-escalates
  1910. # if the *same* task object is still running after _POLLING_ERROR_TASK_STUCK_TIMEOUT. A healthy
  1911. # ladder attempt completes (task done) or chains to a new task well before then, so a single
  1912. # long-lived task is unambiguously wedged. See #66377.
  1913. stuck_task_ref: Optional[asyncio.Task] = None
  1914. stuck_task_since = 0.0
  1915. while True:
  1916. try:
  1917. await asyncio.sleep(HEARTBEAT_INTERVAL)
  1918. if self._teardown_started or self.has_fatal_error:
  1919. return
  1920. # A recovery task hung on an unbounded await gates every other recovery path forever
  1921. # (alive but deaf): force retryable-fatal so the reconnector rebuilds the adapter.
  1922. # Independent wedged-recovery watchdog (#66377): if the tracked recovery task has hung (any
  1923. # await no local bound covers), every other recovery path is gated behind it and returns
  1924. # early forever — the gateway stays alive but deaf.
  1925. recovery_task = self._polling_error_task
  1926. if recovery_task is not None and not recovery_task.done():
  1927. now = time.monotonic()
  1928. if recovery_task is not stuck_task_ref:
  1929. stuck_task_ref = recovery_task
  1930. stuck_task_since = now
  1931. elif now - stuck_task_since > _POLLING_ERROR_TASK_STUCK_TIMEOUT:
  1932. stuck_for = now - stuck_task_since
  1933. logger.error(
  1934. "[%s] Telegram reconnect task wedged for %.0fs with no ladder progress; forcing retryable-fatal so the gateway "
  1935. "reconnects instead of staying silently deaf.",
  1936. self.name, stuck_for)
  1937. with contextlib.suppress(Exception):
  1938. recovery_task.cancel()
  1939. self._set_fatal_error(
  1940. "telegram_network_error",
  1941. f"Telegram reconnect task wedged for {stuck_for:.0f}s; forcing gateway reconnect.",
  1942. retryable=True)
  1943. await self._handoff_polling_fatal_error()
  1944. return
  1945. else:
  1946. stuck_task_ref = None
  1947. bot = self._app.bot if self._app else None
  1948. if bot is None:
  1949. continue
  1950. # No get_me() ⇒ not a live polling client (torn down / test double): exit, don't spin.
  1951. if not callable(getattr(bot, "get_me", None)):
  1952. return
  1953. await asyncio.wait_for(bot.get_me(), PROBE_TIMEOUT)
  1954. # get_me() refreshes PTB's cached bot user: adopt a BotFather rename before routing on it.
  1955. self._bot_identity_checked_at = time.monotonic()
  1956. self._note_bot_username(getattr(bot, "username", None))
  1957. # get_me() OK proves only the send path; a wedged long-poll shows as server-side queue.
  1958. # get_me() succeeded — the general/send request path is healthy. That does NOT prove the
  1959. # getUpdates consumer is alive: PTB can report updater.running=True while the long-poll task
  1960. # is wedged, so DMs queue in the Bot API and never reach handlers (#42909). get_me() is
  1961. # blind to this; get_webhook_info() exposes it via pending_update_count. Escalate only after
  1962. # two consecutive probes see a non-zero queue while we believe we're polling, so a single
  1963. # in-flight update (consumed before the next probe) never trips recovery.
  1964. await self._probe_pending_updates(bot, PROBE_TIMEOUT)
  1965. # An empty queue can't hide a wedge forever: no round-trip past the stall threshold ⇒ dead.
  1966. # Even an empty queue cannot hide a wedged long-poll forever: Telegram answers within ~50s,
  1967. # so a consumer with no successful round-trip past the stall threshold is dead (#92991).
  1968. # Pure local-state check — no Bot API call needed.
  1969. await self._check_polling_stall()
  1970. # Transport health is not dispatch health (#102260). Pure local-state check.
  1971. self._check_ingress_dispatch_stall()
  1972. except asyncio.CancelledError:
  1973. return
  1974. except (TimeoutError, OSError) as probe_err:
  1975. self._schedule_polling_recovery(probe_err, reason="heartbeat probe")
  1976. except Exception as probe_err:
  1977. # Non-connectivity errors (e.g. TelegramError 401) aren't CLOSE-WAIT symptoms.
  1978. if self._looks_like_network_error(probe_err):
  1979. self._schedule_polling_recovery(probe_err, reason="heartbeat probe")
  1980. async def _probe_pending_updates(self, bot, probe_timeout: float) -> None:
  1981. """Detect a wedged or stopped getUpdates consumer via pending_update_count.
  1982. PTB can report ``updater.running`` while the long-poll is stuck; get_me() stays healthy yet DMs queue in the
  1983. Bot API. A stuck queue over two consecutive probes ⇒ dead consumer. Also covers the updater having stopped
  1984. entirely (``running=False``, no reconnect in flight).
  1985. PTB can report ``updater.running == True`` while its long-poll task is silently stuck (e.g. a socket
  1986. that epoll keeps reporting readable on WSL2). ``get_me()`` stays healthy because it uses the general
  1987. request path, so the CLOSE-WAIT heartbeat never fires — yet DMs queue in the Bot API and never reach
  1988. handlers (#42909).
  1989. We detect the stopped updater directly and feed the same ladder (#55769).
  1990. """
  1991. # Polling mode only: in webhook mode Telegram pushes and holds no server-side queue.
  1992. if self._teardown_started or self._webhook_mode:
  1993. return
  1994. # An in-flight reconnect owns recovery — don't double-trigger, and don't misread its brief
  1995. # stop()->start_polling() window (updater.running transiently False) as dead.
  1996. if self._recovery_in_flight():
  1997. self._polling_not_running_count = 0
  1998. return
  1999. updater = getattr(self._app, "updater", None) if self._app else None
  2000. if updater is None:
  2001. self._polling_pending_stuck_count = 0
  2002. return
  2003. if not getattr(updater, "running", False):
  2004. # Long-poll task gone, general-path calls still succeed, so no error_callback/probe ever
  2005. # fires. Debounced over two probes so a just-starting updater never trips it.
  2006. self._polling_pending_stuck_count = 0
  2007. # We are in polling mode with no reconnect in flight, yet PTB's updater has stopped entirely.
  2008. # This is distinct from the wedged-but-running consumer handled below: the long-poll task is
  2009. # gone, get_me()/get_webhook_info() on the general request path still succeed, so no
  2010. # error_callback or connectivity probe ever fires and the gateway silently stops receiving
  2011. # messages while the process stays alive (#55769).
  2012. self._polling_not_running_count += 1
  2013. logger.warning(
  2014. "[%s] Telegram polling heartbeat: updater stopped while in polling mode (stuck probe %d/2)", self.name,
  2015. self._polling_not_running_count)
  2016. if self._polling_not_running_count >= 2:
  2017. self._polling_not_running_count = 0
  2018. self._escalate_stuck_consumer(
  2019. "[%s] Telegram updater is not running (long-poll task gone); triggering polling restart",
  2020. "Telegram updater stopped while in polling mode")
  2021. return
  2022. self._polling_not_running_count = 0
  2023. get_webhook_info = getattr(bot, "get_webhook_info", None)
  2024. if not callable(get_webhook_info):
  2025. return
  2026. try:
  2027. info = await asyncio.wait_for(get_webhook_info(), probe_timeout) # type: ignore[arg-type]
  2028. except (TimeoutError, OSError):
  2029. return # connectivity symptom for the get_me() path, not a stuck-queue signal
  2030. pending = int(getattr(info, "pending_update_count", 0) or 0)
  2031. if pending <= 0:
  2032. self._polling_pending_stuck_count = 0
  2033. return
  2034. self._polling_pending_stuck_count += 1
  2035. logger.warning(
  2036. "[%s] Telegram polling heartbeat: %d update(s) queued but not consumed (stuck probe %d/2)", self.name,
  2037. pending, self._polling_pending_stuck_count)
  2038. if self._polling_pending_stuck_count >= 2:
  2039. self._polling_pending_stuck_count = 0
  2040. self._escalate_stuck_consumer(
  2041. "[%s] getUpdates consumer appears wedged (queue not draining); triggering polling restart",
  2042. "getUpdates consumer wedged: pending updates not draining")
  2043. def _escalate_stuck_consumer(self, log_message: str, reason: str) -> None:
  2044. """Second consecutive stuck probe: restart polling via the network-error ladder (unless tearing down)."""
  2045. if self._teardown_started:
  2046. return
  2047. logger.warning(log_message, self.name)
  2048. self._polling_error_task = asyncio.get_running_loop().create_task(self._handle_polling_network_error(RuntimeError(reason)))
  2049. def _check_ingress_dispatch_stall(self) -> None:
  2050. """Escalate fetched updates PTB's dispatcher is not handing to handlers (#102260, #130407).
  2051. ``received`` and ``dispatched`` count the same population (every fetched update reaches the
  2052. group-99 catch-all: no handler raises ApplicationHandlerStop, no error handler is registered),
  2053. so a backlog with no dispatch progress across ``_INGRESS_DISPATCH_STALL_HEARTBEATS`` heartbeats
  2054. is a wedged dispatcher at any traffic rate. Hands the adapter to the supervisor for a rebuild once
  2055. per stall (``_PollingStallError``: an in-place restart keeps the wedged dispatcher); re-arms on progress.
  2056. """
  2057. if self._webhook_mode or self._teardown_started or self.has_fatal_error or self._recovery_in_flight():
  2058. return
  2059. received = getattr(self, "_updates_received_total", 0)
  2060. dispatched = getattr(self, "_updates_dispatched_total", 0)
  2061. if received <= dispatched or dispatched != getattr(self, "_ingress_dispatched_seen", 0):
  2062. self._ingress_dispatched_seen = dispatched
  2063. self._ingress_stalled_heartbeats = 0
  2064. return
  2065. stalled = getattr(self, "_ingress_stalled_heartbeats", 0)
  2066. if stalled >= _INGRESS_DISPATCH_STALL_HEARTBEATS:
  2067. return # already reported this stall
  2068. self._ingress_stalled_heartbeats = stalled + 1
  2069. if stalled + 1 < _INGRESS_DISPATCH_STALL_HEARTBEATS:
  2070. return
  2071. # No pre-log: the handoff warning and ``_go_fatal_network`` carry this text (see _check_polling_stall).
  2072. generation = getattr(self, "_polling_generation", 0)
  2073. self._schedule_polling_recovery(
  2074. _PollingStallError(
  2075. f"ingress healthy but deaf: PTB dispatcher made no progress for {_INGRESS_DISPATCH_STALL_HEARTBEATS} "
  2076. f"heartbeats with {received - dispatched} update(s) fetched but not dispatched "
  2077. f"({received} received, {dispatched} dispatched, generation {generation})"),
  2078. reason="ingress dispatch stall watchdog")
  2079. async def _check_polling_stall(self) -> None:
  2080. """Watchdog the last successful getUpdates round-trip: a long-poll can wedge without raising
  2081. (CLOSE-WAIT after a route flip) while every other probe stays blind; no round-trip for
  2082. ``_POLLING_STALL_TIMEOUT`` ⇒ raise ``_PollingStallError`` so the recovery path hands the
  2083. adapter to the supervisor for a rebuild instead of reusing the wedged Updater.
  2084. See #92991, #113618.
  2085. """
  2086. if self._webhook_mode or self._teardown_started or self.has_fatal_error or self._recovery_in_flight():
  2087. return
  2088. now = time.monotonic()
  2089. last_progress = getattr(self, "_polling_last_progress_monotonic", None)
  2090. generation_started = getattr(self, "_polling_generation_started_monotonic", None)
  2091. if last_progress is not None:
  2092. stalled_for = now - last_progress
  2093. elif generation_started is not None:
  2094. # No round-trip yet this generation: fallback for when the one-shot verifier could not run.
  2095. stalled_for = now - generation_started
  2096. else:
  2097. return
  2098. if stalled_for <= _POLLING_STALL_TIMEOUT:
  2099. return
  2100. # No pre-log here: the recovery path logs the hand-off and ``_go_fatal_network`` the one
  2101. # error-level line, so a stall does not announce itself twice.
  2102. self._schedule_polling_recovery(
  2103. _PollingStallError(
  2104. f"getUpdates made no progress for {stalled_for:.0f}s (generation {getattr(self, '_polling_generation', 0)}; polling stall watchdog)"),
  2105. reason="polling stall watchdog")
  2106. def _verifier_stale(self, generation: int, progress: asyncio.Event) -> bool:
  2107. """True when a verifier's generation no longer matters (progressed, fatal, replaced, torn down)."""
  2108. return (
  2109. self._teardown_started or progress.is_set() or self.has_fatal_error
  2110. or not self._polling_progress_accepting or generation != self._polling_generation
  2111. or progress is not self._polling_progress_event)
  2112. async def _verify_polling_after_reconnect(self, generation: Optional[int] = None, progress: Optional[asyncio.Event] = None) -> None:
  2113. """Require getUpdates progress, using getMe only to classify failure: a general-path getMe
  2114. success cannot heal polling health. Connectivity failures enter the guarded recovery ladder."""
  2115. PROBE_TIMEOUT = 10
  2116. if self._teardown_started:
  2117. return
  2118. if generation is None:
  2119. generation = self._polling_generation
  2120. if progress is None:
  2121. progress = self._polling_progress_event
  2122. with contextlib.suppress(asyncio.TimeoutError):
  2123. await asyncio.wait_for(progress.wait(), timeout=_POLLING_PROGRESS_TIMEOUT)
  2124. if self._verifier_stale(generation, progress):
  2125. return
  2126. app = self._app
  2127. if not (app and app.updater and app.updater.running):
  2128. logger.warning("[%s] Updater made no getUpdates progress and is not running", self.name)
  2129. self._schedule_polling_recovery(
  2130. RuntimeError("Updater not running after polling progress deadline"),
  2131. reason="polling progress verifier: updater not running")
  2132. return
  2133. try:
  2134. await asyncio.wait_for(app.bot.get_me(), PROBE_TIMEOUT)
  2135. except Exception as probe_err:
  2136. if self._verifier_stale(generation, progress):
  2137. return
  2138. if not self._looks_like_network_error(probe_err):
  2139. logger.warning(
  2140. "[%s] Polling progress verifier hit a non-connectivity error (not retrying): %s", self.name,
  2141. _redact_telegram_error_text(probe_err))
  2142. return
  2143. logger.warning(
  2144. "[%s] Polling progress verifier connectivity probe failed: %s", self.name, _redact_telegram_error_text(probe_err))
  2145. self._schedule_polling_recovery(probe_err, reason="polling progress verifier connectivity failure")
  2146. return
  2147. if self._verifier_stale(generation, progress):
  2148. return
  2149. self._schedule_polling_recovery(
  2150. _PollingStallError("getUpdates made no progress before verifier deadline"),
  2151. reason="polling progress verifier: general path healthy but getUpdates stalled")
  2152. def _disarm_ptb_retry_loop(self) -> None:
  2153. """Synchronously stop PTB's internal polling retry loop.
  2154. PTB's ``network_retry_loop`` calls our ``error_callback`` *synchronously* on a 409 Conflict then polls again; our
  2155. callback only schedules async recovery, so two sessions overlap and Telegram 409s on a ~31s cadence. Setting PTB's
  2156. private ``stop_event`` makes its loop exit on the next tick; ``updater.stop()`` + drain + ``start_polling()`` then build
  2157. a fresh one. Best-effort across PTB spellings. Deliberately NOT flipping ``updater._running``: stop() raises when
  2158. already False, which would skip the real teardown and poison the next start."""
  2159. updater = getattr(self._app, "updater", None) if self._app else None
  2160. if updater is None:
  2161. return
  2162. for attr in ("_Updater__polling_task_stop_event", "_polling_task_stop_event"):
  2163. stop_event = getattr(updater, attr, None)
  2164. if isinstance(stop_event, asyncio.Event):
  2165. if not stop_event.is_set():
  2166. stop_event.set()
  2167. logger.debug("[%s] Disarmed PTB polling retry loop via %s", self.name, attr)
  2168. return
  2169. logger.debug(
  2170. "[%s] Could not disarm PTB polling retry loop (stop_event not found on this PTB version); falling back to async stop()",
  2171. self.name)
  2172. async def _handle_polling_conflict(self, error: Exception) -> None:
  2173. """Recover a 409 Conflict: the previous gateway process was killed but Telegram holds its
  2174. getUpdates session ~30s. Stop, wait (growing delay), drain, restart — MAX_CONFLICT_RETRIES
  2175. times before going fatal; a failed retry must never return silently (limbo)."""
  2176. if self._teardown_started:
  2177. return
  2178. if self.has_fatal_error and self.fatal_error_code == "telegram_polling_conflict":
  2179. return
  2180. self._polling_conflict_count += 1
  2181. MAX_CONFLICT_RETRIES = 5
  2182. # 15s, 25s, 35s, 45s, 55s — clears Telegram's ~30s session window without hammering the API.
  2183. RETRY_DELAY = 10 + (self._polling_conflict_count * 10) # seconds
  2184. if self._polling_conflict_count <= MAX_CONFLICT_RETRIES:
  2185. logger.warning(
  2186. "[%s] Telegram polling conflict (%d/%d) — previous session still "
  2187. "held open on Telegram's servers. Waiting %ds for it to expire. Error: %s",
  2188. self.name, self._polling_conflict_count, MAX_CONFLICT_RETRIES,
  2189. RETRY_DELAY, _redact_telegram_error_text(error))
  2190. # Stop the updater before sleeping (no-op if PTB raised before running was set).
  2191. if not await self._stop_updater_or_go_fatal(self._app, "conflict-retry"):
  2192. return
  2193. await asyncio.sleep(RETRY_DELAY)
  2194. if self._teardown_started:
  2195. return
  2196. await self._drain_polling_connections()
  2197. if self._teardown_started:
  2198. return
  2199. # Stable local ref: a concurrent disconnect() may null self._app across the awaits above.
  2200. app = self._app
  2201. # Capture a stable local reference: self._app can be reassigned to None by a concurrent
  2202. # disconnect() while we're suspended across the awaits above (same race #55992 fixed on the
  2203. # network path). Re-reading self._app after that point would raise AttributeError deep inside
  2204. # start_polling instead of failing fast here, where the except below reschedules or escalates to
  2205. # fatal.
  2206. expected_generation = self._polling_generation + 1
  2207. if not app:
  2208. raise RuntimeError("Telegram application was torn down during conflict reconnect")
  2209. # drop_pending_updates=True makes Telegram terminate any other getUpdates session for this
  2210. # token (zombie or our own prior retry); without it each retry is immediately 409'd.
  2211. # The competing session is either a zombie from the previous gateway process (whose long-poll
  2212. # hasn't expired server-side yet) or our own previous retry's still-expiring session. Without
  2213. # this, each retry starts a new getUpdates session that immediately gets 409'd by the previous
  2214. # one, creating the very conflict we are trying to recover from (#75017).
  2215. self._polling_conflict_recovery_generation = expected_generation
  2216. try:
  2217. await self._start_polling_once(app, drop_pending_updates=True, error_callback=self._polling_error_callback_ref)
  2218. logger.info(
  2219. "[%s] Telegram polling restarted after conflict retry %d/%d; health pending getUpdates progress",
  2220. self.name, self._polling_conflict_count, MAX_CONFLICT_RETRIES)
  2221. return
  2222. except _PollingLifecycleAbort:
  2223. return
  2224. except Exception as retry_err:
  2225. if self._teardown_started:
  2226. return
  2227. logger.warning(
  2228. "[%s] Telegram polling retry %d/%d failed: %s. Scheduling next attempt.", self.name,
  2229. self._polling_conflict_count, MAX_CONFLICT_RETRIES, _redact_telegram_error_text(retry_err))
  2230. # Never return silently: alive-and-"connected" with no polling is limbo.
  2231. if self._polling_conflict_count < MAX_CONFLICT_RETRIES and not self._teardown_started:
  2232. # get_running_loop(): get_event_loop() raises on 3.10+ from PTB's callback context.
  2233. self._restart_polling_in_task(self._handle_polling_conflict(retry_err))
  2234. return
  2235. # Fall through to fatal on the last retry.
  2236. finally:
  2237. if self._polling_conflict_recovery_generation == expected_generation:
  2238. self._polling_conflict_recovery_generation = None
  2239. if self._teardown_started:
  2240. return
  2241. # Retries exhausted — fatal so the runner surfaces it and the user knows to act.
  2242. message = (
  2243. f"Telegram polling could not recover after {MAX_CONFLICT_RETRIES} retries "
  2244. f"({sum(10 + i * 10 for i in range(1, MAX_CONFLICT_RETRIES + 1))}s total wait). "
  2245. "The previous gateway session is still held open on Telegram's servers, "
  2246. "or another process is using the same bot token. To recover: ensure no other Hermes or OpenClaw instance is running "
  2247. "with this token, then restart the gateway with 'hermes gateway restart'.")
  2248. logger.error("[%s] %s Original error: %s", self.name, message, _redact_telegram_error_text(error))
  2249. # Snapshot whether WE transition to fatal: a concurrent retry task suspended past the entry
  2250. # guard reaches this branch too. Only the first transition notifies.
  2251. _already_fatal = self.has_fatal_error and self.fatal_error_code == "telegram_polling_conflict"
  2252. self._set_fatal_error("telegram_polling_conflict", message, retryable=False)
  2253. try:
  2254. if self._app and self._app.updater:
  2255. await _await_with_thread_deadline(self._app.updater.stop(), timeout=_UPDATER_STOP_TIMEOUT)
  2256. except TimeoutError:
  2257. logger.warning("[%s] updater.stop() timed out after exhausting conflict retries (likely CLOSE-WAIT socket); proceeding to fatal notify", self.name)
  2258. except Exception as stop_error:
  2259. logger.warning(
  2260. "[%s] Failed stopping Telegram updater after exhausting conflict retries: %s", self.name, stop_error,
  2261. exc_info=True,
  2262. )
  2263. if not _already_fatal:
  2264. await self._handoff_polling_fatal_error()
  2265. async def _handoff_polling_fatal_error(self) -> None:
  2266. """Notify the runner without letting child teardown cancel this owner: ``disconnect()`` cancels
  2267. the tracked recovery/heartbeat tasks, so release only the current owner from its field."""
  2268. current_task = asyncio.current_task()
  2269. if self._polling_error_task is current_task:
  2270. self._polling_error_task = None
  2271. if getattr(self, "_polling_heartbeat_task", None) is current_task:
  2272. self._polling_heartbeat_task = None
  2273. await self._notify_fatal_error()
  2274. async def _create_dm_topic(
  2275. self, chat_id: int, name: str, icon_color: Optional[int] = None, icon_custom_emoji_id: Optional[str] = None) -> Optional[int]:
  2276. """Create a forum topic in a private (DM) chat (Bot API 9.4+); message_thread_id or None."""
  2277. if not self._bot:
  2278. return None
  2279. try:
  2280. kwargs: dict[str, Any] = {"chat_id": chat_id, "name": name}
  2281. if icon_color is not None:
  2282. kwargs["icon_color"] = icon_color
  2283. if icon_custom_emoji_id:
  2284. kwargs["icon_custom_emoji_id"] = icon_custom_emoji_id
  2285. topic = await self._bot.create_forum_topic(**kwargs)
  2286. thread_id = topic.message_thread_id
  2287. logger.info("[%s] Created DM topic '%s' in chat %s -> thread_id=%s", self.name, name, chat_id, thread_id)
  2288. return thread_id
  2289. except Exception as e:
  2290. error_text = str(e).lower()
  2291. # Telegram has no "list topics" API: an existing topic is mapped from incoming messages.
  2292. if "topic_name_duplicate" in error_text or "already" in error_text:
  2293. logger.info(
  2294. "[%s] DM topic '%s' already exists in chat %s (will be mapped from incoming messages)", self.name, name, chat_id)
  2295. elif "not a forum" in error_text or "forums_disabled" in error_text:
  2296. logger.warning(
  2297. "[%s] Cannot create DM topic '%s' in chat %s: Threaded Mode is not enabled. "
  2298. "The bot owner must open the BotFather Mini App (search 'botfather' in "
  2299. "Telegram, tap Open on the search result) -> My bots -> this bot -> Bot "
  2300. "Settings -> Threads Settings -> enable Threaded Mode. This cannot be enabled "
  2301. "from the DM chat, nor from the BotFather /mybots text menu.",
  2302. self.name, name, chat_id)
  2303. else:
  2304. logger.warning(
  2305. "[%s] Failed to create DM topic '%s' in chat %s: %s", self.name, name, chat_id, _redact_telegram_error_text(e))
  2306. return None
  2307. async def create_handoff_thread(self, parent_chat_id: str, name: str) -> Optional[str]:
  2308. """Create a forum topic for a session handoff; ``message_thread_id`` as str, or None."""
  2309. try:
  2310. chat_id_int = int(parent_chat_id)
  2311. except (TypeError, ValueError):
  2312. return None
  2313. thread_id = await self._create_dm_topic(chat_id_int, name=name)
  2314. return str(thread_id) if thread_id else None
  2315. async def ensure_dm_topic(self, chat_id: str, topic_name: str, force_create: bool = False) -> Optional[str]:
  2316. """Return a private DM topic thread id, creating and persisting it if needed."""
  2317. name = str(topic_name or "").strip()
  2318. if not name:
  2319. return None
  2320. try:
  2321. chat_id_int = int(chat_id)
  2322. except (TypeError, ValueError):
  2323. return None
  2324. cache_key = f"{chat_id_int}:{name}"
  2325. cached = self._dm_topics.get(cache_key)
  2326. if cached and not force_create:
  2327. return str(cached)
  2328. topic_conf: Optional[dict[str, Any]] = None
  2329. chat_entry: Optional[dict[str, Any]] = None
  2330. for entry in self._dm_topics_config:
  2331. if str(entry.get("chat_id")) != str(chat_id_int):
  2332. continue
  2333. chat_entry = entry
  2334. topic_conf = next((c for c in entry.get("topics", []) if c.get("name") == name), None)
  2335. break
  2336. if topic_conf and topic_conf.get("thread_id") and not force_create:
  2337. thread_id = int(topic_conf["thread_id"])
  2338. self._dm_topics[cache_key] = thread_id
  2339. return str(thread_id)
  2340. if chat_entry is None:
  2341. chat_entry = {"chat_id": chat_id_int, "topics": []}
  2342. self._dm_topics_config.append(chat_entry)
  2343. if topic_conf is None:
  2344. topic_conf = {"name": name}
  2345. chat_entry.setdefault("topics", []).append(topic_conf)
  2346. thread_id = await self._create_dm_topic(
  2347. chat_id_int, name=name, icon_color=topic_conf.get("icon_color"), icon_custom_emoji_id=topic_conf.get("icon_custom_emoji_id"))
  2348. if not thread_id:
  2349. return None
  2350. topic_conf["thread_id"] = thread_id
  2351. self._dm_topics[cache_key] = int(thread_id)
  2352. self._persist_dm_topic_thread_id(chat_id_int, name, int(thread_id), replace_existing=force_create)
  2353. return str(thread_id)
  2354. async def rename_dm_topic(self, chat_id: int, thread_id: int, name: str) -> None:
  2355. """Rename a forum topic in a private (DM) chat."""
  2356. if not self._bot:
  2357. return
  2358. try:
  2359. chat_id_arg = int(chat_id)
  2360. except (TypeError, ValueError):
  2361. chat_id_arg = chat_id
  2362. await self._bot.edit_forum_topic(chat_id=chat_id_arg, message_thread_id=int(thread_id), name=name)
  2363. logger.info("[%s] Renamed DM topic in chat %s thread_id=%s -> '%s'", self.name, chat_id, thread_id, name)
  2364. def _persist_dm_topic_thread_id(self, chat_id: int, topic_name: str, thread_id: int, replace_existing: bool = False) -> None:
  2365. """Save a newly created thread_id back into config.yaml so it survives restarts."""
  2366. try:
  2367. from hermes_constants import get_hermes_home
  2368. from hermes_cli.config import atomic_config_write, config_exists, read_user_config_raw
  2369. config_path = get_hermes_home() / "config.yaml"
  2370. if not config_exists(config_path):
  2371. logger.warning("[%s] Config file not found at %s, cannot persist thread_id", self.name, config_path)
  2372. return
  2373. config = read_user_config_raw(config_path)
  2374. # platforms.telegram.extra.dm_topics — create the path for topics not predeclared in config.yaml.
  2375. dm_topics = config.setdefault("platforms", {}).setdefault("telegram", {}).setdefault("extra", {}).setdefault("dm_topics", [])
  2376. changed = False
  2377. matching_chat_entry = None
  2378. for chat_entry in dm_topics:
  2379. try:
  2380. if int(chat_entry.get("chat_id", 0)) != int(chat_id):
  2381. continue
  2382. except (TypeError, ValueError):
  2383. continue
  2384. matching_chat_entry = chat_entry
  2385. topics = chat_entry.setdefault("topics", [])
  2386. t = next((t for t in topics if t.get("name") == topic_name), None)
  2387. if t is None:
  2388. topics.append({"name": topic_name, "thread_id": thread_id})
  2389. changed = True
  2390. elif (replace_existing or not t.get("thread_id")) and t.get("thread_id") != thread_id:
  2391. t["thread_id"] = thread_id
  2392. changed = True
  2393. break
  2394. if matching_chat_entry is None:
  2395. dm_topics.append({"chat_id": chat_id, "topics": [{"name": topic_name, "thread_id": thread_id}]})
  2396. changed = True
  2397. if changed:
  2398. atomic_config_write(config_path, config)
  2399. logger.info("[%s] Persisted thread_id=%s for topic '%s' in config.yaml", self.name, thread_id, topic_name)
  2400. except Exception as e:
  2401. logger.warning("[%s] Failed to persist thread_id to config: %s", self.name, e, exc_info=True)
  2402. async def _setup_dm_topics(self) -> None:
  2403. """Load or create configured DM topics: ``extra['dm_topics']`` is ``[{"chat_id", "topics": [{"name",
  2404. "icon_color", "thread_id"?, "skill"?}]}]``; persisted thread_ids are cached without an API call."""
  2405. for chat_entry in self._dm_topics_config or ():
  2406. chat_id = chat_entry.get("chat_id")
  2407. topics = chat_entry.get("topics", [])
  2408. if not chat_id or not topics:
  2409. continue
  2410. logger.info("[%s] Setting up %d DM topic(s) for chat %s", self.name, len(topics), chat_id)
  2411. for topic_conf in topics:
  2412. topic_name = topic_conf.get("name")
  2413. if not topic_name:
  2414. continue
  2415. cache_key = f"{chat_id}:{topic_name}"
  2416. existing_thread_id = topic_conf.get("thread_id")
  2417. if existing_thread_id:
  2418. self._dm_topics[cache_key] = int(existing_thread_id)
  2419. logger.info("[%s] DM topic loaded from config: %s -> thread_id=%s", self.name, cache_key, existing_thread_id)
  2420. continue
  2421. thread_id = await self._create_dm_topic(
  2422. chat_id=normalize_telegram_chat_id(chat_id), name=topic_name, icon_color=topic_conf.get("icon_color"),
  2423. icon_custom_emoji_id=topic_conf.get("icon_custom_emoji_id"))
  2424. if not thread_id:
  2425. continue
  2426. self._dm_topics[cache_key] = thread_id
  2427. logger.info("[%s] DM topic cached: %s -> thread_id=%s", self.name, cache_key, thread_id)
  2428. self._persist_dm_topic_thread_id(int(chat_id), topic_name, thread_id)
  2429. # Seed message: Telegram's client hides empty topics until they contain one.
  2430. try:
  2431. await _await_with_thread_deadline(
  2432. self._bot.send_message(
  2433. chat_id=normalize_telegram_chat_id(chat_id), message_thread_id=thread_id, text=f"\U0001f4cc {topic_name}"),
  2434. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  2435. except Exception as seed_err:
  2436. logger.debug("[%s] Could not send seed message to topic '%s': %s", self.name, topic_name, seed_err)
  2437. async def _bot_identity_refresh_loop(self) -> None:
  2438. """Keep the cached @username fresh in webhook mode (no heartbeat calls ``get_me()`` there)."""
  2439. while True:
  2440. try:
  2441. await asyncio.sleep(self._BOT_IDENTITY_TTL_SECONDS)
  2442. if self._teardown_started or self.has_fatal_error:
  2443. return
  2444. await self._refresh_bot_identity(force=True)
  2445. except asyncio.CancelledError:
  2446. return
  2447. except Exception:
  2448. logger.debug("[%s] Telegram identity refresh loop iteration failed", self.name, exc_info=True)
  2449. def _start_post_connect_housekeeping(self) -> None:
  2450. """Kick off deferred post-connect housekeeping; idempotent while a task is still running."""
  2451. task = self._post_connect_task
  2452. if task and not task.done():
  2453. return
  2454. self._post_connect_task = asyncio.ensure_future(self._run_post_connect_housekeeping())
  2455. async def _register_command_menu(self) -> None:
  2456. """Register the command menu (from COMMAND_REGISTRY) in every scope — Telegram picks the
  2457. narrowest matching one per chat type; forum topics are handled lazily by _ensure_forum_commands."""
  2458. from telegram import BotCommandScopeAllPrivateChats, BotCommandScopeAllGroupChats, BotCommandScopeDefault
  2459. from hermes_cli.commands_platforms import telegram_menu_commands, telegram_menu_max_commands
  2460. if not self._bot:
  2461. return
  2462. # Telegram allows 100 commands but has an undocumented ~4KB payload limit; default cap 60.
  2463. max_commands = telegram_menu_max_commands()
  2464. # Skill discovery resolves every skill path on disk; a slow filesystem after a reconnect must
  2465. # not hold the gateway loop past the liveness watchdog (#110707). Only the Bot API call stays here.
  2466. menu_commands, hidden_count = await asyncio.to_thread(telegram_menu_commands, max_commands=max_commands)
  2467. bot_commands = self._bot_commands(menu_commands)
  2468. # Descriptions are localized, so the fingerprint carries the language: a display.language change
  2469. # invalidates the lazily registered forum scopes (and any future "skip if unchanged" check).
  2470. fingerprint = self._menu_fingerprint(menu_commands)
  2471. if fingerprint != getattr(self, "_command_menu_fingerprint", ""):
  2472. self._forum_command_registered.clear()
  2473. self._command_menu_fingerprint = fingerprint
  2474. for scope_cls in (BotCommandScopeDefault, BotCommandScopeAllPrivateChats, BotCommandScopeAllGroupChats):
  2475. scope_name = getattr(scope_cls, "__name__", str(scope_cls))
  2476. try:
  2477. await self._bot.set_my_commands(bot_commands, scope=scope_cls())
  2478. logger.info("[%s] set_my_commands OK for scope %s (%d cmds)", self.name, scope_name, len(bot_commands))
  2479. except Exception as scope_err:
  2480. logger.warning("[%s] set_my_commands FAILED for scope %s: %s", self.name, scope_name, scope_err)
  2481. if hidden_count:
  2482. logger.info(
  2483. "[%s] Telegram menu: %d commands registered, %d hidden (over %d limit). Use /commands for full list.",
  2484. self.name, len(menu_commands), hidden_count, max_commands)
  2485. @staticmethod
  2486. def _bot_commands(menu_commands) -> list:
  2487. """``BotCommand`` rows; Telegram caps a description at 256 chars (longer locales get cut)."""
  2488. from telegram import BotCommand
  2489. return [BotCommand(name, str(desc or "")[:_BOT_COMMAND_DESCRIPTION_LIMIT]) for name, desc in menu_commands]
  2490. @staticmethod
  2491. def _menu_fingerprint(menu_commands) -> str:
  2492. import hashlib
  2493. payload = repr([get_language(), [tuple(row) for row in menu_commands]])
  2494. return hashlib.sha256(payload.encode("utf-8")).hexdigest()
  2495. async def _run_post_connect_housekeeping(self) -> None:
  2496. """Command menu, status indicator and DM topics off the connect path; every step is non-fatal.
  2497. DM topics — all off the connect path so a slow Bot API call cannot blow the gateway connect timeout
  2498. (#46298).
  2499. """
  2500. try:
  2501. try:
  2502. await self._register_command_menu()
  2503. except Exception as e:
  2504. logger.warning(
  2505. "[%s] Could not register Telegram command menu: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
  2506. with contextlib.suppress(Exception):
  2507. await self._set_status_indicator(online=True)
  2508. try:
  2509. await self._setup_dm_topics()
  2510. except Exception as topics_err:
  2511. logger.warning("[%s] DM topics setup failed (non-fatal): %s", self.name, topics_err, exc_info=True)
  2512. except asyncio.CancelledError:
  2513. raise
  2514. finally:
  2515. if self._post_connect_task is asyncio.current_task():
  2516. self._post_connect_task = None
  2517. async def _on_platform_update(self, update, context) -> None:
  2518. """Catch-all PTB handler (group 99) firing ``gateway_platform_event`` per inbound update with a
  2519. stable envelope (no raw SDK objects) and an internal auth source. Never raises into PTB."""
  2520. # Admission counts dispatch before any preparation can stop this group. Retain
  2521. # accounting for callers outside that Application boundary (#102260).
  2522. admission = getattr(self, "_update_admission", None)
  2523. if admission is None or admission.get() is None:
  2524. self._updates_dispatched_total = getattr(self, "_updates_dispatched_total", 0) + 1
  2525. handler: Optional[Callable[[dict[str, Any], Any], Awaitable[None]]] = getattr(self, "_platform_event_handler", None)
  2526. if handler is None:
  2527. return
  2528. try:
  2529. from hermes_cli.lifecycle import has_hook
  2530. if not has_hook("gateway_platform_event"):
  2531. return
  2532. event = self._normalize_platform_event(update)
  2533. except Exception:
  2534. self._fail_update_preparation()
  2535. logger.debug("[%s] gateway_platform_event normalize error", self.name, exc_info=True)
  2536. return
  2537. if event is None:
  2538. return
  2539. # The gateway-owned boundary runs the full profile-scoped auth chain before plugin dispatch.
  2540. try:
  2541. source = self._source_for_platform_event_auth(update)
  2542. self._accept_update()
  2543. await handler(event, source)
  2544. except Exception:
  2545. self._fail_update_preparation()
  2546. logger.debug("[%s] gateway_platform_event dispatch error", self.name, exc_info=True)
  2547. def _source_for_platform_event_auth(self, update):
  2548. """Route a supported update to its event-specific auth-source extractor (reactor / editor);
  2549. raises ``ValueError`` for updates without one so the boundary fails closed."""
  2550. if getattr(update, "message_reaction", None) is not None:
  2551. return self._source_from_reaction_for_auth(update)
  2552. edited = getattr(update, "edited_message", None)
  2553. if edited is not None:
  2554. source = self._source_from_message_for_auth(edited)
  2555. # Tolerates missing identities for pairing-flow callers; this boundary must not.
  2556. if not source.user_id or not source.chat_id:
  2557. raise ValueError("gateway_platform_event message_edited requires editor and chat identities")
  2558. return source
  2559. raise ValueError("gateway_platform_event source extraction has no extractor for this update type")
  2560. def _normalize_platform_event(self, update) -> Optional[dict[str, Any]]:
  2561. """Map a PTB update to a ``{platform, event_type, payload}`` envelope (hooks.md contracts), or
  2562. ``None`` for types without one."""
  2563. if getattr(update, "message_reaction", None) is not None:
  2564. return self._normalize_reaction_event(update)
  2565. if getattr(update, "edited_message", None) is not None:
  2566. return self._normalize_message_edited_event(update)
  2567. return None
  2568. @staticmethod
  2569. def _is_id_like(value: Any) -> bool:
  2570. return not isinstance(value, bool) and isinstance(value, (str, int))
  2571. def _normalize_reaction_event(self, update) -> Optional[dict[str, Any]]:
  2572. """``message_reaction`` → ``reaction`` event: emojis (unicode), custom_emoji_ids, chat_id,
  2573. message_id, thread_id (always None — reactions carry none)."""
  2574. mr = getattr(update, "message_reaction", None)
  2575. if mr is None:
  2576. return None
  2577. chat = getattr(mr, "chat", None)
  2578. new_reaction = getattr(mr, "new_reaction", None) or []
  2579. if not isinstance(new_reaction, (list, tuple)):
  2580. return None
  2581. chat_id = getattr(chat, "id", None) if chat is not None else None
  2582. message_id = getattr(mr, "message_id", None)
  2583. if not self._is_id_like(chat_id) or not self._is_id_like(message_id):
  2584. return None
  2585. emojis: list[str] = []
  2586. custom_emoji_ids: list[str] = []
  2587. for r in new_reaction[:64]:
  2588. emoji = getattr(r, "emoji", None)
  2589. if isinstance(emoji, str) and emoji:
  2590. emojis.append(emoji[:64])
  2591. custom_id = getattr(r, "custom_emoji_id", None)
  2592. if self._is_id_like(custom_id):
  2593. custom_emoji_ids.append(str(custom_id)[:128])
  2594. return {
  2595. "platform": "telegram",
  2596. "event_type": "reaction",
  2597. "payload": {
  2598. "emojis": emojis, "custom_emoji_ids": custom_emoji_ids, "chat_id": str(chat_id)[:128],
  2599. "message_id": str(message_id)[:128], "thread_id": None},
  2600. }
  2601. def _normalize_message_edited_event(self, update) -> Optional[dict[str, Any]]:
  2602. """``edited_message`` → ``message_edited`` event (v1, additive): chat_id, message_id, thread_id
  2603. (forum topic), text (edited text or caption, bounded), edited_at (ISO 8601 UTC or None)."""
  2604. message = getattr(update, "edited_message", None)
  2605. if message is None:
  2606. return None
  2607. chat = getattr(message, "chat", None)
  2608. chat_id = getattr(chat, "id", None) if chat is not None else None
  2609. message_id = getattr(message, "message_id", None)
  2610. if not self._is_id_like(chat_id) or not self._is_id_like(message_id):
  2611. return None
  2612. text = getattr(message, "text", None) or getattr(message, "caption", None)
  2613. if not isinstance(text, str):
  2614. text = None
  2615. thread_id = None
  2616. thread_id_raw = getattr(message, "message_thread_id", None)
  2617. if self._is_id_like(thread_id_raw) and bool(getattr(message, "is_topic_message", False)):
  2618. thread_id = str(thread_id_raw)[:128]
  2619. edited_at = None
  2620. edit_date = getattr(message, "edit_date", None)
  2621. try:
  2622. if edit_date is not None and hasattr(edit_date, "isoformat"):
  2623. edited_at = str(edit_date.isoformat())[:64]
  2624. except Exception:
  2625. edited_at = None
  2626. return {
  2627. "platform": "telegram",
  2628. "event_type": "message_edited",
  2629. "payload": {
  2630. "chat_id": str(chat_id)[:128], "message_id": str(message_id)[:128], "thread_id": thread_id,
  2631. "text": text[:8192] if text is not None else None, "edited_at": edited_at},
  2632. }
  2633. def _accept_update(self) -> None:
  2634. """Handoff is irreversible for replay purposes, even if later work raises/cancels."""
  2635. admission = getattr(self, "_update_admission", None)
  2636. claim = admission.get() if admission is not None else None
  2637. if claim is not None:
  2638. claim.accepted = True
  2639. def _fail_update_preparation(self) -> None:
  2640. """A caught preparation error is not a successful no-op or an auth refusal."""
  2641. admission = getattr(self, "_update_admission", None)
  2642. claim = admission.get() if admission is not None else None
  2643. if claim is not None:
  2644. claim.failed = True
  2645. async def handle_message(self, event: MessageEvent) -> None:
  2646. self._accept_update()
  2647. await apply_root_dm_topic_transfer(self, event) # opt-in dm_topic_auto_transfer (#139136)
  2648. await super().handle_message(event)
  2649. def set_topic_transfer_fn(self, fn: Optional[TopicTransferFn]) -> None:
  2650. """Install the runner's root-DM topic-transfer hook (``dm_topic_auto_transfer``); None clears."""
  2651. self._topic_transfer_fn = fn
  2652. def _register_handlers(self, app) -> None:
  2653. """Register every PTB handler on ``app`` (initial connect and the transient-init rebuild)."""
  2654. table = getattr(app, "handlers", None)
  2655. core_before = {g: len(hs) for g, hs in table.items()} if isinstance(table, dict) else {}
  2656. app.add_handler(TelegramMessageHandler(filters.TEXT & ~filters.COMMAND, self._handle_text_message))
  2657. app.add_handler(TelegramMessageHandler(filters.COMMAND, self._handle_command))
  2658. app.add_handler(TelegramMessageHandler(
  2659. filters.LOCATION | getattr(filters, "VENUE", filters.LOCATION), self._handle_location_message))
  2660. app.add_handler(TelegramMessageHandler(
  2661. filters.PHOTO | filters.VIDEO | filters.VIDEO_NOTE | filters.AUDIO | filters.VOICE | filters.Document.ALL | filters.Sticker.ALL,
  2662. self._handle_media_message))
  2663. # Topic-creation service messages carry the new topic id and no text, so no other handler sees
  2664. # them. They are a routing signal only (never an agent turn): see _handle_forum_topic_created.
  2665. created_filter = getattr(getattr(filters, "StatusUpdate", None), "FORUM_TOPIC_CREATED", None)
  2666. if created_filter is not None:
  2667. app.add_handler(TelegramMessageHandler(created_filter, self._handle_forum_topic_created))
  2668. app.add_handler(CallbackQueryHandler(self._handle_callback_query))
  2669. # Inline command picker; inert until the owner enables inline mode via BotFather /setinline.
  2670. app.add_handler(InlineQueryHandler(self._handle_inline_query))
  2671. # gateway_platform_event observer: group 99 observes alongside, never displaces, core handlers.
  2672. app.add_handler(TypeHandler(Update, self._on_platform_update), group=99)
  2673. # Everything appended above is core; a late plugin re-wire must land BEFORE these (#87770).
  2674. if isinstance(table, dict):
  2675. self._core_handler_ids = {id(h) for g, hs in table.items() for h in hs[core_before.get(g, 0):]}
  2676. def _wire_plugin_handlers(self, native: Any = None) -> None:
  2677. """PTB dispatches the FIRST matching handler per group and core registers catch-alls
  2678. (``filters.COMMAND``, ``CallbackQueryHandler``), so a plugin handler appended after connect
  2679. would never fire. Move whatever a late factory added ahead of the first core handler of its
  2680. group, keeping the plugin handlers' own relative order."""
  2681. handlers = getattr(native, "handlers", None)
  2682. core_ids = getattr(self, "_core_handler_ids", None)
  2683. if not isinstance(handlers, dict) or not core_ids:
  2684. super()._wire_plugin_handlers(native) # first wire runs before core registers: nothing to hoist
  2685. return
  2686. before = {g: list(hs) for g, hs in handlers.items()}
  2687. super()._wire_plugin_handlers(native)
  2688. for group, current in handlers.items():
  2689. prior = before.get(group, [])
  2690. prior_ids = {id(h) for h in prior}
  2691. added = [h for h in current if id(h) not in prior_ids]
  2692. first_core = next((i for i, h in enumerate(prior) if id(h) in core_ids), None)
  2693. if not added or first_core is None:
  2694. continue
  2695. current[:] = prior[:first_core] + added + prior[first_core:]
  2696. async def _build_ptb_requests(self) -> tuple:
  2697. """Build the (general, getUpdates) HTTPXRequest pair: fallback-IP transport, explicit proxy, or
  2698. direct DNS; the getUpdates request is instrumented for polling-progress tracking."""
  2699. # PTB's pool_timeout=1s default trips "Pool timeout" on flaky networks; safer defaults + env overrides.
  2700. request_kwargs = {
  2701. "connection_pool_size": env_int("HERMES_TELEGRAM_HTTP_POOL_SIZE", 512),
  2702. "pool_timeout": env_float("HERMES_TELEGRAM_HTTP_POOL_TIMEOUT", 8.0),
  2703. "connect_timeout": env_float("HERMES_TELEGRAM_HTTP_CONNECT_TIMEOUT", 10.0),
  2704. "read_timeout": env_float("HERMES_TELEGRAM_HTTP_READ_TIMEOUT", 20.0),
  2705. "write_timeout": env_float("HERMES_TELEGRAM_HTTP_WRITE_TIMEOUT", 20.0),
  2706. # PTB routes file requests to media_write_timeout; httpx budgets it per socket write (stall
  2707. # tolerance, not bandwidth), so 60s rides out congested-link buffer stalls.
  2708. "media_write_timeout": 60.0,
  2709. }
  2710. # CLOSE_WAIT fd leak: PTB's httpx.AsyncClient has no keepalive tuning; inject platform_httpx_limits()
  2711. # while preserving PTB's max_connections (httpx_kwargs is spread last, so `limits` here wins).
  2712. # CLOSE_WAIT fd leak (#31599, same class as #18451): PTB's HTTPXRequest builds the underlying
  2713. # httpx.AsyncClient with `limits = httpx.Limits(max_connections=connection_pool_size)` and *no*
  2714. # keepalive tuning, so httpx's default keepalive_expiry=5.0 applies. Behind an HTTP proxy
  2715. # (Cloudflare Warp etc.) a peer-initiated FIN can sit in CLOSE_WAIT longer than that, leaking fds in
  2716. # the general request pool (_request[1]) which _drain_polling_connections never resets.
  2717. from gateway.platforms._http_client_limits import platform_httpx_limits
  2718. _base_limits = platform_httpx_limits()
  2719. if _base_limits is not None:
  2720. import httpx as _httpx
  2721. _pool_limits = _httpx.Limits(
  2722. max_connections=request_kwargs["connection_pool_size"],
  2723. max_keepalive_connections=_base_limits.max_keepalive_connections, keepalive_expiry=_base_limits.keepalive_expiry)
  2724. # A long-poll is continuously active, so keepalive expiry can't protect it from a server-side
  2725. # close: never hand getUpdates a pooled socket from a previous poll.
  2726. _updates_limits = _httpx.Limits(
  2727. max_connections=request_kwargs["connection_pool_size"], max_keepalive_connections=0,
  2728. keepalive_expiry=_base_limits.keepalive_expiry)
  2729. else: # pragma: no cover — httpx always present alongside PTB
  2730. _pool_limits = _updates_limits = None
  2731. def _with_limits(httpx_kwargs: Optional[dict] = None) -> dict:
  2732. """Merge tuned limits into httpx client kwargs (proxy/direct branches only; the fallback-IP
  2733. branch must pass limits straight into the transport — httpx ignores client `limits` then)."""
  2734. kwargs = dict(httpx_kwargs or {})
  2735. if _pool_limits is not None and "limits" not in kwargs:
  2736. kwargs["limits"] = _pool_limits
  2737. return kwargs
  2738. disable_fallback = os.getenv("HERMES_TELEGRAM_DISABLE_FALLBACK_IPS", "").strip().lower() in {"1", "true", "yes", "on"}
  2739. fallback_ips = [] if disable_fallback else self._fallback_ips()
  2740. if not fallback_ips and not disable_fallback:
  2741. discovery_timeout = self._env_float_clamped("HERMES_TELEGRAM_FALLBACK_DISCOVERY_TIMEOUT", 5.0, min_value=0.0)
  2742. logger.warning("[%s] Discovering Telegram API fallback IPs via DNS-over-HTTPS…", self.name)
  2743. try:
  2744. fallback_ips = await _await_with_thread_deadline(discover_fallback_ips(), timeout=discovery_timeout)
  2745. except Exception as exc:
  2746. logger.warning(
  2747. "[%s] Telegram fallback-IP discovery failed after %.0fs; "
  2748. "using seed IPv4 Telegram API IPs so a blackholed IPv6 hostname path cannot hang initialize() (#87015): %s",
  2749. self.name, discovery_timeout, _redact_telegram_error_text(exc))
  2750. fallback_ips = list(SEED_FALLBACK_IPS)
  2751. else:
  2752. logger.info("[%s] Auto-discovered Telegram fallback IPs: %s", self.name, ", ".join(fallback_ips))
  2753. proxy_url = resolve_proxy_url(
  2754. "TELEGRAM_PROXY", target_hosts=["api.telegram.org", *fallback_ips],
  2755. configured=self.config.extra.get("proxy_url"))
  2756. def _pair(general_httpx: dict, updates_httpx: dict, **extra) -> tuple:
  2757. return (HTTPXRequest(**request_kwargs, **extra, httpx_kwargs={"verify": platform_ssl_context(), **general_httpx}),
  2758. HTTPXRequest(**request_kwargs, **extra, httpx_kwargs={"verify": platform_ssl_context(), **updates_httpx}))
  2759. if fallback_ips and not proxy_url and not disable_fallback:
  2760. logger.info("[%s] Telegram fallback IPs active: %s", self.name, ", ".join(fallback_ips))
  2761. _transport_kwargs: dict = {"socket_options": tcp_keepalive_socket_options(), "verify": platform_ssl_context()}
  2762. # Keep request/update pools separate to reduce contention during polling reconnect + bot API
  2763. # bootstrap/delete_webhook calls. httpx ignores the client-level `limits` kwarg when a custom
  2764. # `transport` is supplied (#58790). Unlike the proxy/direct branches (which inject limits at the
  2765. # client level via `_with_limits`), this branch MUST pass the tuned limits directly into
  2766. # TelegramFallbackTransport so its inner AsyncHTTPTransport instances honour keepalive_expiry —
  2767. # do not route this through `_with_limits`, httpx would discard it.
  2768. if _pool_limits is not None:
  2769. _transport_kwargs["limits"] = _pool_limits
  2770. _updates_transport_kwargs = dict(_transport_kwargs)
  2771. if _updates_limits is not None:
  2772. _updates_transport_kwargs["limits"] = _updates_limits
  2773. request, get_updates_request = _pair(
  2774. {"transport": TelegramFallbackTransport(fallback_ips, **_transport_kwargs)},
  2775. {"transport": TelegramFallbackTransport(fallback_ips, **_updates_transport_kwargs)})
  2776. elif proxy_url:
  2777. logger.info("[%s] Proxy detected; passing explicitly to HTTPXRequest: %s", self.name, proxy_url)
  2778. request, get_updates_request = _pair(_with_limits(), {"limits": _updates_limits}, proxy=proxy_url)
  2779. else:
  2780. if disable_fallback:
  2781. logger.info("[%s] Telegram fallback-IP transport disabled via env", self.name)
  2782. request, get_updates_request = _pair(_with_limits(), {"limits": _updates_limits})
  2783. return request, self._instrument_polling_request(get_updates_request)
  2784. async def _initialize_app_with_retries(self, builder) -> None:
  2785. """``app.initialize()`` with a bounded retry ladder; rebuilds ``self._app``/``self._bot`` from
  2786. ``builder`` after each failed attempt; OSError when the per-attempt or total watchdog expires."""
  2787. _max_connect = 8
  2788. _init_timeout = env_float("HERMES_TELEGRAM_INIT_TIMEOUT", 30.0) # per attempt
  2789. # Total watchdog: bounds the whole connect loop even if the retry loop silently stalls.
  2790. _total_deadline = asyncio.get_running_loop().time() + _init_timeout * _max_connect + 120.0
  2791. _timed_out = f"Telegram initialization timed out after {_max_connect} attempts ({_init_timeout:.0f}s each)"
  2792. for _attempt in range(_max_connect):
  2793. rebuild_app = False
  2794. try:
  2795. if asyncio.get_running_loop().time() >= _total_deadline:
  2796. raise OSError(
  2797. f"{_timed_out} — total connect watchdog deadline ({_init_timeout * _max_connect + 120.0:.0f}s) exceeded. "
  2798. f"Check network connectivity to api.telegram.org or set HERMES_TELEGRAM_HTTP_CONNECT_TIMEOUT / "
  2799. f"HERMES_TELEGRAM_INIT_TIMEOUT to a lower value.")
  2800. logger.warning("[%s] Connecting to Telegram (attempt %d/%d)…", self.name, _attempt + 1, _max_connect)
  2801. # On timeout the (possibly shielded) initialize() task is abandoned; release the half-built
  2802. # app's httpx client so it isn't leaked across the ladder.
  2803. await _await_with_thread_deadline(
  2804. self._app.initialize(), timeout=_init_timeout, on_abandon=lambda app=self._app: _shutdown_abandoned_app(app),
  2805. label="telegram-init")
  2806. break
  2807. except TimeoutError:
  2808. rebuild_app = True
  2809. if _attempt >= _max_connect - 1:
  2810. raise OSError(
  2811. f"{_timed_out}. Check network connectivity to api.telegram.org "
  2812. f"or set HERMES_TELEGRAM_HTTP_CONNECT_TIMEOUT to a lower value.")
  2813. wait = min(2 ** _attempt, 15)
  2814. logger.warning(
  2815. "[%s] Connect attempt %d/%d timed out after %.0fs — retrying in %ds", self.name, _attempt + 1,
  2816. _max_connect, _init_timeout, wait)
  2817. await asyncio.sleep(wait)
  2818. except Exception as init_err:
  2819. # OSError always retries; anything else only when it looks like a network error.
  2820. rebuild_app = True
  2821. if (not isinstance(init_err, OSError) and not self._looks_like_network_error(init_err)) or _attempt >= _max_connect - 1:
  2822. raise
  2823. wait = min(2 ** _attempt, 15)
  2824. logger.warning(
  2825. "[%s] Connect attempt %d/%d failed: %s — retrying in %ds", self.name, _attempt + 1, _max_connect, init_err, wait)
  2826. await asyncio.sleep(wait)
  2827. except BaseException:
  2828. # CancelledError etc.: log for the operator, then reraise. LAST so the Exception handlers win.
  2829. logger.warning(
  2830. "[%s] Connect attempt %d/%d interrupted by %s — propagating", self.name, _attempt + 1, _max_connect,
  2831. "CancelledError" if isinstance(sys.exc_info()[1], asyncio.CancelledError) else type(sys.exc_info()[1]).__name__)
  2832. raise
  2833. finally:
  2834. # A failed attempt may leave the app half-initialized: rebuild a fresh Application from the
  2835. # same builder for the next attempt and discard the old one.
  2836. if rebuild_app and _attempt < _max_connect - 1:
  2837. old_app = self._app
  2838. self._app = builder.build()
  2839. self._bot = self._app.bot
  2840. # Same order as connect(): plugin handlers first (the wired-set is keyed per app, so
  2841. # the rebuilt app gets them again), then core and the observer in lockstep.
  2842. self._wire_plugin_handlers(self._app)
  2843. self._register_handlers(self._app)
  2844. with contextlib.suppress(Exception):
  2845. await _shutdown_abandoned_app(old_app)
  2846. def _cold_boot_drop_pending(self, *, is_reconnect: bool) -> bool:
  2847. """Whether THIS connection asks Telegram to discard its queued updates.
  2848. A watcher reconnect always preserves them (#46621); a cold boot follows
  2849. ``platforms.telegram.extra.drop_pending_on_cold_boot`` (default true). The decision is logged
  2850. on every cold boot — a command that never ran is otherwise invisible (#71811)."""
  2851. drop_pending = self._drop_pending_on_cold_boot if not is_reconnect else False
  2852. if not is_reconnect:
  2853. logger.info(
  2854. "[%s] Cold boot: %s Telegram updates queued while offline "
  2855. "(platforms.telegram.extra.drop_pending_on_cold_boot: %s)",
  2856. self.name, "dropping" if drop_pending else "preserving",
  2857. "true" if self._drop_pending_on_cold_boot else "false")
  2858. return drop_pending
  2859. async def _start_webhook_mode(self, webhook_url: str, *, is_reconnect: bool) -> None:
  2860. """Start PTB's webhook server (Telegram pushes updates; lets cloud platforms auto-wake suspended
  2861. machines). SECURITY: TELEGRAM_WEBHOOK_SECRET is REQUIRED — without it the endpoint accepts forged
  2862. updates (GHSA-3vpc-7q5r-276h); refuse to start rather than run fail-open."""
  2863. webhook_port = env_int("TELEGRAM_WEBHOOK_PORT", 8443)
  2864. # Default "" → tornado listens on IPv4 + IPv6; "0.0.0.0" is unreachable on IPv6-only networks.
  2865. webhook_host = (os.getenv("TELEGRAM_WEBHOOK_HOST", "").strip() or str((self.config.extra or {}).get("webhook_host") or "").strip())
  2866. webhook_secret = (_get_scoped_secret("TELEGRAM_WEBHOOK_SECRET") or "").strip()
  2867. if not webhook_secret:
  2868. raise RuntimeError(
  2869. "TELEGRAM_WEBHOOK_SECRET is required when TELEGRAM_WEBHOOK_URL is set. Without it, the "
  2870. "webhook endpoint accepts forged updates from anyone who can reach it — see "
  2871. "https://github.com/NousResearch/hermes-agent/security/advisories/GHSA-3vpc-7q5r-276h.\n\n"
  2872. "Generate a secret and set it in your .env:\n export TELEGRAM_WEBHOOK_SECRET=\"$(openssl rand -hex 32)\"\n\n"
  2873. "Then register it with Telegram when setting the webhook via setWebhook's secret_token parameter.")
  2874. from urllib.parse import urlparse
  2875. webhook_path = urlparse(webhook_url).path or "/telegram"
  2876. await self._app.updater.start_webhook(
  2877. listen=webhook_host, port=webhook_port, url_path=webhook_path, webhook_url=webhook_url,
  2878. secret_token=webhook_secret, allowed_updates=Update.ALL_TYPES,
  2879. drop_pending_updates=self._cold_boot_drop_pending(is_reconnect=is_reconnect),
  2880. )
  2881. self._webhook_mode = True
  2882. self._polling_progress_accepting = False
  2883. self._send_path_degraded = False
  2884. logger.info(
  2885. "[%s] Webhook server listening on %s:%d%s", self.name, webhook_host or "* (all interfaces, IPv4+IPv6)",
  2886. webhook_port, webhook_path)
  2887. async def _start_polling_mode(self, *, is_reconnect: bool) -> None:
  2888. """Clear any stale webhook and start resilient long polling."""
  2889. # Best-effort: a transient Bot API error must not fail gateway startup — degrade to recovery.
  2890. await self._delete_webhook_best_effort(require_success=not is_reconnect)
  2891. loop = asyncio.get_running_loop()
  2892. def _polling_error_callback(error: Exception) -> None:
  2893. if self._teardown_started or self._recovery_in_flight():
  2894. return
  2895. if self._looks_like_polling_conflict(error):
  2896. # Stop PTB's network_retry_loop synchronously BEFORE scheduling async recovery, else PTB's
  2897. # retry and our stop->restart overlap and produce a fresh 409.
  2898. self._disarm_ptb_retry_loop()
  2899. self._spawn_polling_recovery(loop, self._handle_polling_conflict(error))
  2900. elif self._looks_like_network_error(error):
  2901. logger.warning(
  2902. "[%s] Telegram network error, scheduling reconnect: %s", self.name, _redact_telegram_error_text(error))
  2903. self._spawn_polling_recovery(loop, self._handle_polling_network_error(error))
  2904. else:
  2905. logger.error("[%s] Telegram polling error: %s", self.name, _redact_telegram_error_text(error), exc_info=True)
  2906. self._polling_error_callback_ref = _polling_error_callback # reused by _handle_polling_conflict
  2907. drop_pending = self._cold_boot_drop_pending(is_reconnect=is_reconnect)
  2908. polling_started = await self._start_polling_resilient(
  2909. drop_pending_updates=drop_pending, error_callback=_polling_error_callback, require_progress=not is_reconnect)
  2910. if not polling_started:
  2911. logger.warning(
  2912. "[%s] Connected in degraded Telegram mode: gateway is alive, polling will be retried in the background", self.name)
  2913. async def connect(self, *, is_reconnect: bool = False) -> bool:
  2914. """Connect via long polling, or a webhook server if ``TELEGRAM_WEBHOOK_URL`` is set.
  2915. ``is_reconnect``: False = cold boot (drop the Bot API queue unless
  2916. ``extra.drop_pending_on_cold_boot`` is false); True = watcher reconnect (preserve
  2917. queued updates, else every message sent during the outage is lost). Webhook env:
  2918. TELEGRAM_WEBHOOK_URL, TELEGRAM_WEBHOOK_PORT (8443), TELEGRAM_WEBHOOK_HOST,
  2919. TELEGRAM_WEBHOOK_SECRET."""
  2920. # Explicit connect() is the only operation allowed to reopen polling after a completed teardown.
  2921. self._polling_teardown_started = False
  2922. self._webhook_mode = False # re-evaluated on every explicit connection
  2923. if not TELEGRAM_AVAILABLE:
  2924. logger.error("[%s] python-telegram-bot not installed. Run: pip install python-telegram-bot", self.name)
  2925. self._set_fatal_error("missing_dependency", "python-telegram-bot not installed", retryable=False)
  2926. return False
  2927. if not self.config.token:
  2928. logger.error("[%s] No bot token configured", self.name)
  2929. self._set_fatal_error("missing_credentials", "No bot token configured", retryable=False)
  2930. return False
  2931. try:
  2932. if not self._acquire_platform_lock('telegram-bot-token', self.config.token, 'Telegram bot token'):
  2933. return False
  2934. from plugins.platforms.telegram.update_admission import TelegramApplication, build_update_processor
  2935. builder = Application.builder().token(self.config.token)
  2936. builder.application_class(TelegramApplication, {"adapter": self})
  2937. custom_base_url = self.config.extra.get("base_url")
  2938. if custom_base_url:
  2939. builder = builder.base_url(custom_base_url)
  2940. builder = builder.base_file_url(self.config.extra.get("base_file_url", custom_base_url))
  2941. logger.info("[%s] Using custom Telegram base_url: %s", self.name, custom_base_url)
  2942. # Local-mode telegram-bot-api returns absolute server-side file paths; PTB needs local_mode=True
  2943. # so download_*() reads from disk instead of a 404ing HTTP GET.
  2944. if self.config.extra.get("local_mode"):
  2945. builder = builder.local_mode(True)
  2946. logger.info("[%s] Using Telegram local_mode (read files from disk)", self.name)
  2947. request, get_updates_request = await self._build_ptb_requests()
  2948. builder = builder.request(request).get_updates_request(get_updates_request)
  2949. # PTB's default processor awaits each update inline, so one slow turn deafens every chat.
  2950. # Concurrent across chats, FIFO within a chat; the builder keeps this instance, so the
  2951. # connect-retry rebuild in _initialize_app_with_retries gets it too.
  2952. builder = builder.concurrent_updates(build_update_processor(self.config.extra, self.name))
  2953. self._app = builder.build()
  2954. self._bot = self._app.bot
  2955. # Plugin PTB handlers go BEFORE core: PTB dispatches the first matching handler per group.
  2956. self._wire_plugin_handlers(self._app)
  2957. self._register_handlers(self._app)
  2958. await self._initialize_app_with_retries(builder)
  2959. await self._app.start()
  2960. # Profile-scoped like TELEGRAM_WEBHOOK_SECRET: under multiplex os.environ holds the DEFAULT
  2961. # profile's URL, and registering it on a secondary bot pushes that bot's updates to the
  2962. # default's listener (and stops polling for it).
  2963. webhook_url = (_get_scoped_secret("TELEGRAM_WEBHOOK_URL") or "").strip()
  2964. if webhook_url:
  2965. await self._start_webhook_mode(webhook_url, is_reconnect=is_reconnect)
  2966. else:
  2967. await self._start_polling_mode(is_reconnect=is_reconnect)
  2968. self._mark_connected()
  2969. # WARNING, not INFO: "Connecting…" above is WARNING and reaches the terminal; an INFO success
  2970. # line made healthy startups look stalled at "attempt 1/8".
  2971. logger.warning("[%s] Connected to Telegram (%s mode)", self.name, "webhook" if self._webhook_mode else "polling")
  2972. # Heartbeat only in polling mode: webhook mode has no long-poll socket to wedge in CLOSE-WAIT.
  2973. # WARNING, not INFO: the "Connecting to Telegram (attempt N/8)…" line above is emitted at
  2974. # WARNING and reaches the terminal (the gateway's default stderr handler is WARNING-only), but
  2975. # this success line was INFO and went to the log file only. A healthy startup therefore looked
  2976. # permanently stalled at "attempt 1/8" on the console — the logging illusion in #90835. Both
  2977. # sides of the connect transition must share a terminal-visible level so a real hang is the
  2978. # *absence* of this line, not ambiguity.
  2979. if not self._webhook_mode:
  2980. self._restart_task_attr("_polling_heartbeat_task", self._polling_heartbeat_loop())
  2981. # Seed the live identity from PTB's initialize() cache; polling rides the heartbeat's get_me(),
  2982. # webhook mode gets a low-frequency refresh loop (else a BotFather rename breaks routing).
  2983. self._note_bot_username(getattr(self._bot, "username", None))
  2984. self._bot_identity_checked_at = time.monotonic()
  2985. if self._webhook_mode:
  2986. self._restart_task_attr("_bot_identity_refresh_task", self._bot_identity_refresh_loop())
  2987. # Command menu / DM topics / status indicator can stall for some tokens: defer to a cancellable
  2988. # task so one slow call can't sink the (gateway-timed) connect while transport is live.
  2989. # Command-menu registration, DM-topic setup, and the status indicator each make Bot API calls
  2990. # that can stall for certain tokens. Running them here — inside the connect() coroutine that the
  2991. # gateway wraps in a connect timeout — means one slow call blows the whole connect and the
  2992. # adapter never comes up, even though polling/webhook is already live (#46298).
  2993. self._start_post_connect_housekeeping()
  2994. return True
  2995. except Exception as e:
  2996. self._release_platform_lock()
  2997. safe_error = _redact_telegram_error_text(e)
  2998. # Classify by exception TYPE (never message text): auth failures can never self-heal, so
  2999. # marking them retryable put agents into a silent eternal reconnect loop.
  3000. if self._looks_like_auth_error(e):
  3001. message = (
  3002. f"Telegram bot token rejected: {safe_error}. "
  3003. "The token is invalid or was revoked — generate a new one "
  3004. "with @BotFather and update TELEGRAM_BOT_TOKEN.")
  3005. self._set_fatal_error("telegram_auth_error", message, retryable=False)
  3006. else:
  3007. self._set_fatal_error("telegram_connect_error", f"Telegram startup failed: {safe_error}", retryable=True)
  3008. logger.error("[%s] Failed to connect to Telegram: %s", self.name, safe_error)
  3009. return False
  3010. async def _set_status_indicator(self, online: bool) -> None:
  3011. """Set the bot's short description to the online/offline text (closest Bot API surface to
  3012. presence). No-op unless ``extra.status_indicator``; failures are debug-logged."""
  3013. if not getattr(self, "_status_indicator_enabled", False):
  3014. return
  3015. bot = self._bot
  3016. if bot is None:
  3017. return
  3018. text = (self._status_online_text if online else self._status_offline_text)[:120] # Telegram cap
  3019. try:
  3020. await bot.set_my_short_description(short_description=text)
  3021. logger.info("[%s] Set bot status indicator to %r", self.name, text)
  3022. except Exception as e:
  3023. logger.debug("[%s] Failed to set bot status indicator to %r: %s", self.name, text, _redact_telegram_error_text(e))
  3024. @staticmethod
  3025. def _collect_live_tasks(candidates, current_task) -> list:
  3026. """Unique, unfinished tasks from ``candidates`` excluding ``current_task`` (so teardown never cancels itself)."""
  3027. seen: set[int] = set()
  3028. out: list[asyncio.Task] = []
  3029. for task in candidates:
  3030. if not task or task.done() or task is current_task or id(task) in seen:
  3031. continue
  3032. seen.add(id(task))
  3033. out.append(task)
  3034. return out
  3035. def _clear_task_attrs_except(self, current_task, *attrs: str) -> None:
  3036. for attr in attrs:
  3037. if getattr(self, attr, None) is not current_task:
  3038. setattr(self, attr, None)
  3039. async def _cancel_pending_delivery_tasks(self) -> None:
  3040. """Cancel every delayed-delivery task family before disconnect completes (media-group, photo-batch, text-batch flushes plus
  3041. polling recovery all sit behind ``asyncio.sleep()`` and would dispatch ``handle_message`` into a torn-down session)."""
  3042. current_task = asyncio.current_task()
  3043. pending_tasks = self._collect_live_tasks(
  3044. [
  3045. *self._media_group_tasks.values(), *self._pending_photo_batch_tasks.values(), *self._pending_text_batch_tasks.values(),
  3046. *getattr(self, "_committed_flush_tasks", {}).values(),
  3047. getattr(self, "_polling_error_task", None), getattr(self, "_polling_progress_verifier_task", None),
  3048. # Hold-queue redispatch must be cancellable+awaitable on teardown too.
  3049. getattr(self, "_held_inbound_redispatch_task", None),
  3050. ],
  3051. current_task)
  3052. awaitable_tasks = [t for t in pending_tasks if asyncio.isfuture(t) or asyncio.iscoroutine(t)]
  3053. # Hold-queue redispatch must be cancellable+awaitable on teardown so it cannot dispatch
  3054. # handle_message into a torn-down session (same lifecycle rule teknium called out on #72037 for
  3055. # shielded flush dispatch).
  3056. for task in pending_tasks:
  3057. task.cancel()
  3058. if awaitable_tasks:
  3059. await asyncio.gather(*awaitable_tasks, return_exceptions=True)
  3060. # Salvage buffered inbound events before clearing maps — unless permanent fatal, where no
  3061. # reconnect can drain and hold would re-orphan them.
  3062. if self._is_permanent_fatal():
  3063. n_pending = len(self._pending_text_batches) + len(self._pending_photo_batches) + len(self._media_group_events)
  3064. if n_pending:
  3065. logger.warning("[Telegram] Non-retryable fatal teardown; discarding %d pending inbound batch(es)", n_pending)
  3066. else:
  3067. for events, where in (
  3068. (self._pending_text_batches, "text-batch-teardown"), (self._pending_photo_batches, "photo-batch-teardown"),
  3069. (self._media_group_events, "media-group-teardown")):
  3070. for event in list(events.values()):
  3071. self._hold_inbound_event(event, where=where)
  3072. for d in (
  3073. self._media_group_tasks, self._media_group_events, self._pending_photo_batch_tasks,
  3074. self._pending_photo_batches, self._pending_text_batch_tasks, self._pending_text_batches):
  3075. d.clear()
  3076. self._clear_task_attrs_except(
  3077. current_task, "_polling_error_task", "_polling_progress_verifier_task", "_held_inbound_redispatch_task")
  3078. async def _await_disconnect_step(self, awaitable, timeout: float, step: str) -> bool:
  3079. """Await one disconnect step; detach on timeout so teardown advances (``wait_for`` would wait for a
  3080. PTB close that swallows ``CancelledError`` on a half-dead socket). Abandoned tasks are observed.
  3081. ``asyncio.wait_for`` cancels an overdue child but then waits for it to exit. Detach at the deadline
  3082. and continue — the abandoned task is observed via ``_consume_abandoned_task``. See #80598.
  3083. """
  3084. task = asyncio.ensure_future(awaitable)
  3085. try:
  3086. done, _pending = await asyncio.wait({task}, timeout=timeout if timeout > 0 else None)
  3087. except asyncio.CancelledError:
  3088. # asyncio.wait does NOT cancel its futures when itself cancelled; don't orphan the inner task.
  3089. task.cancel()
  3090. # Mirror the pattern used by GatewayRunner._await_adapter_cleanup_with_timeout. See #80598.
  3091. task.add_done_callback(_consume_abandoned_task)
  3092. raise
  3093. if task in done:
  3094. with contextlib.suppress(asyncio.CancelledError):
  3095. await task
  3096. return True
  3097. task.cancel()
  3098. task.add_done_callback(_consume_abandoned_task)
  3099. logger.warning("[%s] %s timed out after %.1fs during disconnect; continuing teardown", self.name, step, timeout)
  3100. return False
  3101. def _restart_task_attr(self, attr: str, coro) -> None:
  3102. """Cancel any live task stored at ``self.<attr>`` and start ``coro`` in its place."""
  3103. prior = getattr(self, attr, None)
  3104. if prior and not prior.done():
  3105. prior.cancel()
  3106. setattr(self, attr, asyncio.ensure_future(coro))
  3107. async def _cancel_task_attr(self, attr: str, label: str) -> None:
  3108. """Cancel + bounded-await the task stored at ``self.<attr>`` (may be missing: object.__new__ tests), then clear it."""
  3109. task = getattr(self, attr, None)
  3110. if task and not task.done():
  3111. task.cancel()
  3112. await self._await_disconnect_step(task, _DISCONNECT_STEP_TIMEOUT, label)
  3113. setattr(self, attr, None)
  3114. async def disconnect(self) -> None:
  3115. """Stop polling/webhook, cancel pending delayed deliveries, and disconnect."""
  3116. # Mark disconnected first so the drop guard short-circuits any flush that wins the race.
  3117. self._mark_disconnected()
  3118. self._polling_teardown_started = True
  3119. self._polling_progress_accepting = False
  3120. self._polling_generation = getattr(self, "_polling_generation", 0) + 1
  3121. self._polling_progress_event = asyncio.Event()
  3122. self._send_path_degraded = True
  3123. # Release the bot-token lock immediately so a wedged close cannot block the reconnect watcher.
  3124. # The rest of teardown is best-effort against a half-dead transport. See #80598.
  3125. self._release_platform_lock()
  3126. # Cancel and await both polling lifecycle owners right after the fence, before any other teardown
  3127. # await lets them start a new generation.
  3128. current_task = asyncio.current_task()
  3129. lifecycle_tasks = self._collect_live_tasks(
  3130. [getattr(self, "_polling_error_task", None), getattr(self, "_polling_progress_verifier_task", None)], current_task)
  3131. for task in lifecycle_tasks:
  3132. task.cancel()
  3133. lifecycle_tasks = [t for t in lifecycle_tasks if asyncio.isfuture(t) or asyncio.iscoroutine(t)]
  3134. if lifecycle_tasks:
  3135. await self._await_disconnect_step(
  3136. asyncio.gather(*lifecycle_tasks, return_exceptions=True), _DISCONNECT_STEP_TIMEOUT, "lifecycle-task cancel")
  3137. self._clear_task_attrs_except(current_task, "_polling_error_task", "_polling_progress_verifier_task")
  3138. # Cancellation callbacks may have run while awaited; the fence stays authoritative.
  3139. self._polling_progress_accepting = False
  3140. self._send_path_degraded = True
  3141. # Cancel deferred post-connect housekeeping so it cannot fire into a half-torn-down bot client.
  3142. # Cancel deferred post-connect housekeeping (command-menu / DM-topic / status-indicator Bot API
  3143. # calls) so it cannot fire into a half-torn-down bot client (#46298). getattr guards the
  3144. # object.__new__ test pattern where __init__ (which sets this attr) is never called.
  3145. post_connect_task = getattr(self, "_post_connect_task", None)
  3146. if post_connect_task and not post_connect_task.done():
  3147. post_connect_task.cancel()
  3148. await self._await_disconnect_step(
  3149. asyncio.gather(post_connect_task, return_exceptions=True), _DISCONNECT_STEP_TIMEOUT, "post-connect cancel")
  3150. self._post_connect_task = None
  3151. # Cancel the heartbeat (and webhook-mode identity loop) before tearing down the app.
  3152. await self._cancel_task_attr("_polling_heartbeat_task", "heartbeat cancel")
  3153. await self._cancel_task_attr("_bot_identity_refresh_task", "identity-refresh cancel")
  3154. # Mark the bot "Offline" while its HTTP client is still alive. Opt-in, non-fatal.
  3155. with contextlib.suppress(Exception):
  3156. await self._await_disconnect_step(self._set_status_indicator(online=False), _DISCONNECT_STEP_TIMEOUT, "status-indicator update")
  3157. await self._await_disconnect_step(self._cancel_pending_delivery_tasks(), _DISCONNECT_STEP_TIMEOUT, "pending-delivery cancel")
  3158. if self._app:
  3159. try:
  3160. # Bounded: a CLOSE-WAIT socket can wedge updater.stop() forever; fall through on timeout.
  3161. if self._app.updater and self._app.updater.running:
  3162. try:
  3163. await self._await_disconnect_step(self._app.updater.stop(), _UPDATER_STOP_TIMEOUT, "updater.stop()")
  3164. except Exception as stop_error:
  3165. logger.warning(
  3166. "[%s] updater.stop() failed during disconnect: %s", self.name, _redact_telegram_error_text(stop_error))
  3167. # app.stop()/shutdown() can also block on a half-dead httpx pool.
  3168. # Detach-on-timeout so disconnect always returns (#80598).
  3169. if self._app.running:
  3170. await self._await_disconnect_step(self._app.stop(), _DISCONNECT_STEP_TIMEOUT, "app.stop()")
  3171. await self._await_disconnect_step(self._app.shutdown(), _DISCONNECT_STEP_TIMEOUT, "app.shutdown()")
  3172. except Exception as e:
  3173. logger.warning("[%s] Error during Telegram disconnect: %s", self.name, _redact_telegram_error_text(e))
  3174. self._app = None
  3175. self._bot = None
  3176. # Land the last completed receipts before a replacement adapter reads them.
  3177. flush = getattr(self, "_update_receipt_flush", None)
  3178. if flush is not None and not flush.done():
  3179. await self._await_disconnect_step(asyncio.shield(flush), _DISCONNECT_STEP_TIMEOUT, "update-receipt flush")
  3180. logger.info("[%s] Disconnected from Telegram", self.name)
  3181. def _should_thread_reply(self, reply_to: Optional[str], chunk_index: int) -> bool:
  3182. """Whether this chunk (0 = first) should reply-thread to ``reply_to``, per reply_to_mode."""
  3183. if not reply_to:
  3184. return False
  3185. mode = self._reply_to_mode
  3186. if mode == "off":
  3187. return False
  3188. if mode == "all":
  3189. return True
  3190. return chunk_index == 0 # "first" (default)
  3191. @staticmethod
  3192. def _telegram_error_types() -> tuple:
  3193. """``(NetworkError, BadRequest, TimedOut)`` from PTB, with import-failure fallbacks
  3194. (``OSError``, ``None``, ``None``) so send() still classifies without the SDK."""
  3195. try:
  3196. from telegram.error import NetworkError as _NetErr
  3197. except ImportError:
  3198. _NetErr = OSError # type: ignore[misc,assignment]
  3199. try:
  3200. from telegram.error import BadRequest as _BadReq
  3201. except ImportError:
  3202. _BadReq = None # type: ignore[assignment,misc]
  3203. try:
  3204. from telegram.error import TimedOut as _TimedOut
  3205. except (ImportError, AttributeError):
  3206. _TimedOut = None # type: ignore[assignment,misc]
  3207. return _NetErr, _BadReq, _TimedOut
  3208. async def _send_chunk_markdown_or_plain(self, chunk: str, send_kwargs: dict[str, Any]):
  3209. """MarkdownV2 first; on a parse/markdown rejection resend as stripped plain text."""
  3210. try:
  3211. return await _await_with_thread_deadline(
  3212. self._bot.send_message(text=chunk, parse_mode=ParseMode.MARKDOWN_V2, **send_kwargs),
  3213. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  3214. except Exception as md_error:
  3215. if "parse" in str(md_error).lower() or "markdown" in str(md_error).lower():
  3216. logger.warning("[%s] MarkdownV2 parse failed, falling back to plain text: %s", self.name, md_error)
  3217. return await _await_with_thread_deadline(
  3218. self._bot.send_message(text=_strip_mdv2(chunk), parse_mode=None, **send_kwargs),
  3219. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  3220. raise
  3221. async def _send_chunk_with_retries(
  3222. self, chat_id: str, chunk: str, index: int, reply_to: Optional[str], metadata: Optional[dict[str, Any]],
  3223. thread_id: Optional[str], used_thread_fallback: bool, error_types: tuple):
  3224. """Deliver one chunk: routing, up to 3 attempts, thread-not-found / deleted-anchor / flood handling.
  3225. Returns ``(msg, used_thread_fallback)`` on success or a ``SendResult`` to return verbatim (fail-loud DM-topic
  3226. cases, flood cap); raises anything the caller's classifier should see."""
  3227. _NetErr, _BadReq, _TimedOut = error_types
  3228. retried_thread_not_found = False
  3229. private_dm_topic_send, dm_topic_reply_to_off, reply_to_id = self._chunk_reply_routing(chat_id, reply_to, metadata, thread_id, index)
  3230. if private_dm_topic_send and reply_to_id is None and not dm_topic_reply_to_off:
  3231. return SendResult(success=False, error=self._dm_topic_missing_anchor_error(), retryable=False)
  3232. thread_kwargs = self._thread_kwargs_for_send(
  3233. chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
  3234. if used_thread_fallback and thread_kwargs.get("message_thread_id") is not None:
  3235. thread_kwargs = dict(thread_kwargs)
  3236. thread_kwargs["message_thread_id"] = None
  3237. effective_thread_id = thread_kwargs.get("message_thread_id")
  3238. for _send_attempt in range(3):
  3239. try:
  3240. send_kwargs = {
  3241. "chat_id": normalize_telegram_chat_id(chat_id), "reply_to_message_id": reply_to_id, **thread_kwargs,
  3242. **self._link_preview_kwargs(), **self._notification_kwargs(metadata)}
  3243. return await self._send_chunk_markdown_or_plain(chunk, send_kwargs), used_thread_fallback
  3244. except _NetErr as send_err:
  3245. # BadRequest subclasses NetworkError in PTB but is permanent; handle specific cases.
  3246. if _BadReq and isinstance(send_err, _BadReq):
  3247. if self._is_thread_not_found_error(send_err) and effective_thread_id is not None:
  3248. if private_dm_topic_send or (metadata and metadata.get("telegram_dm_topic_created_for_send")):
  3249. return SendResult(success=False, error=str(send_err), retryable=False)
  3250. # One-off "thread not found" flakes recover on immediate retry: same thread_id once.
  3251. if not retried_thread_not_found:
  3252. retried_thread_not_found = True
  3253. logger.warning("[%s] Thread %s not found, retrying once with same thread_id", self.name, effective_thread_id)
  3254. continue
  3255. # Thread is genuinely gone: retry without it and prune the stale binding.
  3256. logger.warning("[%s] Thread %s not found, retrying without message_thread_id", self.name, effective_thread_id)
  3257. self._prune_stale_dm_topic_binding(chat_id, effective_thread_id, metadata=metadata)
  3258. used_thread_fallback = True
  3259. effective_thread_id = None
  3260. thread_kwargs = {"message_thread_id": None}
  3261. continue
  3262. if "message to be replied not found" in str(send_err).lower() and reply_to_id is not None:
  3263. safe_send_error = _redact_telegram_error_text(send_err)
  3264. if private_dm_topic_send:
  3265. return SendResult(success=False, error=safe_send_error, retryable=False)
  3266. # Reply target deleted; private-topic fallback sends drop anchor + topic id together.
  3267. logger.warning("[%s] Reply target deleted, retrying without reply_to: %s", self.name, safe_send_error)
  3268. reply_to_id = None
  3269. if self._dm_topic_fallback(metadata):
  3270. thread_kwargs = {}
  3271. else:
  3272. thread_kwargs = self._thread_kwargs_for_send(
  3273. chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
  3274. effective_thread_id = thread_kwargs.get("message_thread_id")
  3275. continue
  3276. raise # other BadRequest errors are permanent
  3277. # TimedOut also subclasses NetworkError: a generic timeout may have reached Telegram (don't
  3278. # retry); a wrapped ConnectTimeout or an httpx pool timeout is safe to retry.
  3279. is_pool_timeout = self._looks_like_pool_timeout(send_err)
  3280. if (
  3281. _TimedOut and isinstance(send_err, _TimedOut)
  3282. and not self._looks_like_connect_timeout(send_err) and not is_pool_timeout):
  3283. raise
  3284. if is_pool_timeout:
  3285. await self._drain_general_connections_after_pool_timeout()
  3286. if _send_attempt >= 2:
  3287. raise
  3288. wait = 2 ** _send_attempt
  3289. logger.warning("[%s] Network error on send (attempt %d/3), retrying in %ds: %s",
  3290. self.name, _send_attempt + 1, wait, _redact_telegram_error_text(send_err))
  3291. await asyncio.sleep(wait)
  3292. except Exception as send_err:
  3293. retry_after = getattr(send_err, "retry_after", None)
  3294. if retry_after is not None or "retry after" in str(send_err).lower():
  3295. wait = float(retry_after) if retry_after is not None else 1.0
  3296. safe_send_error = _redact_telegram_error_text(send_err)
  3297. # Never sleep a long server RetryAfter verbatim — it once pinned send() for 97 minutes.
  3298. # Mirror the edit path: a RetryAfter past a few seconds is not something to hold this
  3299. # coroutine open for. Sleeping the server value verbatim pinned send() for 97 minutes in
  3300. # production and froze inbound on every platform when it ran on the gateway boot path
  3301. # (#91969).
  3302. if wait > _FLOOD_INLINE_WAIT_CAP_SECS:
  3303. logger.warning(
  3304. "[%s] Telegram flood control on send (retry_after=%.1fs > %.0fs); failing closed instead of sleeping: %s",
  3305. self.name, wait, _FLOOD_INLINE_WAIT_CAP_SECS, safe_send_error)
  3306. return self._record_send_flood_cooldown(chat_id, wait)
  3307. if _send_attempt < 2:
  3308. logger.warning(
  3309. "[%s] Telegram flood control on send (attempt %d/3), retrying in %.1fs: %s", self.name,
  3310. _send_attempt + 1, wait, safe_send_error)
  3311. await asyncio.sleep(wait)
  3312. continue
  3313. # Retries exhausted and still flooded. Fail closed the same way a long penalty
  3314. # does: raising here handed the caller the platform's own wording instead of the
  3315. # canonical result, so the delivery ledger did not recognise the row as a flood
  3316. # refusal, armed no redelivery timer, and the reply waited for the next restart.
  3317. logger.warning(
  3318. "[%s] Telegram flood control on send persisted across %d attempts; failing "
  3319. "closed so the delivery ledger owns the wait: %s",
  3320. self.name, _send_attempt + 1, safe_send_error)
  3321. return self._record_send_flood_cooldown(chat_id, wait)
  3322. raise
  3323. async def _retrigger_typing(self, chat_id: str, metadata: Optional[dict[str, Any]]) -> None:
  3324. """Re-arm typing after an intermediate send (Telegram clears it when a message lands). Skipped on
  3325. the FINAL reply (``metadata["notify"]``): the refresh loop is gone and no API cancels the bubble.
  3326. Scheduled, never awaited: ``sendChatAction`` is a fire-and-forget UI hint, and awaiting its TLS
  3327. round-trip on the send path after *every* streamed chunk pinned the event loop the ``getUpdates``
  3328. long-polls live on until they rotted into CLOSE-WAIT while the adapter still reported connected
  3329. (#111727). ``_keep_typing`` already refreshes every 2s, so one in-flight re-arm per chat, at most
  3330. one per ``typing_retrigger_min_interval_seconds``, covers the gap a landed message leaves."""
  3331. if (metadata or {}).get("notify") or not getattr(getattr(self, "config", None), "typing_indicator", True):
  3332. return
  3333. # __dict__.setdefault: tests build adapters via object.__new__() (no __init__).
  3334. tasks: dict[str, asyncio.Task] = self.__dict__.setdefault("_telegram_typing_retrigger_tasks", {})
  3335. sent_at: dict[str, float] = self.__dict__.setdefault("_telegram_typing_retrigger_at", {})
  3336. key = str(chat_id)
  3337. in_flight = tasks.get(key)
  3338. if in_flight is not None and not in_flight.done():
  3339. return
  3340. loop = asyncio.get_running_loop()
  3341. now = loop.time()
  3342. # Stamped at scheduling time, not completion, so a burst of chunks cannot all pass while the
  3343. # first round-trip is still open.
  3344. if now - sent_at.get(key, float("-inf")) < getattr(self, "_telegram_typing_retrigger_interval", 2.0):
  3345. return
  3346. sent_at[key] = now
  3347. async def _quiet() -> None:
  3348. with contextlib.suppress(Exception):
  3349. await self.send_typing(chat_id, metadata=metadata)
  3350. task = loop.create_task(_quiet())
  3351. tasks[key] = task
  3352. task.add_done_callback(lambda done: tasks.get(key) is done and tasks.pop(key, None))
  3353. # Shutdown cancels _background_tasks, so a detached re-arm cannot outlive the adapter.
  3354. tracked = getattr(self, "_background_tasks", None)
  3355. if isinstance(tracked, set):
  3356. tracked.add(task)
  3357. task.add_done_callback(tracked.discard)
  3358. async def send(
  3359. self, chat_id: str, content: str, reply_to: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  3360. """Send a message to a Telegram chat."""
  3361. if not self._bot:
  3362. live = self._replacement_telegram_adapter()
  3363. if live is not None:
  3364. return await live.send(chat_id, content, reply_to, metadata)
  3365. if self._is_permanent_fatal() or not await self._wait_for_reconnection():
  3366. return SendResult(success=False, error="Not connected", retryable=not self._is_permanent_fatal())
  3367. live = self._replacement_telegram_adapter()
  3368. if not self._bot and live is not None:
  3369. return await live.send(chat_id, content, reply_to, metadata)
  3370. if not self._bot:
  3371. return SendResult(success=False, error="Not connected", retryable=True)
  3372. # getattr() — tests build adapters via object.__new__() (no __init__).
  3373. if getattr(self, "_send_path_degraded", False):
  3374. return SendResult(success=False, error="send_path_degraded", retryable=True)
  3375. # Skip whitespace-only text to prevent Telegram 400 empty-text errors.
  3376. if not content or not content.strip():
  3377. return SendResult(success=True, message_id=None)
  3378. # One chat at a time (held only around the API calls, never across the reconnect wait above), so
  3379. # two concurrent split replies to one chat cannot interleave their chunks (#114396).
  3380. async with self._chat_send_lock(chat_id):
  3381. return await self._send_text_locked(chat_id, content, reply_to, metadata)
  3382. async def _send_text_locked(
  3383. self, chat_id: str, content: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> SendResult:
  3384. """``send()`` body under the per-chat lock: rich fast-path, else MarkdownV2 chunks."""
  3385. # Re-checked under the lock: a burst queued behind a refused send must not each fire once.
  3386. cooldown = self._send_flood_cooldown_remaining(chat_id)
  3387. if cooldown is not None:
  3388. logger.warning(
  3389. "[%s] Telegram flood control still active for chat %s (%.0fs left); refusing locally without an API call",
  3390. self.name, chat_id, cooldown)
  3391. return _flood_cap_result(cooldown)
  3392. # Shared per-chat budget (#116312): a send WAITS for its slot (a send that waits
  3393. # is delivered; one that is skipped would drop a message).
  3394. slot_remaining = self._chat_outbound_slot_remaining(chat_id)
  3395. if slot_remaining > 0:
  3396. logger.debug(
  3397. "[%s] pacing send for chat %s (shared send+edit budget: slot in %.1fs)",
  3398. self.name, chat_id, slot_remaining)
  3399. await asyncio.sleep(slot_remaining)
  3400. self._hold_chat_outbound_slot(chat_id)
  3401. error_types = self._telegram_error_types()
  3402. chunks: list[str] = []
  3403. delivered: list[str] = []
  3404. try:
  3405. # Bot API 10.1 rich fast-path; falls through to legacy MarkdownV2 on permanent/capability
  3406. # errors or DM-topic skips; returns directly on success or transient failure (no legacy resend).
  3407. if self._should_attempt_rich(content, metadata=metadata):
  3408. rich_result = await self._try_send_rich(chat_id, content, reply_to, metadata)
  3409. if rich_result is not None:
  3410. if rich_result.success:
  3411. await self._retrigger_typing(chat_id, metadata)
  3412. return rich_result
  3413. chunks = self.truncate_message(self.format_message(content), self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)
  3414. if len(chunks) > 1:
  3415. # truncate_message appends a raw " (1/2)" suffix; escape the MarkdownV2-special parentheses.
  3416. chunks = [
  3417. _separate_chunk_indicator_from_fence(re.sub(r" \((\d+)/(\d+)\)$", r" \\(\1/\2\\)", chunk))
  3418. for chunk in chunks
  3419. ]
  3420. return await self._send_chunks(chat_id, chunks, delivered, reply_to, metadata, error_types)
  3421. except Exception as e:
  3422. classified = self._classify_send_exception(e, error_types)
  3423. return self._with_partial_send(classified, chunks[len(delivered):], delivered, tail_certain=classified.retryable)
  3424. def _classify_send_exception(self, e: Exception, error_types: tuple) -> SendResult:
  3425. """The failed ``SendResult`` for an exception escaping the chunk loop. ``retryable`` doubles as
  3426. "non-delivery is certain": a plain ``TimedOut`` may have reached Telegram, so it is neither re-sent
  3427. by ``_send_with_retry`` nor resumed from; a wrapped ConnectTimeout / httpx pool timeout never left."""
  3428. safe_error = _redact_telegram_error_text(e)
  3429. logger.error("[%s] Failed to send Telegram message: %s", self.name, safe_error)
  3430. err_str = str(e).lower()
  3431. error_kind = classify_send_error(e)
  3432. # Content exceeded 4096 chars: fail so the stream consumer enters fallback mode.
  3433. if "message_too_long" in err_str or "too long" in err_str:
  3434. logger.debug("[%s] send() content too long, falling back to new-message continuation", self.name)
  3435. return SendResult(success=False, error="message_too_long", error_kind="too_long")
  3436. _to = error_types[2]
  3437. is_timeout = (_to and isinstance(e, _to)) or "timed out" in err_str
  3438. return SendResult(
  3439. success=False, error=safe_error,
  3440. retryable=(self._looks_like_connect_timeout(e) or self._looks_like_pool_timeout(e) or not is_timeout),
  3441. error_kind=error_kind)
  3442. async def _send_chunks(
  3443. self, chat_id: str, chunks: list[str], delivered: list[str], reply_to: Optional[str],
  3444. metadata: Optional[dict[str, Any]], error_types: tuple) -> SendResult:
  3445. """Deliver formatted ``chunks`` in order, appending each landed message id to ``delivered``
  3446. (pre-seeded with the ids of an earlier partial send when resuming — the chunk index used for
  3447. reply routing continues from there). A mid-loop refusal returns the ``partial_overflow`` result."""
  3448. thread_id = self._metadata_thread_id(metadata)
  3449. requested_thread_id = self._message_thread_id_for_send(thread_id)
  3450. used_thread_fallback = False
  3451. prior = len(delivered)
  3452. for chunk in chunks:
  3453. outcome = await self._send_chunk_with_retries(
  3454. chat_id, chunk, len(delivered), reply_to, metadata, thread_id, used_thread_fallback, error_types)
  3455. if isinstance(outcome, SendResult):
  3456. # Every SendResult returned here is a DEFINITE non-delivery (flood cap, DM-topic refusal);
  3457. # ambiguous timeouts raise instead, so the remainder is safe to resume from.
  3458. return self._with_partial_send(outcome, chunks[len(delivered) - prior:], delivered)
  3459. msg, used_thread_fallback = outcome
  3460. delivered.append(str(msg.message_id))
  3461. await self._retrigger_typing(chat_id, metadata)
  3462. return SendResult(
  3463. success=True, message_id=delivered[0] if delivered else None,
  3464. raw_response={
  3465. "message_ids": list(delivered), "requested_thread_id": requested_thread_id, "thread_fallback": used_thread_fallback})
  3466. @staticmethod
  3467. def _with_partial_send(
  3468. result: SendResult, undelivered: list[str], delivered: list[str], *, tail_certain: bool = True) -> SendResult:
  3469. """Mark a split-send failure that happened after earlier chunks landed with the ``partial_overflow``
  3470. contract (the same key ``_edit_overflow_split`` sets and the stream consumer reads), so no caller
  3471. re-sends the already-visible head. ``undelivered`` (the formatted remainder, for
  3472. :meth:`_resume_partial_send`) is attached only when ``tail_certain``. No-op when nothing landed."""
  3473. if not delivered:
  3474. return result
  3475. raw = dict(result.raw_response) if isinstance(result.raw_response, dict) else {}
  3476. raw.update({
  3477. "partial_overflow": True, "delivered_chunks": len(delivered), "total_chunks": len(delivered) + len(undelivered),
  3478. "last_message_id": delivered[-1], "continuation_message_ids": tuple(delivered[1:])})
  3479. if tail_certain and undelivered:
  3480. raw["undelivered_chunks"] = tuple(undelivered)
  3481. raw["delivered_message_ids"] = tuple(delivered)
  3482. result.raw_response = raw
  3483. return result
  3484. async def _resume_partial_send(
  3485. self, chat_id: str, result: SendResult, *, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> Optional[SendResult]:
  3486. """Send only the chunks a partial ``send()`` could not deliver (``raw_response["undelivered_chunks"]``),
  3487. continuing the message-id sequence; ``None`` when the tail was not certain-undelivered."""
  3488. raw = result.raw_response if isinstance(result.raw_response, dict) else {}
  3489. undelivered = raw.get("undelivered_chunks")
  3490. if not undelivered or not self._bot:
  3491. return None
  3492. delivered = list(raw.get("delivered_message_ids") or ())
  3493. prior = len(delivered)
  3494. async with self._chat_send_lock(chat_id):
  3495. cooldown = self._send_flood_cooldown_remaining(chat_id)
  3496. if cooldown is not None:
  3497. return self._with_partial_send(_flood_cap_result(cooldown), list(undelivered), delivered)
  3498. error_types = self._telegram_error_types()
  3499. try:
  3500. return await self._send_chunks(chat_id, list(undelivered), delivered, reply_to, metadata, error_types)
  3501. except Exception as e:
  3502. classified = self._classify_send_exception(e, error_types)
  3503. return self._with_partial_send(
  3504. classified, list(undelivered)[len(delivered) - prior:], delivered, tail_certain=classified.retryable)
  3505. async def send_or_update_status(
  3506. self, chat_id: str, status_key: str, content: str, *, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  3507. """Send a status message, or edit the previous one with the same ``(chat_id, status_key)``; if the
  3508. edit fails (deleted, too old, …) the cached id is dropped and a fresh message is sent.
  3509. Issue #30045: progress/status callbacks (context-pressure, lifecycle, compression, etc.) used to
  3510. append a fresh bubble on every call. With this method, the first call sends and the message id is
  3511. remembered; subsequent calls with the same (chat_id, status_key) edit that same message in place.
  3512. """
  3513. key = (str(chat_id), str(status_key))
  3514. cached_id = self._status_message_ids.get(key)
  3515. if cached_id is not None:
  3516. result = await self.edit_message(chat_id, cached_id, content, finalize=True, metadata=metadata)
  3517. if result.success:
  3518. # Only write back if nobody evicted/replaced this key during the await.
  3519. if result.message_id and self._status_message_ids.get(key) == cached_id:
  3520. self._status_message_ids[key] = str(result.message_id)
  3521. return result
  3522. self._status_message_ids.pop(key, None)
  3523. result = await self.send(chat_id, content, metadata=metadata)
  3524. if result.success and result.message_id:
  3525. if len(self._status_message_ids) >= self._STATUS_MESSAGE_IDS_MAX:
  3526. # FIFO trim: drop the oldest half to bound memory (mirrors the Slack adapter).
  3527. for stale in list(self._status_message_ids)[: self._STATUS_MESSAGE_IDS_MAX // 2]:
  3528. self._status_message_ids.pop(stale, None)
  3529. self._status_message_ids[key] = str(result.message_id)
  3530. return result
  3531. async def _edit_text(self, chat_id: str, message_id: str, text: str, parse_mode: Any = None) -> None:
  3532. """``editMessageText`` with normalized ids; ``parse_mode=None`` sends plain text."""
  3533. kwargs: dict[str, Any] = {"chat_id": normalize_telegram_chat_id(chat_id), "message_id": int(message_id), "text": text}
  3534. if parse_mode is not None:
  3535. kwargs["parse_mode"] = parse_mode
  3536. await _await_with_thread_deadline(
  3537. self._bot.edit_message_text(**kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  3538. async def _edit_markdown_or_plain(self, chat_id: str, message_id: str, formatted: str, plain: str, warn_fmt: str) -> bool:
  3539. """MarkdownV2 edit with plain-text fallback. Returns True on a "not modified" no-op (caller may
  3540. skip further work); the fallback edit's exceptions propagate."""
  3541. try:
  3542. await self._edit_text(chat_id, message_id, formatted, ParseMode.MARKDOWN_V2)
  3543. except Exception as fmt_err:
  3544. if "not modified" in str(fmt_err).lower():
  3545. return True
  3546. logger.warning(warn_fmt, self.name, _redact_telegram_error_text(fmt_err))
  3547. await self._edit_text(chat_id, message_id, plain)
  3548. return False
  3549. async def edit_message(
  3550. self, chat_id: str, message_id: str, content: str, *, finalize: bool = False, metadata: Optional[dict[str, Any]] = None,
  3551. ) -> SendResult:
  3552. """Edit a previously sent Telegram message.
  3553. Telegram caps a message at 4096 UTF-16 codeunits. Streaming replies that outgrow it must NOT be truncated
  3554. silently nor fail (the consumer would re-send a duplicate): edit with the first chunk, send the rest as
  3555. continuations, and return the final chunk's id as the next edit target."""
  3556. if not self._bot:
  3557. return SendResult(success=False, error="Not connected")
  3558. # Shared per-chat budget (#116312): an interim (preview) edit is SKIPPED when the slot is busy —
  3559. # the text it would show is shown by the next edit anyway, so a burst of edits can't trip flood
  3560. # control. A final edit is never gated (the completed answer is always delivered). Sends wait for
  3561. # their slot; edits defer instead. Over-cap interim edits are exempt: the saturated-preview dedup
  3562. # below already throttles them to one real edit per ~4096-char growth. Consumed only when the
  3563. # edit actually fires. The skip is flagged in raw_response so the stream consumer does not
  3564. # record never-shown text as the visible prefix (a later flood fallback would then drop the
  3565. # tail the user never saw).
  3566. if (
  3567. not finalize
  3568. and utf16_len(content) <= self.MAX_MESSAGE_LENGTH
  3569. and self._chat_outbound_slot_remaining(chat_id) > 0
  3570. ):
  3571. logger.debug(
  3572. "[%s] skipping interim edit for chat %s (shared send+edit budget: slot busy)",
  3573. self.name, chat_id)
  3574. return SendResult(success=True, message_id=message_id, raw_response={"skipped": True})
  3575. self._hold_chat_outbound_slot(chat_id)
  3576. # Rich finalize (Bot API 10.1): edit the preview IN PLACE via rich_message — no fresh send + delete.
  3577. # Before the 4,096 pre-flight because the rich cap is 32,768; falls back to legacy on rejection.
  3578. # Rich finalize (Bot API 10.1): when the completed content has constructs the legacy MarkdownV2 edit
  3579. # degrades (tables → bullet lists, task lists, <details>, block math) and rich is available, edit
  3580. # the preview IN PLACE via editMessageText's rich_message param. No fresh send + delete → no
  3581. # duplicate preview (the problem #46206 reverted the fresh-final path for). Attempted before the
  3582. # 4,096 overflow pre-flight because the rich text cap is 32,768 — a rich table that exceeds the
  3583. # MarkdownV2 limit must not be split into legacy chunks. Falls back to the legacy edit path
  3584. # (overflow split included) on capability/permanent rejection.
  3585. if finalize and self._rich_eligible(content):
  3586. rich_result = await self._try_edit_rich(chat_id, message_id, content, metadata=metadata)
  3587. if rich_result is not None:
  3588. return rich_result
  3589. # Pre-flight: over-limit content is split-and-delivered on finalize; mid-stream we truncate instead
  3590. # (splitting moves the edit target to a continuation → infinite duplication loop).
  3591. # Pre-flight: if content already exceeds the limit, split-and-deliver without round-tripping a
  3592. # doomed edit. During streaming (finalize=False) we truncate instead of splitting — splitting
  3593. # creates continuation messages whose IDs become the new edit target, and on the next token chunk
  3594. # the full accumulated text is re-edited into the continuation, triggering another split → infinite
  3595. # duplication loop (#48648).
  3596. _preview_key = (str(chat_id), str(message_id))
  3597. _saturated_preview = False
  3598. if finalize:
  3599. self._last_overflow_preview.pop(_preview_key, None) # the final edit always delivers full content
  3600. if utf16_len(content) > self.MAX_MESSAGE_LENGTH:
  3601. if finalize:
  3602. return await self._edit_overflow_split(chat_id, message_id, content, finalize=finalize, metadata=metadata)
  3603. content = self._truncate_stream_overflow_preview(content)
  3604. _saturated_preview = True
  3605. # Saturated-preview dedup: past the cap every progressive edit truncates to the same text;
  3606. # re-sending is a visual no-op that still burns flood budget (200s+ penalties).
  3607. if self._last_overflow_preview.get(_preview_key) == content:
  3608. return SendResult(success=True, message_id=message_id)
  3609. elif not finalize:
  3610. # Content shrank back under the cap — clear stale saturation state so dedup can't mask an edit.
  3611. self._last_overflow_preview.pop(_preview_key, None)
  3612. try:
  3613. if not finalize:
  3614. await self._edit_text(chat_id, message_id, content)
  3615. if _saturated_preview:
  3616. self._last_overflow_preview[_preview_key] = content
  3617. return SendResult(success=True, message_id=message_id)
  3618. await self._edit_markdown_or_plain(
  3619. chat_id, message_id, self.format_message(content), _strip_mdv2(content) if content else content,
  3620. "[%s] MarkdownV2 edit failed, falling back to plain text: %s")
  3621. return SendResult(success=True, message_id=message_id)
  3622. except Exception as e:
  3623. err_str = str(e).lower()
  3624. if "not modified" in err_str:
  3625. return SendResult(success=True, message_id=message_id)
  3626. # Reactive split: MarkdownV2 escapes can inflate the payload past the limit even when raw text fit.
  3627. if "message_too_long" in err_str or "too long" in err_str:
  3628. logger.debug(
  3629. "[%s] edit_message overflow (%d UTF-16 > %d), splitting", self.name, utf16_len(content), self.MAX_MESSAGE_LENGTH)
  3630. if finalize:
  3631. return await self._edit_overflow_split(chat_id, message_id, content, finalize=finalize, metadata=metadata)
  3632. # Mid-stream: truncate and retry instead of splitting (saturated-preview dedup as above).
  3633. # See #48648.
  3634. truncated = self._truncate_stream_overflow_preview(content)
  3635. if self._last_overflow_preview.get(_preview_key) == truncated:
  3636. return SendResult(success=True, message_id=message_id)
  3637. await self._edit_text(chat_id, message_id, truncated)
  3638. self._last_overflow_preview[_preview_key] = truncated
  3639. return SendResult(success=True, message_id=message_id)
  3640. # Flood control: short waits retry inline; long waits fail immediately so streaming falls back
  3641. # to a normal final send instead of a clipped partial.
  3642. retry_after = getattr(e, "retry_after", None)
  3643. if retry_after is not None or "retry after" in err_str:
  3644. wait = retry_after if retry_after else 1.0
  3645. if wait > _FLOOD_INLINE_WAIT_CAP_SECS:
  3646. # Log AFTER the cap check: "waiting 33.0s" followed by no wait misled an investigation.
  3647. logger.warning(
  3648. "[%s] Telegram flood control, refusing edit (retry_after %.1fs > %.0fs inline cap)",
  3649. self.name, wait, _FLOOD_INLINE_WAIT_CAP_SECS)
  3650. return _flood_cap_result(wait)
  3651. logger.warning("[%s] Telegram flood control, waiting %.1fs", self.name, wait)
  3652. await asyncio.sleep(wait)
  3653. try:
  3654. await self._edit_text(chat_id, message_id, content)
  3655. return SendResult(success=True, message_id=message_id)
  3656. except Exception as retry_err:
  3657. safe_retry_error = _redact_telegram_error_text(retry_err)
  3658. logger.error("[%s] Edit retry failed after flood wait: %s", self.name, safe_retry_error)
  3659. retry_wait = getattr(retry_err, "retry_after", None)
  3660. if retry_wait is not None or "retry after" in str(retry_err).lower():
  3661. # Still flooded after the inline wait, and typically for much longer than the
  3662. # first refusal asked for. Fail closed canonically so the ledger arms its
  3663. # timer on this delay rather than storing the platform's raw wording, which
  3664. # it would read as an ordinary failure and never redeliver.
  3665. return _flood_cap_result(
  3666. float(retry_wait) if retry_wait is not None else wait)
  3667. return SendResult(success=False, error=safe_retry_error)
  3668. safe_error = _redact_telegram_error_text(e)
  3669. # Transient network errors must not permanently disable progress-message editing.
  3670. _transient_markers = (
  3671. "connecterror", "connect error", "connection error", "networkerror", "network error", "timed out", "readtimeout",
  3672. "writetimeout", "server disconnected", "temporarily unavailable", "temporary failure", "httpx")
  3673. if any(m in err_str for m in _transient_markers):
  3674. logger.warning("[%s] Transient network error editing message %s (will retry): %s", self.name, message_id, safe_error)
  3675. return SendResult(success=False, error=safe_error, retryable=True)
  3676. logger.error("[%s] Failed to edit Telegram message %s: %s", self.name, message_id, safe_error)
  3677. return SendResult(success=False, error=safe_error)
  3678. def _truncate_stream_overflow_preview(self, content: str) -> str:
  3679. """One-message preview for oversized streaming edits (edits must keep targeting the original id;
  3680. final edits use ``_edit_overflow_split``).
  3681. Splitting a mid-stream preview creates continuation messages and moves the active message id, so the
  3682. next accumulated-token edit repeats the overflow cycle (#48648). Final edits still use
  3683. ``_edit_overflow_split`` to deliver the complete response.
  3684. """
  3685. return self.truncate_message(content, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)[0]
  3686. async def _send_overflow_continuation(
  3687. self, chat_id: str, chunk: str, reply_to_id: Optional[int], thread_kwargs: dict[str, Any],
  3688. thread_id: Optional[str], metadata: Optional[dict[str, Any]], finalize: bool):
  3689. """Send one continuation chunk (MarkdownV2 then plain on finalize; raw when streaming); drops the
  3690. reply anchor once on 'reply message not found'. Returns the sent message or None."""
  3691. base = {**self._link_preview_kwargs(), **self._notification_kwargs(metadata)}
  3692. for use_markdown in (True, False) if finalize else (False,):
  3693. try:
  3694. if use_markdown:
  3695. text = _separate_chunk_indicator_from_fence(self.format_message(chunk))
  3696. else:
  3697. # Degrade to stripped text on finalize (raw ** / ``` would render literally); previews stay raw.
  3698. text = _strip_mdv2(chunk) if finalize else chunk
  3699. return await _await_with_thread_deadline(
  3700. self._bot.send_message(
  3701. chat_id=normalize_telegram_chat_id(chat_id), text=text, parse_mode=ParseMode.MARKDOWN_V2 if use_markdown else None,
  3702. reply_to_message_id=reply_to_id, **thread_kwargs, **base),
  3703. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  3704. except Exception as send_err:
  3705. if "reply message not found" in str(send_err).lower():
  3706. # Private DM topic fallback needs anchor + topic id together; forum topics keep thread id.
  3707. retry_thread_kwargs = (
  3708. {} if self._dm_topic_fallback(metadata)
  3709. else self._thread_kwargs_for_send(chat_id, thread_id, metadata, reply_to_message_id=None))
  3710. try:
  3711. return await _await_with_thread_deadline(
  3712. self._bot.send_message(
  3713. chat_id=normalize_telegram_chat_id(chat_id), text=_strip_mdv2(chunk) if finalize else chunk,
  3714. **retry_thread_kwargs, **base),
  3715. timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  3716. except Exception as _retry_err:
  3717. logger.warning(
  3718. "[%s] Overflow continuation no-reply retry failed: %s", self.name, _redact_telegram_error_text(_retry_err))
  3719. return None
  3720. if use_markdown:
  3721. continue # try plain text on next loop iteration
  3722. logger.warning("[%s] Overflow continuation send failed: %s", self.name, _redact_telegram_error_text(send_err))
  3723. return None
  3724. return None
  3725. async def _edit_overflow_split(
  3726. self, chat_id: str, message_id: str, content: str, *, finalize: bool, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  3727. """Split an oversized edit across the existing message + continuations: edit ``message_id`` with
  3728. chunk 1, send the rest as replies to the previous chunk, return ``message_id=<last-chunk-id>`` so
  3729. the consumer keeps editing the newest message. ``success=False`` only if the first-chunk edit fails."""
  3730. chunks = self.truncate_message(content, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)
  3731. if len(chunks) <= 1:
  3732. chunks = [content] # defensive: a single chunk just edits normally
  3733. first_chunk = chunks[0]
  3734. try:
  3735. if finalize:
  3736. await self._edit_markdown_or_plain(
  3737. chat_id, message_id, _separate_chunk_indicator_from_fence(self.format_message(first_chunk)), _strip_mdv2(first_chunk),
  3738. "[%s] Overflow split: MarkdownV2 first-chunk edit failed, falling back to plain text: %s")
  3739. else:
  3740. await self._edit_text(chat_id, message_id, first_chunk)
  3741. except Exception as e:
  3742. if "not modified" not in str(e).lower(): # identical first chunk still sends continuations
  3743. logger.error("[%s] Overflow split: first-chunk edit failed: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
  3744. return SendResult(success=False, error=_redact_telegram_error_text(e))
  3745. # Continuations call self._bot.send_message directly to skip self.send's pre-chunking.
  3746. continuation_ids: list[str] = []
  3747. delivered_chunks = [first_chunk]
  3748. prev_id = message_id
  3749. thread_id = self._metadata_thread_id(metadata)
  3750. for chunk in chunks[1:]:
  3751. reply_to_id = int(prev_id) if prev_id else None
  3752. thread_kwargs = self._thread_kwargs_for_send(chat_id, thread_id, metadata, reply_to_message_id=reply_to_id)
  3753. sent_msg = await self._send_overflow_continuation(chat_id, chunk, reply_to_id, thread_kwargs, thread_id, metadata, finalize)
  3754. if sent_msg is None:
  3755. # Partial delivery: do NOT report success — the consumer would treat it as final delivery.
  3756. logger.warning("[%s] Overflow split: stopped at %d/%d chunks delivered", self.name, 1 + len(continuation_ids), len(chunks))
  3757. delivered_prefix = "".join(re.sub(r" \(\d+/\d+\)$", "", delivered) for delivered in delivered_chunks)
  3758. return SendResult(
  3759. success=False, message_id=prev_id, error="overflow_continuation_failed", retryable=True,
  3760. raw_response={
  3761. "partial_overflow": True, "delivered_chunks": 1 + len(continuation_ids),
  3762. "total_chunks": len(chunks), "last_message_id": prev_id, "delivered_prefix": delivered_prefix,
  3763. "continuation_message_ids": tuple(continuation_ids)},
  3764. continuation_message_ids=tuple(continuation_ids))
  3765. new_id = str(getattr(sent_msg, "message_id", "")) or prev_id
  3766. continuation_ids.append(new_id)
  3767. delivered_chunks.append(chunk)
  3768. prev_id = new_id
  3769. last_id = continuation_ids[-1] if continuation_ids else message_id
  3770. logger.debug("[%s] Overflow split delivered %d chunks; last_id=%s", self.name, 1 + len(continuation_ids), last_id)
  3771. return SendResult(success=True, message_id=last_id, continuation_message_ids=tuple(continuation_ids))
  3772. async def delete_message(self, chat_id: str, message_id: str) -> bool:
  3773. """Delete a bot-posted message (Bot API allows it within 48h); failures are non-fatal.
  3774. Used by the stream consumer's fresh-final cleanup path (ported from openclaw/openclaw#72038) to
  3775. remove long-lived preview messages after sending the completed reply as a fresh message. Telegram's
  3776. Bot API ``deleteMessage`` works for bot-posted messages in the last 48 hours. Failures are non-fatal
  3777. — the caller leaves the preview in place and logs at debug level.
  3778. """
  3779. if not self._bot:
  3780. return False
  3781. try:
  3782. await self._bot.delete_message(chat_id=normalize_telegram_chat_id(chat_id), message_id=int(message_id))
  3783. return True
  3784. except Exception as e:
  3785. logger.debug("[%s] Failed to delete Telegram message %s: %s", self.name, message_id, _redact_telegram_error_text(e))
  3786. return False
  3787. def supports_draft_streaming(self, chat_type: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> bool:
  3788. """sendMessageDraft works for private chats only (Bot API 9.5) and needs PTB >= 22.6; groups and
  3789. older installs use the edit-based path. ``rich_drafts`` controls draft *format*, not availability."""
  3790. if not self._bot or not hasattr(self._bot, "send_message_draft"):
  3791. return False
  3792. return (chat_type or "").lower() in {"dm", "private"}
  3793. async def send_draft(self, chat_id: str, draft_id: int, content: str, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  3794. """Stream a partial message via ``sendRichMessageDraft`` (when rich is enabled and supported) else
  3795. ``sendMessageDraft``; reusing ``draft_id`` animates the preview. The caller sends the final text."""
  3796. if not self._bot:
  3797. return SendResult(success=False, error="not_connected")
  3798. # Rich draft fast-path; any failure degrades to the plain draft below. Drafts have no message_id.
  3799. if self._should_attempt_rich_draft(content) and await self._try_send_rich_draft(chat_id, draft_id, content, metadata):
  3800. return SendResult(success=True, message_id=None)
  3801. if not hasattr(self._bot, "send_message_draft"):
  3802. return SendResult(success=False, error="api_unavailable")
  3803. # Drafts share the regular-send UTF-16 length contract.
  3804. text = content if len(
  3805. content) <= self.MAX_MESSAGE_LENGTH else self.truncate_message(content, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)[0]
  3806. # Same MarkdownV2 conversion as ``send`` (MarkdownV2 then plain) so the draft doesn't snap at the end. Exception: a Rich
  3807. # final with rich drafts disabled previews raw — the legacy formatter would turn pipe tables into bullets.
  3808. plain_rich_preview = bool(
  3809. getattr(self, "_rich_messages_enabled", False) and not getattr(self, "_rich_drafts_enabled", False)
  3810. and self._needs_rich_rendering(text))
  3811. draft_thread_kwargs = self._thread_kwargs_for_draft(chat_id, metadata)
  3812. for use_markdown in ((False,) if plain_rich_preview else (True, False)):
  3813. kwargs: dict[str, Any] = {
  3814. "chat_id": normalize_telegram_chat_id(chat_id), "draft_id": int(draft_id),
  3815. "text": self.format_message(text) if use_markdown else text}
  3816. if use_markdown:
  3817. kwargs["parse_mode"] = ParseMode.MARKDOWN_V2
  3818. kwargs.update(draft_thread_kwargs)
  3819. try:
  3820. if await _await_with_thread_deadline(
  3821. self._bot.send_message_draft(**kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False):
  3822. return SendResult(success=True, message_id=None)
  3823. return SendResult(success=False, error="draft_rejected")
  3824. except Exception as e:
  3825. # MarkdownV2 parse failure → retry once as plain text; anything else returns to the caller,
  3826. # which falls back to edit-based streaming for this response.
  3827. if use_markdown and self._is_bad_request_error(e):
  3828. logger.debug(
  3829. "[%s] sendMessageDraft MarkdownV2 rejected, retrying as plain text (chat=%s draft_id=%s): %s",
  3830. self.name, chat_id, draft_id, _redact_telegram_error_text(e))
  3831. continue
  3832. logger.debug("[%s] sendMessageDraft failed (chat=%s draft_id=%s): %s", self.name, chat_id, draft_id, e)
  3833. return SendResult(success=False, error=_redact_telegram_error_text(e))
  3834. return SendResult(success=False, error="draft_rejected")
  3835. async def _send_message_with_thread_fallback(self, **kwargs):
  3836. """Send a control-style message (approval prompts, pickers), retrying once without
  3837. message_thread_id on 'Message thread not found' (stale thread_id); ``send`` has its own.
  3838. Used for control-style sends (approval prompts, model picker, update prompts) that can carry a stale
  3839. thread_id from a DM reply chain. The streaming send loop has its own equivalent (PR #3390) at the
  3840. body of ``send``; this helper applies the same retry pattern to the non-streaming control paths.
  3841. """
  3842. if not self._bot:
  3843. raise RuntimeError("Not connected")
  3844. message_thread_id = kwargs.get("message_thread_id")
  3845. try:
  3846. return await _await_with_thread_deadline(
  3847. self._bot.send_message(**kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  3848. except Exception as send_err:
  3849. if (message_thread_id is not None and self._is_bad_request_error(send_err) and self._is_thread_not_found_error(send_err)):
  3850. logger.warning(
  3851. "[%s] Thread %s not found for control message, retrying without message_thread_id", self.name, message_thread_id)
  3852. # Same prune as the streaming send path; control sends carry no gateway metadata.
  3853. self._prune_stale_dm_topic_binding(kwargs.get("chat_id"), message_thread_id)
  3854. retry_kwargs = dict(kwargs)
  3855. retry_kwargs.pop("message_thread_id", None)
  3856. return await _await_with_thread_deadline(
  3857. self._bot.send_message(**retry_kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
  3858. raise
  3859. async def _send_control_message(
  3860. self, chat_id: str, text: str, *, parse_mode: Any, thread_id: Optional[str], metadata: Optional[dict[str, Any]],
  3861. reply_markup: Any = None, reply_to_mode: Optional[str] = None):
  3862. """Send a control-style message (prompt/picker) with topic routing + thread fallback."""
  3863. reply_to_id = self._reply_to_message_id_for_send(None, metadata, reply_to_mode=reply_to_mode)
  3864. kwargs: dict[str, Any] = {
  3865. "chat_id": normalize_telegram_chat_id(chat_id), "text": text, "parse_mode": parse_mode, **self._link_preview_kwargs()}
  3866. if reply_markup is not None:
  3867. kwargs["reply_markup"] = reply_markup
  3868. kwargs["reply_to_message_id"] = reply_to_id
  3869. kwargs.update(self._thread_kwargs_for_send(
  3870. chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=reply_to_mode))
  3871. return await self._send_message_with_thread_fallback(**kwargs)
  3872. async def _send_prompt(self, what: str, chat_id: str, metadata: Optional[dict[str, Any]], build, *,
  3873. parse_mode: Any = None, thread_id: Any = None, reply_to_mode: Any = None) -> SendResult:
  3874. """Shared control-prompt shell: not-connected guard, ``build()`` → ``(text, keyboard, on_sent)`` (or a
  3875. SendResult to return as-is), routed send, state hook, redacted failure log."""
  3876. if not self._bot:
  3877. return SendResult(success=False, error="Not connected")
  3878. try:
  3879. built = build()
  3880. if isinstance(built, SendResult):
  3881. return built
  3882. text, keyboard, on_sent = built
  3883. msg = await self._send_control_message(
  3884. chat_id, text, parse_mode=parse_mode if parse_mode is not None else ParseMode.MARKDOWN_V2,
  3885. reply_markup=keyboard, thread_id=thread_id, metadata=metadata, reply_to_mode=reply_to_mode)
  3886. if on_sent is not None:
  3887. on_sent(msg)
  3888. return SendResult(success=True, message_id=str(msg.message_id))
  3889. except Exception as e:
  3890. logger.warning("[%s] %s failed: %s", self.name, what, _redact_telegram_error_text(e))
  3891. return SendResult(success=False, error=_redact_telegram_error_text(e))
  3892. @staticmethod
  3893. def _rows_of_two(buttons: list) -> list:
  3894. """2-per-row layout keeps labels readable on mobile (a 4-button row truncates)."""
  3895. return [buttons[i:i + 2] for i in range(0, len(buttons), 2)]
  3896. async def send_update_prompt(
  3897. self, chat_id: str, prompt: str, default: str = "", session_key: str = "", metadata: Optional[dict[str, Any]] = None) -> SendResult:
  3898. """Send an inline-keyboard Yes/No prompt for the gateway ``/update`` watcher."""
  3899. def build():
  3900. default_hint = t("platform.telegram.prompt.default_hint", default=default) if default else ""
  3901. text = self.format_message(f"☤ *{t('platform.telegram.prompt.update_header')}*\n\n{prompt}{default_hint}")
  3902. keyboard = InlineKeyboardMarkup([[
  3903. InlineKeyboardButton(t("platform.telegram.prompt.affirm"), callback_data="update_prompt:y"),
  3904. InlineKeyboardButton(t("platform.telegram.prompt.negate"), callback_data="update_prompt:n")]])
  3905. return text, keyboard, None
  3906. return await self._send_prompt(
  3907. "send_update_prompt", chat_id, metadata, build, thread_id=self._metadata_thread_id(metadata), reply_to_mode=self._reply_to_mode)
  3908. # Template attrs for the shared _format_exec_approval core (HTML mode). Properties, not class
  3909. # constants: the wording comes from the catalog for the language active at send time, and the
  3910. # translated text is HTML-escaped BEFORE the <b> wrapper (a stray ``&``/``<`` would break the card).
  3911. @property
  3912. def _EA_HEADER(self) -> str:
  3913. return f"⚠️ <b>{_html.escape(t('gateway.exec_approval.header'))}</b>\n\n"
  3914. _EA_CODE_OPEN = "<pre>"
  3915. _EA_CODE_CLOSE = "</pre>\n\n"
  3916. @property
  3917. def _EA_REASON_LABEL(self) -> str:
  3918. return f"{_html.escape(t('gateway.exec_approval.reason_label'))}: "
  3919. @property
  3920. def _EA_SMART_DENY_LINE(self) -> str:
  3921. return "\n\n" + _bold_label_html(t("gateway.exec_approval.smart_deny_line"))
  3922. _EA_REASON_BUDGET = 500 # escaped chars; the reason shares the 4096 cap with the command
  3923. def _ea_escape(self, text: str) -> str:
  3924. return _html.escape(text)
  3925. def _exec_approval_cmd_budget(self, description: str, smart_denied: bool) -> int:
  3926. # Telegram rejects the whole card ("Message is too long") and the gateway then falls back to
  3927. # the text /approve prompt, so budget the preview against what the framing leaves of the cap.
  3928. fixed = utf16_len( # UTF-16 units, like the 4096 chunker in send()
  3929. self._EA_HEADER + self._EA_CODE_OPEN + self._EA_CODE_CLOSE + self._EA_REASON_LABEL
  3930. + self._ea_escape(description) + "..." + self._ea_deadline_line()
  3931. + (self._EA_SMART_DENY_LINE if smart_denied else ""))
  3932. return max(0, self.MAX_MESSAGE_LENGTH - fixed)
  3933. @property
  3934. def _EA_ACTION_LABELS(self) -> dict[str, str]:
  3935. # Shorter than the base wording on purpose: two buttons share a row on mobile.
  3936. return {choice: t(f"platform.telegram.approval.action_{choice}") for choice in ("once", "session", "always", "deny")}
  3937. async def _send_exec_approval_prompt(self, prompt: ExecApprovalPrompt) -> SendResult:
  3938. """Inline-keyboard approval prompt; buttons call ``resolve_gateway_approval()`` like the
  3939. text ``/approve`` flow."""
  3940. def build():
  3941. # Short monotonic ids in callback_data map back to session_key.
  3942. import itertools
  3943. if not hasattr(self, "_approval_counter"):
  3944. self._approval_counter = itertools.count(1)
  3945. approval_id = next(self._approval_counter)
  3946. buttons = [InlineKeyboardButton(label, callback_data=f"ea:{choice}:{approval_id}")
  3947. for label, choice, _ in prompt.actions]
  3948. def on_sent(_msg):
  3949. self._approval_state[approval_id] = prompt.session_key
  3950. if prompt.request_id:
  3951. self._approval_request_ids[approval_id] = prompt.request_id
  3952. return prompt.text, InlineKeyboardMarkup(self._rows_of_two(buttons)), on_sent
  3953. return await self._send_prompt(
  3954. "send_exec_approval", prompt.chat_id, prompt.metadata, build, parse_mode=ParseMode.HTML,
  3955. thread_id=self._metadata_thread_id(prompt.metadata), reply_to_mode=self._reply_to_mode)
  3956. async def send_slash_confirm(
  3957. self, chat_id: str, title: str, message: str, session_key: str, confirm_id: str,
  3958. metadata: Optional[dict[str, Any]] = None) -> SendResult:
  3959. """Render a three-button slash-command confirmation prompt."""
  3960. def build():
  3961. keyboard = InlineKeyboardMarkup([
  3962. [
  3963. InlineKeyboardButton(t("platform.telegram.slash_confirm.approve_once"), callback_data=f"sc:once:{confirm_id}"),
  3964. InlineKeyboardButton(t("platform.telegram.slash_confirm.always_approve"), callback_data=f"sc:always:{confirm_id}")],
  3965. [InlineKeyboardButton(t("platform.telegram.slash_confirm.cancel"), callback_data=f"sc:cancel:{confirm_id}")],
  3966. ])
  3967. # Budget the MarkdownV2 rendering (escaping expands text), not the raw message.
  3968. preview = self.format_message(self._ea_fit(
  3969. message, self.MAX_MESSAGE_LENGTH - utf16_len(self.format_message("...")), escape=self.format_message))
  3970. return preview, keyboard, lambda msg: self._slash_confirm_state.__setitem__(confirm_id, session_key)
  3971. return await self._send_prompt(
  3972. "send_slash_confirm", chat_id, metadata, build, thread_id=self._metadata_thread_id(metadata), reply_to_mode=self._reply_to_mode)
  3973. async def send_clarify(
  3974. self, chat_id: str, question: str, choices: Optional[list], clarify_id: str, session_key: str,
  3975. metadata: Optional[dict[str, Any]] = None) -> SendResult:
  3976. """Render a clarify prompt: numbered buttons per choice plus "✏️ Other (type answer)" (flips to
  3977. text-capture mode); without choices, plain question and the gateway text-intercept captures."""
  3978. def build():
  3979. text = f"❓ {_html.escape(question)}"
  3980. keyboard = None
  3981. if choices:
  3982. # Full option text in the body (mobile truncates button labels); buttons keep numeric labels.
  3983. text += "\n\n" + "\n".join(f"{i + 1}. {_html.escape(str(c))}" for i, c in enumerate(choices))
  3984. # Telegram caps callback_data at 64 bytes; keep "cl:<id>:<idx>" short.
  3985. rows = [[InlineKeyboardButton(str(idx + 1), callback_data=f"cl:{clarify_id}:{idx}")] for idx in range(len(choices))]
  3986. rows.append([InlineKeyboardButton(t("platform.telegram.prompt.other"), callback_data=f"cl:{clarify_id}:other")])
  3987. keyboard = InlineKeyboardMarkup(rows)
  3988. return text, keyboard, lambda msg: self._clarify_state.__setitem__(clarify_id, session_key)
  3989. return await self._send_prompt(
  3990. "send_clarify", chat_id, metadata, build, parse_mode=ParseMode.HTML, thread_id=self._metadata_thread_id(metadata))
  3991. @staticmethod
  3992. def _provider_get_label():
  3993. try:
  3994. from hermes_cli.providers import get_label
  3995. except ImportError:
  3996. def get_label(slug):
  3997. return slug
  3998. return get_label
  3999. async def send_model_picker(
  4000. self, chat_id: str, providers: list, current_model: str, current_provider: str, session_key: str,
  4001. on_model_selected, metadata: Optional[dict[str, Any]] = None) -> SendResult:
  4002. """Send an inline-keyboard model picker: provider → model drill-down, edited in place."""
  4003. def build():
  4004. keyboard, provider_page_info = self._build_provider_keyboard(providers, 0)
  4005. text = self.format_message(
  4006. self._provider_list_text(current_model, self._provider_get_label()(current_provider), provider_page_info)
  4007. )
  4008. def _remember(msg):
  4009. self._model_picker_state[str(chat_id)] = {
  4010. "msg_id": msg.message_id, "providers": providers, "session_key": session_key, "on_model_selected": on_model_selected,
  4011. "current_model": current_model, "current_provider": current_provider, "provider_page": 0}
  4012. return text, keyboard, _remember
  4013. return await self._send_prompt(
  4014. "send_model_picker", chat_id, metadata, build, thread_id=metadata.get("thread_id") if metadata else None,
  4015. reply_to_mode=self._reply_to_mode)
  4016. _PROVIDER_PAGE_SIZE = 10
  4017. async def send_choice_picker(
  4018. self, chat_id: str, title: str, choices: list, session_key: str, on_choice_selected,
  4019. metadata: Optional[dict[str, Any]] = None) -> SendResult:
  4020. """Flat inline-keyboard picker (one tap → one value) for /reasoning, /fast, etc. Each choice dict:
  4021. ``{"value": str, "label": str, "is_current": bool}``."""
  4022. def build():
  4023. buttons = []
  4024. for i, choice in enumerate(choices):
  4025. label = str(choice.get("label") or choice.get("value") or "")
  4026. if choice.get("is_current"):
  4027. label = f"✓ {label}"
  4028. buttons.append(InlineKeyboardButton(label, callback_data=f"cp:{i}"))
  4029. if not buttons:
  4030. return SendResult(success=False, error="No choices")
  4031. keyboard = InlineKeyboardMarkup(self._rows_of_two(buttons))
  4032. def _remember(msg):
  4033. self._choice_picker_state[str(chat_id)] = {
  4034. "msg_id": msg.message_id, "choices": choices, "session_key": session_key, "on_choice_selected": on_choice_selected}
  4035. return self.format_message(title), keyboard, _remember
  4036. return await self._send_prompt(
  4037. "send_choice_picker", chat_id, metadata, build, thread_id=metadata.get("thread_id") if metadata else None,
  4038. reply_to_mode=self._reply_to_mode)
  4039. async def _edit_result_text(self, query, result_text: str) -> None:
  4040. """Replace a picker message with ``result_text`` (MarkdownV2, then plain, then give up), keyboard removed."""
  4041. try:
  4042. await query.edit_message_text(text=self.format_message(result_text), parse_mode=ParseMode.MARKDOWN_V2, reply_markup=None)
  4043. except Exception:
  4044. with contextlib.suppress(Exception):
  4045. await query.edit_message_text(text=result_text, parse_mode=None, reply_markup=None)
  4046. async def _handle_choice_picker_callback(self, query, data: str, chat_id: str) -> None:
  4047. """Handle choice picker button taps (cp:<index>)."""
  4048. state = self._choice_picker_state.get(chat_id)
  4049. if not state:
  4050. await query.answer(text=_toast("platform.telegram.picker.expired_rerun"))
  4051. return
  4052. try:
  4053. choice = state["choices"][int(data[3:])]
  4054. except (ValueError, IndexError):
  4055. await query.answer(text=_toast("platform.telegram.picker.invalid_selection"))
  4056. return
  4057. callback = state.get("on_choice_selected")
  4058. if not callback:
  4059. await query.answer(text=_toast("platform.telegram.picker.expired"))
  4060. return
  4061. try:
  4062. result_text = await callback(chat_id, str(choice.get("value") or ""))
  4063. except Exception as exc:
  4064. logger.error("Choice picker selection failed: %s", exc)
  4065. result_text = t("platform.telegram.picker.apply_error", error=str(exc))
  4066. await self._edit_result_text(query, result_text)
  4067. await query.answer()
  4068. self._choice_picker_state.pop(chat_id, None)
  4069. _MODEL_PAGE_SIZE = 8
  4070. @staticmethod
  4071. def _provider_button(p: dict) -> InlineKeyboardButton:
  4072. count = p.get("total_models", len(p.get("models", [])))
  4073. label = f"{p['name']} ({count})"
  4074. if p.get("is_current"):
  4075. label = f"✓ {label}"
  4076. return InlineKeyboardButton(label, callback_data=f"mp:{p['slug']}")
  4077. @staticmethod
  4078. def _picker_nav_row(page: int, total_pages: int, prefix: str) -> list:
  4079. """``◀ Prev | n/N | Next ▶`` row (``prefix`` = ``mpv``/``mg`` page callback)."""
  4080. nav: list = []
  4081. if page > 0:
  4082. nav.append(InlineKeyboardButton(t("platform.telegram.picker.prev"), callback_data=f"{prefix}:{page - 1}"))
  4083. nav.append(InlineKeyboardButton(f"{page + 1}/{total_pages}", callback_data="mx:noop"))
  4084. if page < total_pages - 1:
  4085. nav.append(InlineKeyboardButton(t("platform.telegram.picker.next"), callback_data=f"{prefix}:{page + 1}"))
  4086. return nav
  4087. @staticmethod
  4088. def _picker_back_cancel_row() -> list:
  4089. return [InlineKeyboardButton(t("platform.telegram.picker.back"), callback_data="mb"), InlineKeyboardButton(t("platform.telegram.picker.cancel"), callback_data="mx")]
  4090. def _paged_keyboard(self, buttons: list, page_meta: dict, nav_prefix: str, tail_row: list) -> tuple:
  4091. rows = self._rows_of_two(buttons)
  4092. if page_meta["total_pages"] > 1:
  4093. rows.append(self._picker_nav_row(page_meta["page"], page_meta["total_pages"], nav_prefix))
  4094. rows.append(tail_row)
  4095. return InlineKeyboardMarkup(rows), page_meta["page_info"]
  4096. def _build_provider_keyboard(self, providers: list, page: int = 0) -> tuple:
  4097. """Paginated top-level provider keyboard folding provider families (Kimi/Moonshot, MiniMax, xAI…)
  4098. into one ``mpg:<gid>`` button via the shared ``group_providers`` fold; singles are ``mp:<slug>``."""
  4099. try:
  4100. from hermes_cli.models_catalog_static import group_providers
  4101. except Exception:
  4102. group_providers = None
  4103. by_slug = {p.get("slug"): p for p in providers}
  4104. buttons: list = []
  4105. if group_providers is not None:
  4106. for row in group_providers([p.get("slug") for p in providers]):
  4107. if row["kind"] == "group":
  4108. members = [by_slug[m] for m in row["members"] if m in by_slug]
  4109. count = sum(m.get("total_models", len(m.get("models", []))) for m in members)
  4110. label = f"{row['label']} ▸ ({count})"
  4111. if any(m.get("is_current") for m in members):
  4112. label = f"✓ {label}"
  4113. buttons.append(InlineKeyboardButton(label, callback_data=f"mpg:{row['group_id']}"))
  4114. else:
  4115. p = by_slug.get(row["slug"])
  4116. if p is not None:
  4117. buttons.append(self._provider_button(p))
  4118. else:
  4119. buttons = [self._provider_button(p) for p in providers]
  4120. page_buttons, page_meta = self._format_choice_page(buttons, page, self._PROVIDER_PAGE_SIZE)
  4121. return self._paged_keyboard(page_buttons, page_meta, "mpv", [InlineKeyboardButton(t("platform.telegram.picker.cancel"), callback_data="mx")])
  4122. def _build_model_keyboard(self, models: list, page: int, provider: Optional[dict] = None) -> tuple:
  4123. """Build paginated model buttons. Returns (keyboard, page_info_text). Window plans: each button
  4124. carries the plan word (``model-b · Large``) from the provider row; other accounts are unchanged."""
  4125. from hermes_cli.model_picker_usage import picker_row_suffix
  4126. page_models, page_meta = self._format_choice_page(models, page, self._MODEL_PAGE_SIZE)
  4127. start = page_meta["start"]
  4128. buttons: list = []
  4129. for i, model_id in enumerate(page_models):
  4130. short = model_id.split("/")[-1] if "/" in model_id else model_id
  4131. suffix = picker_row_suffix(provider, model_id)
  4132. room = 38 - (len(suffix) + 3 if suffix else 0)
  4133. if len(short) > room:
  4134. short = short[:max(room - 3, 8)] + "..."
  4135. if suffix:
  4136. short = f"{short} · {suffix}"
  4137. buttons.append(InlineKeyboardButton(short, callback_data=f"mm:{start + i}"))
  4138. return self._paged_keyboard(buttons, page_meta, "mg", self._picker_back_cancel_row())
  4139. async def _picker_edit(self, query, text_md: str, keyboard) -> None:
  4140. """Re-render the picker message in place (MarkdownV2) and ack the tap."""
  4141. await query.edit_message_text(text=self.format_message(text_md), parse_mode=ParseMode.MARKDOWN_V2, reply_markup=keyboard)
  4142. await query.answer()
  4143. async def _picker_show_models(self, query, state: dict, page: int) -> None:
  4144. """Render the model page for the provider currently selected in ``state``."""
  4145. models = state.get("model_list", [])
  4146. state["model_page"] = page
  4147. pname = state.get("selected_provider_name", "")
  4148. provider_slug = state.get("selected_provider", "")
  4149. provider = next((p for p in state["providers"] if p["slug"] == provider_slug), None)
  4150. keyboard, page_info = self._build_model_keyboard(models, page, provider)
  4151. total = provider.get("total_models", len(models)) if provider else len(models)
  4152. shown = len(models)
  4153. extra = f"\n_{t('platform.telegram.picker.more_available', count=str(total - shown))}_" if total > shown else ""
  4154. # Window plans: the limit line under the provider (absent for every other account).
  4155. plan_line = f"\n⚠️ {provider['plan_line']}" if provider and provider.get("plan_line") else ""
  4156. await self._picker_edit(
  4157. query,
  4158. f"⚙ *{t('platform.telegram.picker.title')}*\n\n"
  4159. f"{t('platform.telegram.picker.provider_label', provider=f'*{pname}*')}{page_info}{plan_line}\n"
  4160. f"{t('platform.telegram.picker.select_model')}{extra}",
  4161. keyboard)
  4162. @staticmethod
  4163. def _provider_list_text(current_model: str, provider_label: str, page_info: str) -> str:
  4164. model = f"`{current_model or t('platform.telegram.picker.unknown_model')}`"
  4165. return (f"⚙ *{t('platform.telegram.picker.title')}*\n\n{t('platform.telegram.picker.current_model', model=model)}\n"
  4166. f"{t('platform.telegram.picker.provider_label', provider=provider_label)}\n\n"
  4167. f"{t('platform.telegram.picker.select_provider')}{page_info}")
  4168. async def _picker_show_providers(self, query, state: dict, page: int, get_label) -> None:
  4169. """Render the (folded, paginated) provider list."""
  4170. keyboard, provider_page_info = self._build_provider_keyboard(state["providers"], page)
  4171. try:
  4172. provider_label = get_label(state["current_provider"])
  4173. except Exception:
  4174. provider_label = state["current_provider"]
  4175. await self._picker_edit(query, self._provider_list_text(state["current_model"], provider_label, provider_page_info), keyboard)
  4176. async def _picker_selection(self, query, state: dict, raw_idx: str) -> Optional[tuple]:
  4177. """Resolve ``mm:``/``mc:`` index → ``(idx, model_id, provider_slug, callback)``; answers + None on error."""
  4178. try:
  4179. idx = int(raw_idx)
  4180. except ValueError:
  4181. await query.answer(text=_toast("platform.telegram.picker.invalid_selection"))
  4182. return None
  4183. model_list = state.get("model_list", [])
  4184. if idx < 0 or idx >= len(model_list):
  4185. await query.answer(text=_toast("platform.telegram.picker.invalid_model_index"))
  4186. return None
  4187. callback = state.get("on_model_selected")
  4188. if not callback:
  4189. await query.answer(text=_toast("platform.telegram.picker.expired"))
  4190. return None
  4191. return idx, model_list[idx], state.get("selected_provider", ""), callback
  4192. async def _picker_switch(self, query, chat_id: str, model_id: str, provider_slug: str, callback) -> None:
  4193. """Perform the model switch, render the result, and drop the picker state."""
  4194. switch_failed = False
  4195. try:
  4196. result_text = await callback(chat_id, model_id, provider_slug)
  4197. except Exception as exc:
  4198. logger.error("Model picker switch failed: %s", exc)
  4199. result_text = t("platform.telegram.picker.switch_error", error=str(exc))
  4200. switch_failed = True
  4201. await self._edit_result_text(query, result_text)
  4202. await query.answer(text=_toast("platform.telegram.picker.switch_failed" if switch_failed else "platform.telegram.picker.switched"))
  4203. self._model_picker_state.pop(chat_id, None)
  4204. @staticmethod
  4205. async def _parse_page(query, raw: str) -> Optional[int]:
  4206. try:
  4207. return int(raw)
  4208. except ValueError:
  4209. await query.answer(text=_toast("platform.telegram.picker.invalid_page"))
  4210. return None
  4211. async def _handle_model_picker_callback(self, query, data: str, chat_id: str) -> None:
  4212. """Handle model picker callbacks (mp:/mpg:/mpv:/mm:/mc:/mb/mx/mg:)."""
  4213. state = self._model_picker_state.get(chat_id)
  4214. if not state:
  4215. await query.answer(text=_toast("platform.telegram.picker.expired_model"))
  4216. return
  4217. get_label = self._provider_get_label()
  4218. if data.startswith("mp:"): # provider selected: show model buttons (page 0)
  4219. provider_slug = data[3:]
  4220. provider = next((p for p in state["providers"] if p["slug"] == provider_slug), None)
  4221. if not provider:
  4222. await query.answer(text=_toast("platform.telegram.picker.provider_not_found"))
  4223. return
  4224. state["selected_provider"] = provider_slug
  4225. state["selected_provider_name"] = provider.get("name", provider_slug)
  4226. state["model_list"] = provider.get("models", [])
  4227. await self._picker_show_models(query, state, 0)
  4228. elif data.startswith("mg:"): # model page navigation
  4229. page = await self._parse_page(query, data[3:])
  4230. if page is not None:
  4231. await self._picker_show_models(query, state, page)
  4232. elif data.startswith("mpv:"): # provider page navigation
  4233. page = await self._parse_page(query, data[4:])
  4234. if page is not None:
  4235. state["provider_page"] = page
  4236. await self._picker_show_providers(query, state, page, get_label)
  4237. elif data.startswith("mc:"): # expensive model confirmed: perform the switch
  4238. sel = await self._picker_selection(query, state, data[3:])
  4239. if sel is not None:
  4240. _idx, model_id, provider_slug, callback = sel
  4241. await self._picker_switch(query, chat_id, model_id, provider_slug, callback)
  4242. elif data.startswith("mm:"): # model selected: warn if expensive, else perform the switch
  4243. sel = await self._picker_selection(query, state, data[3:])
  4244. if sel is None:
  4245. return
  4246. idx, model_id, provider_slug, callback = sel
  4247. try:
  4248. from hermes_cli.model_selection_guards import combined_selection_warning
  4249. # Pricing lookup may hit models.dev on a cache miss — keep it off the event loop.
  4250. warning = await asyncio.to_thread(combined_selection_warning, model_id, provider=provider_slug)
  4251. except Exception:
  4252. warning = None
  4253. if warning is not None:
  4254. keyboard = InlineKeyboardMarkup([
  4255. [InlineKeyboardButton(t("platform.telegram.picker.switch_anyway"), callback_data=f"mc:{idx}")], self._picker_back_cancel_row()])
  4256. await query.edit_message_text(
  4257. text=self.format_message(f"⚠ *{warning.title}*\n\n{warning.message}"),
  4258. parse_mode=ParseMode.MARKDOWN_V2, reply_markup=keyboard)
  4259. await query.answer(text=_toast("platform.telegram.picker.confirm_selection"))
  4260. return
  4261. await self._picker_switch(query, chat_id, model_id, provider_slug, callback)
  4262. elif data.startswith("mpg:"): # provider group selected: show member providers
  4263. group_id = data[4:]
  4264. try:
  4265. from hermes_cli.models_catalog_static import PROVIDER_GROUPS
  4266. _label, _desc, member_slugs = PROVIDER_GROUPS.get(group_id, ("", "", []))
  4267. except Exception:
  4268. _label, member_slugs = "", []
  4269. by_slug = {p["slug"]: p for p in state["providers"]}
  4270. members = [by_slug[m] for m in member_slugs if m in by_slug]
  4271. if not members:
  4272. await query.answer(text=_toast("platform.telegram.picker.group_not_found"))
  4273. return
  4274. rows = self._rows_of_two([self._provider_button(p) for p in members])
  4275. rows.append(self._picker_back_cancel_row())
  4276. await self._picker_edit(
  4277. query,
  4278. f"⚙ *{t('platform.telegram.picker.title')}*\n\n"
  4279. f"{t('platform.telegram.picker.provider_family', family=f'*{_label or group_id}*')}\n\n"
  4280. f"{t('platform.telegram.picker.select_provider')}",
  4281. InlineKeyboardMarkup(rows))
  4282. elif data == "mb": # back to provider list (folds groups)
  4283. await self._picker_show_providers(query, state, int(state.get("provider_page", 0) or 0), get_label)
  4284. elif data == "mx":
  4285. self._model_picker_state.pop(chat_id, None)
  4286. await query.edit_message_text(text=t("platform.telegram.picker.cancelled"), reply_markup=None)
  4287. await query.answer()
  4288. else:
  4289. await query.answer() # e.g. page-counter button "mx:noop"
  4290. async def _notify_clarify_expired(self, query, user_display: str) -> None:
  4291. """Tell the user a clarify tap arrived too late (entry evicted or gateway restarted) — otherwise
  4292. the tap leaves a misleading ✓ the agent never sees."""
  4293. with contextlib.suppress(Exception):
  4294. await query.answer(text=_toast("platform.telegram.prompt.expired_toast"))
  4295. await self._edit_html_quiet(
  4296. query, f"❓ {_html.escape(query.message.text or '')}\n\n<i>{_html.escape(t('platform.telegram.prompt.expired_body'))}</i>")
  4297. @staticmethod
  4298. async def _edit_html_quiet(query, text: str) -> None:
  4299. """HTML edit with the keyboard removed; failures ignored (non-fatal)."""
  4300. with contextlib.suppress(Exception):
  4301. await query.edit_message_text(text=text, parse_mode=ParseMode.HTML, reply_markup=None)
  4302. async def _edit_md_quiet(self, query, text_md: str) -> None:
  4303. """MarkdownV2 edit with the keyboard removed; failures ignored (non-fatal)."""
  4304. with contextlib.suppress(Exception):
  4305. await query.edit_message_text(text=self.format_message(text_md), parse_mode=ParseMode.MARKDOWN_V2, reply_markup=None)
  4306. async def _handle_inline_query(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
  4307. """Answer ``@botname <query>`` with a searchable command/skill picker (the ``/`` menu is capped at
  4308. 60 slots). Results are computed per keystroke, 50 per page; tapping sends ``/cmd`` text as the
  4309. user, so dispatch flows through the normal command path. Inline queries arrive from ANY chat, so
  4310. unauthorized users get an empty list (the skill catalog is not leaked)."""
  4311. inline_query = getattr(update, "inline_query", None)
  4312. if inline_query is None:
  4313. return
  4314. from_user = getattr(inline_query, "from_user", None)
  4315. user_id = str(getattr(from_user, "id", "") or "").strip()
  4316. try:
  4317. # No chat context on inline queries — authorize on user identity alone, DM-shaped.
  4318. authorized = bool(user_id) and self._is_callback_user_authorized(
  4319. user_id, chat_id=user_id, chat_type="private", user_name=getattr(from_user, "username", None))
  4320. except Exception:
  4321. logger.debug("[%s] inline picker auth check failed", self.name, exc_info=True)
  4322. authorized = False
  4323. if not authorized:
  4324. try:
  4325. from plugins.platforms.telegram.inline_picker import CACHE_TIME_SECONDS as _deny_cache
  4326. self._accept_update()
  4327. await inline_query.answer([], cache_time=_deny_cache, is_personal=True)
  4328. except Exception:
  4329. self._fail_update_preparation()
  4330. logger.debug("[%s] inline picker empty answer failed", self.name, exc_info=True)
  4331. return
  4332. try:
  4333. from telegram import InlineQueryResultArticle, InputTextMessageContent
  4334. from plugins.platforms.telegram.inline_picker import CACHE_TIME_SECONDS as _CACHE, build_inline_results
  4335. # Per-keystroke catalog build resolves every skill path; keep it off the loop (#110707).
  4336. results, next_offset = await asyncio.to_thread(
  4337. build_inline_results,
  4338. getattr(inline_query, "query", "") or "", offset=getattr(inline_query, "offset", "") or "")
  4339. articles = [
  4340. InlineQueryResultArticle(
  4341. id=r["id"], title=r["title"], description=r["description"],
  4342. input_message_content=InputTextMessageContent(r["message_text"]))
  4343. for r in results
  4344. ]
  4345. # is_personal: catalogs differ per user (auth, disabled skills) — never share cached pages.
  4346. self._accept_update()
  4347. await inline_query.answer(articles, cache_time=_CACHE, is_personal=True, next_offset=next_offset)
  4348. except Exception:
  4349. self._fail_update_preparation()
  4350. logger.debug("[%s] inline picker answer failed", self.name, exc_info=True)
  4351. @staticmethod
  4352. def _callback_ctx(query) -> dict[str, Any]:
  4353. """Chat/thread/user context of a button tap, for the callback auth gate."""
  4354. query_message = getattr(query, "message", None)
  4355. query_chat = getattr(query_message, "chat", None)
  4356. return {
  4357. "chat_id": getattr(query_message, "chat_id", None), "chat_type": getattr(query_chat, "type", None),
  4358. "thread_id": getattr(query_message, "message_thread_id", None), "user_name": getattr(query.from_user, "first_name", None)}
  4359. async def _callback_authorized(self, query, cb: dict[str, Any], denial_text: str) -> bool:
  4360. """Gate a button tap on the callback allowlist; answers ``denial_text`` when refused."""
  4361. if self._is_callback_user_authorized(
  4362. str(getattr(query.from_user, "id", "")), chat_id=cb["chat_id"],
  4363. chat_type=str(cb["chat_type"]) if cb["chat_type"] is not None else None,
  4364. thread_id=str(cb["thread_id"]) if cb["thread_id"] is not None else None, user_name=cb["user_name"]):
  4365. return True
  4366. await query.answer(text=denial_text)
  4367. return False
  4368. async def _handle_callback_query(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
  4369. """Dispatch inline keyboard button clicks on the callback_data prefix."""
  4370. query = update.callback_query
  4371. if not query or not query.data:
  4372. return
  4373. self._accept_update()
  4374. data = query.data
  4375. cb = self._callback_ctx(query)
  4376. # Model picker / generic choice picker (/reasoning, /fast) need a chat id.
  4377. for prefixes, handler in (
  4378. (("mp:", "mpg:", "mpv:", "mm:", "mc:", "mb", "mx", "mg:"), self._handle_model_picker_callback),
  4379. (("cp:",), self._handle_choice_picker_callback)):
  4380. if data.startswith(prefixes):
  4381. chat_id = str(query.message.chat_id) if query.message else None
  4382. # One auth gate for every chat-id picker: strangers in a shared group must not drive the owner's picker.
  4383. if chat_id and await self._callback_authorized(query, cb, _unauthorized()):
  4384. await handler(query, data, chat_id)
  4385. return
  4386. for prefix, handler in (
  4387. ("gt:", self._handle_gmail_triage_callback), ("ea:", self._handle_exec_approval_callback),
  4388. ("sc:", self._handle_slash_confirm_callback), ("cl:", self._handle_clarify_callback),
  4389. ("update_prompt:", self._handle_update_prompt_callback)):
  4390. if data.startswith(prefix):
  4391. await handler(query, data, cb)
  4392. return
  4393. async def _claim_callback_state(self, query, cb: dict[str, Any], state: dict, key, denial: str, resolved: str, *, pop: bool = True):
  4394. """Auth-gate a button tap, then claim its pending entry; None (after answering) when refused or expired."""
  4395. if not await self._callback_authorized(query, cb, denial):
  4396. return None
  4397. session_key = state.pop(key, None) if pop else state.get(key)
  4398. if not session_key:
  4399. await query.answer(text=resolved)
  4400. return session_key
  4401. async def _handle_exec_approval_callback(self, query, data: str, cb: dict[str, Any]) -> None:
  4402. """``ea:<choice>:<approval_id>`` — resolve a pending exec approval."""
  4403. parts = data.split(":", 2)
  4404. if len(parts) != 3:
  4405. return
  4406. choice = parts[1] # once, session, always, deny
  4407. try:
  4408. approval_id = int(parts[2])
  4409. except (ValueError, IndexError):
  4410. await query.answer(text=_toast("platform.telegram.approval.toast_invalid_data"))
  4411. return
  4412. session_key = await self._claim_callback_state(
  4413. query, cb, self._approval_state, approval_id, _unauthorized(),
  4414. _toast("platform.telegram.approval.toast_already_resolved"))
  4415. if not session_key:
  4416. return
  4417. request_id = self._approval_request_ids.pop(approval_id, None)
  4418. user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
  4419. try:
  4420. # Rendering happens after so the message reflects what actually occurred: a tap that lands after
  4421. # the approval wait timed out (count == 0) must NOT claim "Approved" — the command was already
  4422. # denied and will not run (#63501 regression follow-up: 60s waits made stale taps common).
  4423. from tools.approval import resolve_gateway_approval_card
  4424. count = resolve_gateway_approval_card(session_key, choice, request_id)
  4425. logger.info(
  4426. "Telegram button resolved %d approval(s) for session %s (choice=%s, user=%s)", count, session_key, choice, user_display)
  4427. except Exception as exc:
  4428. logger.error("Failed to resolve gateway approval from Telegram button: %s", exc)
  4429. count = 0
  4430. if count:
  4431. label_key = {"once": "resolved_once", "session": "resolved_session", "always": "resolved_always", "deny": "resolved_deny"}.get(
  4432. choice, "resolved_generic")
  4433. label = t(f"platform.telegram.approval.{label_key}")
  4434. edit_text = t("platform.telegram.approval.resolved_by_user", label=label, user=user_display)
  4435. else:
  4436. label = t("platform.telegram.approval.expired")
  4437. edit_text = t("platform.telegram.approval.expired_detail", label=label)
  4438. await query.answer(text=label[:_TOAST_LIMIT])
  4439. await edit_keeping_prompt(self, query, edit_text)
  4440. # Typing was paused when the approval was sent; the text /approve and /deny paths resume it too.
  4441. if count and cb["chat_id"] is not None:
  4442. self.resume_typing_for_chat(str(cb["chat_id"]))
  4443. async def _handle_slash_confirm_callback(self, query, data: str, cb: dict[str, Any]) -> None:
  4444. """``sc:<choice>:<confirm_id>`` — resolve a slash-command confirmation."""
  4445. parts = data.split(":", 2)
  4446. if len(parts) != 3:
  4447. return
  4448. choice = parts[1] # once, always, cancel
  4449. confirm_id = parts[2]
  4450. session_key = await self._claim_callback_state(
  4451. query, cb, self._slash_confirm_state, confirm_id, _unauthorized(),
  4452. _toast("platform.telegram.slash_confirm.already_resolved"))
  4453. if not session_key:
  4454. return
  4455. label_key = {"once": "slash_confirm.resolved_once", "always": "slash_confirm.resolved_always", "cancel": "slash_confirm.resolved_cancel"}.get(
  4456. choice, "approval.resolved_generic")
  4457. user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
  4458. label = t(f"platform.telegram.{label_key}")
  4459. await query.answer(text=label[:_TOAST_LIMIT])
  4460. decided = t("platform.telegram.approval.resolved_by_user", label=label, user=user_display)
  4461. await edit_keeping_prompt(self, query, decided)
  4462. # The runner stored a handler keyed by session_key; run it and send any returned text as a follow-up.
  4463. try:
  4464. from tools import slash_confirm as _slash_confirm_mod
  4465. result_text = await _slash_confirm_mod.resolve(session_key, confirm_id, choice)
  4466. if result_text and query.message:
  4467. # Inherit the prompt's topic: forums use message_thread_id; private DM-topic lanes need
  4468. # both the topic id and the prompt reply anchor.
  4469. thread_id = getattr(query.message, "message_thread_id", None)
  4470. chat_type = getattr(getattr(query.message, "chat", None), "type", None)
  4471. prompt_message_id = getattr(query.message, "message_id", None)
  4472. send_kwargs: dict[str, Any] = {
  4473. "chat_id": int(query.message.chat_id), "text": self.format_message(result_text),
  4474. "parse_mode": ParseMode.MARKDOWN_V2, **self._link_preview_kwargs()}
  4475. is_private_chat = str(getattr(chat_type, "value", chat_type)).lower() in {
  4476. "private", str(ChatType.PRIVATE).lower(), str(getattr(ChatType.PRIVATE, "value", ChatType.PRIVATE)).lower()}
  4477. if thread_id is not None:
  4478. meta: dict[str, Any] = {"thread_id": str(thread_id)}
  4479. reply_to_id = None
  4480. if is_private_chat and prompt_message_id is not None:
  4481. reply_to_id = send_kwargs["reply_to_message_id"] = int(prompt_message_id)
  4482. meta["telegram_dm_topic_reply_fallback"] = True
  4483. send_kwargs.update(self._thread_kwargs_for_send(
  4484. str(
  4485. query.message.chat_id
  4486. ), str(thread_id), meta, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode))
  4487. await self._send_message_with_thread_fallback(**send_kwargs)
  4488. except Exception as exc:
  4489. logger.error("[%s] slash-confirm callback failed: %s", self.name, exc, exc_info=True)
  4490. async def _handle_clarify_callback(self, query, data: str, cb: dict[str, Any]) -> None:
  4491. """``cl:<clarify_id>:<idx|other>`` — resolve a clarify prompt or flip to text capture."""
  4492. parts = data.split(":", 2)
  4493. if len(parts) != 3:
  4494. return
  4495. clarify_id = parts[1]
  4496. choice_token = parts[2]
  4497. session_key = await self._claim_callback_state(
  4498. query, cb, self._clarify_state, clarify_id, _unauthorized(),
  4499. _toast("platform.telegram.slash_confirm.already_resolved"), pop=False)
  4500. if not session_key:
  4501. return
  4502. user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
  4503. if choice_token == "other":
  4504. # Flip to text-capture: the gateway's text-intercept resolves the clarify with the next message.
  4505. # Do NOT pop _clarify_state yet — still needed if the entry gets cleared by something else.
  4506. flipped = False
  4507. try:
  4508. from tools.clarify_gateway import mark_awaiting_text
  4509. flipped = mark_awaiting_text(clarify_id)
  4510. except Exception as exc:
  4511. logger.warning("[%s] mark_awaiting_text failed: %s", self.name, exc)
  4512. if not flipped:
  4513. # Entry evicted / gateway restarted — a typed answer would go nowhere.
  4514. self._clarify_state.pop(clarify_id, None)
  4515. await self._notify_clarify_expired(query, user_display)
  4516. return
  4517. await query.answer(text=_toast("platform.telegram.prompt.type_answer"))
  4518. await self._edit_html_quiet(
  4519. query,
  4520. f"❓ {query.message.text or ''}\n\n"
  4521. f"<i>{_html.escape(t('platform.telegram.prompt.awaiting_typed', user=user_display))}</i>")
  4522. return
  4523. # Numeric choice → resolve immediately with the chosen text
  4524. try:
  4525. idx = int(choice_token)
  4526. except (ValueError, TypeError):
  4527. await query.answer(text=_toast("platform.telegram.prompt.invalid_choice"))
  4528. return
  4529. resolved_text: Optional[str] = None
  4530. try:
  4531. from tools.clarify_gateway import get_pending
  4532. entry = get_pending(clarify_id)
  4533. if entry and entry.choices and 0 <= idx < len(entry.choices):
  4534. resolved_text = entry.choices[idx]
  4535. except Exception:
  4536. resolved_text = None
  4537. if resolved_text is None:
  4538. # Race (timeout / session reset): echo the index so the agent sees an intentional response.
  4539. resolved_text = f"choice {idx + 1}"
  4540. self._clarify_state.pop(clarify_id, None)
  4541. try:
  4542. from tools.clarify_gateway import resolve_gateway_clarify
  4543. resolved = resolve_gateway_clarify(clarify_id, resolved_text)
  4544. except Exception as exc:
  4545. logger.error("[%s] resolve_gateway_clarify failed: %s", self.name, exc)
  4546. resolved = False
  4547. if resolved:
  4548. await query.answer(text=f"✓ {resolved_text[:60]}")
  4549. await self._edit_html_quiet(
  4550. query, f"❓ {_html.escape(query.message.text or '')}\n\n<b>{_html.escape(user_display)}:</b> {_html.escape(resolved_text)}")
  4551. logger.info("Telegram clarify button resolved (id=%s, choice=%r, user=%s)", clarify_id, resolved_text, user_display)
  4552. else:
  4553. # Entry evicted / gateway restarted between ask and tap.
  4554. await self._notify_clarify_expired(query, user_display)
  4555. logger.warning("Telegram clarify button: resolve_gateway_clarify returned False (id=%s)", clarify_id)
  4556. async def _handle_update_prompt_callback(self, query, data: str, cb: dict[str, Any]) -> None:
  4557. """``update_prompt:<y|n>`` — forward the answer to the update process."""
  4558. answer = data.split(":", 1)[1] # "y" or "n"
  4559. if not await self._callback_authorized(query, cb, _unauthorized()):
  4560. return
  4561. await query.answer(text=_toast("platform.telegram.prompt.update_sent", answer=answer))
  4562. word = t("platform.telegram.prompt.affirm_word" if answer == "y" else "platform.telegram.prompt.negate_word")
  4563. await self._edit_md_quiet(query, f"☤ {t('platform.telegram.prompt.update_answered', answer=f'*{word}*')}")
  4564. try:
  4565. from hermes_constants import get_hermes_home
  4566. response_path = get_hermes_home() / ".update_response"
  4567. tmp = response_path.with_suffix(".tmp")
  4568. tmp.write_text(answer, encoding="utf-8")
  4569. tmp.replace(response_path)
  4570. logger.info("Telegram update prompt answered '%s' by user %s", answer, getattr(query.from_user, "id", "unknown"))
  4571. except Exception as exc:
  4572. logger.error("Failed to write update response from callback: %s", exc)
  4573. # `gt:<verb>` -> (script in ~/.hermes/scripts/gmail-triage/, extra-args, success-label, is_state). The callback
  4574. # `arg` is always the first positional arg. is_state=True keeps the keyboard tappable (sticky sender rule);
  4575. # False strips it on success (per-email one-shot).
  4576. # The success label is a ``platform.telegram.gmail_triage.*`` catalog key, resolved per tap.
  4577. _GT_VERB_DISPATCH = {
  4578. "send": ("send-draft.sh", [], "done_send", False),
  4579. "archive": ("archive.sh", [], "done_archive", False),
  4580. "draft": ("draft-blank.sh", [], "done_draft", False),
  4581. "spam": ("spam.sh", [], "done_spam", False),
  4582. "mute": ("mute-add.sh", ["email"], "done_mute", True),
  4583. "mute-domain": ("mute-add.sh", ["domain"], "done_mute_domain", True),
  4584. "trust": ("trusted-ops-add.sh", ["email"], "done_trust", True),
  4585. "trust-domain": ("trusted-ops-add.sh", ["domain"], "done_trust_domain", True),
  4586. "vip": ("vip-add.sh", ["email"], "done_vip", True),
  4587. "vip-domain": ("vip-add.sh", ["domain"], "done_vip_domain", True)}
  4588. async def _handle_gmail_triage_callback(self, query, data: str, cb: dict[str, Any]) -> None:
  4589. """Dispatch a gmail-triage inline-button callback (gt:verb:arg)."""
  4590. parts = data.split(":", 2)
  4591. if len(parts) != 3:
  4592. await query.answer(text=_toast("platform.telegram.gmail_triage.invalid_data"))
  4593. return
  4594. verb, arg = parts[1], parts[2]
  4595. if not await self._callback_authorized(query, cb, _unauthorized()):
  4596. return
  4597. entry = self._GT_VERB_DISPATCH.get(verb)
  4598. if not entry:
  4599. await query.answer(text=_toast("platform.telegram.gmail_triage.unknown_verb", verb=verb))
  4600. return
  4601. script_name, extra_args, success_key, is_state_verb = entry
  4602. from hermes_constants import get_hermes_home
  4603. script_path = get_hermes_home() / "scripts" / "gmail-triage" / script_name
  4604. if not script_path.exists():
  4605. await query.answer(text=_toast("platform.telegram.gmail_triage.script_missing", script=script_name))
  4606. logger.error("[%s] gmail-triage script missing: %s", self.name, script_path)
  4607. return
  4608. success = False
  4609. try:
  4610. # A user script under HERMES_HOME the agent can write: scrubbed like cron and quick-command scripts.
  4611. from tools.environments.local import build_subprocess_env
  4612. proc = await asyncio.create_subprocess_exec(
  4613. str(script_path), arg, *extra_args, stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
  4614. env=build_subprocess_env(strip_launch_profile=True))
  4615. _stdout_bytes, stderr_bytes = await asyncio.wait_for(proc.communicate(), timeout=60)
  4616. if proc.returncode == 0:
  4617. label = t(f"platform.telegram.gmail_triage.{success_key}")
  4618. success = True
  4619. logger.info("[%s] gmail-triage callback ok: verb=%s arg=%s", self.name, verb, arg)
  4620. else:
  4621. stderr_text = stderr_bytes.decode("utf-8", errors="replace").strip()
  4622. last_line = stderr_text.splitlines()[-1] if stderr_text else t("platform.telegram.gmail_triage.exit_code", code=str(proc.returncode))
  4623. label = t("platform.telegram.gmail_triage.failed", verb=verb, detail=last_line[:80])
  4624. logger.error(
  4625. "[%s] gmail-triage callback failed: verb=%s arg=%s rc=%s stderr=%s", self.name, verb, arg, proc.returncode, stderr_text)
  4626. except TimeoutError:
  4627. label = t("platform.telegram.gmail_triage.timed_out", verb=verb)
  4628. logger.error("[%s] gmail-triage callback timed out: verb=%s arg=%s", self.name, verb, arg)
  4629. except Exception as exc:
  4630. label = t("platform.telegram.gmail_triage.error", verb=verb, error=str(exc))
  4631. logger.error("[%s] gmail-triage callback exception: verb=%s arg=%s err=%s", self.name, verb, arg, exc, exc_info=True)
  4632. await query.answer(text=label[:_TOAST_LIMIT])
  4633. if not success:
  4634. return
  4635. original_text = (query.message.text or "") if query.message else ""
  4636. user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
  4637. appended = f"{original_text}\n{t('platform.telegram.gmail_triage.appended_by', label=label, user=user_display)}"
  4638. # Sticky state verbs keep the keyboard so further actions can stack; one-shots strip it (can't fire twice).
  4639. with contextlib.suppress(Exception):
  4640. await query.edit_message_text(text=appended, **({} if is_state_verb else {"reply_markup": None}))
  4641. def _missing_media_path_error(self, label: str, path: str) -> str:
  4642. """File-not-found error for MEDIA delivery; /workspace-style paths often exist only in the sandbox."""
  4643. error = f"{label} file not found: {path}"
  4644. if path.startswith(("/workspace/", "/output/", "/outputs/")):
  4645. error += (
  4646. " (path may only exist inside the Docker sandbox. "
  4647. "Bind-mount a host directory and emit the host-visible path in MEDIA: for gateway file delivery.)")
  4648. return error
  4649. @staticmethod
  4650. def _sniff_raster_format(image_path: str) -> Optional[str]:
  4651. """Identify convertible raster formats by magic bytes.
  4652. Replacement for stdlib ``imghdr`` (removed in Python 3.13). Returns
  4653. one of ``png``/``gif``/``webp``/``bmp``/``tiff`` or None.
  4654. """
  4655. try:
  4656. with open(image_path, "rb") as fh:
  4657. head = fh.read(16)
  4658. except OSError:
  4659. return None
  4660. if head.startswith(b"\x89PNG\r\n\x1a\n"):
  4661. return "png"
  4662. if head.startswith((b"GIF87a", b"GIF89a")):
  4663. # GIFs are excluded: converting flattens animations to one frame.
  4664. return None
  4665. if head.startswith(b"RIFF") and head[8:12] == b"WEBP":
  4666. return "webp"
  4667. if head.startswith(b"BM"):
  4668. return "bmp"
  4669. if head.startswith((b"II*\x00", b"MM\x00*")):
  4670. return "tiff"
  4671. return None
  4672. def _compress_image_to_jpeg(self, image_path: str) -> Optional[str]:
  4673. """Pre-compress a large image to progressive JPEG before upload.
  4674. Behind an HTTP proxy (e.g. tgapi.indevs.in) the PTB
  4675. media_write_timeout (~20s) is easily exceeded by raw PNGs > 1-2MB.
  4676. A progressive JPEG at ~85% quality keeps the upload well under the
  4677. timeout while remaining visually equivalent for photos / info-graphics.
  4678. Returns the path to a temporary JPEG, or None when the original can be
  4679. used as-is (already small / already JPEG / Pillow not available). The
  4680. caller is responsible for cleaning up the returned temp file.
  4681. """
  4682. import tempfile
  4683. try:
  4684. file_size = os.path.getsize(image_path)
  4685. except OSError:
  4686. return None
  4687. ext = os.path.splitext(image_path)[1].lower()
  4688. # Already JPEG — no gain in converting back
  4689. if ext in (".jpg", ".jpeg"):
  4690. return None
  4691. # Skip tiny files; conversion cost > upload benefit
  4692. if file_size < self._IMG_COMPRESS_THRESHOLD_BYTES:
  4693. return None
  4694. # Only convert raster image formats (png, gif, webp, bmp, tiff).
  4695. # Magic-byte sniff instead of the stdlib imghdr module, which was
  4696. # removed in Python 3.13.
  4697. if self._sniff_raster_format(image_path) is None:
  4698. return None
  4699. try:
  4700. from PIL import Image
  4701. except Exception:
  4702. # Pillow missing: fall back to uploading the original (may timeout)
  4703. logger.warning("[%s] Pillow not available for image compression", self.name)
  4704. return None
  4705. try:
  4706. # Close the source handle before returning: convert()/resize() produce new images, so
  4707. # the original file object would otherwise stay open until garbage collection.
  4708. with Image.open(image_path) as src:
  4709. if src.mode in ("RGBA", "LA", "P"):
  4710. # Alpha-capable modes: composite onto a white background so
  4711. # transparency doesn't render as black in the JPEG.
  4712. background = Image.new("RGB", src.size, (255, 255, 255))
  4713. layer = src.convert("RGBA") if src.mode == "P" else src
  4714. if layer.mode in ("RGBA", "LA"):
  4715. background.paste(layer, mask=layer.split()[-1])
  4716. else:
  4717. background.paste(layer)
  4718. img = background
  4719. else:
  4720. img = src.convert("RGB")
  4721. max_w, max_h = img.size
  4722. max_dim = max(max_w, max_h)
  4723. if max_dim > self._IMG_MAX_DIMENSION:
  4724. scale = self._IMG_MAX_DIMENSION / max_dim
  4725. img = img.resize((int(max_w * scale), int(max_h * scale)), Image.LANCZOS)
  4726. fd, tmp = tempfile.mkstemp(
  4727. suffix=".jpg",
  4728. prefix="tg_compress_",
  4729. )
  4730. os.close(fd)
  4731. img.save(
  4732. tmp,
  4733. "JPEG",
  4734. quality=self._IMG_JPEG_QUALITY,
  4735. progressive=True,
  4736. optimize=True,
  4737. )
  4738. logger.info(
  4739. "[%s] Pre-compressed %s (%.1fKB → %s %.1fKB) for Telegram upload",
  4740. self.name,
  4741. image_path,
  4742. file_size / 1024,
  4743. tmp,
  4744. os.path.getsize(tmp) / 1024,
  4745. )
  4746. return tmp
  4747. except Exception as e:
  4748. logger.warning(
  4749. "[%s] Image compression failed, uploading original: %s",
  4750. self.name,
  4751. e,
  4752. )
  4753. return None
  4754. def _telegram_media_too_large_note(self, label: str, file_size: Any, max_bytes: int) -> str:
  4755. limit_mb = max(1, max_bytes // (1024 * 1024))
  4756. try:
  4757. size_text = f"{int(file_size or 0) / (1024 * 1024):.1f} MB"
  4758. except (TypeError, ValueError):
  4759. size_text = "unknown size"
  4760. return f"[Telegram {label} skipped: file size {size_text} exceeds the {limit_mb} MB limit. Ask the user to send a smaller file.]"
  4761. @staticmethod
  4762. def _int_or_zero(value: Any) -> int:
  4763. try:
  4764. return int(value or 0)
  4765. except (TypeError, ValueError):
  4766. return 0
  4767. def _telegram_media_size_allowed(self, source: Any, label: str) -> tuple[bool, Optional[str]]:
  4768. """Validate Telegram media size before downloading into memory."""
  4769. max_bytes = int(getattr(self, "_max_doc_bytes", 20 * 1024 * 1024) or 20 * 1024 * 1024)
  4770. size = self._int_or_zero(getattr(source, "file_size", None))
  4771. if size <= 0 or size <= max_bytes:
  4772. return True, None
  4773. return False, self._telegram_media_too_large_note(label, size, max_bytes)
  4774. def _media_send_kwargs(
  4775. self, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> tuple[Optional[int], dict[str, Any]]:
  4776. """Return ``(reply_to_id, base_kwargs)`` shared by every native media send."""
  4777. reply_to_id = self._reply_to_message_id_for_send(reply_to, metadata, reply_to_mode=self._reply_to_mode)
  4778. thread_kwargs = self._thread_kwargs_for_send(
  4779. chat_id, self._metadata_thread_id(metadata), metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
  4780. return reply_to_id, {
  4781. "chat_id": normalize_telegram_chat_id(chat_id), "reply_to_message_id": reply_to_id,
  4782. "read_timeout": _MEDIA_SEND_READ_TIMEOUT, **thread_kwargs, **self._notification_kwargs(metadata)}
  4783. async def _send_media(
  4784. self, send_fn: Any, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]], media_label: str,
  4785. reset_media: Optional[Any] = None, **media_kwargs: Any) -> Any:
  4786. """Send one native media payload with thread routing + DM-topic anchor retry."""
  4787. reply_to_id, kwargs = self._media_send_kwargs(chat_id, reply_to, metadata)
  4788. return await self._send_with_dm_topic_reply_anchor_retry(
  4789. send_fn, {**kwargs, **media_kwargs}, metadata, reply_to_id, media_label, reset_media=reset_media)
  4790. @staticmethod
  4791. def _caption_1024(caption: Optional[str]) -> Optional[str]:
  4792. return caption[:1024] if caption else None
  4793. async def _send_voice_bubble(self, audio_file, chat_id, reply_to, metadata, caption, duration_secs):
  4794. """sendVoice with caption variants: MarkdownV2 when it fits 1024 chars, plain fallback when the
  4795. Bot API rejects the entities; anything else is a real error."""
  4796. # Render caption markdown (#32029): auto-TTS captions carry the agent's markdown reply, which showed
  4797. # literal *asterisks* and [links](...) without a parse_mode. Format to MarkdownV2 when it fits the
  4798. # 1024-char caption cap; fall back to the raw text (previous behaviour) when formatting would
  4799. # overflow or the Bot API rejects the entities.
  4800. _caption_variants: list[tuple] = []
  4801. if caption:
  4802. try:
  4803. _formatted_caption = self.format_message(caption)
  4804. if utf16_len(_formatted_caption) <= 1024:
  4805. _caption_variants.append((_formatted_caption, ParseMode.MARKDOWN_V2))
  4806. except Exception:
  4807. logger.debug("[%s] voice caption MarkdownV2 formatting failed; sending plain caption", self.name, exc_info=True)
  4808. _caption_variants.append((caption[:1024], None))
  4809. else:
  4810. _caption_variants.append((None, None))
  4811. _last_parse_error: Optional[Exception] = None
  4812. for _cap_text, _cap_parse_mode in _caption_variants:
  4813. try:
  4814. return await self._send_media(
  4815. self._bot.send_voice, chat_id, reply_to, metadata, "voice", reset_media=lambda: audio_file.seek(0),
  4816. voice=audio_file, caption=_cap_text, parse_mode=_cap_parse_mode, duration=duration_secs)
  4817. except Exception as _cap_error:
  4818. err = str(_cap_error).lower()
  4819. if _cap_parse_mode is not None and ("parse" in err or "entit" in err):
  4820. logger.warning(
  4821. "[%s] voice caption MarkdownV2 rejected, retrying plain: %s", self.name, _redact_telegram_error_text(_cap_error))
  4822. _last_parse_error = _cap_error
  4823. audio_file.seek(0)
  4824. continue
  4825. raise
  4826. raise _last_parse_error or RuntimeError("Telegram send_voice failed for all caption variants")
  4827. async def send_voice(
  4828. self, chat_id: str, audio_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
  4829. metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
  4830. """Send audio as a native Telegram voice message or audio file."""
  4831. if not self._bot:
  4832. return SendResult(success=False, error="Not connected")
  4833. _transcoded_voice_path: Optional[str] = None
  4834. try:
  4835. if not os.path.exists(audio_path):
  4836. return SendResult(success=False, error=self._missing_media_path_error("Audio", audio_path))
  4837. # sendVoice only accepts Ogg/Opus: an explicit voice-bubble request (is_voice) transcodes via
  4838. # ffmpeg; otherwise route by extension (.mp3/.m4a → sendAudio, others → document).
  4839. if kwargs.get("is_voice") and os.path.splitext(audio_path)[1].lower() not in (".ogg", ".opus"):
  4840. from gateway.platforms.base import transcode_to_ogg_opus
  4841. _transcoded_voice_path = await asyncio.to_thread(transcode_to_ogg_opus, audio_path)
  4842. if _transcoded_voice_path:
  4843. audio_path = _transcoded_voice_path
  4844. else:
  4845. logger.warning(
  4846. "[%s] voice transcode unavailable for %s — sending original format (install ffmpeg for voice bubbles)",
  4847. self.name, os.path.basename(audio_path))
  4848. # Telegram drops duration for long clips (~5 min+, shows 0:00).
  4849. _duration_secs = await asyncio.to_thread(_probe_voice_duration_seconds, audio_path)
  4850. with open(audio_path, "rb") as audio_file: # noqa: ASYNC230 -- file handle is streamed to the upload; a local open() is non-blocking in practice
  4851. ext = os.path.splitext(audio_path)[1].lower()
  4852. if ext in {".ogg", ".opus"}: # round playable voice bubble
  4853. msg = await self._send_voice_bubble(audio_file, chat_id, reply_to, metadata, caption, _duration_secs)
  4854. elif ext in {".mp3", ".m4a"}: # Bot API sendAudio only accepts MP3 / M4A
  4855. msg = await self._send_media(
  4856. self._bot.send_audio, chat_id, reply_to, metadata, "audio", reset_media=lambda: audio_file.seek(0),
  4857. audio=audio_file, caption=self._caption_1024(caption), duration=_duration_secs)
  4858. else: # formats Telegram can't play natively (.wav, .flac, ...)
  4859. return await self.send_document(
  4860. chat_id=chat_id, file_path=audio_path, caption=caption, reply_to=reply_to, metadata=metadata)
  4861. return SendResult(success=True, message_id=str(msg.message_id))
  4862. except Exception as e:
  4863. logger.error(
  4864. "[%s] Failed to send Telegram voice/audio, falling back to base adapter: %s", self.name,
  4865. _redact_telegram_error_text(e), exc_info=True)
  4866. return await super().send_voice(chat_id, audio_path, caption, reply_to, metadata=metadata)
  4867. finally:
  4868. if _transcoded_voice_path:
  4869. with contextlib.suppress(OSError):
  4870. os.unlink(_transcoded_voice_path)
  4871. async def send_multiple_images(
  4872. self, chat_id: str, images: list[tuple], metadata: Optional[dict[str, Any]] = None, human_delay: float = 0.0) -> SendResult:
  4873. """Send images as Telegram albums (``send_media_group``, 10 per chunk). Animated GIFs can't join a
  4874. media group (need ``send_animation``) so they go via the base per-image path, as does a failed chunk."""
  4875. if not self._bot:
  4876. return SendResult(success=False, error="Not connected")
  4877. if not images:
  4878. return SendResult(success=False, error="no images to send")
  4879. try:
  4880. from telegram import InputMediaPhoto
  4881. except Exception as exc: # pragma: no cover - missing SDK
  4882. logger.warning("[%s] InputMediaPhoto unavailable, falling back to per-image send: %s", self.name, exc)
  4883. return await super().send_multiple_images(chat_id, images, metadata, human_delay)
  4884. is_anim = lambda url: not url.startswith("file://") and self._is_animation_url(url)
  4885. animations = [img for img in images if is_anim(img[0])]
  4886. photos = [img for img in images if not is_anim(img[0])]
  4887. delivered = False
  4888. if animations:
  4889. anim_result = await super().send_multiple_images(chat_id, animations, metadata, human_delay=human_delay)
  4890. delivered = anim_result.success
  4891. if not photos:
  4892. return SendResult(success=delivered, error=None if delivered else "all images failed to send")
  4893. from urllib.parse import unquote as _unquote
  4894. CHUNK = 10 # Telegram's album limit
  4895. chunks = [photos[i:i + CHUNK] for i in range(0, len(photos), CHUNK)]
  4896. for chunk_idx, chunk in enumerate(chunks):
  4897. if human_delay > 0 and chunk_idx > 0:
  4898. await asyncio.sleep(human_delay)
  4899. media: list[Any] = []
  4900. opened_files: list[Any] = []
  4901. temp_paths: list[str] = []
  4902. try:
  4903. for image_url, alt_text in chunk:
  4904. source: Any = image_url
  4905. if image_url.startswith("file://"):
  4906. local_path = _unquote(image_url[7:])
  4907. if not os.path.exists(local_path):
  4908. logger.warning("[%s] Skipping missing image in media group: %s", self.name, local_path)
  4909. continue
  4910. # Pre-compress large raster images so the media-group upload stays under
  4911. # media_write_timeout; the temp JPEG is removed after the send.
  4912. compressed = self._compress_image_to_jpeg(local_path)
  4913. if compressed:
  4914. temp_paths.append(compressed)
  4915. local_path = compressed
  4916. source = open(local_path, "rb") # noqa: ASYNC230 -- file handle is streamed to the upload; a local open() is non-blocking in practice
  4917. opened_files.append(source)
  4918. media.append(InputMediaPhoto(media=source, caption=self._caption_1024(alt_text)))
  4919. if not media:
  4920. continue
  4921. logger.info("[%s] Sending media group of %d photo(s) (chunk %d/%d)", self.name, len(media), chunk_idx + 1, len(chunks))
  4922. reply_to_id, send_kwargs = self._media_send_kwargs(chat_id, None, metadata)
  4923. def _reset_opened_files() -> None:
  4924. for fh in opened_files:
  4925. with contextlib.suppress(Exception):
  4926. fh.seek(0)
  4927. await self._send_with_dm_topic_reply_anchor_retry(
  4928. self._bot.send_media_group, {**send_kwargs, "media": media}, metadata, reply_to_id,
  4929. "media group", reset_media=_reset_opened_files)
  4930. delivered = True
  4931. except Exception as e:
  4932. logger.warning(
  4933. "[%s] send_media_group failed (chunk %d/%d), falling back to per-image: %s", self.name,
  4934. chunk_idx + 1, len(chunks), _redact_telegram_error_text(e), exc_info=True)
  4935. fallback = await super().send_multiple_images(chat_id, chunk, metadata, human_delay=human_delay)
  4936. delivered = delivered or fallback.success
  4937. finally:
  4938. for fh in opened_files:
  4939. with contextlib.suppress(Exception):
  4940. fh.close()
  4941. for tmp in temp_paths:
  4942. with contextlib.suppress(OSError):
  4943. os.remove(tmp)
  4944. return SendResult(success=delivered, error=None if delivered else "all images failed to send")
  4945. async def send_image_file(
  4946. self, chat_id: str, image_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
  4947. metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
  4948. """Send a local image file natively as a Telegram photo."""
  4949. # Pre-compress large raster images to progressive JPEG once; the photo send and the document
  4950. # fallback both reuse the compressed file so either upload stays under media_write_timeout.
  4951. compressed = self._compress_image_to_jpeg(image_path)
  4952. actual_path = compressed or image_path
  4953. doc_name = os.path.splitext(os.path.basename(image_path))[0] + ".jpg" if compressed else os.path.basename(image_path)
  4954. async def _photo_failed(e: Exception) -> SendResult:
  4955. error_str = str(e)
  4956. # Dimension errors are expected for valid images Telegram refuses as photos → INFO.
  4957. if "Photo_invalid_dimensions" in error_str or "PHOTO_INVALID_DIMENSIONS" in error_str:
  4958. logger.info("[%s] Image dimensions exceed Telegram photo limits, sending as document: %s", self.name, image_path)
  4959. else:
  4960. logger.warning(
  4961. "[%s] Failed to send Telegram local image as photo, trying document fallback: %s", self.name,
  4962. _redact_telegram_error_text(e), exc_info=True)
  4963. # Document has no dimension limit (50MB only); if even that fails, base adapter text.
  4964. try:
  4965. return await self.send_document(
  4966. chat_id=chat_id, file_path=actual_path, caption=caption, file_name=doc_name,
  4967. reply_to=reply_to, metadata=metadata)
  4968. except Exception as doc_err:
  4969. logger.error(
  4970. "[%s] Failed to send Telegram local image as document, falling back to base adapter: %s",
  4971. self.name, doc_err, exc_info=True)
  4972. return await super(TelegramAdapter, self).send_image_file(chat_id, image_path, caption, reply_to, metadata=metadata)
  4973. try:
  4974. return await self._send_local_file(
  4975. "Image", actual_path, chat_id, reply_to, metadata, "photo",
  4976. lambda f: {"photo": f, "caption": self._caption_1024(caption)}, _photo_failed)
  4977. finally:
  4978. if compressed:
  4979. with contextlib.suppress(OSError):
  4980. os.remove(compressed)
  4981. async def _send_local_file(
  4982. self, label: str, path: str, chat_id, reply_to, metadata, media_key: str, build_kwargs, on_error,
  4983. ) -> SendResult:
  4984. """Shared shell for native local-file sends: existence check, open, send with routing, then
  4985. ``await on_error(exc)`` on any failure. ``build_kwargs(f)`` supplies the media kwargs."""
  4986. if not self._bot:
  4987. return SendResult(success=False, error="Not connected")
  4988. try:
  4989. if not os.path.exists(path):
  4990. return SendResult(success=False, error=self._missing_media_path_error(label, path))
  4991. with open(path, "rb") as f: # noqa: ASYNC230 -- file handle is streamed to the upload; a local open() is non-blocking in practice
  4992. msg = await self._send_media(
  4993. getattr(self._bot, f"send_{media_key}"), chat_id, reply_to, metadata, media_key,
  4994. reset_media=lambda: f.seek(0), **build_kwargs(f))
  4995. return SendResult(success=True, message_id=str(msg.message_id))
  4996. except Exception as e:
  4997. return await on_error(e)
  4998. async def _warn_then(self, media_key: str, e: Exception, fallback) -> SendResult:
  4999. logger.warning("[%s] Failed to send %s: %s", self.name, media_key, _redact_telegram_error_text(e))
  5000. return await fallback
  5001. async def send_document(
  5002. self, chat_id: str, file_path: str, caption: Optional[str] = None, file_name: Optional[str] = None,
  5003. reply_to: Optional[str] = None, metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
  5004. """Send a document/file natively as a Telegram file attachment."""
  5005. return await self._send_local_file(
  5006. "File", file_path, chat_id, reply_to, metadata, "document",
  5007. lambda f: {"document": f, "filename": file_name or os.path.basename(file_path), "caption": self._caption_1024(caption)},
  5008. lambda e: self._warn_then(
  5009. "document", e, super(
  5010. TelegramAdapter, self,
  5011. ).send_document(chat_id, file_path, caption, file_name, reply_to, metadata=metadata)))
  5012. async def send_video(
  5013. self, chat_id: str, video_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
  5014. metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
  5015. """Send a video natively as a Telegram video message.
  5016. Real geometry and a JPEG thumbnail ride along explicitly. Telegram only runs its own video
  5017. processing for uploads under roughly 10 MB; larger files come back as an unprocessed
  5018. ``320x320`` video with ``duration=0`` and no thumbnail, which clients then draw as a square
  5019. tile whatever the true aspect ratio (portrait reels and 16:9 clips alike).
  5020. """
  5021. geometry = await asyncio.to_thread(_probe_video_geometry, video_path)
  5022. thumb_path = (
  5023. await asyncio.to_thread(_video_thumbnail_jpeg, video_path, geometry.get("duration"))
  5024. if geometry else None
  5025. )
  5026. try:
  5027. def build_kwargs(f):
  5028. payload = {"video": f, "caption": self._caption_1024(caption), **geometry}
  5029. if thumb_path:
  5030. # A path (not an open handle) so python-telegram-bot loads the bytes once and a
  5031. # retry after a stale topic anchor still has a thumbnail to send.
  5032. payload["thumbnail"] = thumb_path
  5033. return payload
  5034. return await self._send_local_file(
  5035. "Video", video_path, chat_id, reply_to, metadata, "video", build_kwargs,
  5036. lambda e: self._warn_then(
  5037. "video", e, super(TelegramAdapter, self).send_video(chat_id, video_path, caption, reply_to, metadata=metadata),
  5038. ))
  5039. finally:
  5040. if thumb_path:
  5041. with contextlib.suppress(OSError):
  5042. os.remove(thumb_path)
  5043. async def send_image(
  5044. self, chat_id: str, image_url: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
  5045. metadata: Optional[dict[str, Any]] = None) -> SendResult:
  5046. """Send a URL image as a Telegram photo: URL send (<5MB) → download+upload (≤10MB) → base text."""
  5047. if not self._bot:
  5048. return SendResult(success=False, error="Not connected")
  5049. from tools.url_safety import is_safe_url
  5050. if not is_safe_url(image_url):
  5051. logger.warning("[%s] Blocked unsafe image URL (SSRF protection)", self.name)
  5052. return await super().send_image(chat_id, image_url, caption, reply_to, metadata=metadata)
  5053. photo_caption = self._caption_1024(caption)
  5054. try:
  5055. msg = await self._send_media(
  5056. self._bot.send_photo, chat_id, reply_to, metadata, "URL photo", photo=image_url, caption=photo_caption)
  5057. return SendResult(success=True, message_id=str(msg.message_id))
  5058. except Exception as e:
  5059. logger.warning(
  5060. "[%s] URL-based send_photo failed, trying file upload: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
  5061. try:
  5062. from gateway.platforms.base import ssrf_redirect_guard
  5063. from tools.url_safety import create_ssrf_safe_async_client
  5064. async with create_ssrf_safe_async_client(timeout=30.0, event_hooks={"response": [ssrf_redirect_guard]}) as client:
  5065. resp = await client.get(image_url)
  5066. resp.raise_for_status()
  5067. image_data = resp.content
  5068. msg = await self._send_media(
  5069. self._bot.send_photo, chat_id, reply_to, metadata, "uploaded photo", photo=image_data, caption=photo_caption)
  5070. return SendResult(success=True, message_id=str(msg.message_id))
  5071. except Exception as e2:
  5072. logger.error("[%s] File upload send_photo also failed: %s", self.name, e2, exc_info=True)
  5073. return await super().send_image(chat_id, image_url, caption, reply_to, metadata=metadata)
  5074. async def send_animation(
  5075. self, chat_id: str, animation_url: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
  5076. metadata: Optional[dict[str, Any]] = None) -> SendResult:
  5077. """Send an animated GIF natively as a Telegram animation (auto-plays inline)."""
  5078. if not self._bot:
  5079. return SendResult(success=False, error="Not connected")
  5080. try:
  5081. msg = await self._send_media(
  5082. self._bot.send_animation, chat_id, reply_to, metadata, "animation", animation=animation_url,
  5083. caption=self._caption_1024(caption))
  5084. return SendResult(success=True, message_id=str(msg.message_id))
  5085. except Exception as e:
  5086. logger.error(
  5087. "[%s] Failed to send Telegram animation, falling back to photo: %s", self.name,
  5088. _redact_telegram_error_text(e), exc_info=True)
  5089. return await self.send_image(chat_id, animation_url, caption, reply_to, metadata=metadata)
  5090. @staticmethod
  5091. def _is_transient_typing_error(exc: Exception) -> bool:
  5092. """Return True for Telegram typing errors worth cooling down."""
  5093. if getattr(exc, "retry_after", None) is not None:
  5094. return True
  5095. status_code = getattr(exc, "status_code", None) or getattr(exc, "code", None)
  5096. if isinstance(status_code, int) and (status_code == 429 or status_code >= 500):
  5097. return True
  5098. text = str(exc).lower()
  5099. if any(marker in text for marker in ("too many requests", "rate limit", "timed out", "timeout", "temporar")):
  5100. return True
  5101. return isinstance(exc, (OSError, TimeoutError, ConnectionError, asyncio.TimeoutError))
  5102. def _record_typing_cooldown(self, chat_id: str, exc: Exception) -> None:
  5103. """Suppress Telegram typing refreshes for this chat after transient failures."""
  5104. if not hasattr(self, "_telegram_typing_cooldown_until"):
  5105. self._telegram_typing_cooldown_until = {}
  5106. retry_after = getattr(exc, "retry_after", None)
  5107. try:
  5108. delay = float(retry_after) if retry_after is not None else self._telegram_typing_cooldown_seconds
  5109. except (TypeError, ValueError):
  5110. delay = self._telegram_typing_cooldown_seconds
  5111. self._telegram_typing_cooldown_until[str(chat_id)] = asyncio.get_running_loop().time() + max(1.0, min(delay, 300.0))
  5112. def _typing_in_cooldown(self, chat_id: str) -> bool:
  5113. if not hasattr(self, "_telegram_typing_cooldown_until"):
  5114. self._telegram_typing_cooldown_until = {}
  5115. self._telegram_typing_cooldown_seconds = 30.0
  5116. until = self._telegram_typing_cooldown_until.get(str(chat_id))
  5117. if until is None:
  5118. return False
  5119. if asyncio.get_running_loop().time() < until:
  5120. return True
  5121. self._telegram_typing_cooldown_until.pop(str(chat_id), None)
  5122. return False
  5123. # --- per-chat send ordering + flood cooldown (#114396) ---------------------------------------------
  5124. # Both keyed by the Bot-API-normalized chat id: the text path passes the raw chat_id while the media
  5125. # funnel's send_kwargs carry the normalized value. ``__dict__.setdefault``: tests build adapters via
  5126. # ``object.__new__()`` (no __init__).
  5127. @contextlib.asynccontextmanager
  5128. async def _chat_send_lock(self, chat_id: Any):
  5129. """FIFO per-chat gate around outgoing API calls, reentrant within one asyncio task (media paths
  5130. nest: send_voice → send_document, and ``super().send_*`` fallbacks reach ``send()``; a plain
  5131. ``asyncio.Lock`` re-acquired by its holder would wedge that chat's sends for good)."""
  5132. key = str(normalize_telegram_chat_id(chat_id))
  5133. locks: dict[str, asyncio.Lock] = self.__dict__.setdefault("_telegram_chat_send_locks", {})
  5134. owners: dict[str, asyncio.Task] = self.__dict__.setdefault("_telegram_chat_send_lock_owners", {})
  5135. task = asyncio.current_task()
  5136. if owners.get(key) is task:
  5137. yield
  5138. return
  5139. lock = locks.setdefault(key, asyncio.Lock())
  5140. async with lock:
  5141. owners[key] = task
  5142. try:
  5143. yield
  5144. finally:
  5145. owners.pop(key, None)
  5146. if not getattr(lock, "_waiters", None): # nobody queued: drop the entry (bounded dict)
  5147. locks.pop(key, None)
  5148. def _record_send_flood_cooldown(self, chat_id: Any, wait: float) -> SendResult:
  5149. """A send refused with ``retry_after=wait`` arms a per-chat window during which ``send()`` fails
  5150. closed locally (same ``flood_control:<s>`` result, so ledger recognition and redelivery timing are
  5151. unchanged) instead of firing more requests into a penalty Telegram lengthens while it is hammered."""
  5152. until: dict[str, float] = self.__dict__.setdefault("_telegram_send_cooldown_until", {})
  5153. until[str(normalize_telegram_chat_id(chat_id))] = asyncio.get_running_loop().time() + max(1.0, min(float(wait), 300.0))
  5154. return _flood_cap_result(wait)
  5155. def _send_flood_cooldown_remaining(self, chat_id: Any) -> Optional[float]:
  5156. """Seconds left in this chat's flood window, or ``None`` when sends may go out."""
  5157. until: dict[str, float] = self.__dict__.setdefault("_telegram_send_cooldown_until", {})
  5158. key = str(normalize_telegram_chat_id(chat_id))
  5159. deadline = until.get(key)
  5160. if deadline is None:
  5161. return None
  5162. remaining = deadline - asyncio.get_running_loop().time()
  5163. if remaining > 0:
  5164. return remaining
  5165. until.pop(key, None)
  5166. return None
  5167. # --- shared per-chat send+edit pacing budget (#116312) -----------------------------------------
  5168. # One slot per chat that sendMessage AND editMessageText both draw from (Telegram counts them
  5169. # against the same per-chat allowance). ``_telegram_chat_outbound_slot_until`` maps the
  5170. # normalized chat id to the loop-time when the next outbound call may fire.
  5171. def _chat_outbound_slot_remaining(self, chat_id: Any) -> float:
  5172. """Seconds until this chat's shared send+edit slot is open again (0 = may fire now)."""
  5173. slot_until: dict[str, float] = self.__dict__.setdefault("_telegram_chat_outbound_slot_until", {})
  5174. key = str(normalize_telegram_chat_id(chat_id))
  5175. deadline = slot_until.get(key)
  5176. if deadline is None:
  5177. return 0.0
  5178. remaining = deadline - asyncio.get_running_loop().time()
  5179. if remaining <= 0:
  5180. slot_until.pop(key, None) # expired — bounded dict, like the send locks
  5181. return 0.0
  5182. return remaining
  5183. def _hold_chat_outbound_slot(self, chat_id: Any) -> None:
  5184. """Arm/re-arm this chat's slot after an actual send/edit API call fires."""
  5185. slot_until: dict[str, float] = self.__dict__.setdefault("_telegram_chat_outbound_slot_until", {})
  5186. budget = getattr(self, "_telegram_chat_outbound_slot_secs", _TELEGRAM_CHAT_OUTBOUND_BUDGET_SECS)
  5187. slot_until[str(normalize_telegram_chat_id(chat_id))] = (
  5188. asyncio.get_running_loop().time() + max(0.0, budget))
  5189. async def send_typing(self, chat_id: str, metadata: Optional[dict[str, Any]] = None) -> None:
  5190. """Send typing indicator."""
  5191. if not self._bot or self._typing_in_cooldown(chat_id):
  5192. return
  5193. _is_dm_topic: bool = False
  5194. message_thread_id: Optional[int] = None
  5195. async def _action(**kw) -> None:
  5196. await self._bot.send_chat_action(chat_id=normalize_telegram_chat_id(chat_id), action="typing", **kw)
  5197. self._telegram_typing_cooldown_until.pop(str(chat_id), None)
  5198. try:
  5199. _is_dm_topic = self._dm_topic_fallback(metadata)
  5200. message_thread_id = self._message_thread_id_for_typing(self._metadata_thread_id(metadata))
  5201. await _action(message_thread_id=message_thread_id)
  5202. except Exception as e:
  5203. # DM topic lanes: Telegram may reject message_thread_id — retry without it so the indicator at
  5204. # least appears in the main DM view.
  5205. if _is_dm_topic and message_thread_id is not None:
  5206. try:
  5207. await _action()
  5208. return
  5209. except Exception as fallback_exc:
  5210. if self._is_transient_typing_error(fallback_exc):
  5211. self._record_typing_cooldown(chat_id, fallback_exc)
  5212. elif self._is_transient_typing_error(e):
  5213. self._record_typing_cooldown(chat_id, e)
  5214. logger.debug("[%s] Failed to send Telegram typing indicator: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
  5215. async def get_chat_info(self, chat_id: str) -> dict[str, Any]:
  5216. """Get information about a Telegram chat."""
  5217. if not self._bot:
  5218. return {"name": "Unknown", "type": "dm"}
  5219. try:
  5220. chat = await self._bot.get_chat(normalize_telegram_chat_id(chat_id))
  5221. chat_type = "dm"
  5222. if chat.type == ChatType.GROUP:
  5223. chat_type = "group"
  5224. elif chat.type == ChatType.SUPERGROUP:
  5225. chat_type = "forum" if chat.is_forum else "group"
  5226. elif chat.type == ChatType.CHANNEL:
  5227. chat_type = "channel"
  5228. return {
  5229. "name": chat.title or chat.full_name or str(chat_id), "type": chat_type, "username": chat.username,
  5230. "is_forum": getattr(chat, "is_forum", False)}
  5231. except Exception as e:
  5232. logger.error("[%s] Failed to get Telegram chat info for %s: %s", self.name, chat_id, _redact_telegram_error_text(
  5233. e), exc_info=True)
  5234. return {"name": str(chat_id), "type": "dm", "error": str(e)}
  5235. def format_message(self, content: str) -> str:
  5236. """Convert standard markdown to Telegram MarkdownV2: code is stashed behind placeholders first (never
  5237. modified), markdown constructs become MarkdownV2 syntax, everything else is escaped."""
  5238. if not content:
  5239. return content
  5240. placeholders: dict = {}
  5241. counter = [0]
  5242. def _ph(value: str) -> str:
  5243. """Stash *value* behind a placeholder token that survives escaping."""
  5244. key = f"\x00PH{counter[0]}\x00"
  5245. counter[0] += 1
  5246. placeholders[key] = value
  5247. return key
  5248. def _ph_wrap(open_: str, close: str):
  5249. return lambda m: _ph(f"{open_}{_escape_mdv2(m.group(1))}{close}")
  5250. # 0) Rewrite GFM-style pipe tables into Telegram-friendly row groups
  5251. # before the normal MarkdownV2 conversions run.
  5252. text = _wrap_markdown_tables(content)
  5253. # 1) Protect fenced code blocks (``` ... ```)
  5254. # Per MarkdownV2 spec, \ and ` inside pre/code must be escaped.
  5255. # A fence still opens on its own line — the opening ``` must be the
  5256. # first triple-backtick run on that line and must end it — but the
  5257. # line may carry arbitrary leading whitespace (list/blockquote-
  5258. # nested code indents fences by 4+ spaces) or lead-in prose
  5259. # ("Here is the code: ```"), both of which the line-start-only
  5260. # anchor silently downgraded from <pre> to escaped literal text.
  5261. # Requiring the rest of the opening line to be backtick-free is
  5262. # what keeps *inline* triple backticks (e.g. "the syntax is
  5263. # ```x``` inline") out of the match: a closing run can never sit
  5264. # on the same line as the opener, and the tempered prefix cannot
  5265. # skip past an earlier run to a later one. The closing fence must
  5266. # sit on its own line (any indent), with optional trailing
  5267. # whitespace and an optional ``\r`` so CRLF-terminated fences
  5268. # (Windows-authored content) match too.
  5269. def _protect_fenced(m):
  5270. prefix = m.group(1) # lead-in text / indent before the opening fence
  5271. opening = m.group(2) # opening ``` (with optional language) and newline
  5272. body = m.group(3) # code body (may be empty)
  5273. closing = m.group(4) # closing fence (with its indent)
  5274. body = body.replace('\\', '\\\\').replace('`', '\\`')
  5275. return prefix + _ph(opening + body + closing)
  5276. text = re.sub(
  5277. r'(?m)^((?:(?!```)[^\n])*)(```[^`\n]*\n)([\s\S]*?)(^[ \t]*```)[ \t]*\r?$',
  5278. _protect_fenced,
  5279. text,
  5280. )
  5281. # 2) Protect inline code (`...`)
  5282. # Escape \ inside inline code per MarkdownV2 spec.
  5283. text = re.sub(
  5284. r'(`[^`]+`)',
  5285. lambda m: _ph(m.group(0).replace('\\', '\\\\')),
  5286. text,
  5287. )
  5288. # 3) Convert markdown links – escape the display text; inside the URL
  5289. # only ')' and '\' need escaping per the MarkdownV2 spec.
  5290. def _convert_link(m):
  5291. url = m.group(2).replace('\\', '\\\\').replace(')', '\\)')
  5292. return _ph(f'[{_escape_mdv2(m.group(1))}]({url})')
  5293. text = re.sub(r'\[([^\]]+)\]\(([^()]*(?:\([^()]*\)[^()]*)*)\)', _convert_link, text)
  5294. # 4) Headers (## Title) → bold *Title*, stripping redundant ** inside the header
  5295. def _convert_header(m):
  5296. inner = re.sub(r'\*\*(.+?)\*\*', r'\1', m.group(1).strip())
  5297. return _ph(f'*{_escape_mdv2(inner)}*')
  5298. text = re.sub(r'^#{1,6}\s+(.+)$', _convert_header, text, flags=re.MULTILINE)
  5299. # 5) Bold **text** → *text*; 6) Italic *text* → _text_ ([^*\n]+ keeps matches on one line, or *
  5300. # bullet lists corrupt); 7) Strikethrough ~~text~~ → ~text~; 8) Spoiler ||text|| kept as-is.
  5301. text = re.sub(r'\*\*(.+?)\*\*', _ph_wrap('*', '*'), text)
  5302. text = re.sub(r'\*([^*\n]+)\*', _ph_wrap('_', '_'), text)
  5303. text = re.sub(r'~~(.+?)~~', _ph_wrap('~', '~'), text)
  5304. text = re.sub(r'\|\|(.+?)\|\|', _ph_wrap('||', '||'), text)
  5305. # 9) Blockquotes: protect leading > from escaping; expandable quotes (**> starts, trailing || ends).
  5306. def _convert_blockquote(m):
  5307. prefix, content = m.group(1), m.group(2) # prefix: >, >>, >>>, **>, **>> …
  5308. if prefix.startswith('**') and content.endswith('||'):
  5309. return _ph(f'{prefix} {_escape_mdv2(content[:-2])}||')
  5310. return _ph(f'{prefix} {_escape_mdv2(content)}')
  5311. text = re.sub(r'^((?:\*\*)?>{1,3}) (.+)$', _convert_blockquote, text, flags=re.MULTILINE)
  5312. # 10) Escape remaining special characters in plain text
  5313. text = _escape_mdv2(text)
  5314. # 11) Restore placeholders in reverse insertion order so nested placeholders resolve.
  5315. for key in reversed(list(placeholders.keys())):
  5316. text = text.replace(key, placeholders[key])
  5317. # 12) Safety net: escape bare ( ) { } that slipped through, but never inside ``` or ` spans.
  5318. _safe_parts = []
  5319. for _idx, _seg in enumerate(re.split(r'(```[\s\S]*?```|`[^`]+`)', text)):
  5320. if _idx % 2 == 1:
  5321. _safe_parts.append(_seg) # inside code — untouched
  5322. else:
  5323. _safe_parts.append(re.sub(r'[(){}]', lambda m, _seg=_seg: TelegramAdapter._escape_bare_bracket(m, _seg), _seg))
  5324. return ''.join(_safe_parts)
  5325. @staticmethod
  5326. def _escape_bare_bracket(m, seg: str) -> str:
  5327. """Escape a bare ( ) { } unless it is already escaped or delimits a ``[text](url)`` link."""
  5328. s = m.start()
  5329. ch = m.group(0)
  5330. if s > 0 and seg[s - 1] == '\\': # already escaped
  5331. return ch
  5332. if ch == '(' and s > 0 and seg[s - 1] == ']': # opens a link [text](url)
  5333. return ch
  5334. if ch == ')': # closes a link URL? walk back matching depth
  5335. before = seg[:s]
  5336. if '](http' in before or '](' in before:
  5337. depth = 0
  5338. for j in range(s - 1, max(s - 2000, -1), -1):
  5339. if seg[j] == '(':
  5340. depth -= 1
  5341. if depth < 0:
  5342. if j > 0 and seg[j - 1] == ']':
  5343. return ch
  5344. break
  5345. elif seg[j] == ')':
  5346. depth += 1
  5347. return '\\' + ch
  5348. # ── Group mention gating ──────────────────────────────────────────────
  5349. def _extra_bool(self, key: str, env_name: str, default: str, *fallback_keys: str) -> bool:
  5350. """Boolean gate: scoped ``env_name`` → ``config.extra[key]`` (then ``fallback_keys``) → ``default``."""
  5351. configured = _extra_or_secret(self.config.extra, key, env_name, None)
  5352. for alt in fallback_keys:
  5353. if configured is None:
  5354. configured = self.config.extra.get(alt)
  5355. if configured is None:
  5356. configured = default
  5357. if isinstance(configured, bool):
  5358. return configured
  5359. return str(configured).strip().lower() in {"true", "1", "yes", "on"}
  5360. def _extra_str_set(self, key: str, env_name: str) -> set[str]:
  5361. """Comma/list allowlist: scoped ``env_name`` → ``config.extra[key]`` → empty."""
  5362. raw = _extra_or_secret(self.config.extra, key, env_name, "", blank_is_unset=False)
  5363. raw = _decode_json_list_literal(raw)
  5364. if isinstance(raw, list):
  5365. return {str(part).strip() for part in raw if str(part).strip()}
  5366. return {part.strip() for part in str(raw).split(",") if part.strip()}
  5367. def _telegram_require_mention(self) -> bool:
  5368. """Return whether group chats should require an explicit bot trigger."""
  5369. return self._extra_bool("require_mention", "TELEGRAM_REQUIRE_MENTION", "false")
  5370. def _telegram_observe_unmentioned_group_messages(self) -> bool:
  5371. """Store skipped unmentioned group messages as context (observe chatter, dispatch only when addressed)."""
  5372. return self._extra_bool(
  5373. "observe_unmentioned_group_messages", "TELEGRAM_OBSERVE_UNMENTIONED_GROUP_MESSAGES", "false",
  5374. "ingest_unmentioned_group_messages")
  5375. def _telegram_guest_mode(self) -> bool:
  5376. """Return whether non-allowlisted groups may trigger via direct @mention."""
  5377. return self._extra_bool("guest_mode", "TELEGRAM_GUEST_MODE", "false")
  5378. def _telegram_exclusive_bot_mentions(self) -> bool:
  5379. """Return whether explicit @...bot mentions exclusively route group messages."""
  5380. return self._extra_bool("exclusive_bot_mentions", "TELEGRAM_EXCLUSIVE_BOT_MENTIONS", "true")
  5381. def _telegram_bots_require_mention(self) -> bool:
  5382. """Whether another bot's message must explicitly @mention us (a quote-reply alone won't);
  5383. breaks two-bot reply loops in groups while human replies stay unaffected."""
  5384. return self._extra_bool(
  5385. "bots_require_mention", "TELEGRAM_BOTS_REQUIRE_MENTION", "false"
  5386. )
  5387. def _telegram_free_response_chats(self) -> set[str]:
  5388. return self._extra_str_set("free_response_chats", "TELEGRAM_FREE_RESPONSE_CHATS")
  5389. def _telegram_free_response_topics(self) -> set[str]:
  5390. """Topic-level free-response entries as ``<chat_id>:<thread_id>`` (General topic = ``1``)."""
  5391. return self._extra_str_set("free_response_topics", "TELEGRAM_FREE_RESPONSE_TOPICS")
  5392. def _telegram_is_free_response_topic(self, message: Message) -> bool:
  5393. """True when the message's chat/topic pair is in ``free_response_topics``."""
  5394. topics = self._telegram_free_response_topics()
  5395. if not topics:
  5396. return False
  5397. chat_id = self._chat_id_str(message)
  5398. if not chat_id:
  5399. return False
  5400. return f"{chat_id}:{self._topic_id_or_general(self._effective_message_thread_id(message))}" in topics
  5401. def _telegram_allowed_chats(self) -> set[str]:
  5402. """Group chat IDs the bot responds in (non-empty: others need ``guest_mode`` + @mention; DMs never
  5403. filtered; empty = no restriction)."""
  5404. return self._extra_str_set("allowed_chats", "TELEGRAM_ALLOWED_CHATS")
  5405. def _telegram_group_allowed_chats(self) -> set[str]:
  5406. """Return Telegram chats authorized at group scope."""
  5407. return self._extra_str_set("group_allowed_chats", "TELEGRAM_GROUP_ALLOWED_CHATS")
  5408. def _telegram_observe_allowed_chats(self) -> set[str]:
  5409. """Chats where observed group context may use a shared source: ``group_allowed_chats`` ∩
  5410. ``allowed_chats`` (when set)."""
  5411. group_allowed = self._telegram_group_allowed_chats()
  5412. if not group_allowed:
  5413. return set()
  5414. response_allowed = self._telegram_allowed_chats()
  5415. return group_allowed & response_allowed if response_allowed else group_allowed
  5416. def _telegram_allowed_topics(self) -> set[str]:
  5417. """Forum topic IDs this bot handles (non-empty: other topics ignored; DMs never filtered; missing
  5418. ``message_thread_id`` == General topic ``1``)."""
  5419. return self._extra_str_set("allowed_topics", "TELEGRAM_ALLOWED_TOPICS")
  5420. def _telegram_ignored_threads(self) -> set[int]:
  5421. """Thread ids to skip: scoped ``TELEGRAM_IGNORED_THREADS`` → ``config.extra`` → none."""
  5422. raw = _extra_or_secret(self.config.extra, "ignored_threads", "TELEGRAM_IGNORED_THREADS", "", blank_is_unset=False)
  5423. raw = _decode_json_list_literal(raw)
  5424. ignored: set[int] = set()
  5425. for value in (raw if isinstance(raw, list) else str(raw).split(",")):
  5426. text = str(value).strip()
  5427. if not text:
  5428. continue
  5429. try:
  5430. ignored.add(int(text))
  5431. except (TypeError, ValueError):
  5432. logger.warning("[%s] Ignoring invalid Telegram thread id: %r", self.name, value)
  5433. return ignored
  5434. def _compile_mention_patterns(self) -> list[re.Pattern]:
  5435. """Compile optional regex wake-word patterns for group triggers."""
  5436. # Scoped env → the profile's YAML → none. Only the env rung is a serialized string (JSON list,
  5437. # newline- or comma-separated); a YAML string is one literal pattern and is left intact.
  5438. env_raw = _scoped_gate_env("TELEGRAM_MENTION_PATTERNS", "").strip()
  5439. if env_raw:
  5440. try:
  5441. patterns = json.loads(env_raw)
  5442. except Exception:
  5443. patterns = [part.strip() for part in env_raw.splitlines() if part.strip()]
  5444. if not patterns:
  5445. patterns = [part.strip() for part in env_raw.split(",") if part.strip()]
  5446. else:
  5447. patterns = self.config.extra.get("mention_patterns")
  5448. if patterns is None:
  5449. return [] # before touching ``self.name``: tests build bare adapters via object.__new__
  5450. return compile_mention_patterns(patterns, log_prefix=self.name, platform_label="telegram", display_label="Telegram", logger_=logger)
  5451. @staticmethod
  5452. def _chat_type_str(chat) -> str:
  5453. """PTB enum or plain-string ``chat.type`` → bare lowercase name (``supergroup``)."""
  5454. return str(getattr(chat, "type", "")).split(".")[-1].lower() if chat else ""
  5455. @staticmethod
  5456. def _chat_id_str(message) -> str:
  5457. return str(getattr(getattr(message, "chat", None), "id", ""))
  5458. @classmethod
  5459. def _topic_id_or_general(cls, thread_id) -> str:
  5460. return str(thread_id) if thread_id is not None else cls._GENERAL_TOPIC_THREAD_ID
  5461. def _is_group_chat(self, message: Message) -> bool:
  5462. chat = getattr(message, "chat", None)
  5463. return bool(chat) and self._chat_type_str(chat) in {"group", "supergroup"}
  5464. @classmethod
  5465. def _effective_message_thread_id(cls, message: Message) -> Optional[str]:
  5466. """Routable thread id: forum General-topic messages arrive with ``message_thread_id=None`` but
  5467. Telegram addresses that topic as ``1``; plain group/DM replies carry a reply-UI anchor that is NOT
  5468. a routing id. Gating, skill binding and outbound routing must all agree on this value."""
  5469. chat = getattr(message, "chat", None)
  5470. chat_type = cls._chat_type_str(chat)
  5471. raw = getattr(message, "message_thread_id", None)
  5472. is_topic_message = bool(getattr(message, "is_topic_message", False))
  5473. is_group = chat_type in ("group", "supergroup")
  5474. is_forum_group = is_group and getattr(chat, "is_forum", False) is True
  5475. if raw is not None:
  5476. if is_forum_group or (is_group and is_topic_message) or (chat_type == "private" and is_topic_message):
  5477. return str(raw)
  5478. return None
  5479. return cls._GENERAL_TOPIC_THREAD_ID if is_forum_group else None
  5480. # Decides only whether a FOREIGN @handle is bot-shaped; our own handle is matched by identity, never
  5481. # shape (collectible/Fragment bot usernames need not end in "bot").
  5482. _FOREIGN_BOT_HANDLE_RE = re.compile(r"[a-z0-9_]{2,29}bot", re.IGNORECASE)
  5483. _BOT_IDENTITY_TTL_SECONDS = 300.0 # how long an observed identity is trusted before re-check
  5484. def _current_bot_username(self) -> str:
  5485. """This bot's live @username (lowercased, no ``@``): the last observed handle beats PTB's
  5486. ``get_me()`` cache, which keeps a stale handle after a BotFather rename."""
  5487. observed = getattr(self, "_bot_username_observed", None)
  5488. if observed:
  5489. return observed
  5490. return (getattr(self._bot, "username", None) or "").lstrip("@").lower()
  5491. def _note_bot_username(self, username: Optional[str]) -> None:
  5492. """Record the bot's current @username, logging real renames."""
  5493. handle = (username or "").lstrip("@").lower()
  5494. if not handle:
  5495. return
  5496. previous = getattr(self, "_bot_username_observed", None)
  5497. if previous == handle:
  5498. return
  5499. self._bot_username_observed = handle
  5500. self._bot_identity_checked_at = time.monotonic()
  5501. if previous:
  5502. logger.info(
  5503. "[%s] Telegram bot username changed: @%s -> @%s (mention routing now follows the new handle)", self.name, previous, handle)
  5504. def _observe_bot_identity_from_message(self, message: Message) -> None:
  5505. """Learn our own handle from a message Telegram says we authored (own messages and
  5506. ``reply_to_message``); only trusted when the user id matches this bot."""
  5507. bot_id = getattr(self._bot, "id", None)
  5508. if bot_id is None:
  5509. return
  5510. for candidate in (getattr(message, "from_user", None), getattr(getattr(message, "reply_to_message", None), "from_user", None)):
  5511. if candidate is not None and getattr(candidate, "id", None) == bot_id:
  5512. self._note_bot_username(getattr(candidate, "username", None))
  5513. def _bot_identity_is_fresh(self) -> bool:
  5514. """True when identity was re-read within the TTL. ``None`` (never checked) is always stale — do
  5515. not fold it into ``0.0``: monotonic clocks have an arbitrary epoch."""
  5516. checked_at = getattr(self, "_bot_identity_checked_at", None)
  5517. return checked_at is not None and (time.monotonic() - checked_at) < self._BOT_IDENTITY_TTL_SECONDS
  5518. async def _refresh_bot_identity(self, *, force: bool = False) -> None:
  5519. """Re-read bot identity when the cache may be stale (``get_me()`` rewrites PTB's ``Bot._bot_user``
  5520. in place). Best-effort: a failed probe keeps the last known handle."""
  5521. bot = self._bot
  5522. if bot is None or not callable(getattr(bot, "get_me", None)):
  5523. return
  5524. if not force and self._bot_identity_is_fresh():
  5525. return
  5526. try:
  5527. me = await asyncio.wait_for(bot.get_me(), self._BOT_IDENTITY_PROBE_TIMEOUT)
  5528. except asyncio.CancelledError:
  5529. raise
  5530. except Exception as exc:
  5531. logger.debug(
  5532. "[%s] Telegram identity refresh failed (keeping @%s): %s", self.name, self._current_bot_username() or "unknown", exc)
  5533. return
  5534. self._bot_identity_checked_at = time.monotonic()
  5535. self._note_bot_username(getattr(me, "username", None))
  5536. _BOT_IDENTITY_PROBE_TIMEOUT = 15.0
  5537. def _is_reply_to_bot(self, message: Message) -> bool:
  5538. if not self._bot or not getattr(message, "reply_to_message", None):
  5539. return False
  5540. reply_user = getattr(message.reply_to_message, "from_user", None)
  5541. return bool(reply_user and getattr(reply_user, "id", None) == getattr(self._bot, "id", None))
  5542. @staticmethod
  5543. def _entity_sources(message: Message):
  5544. """``(text, entities)`` pairs for the message text and caption."""
  5545. yield getattr(message, "text", None) or "", getattr(message, "entities", None) or []
  5546. yield getattr(message, "caption", None) or "", getattr(message, "caption_entities", None) or []
  5547. @staticmethod
  5548. def _entity_type(entity) -> str:
  5549. return str(getattr(entity, "type", "")).split(".")[-1].lower()
  5550. @classmethod
  5551. def _entity_span(cls, source_text: str, entity) -> Optional[str]:
  5552. """The entity's text, or None when its offsets are unusable."""
  5553. # Telegram's official group-disambiguation form for slash commands (``/cmd@botname``) is emitted as
  5554. # a single ``bot_command`` entity covering the whole span — there is no accompanying ``mention``
  5555. # entity. Treat it as a direct address to this bot when the ``@botname`` suffix matches. This is the
  5556. # form Telegram's own command menu autocomplete produces in groups, so dropping it at the mention
  5557. # gate would break /new, /reset, /help, ... for every group that has ``require_mention`` enabled
  5558. # (#15415).
  5559. offset = int(getattr(entity, "offset", -1))
  5560. length = int(getattr(entity, "length", 0))
  5561. if offset < 0 or length <= 0:
  5562. return None
  5563. return cls._telegram_entity_text(source_text, offset, length)
  5564. @classmethod
  5565. def _extract_bot_mention_usernames(cls, message: Message, self_username: str = "") -> set[str]:
  5566. """Explicit bot usernames mentioned in text/captions: foreign handles count only when bot-shaped
  5567. (``...bot``), ``self_username`` opts our OWN handle in regardless of shape. Entity mentions are
  5568. authoritative; the raw-text fallback is deliberately narrow."""
  5569. mentioned_bot_usernames: set[str] = set()
  5570. own = (self_username or "").lstrip("@").lower()
  5571. def _is_bot_handle(handle: str) -> bool:
  5572. if not handle:
  5573. return False
  5574. if own and handle == own:
  5575. return True
  5576. return bool(cls._FOREIGN_BOT_HANDLE_RE.fullmatch(handle))
  5577. for source_text, entities in cls._entity_sources(message):
  5578. for entity in entities:
  5579. entity_type = cls._entity_type(entity)
  5580. if entity_type not in {"mention", "bot_command"}:
  5581. continue
  5582. entity_text = cls._entity_span(source_text, entity)
  5583. if entity_text is None:
  5584. continue
  5585. entity_text = entity_text.strip()
  5586. if entity_type == "mention":
  5587. handle = entity_text.lstrip("@").lower()
  5588. if _is_bot_handle(handle):
  5589. mentioned_bot_usernames.add(handle)
  5590. continue
  5591. # /cmd@botname is one bot_command entity; its suffix is an explicit bot address.
  5592. at_index = entity_text.find("@")
  5593. if at_index < 0:
  5594. continue
  5595. command_target = entity_text[at_index + 1:].strip().lower()
  5596. if _is_bot_handle(command_target):
  5597. mentioned_bot_usernames.add(command_target)
  5598. # Entity-less fallback only: if Telegram supplied entities, trust them (no URL/code rescue).
  5599. for raw_text, entities in cls._entity_sources(message):
  5600. if not raw_text or entities:
  5601. continue
  5602. for match in re.finditer(r"(?i)(?<![A-Za-z0-9_`/])@([A-Za-z0-9_]{2,31})\b", raw_text):
  5603. handle = match.group(1).lower()
  5604. if _is_bot_handle(handle):
  5605. mentioned_bot_usernames.add(handle)
  5606. return mentioned_bot_usernames
  5607. @staticmethod
  5608. def _telegram_entity_text(source_text: str, offset: int, length: int) -> str:
  5609. """Return a Telegram entity span using UTF-16 code-unit offsets."""
  5610. if offset < 0 or length <= 0:
  5611. return ""
  5612. try:
  5613. return source_text.encode("utf-16-le")[offset * 2:(offset + length) * 2].decode("utf-16-le")
  5614. except UnicodeDecodeError:
  5615. return ""
  5616. def _message_mentions_bot(self, message: Message) -> bool:
  5617. if not self._bot:
  5618. return False
  5619. bot_username = self._current_bot_username()
  5620. bot_id = getattr(self._bot, "id", None)
  5621. expected = f"@{bot_username}" if bot_username else None
  5622. # Server-side MessageEntity values are authoritative: raw substrings like "foo@hermes_bot.example"
  5623. # or handles inside URLs/code are not mentions.
  5624. for source_text, entities in self._entity_sources(message):
  5625. for entity in entities:
  5626. entity_type = self._entity_type(entity)
  5627. if entity_type == "mention" and expected:
  5628. span = self._entity_span(source_text, entity)
  5629. if span is not None and span.strip().lower() == expected:
  5630. return True
  5631. elif entity_type == "text_mention":
  5632. user = getattr(entity, "user", None)
  5633. if user and getattr(user, "id", None) == bot_id:
  5634. return True
  5635. elif entity_type == "bot_command" and expected:
  5636. # ``/cmd@botname`` (what the group command menu produces) must count as a direct address.
  5637. command_text = self._entity_span(source_text, entity)
  5638. if command_text is None:
  5639. continue
  5640. at_index = command_text.find("@")
  5641. if at_index >= 0 and command_text[at_index:].strip().lower() == expected:
  5642. return True
  5643. if bot_username:
  5644. return bot_username in self._extract_bot_mention_usernames(message, bot_username)
  5645. return False
  5646. def _schedule_bot_identity_recheck(self) -> None:
  5647. """Fire a TTL-guarded identity refresh in the background when routing is about to discard a
  5648. message naming other bots but not us (the symptom of a stale handle after a rename)."""
  5649. existing = getattr(self, "_bot_identity_refresh_task", None)
  5650. if (existing is not None and not existing.done()) or self._bot_identity_is_fresh():
  5651. return
  5652. try:
  5653. loop = asyncio.get_running_loop()
  5654. except RuntimeError:
  5655. return
  5656. task = loop.create_task(self._refresh_bot_identity())
  5657. self._bot_identity_refresh_task = task
  5658. tracked = getattr(self, "_background_tasks", None)
  5659. if isinstance(tracked, set):
  5660. tracked.add(task)
  5661. task.add_done_callback(tracked.discard)
  5662. def _explicit_bot_mentions_exclude_self(self, message: Message) -> bool:
  5663. """True when explicit bot handles target other bots, not this one (``@bot3 hi @bot4`` must not
  5664. wake ``@bot1`` via reply/wake-word fallbacks)."""
  5665. if not self._bot:
  5666. return False
  5667. bot_username = self._current_bot_username()
  5668. if not bot_username:
  5669. return False
  5670. mentioned_bot_usernames = self._extract_bot_mention_usernames(message, bot_username)
  5671. excludes_self = bool(mentioned_bot_usernames) and bot_username not in mentioned_bot_usernames
  5672. if excludes_self:
  5673. # Either truly for another bot, or our handle is stale after a rename — re-check out of band.
  5674. self._schedule_bot_identity_recheck()
  5675. return excludes_self
  5676. def _message_matches_mention_patterns(self, message: Message) -> bool:
  5677. if not self._mention_patterns:
  5678. return False
  5679. return any(
  5680. pattern.search(candidate)
  5681. for candidate in (getattr(message, "text", None), getattr(message, "caption", None)) if candidate
  5682. for pattern in self._mention_patterns)
  5683. def _is_guest_mention(self, message: Message) -> bool:
  5684. """Guest-mode bypass: explicit bot mention (caller already verified group chat)."""
  5685. return self._telegram_guest_mode() and self._message_mentions_bot(message)
  5686. def _clean_bot_trigger_text(self, text: Optional[str]) -> Optional[str]:
  5687. bot_username = self._current_bot_username()
  5688. if not text or not bot_username:
  5689. return text
  5690. cleaned = re.sub(rf"(?i)@{re.escape(bot_username)}\b[,:\-]*\s*", "", text).strip()
  5691. return cleaned or text
  5692. def _topic_gates_pass(self, thread_id, *, warn_non_numeric: bool) -> Optional[bool]:
  5693. """``allowed_topics`` / ``ignored_threads`` gates; False = blocked, None = undecided."""
  5694. allowed_topics = self._telegram_allowed_topics()
  5695. if allowed_topics and self._topic_id_or_general(thread_id) not in allowed_topics:
  5696. return False
  5697. if thread_id is not None:
  5698. try:
  5699. if int(thread_id) in self._telegram_ignored_threads():
  5700. return False
  5701. except (TypeError, ValueError):
  5702. if not warn_non_numeric:
  5703. return False
  5704. logger.warning("[%s] Ignoring non-numeric Telegram message_thread_id: %r", self.name, thread_id)
  5705. return None
  5706. def _bot_sender_suppressed(self, message: Message) -> bool:
  5707. """True when ``bots_require_mention`` vetoes this other-bot message: another bot must
  5708. explicitly @mention us, its quote-replies and plain chatter do not count (two bots
  5709. answering each other's replies never stop otherwise)."""
  5710. return bool(
  5711. self._telegram_bots_require_mention()
  5712. and self._sender_is_other_bot(message)
  5713. and not self._message_mentions_bot(message)
  5714. )
  5715. def _should_observe_unmentioned_group_message(self, message: Message) -> bool:
  5716. """Return True when a group message should be stored but not dispatched."""
  5717. if self._is_own_message(message) or not self._telegram_observe_unmentioned_group_messages() or not self._is_group_chat(message):
  5718. return False
  5719. if self._topic_gates_pass(getattr(message, "message_thread_id", None), warn_non_numeric=False) is False:
  5720. return False
  5721. chat_id_str = self._chat_id_str(message)
  5722. if self._telegram_exclusive_bot_mentions() and self._explicit_bot_mentions_exclude_self(message):
  5723. return False
  5724. # Observed context is shared at chat/topic scope, so require an explicit chat allowlist.
  5725. allowed = self._telegram_observe_allowed_chats()
  5726. if not allowed or chat_id_str not in allowed:
  5727. return False
  5728. # Free-response chats/topics dispatch every message, so they are never observed.
  5729. if chat_id_str in self._telegram_free_response_chats() or self._telegram_is_free_response_topic(message):
  5730. return False
  5731. # Only observe messages the require_mention gate would skip. The bot-to-bot loop breaker
  5732. # in ``_should_process_message`` skips another bot's message too, so a sibling bot
  5733. # addressing us by wake word (or quote-reply) is never dispatched and must still be
  5734. # kept as observed context (#115119).
  5735. if self._bot_sender_suppressed(message):
  5736. return True
  5737. if not self._telegram_require_mention() or self._is_reply_to_bot(message) or self._message_mentions_bot(message):
  5738. return False
  5739. return not self._message_matches_mention_patterns(message)
  5740. def _telegram_group_observe_shared_source(self, source):
  5741. """Return a chat/topic-scoped source for observed Telegram group context."""
  5742. return dataclasses.replace(source, user_id=None, user_name=None, user_id_alt=None)
  5743. def _telegram_group_observe_attributed_text(self, event: MessageEvent) -> str:
  5744. user_id = event.source.user_id or "unknown"
  5745. return f"[{event.source.user_name or user_id}|{user_id}]\n{event.text or ''}"
  5746. def _telegram_group_observe_channel_prompt(self) -> str:
  5747. username = self._current_bot_username() or "unknown"
  5748. bot_id = getattr(getattr(self, "_bot", None), "id", None) or "unknown"
  5749. return (
  5750. "You are handling a Telegram group chat message.\n"
  5751. f"- Your identity: user_id={bot_id}, @-mention name in this group=@{username}\n"
  5752. "- observed Telegram group context may be provided in a separate context-only block "
  5753. "before the current message; it is not necessarily addressed to you.\n"
  5754. "- Treat only the current new message as a request explicitly directed at you, "
  5755. "and use observed context only when the current message asks for it.")
  5756. def _apply_telegram_group_observe_attribution(self, event: MessageEvent) -> MessageEvent:
  5757. """Align triggered group turns with observed-history attribution."""
  5758. if not self._telegram_observe_unmentioned_group_messages():
  5759. return event
  5760. raw_message = getattr(event, "raw_message", None)
  5761. if not raw_message or not self._is_group_chat(raw_message):
  5762. return event
  5763. allowed = self._telegram_observe_allowed_chats()
  5764. if not allowed or self._chat_id_str(raw_message) not in allowed:
  5765. return event
  5766. observe_prompt = self._telegram_group_observe_channel_prompt()
  5767. channel_prompt = f"{event.channel_prompt}\n\n{observe_prompt}" if event.channel_prompt else observe_prompt
  5768. if event.message_type == MessageType.COMMAND:
  5769. # Commands keep the original source (user_id) so _check_slash_access can identify the sender.
  5770. return dataclasses.replace(event, channel_prompt=channel_prompt)
  5771. return dataclasses.replace(
  5772. event, text=self._telegram_group_observe_attributed_text(event),
  5773. source=self._telegram_group_observe_shared_source(event.source), channel_prompt=channel_prompt)
  5774. def _media_message_type(self, msg: Message) -> MessageType:
  5775. """Classify a Telegram media message into a MessageType (first present attachment wins)."""
  5776. for attr, mtype in (
  5777. ("sticker", MessageType.STICKER), ("photo", MessageType.PHOTO), ("video", MessageType.VIDEO),
  5778. ("video_note", MessageType.VIDEO), ("audio", MessageType.AUDIO), ("voice", MessageType.VOICE)):
  5779. if getattr(msg, attr, None):
  5780. return mtype
  5781. return MessageType.DOCUMENT
  5782. _CACHED_KIND_TO_MESSAGE_TYPE = {"image": MessageType.PHOTO, "video": MessageType.VIDEO, "audio": MessageType.AUDIO}
  5783. async def _download_observed_media(self, msg: Any, what: str):
  5784. """Download ``msg``'s attachment into the media cache (bounded by ``_max_doc_bytes``). Returns ``(status, cached)``:
  5785. ``"none"``, ``"oversized"`` (cached = raw file_size), ``"failed"``, ``"unreadable"`` or ``"ok"``."""
  5786. from gateway.platforms.base import cache_media_bytes_async
  5787. source, filename, mime, kind = self._observed_media_source(msg)
  5788. if source is None:
  5789. return "none", None
  5790. file_size = getattr(source, "file_size", None)
  5791. if not (0 < self._int_or_zero(file_size) <= getattr(self, "_max_doc_bytes", 20 * 1024 * 1024)):
  5792. return "oversized", file_size
  5793. try:
  5794. file_obj = await source.get_file()
  5795. data = bytes(await file_obj.download_as_bytearray())
  5796. if not filename:
  5797. filename = os.path.basename(getattr(file_obj, "file_path", "") or "")
  5798. cached = await cache_media_bytes_async(data, filename=filename, mime_type=mime, default_kind=kind)
  5799. except Exception as exc:
  5800. logger.warning("[Telegram] Failed to cache %s: %s", what, _redact_telegram_error_text(exc), exc_info=True)
  5801. return "failed", None
  5802. if cached is None:
  5803. return "unreadable", None
  5804. return "ok", cached
  5805. async def _cache_observed_media(self, msg: Message, event: MessageEvent) -> None:
  5806. """Cache an unmentioned group attachment and annotate the observed text; oversized or unsupported
  5807. attachments are noted in the transcript without downloading."""
  5808. status, cached = await self._download_observed_media(msg, "observed group media")
  5809. if status == "oversized":
  5810. limit_mb = getattr(self, "_max_doc_bytes", 20 * 1024 * 1024) // (1024 * 1024)
  5811. event.text = self._append_observed_note(
  5812. event.text, f"[Observed Telegram attachment too large or unverifiable. Maximum: {limit_mb} MB.]")
  5813. logger.info("[Telegram] Observed group attachment skipped (size=%s)", cached)
  5814. return
  5815. if status == "unreadable": # only images that fail validation reach here
  5816. event.text = self._append_observed_note(event.text, "[Observed Telegram attachment could not be read, not cached.]")
  5817. return
  5818. if status == "ok":
  5819. event.media_urls = []
  5820. event.media_types = []
  5821. self._attach_cached(event, cached, cached.context_note(), "[Telegram] Cached observed group %s at %s")
  5822. def _attach_cached(self, event: MessageEvent, cached, note: str, log_fmt: str) -> None:
  5823. """Append a cached attachment to the event (message type follows the kind only for the first one)."""
  5824. event.media_urls.append(cached.path)
  5825. event.media_types.append(cached.media_type)
  5826. if len(event.media_urls) == 1 and cached.kind in self._CACHED_KIND_TO_MESSAGE_TYPE:
  5827. event.message_type = self._CACHED_KIND_TO_MESSAGE_TYPE[cached.kind]
  5828. event.text = self._append_observed_note(event.text, note)
  5829. logger.info(log_fmt, cached.kind, cached.path)
  5830. async def _cache_replied_media(self, msg: Any, event: MessageEvent) -> None:
  5831. """Cache media from the message this turn replies to, if any."""
  5832. reply_msg = getattr(msg, "reply_to_message", None)
  5833. if reply_msg is None:
  5834. return
  5835. status, cached = await self._download_observed_media(reply_msg, "replied-to media")
  5836. if status == "ok":
  5837. self._attach_cached(
  5838. event, cached, f"[Replied-to {cached.kind} '{cached.display_name}' saved at: {cached.path}]",
  5839. "[Telegram] Cached replied-to %s at %s")
  5840. def _observed_media_source(self, msg: Message):
  5841. """Return (telegram_file_source, filename, mime, default_kind) or Nones."""
  5842. if msg.photo:
  5843. return msg.photo[-1], "", "", "image"
  5844. if video := msg.video or getattr(msg, "video_note", None):
  5845. return video, "", "video/mp4", "video"
  5846. if msg.voice:
  5847. return msg.voice, "voice.ogg", "audio/ogg", "audio"
  5848. if msg.audio:
  5849. return msg.audio, getattr(msg.audio, "file_name", "") or "", "", "audio"
  5850. if msg.document:
  5851. doc = msg.document
  5852. return doc, doc.file_name or "", (doc.mime_type or "").lower(), None
  5853. return None, "", "", None
  5854. @staticmethod
  5855. def _append_observed_note(existing: Optional[str], note: str) -> str:
  5856. if not note:
  5857. return existing or ""
  5858. return f"{existing}\n\n{note}" if existing else note
  5859. async def _surface_media_cache_failure(
  5860. self, msg: Message, event: MessageEvent, kind: str, exc: Exception, display_name: Optional[str] = None) -> None:
  5861. """Surface a failed media download to BOTH the user (reply asking to retry) and the agent (observed
  5862. note) — otherwise the turn dispatches silently with empty media_urls.
  5863. This (1) replies to the user in Telegram so they know to retry, and (2) appends an agent-visible
  5864. notice to event.text via the existing observed-note channel so the agent knows an attachment was
  5865. attempted and failed — never a silent empty turn. No new event fields (the structured-event refactor
  5866. is out of scope per #23045).
  5867. """
  5868. # Inbound media fails before handle_message binds the routed profile.
  5869. with self._media_delivery_scope(event.source):
  5870. named = t("platform.telegram.media.named_suffix", name=display_name) if display_name else ""
  5871. # ``kind`` is the English label the agent-visible note below keeps; the chat reply is localized.
  5872. notice = self.warning_text(t(
  5873. "platform.telegram.media.download_failed",
  5874. kind=t(_MEDIA_KIND_KEYS[kind]) if kind in _MEDIA_KIND_KEYS else kind,
  5875. name=named, error=exc.__class__.__name__))
  5876. if notice:
  5877. try:
  5878. self._accept_update()
  5879. await msg.reply_text(notice)
  5880. except Exception as reply_err:
  5881. logger.warning("[Telegram] Failed to notify user about %s cache failure: %s", kind, reply_err, exc_info=True)
  5882. # The agent-visible note is execution evidence, not a channel diagnostic; it stays in both modes.
  5883. event.text = self._append_observed_note(
  5884. event.text,
  5885. f"[The user attempted to send a {kind}{named} but it could not be downloaded ({exc.__class__.__name__}); they have been asked to retry.]"
  5886. if notice else f"[The user attempted to send a {kind}{named} but it could not be downloaded.]",
  5887. )
  5888. def _observe_unmentioned_group_message(
  5889. self, message: Message, msg_type: MessageType, update_id: Optional[int] = None, event: Optional[MessageEvent] = None) -> None:
  5890. """Append skipped group chatter to the target session without dispatching."""
  5891. store = getattr(self, "_session_store", None)
  5892. if not store:
  5893. return
  5894. adapter_name = getattr(self, "name", "telegram")
  5895. try:
  5896. event = event or self._build_message_event(message, msg_type, update_id=update_id)
  5897. session_entry = store.get_or_create_session(self._telegram_group_observe_shared_source(event.source))
  5898. entry = {
  5899. "role": "user", "content": self._telegram_group_observe_attributed_text(event),
  5900. "timestamp": datetime.now(tz=UTC).isoformat(), "observed": True}
  5901. if event.message_id:
  5902. entry["message_id"] = str(event.message_id)
  5903. self._accept_update()
  5904. store.append_to_transcript(session_entry.session_id, entry)
  5905. logger.info(
  5906. "[%s] Telegram group message observed (no bot trigger): chat=%s from=%s", adapter_name,
  5907. getattr(getattr(message, "chat", None), "id", "unknown"), event.source.user_id or "unknown")
  5908. except Exception as exc:
  5909. self._fail_update_preparation()
  5910. logger.warning("[%s] Failed to observe Telegram group message: %s", adapter_name, exc)
  5911. def _is_own_message(self, message: Message) -> bool:
  5912. """True when sent by this bot itself (echoed getUpdates must not count as incoming unread)."""
  5913. if not self._bot:
  5914. return False
  5915. from_user = getattr(message, "from_user", None)
  5916. if from_user is None:
  5917. return False
  5918. bot_id = getattr(self._bot, "id", None)
  5919. user_id = getattr(from_user, "id", None)
  5920. return bot_id is not None and user_id is not None and bot_id == user_id
  5921. def _sender_is_other_bot(self, message: Message) -> bool:
  5922. """True when the sender is a bot other than this one (this bot's own echoes are already
  5923. filtered by ``_is_own_message``)."""
  5924. sender = getattr(message, "from_user", None)
  5925. if sender is None or not getattr(sender, "is_bot", False):
  5926. return False
  5927. bot_id = getattr(self._bot, "id", None)
  5928. sender_id = getattr(sender, "id", None)
  5929. return bot_id is None or sender_id is None or sender_id != bot_id
  5930. def _should_process_message(self, message: Message, *, is_command: bool = False) -> bool:
  5931. """Apply Telegram group trigger rules: DMs unrestricted; group messages pass ``allowed_chats`` (hard gate; only
  5932. the ``guest_mode`` @mention bypass crosses it) and then any of free_response chat/topic, ``require_mention``
  5933. off, reply to the bot, @mention (incl. ``/cmd@botname``), or a wake-word match."""
  5934. # Learn the live handle BEFORE any mention gate routes on it, then drop our own echoed messages.
  5935. # Filter out the bot's own messages (returned by getUpdates in some environments like
  5936. # groups/supergroups where the bot can see its own messages). Without this, outbound messages are
  5937. # counted as incoming unread in the Hermes inbox (#52363). Otherwise a BotFather rename leaves the
  5938. # stale handle in place and the exclusive-mention gate reads a message addressed to us as one
  5939. # addressed to some other bot.
  5940. self._observe_bot_identity_from_message(message)
  5941. if self._is_own_message(message):
  5942. return False
  5943. if not self._is_group_chat(message):
  5944. return True
  5945. thread_id = self._effective_message_thread_id(message)
  5946. if self._topic_gates_pass(thread_id, warn_non_numeric=True) is False:
  5947. return False
  5948. chat_id_str = self._chat_id_str(message)
  5949. if self._telegram_exclusive_bot_mentions() and self._explicit_bot_mentions_exclude_self(message):
  5950. return False
  5951. # Resolve once; _message_mentions_bot is not re-called below in guest mode.
  5952. guest_mention = self._is_guest_mention(message)
  5953. # allowed_chats whitelist: outside chats pass only via the guest-mode explicit mention.
  5954. allowed = self._telegram_allowed_chats()
  5955. if allowed and chat_id_str not in allowed:
  5956. return guest_mention
  5957. if guest_mention or chat_id_str in self._telegram_free_response_chats() or self._telegram_is_free_response_topic(message):
  5958. return True
  5959. # Bot-to-bot loop breaker: another bot must explicitly @mention us; its quote-reply or
  5960. # plain chatter does not count (two bots answering each other's replies never stop otherwise).
  5961. if self._bot_sender_suppressed(message):
  5962. return False
  5963. if not self._telegram_require_mention() or self._is_reply_to_bot(message):
  5964. return True
  5965. if not self._telegram_guest_mode() and self._message_mentions_bot(message):
  5966. return True
  5967. return self._message_matches_mention_patterns(message)
  5968. async def _ensure_forum_commands(self, message) -> None:
  5969. """Lazy-register bot commands for forum supergroups (topics don't inherit AllGroupChats scope;
  5970. Telegram resolves via BotCommandScopeChat)."""
  5971. async with self._forum_lock:
  5972. try:
  5973. chat = getattr(message, "chat", None)
  5974. if not chat or not getattr(chat, "is_forum", False):
  5975. return
  5976. chat_id = int(chat.id)
  5977. if chat_id in self._forum_command_registered:
  5978. return
  5979. from telegram import BotCommandScopeChat
  5980. from hermes_cli.commands_platforms import telegram_menu_commands, telegram_menu_max_commands
  5981. menu_commands, _ = await asyncio.to_thread(
  5982. telegram_menu_commands, max_commands=telegram_menu_max_commands())
  5983. bot_commands = self._bot_commands(menu_commands)
  5984. await self._bot.set_my_commands(bot_commands, scope=BotCommandScopeChat(chat_id=chat_id))
  5985. self._forum_command_registered.add(chat_id)
  5986. logger.info("[%s] Lazy-registered %d commands for forum chat %s", self.name, len(bot_commands), chat_id)
  5987. except Exception as e:
  5988. logger.warning("[%s] Forum command lazy-registration failed: %s", self.name, _redact_telegram_error_text(e))
  5989. def _effective_update_message(self, update: Update) -> Optional[Message]:
  5990. """Message-like payload for normal messages and channel posts (``update.channel_post``)."""
  5991. return getattr(update, "effective_message", None) or getattr(update, "message", None)
  5992. def _log_blocked_user(self, msg, *, level=logging.WARNING, what: str = "unauthorized user") -> None:
  5993. logger.log(
  5994. level, "[Telegram] Blocked %s %s in chat %s", what, getattr(getattr(msg, "from_user", None), "id", None),
  5995. getattr(getattr(msg, "chat", None), "id", None))
  5996. def _gate_or_observe(self, msg, update, msg_type: MessageType) -> bool:
  5997. """Group trigger gate; observes unmentioned chatter when configured. True = proceed."""
  5998. if self._should_process_message(msg):
  5999. return True
  6000. if self._should_observe_unmentioned_group_message(msg):
  6001. self._observe_unmentioned_group_message(msg, msg_type, update_id=update.update_id)
  6002. return False
  6003. async def _build_triggered_event(self, msg, update, msg_type: MessageType) -> MessageEvent:
  6004. """Event for an addressed text/command: trigger text cleaned (sole addressee only), replied-to
  6005. media cached, attribution applied."""
  6006. from plugins.platforms.telegram.telegram_context import group_trigger_text
  6007. event = self._build_message_event(msg, msg_type, update_id=update.update_id)
  6008. event.text = group_trigger_text(self, msg, event.text)
  6009. await self._cache_replied_media(msg, event)
  6010. return self._apply_telegram_group_observe_attribution(event)
  6011. async def _handle_text_message(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
  6012. """Handle incoming text; buffers client-split chunks into one MessageEvent."""
  6013. msg = self._effective_update_message(update)
  6014. if not msg or not msg.text:
  6015. return
  6016. # Auth check first: blocked users must not reach batching, the observed transcript, or the agent.
  6017. if not self._is_user_authorized_from_message(msg):
  6018. self._log_blocked_user(msg)
  6019. return
  6020. if not self._gate_or_observe(msg, update, MessageType.TEXT):
  6021. return
  6022. await self._ensure_forum_commands(update.message)
  6023. self._enqueue_text_event(await self._build_triggered_event(msg, update, MessageType.TEXT))
  6024. async def _handle_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
  6025. """Handle incoming command messages."""
  6026. msg = self._effective_update_message(update)
  6027. if not msg or not msg.text:
  6028. return
  6029. if not self._should_process_message(msg, is_command=True):
  6030. return
  6031. if not self._is_user_authorized_from_message(msg):
  6032. self._log_blocked_user(msg)
  6033. return
  6034. await self._ensure_forum_commands(msg)
  6035. event = await self._build_triggered_event(msg, update, MessageType.COMMAND)
  6036. # A >4096-char command paste arrives as a near-limit COMMAND chunk plus TEXT continuations; dispatching
  6037. # immediately would orphan them. Near-limit commands go through text batching.
  6038. if len(event.text or "") >= self._SPLIT_THRESHOLD:
  6039. self._enqueue_text_event(event)
  6040. return
  6041. await self.handle_message(event)
  6042. async def _handle_location_message(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
  6043. """Handle incoming location/venue pin messages."""
  6044. msg = self._effective_update_message(update)
  6045. if not msg:
  6046. return
  6047. if not self._is_user_authorized_from_message(msg):
  6048. self._log_blocked_user(msg)
  6049. return
  6050. if not self._gate_or_observe(msg, update, MessageType.LOCATION):
  6051. return
  6052. venue = getattr(msg, "venue", None)
  6053. location = getattr(venue, "location", None) if venue else getattr(msg, "location", None)
  6054. if not location:
  6055. return
  6056. lat = getattr(location, "latitude", None)
  6057. lon = getattr(location, "longitude", None)
  6058. if lat is None or lon is None:
  6059. return
  6060. parts = ["[The user shared a location pin.]"]
  6061. if venue:
  6062. title = getattr(venue, "title", None)
  6063. address = getattr(venue, "address", None)
  6064. if title:
  6065. parts.append(f"Venue: {title}")
  6066. if address:
  6067. parts.append(f"Address: {address}")
  6068. parts += [
  6069. f"latitude: {lat}", f"longitude: {lon}", f"Map: https://www.google.com/maps/search/?api=1&query={lat},{lon}",
  6070. "Ask what they'd like to find nearby (restaurants, cafes, etc.) and any preferences."]
  6071. event = self._build_message_event(msg, MessageType.LOCATION, update_id=update.update_id)
  6072. event.text = "\n".join(parts)
  6073. await self.handle_message(self._apply_telegram_group_observe_attribution(event))
  6074. # -- Text message aggregation (handles Telegram client-side splits) --
  6075. def _text_batch_key(self, event: MessageEvent) -> str:
  6076. """Session-scoped batching key; topic recovery first so DM-topic batches coalesce on the recovered lane."""
  6077. self._apply_topic_recovery(event)
  6078. return super()._text_batch_key(event)
  6079. def _enqueue_text_event(self, event: MessageEvent) -> None:
  6080. """Buffer a text chunk, or hold it while delayed delivery must be dropped."""
  6081. if self._should_drop_delayed_delivery():
  6082. self._hold_inbound_event(event, where="text-enqueue")
  6083. return
  6084. super()._enqueue_text_event(event)
  6085. self._accept_update()
  6086. async def _flush_buffered(self, pending: dict, tasks: dict, key: str, delay: float, where: str, log_fn=None) -> None:
  6087. """Shared delayed-flush body: sleep, pop, hold if teardown started, else dispatch. A cancel after
  6088. the pop but before durable dispatch re-holds the event (never lose it)."""
  6089. current_task = asyncio.current_task()
  6090. committed = self.__dict__.setdefault("_committed_flush_tasks", {})
  6091. event = None
  6092. try:
  6093. await asyncio.sleep(delay)
  6094. # Superseded flush (a newer chunk re-armed the timer while our sleep was already done):
  6095. # CancelledError only lands at the next await, so check synchronously before the pop.
  6096. owner = tasks.get(key)
  6097. if owner is not None and owner is not current_task:
  6098. return
  6099. event = pending.pop(key, None)
  6100. if not event:
  6101. return
  6102. # Committed: leave the per-key slot so a newer chunk re-arms without cancelling this
  6103. # dispatch mid-flight (e.g. during a root-DM createForumTopic, whose ID would be lost).
  6104. tasks.pop(key, None)
  6105. prior = committed.get((where, key))
  6106. committed[(where, key)] = current_task
  6107. if prior is not None and not prior.done():
  6108. await asyncio.wait({prior}) # dispatch in order, after the earlier batch settles
  6109. if self._should_drop_delayed_delivery():
  6110. self._hold_inbound_event(event, where=f"{where}-flush")
  6111. event = None
  6112. return
  6113. if log_fn is not None:
  6114. log_fn(event)
  6115. await self.handle_message(event)
  6116. event = None
  6117. except asyncio.CancelledError:
  6118. if event is not None:
  6119. self._hold_inbound_event(event, where=f"{where}-flush-cancelled")
  6120. raise
  6121. finally:
  6122. if committed.get((where, key)) is current_task:
  6123. committed.pop((where, key), None)
  6124. if tasks.get(key) is current_task:
  6125. tasks.pop(key, None)
  6126. def _text_batch_delay_for(self, pending: Optional[MessageEvent]) -> float:
  6127. """Adaptive delay: near-split-point last chunk → long delay (continuation almost certain);
  6128. short/medium totals → capped fast delays; else configured cap (all min()'d with the operator cap)."""
  6129. last_len = getattr(pending, "_last_chunk_len", 0) if pending else 0
  6130. total_len = len(getattr(pending, "text", "") or "") if pending else 0
  6131. if last_len >= self._SPLIT_THRESHOLD:
  6132. return self._text_batch_split_delay_seconds
  6133. if total_len <= self._TEXT_BATCH_FAST_LEN:
  6134. return min(self._text_batch_delay_seconds, self._TEXT_BATCH_FAST_DELAY_S)
  6135. if total_len <= self._TEXT_BATCH_SHORT_LEN:
  6136. return min(self._text_batch_delay_seconds, self._TEXT_BATCH_SHORT_DELAY_S)
  6137. return self._text_batch_delay_seconds
  6138. async def _flush_text_batch(self, key: str) -> None:
  6139. """Telegram keeps its own flush body: a cancel after the pop must HOLD the event and re-raise
  6140. (PTB already acked the update; the hold queue redispatches after reconnect) rather than shield
  6141. the dispatch — teardown must be able to stop a flush from reaching a torn-down session."""
  6142. await self._flush_buffered(
  6143. self._pending_text_batches, self._pending_text_batch_tasks, key,
  6144. self._text_batch_delay_for(self._pending_text_batches.get(key)), "text",
  6145. lambda ev: logger.info("[Telegram] Flushing text batch %s (%d chars)", key, len(ev.text or "")))
  6146. # -- Photo batching --
  6147. def _photo_batch_key(self, event: MessageEvent, msg: Message) -> str:
  6148. """Return a batching key for Telegram photos/albums."""
  6149. session_key = self._event_session_key(event)
  6150. media_group_id = getattr(msg, "media_group_id", None)
  6151. return f"{session_key}:album:{media_group_id}" if media_group_id else f"{session_key}:photo-burst"
  6152. async def _flush_photo_batch(self, batch_key: str) -> None:
  6153. """Send a buffered photo burst/album as a single MessageEvent."""
  6154. await self._flush_buffered(
  6155. self._pending_photo_batches, self._pending_photo_batch_tasks, batch_key, self._media_batch_delay_seconds, "photo",
  6156. lambda ev: logger.info("[Telegram] Flushing photo batch %s with %d image(s)", batch_key, len(ev.media_urls)))
  6157. def _merge_into_pending(self, pending: dict, key: str, event: MessageEvent) -> None:
  6158. """Merge ``event`` into ``pending[key]`` (media, caption, reply expectation) or seed it."""
  6159. if (existing := pending.get(key)) is None:
  6160. pending[key] = event
  6161. return
  6162. existing.absorb_reply_expected(event)
  6163. existing.media_urls.extend(event.media_urls)
  6164. existing.media_types.extend(event.media_types)
  6165. if event.text:
  6166. existing.text = self._merge_caption(existing.text, event.text)
  6167. def _enqueue_photo_event(self, batch_key: str, event: MessageEvent) -> None:
  6168. """Merge photo events into a pending batch and schedule flush."""
  6169. if self._should_drop_delayed_delivery():
  6170. self._hold_inbound_event(event, where="photo-enqueue")
  6171. return
  6172. self._merge_into_pending(self._pending_photo_batches, batch_key, event)
  6173. self._accept_update()
  6174. prior_task = self._pending_photo_batch_tasks.get(batch_key)
  6175. if prior_task and not prior_task.done():
  6176. prior_task.cancel()
  6177. self._pending_photo_batch_tasks[batch_key] = asyncio.create_task(self._flush_photo_batch(batch_key))
  6178. async def _route_photo_event(self, msg, event: MessageEvent) -> None:
  6179. """Album items debounce on media_group_id; singles go through the photo burst batcher."""
  6180. if self._drop_unresolved(event): # identity FIRST: the batch lane is derived from it
  6181. return
  6182. media_group_id = getattr(msg, "media_group_id", None)
  6183. if media_group_id:
  6184. await self._queue_media_group_event(str(media_group_id), event)
  6185. else:
  6186. self._enqueue_photo_event(self._photo_batch_key(event, msg), event)
  6187. @staticmethod
  6188. def _ext_from_path(file_path: Optional[str], candidates, default: str) -> str:
  6189. """First extension in ``candidates`` that ``file_path`` ends with (case-insensitive), else default."""
  6190. if file_path:
  6191. lowered = file_path.lower()
  6192. for candidate in candidates:
  6193. if lowered.endswith(candidate):
  6194. return candidate
  6195. return default
  6196. async def _cache_inbound_av(self, msg, event: MessageEvent, source: Any, label: str, kind: str, ext: str, mime: str) -> bool:
  6197. """Download a voice/audio/video attachment into the local cache. Returns True when the event was
  6198. already dispatched (oversized attachment), so the caller must return."""
  6199. try:
  6200. allowed, note = self._telegram_media_size_allowed(source, label)
  6201. if not allowed:
  6202. event.text = self._append_observed_note(event.text, note or "")
  6203. logger.info("[Telegram] Skipped oversized user %s (size=%s)", kind, getattr(source, "file_size", None))
  6204. await self.handle_message(event)
  6205. return True
  6206. file_obj = await source.get_file()
  6207. data = await file_obj.download_as_bytearray()
  6208. if kind == "video":
  6209. ext = self._ext_from_path(getattr(file_obj, "file_path", None), SUPPORTED_VIDEO_TYPES, ext)
  6210. cached_path = await cache_video_from_bytes_async(bytes(data), ext=ext)
  6211. mime = SUPPORTED_VIDEO_TYPES.get(ext, "video/mp4")
  6212. else:
  6213. cached_path = await cache_audio_from_bytes_async(bytes(data), ext=ext)
  6214. event.media_urls = [cached_path]
  6215. event.media_types = [mime]
  6216. logger.info("[Telegram] Cached user %s at %s", kind, cached_path)
  6217. except Exception as e:
  6218. logger.warning("[Telegram] Failed to cache %s: %s", kind, _redact_telegram_error_text(e), exc_info=True)
  6219. await self._surface_media_cache_failure(msg, event, label, e)
  6220. return False
  6221. async def _dispatch_with_text(self, event: MessageEvent, text: str) -> bool:
  6222. """Replace the event text with a user-facing note and dispatch it; returns True (handled)."""
  6223. event.text = text
  6224. await self.handle_message(event)
  6225. return True
  6226. @staticmethod
  6227. def _set_cached_media(event: MessageEvent, path: str, mime: str, mtype: MessageType, log_fmt: str) -> None:
  6228. event.media_urls = [path]
  6229. event.media_types = [mime]
  6230. event.message_type = mtype
  6231. logger.info(log_fmt, path)
  6232. async def _cache_inbound_document(self, msg, event: MessageEvent) -> bool:
  6233. """Cache a document attachment (image → photo path, video, else generic media + text injection).
  6234. Returns True when the event was already dispatched/routed so the caller must return."""
  6235. doc = msg.document
  6236. try:
  6237. original_filename = doc.file_name or ""
  6238. ext = os.path.splitext(original_filename)[1].lower() if original_filename else ""
  6239. doc_mime = (doc.mime_type or "").lower() # some clients send "IMAGE/PNG"
  6240. if not ext and doc_mime:
  6241. ext = _TELEGRAM_IMAGE_MIME_TO_EXT.get(doc_mime, "")
  6242. if not ext:
  6243. ext = {v: k for k, v in SUPPORTED_DOCUMENT_TYPES.items()}.get(doc_mime, "")
  6244. display = original_filename or doc_mime or ext or 'unknown'
  6245. # Size check before the image branch so image documents can't bypass the limit.
  6246. if not doc.file_size or doc.file_size > self._max_doc_bytes:
  6247. logger.info("[Telegram] Document too large: %s bytes", doc.file_size)
  6248. return await self._dispatch_with_text(
  6249. event, f"The document is too large or its size could not be verified. Maximum: {self._max_doc_bytes // (1024 * 1024)} MB.")
  6250. # Screenshots/photos sent as documents take the image cache + batching path.
  6251. if ext in _TELEGRAM_IMAGE_EXTENSIONS or doc_mime.startswith("image/"):
  6252. file_obj = await doc.get_file()
  6253. image_bytes = await file_obj.download_as_bytearray()
  6254. image_ext = ext if ext in _TELEGRAM_IMAGE_EXTENSIONS else _TELEGRAM_IMAGE_MIME_TO_EXT.get(doc_mime, ".jpg")
  6255. try:
  6256. cached_path = await cache_image_from_bytes_async(bytes(image_bytes), ext=image_ext)
  6257. except ValueError as e:
  6258. logger.warning("[Telegram] Failed to cache image document: %s", _redact_telegram_error_text(e), exc_info=True)
  6259. return await self._dispatch_with_text(event, f"Image document '{display}' could not be read as an image.")
  6260. self._set_cached_media(
  6261. event, cached_path, doc_mime if doc_mime.startswith(
  6262. "image/"
  6263. ) else _TELEGRAM_IMAGE_EXT_TO_MIME.get(image_ext, "image/jpeg"),
  6264. MessageType.PHOTO, "[Telegram] Cached user image-document at %s")
  6265. await self._route_photo_event(msg, event)
  6266. return True
  6267. if not ext and doc.mime_type:
  6268. ext = {v: k for k, v in SUPPORTED_VIDEO_TYPES.items()}.get(doc.mime_type, "")
  6269. if not ext and doc.mime_type:
  6270. # .jpg and .jpeg both map to image/jpeg; keep the first ext seen.
  6271. image_mime_to_ext: dict[str, str] = {}
  6272. for _ext, _mime in SUPPORTED_IMAGE_DOCUMENT_TYPES.items():
  6273. image_mime_to_ext.setdefault(_mime, _ext)
  6274. ext = image_mime_to_ext.get(doc.mime_type, "")
  6275. if ext in SUPPORTED_VIDEO_TYPES:
  6276. file_obj = await doc.get_file()
  6277. video_bytes = await file_obj.download_as_bytearray()
  6278. self._set_cached_media(
  6279. event, await cache_video_from_bytes_async(bytes(video_bytes), ext=ext), SUPPORTED_VIDEO_TYPES[ext], MessageType.VIDEO,
  6280. "[Telegram] Cached user video document at %s")
  6281. await self.handle_message(event)
  6282. return True
  6283. # Any file type is accepted (authorization is the gate, not the extension); unknown types get
  6284. # application/octet-stream. Image documents already returned above.
  6285. file_obj = await doc.get_file()
  6286. raw_bytes = bytes(await file_obj.download_as_bytearray())
  6287. from gateway.platforms.base import cache_media_bytes_async
  6288. cached = await cache_media_bytes_async(raw_bytes, filename=original_filename or f"document{ext or '.bin'}", mime_type=doc_mime)
  6289. if cached is None:
  6290. return await self._dispatch_with_text(event, f"Document '{display}' could not be cached.")
  6291. event.media_urls = [cached.path]
  6292. event.media_types = [cached.media_type]
  6293. event.media_text_inlined = [False] # flipped below once the text is actually injected
  6294. if cached.kind == "audio":
  6295. event.message_type = MessageType.AUDIO
  6296. logger.info("[Telegram] Cached user %s at %s (%s)", cached.kind, cached.path, cached.media_type)
  6297. # Inject text-readable content (≤100 KB). Gate on extension/MIME, NOT a blind UTF-8 decode:
  6298. # PDF/zip/docx have decodable ASCII headers. Binary files are surfaced as a cached path only.
  6299. MAX_TEXT_INJECT_BYTES = 100 * 1024
  6300. _is_text = ext in _TEXT_INJECT_EXTENSIONS or (doc_mime or "").startswith("text/")
  6301. if _is_text and len(raw_bytes) <= MAX_TEXT_INJECT_BYTES:
  6302. try:
  6303. text_content = raw_bytes.decode("utf-8")
  6304. display_name = re.sub(r'[^\w.\- ]', '_', original_filename or f"document{ext or '.txt'}")
  6305. injection = f"[Content of {display_name}]:\n{text_content}"
  6306. event.text = f"{injection}\n\n{event.text}" if event.text else injection
  6307. event.media_text_inlined = [True]
  6308. except UnicodeDecodeError:
  6309. pass # binary — agent has the cached path
  6310. except Exception as e:
  6311. logger.warning("[Telegram] Failed to cache document: %s", _redact_telegram_error_text(e), exc_info=True)
  6312. await self._surface_media_cache_failure(msg, event, "attachment", e, display_name=getattr(doc, "file_name", None) or None)
  6313. return False
  6314. async def _handle_media_message(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
  6315. """Handle incoming media messages, downloading images to local cache."""
  6316. msg = update.message
  6317. if not msg:
  6318. return
  6319. if not self._is_user_authorized_from_message(msg):
  6320. self._log_blocked_user(msg, level=logging.INFO, what="media from unauthorized user")
  6321. return
  6322. if not self._should_process_message(msg):
  6323. if self._should_observe_unmentioned_group_message(msg):
  6324. _event = self._build_message_event(msg, self._media_message_type(msg), update_id=update.update_id)
  6325. if msg.caption:
  6326. _event.text = self._clean_bot_trigger_text(expand_link_entities(msg))
  6327. await self._cache_observed_media(msg, _event)
  6328. self._observe_unmentioned_group_message(msg, _event.message_type, update_id=update.update_id, event=_event)
  6329. return
  6330. event = self._build_message_event(msg, self._media_message_type(msg), update_id=update.update_id)
  6331. if msg.caption:
  6332. from plugins.platforms.telegram.telegram_context import group_trigger_text
  6333. event.text = group_trigger_text(self, msg, expand_link_entities(msg))
  6334. # Stickers: _handle_sticker overwrites event.text with its vision description, so observe attribution must run after it.
  6335. if msg.sticker:
  6336. await self._handle_sticker(msg, event)
  6337. await self.handle_message(self._apply_telegram_group_observe_attribution(event))
  6338. return
  6339. event = self._apply_telegram_group_observe_attribution(event)
  6340. # Cache photo locally: Telegram's file URLs expire (~1 hour) before vision may run.
  6341. if msg.photo:
  6342. try:
  6343. file_obj = await msg.photo[-1].get_file() # PhotoSize list sorted by size; largest last
  6344. image_bytes = await file_obj.download_as_bytearray()
  6345. ext = self._ext_from_path(file_obj.file_path, [".png", ".webp", ".gif", ".jpeg", ".jpg"], ".jpg")
  6346. self._set_cached_media(
  6347. event, await cache_image_from_bytes_async(bytes(image_bytes), ext=ext), f"image/{ext.lstrip('.')}", event.message_type,
  6348. "[Telegram] Cached user photo at %s")
  6349. await self._route_photo_event(msg, event)
  6350. return
  6351. except Exception as e:
  6352. logger.warning("[Telegram] Failed to cache photo: %s", _redact_telegram_error_text(e), exc_info=True)
  6353. await self._surface_media_cache_failure(msg, event, "photo", e)
  6354. # Voice/audio cached for STT transcription; video for vision.
  6355. if msg.voice:
  6356. if await self._cache_inbound_av(msg, event, msg.voice, "voice message", "voice", ".ogg", "audio/ogg"):
  6357. return
  6358. elif msg.audio:
  6359. if await self._cache_inbound_av(msg, event, msg.audio, "audio file", "audio", ".mp3", "audio/mp3"):
  6360. return
  6361. elif video := msg.video or getattr(msg, "video_note", None):
  6362. if await self._cache_inbound_av(msg, event, video, "video file" if msg.video else "video note", "video", ".mp4", "video/mp4"):
  6363. return
  6364. elif msg.document and await self._cache_inbound_document(msg, event):
  6365. return
  6366. media_group_id = getattr(msg, "media_group_id", None)
  6367. if media_group_id:
  6368. await self._queue_media_group_event(str(media_group_id), event)
  6369. return
  6370. await self.handle_message(event)
  6371. async def _queue_media_group_event(self, media_group_id: str, event: MessageEvent) -> None:
  6372. """Debounce album items (shared media_group_id) into one MessageEvent so the second image isn't
  6373. treated as a new message interrupting the first."""
  6374. if self._should_drop_delayed_delivery():
  6375. self._hold_inbound_event(event, where="media-group-enqueue")
  6376. return
  6377. self._merge_into_pending(self._media_group_events, media_group_id, event)
  6378. self._accept_update()
  6379. prior_task = self._media_group_tasks.get(media_group_id)
  6380. if prior_task:
  6381. prior_task.cancel()
  6382. self._media_group_tasks[media_group_id] = asyncio.create_task(self._flush_media_group_event(media_group_id))
  6383. async def _flush_media_group_event(self, media_group_id: str) -> None:
  6384. await self._flush_buffered(
  6385. self._media_group_events, self._media_group_tasks, media_group_id, self.MEDIA_GROUP_WAIT_SECONDS, "media-group")
  6386. async def _handle_sticker(self, msg: Message, event: MessageEvent) -> None:
  6387. """Describe a sticker via vision, cached by file_unique_id; animated/video stickers get an emoji placeholder."""
  6388. from gateway.sticker_cache import (
  6389. get_cached_description, cache_sticker_description_async, build_sticker_injection,
  6390. build_animated_sticker_injection, STICKER_VISION_PROMPT)
  6391. sticker = msg.sticker
  6392. emoji = sticker.emoji or ""
  6393. set_name = sticker.set_name or ""
  6394. if sticker.is_animated or sticker.is_video:
  6395. event.text = build_animated_sticker_injection(emoji)
  6396. return
  6397. cached = get_cached_description(sticker.file_unique_id)
  6398. if cached:
  6399. event.text = build_sticker_injection(cached["description"], cached.get("emoji", emoji), cached.get("set_name", set_name))
  6400. logger.info("[Telegram] Sticker cache hit: %s", sticker.file_unique_id)
  6401. return
  6402. fallback = f"a sticker with emoji {emoji}" if emoji else "a sticker"
  6403. try:
  6404. file_obj = await sticker.get_file()
  6405. image_bytes = await file_obj.download_as_bytearray()
  6406. cached_path = await cache_image_from_bytes_async(bytes(image_bytes), ext=".webp")
  6407. logger.info("[Telegram] Analyzing sticker at %s", cached_path)
  6408. from tools.vision_tools import vision_analyze_tool
  6409. self._accept_update() # Cancellation cannot undo an already submitted auxiliary LLM request.
  6410. result = json.loads(await vision_analyze_tool(image_url=cached_path, user_prompt=STICKER_VISION_PROMPT))
  6411. if result.get("success"):
  6412. description = result.get("analysis", "a sticker")
  6413. await cache_sticker_description_async(sticker.file_unique_id, description, emoji, set_name)
  6414. event.text = build_sticker_injection(description, emoji, set_name)
  6415. else:
  6416. event.text = build_sticker_injection(fallback, emoji, set_name)
  6417. except Exception as e:
  6418. logger.warning("[Telegram] Sticker analysis error: %s", _redact_telegram_error_text(e), exc_info=True)
  6419. event.text = build_sticker_injection(fallback, emoji, set_name)
  6420. def _reload_dm_topics_from_config(self) -> None:
  6421. """Re-read dm_topics from config.yaml so externally created topics work without restart."""
  6422. try:
  6423. from hermes_cli.config import load_config_readonly # canonical loader: managed overlay + ${VAR}
  6424. dm_topics = load_config_readonly().get("platforms", {}).get("telegram", {}).get("extra", {}).get("dm_topics", [])
  6425. if not dm_topics:
  6426. self._dm_topics_config = []
  6427. self._dm_topic_chat_ids = set()
  6428. return
  6429. self._dm_topics_config = dm_topics
  6430. self._dm_topic_chat_ids = {str(chat_entry["chat_id"]) for chat_entry in dm_topics if "chat_id" in chat_entry}
  6431. for chat_entry in dm_topics:
  6432. cid = chat_entry.get("chat_id")
  6433. if not cid:
  6434. continue
  6435. for t in chat_entry.get("topics", []):
  6436. tid = t.get("thread_id")
  6437. name = t.get("name")
  6438. if tid and name and f"{cid}:{name}" not in self._dm_topics:
  6439. self._dm_topics[f"{cid}:{name}"] = int(tid)
  6440. logger.info("[%s] Hot-loaded DM topic from config: %s -> thread_id=%s", self.name, f"{cid}:{name}", tid)
  6441. except Exception as e:
  6442. logger.debug("[%s] Failed to reload dm_topics from config: %s", self.name, e)
  6443. def _get_dm_topic_info(self, chat_id: str, thread_id: Optional[str]) -> Optional[dict[str, Any]]:
  6444. """Return the DM topic config dict (name, skill, ...) for this thread_id, or None."""
  6445. if not thread_id:
  6446. return None
  6447. thread_id_int = int(thread_id)
  6448. def _lookup() -> Optional[dict[str, Any]]:
  6449. for key, cached_tid in self._dm_topics.items():
  6450. if cached_tid == thread_id_int and key.startswith(f"{chat_id}:"):
  6451. topic_name = key.split(":", 1)[1]
  6452. for chat_entry in self._dm_topics_config:
  6453. if str(chat_entry.get("chat_id")) == chat_id:
  6454. for t in chat_entry.get("topics", []):
  6455. if t.get("name") == topic_name:
  6456. return t
  6457. return {"name": topic_name}
  6458. return None
  6459. found = _lookup()
  6460. if found is not None:
  6461. return found
  6462. self._reload_dm_topics_from_config() # cache miss — topics may have been added externally
  6463. return _lookup()
  6464. def _cache_dm_topic_from_message(self, chat_id: str, thread_id: str, topic_name: str) -> None:
  6465. """Cache a thread_id -> topic_name mapping discovered from an incoming message."""
  6466. cache_key = f"{chat_id}:{topic_name}"
  6467. if cache_key not in self._dm_topics:
  6468. self._dm_topics[cache_key] = int(thread_id)
  6469. logger.info("[%s] Cached DM topic from message: %s -> thread_id=%s", self.name, cache_key, thread_id)
  6470. @classmethod
  6471. def _flatten_rich_inline_text(cls, value: Any) -> str:
  6472. """Best-effort plaintext flattener for Bot API rich-message inline nodes."""
  6473. if value is None:
  6474. return ""
  6475. if isinstance(value, str):
  6476. return value
  6477. if isinstance(value, list):
  6478. return "".join(cls._flatten_rich_inline_text(item) for item in value)
  6479. if isinstance(value, dict):
  6480. for key in ("text", "children"):
  6481. if value.get(key) is not None:
  6482. return cls._flatten_rich_inline_text(value[key])
  6483. return ""
  6484. @classmethod
  6485. def _flatten_rich_blocks(cls, blocks: Any) -> str:
  6486. """Best-effort plaintext flattener for Bot API rich-message blocks."""
  6487. if not isinstance(blocks, list):
  6488. return ""
  6489. lines: list[str] = []
  6490. for block in blocks:
  6491. if not isinstance(block, dict):
  6492. continue
  6493. if block.get("type") == "list":
  6494. for item in block.get("items", []):
  6495. if not isinstance(item, dict):
  6496. continue
  6497. item_lines = cls._flatten_rich_blocks(item.get("blocks")).splitlines()
  6498. if not item_lines:
  6499. continue
  6500. label = item.get("label")
  6501. lines.append(f"{label} {item_lines[0]}".strip() if label else item_lines[0])
  6502. lines.extend(item_lines[1:])
  6503. continue
  6504. text = cls._flatten_rich_inline_text(block.get("text"))
  6505. if text:
  6506. lines.extend(text.splitlines())
  6507. return "\n".join(line.rstrip() for line in lines if line)
  6508. @classmethod
  6509. def _extract_rich_reply_text(cls, reply_to_message: Any) -> Optional[str]:
  6510. """Return plaintext echoed by Telegram's rich_message reply payload."""
  6511. try:
  6512. getter = getattr(getattr(reply_to_message, "api_kwargs", None), "get", None)
  6513. if not callable(getter):
  6514. return None
  6515. rich_getter = getattr(getter("rich_message"), "get", None)
  6516. if not callable(rich_getter):
  6517. return None
  6518. return cls._flatten_rich_blocks(rich_getter("blocks")).strip() or None
  6519. except Exception:
  6520. return None
  6521. def _resolve_topic_binding(self, message: Message, chat_type: str, thread_id_str: Optional[str]) -> tuple:
  6522. """Return ``(chat_topic, topic_skill)`` for a DM topic or bound forum topic (else Nones)."""
  6523. chat = message.chat
  6524. chat_topic = None
  6525. topic_skill = None
  6526. if chat_type == "dm" and thread_id_str:
  6527. topic_info = self._get_dm_topic_info(str(chat.id), thread_id_str)
  6528. if topic_info:
  6529. chat_topic = topic_info.get("name")
  6530. topic_skill = topic_info.get("skill")
  6531. # forum_topic_created service messages also reveal topic names
  6532. if hasattr(message, "forum_topic_created") and message.forum_topic_created:
  6533. created_name = message.forum_topic_created.name
  6534. if created_name:
  6535. self._cache_dm_topic_from_message(str(chat.id), thread_id_str, created_name)
  6536. if not chat_topic:
  6537. chat_topic = created_name
  6538. elif chat_type == "group" and thread_id_str:
  6539. # Forum topic skill binding via config.extra['group_topics']; accepts both
  6540. # [{"chat_id": ..., "topics": [...]}] and legacy {"-100...": [{"thread_id": 12}]}.
  6541. group_topics_config = self.config.extra.get("group_topics", [])
  6542. if isinstance(group_topics_config, dict):
  6543. group_topics_iter = [{"chat_id": cfg_chat_id, "topics": topics} for cfg_chat_id, topics in group_topics_config.items()]
  6544. elif isinstance(group_topics_config, list):
  6545. group_topics_iter = [entry for entry in group_topics_config if isinstance(entry, dict)]
  6546. else:
  6547. group_topics_iter = []
  6548. for chat_entry in group_topics_iter:
  6549. if str(chat_entry.get("chat_id", "")) != str(chat.id):
  6550. continue
  6551. topics = chat_entry.get("topics", [])
  6552. for topic in (topics if isinstance(topics, list) else []):
  6553. if not isinstance(topic, dict):
  6554. continue
  6555. tid = topic.get("thread_id")
  6556. if tid is not None and str(tid) == thread_id_str:
  6557. chat_topic = topic.get("name")
  6558. topic_skill = topic.get("skill")
  6559. break
  6560. break
  6561. return chat_topic, topic_skill
  6562. def _reply_context(self, message: Message) -> tuple:
  6563. """``(reply_to_id, reply_to_text)`` for the replied-to message: Telegram's native partial quote
  6564. first, then text/caption, rich echo, then the sent index."""
  6565. if not message.reply_to_message:
  6566. return None, None
  6567. reply_to_id = str(message.reply_to_message.message_id)
  6568. quote = getattr(message, "quote", None)
  6569. quote_text = getattr(quote, "text", None) if quote is not None else None
  6570. if quote_text:
  6571. return reply_to_id, quote_text
  6572. reply_to_text = message.reply_to_message.text or message.reply_to_message.caption or None
  6573. if not reply_to_text:
  6574. reply_to_text = self._extract_rich_reply_text(message.reply_to_message)
  6575. if not reply_to_text:
  6576. try:
  6577. from gateway import rich_sent_store
  6578. reply_to_text = rich_sent_store.lookup(str(message.chat.id), reply_to_id)
  6579. except Exception:
  6580. # Extract reply context if this message is a reply. Prefer Telegram's native partial quote
  6581. # (message.quote, TextQuote) so a user replying to a single selected substring of a prior
  6582. # multi-section message doesn't get the whole replied-to message injected into the agent's
  6583. # context — which can cause the agent to act on unrelated actionable-looking text the user
  6584. # didn't quote (#22619). Fall back to the full replied-to message text / caption when no
  6585. # native quote is present.
  6586. reply_to_text = None
  6587. return reply_to_id, reply_to_text
  6588. def _build_message_event(self, message: Message, msg_type: MessageType, update_id: Optional[int] = None) -> MessageEvent:
  6589. """Build a MessageEvent from a Telegram message. ``update_id`` lets ``/restart`` record the
  6590. triggering offset so the new gateway process advances past it."""
  6591. chat = message.chat
  6592. user = message.from_user
  6593. telegram_chat_type = self._chat_type_str(chat) # str() so PTB enums and plain-string mocks both work
  6594. chat_type = "group" if telegram_chat_type in {"group", "supergroup"} else ("channel" if telegram_chat_type == "channel" else "dm")
  6595. # Shared normalizer so gating and session routing agree (reply-UI anchors dropped, General → "1").
  6596. # Resolve routable thread id for DM topics and forum group topics via the shared normalizer, so
  6597. # gating and session routing agree on one value. Only real topic/forum messages keep a thread id;
  6598. # ordinary reply-UI anchors are dropped (they are not durable session threads and sends against them
  6599. # hit 'Message thread not found', #3206), while forum General-topic messages
  6600. # (message_thread_id=None) normalize to the General-topic id so replies route back to General
  6601. # (#22423).
  6602. thread_id_str = self._effective_message_thread_id(message)
  6603. # A new private-chat topic's opening message can arrive with no thread id at all (#136567); the
  6604. # creation it follows is then the only source of the topic id. Adopt it before anything downstream
  6605. # (gating, batching, session routing, topic recovery) reads the event source.
  6606. thread_id_str = self._adopt_created_topic_thread(message, chat_type, thread_id_str)
  6607. chat_topic, topic_skill = self._resolve_topic_binding(message, chat_type, thread_id_str)
  6608. has_full_name = hasattr(chat, "full_name")
  6609. if user:
  6610. user_name = user.full_name
  6611. elif has_full_name and chat_type == "dm":
  6612. user_name = chat.full_name
  6613. else:
  6614. user_name = chat.title if chat_type == "channel" else None
  6615. source = self.build_source(
  6616. chat_id=str(chat.id), chat_name=chat.title or (chat.full_name if has_full_name else None), chat_type=chat_type,
  6617. user_id=(str(user.id) if user else (str(chat.id) if chat_type in {"dm", "channel"} else None)),
  6618. user_name=user_name, thread_id=thread_id_str, chat_topic=chat_topic, message_id=str(message.message_id),
  6619. is_bot=bool(getattr(user, "is_bot", False)) if user else False)
  6620. reply_to_id, reply_to_text = self._reply_context(message)
  6621. from gateway.platforms.base import resolve_channel_prompt # per-channel/topic ephemeral prompt
  6622. from plugins.platforms.telegram.telegram_context import group_identity_prompt, group_reply_expected
  6623. _chat_id_str = str(chat.id)
  6624. channel_prompt = resolve_channel_prompt(self.config.extra, thread_id_str or _chat_id_str, _chat_id_str if thread_id_str else None)
  6625. return MessageEvent(
  6626. text=expand_link_entities(message), message_type=msg_type, source=source, raw_message=message,
  6627. message_id=str(message.message_id), platform_update_id=update_id,
  6628. reply_to_message_id=reply_to_id, reply_to_text=reply_to_text, auto_skill=topic_skill,
  6629. channel_prompt=group_identity_prompt(self, message, channel_prompt), reply_expected=group_reply_expected(self, message, chat_type),
  6630. edit_revision=self._edit_revision(message, update_id), timestamp=message.date)
  6631. @staticmethod
  6632. def _edit_revision(message, update_id: Optional[int]) -> Optional[str]:
  6633. """Identity of one edit of a message, None for a new send. ``edit_date`` has whole-second
  6634. resolution, so two edits in one second would share it: the update id is unique per edit and
  6635. stable on redelivery; without one, the edited text tells same-second edits apart."""
  6636. edit_date = getattr(message, "edit_date", None)
  6637. if not hasattr(edit_date, "timestamp"):
  6638. return None
  6639. if update_id is not None:
  6640. return f"u{update_id}"
  6641. text = (getattr(message, "text", None) or getattr(message, "caption", None) or "").encode()
  6642. return f"{int(edit_date.timestamp())}.{hashlib.sha256(text).hexdigest()[:16]}"
  6643. # -- Message reactions (processing lifecycle) --
  6644. def _reactions_enabled(self) -> bool:
  6645. """Reactions: scoped ``TELEGRAM_REACTIONS`` → ``extra.reactions`` (YAML, per profile) → off.
  6646. An explicit env var wins over YAML, so the stock ``reactions: false`` every install
  6647. materializes cannot silently kill a documented ``TELEGRAM_REACTIONS=true`` (#109032). Under
  6648. multiplex a scoped miss falls to the profile's own YAML, never another profile's env (#72348).
  6649. """
  6650. configured = _extra_or_secret(self.config.extra, "reactions", "TELEGRAM_REACTIONS", None)
  6651. if configured is None:
  6652. return False
  6653. return str(configured).lower() not in {"false", "0", "no"}
  6654. async def _set_reaction(self, chat_id: str, message_id: str, emoji: Optional[str]) -> bool:
  6655. """Set a single emoji reaction (``None`` clears all bot-set reactions, the documented Bot API way)."""
  6656. if not self._bot:
  6657. return False
  6658. try:
  6659. await self._bot.set_message_reaction(chat_id=normalize_telegram_chat_id(chat_id), message_id=int(message_id), reaction=emoji)
  6660. return True
  6661. except Exception as e:
  6662. if emoji is None:
  6663. logger.debug("[%s] clear reactions failed: %s", self.name, _redact_telegram_error_text(e))
  6664. else:
  6665. logger.debug("[%s] set_message_reaction failed (%s): %s", self.name, emoji, _redact_telegram_error_text(e))
  6666. return False
  6667. async def _clear_reactions(self, chat_id: str, message_id: str) -> bool:
  6668. """Clear all bot-set reactions."""
  6669. return await self._set_reaction(chat_id, message_id, None)
  6670. async def on_processing_start(self, event: MessageEvent) -> None:
  6671. """Add an in-progress reaction when message processing begins."""
  6672. if not self._reactions_enabled():
  6673. return
  6674. chat_id = getattr(event.source, "chat_id", None)
  6675. message_id = getattr(event, "message_id", None)
  6676. if chat_id and message_id:
  6677. await self._set_reaction(chat_id, message_id, "\U0001f440")
  6678. async def on_processing_complete(self, event: MessageEvent, outcome: ProcessingOutcome) -> None:
  6679. """Swap the in-progress reaction for a final success/failure reaction (set_message_reaction
  6680. replaces, not adds); CANCELLED explicitly clears the 👀."""
  6681. if not self._reactions_enabled():
  6682. return
  6683. chat_id = getattr(event.source, "chat_id", None)
  6684. message_id = getattr(event, "message_id", None)
  6685. if not (chat_id and message_id):
  6686. return
  6687. if outcome == ProcessingOutcome.CANCELLED:
  6688. await self._clear_reactions(chat_id, message_id)
  6689. else:
  6690. await self._set_reaction(chat_id, message_id, "\U0001f44d" if outcome == ProcessingOutcome.SUCCESS else "\U0001f44e")
  6691. # -- Plugin registration glue: register(ctx) plus the hook implementations (adapter factory, YAML→env/extra
  6692. # config, setup wizard, standalone sender).
  6693. # ────────────────────────────────────────────────────────────────────────── Plugin migration glue (#41112 /
  6694. # #3823) Added when the Telegram adapter (+ its telegram_network satellite) moved from gateway/platforms/
  6695. # into this bundled plugin. Mirrors the Discord (#24356) / Slack migrations: a register(ctx) entry point
  6696. # plus hook implementations that replace the per-platform core touchpoints (the Platform.TELEGRAM branch in
  6697. # gateway/run.py, the telegram_cfg YAML→env/extra block in gateway/config.py, the _setup_telegram wizard +
  6698. # _PLATFORMS["telegram"] static dict in hermes_cli/{setup,gateway}.py, and the _send_telegram dispatch in
  6699. # tools/send_message_tool.py). Telegram uses the generic token connected check, so no is_connected override
  6700. # is needed. ──────────────────────────────────────────────────────────────────────────
  6701. def _resolve_notifications_mode() -> str:
  6702. """Notification mode (all/important) from env, else config.yaml display.platforms.telegram.notifications."""
  6703. mode = os.getenv("HERMES_TELEGRAM_NOTIFICATIONS", "")
  6704. if not mode:
  6705. try:
  6706. from gateway.config import load_gateway_config
  6707. from gateway.run import cfg_get
  6708. _raw = cfg_get(load_gateway_config(), "display", "platforms", "telegram", "notifications")
  6709. if _raw not in {None, ""}:
  6710. mode = str(_raw).strip().lower()
  6711. except Exception:
  6712. pass
  6713. mode = mode or "important"
  6714. if mode not in {"all", "important"}:
  6715. logger.warning("Unknown telegram notifications mode '%s', defaulting to 'important' (valid: all, important)", mode)
  6716. mode = "important"
  6717. return mode
  6718. def _build_adapter(config):
  6719. """Construct TelegramAdapter and apply the notification mode."""
  6720. adapter = TelegramAdapter(config)
  6721. try:
  6722. adapter._notifications_mode = _resolve_notifications_mode()
  6723. except Exception:
  6724. adapter._notifications_mode = "important"
  6725. return adapter
  6726. def _is_connected(config) -> bool:
  6727. """Connected when a bot token is configured (env or PlatformConfig.token); the SDK being importable is
  6728. not enough or the plugin-enable pass would enable Telegram on any machine with it installed."""
  6729. token = getattr(config, "token", None)
  6730. if not token:
  6731. import hermes_cli.gateway as gateway_mod
  6732. token = gateway_mod.get_env_value("TELEGRAM_BOT_TOKEN") or ""
  6733. return bool(str(token).strip())
  6734. async def _standalone_send(pconfig, chat_id, message, *, thread_id=None, media_files=None, force_document=False):
  6735. """Out-of-process delivery (standalone_sender_fn) so deliver=telegram cron jobs succeed without the
  6736. gateway; delegates to the REST ``_send_telegram`` sender."""
  6737. token = getattr(pconfig, "token", None)
  6738. if not token:
  6739. from agent.secret_scope import get_secret # profile-scoped: never borrow another profile's token
  6740. token = get_secret("TELEGRAM_BOT_TOKEN", "") or ""
  6741. disable_link_previews = bool(getattr(pconfig, "extra", {}) and pconfig.extra.get("disable_link_previews"))
  6742. from tools.send_message_tool import _send_telegram
  6743. return await _send_telegram(
  6744. token, chat_id, message, media_files=media_files, thread_id=thread_id,
  6745. disable_link_previews=disable_link_previews, force_document=force_document)
  6746. def interactive_setup() -> None:
  6747. """Configure Telegram credentials and allowlist via the CLI setup wizard (lazy import)."""
  6748. setup_platforms._setup_telegram()
  6749. def _apply_yaml_config(yaml_cfg: dict, telegram_cfg: dict) -> dict | None:
  6750. """Translate config.yaml telegram: keys into TELEGRAM_* env vars and PlatformConfig.extra. Env vars
  6751. take precedence over YAML. Returns extras to merge into PlatformConfig.extra, or None.
  6752. Implements the apply_yaml_config_fn contract (#24849). Mirrors the legacy telegram_cfg block from
  6753. gateway/config.py::load_gateway_config().
  6754. """
  6755. import json as _json
  6756. from gateway.platforms._shared import yaml_env_setter
  6757. extras: dict = {}
  6758. # Under multiplex a secondary profile's settings must NOT hit the process-global env (first-writer-wins
  6759. # would pin them for every profile, #72348); yaml_env_setter skips the write under its scope and the
  6760. # values flow via extra/secret scope instead.
  6761. _set_env = yaml_env_setter()
  6762. def _bridge_lower(key: str, env: str) -> None:
  6763. if key in telegram_cfg:
  6764. extras.setdefault(key, telegram_cfg[key])
  6765. _set_env(env, str(telegram_cfg[key]).lower())
  6766. def _bridge_gate(key: str, env: str, value: Any, *, seed_extra: bool = False) -> None:
  6767. """CSV allowlist gate: list → comma-joined; env write skipped under multiplex secret scope."""
  6768. if value is None:
  6769. return
  6770. if seed_extra:
  6771. extras.setdefault(key, value)
  6772. _set_env(env, value)
  6773. for _topic_key in ("disable_topic_auto_rename", "dm_topic_auto_transfer"):
  6774. if _topic_key in telegram_cfg:
  6775. extras.setdefault(_topic_key, telegram_cfg[_topic_key])
  6776. _effective_rm = telegram_cfg.get("require_mention", yaml_cfg.get("require_mention"))
  6777. if _effective_rm is not None:
  6778. _set_env("TELEGRAM_REQUIRE_MENTION", str(_effective_rm).lower())
  6779. if "mention_patterns" in telegram_cfg:
  6780. _set_env("TELEGRAM_MENTION_PATTERNS", _json.dumps(telegram_cfg["mention_patterns"]))
  6781. for key, env in (
  6782. ("exclusive_bot_mentions", "TELEGRAM_EXCLUSIVE_BOT_MENTIONS"), ("allow_bots", "TELEGRAM_ALLOW_BOTS"),
  6783. ("bots_require_mention", "TELEGRAM_BOTS_REQUIRE_MENTION"),
  6784. ("guest_mode", "TELEGRAM_GUEST_MODE", ), ("observe_unmentioned_group_messages", "TELEGRAM_OBSERVE_UNMENTIONED_GROUP_MESSAGES")):
  6785. _bridge_lower(key, env)
  6786. # No extras seed for allowed_chats / allowed_topics / group_allowed_chats: the shared-key loop already
  6787. # bridges them with their original type and this merge would clobber it.
  6788. for key, env, seed in (
  6789. ("free_response_chats", "TELEGRAM_FREE_RESPONSE_CHATS", True), ("free_response_topics", "TELEGRAM_FREE_RESPONSE_TOPICS", False),
  6790. ("allowed_chats", "TELEGRAM_ALLOWED_CHATS", False), ("allowed_topics", "TELEGRAM_ALLOWED_TOPICS", False),
  6791. ("ignored_threads", "TELEGRAM_IGNORED_THREADS", True)):
  6792. _bridge_gate(key, env, telegram_cfg.get(key), seed_extra=seed)
  6793. _bridge_lower("reactions", "TELEGRAM_REACTIONS")
  6794. if "proxy_url" in telegram_cfg:
  6795. # Seeded into extra so ``_build_ptb_requests`` keeps a secondary's route without the env bridge.
  6796. extras.setdefault("proxy_url", str(telegram_cfg["proxy_url"]).strip())
  6797. _set_env("TELEGRAM_PROXY", str(telegram_cfg["proxy_url"]).strip())
  6798. _telegram_extra = telegram_cfg.get("extra") if isinstance(telegram_cfg.get("extra"), dict) else {}
  6799. _telegram_rtm = telegram_cfg["reply_to_mode"] if "reply_to_mode" in telegram_cfg else _telegram_extra.get("reply_to_mode")
  6800. if _telegram_rtm is not None:
  6801. _set_env("TELEGRAM_REPLY_TO_MODE", "off" if _telegram_rtm is False else str(_telegram_rtm).lower())
  6802. _bridge_gate("allow_from", "TELEGRAM_ALLOWED_USERS", telegram_cfg.get("allow_from"))
  6803. _bridge_gate("group_allow_from", "TELEGRAM_GROUP_ALLOWED_USERS", telegram_cfg.get("group_allow_from") or _telegram_extra.get("group_allow_from"))
  6804. _bridge_gate(
  6805. "group_allowed_chats", "TELEGRAM_GROUP_ALLOWED_CHATS",
  6806. telegram_cfg.get("group_allowed_chats") or _telegram_extra.get("group_allowed_chats"))
  6807. for _key in ("guest_mode", "disable_link_previews", "observe_unmentioned_group_messages", "free_response_topics"):
  6808. if _key in telegram_cfg:
  6809. extras.setdefault(_key, telegram_cfg[_key])
  6810. # Pass through telegram-specific extra keys but EXCLUDE generic shared-config keys: _merge_platform_map
  6811. # already applied top-level-over-nested precedence and re-emitting them via dict.update() would undo it.
  6812. _GENERIC_MERGE_KEYS = {
  6813. "reply_prefix", "reply_in_thread", "reply_to_mode", "unauthorized_dm_behavior", "notice_delivery",
  6814. "require_mention", "channel_skill_bindings", "channel_prompts", "gateway_restart_notification", "allow_from",
  6815. "allow_admin_from", "dm_policy", "group_policy"}
  6816. for _k, _v in _telegram_extra.items():
  6817. if _k not in _GENERIC_MERGE_KEYS:
  6818. extras.setdefault(_k, _v)
  6819. return extras or None
  6820. def register(ctx) -> None:
  6821. """Plugin entry point — called by the Hermes plugin system."""
  6822. from .adapter_hint import PLATFORM_HINT
  6823. from .dashboard_card import DASHBOARD_CARD
  6824. from . import gateway_env # env -> GatewayConfig bridge (gateway.config_env)
  6825. ctx.register_platform(
  6826. name="telegram", label="Telegram", adapter_factory=_build_adapter, check_fn=telegram_deps_present,
  6827. **DASHBOARD_CARD,
  6828. ensure_deps_fn=check_telegram_requirements, is_connected=_is_connected, required_env=["TELEGRAM_BOT_TOKEN"],
  6829. install_hint=t("platform.telegram.install_hint"), setup_fn=interactive_setup, apply_yaml_config_fn=_apply_yaml_config, cron_deliver_env_var="TELEGRAM_HOME_CHANNEL",
  6830. standalone_sender_fn=_standalone_send, max_message_length=4096, emoji="✈️", allow_update_command=True,
  6831. platform_hint=PLATFORM_HINT,
  6832. display_tier="high",
  6833. # Mobile inbox: quiet tool_progress / busy-ack, but keep interim commentary and heartbeats.
  6834. display_overrides={"tool_progress": "off", "busy_ack_detail": False},
  6835. target_ref_syntax_fn=parse_send_target,
  6836. env_overrides=gateway_env.ENV_OVERRIDES,
  6837. env_enable_credentials=gateway_env.ENV_ENABLE_CREDENTIALS, **gateway_env.AUTHZ,
  6838. )