| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385 |
- # health: allow FILE_LINES -- platform-tables refactor: core's PLATFORM_HINTS/_PLATFORM_DEFAULTS rows for this platform moved onto its register() call (hint text lives in adapter_hint.py); the growth is only those kwargs
- """Telegram platform adapter (python-telegram-bot): inbound messages/media/commands, outbound replies."""
- import asyncio
- import contextlib
- import dataclasses
- import hashlib
- import inspect
- import json
- import logging
- import os
- import html as _html
- import re
- import time
- from contextvars import ContextVar
- from datetime import datetime, timezone, UTC
- from typing import Any, Awaitable, Callable, Dict, Iterator, List, Optional, Set
- from hermes_cli import setup_platforms
- logger = logging.getLogger(__name__)
- from agent.deadline import run_bounded_async
- from agent.ssl_verify import platform_ssl_context
- from gateway.platforms._shared import (
- decode_json_list_literal as _decode_json_list_literal,
- extra_or_secret as _extra_or_secret, get_scoped_secret as _get_scoped_secret,
- platform_gate_env as _scoped_gate_env,
- )
- def _redact_telegram_error_text(error: object) -> str:
- """Redact secrets from Telegram transport errors before logging or returning them."""
- text = "" if error is None else str(error)
- if not text:
- # httpx timeout exceptions (ConnectTimeout, ReadTimeout, ...) stringify to "" — keep the
- # class name so failure lines never log an empty reason (#111211).
- return f"<{type(error).__name__}>" if error is not None else text
- try:
- from agent.redact import redact_sensitive_text
- return redact_sensitive_text(text, force=True)
- except Exception:
- return "<telegram error redacted>"
- def _consume_abandoned_task(task: asyncio.Task) -> None:
- """Observe a detached task's terminal exception to avoid noisy loop logs."""
- try:
- task.exception()
- except asyncio.CancelledError:
- pass
- except Exception:
- logger.debug("Abandoned Telegram init task failed after timeout", exc_info=True)
- async def _await_with_thread_deadline(
- awaitable, timeout: float, *, on_abandon=None, label: str = "telegram", dump_on_blocked_loop: bool = True,
- ):
- """Wall-clock deadline that survives a blocked loop / cancellation-shielded PTB+httpcore init.
- ``on_abandon`` runs detached so an abandoned initialize() can't leak an httpx pool. Raises
- ``asyncio.TimeoutError`` on expiry (feeds the PTB retry ladder). Send/media call sites pass
- ``dump_on_blocked_loop=False``: the bug they bound is a shielded socket on a LIVE loop, and the init
- wrapper already reports a blocked loop, so N in-flight sends must not each arm a stack-dump timer.
- Thin wrapper over :func:`agent.deadline.run_bounded_async` (#85125 Phase 2f) — this adapter's private
- implementation was the ancestor of that primitive and is now consolidated onto it. The unified layer
- keeps every property the 9 call sites here rely on: thread-timer deadline that survives a blocked event
- loop (#63309), abandonment of cancellation-shielded tasks (PTB/httpcore init inside anyio scopes),
- detached best-effort ``on_abandon`` cleanup so an abandoned initialize() can't leak an httpx pool per
- retry attempt, and off-loop stack-dump diagnostics when the loop never processes the expiry.
- """
- result = await run_bounded_async(
- awaitable, timeout, label=label, on_abandon=on_abandon, dump_on_blocked_loop=dump_on_blocked_loop)
- if result.timed_out:
- raise TimeoutError(f"timed out after {timeout:.0f}s ({label})")
- return result.value
- def _iter_exception_graph(error: BaseException) -> Iterator[BaseException]:
- """Yield ``error`` and every ``__cause__``/``__context__`` ancestor (DFS, cycle-safe) —
- PTB wraps httpx errors, so classifiers must inspect the whole graph."""
- seen: set[int] = set()
- stack: list[BaseException] = [error]
- while stack:
- cur = stack.pop()
- ident = id(cur)
- if ident in seen:
- continue
- seen.add(ident)
- yield cur
- stack.extend(x for x in (getattr(cur, "__cause__", None), getattr(cur, "__context__", None)) if x is not None)
- async def _shutdown_abandoned_app(app) -> None:
- """Release a half-built PTB app's httpx transports after an abandoned init: ``app.shutdown()``
- no-ops when ``_initialized`` was never set, so the request transports are closed directly."""
- if app is None:
- return
- try:
- await app.shutdown()
- except Exception:
- logger.debug("Abandoned Telegram app.shutdown() failed", exc_info=True)
- bot = getattr(app, "bot", None)
- for request in (getattr(bot, "_request", None) if bot is not None else None) or ():
- shutdown = getattr(request, "shutdown", None)
- if shutdown is None:
- continue
- try:
- result = shutdown()
- if asyncio.iscoroutine(result) or asyncio.isfuture(result):
- await result
- except Exception:
- logger.debug("Abandoned Telegram request shutdown failed", exc_info=True)
- try:
- from telegram import Update, Bot, Message, InlineKeyboardButton, InlineKeyboardMarkup
- try:
- from telegram import LinkPreviewOptions
- except ImportError:
- LinkPreviewOptions = None
- from telegram.ext import (
- Application, CommandHandler, CallbackQueryHandler, InlineQueryHandler, MessageHandler as TelegramMessageHandler,
- ContextTypes, TypeHandler, filters)
- from telegram.constants import ParseMode, ChatType
- from telegram.request import HTTPXRequest
- TELEGRAM_AVAILABLE = True
- except ImportError:
- TELEGRAM_AVAILABLE = False
- Update = Bot = Message = InlineKeyboardButton = InlineKeyboardMarkup = Application = Any
- CommandHandler = CallbackQueryHandler = InlineQueryHandler = TypeHandler = TelegramMessageHandler = HTTPXRequest = Any
- LinkPreviewOptions = filters = ParseMode = ChatType = None
- # Mock so ContextTypes.DEFAULT_TYPE annotations don't crash class definition without the lib.
- class _MockContextTypes:
- DEFAULT_TYPE = Any
- ContextTypes = _MockContextTypes
- import sys
- from pathlib import Path as _Path
- sys.path.insert(0, str(_Path(__file__).resolve().parents[3]))
- from gateway.authz_mixin import _coerce_allow_set
- from agent.i18n import get_language, t
- from gateway.config import Platform, PlatformConfig
- from gateway.platforms.base import (
- BasePlatformAdapter, ExecApprovalPrompt, SendResult, classify_send_error, unauthorized_action_notice,
- cache_image_from_bytes_async, cache_audio_from_bytes_async, cache_video_from_bytes_async, resolve_proxy_url, SUPPORTED_VIDEO_TYPES,
- SUPPORTED_DOCUMENT_TYPES, SUPPORTED_IMAGE_DOCUMENT_TYPES, _TEXT_INJECT_EXTENSIONS, utf16_len,
- )
- # Telegram truncates ``answerCallbackQuery`` text at 200 chars; ``BotCommand`` descriptions at 256.
- _TOAST_LIMIT = 200
- _BOT_COMMAND_DESCRIPTION_LIMIT = 256
- def _unauthorized() -> str:
- """Every refused button tap answers with the same sentence — resolved per tap so the active
- language applies (never bound at import)."""
- return unauthorized_action_notice(Platform.TELEGRAM)[:_TOAST_LIMIT]
- def _toast(key: str, **kwargs: Any) -> str:
- """``t()`` for ``query.answer(text=...)`` payloads, cut at Telegram's 200-char toast cap."""
- return t(key, **kwargs)[:_TOAST_LIMIT]
- def _bold_label_html(line: str) -> str:
- """HTML-bold the ``Label:`` prefix of a translated line (whole line when it has no colon)."""
- label, sep, rest = line.partition(":")
- if not sep:
- return f"<b>{_html.escape(line)}</b>"
- return f"<b>{_html.escape(label)}:</b>{_html.escape(rest)}"
- # Inbound-media ``kind`` labels are dual-use: the English word stays in the agent-visible observed note,
- # the chat reply shows the catalog translation.
- _MEDIA_KIND_KEYS = {
- "attachment": "platform.telegram.media.kind_attachment", "photo": "platform.telegram.media.kind_photo",
- "voice message": "platform.telegram.media.kind_voice", "audio file": "platform.telegram.media.kind_audio",
- "video file": "platform.telegram.media.kind_video", "video note": "platform.telegram.media.kind_video"}
- from gateway.platforms.event import MessageEvent, MessageType, ProcessingOutcome
- from plugins.platforms.telegram.telegram_entities import expand_link_entities
- from plugins.platforms.telegram.telegram_held_inbound import TelegramHeldInboundMixin
- from plugins.platforms.telegram.telegram_ids import normalize_telegram_chat_id, parse_send_target, parse_telegram_message_id
- from plugins.platforms.telegram.telegram_resolved_prompt import edit_keeping_prompt
- from plugins.platforms.telegram.telegram_topic_transfer import TopicTransferFn, apply_root_dm_topic_transfer
- from plugins.platforms.telegram.telegram_network import (
- SEED_FALLBACK_IPS, TelegramFallbackTransport, discover_fallback_ips, parse_fallback_ip_env, tcp_keepalive_socket_options)
- from plugins.platforms.telegram.telegram_topic_routing import TelegramTopicRoutingMixin
- from utils import env_float, env_int
- _TELEGRAM_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".gif"}
- # Max seconds a send/edit may sleep inline on a flood-control RetryAfter; longer penalties fail
- # closed with ``flood_control:{wait}`` so the caller's retry machinery owns the wait.
- # Longer server penalties fail closed with a ``flood_control:{wait}`` SendResult so the caller's retry
- # machinery (delivery ledger, streaming fallback) owns the wait instead of the coroutine pinning its worker
- # — a 97-minute penalty on the boot path froze inbound on every platform (#91969).
- _FLOOD_INLINE_WAIT_CAP_SECS = 5.0
- # Shared per-chat outbound budget (#116312): Telegram counts an editMessageText against the
- # SAME per-chat allowance as a sendMessage, but streaming previews used to pace only edits at
- # DEFAULT_STREAMING_EDIT_INTERVAL = 0.8s (1.25 msg/s into one chat before any reply was sent)
- # — that was 83% of measured flood penalties. One shared slot per chat: a SEND waits for its
- # slot (skipping a send would drop a message), an INTERIM edit is skipped (the next tick shows
- # the same text anyway), and the FINAL edit is never gated (the answer itself is never
- # withheld). Tunable: validated in production by the issue reporter at 0 flood events.
- _TELEGRAM_CHAT_OUTBOUND_BUDGET_SECS = 1.0
- def _flood_cap_result(wait: float) -> SendResult:
- """The shared fail-closed SendResult for an over-cap flood wait."""
- return SendResult(success=False, error=f"flood_control:{wait}", retry_after=float(wait))
- _TELEGRAM_IMAGE_MIME_TO_EXT = {"image/png": ".png", "image/jpeg": ".jpg", "image/jpg": ".jpg", "image/webp": ".webp", "image/gif": ".gif"}
- _TELEGRAM_IMAGE_EXT_TO_MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif"}
- def _coerce_duration_seconds(value: Any) -> Optional[int]:
- """Round a raw length to whole positive seconds, or None if unusable."""
- try:
- secs = round(float(value))
- except (TypeError, ValueError):
- return None
- return secs if secs > 0 else None
- def _probe_voice_duration_seconds(path: str) -> Optional[int]:
- """Best-effort whole-second audio length (wave → mutagen → ffprobe; None if unreadable).
- Telegram renders long clips as 0:00 without an explicit duration. Blocking: use ``to_thread``."""
- if os.path.splitext(path)[1].lower() == ".wav":
- try:
- import wave
- with wave.open(path, "rb") as wf:
- rate = wf.getframerate() or 0
- secs = _coerce_duration_seconds(wf.getnframes() / float(rate)) if rate else None
- if secs is not None:
- return secs
- except Exception:
- pass
- try:
- import mutagen
- secs = _coerce_duration_seconds(getattr(getattr(mutagen.File(path), "info", None), "length", None))
- if secs is not None:
- return secs
- except Exception:
- pass
- try:
- import shutil
- import subprocess
- if shutil.which("ffprobe"):
- proc = subprocess.run(
- ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", path],
- stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=5, check=False)
- if proc.returncode == 0:
- return _coerce_duration_seconds(proc.stdout.strip())
- except Exception:
- pass
- return None
- def _probe_video_geometry(path: str) -> dict[str, int]:
- """``{"width", "height", "duration"}`` for a local video; ``{}`` when ffprobe can't read it.
- Telegram runs its own video processing only for uploads under roughly 10 MB; above that it
- stores the file as an unprocessed ``320x320`` video with ``duration=0``, so the message must
- carry the real geometry or clients draw a square tile for any aspect ratio.
- """
- try:
- import shutil
- import subprocess
- if not shutil.which("ffprobe"):
- return {}
- proc = subprocess.run(
- ["ffprobe", "-v", "error", "-select_streams", "v:0",
- "-show_entries", "stream=width,height", "-show_entries", "format=duration",
- "-of", "json", path],
- stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=20, check=False)
- if proc.returncode != 0:
- return {}
- blob = json.loads(proc.stdout or "{}")
- streams = blob.get("streams") or []
- if not streams:
- return {}
- geometry = {"width": int(streams[0]["width"]), "height": int(streams[0]["height"])}
- duration = _coerce_duration_seconds((blob.get("format") or {}).get("duration"))
- if duration:
- geometry["duration"] = duration
- return geometry
- except Exception:
- logger.debug("[Telegram] video geometry probe failed for %s", path, exc_info=True)
- return {}
- def _video_thumbnail_jpeg(path: str, duration: Optional[int]) -> Optional[str]:
- """Write a 320px-wide JPEG frame for Telegram's ``thumbnail`` field; None on failure.
- Telegram keeps a supplied thumbnail for the uploads it did not process itself — without one the
- chat shows a square placeholder tile until the video is opened.
- """
- out = None
- try:
- import shutil
- import subprocess
- import tempfile
- if not shutil.which("ffmpeg"):
- return None
- seek = max(1, int((duration or 3) * 0.25))
- fd, out = tempfile.mkstemp(suffix=".jpg", prefix="hermes-tg-thumb-")
- os.close(fd)
- proc = subprocess.run(
- ["ffmpeg", "-y", "-ss", str(seek), "-i", path, "-frames:v", "1",
- "-vf", "scale=320:-2", "-q:v", "6", out],
- stdin=subprocess.DEVNULL, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, check=False)
- if proc.returncode != 0 or not os.path.getsize(out):
- with contextlib.suppress(OSError):
- os.remove(out)
- return None
- return out
- except Exception:
- logger.debug("[Telegram] video thumbnail extraction failed for %s", path, exc_info=True)
- if out:
- with contextlib.suppress(OSError):
- os.remove(out)
- return None
- def telegram_deps_present() -> bool:
- """PASSIVE registry ``check_fn``: is python-telegram-bot importable? Never installs
- (``check_telegram_requirements`` is the active ``ensure_deps_fn``).
- Registry ``check_fn`` — called from status displays and config loading, so it must never install
- anything. The ACTIVE lazy-installer (``check_telegram_requirements``) is registered as
- ``ensure_deps_fn`` and runs from ``create_adapter()`` when this returns False (#79812).
- """
- return TELEGRAM_AVAILABLE
- def check_telegram_requirements() -> bool:
- """Check if Telegram dependencies are available.
- If python-telegram-bot is missing, attempts to lazy-install it via
- ``pm.ensure_import("telegram")``. After a successful
- install, re-imports the SDK and flips ``TELEGRAM_AVAILABLE`` to True
- so the adapter's class-level type aliases get rebound.
- """
- global TELEGRAM_AVAILABLE, Update, Bot, Message, InlineKeyboardButton
- global InlineKeyboardMarkup, LinkPreviewOptions, Application
- global CommandHandler, CallbackQueryHandler, InlineQueryHandler, TelegramMessageHandler
- global ContextTypes, filters, ParseMode, ChatType, HTTPXRequest, TypeHandler
- if TELEGRAM_AVAILABLE:
- return True
- try:
- from pm import ensure_import as _lazy_ensure
- _lazy_ensure("telegram")
- except Exception:
- return False
- try:
- import importlib
- _tg, _ext, _const, _req = (
- importlib.import_module(m) for m in ("telegram", "telegram.ext", "telegram.constants", "telegram.request"))
- Update, Bot, Message, InlineKeyboardButton, InlineKeyboardMarkup = (
- getattr(_tg, n) for n in ("Update", "Bot", "Message", "InlineKeyboardButton", "InlineKeyboardMarkup"))
- LinkPreviewOptions = getattr(_tg, "LinkPreviewOptions", None)
- Application, CommandHandler, CallbackQueryHandler, InlineQueryHandler, TelegramMessageHandler = (
- getattr(_ext, n) for n in ("Application", "CommandHandler", "CallbackQueryHandler", "InlineQueryHandler", "MessageHandler"))
- ContextTypes, filters, TypeHandler = _ext.ContextTypes, _ext.filters, _ext.TypeHandler
- ParseMode, ChatType = _const.ParseMode, _const.ChatType
- HTTPXRequest = _req.HTTPXRequest
- except (ImportError, AttributeError):
- return False
- TELEGRAM_AVAILABLE = True
- return True
- # Every char MarkdownV2 requires backslash-escaped outside code spans/fences.
- _MDV2_ESCAPE_RE = re.compile(r'([_*\[\]()~`>#\+\-=|{}.!\\])')
- def _escape_mdv2(text: str) -> str:
- """Escape Telegram MarkdownV2 special characters with a preceding backslash."""
- return _MDV2_ESCAPE_RE.sub(r'\\\1', text)
- def _strip_mdv2(text: str) -> str:
- """Strip MarkdownV2 escapes and formatting markers for the plain-text fallback."""
- cleaned = re.sub(r'\\([_*\[\]()~`>#\+\-=|{}.!\\])', r'\1', text) # escape backslashes
- cleaned = re.sub(r'\*\*([^*]+)\*\*', r'\1', cleaned) # **bold** BEFORE MarkdownV2 *bold*
- cleaned = re.sub(r'\*([^*]+)\*', r'\1', cleaned)
- cleaned = re.sub(r'(?<!\w)_([^_]+)_(?!\w)', r'\1', cleaned) # italic; word-bounded so snake_case survives
- cleaned = re.sub(r'~([^~]+)~', r'\1', cleaned) # strikethrough
- cleaned = re.sub(r'\|\|([^|]+)\|\|', r'\1', cleaned) # spoiler
- return cleaned
- _CHUNK_INDICATOR_ON_FENCE_RE = re.compile(r'(?m)^``` (?P<indicator>(?:\\)?\(\d+/\d+(?:\\)?\))$')
- def _separate_chunk_indicator_from_fence(text: str) -> str:
- """Move a ``(N/M)`` chunk marker that ``truncate_message()`` appended to a synthesized closing
- fence onto its own line — Telegram rejects ````` \\(1/2\\)`` as a fence."""
- return _CHUNK_INDICATOR_ON_FENCE_RE.sub(r'```\n\g<indicator>', text)
- # MarkdownV2 has no table syntax, so pipe tables become bullet groups via convert_table_to_bullets().
- from gateway.platforms.helpers import (
- TABLE_SEPARATOR_RE as _TABLE_SEPARATOR_RE, compile_mention_patterns, convert_table_to_bullets as _wrap_markdown_tables)
- # Rich-message regions whose internal newlines must stay bare (Telegram renders them natively):
- # fenced code blocks OR GFM pipe-table blocks (header row, delimiter row, data rows).
- _RICH_PROTECTED_REGION_RE = re.compile(
- r'(?:```[^\n]*\n[\s\S]*?```)' # fenced code block
- r'|(?:^[^\n]*\|[^\n]*\n' # table header row (has a pipe)
- r'[ \t]*\|?[ \t]*:?-+:?[ \t]*(?:\|[ \t]*:?-+:?[ \t]*)+\|?[ \t]*' # delimiter
- r'(?:\n[^\n]*\|[^\n]*)*)', # data rows (newline-led, trailing \n left for prose)
- re.MULTILINE)
- def _rich_normalize_linebreaks(text: str) -> str:
- """Convert lone ``\\n`` (a Markdown soft break) to hard breaks for sendRichMessage; ``\\n\\n``,
- fenced code and pipe tables are left untouched."""
- if not text or '\n' not in text:
- return text
- out: list[str] = []
- pos = 0
- for m in _RICH_PROTECTED_REGION_RE.finditer(text):
- out.append(re.sub(r'(?<!\n)\n(?!\n)', ' \n', text[pos:m.start()]))
- out.append(m.group(0)) # protected region kept verbatim
- pos = m.end()
- out.append(re.sub(r'(?<!\n)\n(?!\n)', ' \n', text[pos:]))
- return ''.join(out)
- # Internal safety bounds (not user knobs): no reconnect/teardown path may hang on a dead CLOSE-WAIT
- # socket PTB's polling task is blocked on in epoll.
- _UPDATER_STOP_TIMEOUT = 15.0 # `await updater.stop()`, applied identically at every site
- _DISCONNECT_STEP_TIMEOUT = 2.0 # other disconnect() steps: short, so a swallowed cancel can't burn the fatal budget
- _UPDATER_START_TIMEOUT = 30.0 # start_polling() can hang on a degraded pool after a drain
- # Initial connect is unhealthy until getUpdates completes one round trip; bootstrap fails closed so
- # GatewayRunner disposes the adapter and retries fresh.
- # Per-step bound for disconnect() awaits that are not updater.stop() itself. Kept short so a
- # cancellation-swallowing lifecycle/PTB close cannot burn the gateway's whole fatal-handler budget before
- # the reconnect queue is useful (#80598). updater.stop() keeps the longer _UPDATER_STOP_TIMEOUT.
- # start_polling() can also hang when the connection pool is in a degraded state after
- # _drain_polling_connections(), particularly when both primary and fallback Telegram endpoints are
- # unreachable. Bounding start_polling() prevents the reconnect ladder from stalling indefinitely and allows
- # the heartbeat loop to trigger its own recovery path. Refs: NousResearch/hermes-agent#59614
- _INITIAL_POLLING_PROGRESS_TIMEOUT = 60.0
- # Bounded drain (shutdown()/initialize() of the getUpdates request) so a wedged socket can't freeze
- # _polling_error_task and gate every escalation path behind its in-flight guard.
- # shutdown()/initialize() on the getUpdates httpx request close and rebuild the connection pool. When a
- # connection is wedged on a stale CLOSE-WAIT socket that close can block forever, hanging
- # _drain_polling_connections() and freezing the whole reconnect ladder (the tracked _polling_error_task
- # never completes, so every escalation path stays gated behind its in-flight guard). Bound the drain so the
- # ladder always advances toward the fatal-restart escalation. Matches _UPDATER_STOP_TIMEOUT. Refs:
- # NousResearch/hermes-agent#66377
- _DRAIN_TIMEOUT = 15.0
- # Wedged-recovery watchdog: healthy worst case is stop + 2x drain + start + 60s backoff ≈ 135s, so
- # 300s in flight is unambiguously stuck and the heartbeat force-escalates.
- # Every recovery path (the reconnect ladder's re-entry, the pending-update probe, PTB's error callback)
- # gates new recovery on ``_polling_error_task.done()``; if that task ever wedges on a hung await that no
- # local bound covers, the whole gateway goes silently deaf with nothing retrying. The heartbeat loop
- # force-escalates a recovery task that stays in-flight far longer than any healthy ladder attempt could take
- # — stop (_UPDATER_STOP_TIMEOUT) + drain (2x_DRAIN_TIMEOUT) + start (_UPDATER_START_TIMEOUT) + max backoff
- # (60s) is ~135s, so 300s is unambiguously stuck. See #66377.
- _POLLING_ERROR_TASK_STUCK_TIMEOUT = 300.0
- _POLLING_PROGRESS_TIMEOUT = 60.0 # generation unhealthy until getUpdates returns; exceeds one idle long-poll
- # Telegram answers a long-poll within ~50s; no round-trip for ~3x that while get_me() is healthy and
- # nothing is queued means a consumer wedged on a socket that never raises (CLOSE-WAIT behind a route flip).
- # Telegram holds a long-poll open for at most ~50s before answering (empty or not), so a healthy idle poller
- # completes a getUpdates round-trip well inside this window. If no round-trip has completed for longer than
- # this — while get_me() on the general request path stays healthy and no updates are queued server-side —
- # the long-poll consumer is wedged on a socket that never raises (CLOSE-WAIT behind a TUN/proxy route flip,
- # #92991) and no other probe can see it. ~3x the worst-case poll window leaves ample margin against false
- # positives while still recovering within a few heartbeat intervals.
- _POLLING_STALL_TIMEOUT = 150.0
- # Ingress dispatch stall (#102260, #130407): the transport probes prove getUpdates round-trips complete, not
- # that PTB's dispatcher ever handed the fetched updates to a handler. A backlog with no dispatch progress for
- # four heartbeats (360s) hands the adapter to the supervisor for a rebuild (an in-place restart keeps the
- # wedged dispatcher). Sized past _POLLING_ERROR_TASK_STUCK_TIMEOUT (300s), the bound on a slow handler: updates
- # of one chat still dispatch in order (PerChatUpdateProcessor), so e.g. a sticker vision_analyze (120s default)
- # must not trip it. Re-arms on progress.
- _INGRESS_DISPATCH_STALL_HEARTBEATS = 4
- # sendVideo transcodes before answering, outlasting the 20s read timeout; also how long a user waits
- # to hear the attachment failed, so kept modest.
- _MEDIA_SEND_READ_TIMEOUT = 60.0
- # Text send used to hang forever on a shielded httpcore socket (NordVPN/Telegram sticky IP).
- # A wedged send froze getUpdates on the same loop. Bound every text send/edit and media upload.
- _TEXT_SEND_DEADLINE = 30.0
- # Wall-clock cap on one media upload (whole request: pool wait + connect + body upload + server
- # processing). NOT `_MEDIA_SEND_READ_TIMEOUT`: that is httpx's per-phase stall budget (time-to-first-byte
- # after the body is sent), whereas this bounds the entire call, so it must leave room for bandwidth. The
- # Bot API upload cap is 50 MB; at ~2 Mbit/s that is ~200 s, plus connect (10 s) and sendVideo transcoding
- # (up to the 60 s read timeout) — 300 s covers it. It is also >2x the sum of the per-phase httpx budgets
- # (pool 8 + connect 10 + media_write 60 + read 60 = 138 s), so it only fires when a shielded socket has
- # stopped raising at all, never on a merely slow link.
- # On expiry `run_bounded_async` cancels and then abandons the upload task (never awaited), inside
- # `_chat_send_lock`: the lock is released while the abandoned task drains. Awaiting the cancel with a grace
- # period would re-hang the lock on exactly the wedged socket this bounds, so the rare late landing is
- # accepted; httpx's own timeouts free the pool slot.
- _MEDIA_SEND_DEADLINE = 300.0
- _POLLING_GENERATION_CONTEXT: ContextVar[Optional[int]] = ContextVar("telegram_polling_generation", default=None)
- class _PollingLifecycleAbort(RuntimeError):
- """Internal control flow for polling startup fenced by teardown."""
- class _PollingStallError(RuntimeError):
- """A confirmed getUpdates stall (watchdog or post-reconnect verifier), as opposed to a transport drop.
- Typed so the recovery ladder can hand the adapter to the supervisor instead of classifying log text:
- restarting the same Updater cannot heal a wedged long-poll consumer whose stop() did not quiesce (#113618).
- """
- class TelegramAdapter(TelegramHeldInboundMixin, TelegramTopicRoutingMixin, BasePlatformAdapter):
- """Telegram bot adapter: users/groups, MarkdownV2 replies, forum topics, media."""
- # Bound for the per-(chat_id, status_key) status-message cache; FIFO half-trim on overflow.
- _STATUS_MESSAGE_IDS_MAX = 2000
- MAX_MESSAGE_LENGTH = 4096
- supports_code_blocks = True # MarkdownV2 renders fenced code blocks
- splits_long_messages = True # send() chunks via truncate_message(MAX_MESSAGE_LENGTH)
- RICH_MESSAGE_MAX_CHARS = 32768 # Bot API 10.1 rich cap; above it use legacy chunking
- _SPLIT_THRESHOLD = 4000 # chunk near this length ⇒ a client-side split continuation is almost certain
- MEDIA_GROUP_WAIT_SECONDS = 0.8
- HELD_INBOUND_MAX = 64 # inbound events held across a disconnect window; oldest dropped first
- _GENERAL_TOPIC_THREAD_ID = "1"
- # send() can race a disconnect blip; failing "Not connected" (retryable=False) parks the answer in the
- # delivery ledger until next boot, so wait briefly for _bot (or a replacement adapter) instead.
- _RECONNECT_WAIT_SECONDS = 15.0
- _RECONNECT_POLL_INTERVAL = 0.5
- # Large-image compression for Telegram photo sends. Behind an HTTP proxy the PTB
- # media_write_timeout is easily exceeded by raw PNGs > 1-2MB; pre-compressing to
- # progressive JPEG keeps the upload well under the timeout and reduces bandwidth.
- _IMG_JPEG_QUALITY = 85
- _IMG_MAX_DIMENSION = 1600 # above this, resize before JPEG
- _IMG_COMPRESS_THRESHOLD_BYTES = 1_048_576 # 1MB
- # edit_message applies MarkdownV2 only on finalize=True; without this flag stream_consumer skips
- # the final edit when raw text is unchanged.
- # Fixes #25710.
- REQUIRES_EDIT_FINALIZE: bool = True
- FALLBACK_ON_FINAL_EDIT_FLOOD: bool = True # retrying a final edit burns the same flood budget
- RESEND_FINAL_ON_EMPTY_STREAM_FALLBACK: bool = True # a failed final edit may leave a partial preview
- # Adaptive text-batch ingress ("feels instant"): ≤320 codepoints settle in ~180ms, ≤1024 in ~240ms,
- # longer waits the configured cap; always clamped to ``_text_batch_delay_seconds``.
- _TEXT_BATCH_FAST_LEN = 320
- _TEXT_BATCH_FAST_DELAY_S = 0.18
- _TEXT_BATCH_SHORT_LEN = 1024
- _TEXT_BATCH_SHORT_DELAY_S = 0.24
- @staticmethod
- def _env_float_clamped(name: str, default: float, *, min_value: Optional[float] = None, max_value: Optional[float] = None) -> float:
- """Read a float env var; non-finite → default; clamp to bounds (safe for asyncio.sleep)."""
- import math
- raw = os.getenv(name)
- try:
- value = float(raw) if raw is not None else float(default)
- except (TypeError, ValueError):
- value = float(default)
- if not math.isfinite(value):
- value = float(default)
- if min_value is not None:
- value = max(value, min_value)
- if max_value is not None:
- value = min(value, max_value)
- return value
- @property
- def _teardown_started(self) -> bool:
- """True once disconnect() fenced polling (tolerates object.__new__ test adapters)."""
- return getattr(self, "_polling_teardown_started", False)
- @property
- def message_len_fn(self):
- """Telegram measures message length in UTF-16 code units."""
- return utf16_len
- def __init__(self, config: PlatformConfig):
- super().__init__(config, Platform.TELEGRAM)
- extra = self.config.extra
- self._app: Optional[Application] = None
- self._seen_update_ids: dict = {}
- self._inflight_update_ids: dict = {}
- self._update_admission = None
- # Completed update IDs survive adapter replacement and restarts (update_admission.py).
- # Resolved now: secondary profiles construct adapters inside their own home scope.
- from hermes_constants import get_hermes_home
- self._update_receipt_dir = get_hermes_home()
- self._update_receipts_loaded: set = set()
- self._update_receipts_dirty: set = set()
- self._update_receipt_flush: Optional[asyncio.Task] = None
- self._bot: Optional[Bot] = None
- self._webhook_mode: bool = False
- self._mention_patterns = self._compile_mention_patterns()
- self._reply_to_mode: str = getattr(config, 'reply_to_mode', 'first') or 'first'
- self._disable_link_previews: bool = self._coerce_bool_extra("disable_link_previews", False)
- # Bot API 10.1 Rich Messages render what MarkdownV2 degrades (tables, task lists, <details>, block
- # math). Opt-in: current clients make rich messages hard to copy as plain text. rich_drafts is a
- # separate opt-in (Desktop can leave rich draft frames overlaid): off keeps native draft transport
- # but skips rich draft rendering; the final reply still lands via sendRichMessage.
- self._rich_messages_enabled: bool = self._coerce_bool_extra("rich_messages", False)
- # CJK stays on legacy MarkdownV2 by default (Desktop/macOS garble, #47653); opt-in for unaffected clients.
- self._allow_cjk_rich_messages: bool = self._coerce_bool_extra("allow_cjk_rich_messages", False)
- self._rich_drafts_enabled: bool = self._coerce_bool_extra("rich_drafts", False)
- self._rich_send_disabled = self._rich_draft_disabled = False # latched after a capability failure
- # Transient sendChatAction failures recur on every keep-typing tick; back off per chat.
- self._telegram_typing_cooldown_until: dict[str, float] = {}
- self._telegram_typing_cooldown_seconds: float = self._coerce_float_extra(
- "typing_cooldown_seconds", 30.0, min_value=1.0, max_value=300.0)
- # Post-send typing re-arm: scheduled, deduped and rate-limited per chat. Awaiting a
- # sendChatAction round-trip on the send path shares the loop with the getUpdates long-polls,
- # and under concurrent streaming it starved them until they rotted into CLOSE-WAIT (#111727).
- self._telegram_typing_retrigger_tasks: dict[str, asyncio.Task] = {}
- self._telegram_typing_retrigger_at: dict[str, float] = {}
- # Telegram's bubble lasts ~5s and _keep_typing already refreshes every 2s, so the re-arm only
- # has to cover the gap left by a landed message. 0 restores a call per intermediate send.
- self._telegram_typing_retrigger_interval: float = self._coerce_float_extra(
- "typing_retrigger_min_interval_seconds", 2.0, min_value=0.0, max_value=30.0)
- # Buffer album/photo bursts into a single MessageEvent instead of self-interrupting turns.
- self._media_batch_delay_seconds = env_float("HERMES_TELEGRAM_MEDIA_BATCH_DELAY_SECONDS", 0.8)
- self._pending_photo_batches: dict[str, MessageEvent] = {}
- self._pending_photo_batch_tasks: dict[str, asyncio.Task] = {}
- self._media_group_events: dict[str, MessageEvent] = {}
- self._media_group_tasks: dict[str, asyncio.Task] = {}
- # Flushes past their pop, by (family, key): a newer chunk must not cancel them (teardown still does).
- self._committed_flush_tasks: dict[tuple[str, str], asyncio.Task] = {}
- # Aggregate client-side splits of long messages into one MessageEvent; bounds are conservative
- # for Telegram's ~1 edit/s flood envelope.
- self._text_batch_delay_seconds = self._env_float_clamped(
- "HERMES_TELEGRAM_TEXT_BATCH_DELAY_SECONDS", self._TEXT_BATCH_DEFAULT_DELAY_S,
- min_value=0.08, max_value=self._TEXT_BATCH_MAX_DELAY_S)
- self._text_batch_split_delay_seconds = self._env_float_clamped(
- "HERMES_TELEGRAM_TEXT_BATCH_SPLIT_DELAY_SECONDS", self._TEXT_BATCH_DEFAULT_SPLIT_DELAY_S,
- min_value=self._text_batch_delay_seconds, max_value=self._TEXT_BATCH_MAX_SPLIT_DELAY_S)
- self._drop_delayed_deliveries = False
- # Held across disconnect: PTB advances the offset before our drop-guard runs, so Telegram won't
- # redeliver — dropping is permanent loss (see _hold_inbound_event).
- self._held_inbound_events: list[MessageEvent] = []
- self._held_inbound_redispatch_task: Optional[asyncio.Task] = None
- self._polling_error_task: Optional[asyncio.Task] = None
- self._polling_progress_verifier_task: Optional[asyncio.Task] = None
- self._polling_heartbeat_task: Optional[asyncio.Task] = None
- self._bot_identity_refresh_task: Optional[asyncio.Task] = None
- self._post_connect_task: Optional[asyncio.Task] = None # command menu + DM topics, off the connect path
- self._polling_conflict_count = self._polling_network_error_count = self._polling_generation = 0
- self._polling_conflict_recovery_generation: Optional[int] = None
- self._polling_progress_event = asyncio.Event()
- self._polling_progress_accepting = self._polling_teardown_started = False
- self._polling_error_callback_ref = None
- # Stall watchdog: generation start and last successful getUpdates (None = unknown).
- # Monotonic timestamps for the polling stall watchdog (#92991): when the current polling generation
- # began, and when the last successful getUpdates round-trip completed.
- self._polling_generation_started_monotonic: Optional[float] = None
- self._polling_last_progress_monotonic: Optional[float] = None
- # Ingress accounting (#102260): received (getUpdates wire) vs dispatched (PTB admission).
- self._updates_received_total: int = 0
- self._updates_dispatched_total: int = 0
- self._ingress_dispatched_seen: int = 0
- self._ingress_stalled_heartbeats: int = 0
- # Live @username: PTB caches getMe() at initialize() and only rewrites it inside get_me(), so a
- # BotFather rename leaves self._bot.username stale; routing reads _current_bot_username().
- self._bot_username_observed: Optional[str] = None
- # None = never checked. Must NOT be 0.0: compared against time.monotonic(), which on a fresh host
- # starts near zero, so 0.0 would suppress the first refresh for a TTL.
- self._bot_identity_checked_at: Optional[float] = None
- # Consecutive heartbeat probes seeing queued updates the poller isn't consuming (get_me() can't
- # see a wedged getUpdates) / finding the updater stopped with no reconnect in flight; escalate after two.
- self._polling_pending_stuck_count = self._polling_not_running_count = 0
- # Degraded until getUpdates makes progress; while True, send() short-circuits to failure so callers
- # (cron live-adapter branch) fall through to standalone delivery.
- # Consecutive heartbeat probes that saw queued updates the running poller is not consuming. get_me()
- # can't see this — the send path is healthy while the getUpdates consumer is wedged — so the
- # heartbeat also probes get_webhook_info().pending_update_count and escalates to recovery after two
- # consecutive stuck probes (#42909).
- # Consecutive heartbeat probes that found the updater stopped entirely (running=False) while we are
- # in polling mode with no reconnect in flight. Distinct from the wedged-but-running case above: the
- # long-poll task is simply gone, so neither the connectivity probe nor PTB's error_callback ever
- # fires and the gateway silently stops receiving messages with the process still alive (#55769).
- self._send_path_degraded: bool = False
- self._general_request_drain_lock = asyncio.Lock()
- self._dm_topics: dict[str, int] = {} # topic_name -> message_thread_id
- # (chat_id, sender_id) -> (new topic id, last Message.date, monotonic record time) for client-created
- # DM topics whose thread-less opener may still arrive (#136567); see TelegramTopicRoutingMixin.
- self._pending_created_topics: dict[tuple[str, str], tuple[str, float, float]] = {}
- self._forum_command_registered: set[int] = set() # forum chats with commands registered
- self._command_menu_fingerprint: str = "" # language + menu payload last pushed via set_my_commands
- self._forum_lock = asyncio.Lock()
- # Status indicator: bot short description "Online"/"Offline" on connect/clean disconnect. Off by
- # default because it mutates the GLOBAL profile; opt in via extra.status_indicator.
- self._status_indicator_enabled: bool = bool(extra.get("status_indicator", False))
- self._status_online_text: str = str(extra.get("status_online", "Online"))
- self._status_offline_text: str = str(extra.get("status_offline", "Offline"))
- # Cold-boot queue: drop server-side pending updates on first boot (default True,
- # preserves historical behaviour). Set extra.drop_pending_on_cold_boot: false to
- # receive messages sent while the gateway was offline (e.g. nightly-off hosts).
- # Watcher reconnects always preserve the queue regardless of this setting.
- self._drop_pending_on_cold_boot: bool = self._coerce_bool_extra("drop_pending_on_cold_boot", True)
- self._dm_topics_config: list[dict[str, Any]] = extra.get("dm_topics", [])
- # chat_ids with DM topics configured (O(1) root-DM ignore check)
- self._dm_topic_chat_ids: set[str] = {str(e["chat_id"]) for e in self._dm_topics_config if "chat_id" in e}
- # getFile cap: 20MB on the public Bot API, 2GB on a local telegram-bot-api (base_url).
- self._max_doc_bytes: int = 2 * 1024 * 1024 * 1024 if extra.get("base_url") else 20 * 1024 * 1024
- self._model_picker_state: dict[str, dict] = {} # per-chat interactive picker state
- self._choice_picker_state: dict[str, dict] = {}
- self._approval_state: dict[int, str] = {} # message_id → session_key
- self._approval_request_ids: dict[int, str] = {} # message_id → tools.approval request id
- self._slash_confirm_state: dict[str, str] = {} # confirm_id → session_key
- self._clarify_state: dict[str, str] = {} # clarify_id → session_key
- # "important" (default): only final responses, approvals and slash confirmations notify;
- # "all": every message notifies (display.platforms.telegram.notifications).
- self._notifications_mode: str = "important"
- # send_or_update_status(): {(chat_id, status_key) -> message_id} so repeat calls edit in place.
- # send_or_update_status() bookkeeping: {(chat_id, status_key) -> bot message_id} Tracks status
- # bubbles owned by this adapter so subsequent calls with the same key edit the same message instead
- # of appending new ones (#30045).
- self._status_message_ids: dict[tuple, str] = {}
- # Last truncated mid-stream preview per (chat_id, message_id): past the 4096 cap every edit
- # truncates to the SAME text, and resending burns flood budget. Dropped on finalize.
- self._last_overflow_preview: dict[tuple, str] = {}
- @property
- def send_path_degraded(self) -> bool:
- # True from polling-generation start until the first getUpdates
- # round-trip is proven (_record_polling_progress), and again at every
- # polling-death site. getattr: tests build adapters via object.__new__().
- return bool(getattr(self, "_send_path_degraded", False))
- def _mark_connected(self) -> None:
- self._drop_delayed_deliveries = False
- super()._mark_connected()
- self._schedule_held_inbound_redispatch() # PTB will not redeliver these events
- def _mark_disconnected(self) -> None:
- self._drop_delayed_deliveries = True
- super()._mark_disconnected()
- def _set_fatal_error(self, code: str, message: str, *, retryable: bool) -> None:
- self._drop_delayed_deliveries = True
- super()._set_fatal_error(code, message, retryable=retryable)
- # Permanent fatal: no reconnect will drain, so discard the hold queue (later holds are refused).
- # Discard the hold queue now and refuse further holds (teardown salvage / late enqueue must not
- # re-populate a queue that can never drain — review #83878).
- if not retryable:
- held = getattr(self, "_held_inbound_events", None)
- n = len(held) if held else 0
- if held:
- held.clear()
- if n:
- logger.warning("[Telegram] Non-retryable fatal (%s); discarding %d held inbound message(s)", code, n)
- def _is_permanent_fatal(self) -> bool:
- """True after non-retryable fatal — holds must discard, not queue."""
- if not getattr(self, "_fatal_error_code", None):
- return False
- return not bool(getattr(self, "_fatal_error_retryable", True))
- def _replacement_telegram_adapter(self) -> "Optional[TelegramAdapter]":
- """Live adapter if the reconnect watcher replaced us in ``runner.adapters`` (an in-flight
- ``send()`` still holds the old instance whose ``_bot`` stays None)."""
- runner = getattr(self, "gateway_runner", None)
- adapters = getattr(runner, "adapters", None) or {}
- live = adapters.get(self.platform)
- if live is not None and live is not self and getattr(live, "_bot", None):
- return live
- return None
- async def _wait_for_reconnection(self) -> bool:
- """Wait for ``_bot`` or a replacement adapter; False on expiry or permanent fatal."""
- if self._bot or self._replacement_telegram_adapter() is not None:
- return True
- if self._is_permanent_fatal():
- return False
- wait_s = float(getattr(self, "_RECONNECT_WAIT_SECONDS", 15.0))
- poll_s = float(getattr(self, "_RECONNECT_POLL_INTERVAL", 0.5))
- logger.info("[%s] Not connected — waiting for reconnection (up to %.0fs)", self.name, wait_s)
- waited = 0.0
- while waited < wait_s:
- await asyncio.sleep(poll_s)
- waited += poll_s
- if self._is_permanent_fatal():
- return False
- if self._bot or self._replacement_telegram_adapter() is not None:
- logger.info("[%s] Reconnected after %.1fs", self.name, waited)
- return True
- logger.warning("[%s] Still not connected after %.0fs", self.name, wait_s)
- return False
- def _should_drop_delayed_delivery(self) -> bool:
- """True once teardown/fatal started: delayed flushes must not dispatch onto a torn-down session.
- Callers must NOT destroy the event (PTB already advanced the offset) — hold and redispatch."""
- return bool(getattr(self, "_drop_delayed_deliveries", False))
- def _notification_kwargs(self, metadata: Optional[dict[str, Any]]) -> dict[str, Any]:
- """In "important" mode return disable_notification=True unless ``notify`` or ``is_approval_prompt`` (#132516)."""
- if getattr(self, "_notifications_mode", "important") != "important" or (metadata or {}).get("notify") \
- or (metadata or {}).get("is_approval_prompt"):
- return {}
- return {"disable_notification": True}
- @staticmethod
- def _normalize_chat_type(chat_type: Any, *, is_forum: bool) -> str:
- """Telegram chat type → gateway chat type (``private``→``dm``, ``supergroup``→forum/group)."""
- normalized = str(chat_type or "dm").strip().lower() or "dm"
- if normalized == "private":
- return "dm"
- if normalized == "supergroup":
- return "forum" if is_forum else "group"
- return normalized
- def _legacy_runner_auth_fn(self):
- """``runner._is_user_authorized`` resolved off the bound handler (bare-adapter tests, direct
- embedding); None under multiplex where the handler is a profile closure."""
- # Resolve through the runner's full auth chain (platform + group allowlists, pairing store,
- # allow-all flags). Prefer the platform-bound callback registered via set_authorization_check: it
- # routes to GatewayRunner._is_user_authorized AND survives multiplex handler wrapping, whereas the
- # bound-handler __self__ lookup is None when the primary handler is a profile closure — which
- # silently dropped the chat allowlist and default-denied allowlisted group members under
- # multiplex_profiles (#87132). Fall back to the bound handler for setups without a registered
- # callback.
- runner = getattr(getattr(self, "_message_handler", None), "__self__", None)
- auth_fn = getattr(runner, "_is_user_authorized", None)
- return auth_fn if callable(auth_fn) else None
- @staticmethod
- def _env_allowlist_decision(user_id: str) -> Optional[bool]:
- """TELEGRAM_ALLOWED_USERS decision; None when no allowlist is configured."""
- allowed_csv = _scoped_gate_env("TELEGRAM_ALLOWED_USERS").strip()
- if not allowed_csv:
- return None
- allowed_ids = {uid.strip() for uid in allowed_csv.split(",") if uid.strip()}
- return "*" in allowed_ids or user_id in allowed_ids
- def _is_callback_user_authorized(
- self, user_id: str, *, chat_id: Optional[str] = None, chat_type: Optional[str] = None,
- thread_id: Optional[str] = None, user_name: Optional[str] = None) -> bool:
- """Return whether a Telegram inline-button caller may perform gated actions."""
- normalized_user_id = str(user_id or "").strip()
- if not normalized_user_id:
- return False
- normalized_chat_type = self._normalize_chat_type(chat_type, is_forum=thread_id is not None)
- # Preferred: the auth callback GatewayRunner injects (set_authorization_check) → full
- # _is_user_authorized chain; also works for a multiplexed adapter whose _message_handler is a
- # profile closure. getattr tolerates partially-constructed adapters (object.__new__ in tests).
- if getattr(self, "_authorization_check", None) is not None:
- injected = self._is_sender_authorized(
- normalized_user_id, chat_type=normalized_chat_type, chat_id=str(chat_id or normalized_user_id),
- thread_id=str(thread_id) if thread_id is not None else None)
- if injected is not None:
- return injected
- auth_fn = self._legacy_runner_auth_fn()
- if auth_fn is not None:
- try:
- from gateway.session import SessionSource
- source = SessionSource(
- platform=Platform.TELEGRAM, chat_id=str(chat_id or normalized_user_id), chat_type=normalized_chat_type,
- user_id=normalized_user_id, user_name=str(user_name).strip() if user_name else None,
- thread_id=str(thread_id) if thread_id is not None else None)
- return bool(auth_fn(source))
- except Exception:
- logger.debug(
- "[Telegram] Falling back to env-only callback auth for user %s", normalized_user_id, exc_info=True)
- decision = self._env_allowlist_decision(normalized_user_id)
- if decision is None:
- # Fail-closed: no allowlist means deny unless GATEWAY_ALLOW_ALL_USERS is set.
- # The runner auth path in _is_user_authorized() handles GATEWAY_ALLOW_ALL_USERS; this fallback
- # must not silently allow everyone (fixes #24457).
- return _scoped_gate_env("GATEWAY_ALLOW_ALL_USERS").lower() in {"true", "1", "yes"}
- return decision
- def _source_from_message_for_auth(self, message: Message):
- """Build the SessionSource the gateway auth path expects; identity comes from ``from_user``,
- falling back to ``sender_chat`` for channel posts so an unauthorized channel can't inject."""
- from gateway.session import SessionSource
- user = getattr(message, "from_user", None)
- chat = getattr(message, "chat", None)
- user_id = str(getattr(user, "id", "")).strip() or None
- # Carry is_bot so the runner's ``*_ALLOW_BOTS`` branch is reachable, as in build_source.
- is_bot = bool(getattr(user, "is_bot", False)) if user is not None else False
- user_name = str(getattr(user, "username", "") or getattr(user, "full_name", "") or "").strip() or None
- if not user_id: # channel post — authorize the sender chat instead
- sender_chat = getattr(message, "sender_chat", None)
- if sender_chat is not None:
- user_id = str(getattr(sender_chat, "id", "")).strip() or None
- if not user_name:
- user_name = str(getattr(sender_chat, "title", "") or "").strip() or None
- chat_id = str(getattr(chat, "id", "")).strip() or user_id
- thread_id_raw = getattr(message, "message_thread_id", None)
- is_topic_message = bool(getattr(message, "is_topic_message", False))
- is_forum_group = getattr(chat, "is_forum", False) is True
- chat_type = self._normalize_chat_type(
- getattr(chat, "type", "dm"), is_forum=thread_id_raw is not None and (is_topic_message or is_forum_group))
- thread_id = None
- if thread_id_raw is not None and (
- (chat_type == "forum" and (is_topic_message or is_forum_group)) or (chat_type == "dm" and is_topic_message)):
- thread_id = str(thread_id_raw)
- return SessionSource(
- platform=Platform.TELEGRAM, chat_id=chat_id or "", chat_type=chat_type, user_id=user_id,
- user_name=user_name, thread_id=thread_id, is_bot=is_bot)
- def _source_from_reaction_for_auth(self, update):
- """SessionSource for a ``message_reaction`` update's actor (``user`` or ``actor_chat``).
- Raises ``ValueError`` when actor, chat or message identity is absent so the post-auth boundary fails closed."""
- mr = getattr(update, "message_reaction", None)
- if mr is None:
- raise ValueError("gateway_platform_event source extraction requires a message_reaction update")
- user = getattr(mr, "user", None) or getattr(mr, "actor_chat", None)
- chat = getattr(mr, "chat", None)
- user_id = str(getattr(user, "id", "")).strip() or None
- user_name = str(getattr(user, "username", "") or getattr(user, "full_name", "") or getattr(user, "title", "")).strip() or None
- chat_id = str(getattr(chat, "id", "")).strip() or None
- message_id = getattr(mr, "message_id", None)
- if not user_id or not chat_id or message_id is None or not str(message_id).strip():
- raise ValueError("gateway_platform_event reaction requires actor, chat, and message identities")
- # Reactions carry no message_thread_id; is_forum is the only forum signal.
- chat_type = self._normalize_chat_type(getattr(chat, "type", "dm"), is_forum=getattr(chat, "is_forum", False) is True)
- return self.build_source(
- chat_id=chat_id, chat_type=chat_type, user_id=user_id, user_name=user_name, thread_id=None, message_id=str(message_id))
- def _telegram_auth_env_configured(self) -> bool:
- """Return True when Telegram auth env vars make an early decision safe."""
- keys = (
- "TELEGRAM_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_CHATS",
- "TELEGRAM_ALLOW_ALL_USERS", "GATEWAY_ALLOWED_USERS", "GATEWAY_ALLOW_ALL_USERS")
- return any(_scoped_gate_env(key).strip() for key in keys)
- def _should_pass_unauthorized_dm_for_pairing(self, source) -> bool:
- """True when an unauthorized DM must still reach the gateway for an outbound reply
- (``unauthorized_dm_behavior`` resolves to anything but ``ignore`` — a pairing code or a
- one-time decline — incl. an allowlist plus an explicit platform override)."""
- if source.chat_type != "dm":
- return False
- # Bound-handler ``__self__`` is None under multiplex; ``gateway_runner`` survives that wrapping.
- runner = getattr(getattr(self, "_message_handler", None), "__self__", None) or getattr(self, "gateway_runner", None)
- behavior_fn = getattr(runner, "_get_unauthorized_dm_behavior", None)
- if callable(behavior_fn):
- try:
- profile = getattr(source, "profile", None) or getattr(self, "_owner_profile", None)
- return behavior_fn(Platform.TELEGRAM, profile=profile) != "ignore"
- except Exception:
- logger.debug("[Telegram] Failed to resolve unauthorized DM behavior; falling back to adapter-local override", exc_info=True)
- extra = getattr(getattr(self, "config", None), "extra", None) or {}
- return str(extra.get("unauthorized_dm_behavior", "")).strip().lower() in ("pair", "decline")
- def _is_user_authorized_from_message(self, message: Message) -> bool:
- """Intake auth prefilter, run BEFORE batching/event construction/group observation.
- Only rejects when it can make the same context-aware decision the runner would; unknown DMs pass through when
- there is no allowlist or pairing is the unauthorized-DM behavior."""
- source = self._source_from_message_for_auth(message)
- user_id = source.user_id
- # No identity → service message or channel post without sender_chat; defer to message gating.
- if not user_id:
- return True
- authorized: Optional[bool] = None
- # Adapter-level allow_from (DMs) / group_allow_from (groups) are the sole authority if set.
- adapter_allow_from = self.config.extra.get(
- "group_allow_from" if (source.chat_type or "") in ("group", "forum", "channel") else "allow_from")
- if adapter_allow_from is not None:
- allowed = _coerce_allow_set(adapter_allow_from)
- authorized = user_id in allowed or "*" in allowed
- # Instance-level override only (tests): the class method _is_callback_user_authorized is for
- # inline buttons and must not become a user-id-only shortcut for real messages.
- if authorized is None:
- callback_auth = self.__dict__.get("_is_callback_user_authorized")
- if callable(callback_auth):
- with contextlib.suppress(Exception):
- authorized = bool(callback_auth(
- user_id, chat_id=source.chat_id, chat_type=source.chat_type, thread_id=source.thread_id,
- user_name=source.user_name))
- if authorized is None:
- # Runner's full auth chain; prefer the set_authorization_check callback (survives multiplex
- # handler wrapping, unlike bound-handler __self__).
- auth_fn = self._legacy_runner_auth_fn()
- has_callback = getattr(self, "_authorization_check", None) is not None
- if has_callback or auth_fn is not None:
- # No allowlist → unknown DMs must reach pairing, not be default-denied here.
- if not self._telegram_auth_env_configured():
- return True
- decision = self._is_sender_authorized(
- user_id, chat_type=source.chat_type, chat_id=source.chat_id, is_bot=source.is_bot,
- thread_id=source.thread_id) if has_callback else None
- if decision is not None:
- authorized = decision
- elif auth_fn is not None:
- try:
- authorized = bool(auth_fn(source))
- except Exception:
- logger.debug("[Telegram] Falling back to env-only auth for user %s", user_id, exc_info=True)
- if authorized is None:
- authorized = self._env_allowlist_decision(user_id)
- if authorized is None:
- return True
- if authorized:
- return True
- # Unauthorized DM the gateway would pair: forward so pairing can run.
- return self._should_pass_unauthorized_dm_for_pairing(source)
- @classmethod
- def _metadata_thread_id(cls, metadata: Optional[dict[str, Any]]) -> Optional[str]:
- thread_id = (metadata or {}).get("thread_id") or (metadata or {}).get("message_thread_id")
- return str(thread_id) if thread_id is not None else None
- @classmethod
- def _metadata_direct_messages_topic_id(cls, metadata: Optional[dict[str, Any]]) -> Optional[str]:
- topic_id = (metadata or {}).get("direct_messages_topic_id") or (metadata or {}).get("telegram_direct_messages_topic_id")
- return str(topic_id) if topic_id is not None else None
- @classmethod
- def _metadata_reply_to_message_id(cls, metadata: Optional[dict[str, Any]]) -> Optional[int]:
- return parse_telegram_message_id((metadata or {}).get("telegram_reply_to_message_id"))
- @staticmethod
- def _dm_topic_fallback(metadata: Optional[dict[str, Any]]) -> bool:
- """True for Hermes private-chat topic lanes (``telegram_dm_topic_reply_fallback``)."""
- return bool(metadata and metadata.get("telegram_dm_topic_reply_fallback"))
- @classmethod
- def _is_private_dm_topic_send(cls, chat_id: str, thread_id: Optional[str], metadata: Optional[dict[str, Any]]) -> bool:
- if cls._metadata_direct_messages_topic_id(metadata) is not None:
- return cls._dm_topic_fallback(metadata) and cls._metadata_reply_to_message_id(metadata) is not None
- if metadata and metadata.get("telegram_dm_topic_created_for_send"):
- return False
- return bool(thread_id) and cls._dm_topic_fallback(metadata)
- @staticmethod
- def _dm_topic_missing_anchor_error() -> str:
- return "Telegram DM topic delivery requires a reply anchor; refusing to send outside the requested topic"
- @classmethod
- def _reply_to_message_id_for_send(
- cls, reply_to: Optional[str], metadata: Optional[dict[str, Any]] = None, reply_to_mode: Optional[str] = None) -> Optional[int]:
- if (anchor := parse_telegram_message_id(reply_to)) is not None:
- return anchor
- if cls._dm_topic_fallback(metadata) and reply_to_mode != "off":
- return cls._metadata_reply_to_message_id(metadata)
- return None
- @classmethod
- def _thread_kwargs_for_send(
- cls, chat_id: str, thread_id: Optional[str], metadata: Optional[dict[str, Any]] = None,
- reply_to_message_id: Optional[int] = None, reply_to_mode: Optional[str] = None) -> dict[str, Any]:
- """Telegram send kwargs for forum and direct-message topic routing.
- Forum topics use ``message_thread_id``; native Bot API DM topics opt in via explicit ``direct_messages_topic_id``
- metadata; Hermes private-chat topic lanes are marked ``telegram_dm_topic_reply_fallback``. Anchor-less synthetic sends
- prefer the Hermes topic's ``message_thread_id`` (the native DM-topic id renders in a different chat lane).
- ``reply_to_mode="off"`` suppresses the anchor but keeps ``message_thread_id``.
- Live replies send the private topic thread id together with a reply anchor. Synthetic/resumed sends
- without an anchor (loop wakeups, background-process notifications, queued follow-ups after a gateway
- restart) prefer the Hermes topic's ``message_thread_id`` so they stay in the active topic lane
- (#87051); ``direct_messages_topic_id`` is only used when no topic thread resolves, since the native
- DM-topic id does not match the Hermes topic lane and can render the message in a different chat
- lane.
- """
- fallback = cls._dm_topic_fallback(metadata)
- if fallback and reply_to_mode != "off":
- if reply_to_message_id is None:
- reply_to_message_id = cls._metadata_reply_to_message_id(metadata)
- if reply_to_message_id is None:
- # Anchor-less synthetic send: prefer the Hermes topic thread id (see docstring).
- # Anchor-less synthetic sends (loop wakeups, watch notifications, restart-resumed
- # follow-ups) must stay in the active topic lane: prefer the Hermes topic thread id when it
- # resolves (#87051). Routing via direct_messages_topic_id here sent these to a different
- # lane than the topic the session runs in.
- thread_message_id = cls._message_thread_id_for_send(thread_id)
- if thread_message_id is not None:
- return {"message_thread_id": thread_message_id}
- return cls._direct_topic_kwargs(metadata) or {}
- elif not fallback:
- direct_kwargs = cls._direct_topic_kwargs(metadata)
- if direct_kwargs is not None:
- return direct_kwargs
- return {"message_thread_id": cls._message_thread_id_for_send(thread_id)}
- @classmethod
- def _direct_topic_kwargs(cls, metadata: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
- """Native Bot API DM-topic routing kwargs, or None when no ``direct_messages_topic_id``."""
- direct_topic_id = cls._metadata_direct_messages_topic_id(metadata)
- if direct_topic_id is None:
- return None
- return {"message_thread_id": None, "direct_messages_topic_id": int(direct_topic_id)}
- def _thread_kwargs_for_draft(self, chat_id: str, metadata: Optional[dict[str, Any]]) -> dict[str, Any]:
- """Routing kwargs for ``sendMessageDraft`` / ``sendRichMessageDraft`` (integer
- ``message_thread_id`` for DM topics — Telegram rejects the raw string ``thread_id``)."""
- kwargs = self._thread_kwargs_for_send(
- chat_id, self._metadata_thread_id(metadata), metadata, reply_to_message_id=self._reply_to_message_id_for_send(None, metadata),
- reply_to_mode=getattr(self, "_reply_to_mode", None))
- return {k: v for k, v in kwargs.items() if v is not None}
- @classmethod
- def _message_thread_id_for_send(cls, thread_id: Optional[str]) -> Optional[int]:
- if not thread_id or str(thread_id) == cls._GENERAL_TOPIC_THREAD_ID:
- return None
- return int(thread_id)
- @classmethod
- def _message_thread_id_for_typing(cls, thread_id: Optional[str]) -> Optional[int]:
- # Deliberately asymmetric with _message_thread_id_for_send: sendMessage rejects message_thread_id=1
- # (forum General), but sendChatAction NEEDS it to place the typing bubble in General.
- return int(thread_id) if thread_id else None
- @staticmethod
- def _is_thread_not_found_error(error: Exception) -> bool:
- return "thread not found" in str(error).lower()
- def _prune_stale_dm_topic_binding(self, chat_id: Any, thread_id: Any, *, metadata: Optional[dict[str, Any]] = None) -> None:
- """Drop the stale ``telegram_dm_topic_bindings`` row for a topic Telegram confirmed deleted, else
- ``_recover_telegram_topic_thread_id`` keeps steering inbound to the dead thread. Best-effort.
- Rows are namespaced by profile: the send's ``hermes_profile`` wins over the adapter's stamp.
- Without this prune the recovery logic in ``gateway.run._recover_telegram_topic_thread_id`` keeps
- steering future inbound messages to the dead thread (the bug behind #31501 — tool progress,
- approvals, replies all end up in the wrong place even though the user has moved on to a fresh
- topic). Best-effort: we never raise from a send-fallback path — a failed cleanup must not turn into
- a failed user-facing send.
- Under ``gateway.profile_routes`` the transport adapter may not be the profile that wrote the
- binding, so the send's ``hermes_profile`` metadata wins over the adapter's own profile stamp;
- single-profile bots fall back to ``"default"``. See #76423.
- """
- if chat_id is None or thread_id is None:
- return
- db = getattr(getattr(self, "_session_store", None), "_db", None)
- if db is None or not hasattr(db, "delete_telegram_topic_binding"):
- return
- try:
- profile_name = (metadata or {}).get("hermes_profile") or getattr(self, "_hermes_profile_name", None) or "default"
- removed = db.delete_telegram_topic_binding(chat_id=str(chat_id), thread_id=str(thread_id), profile_name=profile_name)
- except Exception:
- logger.debug(
- "[%s] delete_telegram_topic_binding failed for chat=%s thread=%s — skipping prune",
- self.name, chat_id, thread_id, exc_info=True)
- return
- if removed:
- logger.info(
- "[%s] Pruned stale Telegram DM topic binding chat=%s thread=%s (Bot API: thread not found)", self.name, chat_id, thread_id)
- @staticmethod
- def _is_bad_request_error(error: Exception) -> bool:
- name = error.__class__.__name__.lower()
- if name == "badrequest" or name.endswith("badrequest"):
- return True
- try:
- from telegram.error import BadRequest
- return isinstance(error, BadRequest)
- except ImportError:
- return False
- @classmethod
- def _should_retry_without_dm_topic_reply_anchor(
- cls, error: Exception, metadata: Optional[dict[str, Any]], reply_to_message_id: Optional[int]) -> bool:
- """True when a DM-topic send should be retried with routing stripped: (1) stale anchor — reply
- target deleted; (2) anchor-less synthetic send whose ``direct_messages_topic_id`` Bot API rejects.
- 2. The synthetic-event case (added when #27937 introduced ``direct_messages_topic_id`` fallback for
- sends without an anchor): if Bot API rejects the topic id itself with any BadRequest that mentions
- topic/thread routing, we retry without routing rather than dropping the message.
- """
- if not cls._dm_topic_fallback(metadata) or not cls._is_bad_request_error(error):
- return False
- err_lower = str(error).lower()
- if reply_to_message_id is not None and "message to be replied not found" in err_lower:
- return True
- if not metadata.get("direct_messages_topic_id"): # topic id rejected → plain DM send
- return False
- topic_markers = (
- "direct_messages_topic", "message thread not found", "thread not found", "topic_closed", "topic_deleted", "topic not found")
- return any(marker in err_lower for marker in topic_markers)
- async def _send_with_dm_topic_reply_anchor_retry(
- self, send_fn: Any, send_kwargs: dict[str, Any], metadata: Optional[dict[str, Any]],
- reply_to_message_id: Optional[int], media_label: str, reset_media: Optional[Any] = None) -> Any:
- """Retry stale private-topic media replies once without the topic anchor. Serialized per chat with
- ``send()`` so a file upload cannot land between two chunks of the text it accompanies."""
- async with self._chat_send_lock(send_kwargs.get("chat_id")):
- try:
- return await _await_with_thread_deadline(
- send_fn(**send_kwargs), timeout=_MEDIA_SEND_DEADLINE, label="telegram-media-send", dump_on_blocked_loop=False)
- except Exception as send_err:
- if not self._should_retry_without_dm_topic_reply_anchor(send_err, metadata, reply_to_message_id):
- raise
- logger.warning(
- "[%s] Reply target deleted for Telegram %s, retrying without reply/topic anchor: %s",
- self.name, media_label, _redact_telegram_error_text(send_err))
- if reset_media is not None:
- reset_media()
- retry_kwargs = dict(send_kwargs)
- retry_kwargs["reply_to_message_id"] = None
- retry_kwargs.pop("message_thread_id", None)
- retry_kwargs.pop("direct_messages_topic_id", None)
- return await _await_with_thread_deadline(
- send_fn(**retry_kwargs), timeout=_MEDIA_SEND_DEADLINE, label="telegram-media-send", dump_on_blocked_loop=False)
- def _fallback_ips(self) -> list[str]:
- """Return validated fallback IPs from config (populated by _apply_env_overrides)."""
- configured = self.config.extra.get("fallback_ips", []) if getattr(self.config, "extra", None) else []
- if isinstance(configured, str):
- configured = configured.split(",")
- return parse_fallback_ip_env(",".join(str(v) for v in configured) if configured else None)
- @staticmethod
- def _looks_like_polling_conflict(error: Exception) -> bool:
- text = str(error).lower()
- return (
- error.__class__.__name__.lower() == "conflict"
- or "terminated by other getupdates request" in text
- or "another bot instance is running" in text)
- @staticmethod
- def _looks_like_auth_error(error: Exception) -> bool:
- """True for terminal credential failures (InvalidToken, Forbidden) → retryable=False. Type-based
- only, never message text; BadRequest/RetryAfter are transient at connect time."""
- if error.__class__.__name__.lower() in {"invalidtoken", "forbidden"}:
- return True
- try:
- from telegram.error import Forbidden, InvalidToken
- return isinstance(error, (InvalidToken, Forbidden))
- except ImportError:
- return False
- @staticmethod
- def _looks_like_network_error(error: Exception) -> bool:
- """Return True for transient transport failures that warrant reconnect."""
- name = error.__class__.__name__.lower()
- if name in {"badrequest", "invalidtoken", "forbidden", "retryafter"}:
- return False
- if name in {"networkerror", "timedout", "connectionerror"}:
- return True
- try:
- from telegram.error import BadRequest, Forbidden, InvalidToken, NetworkError, RetryAfter, TimedOut
- if isinstance(error, (BadRequest, InvalidToken, Forbidden, RetryAfter)):
- return False
- if isinstance(error, (NetworkError, TimedOut)):
- return True
- except ImportError:
- pass
- return isinstance(error, OSError)
- @staticmethod
- def _exception_graph_matches(error: Exception, name_marker: str, *text_markers: str) -> bool:
- """True when any exception in ``error``'s cause/context graph matches by class name or text."""
- for cur in _iter_exception_graph(error):
- text = str(cur).lower()
- if name_marker in cur.__class__.__name__.lower() or any(m in text for m in text_markers):
- return True
- return False
- @classmethod
- def _looks_like_connect_timeout(cls, error: Exception) -> bool:
- """True when a TimedOut wraps a ConnectTimeout: TCP never connected, so re-sending is safe
- (a plain TimedOut may have reached Telegram and must not be re-sent)."""
- return cls._exception_graph_matches(error, "connecttimeout", "connect timeout", "connect timed out")
- @staticmethod
- def _looks_like_pool_timeout(error: Exception) -> bool:
- """True when a TimedOut wraps ``httpx.PoolTimeout``: PTB says "Request was *not* sent", so
- re-sending cannot duplicate. Matches class AND text to survive rewording."""
- for cur in _iter_exception_graph(error):
- name = cur.__class__.__name__.lower()
- text = str(cur).lower()
- if "pooltimeout" in name or "pool timeout" in text or ("connection pool" in text and "occupied" in text):
- return True
- return False
- def _coerce_bool_extra(self, key: str, default: bool = False) -> bool:
- value = self.config.extra.get(key) if getattr(self.config, "extra", None) else None
- if value is None:
- return default
- if isinstance(value, str):
- lowered = value.strip().lower()
- if lowered in {"true", "1", "yes", "on"}:
- return True
- if lowered in {"false", "0", "no", "off"}:
- return False
- return default
- return bool(value)
- def _link_preview_kwargs(self) -> dict[str, Any]:
- if not getattr(self, "_disable_link_previews", False):
- return {}
- if LinkPreviewOptions is not None:
- return {"link_preview_options": LinkPreviewOptions(is_disabled=True)}
- return {"disable_web_page_preview": True}
- # --- Bot API 10.1 Rich Messages (sendRichMessage): final/new-message replies opportunistically send
- # RAW agent markdown so tables, task lists, <details>, math render natively; legacy MarkdownV2 send()
- # is the fallback. Streaming edits stay on the MarkdownV2 edit path.
- def _content_fits_rich_limits(self, content: str) -> bool:
- """Pre-check the 32,768-char cap only; other rich limits surface as BadRequest (permanent)."""
- return len(content) <= self.RICH_MESSAGE_MAX_CHARS
- def _bot_supports_rich(self) -> bool:
- """True when ``do_api_request`` is an *async* callable (real Bot or AsyncMock); plain MagicMock
- and SimpleNamespace bots resolve False → legacy path."""
- return inspect.iscoroutinefunction(getattr(self._bot, "do_api_request", None))
- _RICH_DETAILS_RE = re.compile(r"<details\b[^>]*>.*?</details>", re.IGNORECASE | re.DOTALL)
- _RICH_MATH_IN_DETAILS_RE = re.compile(
- r"(\$\$.*?\$\$|\\\[.*?\\\]|\\\(.*?\\\)|"
- r"\\(?:sum|frac|alpha|beta|gamma|delta|theta|lambda|mu|pi|sigma|"
- r"int|prod|sqrt|lim|infty|begin\{(?:equation|align|matrix|cases)\}))",
- re.IGNORECASE | re.DOTALL)
- # Hiragana/Katakana, CJK Ext A, CJK Unified, Hangul, CJK Compatibility, CJK ext/compat supplement.
- _RICH_CJK_RE = re.compile("[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff\U00020000-\U000323af]")
- def _has_telegram_desktop_details_math_crash_shape(self, content: str) -> bool:
- """Math inside <details> crashes Telegram Desktop 6.9.1 (tdesktop#30808); the Bot API accepts
- the payload, so rich delivery must be skipped up front."""
- if not content:
- return False
- return any(self._RICH_MATH_IN_DETAILS_RE.search(block) for block in self._RICH_DETAILS_RE.findall(content))
- def _has_telegram_desktop_cjk_rich_garble_shape(self, content: str) -> bool:
- """True for CJK content: Telegram Mac/Desktop rich rendering leaves overlapping glyphs.
- Telegram Mac/Desktop Bot API 10.1 rich-message rendering currently leaves overlapping draft/overlay
- glyph artifacts for CJK text (#47653). The legacy MarkdownV2 path renders the same text cleanly, so
- skip rich delivery up front until affected clients age out.
- """
- return bool(content and self._RICH_CJK_RE.search(content))
- def _needs_rich_rendering(self, content: str) -> bool:
- """True for constructs MarkdownV2 degrades: pipe tables, task lists, <details>, block math.
- Ordinary replies stay on MarkdownV2 so clients render consistent font weight/spacing.
- The rich endpoint is reserved for constructs where raw markdown materially improves output: pipe
- tables (MarkdownV2 has no table syntax and rewrites them into bullet lists), GFM task lists,
- collapsible ``<details>`` blocks, and block math. Adapted from #45995 (@YonganZhang).
- """
- if not content:
- return False
- if any(_TABLE_SEPARATOR_RE.match(line) for line in content.splitlines()):
- return True
- if re.search(r"(?m)^\s*[-*]\s+\[[ xX]\]\s+", content):
- return True
- if re.search(r"(?m)^<details\b|^</details>|^<summary\b|^</summary>", content):
- return True
- return "$$" in content
- def _rich_delivery_enabled(self) -> bool:
- """Whether rich delivery is allowed (``rich_messages`` opt-in)."""
- return bool(getattr(self, "_rich_messages_enabled", True))
- def _rich_content_ok(self, content: str) -> bool:
- """Shape checks shared by rich sends and rich drafts (non-blank, no Desktop crash/garble
- shapes, under the cap, async-capable bot)."""
- return bool(
- content and content.strip()
- and not self._has_telegram_desktop_details_math_crash_shape(content)
- and (
- getattr(self, "_allow_cjk_rich_messages", False)
- or not self._has_telegram_desktop_cjk_rich_garble_shape(content)
- )
- and self._content_fits_rich_limits(content)
- and self._bot_supports_rich())
- def _rich_eligible(self, content: str) -> bool:
- """Rich eligibility ignoring ``expect_edits`` (a streamed preview's FINAL edit still upgrades)."""
- return bool(
- self._rich_delivery_enabled()
- and not getattr(self, "_rich_send_disabled", False)
- and content and content.strip()
- and self._needs_rich_rendering(content)
- and self._rich_content_ok(content))
- def _should_attempt_rich(self, content: str, metadata: Optional[dict[str, Any]] = None) -> bool:
- return bool(not (metadata or {}).get("expect_edits") and self._rich_eligible(content))
- def prefers_fresh_final_streaming(self, content: str, metadata: Optional[dict[str, Any]] = None) -> bool:
- """Replace a streamed preview with a fresh rich final — DM topics only. Root DMs stay off (a live
- draft has no preview id); DM *topics* degrade to edit-in-place whose MarkdownV2 preview Telegram
- refuses to rich-edit, so a fresh sendRichMessage + delete is the only way to keep native tables.
- Root DMs keep this off (#46206 / #47048): successful draft streaming has no preview ``message_id``,
- so the hook is not consulted, and in-place ``editMessageText.rich_message`` would duplicate a live
- draft turn. Private DM *topics* often reject ``sendMessageDraft``; the consumer then degrades to
- edit-in-place. Telegram rejects a rich edit of that plain MarkdownV2 preview, and the fallback
- formatter permanently turns pipe tables into bullet lists.
- """
- metadata = metadata or {}
- if not (metadata.get("telegram_dm_topic_reply_fallback") or self._metadata_direct_messages_topic_id(metadata)):
- return False
- return self._rich_eligible(content)
- def _rich_transport_available(self) -> bool:
- return bool(
- getattr(self, "_rich_messages_enabled", True) and not getattr(self, "_rich_send_disabled", False) and self._bot_supports_rich())
- def streaming_overflow_limit(self) -> Optional[int]:
- """Let the stream consumer accumulate up to the rich cap so a reply that fits one sendRichMessage
- isn't fragmented at 4,096. None (→ legacy limit) if rich is unavailable."""
- return self.RICH_MESSAGE_MAX_CHARS if self._rich_transport_available() else None
- def _rich_message_payload(self, content: str, *, skip_entity_detection: bool = False) -> dict[str, Any]:
- """``InputRichMessage`` from RAW markdown — never ``format_message(content)``, whose MarkdownV2
- escaping destroys table pipes."""
- payload: dict[str, Any] = {"markdown": _rich_normalize_linebreaks(content)}
- if skip_entity_detection:
- payload["skip_entity_detection"] = True
- return payload
- def _is_rich_capability_error(self, exc: Exception) -> bool:
- """True ⇒ the rich endpoint itself is unavailable (old PTB/server); latches rich off.
- Per-message BadRequests (parser/limit) are NOT capability errors."""
- if exc.__class__.__name__.lower() in {"endpointnotfound", "invalidtoken"}:
- return True
- if isinstance(exc, (AttributeError, TypeError, NotImplementedError)) or getattr(exc, "error_code", None) == 404:
- return True
- s = str(exc).lower()
- if ("method" in s or "endpoint" in s) and ("not found" in s or "does not exist" in s):
- return True
- return "no such method" in s
- def _is_rich_fallback_error(self, exc: Exception) -> bool:
- """True ⇒ permanent/capability error ⇒ safe to fall back to legacy. Conservative: anything not
- clearly permanent is transient — the rich request may have reached Telegram (duplicate risk)."""
- if self._is_bad_request_error(exc) or self._is_rich_capability_error(exc):
- return True
- s = str(exc).lower()
- return "unsupported" in s or "not implemented" in s
- def _chunk_reply_routing(
- self, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]], thread_id: Optional[str], index: int) -> tuple:
- """Reply-anchor routing for chunk ``index``: ``(private_dm_topic_send, anchor_off, reply_to_id)``.
- ``anchor_off``: reply_to_mode="off" on the DM-topic fallback path opts into "message_thread_id
- alone is enough" — don't fail loud because the anchor was suppressed by config."""
- metadata_reply_to = self._metadata_reply_to_message_id(metadata)
- private_dm_topic_send = self._is_private_dm_topic_send(chat_id, thread_id, metadata)
- dm_topic_reply_to_off = private_dm_topic_send and self._reply_to_mode == "off" and self._dm_topic_fallback(metadata)
- reply_to_source = next((str(i) for i in (parse_telegram_message_id(reply_to), metadata_reply_to if private_dm_topic_send else None) if i is not None), None)
- if private_dm_topic_send:
- should_thread = reply_to_source is not None and self._reply_to_mode != "off"
- else:
- should_thread = self._should_thread_reply(reply_to_source, index)
- reply_to_id = int(reply_to_source) if should_thread and reply_to_source else None
- return private_dm_topic_send, dm_topic_reply_to_off, reply_to_id
- def _compute_single_send_routing(
- self, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]], thread_id: Optional[str]) -> Optional[tuple]:
- """Routing for a single (rich) send — mirrors send()'s index-0 block. Returns ``(reply_to_id,
- thread_kwargs)`` or ``None`` = skip rich, legacy owns the DM-topic fail-loud SendResult."""
- private_dm_topic_send, dm_topic_reply_to_off, reply_to_id = self._chunk_reply_routing(chat_id, reply_to, metadata, thread_id, 0)
- thread_kwargs = self._thread_kwargs_for_send(
- chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
- # Synthetic/resumed sends via direct_messages_topic_id need no reply anchor.
- if (
- private_dm_topic_send and reply_to_id is None and not dm_topic_reply_to_off
- and not thread_kwargs.get("direct_messages_topic_id")):
- return None
- return reply_to_id, thread_kwargs
- @staticmethod
- def _is_timed_out(exc: Exception) -> bool:
- """PTB ``TimedOut`` (when importable) or a "timed out" message."""
- try:
- from telegram.error import TimedOut as _TimedOut
- except (ImportError, AttributeError):
- _TimedOut = None
- return bool((_TimedOut and isinstance(exc, _TimedOut)) or "timed out" in str(exc).lower())
- def _rich_transient_result(self, exc: Exception, what: str, *, retry_after: Any = None) -> SendResult:
- """SendResult for a transient/unknown rich-API failure (request may have reached Telegram, so the
- caller must NOT legacy-resend); retry semantics mirror legacy send()."""
- safe_error = _redact_telegram_error_text(exc)
- logger.warning("[%s] %s transient failure (no legacy resend): %s", self.name, what, safe_error)
- return SendResult(
- success=False, error=safe_error,
- retryable=(self._looks_like_connect_timeout(exc) or not self._is_timed_out(exc)), retry_after=retry_after)
- @staticmethod
- async def _record_rich_sent(chat_id: Any, message_id: Any, content: str) -> None:
- """Index rich content we sent: Telegram won't echo it back in reply_to_message.
- Awaited so the store's read-modify-write + ``os.replace`` runs off the loop."""
- try:
- from gateway import rich_sent_store
- await rich_sent_store.record_async(str(chat_id), str(message_id), content)
- except Exception:
- pass
- async def _try_send_rich(
- self, chat_id: str, content: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> Optional[SendResult]:
- """Attempt a single ``sendRichMessage``. Returns a SendResult (success, or a transient failure the
- caller must NOT legacy-resend), or ``None`` = fall back to legacy MarkdownV2."""
- thread_id = self._metadata_thread_id(metadata)
- routing = self._compute_single_send_routing(chat_id, reply_to, metadata, thread_id)
- if routing is None:
- return None
- reply_to_id, thread_kwargs = routing
- payload = self._rich_payload_base(chat_id, content)
- # Only non-None routing keys: direct_messages_topic_id is paired with message_thread_id=None.
- payload.update({k: v for k, v in thread_kwargs.items() if v is not None})
- payload.update(self._notification_kwargs(metadata))
- if reply_to_id is not None:
- # sendRichMessage takes reply_parameters, NOT reply_to_message_id (silently ignored → anchor dropped).
- payload["reply_parameters"] = {"message_id": reply_to_id}
- try:
- # Raw Bot API result: return_type=Message would make PTB deserialize a 10.1 shape it doesn't
- # fully model; a post-delivery parse error ≠ send failure.
- msg = await _await_with_thread_deadline(
- self._bot.do_api_request("sendRichMessage", api_kwargs=payload),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- except Exception as exc:
- if self._rich_rejected(exc, "sendRichMessage", "MarkdownV2"):
- return None
- # Honor Telegram's flood-control retry_after over the base retry schedule.
- _retry_after = getattr(exc, "retry_after", None)
- if _retry_after is None:
- _m = re.search(r"retry\s+(?:in\s+)?(\d+)", str(exc).lower(), re.IGNORECASE)
- if _m:
- _retry_after = float(_m.group(1))
- return self._rich_transient_result(exc, "sendRichMessage", retry_after=_retry_after)
- if isinstance(msg, dict):
- message_id = msg.get("message_id")
- if message_id is None:
- message_id = (msg.get("result") or {}).get("message_id")
- else:
- message_id = getattr(msg, "message_id", None)
- if message_id is not None:
- await self._record_rich_sent(chat_id, message_id, content)
- return SendResult(success=True, message_id=str(message_id) if message_id is not None else None)
- def _rich_payload_base(self, chat_id: str, content: str) -> dict[str, Any]:
- payload: dict[str, Any] = {"chat_id": normalize_telegram_chat_id(chat_id), "rich_message": self._rich_message_payload(content)}
- if getattr(self, "_disable_link_previews", False):
- payload["link_preview_options"] = {"is_disabled": True}
- return payload
- def _rich_rejected(self, exc: Exception, what: str, fallback: str) -> bool:
- """True for a permanent/capability rich-API failure (caller falls back to legacy); capability
- errors latch rich off so no doomed roundtrip repeats per send."""
- if not self._is_rich_fallback_error(exc):
- return False
- if self._is_rich_capability_error(exc):
- self._rich_send_disabled = True
- logger.debug("[%s] %s rejected (%s) — falling back to %s", self.name, what, _redact_telegram_error_text(exc), fallback)
- return True
- async def _try_edit_rich(
- self, chat_id: str, message_id: str, content: str, metadata: Optional[dict[str, Any]] = None) -> Optional[SendResult]:
- """Edit a message in place as rich (``editMessageText`` + ``rich_message``) so a streamed preview
- finalizes without send+delete. Same contract as :meth:`_try_send_rich`."""
- # No topic routing on edits: message_thread_id/direct_messages_topic_id make Telegram reject it.
- payload = {**self._rich_payload_base(chat_id, content), "message_id": int(message_id)}
- try:
- await _await_with_thread_deadline(
- self._bot.do_api_request("editMessageText", api_kwargs=payload),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- except Exception as exc:
- # "Message is not modified" = successful no-op; skip the redundant legacy edit.
- if "not modified" in str(exc).lower():
- if self._is_rich_fallback_error(exc) and self._is_rich_capability_error(exc):
- self._rich_send_disabled = True
- return SendResult(success=True, message_id=message_id)
- if self._rich_rejected(exc, "rich editMessageText", "MarkdownV2 edit"):
- return None
- return self._rich_transient_result(exc, "rich editMessageText")
- # Mirror the fresh-send index: a streamed final finalized via edit is otherwise never recorded.
- await self._record_rich_sent(chat_id, message_id, content)
- return SendResult(success=True, message_id=message_id)
- def _should_attempt_rich_draft(self, content: str) -> bool:
- return bool(
- getattr(self, "_rich_messages_enabled", True)
- and getattr(self, "_rich_drafts_enabled", False)
- and not getattr(self, "_rich_send_disabled", False)
- and not getattr(self, "_rich_draft_disabled", False)
- and self._rich_content_ok(content))
- async def _try_send_rich_draft(self, chat_id: str, draft_id: int, content: str, metadata: Optional[dict[str, Any]]) -> bool:
- """Emit one ``sendRichMessageDraft`` frame; True on success. Frames are ephemeral, so any failure
- returns False and the caller renders the legacy draft; capability failures latch off."""
- payload: dict[str, Any] = {
- "chat_id": normalize_telegram_chat_id(chat_id), "draft_id": int(draft_id), "rich_message": self._rich_message_payload(content)}
- payload.update(self._thread_kwargs_for_draft(chat_id, metadata))
- try:
- return bool(await _await_with_thread_deadline(
- self._bot.do_api_request("sendRichMessageDraft", api_kwargs=payload),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False))
- except Exception as exc:
- if self._is_rich_capability_error(exc):
- self._rich_draft_disabled = True
- logger.debug(
- "[%s] sendRichMessageDraft unsupported (%s) — using legacy drafts", self.name, _redact_telegram_error_text(exc))
- else:
- logger.debug(
- "[%s] sendRichMessageDraft transient failure (%s) — legacy draft this frame", self.name,
- _redact_telegram_error_text(exc))
- return False
- async def _drain_polling_connections(self) -> None:
- """Reset the httpx pool used for getUpdates polling before a reconnect.
- Half-closed connections (esp. via proxies) occupy pool slots until "Pool timeout: All connections in the connection pool
- are occupied". Only ``_request[0]`` (getUpdates) is reset; the general request stays untouched so concurrent sends are
- never interrupted. Relies on PTB 22.x's private ``(get_updates, general)`` tuple — review on PTB 23+."""
- if not (self._app and self._app.bot):
- return
- try:
- polling_req = self._app.bot._request[0]
- except Exception:
- return
- # Bounded wall-clock deadline (not asyncio.wait_for): httpcore's pool close runs under
- # AsyncShieldCancellation and a wedged CLOSE-WAIT socket can hang it forever.
- if not await self._bounded_request_step(polling_req.shutdown(), "Polling request shutdown failed/timed out (non-fatal)"):
- # initialize() only rebuilds the client when ``client.is_closed``; an abandoned aclose()
- # leaves it false, so start_polling would reuse the CLOSE-WAIT socket (alive but deaf).
- # Swap in a fresh client before initialize(). See #87057.
- self._orphan_and_rebuild_polling_client(polling_req)
- if await self._bounded_request_step(polling_req.initialize(), "Polling request re-initialize failed/timed out (non-fatal)"):
- logger.debug("[%s] Polling request pool drained before reconnect", self.name)
- else:
- self._orphan_and_rebuild_polling_client(polling_req)
- async def _bounded_request_step(self, awaitable, failure_msg: str) -> bool:
- """Await a request shutdown()/initialize() under ``_DRAIN_TIMEOUT``; False (debug-logged) on failure."""
- try:
- await _await_with_thread_deadline(awaitable, timeout=_DRAIN_TIMEOUT)
- return True
- except Exception:
- logger.debug("[%s] " + failure_msg, self.name, exc_info=True)
- return False
- def _orphan_and_rebuild_polling_client(self, polling_req) -> None:
- """Replace a wedged HTTPXRequest client after a hung aclose(): swap in a fresh client and close
- the old one in a detached, bounded task so it can't block the reconnect ladder.
- PTB's ``HTTPXRequest.initialize()`` only calls ``_build_client()`` when the current client reports
- ``is_closed``. If ``shutdown()`` was abandoned on a CLOSE-WAIT socket, that flag stays false and the
- next ``start_polling()`` reuses the dead getUpdates connection (#87057).
- """
- old = getattr(polling_req, "_client", None)
- build = getattr(polling_req, "_build_client", None)
- if old is None or not callable(build) or getattr(old, "is_closed", True):
- return
- try:
- polling_req._client = build()
- except Exception:
- logger.debug("[%s] Failed to rebuild polling HTTP client after hung drain", self.name, exc_info=True)
- return
- logger.warning("[%s] Replaced wedged getUpdates HTTP client after drain timeout (likely CLOSE-WAIT socket)", self.name)
- async def _orphan_aclose() -> None:
- try:
- aclose = getattr(old, "aclose", None)
- if not callable(aclose):
- return
- # Same cancellation-swallowing httpcore scope as shutdown(): wall-clock deadline.
- await _await_with_thread_deadline(aclose(), timeout=_DRAIN_TIMEOUT)
- except Exception:
- logger.debug("[%s] Orphan polling client aclose failed (non-fatal)", self.name, exc_info=True)
- try:
- task = asyncio.ensure_future(_orphan_aclose())
- self._background_tasks.add(task)
- task.add_done_callback(self._background_tasks.discard)
- task.add_done_callback(_consume_abandoned_task)
- except Exception:
- pass
- def _fence_polling(self) -> None:
- """Mark polling closed: no progress accepted, send path degraded."""
- self._polling_progress_accepting = False
- self._send_path_degraded = True
- def _begin_polling_generation(self) -> tuple[int, asyncio.Event]:
- """Start accepting progress for a new getUpdates polling generation."""
- if self._teardown_started:
- self._fence_polling()
- progress = getattr(self, "_polling_progress_event", None)
- if progress is None:
- progress = self._polling_progress_event = asyncio.Event()
- return getattr(self, "_polling_generation", 0), progress
- verifier = getattr(self, "_polling_progress_verifier_task", None)
- if verifier is not None and not verifier.done():
- verifier.cancel()
- self._polling_progress_verifier_task = None
- self._polling_generation = getattr(self, "_polling_generation", 0) + 1
- self._polling_progress_event = asyncio.Event()
- self._polling_progress_accepting = True
- self._send_path_degraded = True
- # Reset stall-watchdog timestamps: no proven progress yet, age measured from here.
- # See #92991.
- self._polling_generation_started_monotonic = time.monotonic()
- self._polling_last_progress_monotonic = None
- # Re-base the backlog per generation. On an in-place updater restart PTB keeps the old
- # update_queue, so old dispatches can briefly exceed received; the check treats that as no backlog.
- self._updates_received_total = self._updates_dispatched_total = 0
- self._ingress_dispatched_seen = self._ingress_stalled_heartbeats = 0
- return self._polling_generation, self._polling_progress_event
- def _record_polling_progress(self, generation: int) -> bool:
- """Record successful getUpdates I/O for the current generation only; True when accepted."""
- if self._teardown_started or not self._polling_progress_accepting or generation != self._polling_generation:
- return False
- if not self._polling_progress_event.is_set():
- # First confirmed round-trip resolves the "health pending" line both reconnect paths end on.
- # After network-error WARNINGs the line must read as the matching recovery event (#111211).
- state = "recovered" if self._polling_network_error_count else "confirmed healthy"
- logger.info("[%s] Telegram polling %s: getUpdates progressing (generation %d)", self.name, state, generation)
- self._polling_progress_event.set()
- self._polling_last_progress_monotonic = time.monotonic()
- self._polling_network_error_count = 0
- if generation == self._polling_conflict_recovery_generation:
- self._polling_conflict_recovery_generation = None
- else:
- self._polling_conflict_count = 0
- # First proof getUpdates is flowing for this generation: flip a
- # published "retrying" (degraded connect, reconnect stamp, or the
- # mid-session recovery below) back to "connected" (#101391).
- if self._send_path_degraded and getattr(self, "_running", False) and not self.has_fatal_error:
- self._write_runtime_status_safe(
- "connected", platform_state="connected", error_code=None, error_message=None,
- )
- self._send_path_degraded = False
- return True
- def _observe_polling_request_result(self, request, generation, result):
- """Record getUpdates progress from an observed do_request result (purely observational: PTB still
- parses the untouched payload and owns any resulting exception)."""
- status_code, payload = result
- if generation is None or not (200 <= status_code < 300):
- return
- try:
- # The request's own parser keeps health observation in agreement with PTB.
- envelope = request.parse_json_payload(payload)
- except Exception:
- return
- if isinstance(envelope, dict) and envelope.get("ok") is True and "result" in envelope:
- if self._record_polling_progress(generation):
- self._record_updates_received(envelope.get("result"))
- def _record_updates_received(self, result) -> None:
- """Count updates Telegram handed us on the getUpdates wire (#102260). Only reached for the
- accepted generation, so a late response from a fenced poll cannot inflate the backlog."""
- if isinstance(result, list) and result:
- self._updates_received_total += len(result)
- def _instrument_polling_request(self, request):
- """Instrument one dedicated PTB getUpdates request with progress tracking.
- PTB request classes use ``__slots__`` (no ``__dict__`` on 3.13), so re-tag the instance to a thin ``__slots__ = ()``
- subclass overriding ``do_request`` — identical layout makes the ``__class__`` swap legal; works for test doubles too.
- On Python 3.13 their instances no longer carry a ``__dict__`` (the ``AbstractAsyncContextManager``
- MRO stopped yielding one), so ``request.do_request = wrapper`` raises ``AttributeError:
- 'HTTPXRequest' object attribute 'do_request' is read-only`` and the whole Telegram connect fails
- (#64482). It only appeared to work on Python 3.12, where those instances still had a ``__dict__``.
- """
- adapter = self
- class _InstrumentedPollingRequest(type(request)):
- __slots__ = ()
- async def do_request(self, *args, **kwargs):
- generation = _POLLING_GENERATION_CONTEXT.get()
- result = await super().do_request(*args, **kwargs)
- adapter._observe_polling_request_result(self, generation, result)
- return result
- request.__class__ = _InstrumentedPollingRequest
- return request
- async def _start_polling_once(
- self, app, *, drop_pending_updates: bool, error_callback, abandon_app_on_timeout: bool = False,
- schedule_verifier: bool = True) -> tuple[int, asyncio.Event]:
- """Start one generation and verify real getUpdates progress. Returns this generation's
- ``(generation, progress_event)`` so readiness-gating callers bind to exactly it."""
- if self._teardown_started:
- raise _PollingLifecycleAbort("Telegram polling teardown started")
- generation, progress = self._begin_polling_generation()
- if not self._polling_progress_accepting:
- raise _PollingLifecycleAbort("Telegram polling teardown started")
- def _generation_error_callback(error: Exception) -> None:
- if self._teardown_started or generation != self._polling_generation or error_callback is None:
- return
- callback_context_token = _POLLING_GENERATION_CONTEXT.set(None)
- try:
- error_callback(error)
- finally:
- _POLLING_GENERATION_CONTEXT.reset(callback_context_token)
- context_token = _POLLING_GENERATION_CONTEXT.set(generation)
- try:
- # asyncio.wait_for can wait forever on httpcore/AnyIO shielded scopes; use the wall-deadline
- # helper and abandon the partial updater (caller rebuilds).
- await _await_with_thread_deadline(
- app.updater.start_polling(
- allowed_updates=Update.ALL_TYPES, drop_pending_updates=drop_pending_updates, error_callback=_generation_error_callback),
- timeout=_UPDATER_START_TIMEOUT,
- on_abandon=((lambda app=app: _shutdown_abandoned_app(app)) if abandon_app_on_timeout else None))
- finally:
- _POLLING_GENERATION_CONTEXT.reset(context_token)
- if self._teardown_started:
- self._fence_polling()
- raise _PollingLifecycleAbort("Telegram polling teardown started")
- if schedule_verifier:
- self._schedule_polling_progress_verifier(generation, progress)
- return generation, progress
- def _schedule_polling_progress_verifier(self, generation: int, progress: asyncio.Event) -> None:
- """Own exactly one tracked verifier for the current generation."""
- if self._teardown_started:
- self._fence_polling()
- return
- previous = getattr(self, "_polling_progress_verifier_task", None)
- if previous is not None and not previous.done():
- previous.cancel()
- task = asyncio.get_running_loop().create_task(self._verify_polling_after_reconnect(generation, progress))
- self._polling_progress_verifier_task = task
- self._background_tasks.add(task)
- def _clear_finished_verifier(finished: asyncio.Task) -> None:
- self._background_tasks.discard(finished)
- if self._polling_progress_verifier_task is finished:
- self._polling_progress_verifier_task = None
- task.add_done_callback(_clear_finished_verifier)
- def _get_general_request_drain_lock(self) -> asyncio.Lock:
- lock = getattr(self, "_general_request_drain_lock", None)
- if lock is None:
- lock = self._general_request_drain_lock = asyncio.Lock()
- return lock
- async def _drain_general_connections_after_pool_timeout(self) -> None:
- """Reset the general Bot API pool (``_request[1]``) after a confirmed send pool timeout — PTB
- guarantees the request was not sent, so resetting before retrying is safe."""
- bot = getattr(getattr(self, "_app", None), "bot", None)
- if bot is None:
- bot = getattr(self, "_bot", None)
- if bot is None:
- return
- try:
- general_req = bot._request[1]
- except Exception:
- return
- async with self._get_general_request_drain_lock():
- await self._bounded_request_step(
- general_req.shutdown(), "General request shutdown failed/timed out after pool timeout (non-fatal)")
- if await self._bounded_request_step(
- general_req.initialize(), "General request re-initialize failed/timed out after pool timeout (non-fatal)"):
- logger.warning("[%s] General request pool drained after Telegram pool timeout", self.name)
- def _spawn_polling_recovery(self, loop, coro) -> None:
- """Start ``coro`` as the tracked in-flight recovery task (reentrancy guard)."""
- self._polling_error_task = loop.create_task(coro)
- self._background_tasks.add(self._polling_error_task)
- self._polling_error_task.add_done_callback(self._background_tasks.discard)
- def _recovery_in_flight(self) -> bool:
- return bool(self._polling_error_task and not self._polling_error_task.done())
- def _schedule_polling_recovery(self, error: Exception, *, reason: str) -> None:
- """Schedule background polling recovery without failing gateway startup: a transient bootstrap
- failure degrades only this adapter; the reconnect ladder recovers in the background."""
- if self._teardown_started or self.has_fatal_error:
- return
- if self._recovery_in_flight():
- logger.debug(
- "[%s] Telegram polling recovery already scheduled; ignoring %s: %s", self.name, reason, _redact_telegram_error_text(error))
- return
- self._send_path_degraded = True
- # Polling died mid-session on an adapter that published "connected"
- # at connect time. Without this, gateway_state.json keeps saying
- # connected for as long as the recovery ladder runs (#101391: 11 h).
- if getattr(self, "_running", False):
- self._mark_degraded()
- if isinstance(error, _PollingStallError):
- # Not a retry promise: the recovery path hands a confirmed stall straight to the supervisor
- # (``_go_fatal_network`` logs the single error-level line for it).
- logger.warning(
- "[%s] Telegram polling stall confirmed (%s); handing off to the supervisor for an adapter rebuild. "
- "Error: %s", self.name, reason, _redact_telegram_error_text(error))
- else:
- logger.warning(
- "[%s] Telegram polling degraded (%s); gateway stays alive and will retry. Error: %s", self.name, reason,
- _redact_telegram_error_text(error))
- self._spawn_polling_recovery(asyncio.get_running_loop(), self._handle_polling_network_error(error))
- async def _delete_webhook_best_effort(self, *, require_success: bool = False) -> bool:
- """Clear a stale webhook; ``require_success`` (cold start) raises so GatewayRunner disposes the
- partial adapter, while reconnects recover transient errors in background."""
- if not self._bot:
- return False
- delete_webhook = getattr(self._bot, "delete_webhook", None)
- if not callable(delete_webhook):
- return True
- try:
- # Same shielded-cancellation class as initialize/start_polling: never let it pin connect.
- await _await_with_thread_deadline(delete_webhook(drop_pending_updates=False), timeout=_UPDATER_START_TIMEOUT)
- return True
- except Exception as err:
- if not self._looks_like_network_error(err):
- raise
- if require_success:
- raise OSError("Telegram deleteWebhook did not complete during initial connect") from err
- logger.warning(
- "[%s] deleteWebhook failed with a recoverable network error; continuing to polling so getUpdates/retry can recover: %s",
- self.name, _redact_telegram_error_text(err))
- self._send_path_degraded = True
- return False
- async def _await_cold_start_readiness(self, progress: asyncio.Event, strict_error_event: asyncio.Event, strict_error: list) -> None:
- """Cold start: wait for THIS generation's first getUpdates success or the first polling error;
- raises OSError so GatewayRunner disposes the partial adapter and retries fresh."""
- progress_wait = asyncio.ensure_future(progress.wait())
- error_wait = asyncio.ensure_future(strict_error_event.wait())
- try:
- # Losers are NOT cancelled here; the finally below does it.
- await _await_with_thread_deadline(
- asyncio.wait({progress_wait, error_wait}, return_when=asyncio.FIRST_COMPLETED), timeout=_INITIAL_POLLING_PROGRESS_TIMEOUT)
- except TimeoutError as exc:
- raise OSError(
- "Telegram getUpdates made no progress within "
- f"{_INITIAL_POLLING_PROGRESS_TIMEOUT:.0f}s during initial "
- "connect — failing startup so the gateway retries with a fresh adapter (#67498)"
- ) from exc
- finally:
- for fut in (progress_wait, error_wait):
- if not fut.done():
- fut.cancel()
- await asyncio.gather(progress_wait, error_wait, return_exceptions=True)
- if strict_error and not progress.is_set():
- raise OSError(
- "Telegram polling errored before first getUpdates success during initial connect: "
- f"{_redact_telegram_error_text(strict_error[0])}"
- ) from strict_error[0]
- if not progress.is_set():
- raise OSError("Telegram getUpdates did not become ready during initial connect")
- async def _start_polling_resilient(self, *, drop_pending_updates: bool, error_callback, require_progress: bool = False) -> bool:
- """Start PTB polling; ``require_progress`` (initial connect) demands real readiness. Reconnects
- may recover in background; on cold start a bootstrap failure raises (see _await_cold_start_readiness)."""
- if self._teardown_started:
- return False
- if not (self._app and self._app.updater):
- raise RuntimeError("Telegram application/updater not initialized")
- # Strict cold start: background recovery must not run while the readiness gate waits, else a G1
- # error starts G2 on the same partial app and GatewayRunner never disposes it.
- strict_error: list[BaseException] = []
- strict_error_event = asyncio.Event()
- strict_gate_open = True
- effective_callback = error_callback
- if require_progress:
- loop = asyncio.get_running_loop()
- def _strict_error_callback(error: Exception) -> None:
- # Once the gate closes, delegate so later errors still reach background recovery.
- if not strict_gate_open:
- if error_callback is not None:
- error_callback(error)
- return
- if not strict_error:
- strict_error.append(error)
- # Called from the polling task; set on the loop to wake the strict waiter.
- loop.call_soon_threadsafe(strict_error_event.set)
- effective_callback = _strict_error_callback
- try:
- # Same watchdog bound as the reconnect ladders; the TimeoutError is an OSError subclass, so
- # the except below classifies it as a network error → background recovery.
- # Same watchdog bound as the reconnect ladders: a wedged httpx connection pool can hang
- # start_polling() forever at bootstrap too (#59614).
- _generation, progress = await self._start_polling_once(
- self._app, drop_pending_updates=drop_pending_updates, error_callback=effective_callback,
- abandon_app_on_timeout=require_progress,
- # The strict gate IS the cold-start verifier; a background one would race it.
- schedule_verifier=not require_progress)
- if require_progress:
- await self._await_cold_start_readiness(progress, strict_error_event, strict_error)
- # Readiness proven — close the gate so later errors reach background recovery.
- strict_gate_open = False
- self._polling_error_callback_ref = error_callback
- return True
- except _PollingLifecycleAbort:
- return False
- except Exception as err:
- if self._teardown_started:
- return False
- if require_progress:
- raise
- if self._looks_like_polling_conflict(err):
- logger.warning(
- "[%s] Telegram polling bootstrap conflict; gateway stays alive while conflict retry runs: %s",
- self.name, _redact_telegram_error_text(err))
- self._spawn_polling_recovery(asyncio.get_running_loop(), self._handle_polling_conflict(err))
- return False
- if self._looks_like_network_error(err):
- self._schedule_polling_recovery(err, reason="polling bootstrap")
- return False
- raise
- async def _go_fatal_network(self, message: str, log_message: str, *log_args) -> None:
- """Retryable ``telegram_network_error`` fatal + runner handoff (supervisor rebuilds the adapter)."""
- logger.error(log_message, *log_args)
- self._set_fatal_error("telegram_network_error", message, retryable=True)
- await self._handoff_polling_fatal_error()
- async def _stop_updater_or_go_fatal(self, app, what: str) -> bool:
- """Bounded ``updater.stop()`` before a recovery restart; False = went fatal, caller returns.
- Wall-clock deadline, not asyncio.wait_for: a CLOSE-WAIT socket wedges stop() on epoll and PTB/AnyIO shielded cleanup
- hangs wait_for. On timeout the Updater's lifecycle lock may still be held, so rebuild the adapter instead."""
- try:
- if app and app.updater and app.updater.running:
- try:
- await _await_with_thread_deadline(app.updater.stop(), timeout=_UPDATER_STOP_TIMEOUT)
- except TimeoutError:
- message = (
- f"Telegram updater.stop() did not finish before the {what} deadline; "
- "rebuilding the adapter instead of reusing an Updater whose lifecycle lock may still be held.")
- await self._go_fatal_network(message, "[%s] %s (likely CLOSE-WAIT socket)", self.name, message)
- return False
- except Exception:
- pass
- return True
- def _restart_polling_in_task(self, coro) -> None:
- """Run a recovery coroutine as the tracked in-flight ``_polling_error_task``."""
- self._polling_error_task = asyncio.get_running_loop().create_task(coro)
- async def _handle_polling_network_error(self, error: Exception) -> None:
- """Reconnect polling after a transient network interruption (NetworkError/TimedOut).
- Host connectivity loss (sleep, WiFi switch, VPN) kills the long-poll silently. Exponential back-off (5s→60s
- cap) up to MAX_NETWORK_RETRIES, then retryable-fatal so the supervisor restarts the gateway.
- A confirmed polling stall (``_PollingStallError``) skips the ladder entirely: the Updater's long-poll
- action never quiesced, so it is handed to the supervisor for a rebuild before any backoff."""
- if self._teardown_started or self.has_fatal_error:
- return
- if isinstance(error, _PollingStallError):
- # Not a retry: no counter bump, no backoff, no in-place stop/drain. The supervisor's rebuild
- # runs disconnect(), which performs the same bounded updater.stop() and app.shutdown().
- message = (
- f"Telegram polling stall confirmed ({_redact_telegram_error_text(error)}); rebuilding the adapter instead of reusing "
- "a wedged Updater/dispatcher in place."
- )
- await self._go_fatal_network(message, "[%s] %s (rebuilding adapter via supervisor)", self.name, message)
- return
- MAX_NETWORK_RETRIES = 10
- BASE_DELAY = 5
- MAX_DELAY = 60
- self._polling_network_error_count += 1
- self._send_path_degraded = True
- attempt = self._polling_network_error_count
- if attempt > MAX_NETWORK_RETRIES:
- message = (
- f'Telegram polling could not reconnect after {MAX_NETWORK_RETRIES:d} network error retries. Escalating to gateway recovery.')
- await self._go_fatal_network(message, "[%s] %s Last error: %s", self.name, message, _redact_telegram_error_text(error))
- return
- delay = min(BASE_DELAY * (2 ** (attempt - 1)), MAX_DELAY)
- logger.warning(
- "[%s] Telegram network error (attempt %d/%d), reconnecting in %ds. Error: %s", self.name, attempt,
- MAX_NETWORK_RETRIES, delay, _redact_telegram_error_text(error))
- await asyncio.sleep(delay)
- if self._teardown_started:
- return
- # Stable local ref: a concurrent disconnect() may set self._app = None while we await.
- app = self._app
- # Unguarded stop() on a CLOSE-WAIT socket would leave _polling_error_task perpetually
- # "in-flight" so every probe skips reconnect for hours.
- if not await self._stop_updater_or_go_fatal(app, "network-recovery") or self._teardown_started:
- return
- # start_polling() bootstraps through the *general* pool before getUpdates; a confirmed pool timeout means the request
- # was never sent, so rebuilding that pool is safe. Generic network errors stay polling-only (sends untouched).
- if self._looks_like_pool_timeout(error):
- await self._drain_general_connections_after_pool_timeout()
- if self._teardown_started:
- return
- await self._drain_polling_connections()
- if self._teardown_started:
- return
- try:
- if not app:
- raise RuntimeError("Telegram application was torn down during reconnect")
- await self._start_polling_once(app, drop_pending_updates=False, error_callback=self._polling_error_callback_ref)
- logger.info(
- "[%s] Telegram polling restarted after network error (attempt %d); health pending getUpdates progress", self.name, attempt)
- except _PollingLifecycleAbort:
- return
- except Exception as retry_err:
- if self._teardown_started:
- return
- logger.warning("[%s] Telegram polling reconnect failed: %s", self.name, _redact_telegram_error_text(retry_err))
- # Polling is dead and no more error callbacks will fire — chain the retry ourselves.
- if not self.has_fatal_error and not self._teardown_started:
- task = asyncio.ensure_future(self._handle_polling_network_error(retry_err))
- self._background_tasks.add(task)
- task.add_done_callback(self._background_tasks.discard)
- # The chained retry IS the in-flight recovery: it must replace the reentrancy guard.
- self._polling_error_task = task
- async def _polling_heartbeat_loop(self) -> None:
- """Detect dead Telegram TCP sockets (CLOSE-WAIT) by periodic probing.
- In CLOSE-WAIT epoll still reports the long-poll socket readable and nothing raises, so PTB's
- ``error_callback`` never fires. Probe ``get_me()`` on the *general* path (never the getUpdates pool);
- connect-level failures feed ``_handle_polling_network_error``. Runs for the connection's lifetime, catching
- steady-state wedges the one-shot verifier can't."""
- HEARTBEAT_INTERVAL = 90 # seconds between probes
- PROBE_TIMEOUT = 15 # seconds before declaring the path dead
- # Wedged-recovery watchdog: note when a recovery task is first seen in-flight and force-escalate
- # if the *same* task object still runs past the stuck timeout.
- # Tracked locally so no _polling_error_task assignment site needs to stamp a timestamp: the
- # heartbeat notes when it first observes a given recovery task still in-flight, and force-escalates
- # if the *same* task object is still running after _POLLING_ERROR_TASK_STUCK_TIMEOUT. A healthy
- # ladder attempt completes (task done) or chains to a new task well before then, so a single
- # long-lived task is unambiguously wedged. See #66377.
- stuck_task_ref: Optional[asyncio.Task] = None
- stuck_task_since = 0.0
- while True:
- try:
- await asyncio.sleep(HEARTBEAT_INTERVAL)
- if self._teardown_started or self.has_fatal_error:
- return
- # A recovery task hung on an unbounded await gates every other recovery path forever
- # (alive but deaf): force retryable-fatal so the reconnector rebuilds the adapter.
- # Independent wedged-recovery watchdog (#66377): if the tracked recovery task has hung (any
- # await no local bound covers), every other recovery path is gated behind it and returns
- # early forever — the gateway stays alive but deaf.
- recovery_task = self._polling_error_task
- if recovery_task is not None and not recovery_task.done():
- now = time.monotonic()
- if recovery_task is not stuck_task_ref:
- stuck_task_ref = recovery_task
- stuck_task_since = now
- elif now - stuck_task_since > _POLLING_ERROR_TASK_STUCK_TIMEOUT:
- stuck_for = now - stuck_task_since
- logger.error(
- "[%s] Telegram reconnect task wedged for %.0fs with no ladder progress; forcing retryable-fatal so the gateway "
- "reconnects instead of staying silently deaf.",
- self.name, stuck_for)
- with contextlib.suppress(Exception):
- recovery_task.cancel()
- self._set_fatal_error(
- "telegram_network_error",
- f"Telegram reconnect task wedged for {stuck_for:.0f}s; forcing gateway reconnect.",
- retryable=True)
- await self._handoff_polling_fatal_error()
- return
- else:
- stuck_task_ref = None
- bot = self._app.bot if self._app else None
- if bot is None:
- continue
- # No get_me() ⇒ not a live polling client (torn down / test double): exit, don't spin.
- if not callable(getattr(bot, "get_me", None)):
- return
- await asyncio.wait_for(bot.get_me(), PROBE_TIMEOUT)
- # get_me() refreshes PTB's cached bot user: adopt a BotFather rename before routing on it.
- self._bot_identity_checked_at = time.monotonic()
- self._note_bot_username(getattr(bot, "username", None))
- # get_me() OK proves only the send path; a wedged long-poll shows as server-side queue.
- # get_me() succeeded — the general/send request path is healthy. That does NOT prove the
- # getUpdates consumer is alive: PTB can report updater.running=True while the long-poll task
- # is wedged, so DMs queue in the Bot API and never reach handlers (#42909). get_me() is
- # blind to this; get_webhook_info() exposes it via pending_update_count. Escalate only after
- # two consecutive probes see a non-zero queue while we believe we're polling, so a single
- # in-flight update (consumed before the next probe) never trips recovery.
- await self._probe_pending_updates(bot, PROBE_TIMEOUT)
- # An empty queue can't hide a wedge forever: no round-trip past the stall threshold ⇒ dead.
- # Even an empty queue cannot hide a wedged long-poll forever: Telegram answers within ~50s,
- # so a consumer with no successful round-trip past the stall threshold is dead (#92991).
- # Pure local-state check — no Bot API call needed.
- await self._check_polling_stall()
- # Transport health is not dispatch health (#102260). Pure local-state check.
- self._check_ingress_dispatch_stall()
- except asyncio.CancelledError:
- return
- except (TimeoutError, OSError) as probe_err:
- self._schedule_polling_recovery(probe_err, reason="heartbeat probe")
- except Exception as probe_err:
- # Non-connectivity errors (e.g. TelegramError 401) aren't CLOSE-WAIT symptoms.
- if self._looks_like_network_error(probe_err):
- self._schedule_polling_recovery(probe_err, reason="heartbeat probe")
- async def _probe_pending_updates(self, bot, probe_timeout: float) -> None:
- """Detect a wedged or stopped getUpdates consumer via pending_update_count.
- PTB can report ``updater.running`` while the long-poll is stuck; get_me() stays healthy yet DMs queue in the
- Bot API. A stuck queue over two consecutive probes ⇒ dead consumer. Also covers the updater having stopped
- entirely (``running=False``, no reconnect in flight).
- PTB can report ``updater.running == True`` while its long-poll task is silently stuck (e.g. a socket
- that epoll keeps reporting readable on WSL2). ``get_me()`` stays healthy because it uses the general
- request path, so the CLOSE-WAIT heartbeat never fires — yet DMs queue in the Bot API and never reach
- handlers (#42909).
- We detect the stopped updater directly and feed the same ladder (#55769).
- """
- # Polling mode only: in webhook mode Telegram pushes and holds no server-side queue.
- if self._teardown_started or self._webhook_mode:
- return
- # An in-flight reconnect owns recovery — don't double-trigger, and don't misread its brief
- # stop()->start_polling() window (updater.running transiently False) as dead.
- if self._recovery_in_flight():
- self._polling_not_running_count = 0
- return
- updater = getattr(self._app, "updater", None) if self._app else None
- if updater is None:
- self._polling_pending_stuck_count = 0
- return
- if not getattr(updater, "running", False):
- # Long-poll task gone, general-path calls still succeed, so no error_callback/probe ever
- # fires. Debounced over two probes so a just-starting updater never trips it.
- self._polling_pending_stuck_count = 0
- # We are in polling mode with no reconnect in flight, yet PTB's updater has stopped entirely.
- # This is distinct from the wedged-but-running consumer handled below: the long-poll task is
- # gone, get_me()/get_webhook_info() on the general request path still succeed, so no
- # error_callback or connectivity probe ever fires and the gateway silently stops receiving
- # messages while the process stays alive (#55769).
- self._polling_not_running_count += 1
- logger.warning(
- "[%s] Telegram polling heartbeat: updater stopped while in polling mode (stuck probe %d/2)", self.name,
- self._polling_not_running_count)
- if self._polling_not_running_count >= 2:
- self._polling_not_running_count = 0
- self._escalate_stuck_consumer(
- "[%s] Telegram updater is not running (long-poll task gone); triggering polling restart",
- "Telegram updater stopped while in polling mode")
- return
- self._polling_not_running_count = 0
- get_webhook_info = getattr(bot, "get_webhook_info", None)
- if not callable(get_webhook_info):
- return
- try:
- info = await asyncio.wait_for(get_webhook_info(), probe_timeout) # type: ignore[arg-type]
- except (TimeoutError, OSError):
- return # connectivity symptom for the get_me() path, not a stuck-queue signal
- pending = int(getattr(info, "pending_update_count", 0) or 0)
- if pending <= 0:
- self._polling_pending_stuck_count = 0
- return
- self._polling_pending_stuck_count += 1
- logger.warning(
- "[%s] Telegram polling heartbeat: %d update(s) queued but not consumed (stuck probe %d/2)", self.name,
- pending, self._polling_pending_stuck_count)
- if self._polling_pending_stuck_count >= 2:
- self._polling_pending_stuck_count = 0
- self._escalate_stuck_consumer(
- "[%s] getUpdates consumer appears wedged (queue not draining); triggering polling restart",
- "getUpdates consumer wedged: pending updates not draining")
- def _escalate_stuck_consumer(self, log_message: str, reason: str) -> None:
- """Second consecutive stuck probe: restart polling via the network-error ladder (unless tearing down)."""
- if self._teardown_started:
- return
- logger.warning(log_message, self.name)
- self._polling_error_task = asyncio.get_running_loop().create_task(self._handle_polling_network_error(RuntimeError(reason)))
- def _check_ingress_dispatch_stall(self) -> None:
- """Escalate fetched updates PTB's dispatcher is not handing to handlers (#102260, #130407).
- ``received`` and ``dispatched`` count the same population (every fetched update reaches the
- group-99 catch-all: no handler raises ApplicationHandlerStop, no error handler is registered),
- so a backlog with no dispatch progress across ``_INGRESS_DISPATCH_STALL_HEARTBEATS`` heartbeats
- is a wedged dispatcher at any traffic rate. Hands the adapter to the supervisor for a rebuild once
- per stall (``_PollingStallError``: an in-place restart keeps the wedged dispatcher); re-arms on progress.
- """
- if self._webhook_mode or self._teardown_started or self.has_fatal_error or self._recovery_in_flight():
- return
- received = getattr(self, "_updates_received_total", 0)
- dispatched = getattr(self, "_updates_dispatched_total", 0)
- if received <= dispatched or dispatched != getattr(self, "_ingress_dispatched_seen", 0):
- self._ingress_dispatched_seen = dispatched
- self._ingress_stalled_heartbeats = 0
- return
- stalled = getattr(self, "_ingress_stalled_heartbeats", 0)
- if stalled >= _INGRESS_DISPATCH_STALL_HEARTBEATS:
- return # already reported this stall
- self._ingress_stalled_heartbeats = stalled + 1
- if stalled + 1 < _INGRESS_DISPATCH_STALL_HEARTBEATS:
- return
- # No pre-log: the handoff warning and ``_go_fatal_network`` carry this text (see _check_polling_stall).
- generation = getattr(self, "_polling_generation", 0)
- self._schedule_polling_recovery(
- _PollingStallError(
- f"ingress healthy but deaf: PTB dispatcher made no progress for {_INGRESS_DISPATCH_STALL_HEARTBEATS} "
- f"heartbeats with {received - dispatched} update(s) fetched but not dispatched "
- f"({received} received, {dispatched} dispatched, generation {generation})"),
- reason="ingress dispatch stall watchdog")
- async def _check_polling_stall(self) -> None:
- """Watchdog the last successful getUpdates round-trip: a long-poll can wedge without raising
- (CLOSE-WAIT after a route flip) while every other probe stays blind; no round-trip for
- ``_POLLING_STALL_TIMEOUT`` ⇒ raise ``_PollingStallError`` so the recovery path hands the
- adapter to the supervisor for a rebuild instead of reusing the wedged Updater.
- See #92991, #113618.
- """
- if self._webhook_mode or self._teardown_started or self.has_fatal_error or self._recovery_in_flight():
- return
- now = time.monotonic()
- last_progress = getattr(self, "_polling_last_progress_monotonic", None)
- generation_started = getattr(self, "_polling_generation_started_monotonic", None)
- if last_progress is not None:
- stalled_for = now - last_progress
- elif generation_started is not None:
- # No round-trip yet this generation: fallback for when the one-shot verifier could not run.
- stalled_for = now - generation_started
- else:
- return
- if stalled_for <= _POLLING_STALL_TIMEOUT:
- return
- # No pre-log here: the recovery path logs the hand-off and ``_go_fatal_network`` the one
- # error-level line, so a stall does not announce itself twice.
- self._schedule_polling_recovery(
- _PollingStallError(
- f"getUpdates made no progress for {stalled_for:.0f}s (generation {getattr(self, '_polling_generation', 0)}; polling stall watchdog)"),
- reason="polling stall watchdog")
- def _verifier_stale(self, generation: int, progress: asyncio.Event) -> bool:
- """True when a verifier's generation no longer matters (progressed, fatal, replaced, torn down)."""
- return (
- self._teardown_started or progress.is_set() or self.has_fatal_error
- or not self._polling_progress_accepting or generation != self._polling_generation
- or progress is not self._polling_progress_event)
- async def _verify_polling_after_reconnect(self, generation: Optional[int] = None, progress: Optional[asyncio.Event] = None) -> None:
- """Require getUpdates progress, using getMe only to classify failure: a general-path getMe
- success cannot heal polling health. Connectivity failures enter the guarded recovery ladder."""
- PROBE_TIMEOUT = 10
- if self._teardown_started:
- return
- if generation is None:
- generation = self._polling_generation
- if progress is None:
- progress = self._polling_progress_event
- with contextlib.suppress(asyncio.TimeoutError):
- await asyncio.wait_for(progress.wait(), timeout=_POLLING_PROGRESS_TIMEOUT)
- if self._verifier_stale(generation, progress):
- return
- app = self._app
- if not (app and app.updater and app.updater.running):
- logger.warning("[%s] Updater made no getUpdates progress and is not running", self.name)
- self._schedule_polling_recovery(
- RuntimeError("Updater not running after polling progress deadline"),
- reason="polling progress verifier: updater not running")
- return
- try:
- await asyncio.wait_for(app.bot.get_me(), PROBE_TIMEOUT)
- except Exception as probe_err:
- if self._verifier_stale(generation, progress):
- return
- if not self._looks_like_network_error(probe_err):
- logger.warning(
- "[%s] Polling progress verifier hit a non-connectivity error (not retrying): %s", self.name,
- _redact_telegram_error_text(probe_err))
- return
- logger.warning(
- "[%s] Polling progress verifier connectivity probe failed: %s", self.name, _redact_telegram_error_text(probe_err))
- self._schedule_polling_recovery(probe_err, reason="polling progress verifier connectivity failure")
- return
- if self._verifier_stale(generation, progress):
- return
- self._schedule_polling_recovery(
- _PollingStallError("getUpdates made no progress before verifier deadline"),
- reason="polling progress verifier: general path healthy but getUpdates stalled")
- def _disarm_ptb_retry_loop(self) -> None:
- """Synchronously stop PTB's internal polling retry loop.
- PTB's ``network_retry_loop`` calls our ``error_callback`` *synchronously* on a 409 Conflict then polls again; our
- callback only schedules async recovery, so two sessions overlap and Telegram 409s on a ~31s cadence. Setting PTB's
- private ``stop_event`` makes its loop exit on the next tick; ``updater.stop()`` + drain + ``start_polling()`` then build
- a fresh one. Best-effort across PTB spellings. Deliberately NOT flipping ``updater._running``: stop() raises when
- already False, which would skip the real teardown and poison the next start."""
- updater = getattr(self._app, "updater", None) if self._app else None
- if updater is None:
- return
- for attr in ("_Updater__polling_task_stop_event", "_polling_task_stop_event"):
- stop_event = getattr(updater, attr, None)
- if isinstance(stop_event, asyncio.Event):
- if not stop_event.is_set():
- stop_event.set()
- logger.debug("[%s] Disarmed PTB polling retry loop via %s", self.name, attr)
- return
- logger.debug(
- "[%s] Could not disarm PTB polling retry loop (stop_event not found on this PTB version); falling back to async stop()",
- self.name)
- async def _handle_polling_conflict(self, error: Exception) -> None:
- """Recover a 409 Conflict: the previous gateway process was killed but Telegram holds its
- getUpdates session ~30s. Stop, wait (growing delay), drain, restart — MAX_CONFLICT_RETRIES
- times before going fatal; a failed retry must never return silently (limbo)."""
- if self._teardown_started:
- return
- if self.has_fatal_error and self.fatal_error_code == "telegram_polling_conflict":
- return
- self._polling_conflict_count += 1
- MAX_CONFLICT_RETRIES = 5
- # 15s, 25s, 35s, 45s, 55s — clears Telegram's ~30s session window without hammering the API.
- RETRY_DELAY = 10 + (self._polling_conflict_count * 10) # seconds
- if self._polling_conflict_count <= MAX_CONFLICT_RETRIES:
- logger.warning(
- "[%s] Telegram polling conflict (%d/%d) — previous session still "
- "held open on Telegram's servers. Waiting %ds for it to expire. Error: %s",
- self.name, self._polling_conflict_count, MAX_CONFLICT_RETRIES,
- RETRY_DELAY, _redact_telegram_error_text(error))
- # Stop the updater before sleeping (no-op if PTB raised before running was set).
- if not await self._stop_updater_or_go_fatal(self._app, "conflict-retry"):
- return
- await asyncio.sleep(RETRY_DELAY)
- if self._teardown_started:
- return
- await self._drain_polling_connections()
- if self._teardown_started:
- return
- # Stable local ref: a concurrent disconnect() may null self._app across the awaits above.
- app = self._app
- # Capture a stable local reference: self._app can be reassigned to None by a concurrent
- # disconnect() while we're suspended across the awaits above (same race #55992 fixed on the
- # network path). Re-reading self._app after that point would raise AttributeError deep inside
- # start_polling instead of failing fast here, where the except below reschedules or escalates to
- # fatal.
- expected_generation = self._polling_generation + 1
- if not app:
- raise RuntimeError("Telegram application was torn down during conflict reconnect")
- # drop_pending_updates=True makes Telegram terminate any other getUpdates session for this
- # token (zombie or our own prior retry); without it each retry is immediately 409'd.
- # The competing session is either a zombie from the previous gateway process (whose long-poll
- # hasn't expired server-side yet) or our own previous retry's still-expiring session. Without
- # this, each retry starts a new getUpdates session that immediately gets 409'd by the previous
- # one, creating the very conflict we are trying to recover from (#75017).
- self._polling_conflict_recovery_generation = expected_generation
- try:
- await self._start_polling_once(app, drop_pending_updates=True, error_callback=self._polling_error_callback_ref)
- logger.info(
- "[%s] Telegram polling restarted after conflict retry %d/%d; health pending getUpdates progress",
- self.name, self._polling_conflict_count, MAX_CONFLICT_RETRIES)
- return
- except _PollingLifecycleAbort:
- return
- except Exception as retry_err:
- if self._teardown_started:
- return
- logger.warning(
- "[%s] Telegram polling retry %d/%d failed: %s. Scheduling next attempt.", self.name,
- self._polling_conflict_count, MAX_CONFLICT_RETRIES, _redact_telegram_error_text(retry_err))
- # Never return silently: alive-and-"connected" with no polling is limbo.
- if self._polling_conflict_count < MAX_CONFLICT_RETRIES and not self._teardown_started:
- # get_running_loop(): get_event_loop() raises on 3.10+ from PTB's callback context.
- self._restart_polling_in_task(self._handle_polling_conflict(retry_err))
- return
- # Fall through to fatal on the last retry.
- finally:
- if self._polling_conflict_recovery_generation == expected_generation:
- self._polling_conflict_recovery_generation = None
- if self._teardown_started:
- return
- # Retries exhausted — fatal so the runner surfaces it and the user knows to act.
- message = (
- f"Telegram polling could not recover after {MAX_CONFLICT_RETRIES} retries "
- f"({sum(10 + i * 10 for i in range(1, MAX_CONFLICT_RETRIES + 1))}s total wait). "
- "The previous gateway session is still held open on Telegram's servers, "
- "or another process is using the same bot token. To recover: ensure no other Hermes or OpenClaw instance is running "
- "with this token, then restart the gateway with 'hermes gateway restart'.")
- logger.error("[%s] %s Original error: %s", self.name, message, _redact_telegram_error_text(error))
- # Snapshot whether WE transition to fatal: a concurrent retry task suspended past the entry
- # guard reaches this branch too. Only the first transition notifies.
- _already_fatal = self.has_fatal_error and self.fatal_error_code == "telegram_polling_conflict"
- self._set_fatal_error("telegram_polling_conflict", message, retryable=False)
- try:
- if self._app and self._app.updater:
- await _await_with_thread_deadline(self._app.updater.stop(), timeout=_UPDATER_STOP_TIMEOUT)
- except TimeoutError:
- logger.warning("[%s] updater.stop() timed out after exhausting conflict retries (likely CLOSE-WAIT socket); proceeding to fatal notify", self.name)
- except Exception as stop_error:
- logger.warning(
- "[%s] Failed stopping Telegram updater after exhausting conflict retries: %s", self.name, stop_error,
- exc_info=True,
- )
- if not _already_fatal:
- await self._handoff_polling_fatal_error()
- async def _handoff_polling_fatal_error(self) -> None:
- """Notify the runner without letting child teardown cancel this owner: ``disconnect()`` cancels
- the tracked recovery/heartbeat tasks, so release only the current owner from its field."""
- current_task = asyncio.current_task()
- if self._polling_error_task is current_task:
- self._polling_error_task = None
- if getattr(self, "_polling_heartbeat_task", None) is current_task:
- self._polling_heartbeat_task = None
- await self._notify_fatal_error()
- async def _create_dm_topic(
- self, chat_id: int, name: str, icon_color: Optional[int] = None, icon_custom_emoji_id: Optional[str] = None) -> Optional[int]:
- """Create a forum topic in a private (DM) chat (Bot API 9.4+); message_thread_id or None."""
- if not self._bot:
- return None
- try:
- kwargs: dict[str, Any] = {"chat_id": chat_id, "name": name}
- if icon_color is not None:
- kwargs["icon_color"] = icon_color
- if icon_custom_emoji_id:
- kwargs["icon_custom_emoji_id"] = icon_custom_emoji_id
- topic = await self._bot.create_forum_topic(**kwargs)
- thread_id = topic.message_thread_id
- logger.info("[%s] Created DM topic '%s' in chat %s -> thread_id=%s", self.name, name, chat_id, thread_id)
- return thread_id
- except Exception as e:
- error_text = str(e).lower()
- # Telegram has no "list topics" API: an existing topic is mapped from incoming messages.
- if "topic_name_duplicate" in error_text or "already" in error_text:
- logger.info(
- "[%s] DM topic '%s' already exists in chat %s (will be mapped from incoming messages)", self.name, name, chat_id)
- elif "not a forum" in error_text or "forums_disabled" in error_text:
- logger.warning(
- "[%s] Cannot create DM topic '%s' in chat %s: Threaded Mode is not enabled. "
- "The bot owner must open the BotFather Mini App (search 'botfather' in "
- "Telegram, tap Open on the search result) -> My bots -> this bot -> Bot "
- "Settings -> Threads Settings -> enable Threaded Mode. This cannot be enabled "
- "from the DM chat, nor from the BotFather /mybots text menu.",
- self.name, name, chat_id)
- else:
- logger.warning(
- "[%s] Failed to create DM topic '%s' in chat %s: %s", self.name, name, chat_id, _redact_telegram_error_text(e))
- return None
- async def create_handoff_thread(self, parent_chat_id: str, name: str) -> Optional[str]:
- """Create a forum topic for a session handoff; ``message_thread_id`` as str, or None."""
- try:
- chat_id_int = int(parent_chat_id)
- except (TypeError, ValueError):
- return None
- thread_id = await self._create_dm_topic(chat_id_int, name=name)
- return str(thread_id) if thread_id else None
- async def ensure_dm_topic(self, chat_id: str, topic_name: str, force_create: bool = False) -> Optional[str]:
- """Return a private DM topic thread id, creating and persisting it if needed."""
- name = str(topic_name or "").strip()
- if not name:
- return None
- try:
- chat_id_int = int(chat_id)
- except (TypeError, ValueError):
- return None
- cache_key = f"{chat_id_int}:{name}"
- cached = self._dm_topics.get(cache_key)
- if cached and not force_create:
- return str(cached)
- topic_conf: Optional[dict[str, Any]] = None
- chat_entry: Optional[dict[str, Any]] = None
- for entry in self._dm_topics_config:
- if str(entry.get("chat_id")) != str(chat_id_int):
- continue
- chat_entry = entry
- topic_conf = next((c for c in entry.get("topics", []) if c.get("name") == name), None)
- break
- if topic_conf and topic_conf.get("thread_id") and not force_create:
- thread_id = int(topic_conf["thread_id"])
- self._dm_topics[cache_key] = thread_id
- return str(thread_id)
- if chat_entry is None:
- chat_entry = {"chat_id": chat_id_int, "topics": []}
- self._dm_topics_config.append(chat_entry)
- if topic_conf is None:
- topic_conf = {"name": name}
- chat_entry.setdefault("topics", []).append(topic_conf)
- thread_id = await self._create_dm_topic(
- chat_id_int, name=name, icon_color=topic_conf.get("icon_color"), icon_custom_emoji_id=topic_conf.get("icon_custom_emoji_id"))
- if not thread_id:
- return None
- topic_conf["thread_id"] = thread_id
- self._dm_topics[cache_key] = int(thread_id)
- self._persist_dm_topic_thread_id(chat_id_int, name, int(thread_id), replace_existing=force_create)
- return str(thread_id)
- async def rename_dm_topic(self, chat_id: int, thread_id: int, name: str) -> None:
- """Rename a forum topic in a private (DM) chat."""
- if not self._bot:
- return
- try:
- chat_id_arg = int(chat_id)
- except (TypeError, ValueError):
- chat_id_arg = chat_id
- await self._bot.edit_forum_topic(chat_id=chat_id_arg, message_thread_id=int(thread_id), name=name)
- logger.info("[%s] Renamed DM topic in chat %s thread_id=%s -> '%s'", self.name, chat_id, thread_id, name)
- def _persist_dm_topic_thread_id(self, chat_id: int, topic_name: str, thread_id: int, replace_existing: bool = False) -> None:
- """Save a newly created thread_id back into config.yaml so it survives restarts."""
- try:
- from hermes_constants import get_hermes_home
- from hermes_cli.config import atomic_config_write, config_exists, read_user_config_raw
- config_path = get_hermes_home() / "config.yaml"
- if not config_exists(config_path):
- logger.warning("[%s] Config file not found at %s, cannot persist thread_id", self.name, config_path)
- return
- config = read_user_config_raw(config_path)
- # platforms.telegram.extra.dm_topics — create the path for topics not predeclared in config.yaml.
- dm_topics = config.setdefault("platforms", {}).setdefault("telegram", {}).setdefault("extra", {}).setdefault("dm_topics", [])
- changed = False
- matching_chat_entry = None
- for chat_entry in dm_topics:
- try:
- if int(chat_entry.get("chat_id", 0)) != int(chat_id):
- continue
- except (TypeError, ValueError):
- continue
- matching_chat_entry = chat_entry
- topics = chat_entry.setdefault("topics", [])
- t = next((t for t in topics if t.get("name") == topic_name), None)
- if t is None:
- topics.append({"name": topic_name, "thread_id": thread_id})
- changed = True
- elif (replace_existing or not t.get("thread_id")) and t.get("thread_id") != thread_id:
- t["thread_id"] = thread_id
- changed = True
- break
- if matching_chat_entry is None:
- dm_topics.append({"chat_id": chat_id, "topics": [{"name": topic_name, "thread_id": thread_id}]})
- changed = True
- if changed:
- atomic_config_write(config_path, config)
- logger.info("[%s] Persisted thread_id=%s for topic '%s' in config.yaml", self.name, thread_id, topic_name)
- except Exception as e:
- logger.warning("[%s] Failed to persist thread_id to config: %s", self.name, e, exc_info=True)
- async def _setup_dm_topics(self) -> None:
- """Load or create configured DM topics: ``extra['dm_topics']`` is ``[{"chat_id", "topics": [{"name",
- "icon_color", "thread_id"?, "skill"?}]}]``; persisted thread_ids are cached without an API call."""
- for chat_entry in self._dm_topics_config or ():
- chat_id = chat_entry.get("chat_id")
- topics = chat_entry.get("topics", [])
- if not chat_id or not topics:
- continue
- logger.info("[%s] Setting up %d DM topic(s) for chat %s", self.name, len(topics), chat_id)
- for topic_conf in topics:
- topic_name = topic_conf.get("name")
- if not topic_name:
- continue
- cache_key = f"{chat_id}:{topic_name}"
- existing_thread_id = topic_conf.get("thread_id")
- if existing_thread_id:
- self._dm_topics[cache_key] = int(existing_thread_id)
- logger.info("[%s] DM topic loaded from config: %s -> thread_id=%s", self.name, cache_key, existing_thread_id)
- continue
- thread_id = await self._create_dm_topic(
- chat_id=normalize_telegram_chat_id(chat_id), name=topic_name, icon_color=topic_conf.get("icon_color"),
- icon_custom_emoji_id=topic_conf.get("icon_custom_emoji_id"))
- if not thread_id:
- continue
- self._dm_topics[cache_key] = thread_id
- logger.info("[%s] DM topic cached: %s -> thread_id=%s", self.name, cache_key, thread_id)
- self._persist_dm_topic_thread_id(int(chat_id), topic_name, thread_id)
- # Seed message: Telegram's client hides empty topics until they contain one.
- try:
- await _await_with_thread_deadline(
- self._bot.send_message(
- chat_id=normalize_telegram_chat_id(chat_id), message_thread_id=thread_id, text=f"\U0001f4cc {topic_name}"),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- except Exception as seed_err:
- logger.debug("[%s] Could not send seed message to topic '%s': %s", self.name, topic_name, seed_err)
- async def _bot_identity_refresh_loop(self) -> None:
- """Keep the cached @username fresh in webhook mode (no heartbeat calls ``get_me()`` there)."""
- while True:
- try:
- await asyncio.sleep(self._BOT_IDENTITY_TTL_SECONDS)
- if self._teardown_started or self.has_fatal_error:
- return
- await self._refresh_bot_identity(force=True)
- except asyncio.CancelledError:
- return
- except Exception:
- logger.debug("[%s] Telegram identity refresh loop iteration failed", self.name, exc_info=True)
- def _start_post_connect_housekeeping(self) -> None:
- """Kick off deferred post-connect housekeeping; idempotent while a task is still running."""
- task = self._post_connect_task
- if task and not task.done():
- return
- self._post_connect_task = asyncio.ensure_future(self._run_post_connect_housekeeping())
- async def _register_command_menu(self) -> None:
- """Register the command menu (from COMMAND_REGISTRY) in every scope — Telegram picks the
- narrowest matching one per chat type; forum topics are handled lazily by _ensure_forum_commands."""
- from telegram import BotCommandScopeAllPrivateChats, BotCommandScopeAllGroupChats, BotCommandScopeDefault
- from hermes_cli.commands_platforms import telegram_menu_commands, telegram_menu_max_commands
- if not self._bot:
- return
- # Telegram allows 100 commands but has an undocumented ~4KB payload limit; default cap 60.
- max_commands = telegram_menu_max_commands()
- # Skill discovery resolves every skill path on disk; a slow filesystem after a reconnect must
- # not hold the gateway loop past the liveness watchdog (#110707). Only the Bot API call stays here.
- menu_commands, hidden_count = await asyncio.to_thread(telegram_menu_commands, max_commands=max_commands)
- bot_commands = self._bot_commands(menu_commands)
- # Descriptions are localized, so the fingerprint carries the language: a display.language change
- # invalidates the lazily registered forum scopes (and any future "skip if unchanged" check).
- fingerprint = self._menu_fingerprint(menu_commands)
- if fingerprint != getattr(self, "_command_menu_fingerprint", ""):
- self._forum_command_registered.clear()
- self._command_menu_fingerprint = fingerprint
- for scope_cls in (BotCommandScopeDefault, BotCommandScopeAllPrivateChats, BotCommandScopeAllGroupChats):
- scope_name = getattr(scope_cls, "__name__", str(scope_cls))
- try:
- await self._bot.set_my_commands(bot_commands, scope=scope_cls())
- logger.info("[%s] set_my_commands OK for scope %s (%d cmds)", self.name, scope_name, len(bot_commands))
- except Exception as scope_err:
- logger.warning("[%s] set_my_commands FAILED for scope %s: %s", self.name, scope_name, scope_err)
- if hidden_count:
- logger.info(
- "[%s] Telegram menu: %d commands registered, %d hidden (over %d limit). Use /commands for full list.",
- self.name, len(menu_commands), hidden_count, max_commands)
- @staticmethod
- def _bot_commands(menu_commands) -> list:
- """``BotCommand`` rows; Telegram caps a description at 256 chars (longer locales get cut)."""
- from telegram import BotCommand
- return [BotCommand(name, str(desc or "")[:_BOT_COMMAND_DESCRIPTION_LIMIT]) for name, desc in menu_commands]
- @staticmethod
- def _menu_fingerprint(menu_commands) -> str:
- import hashlib
- payload = repr([get_language(), [tuple(row) for row in menu_commands]])
- return hashlib.sha256(payload.encode("utf-8")).hexdigest()
- async def _run_post_connect_housekeeping(self) -> None:
- """Command menu, status indicator and DM topics off the connect path; every step is non-fatal.
- DM topics — all off the connect path so a slow Bot API call cannot blow the gateway connect timeout
- (#46298).
- """
- try:
- try:
- await self._register_command_menu()
- except Exception as e:
- logger.warning(
- "[%s] Could not register Telegram command menu: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
- with contextlib.suppress(Exception):
- await self._set_status_indicator(online=True)
- try:
- await self._setup_dm_topics()
- except Exception as topics_err:
- logger.warning("[%s] DM topics setup failed (non-fatal): %s", self.name, topics_err, exc_info=True)
- except asyncio.CancelledError:
- raise
- finally:
- if self._post_connect_task is asyncio.current_task():
- self._post_connect_task = None
- async def _on_platform_update(self, update, context) -> None:
- """Catch-all PTB handler (group 99) firing ``gateway_platform_event`` per inbound update with a
- stable envelope (no raw SDK objects) and an internal auth source. Never raises into PTB."""
- # Admission counts dispatch before any preparation can stop this group. Retain
- # accounting for callers outside that Application boundary (#102260).
- admission = getattr(self, "_update_admission", None)
- if admission is None or admission.get() is None:
- self._updates_dispatched_total = getattr(self, "_updates_dispatched_total", 0) + 1
- handler: Optional[Callable[[dict[str, Any], Any], Awaitable[None]]] = getattr(self, "_platform_event_handler", None)
- if handler is None:
- return
- try:
- from hermes_cli.lifecycle import has_hook
- if not has_hook("gateway_platform_event"):
- return
- event = self._normalize_platform_event(update)
- except Exception:
- self._fail_update_preparation()
- logger.debug("[%s] gateway_platform_event normalize error", self.name, exc_info=True)
- return
- if event is None:
- return
- # The gateway-owned boundary runs the full profile-scoped auth chain before plugin dispatch.
- try:
- source = self._source_for_platform_event_auth(update)
- self._accept_update()
- await handler(event, source)
- except Exception:
- self._fail_update_preparation()
- logger.debug("[%s] gateway_platform_event dispatch error", self.name, exc_info=True)
- def _source_for_platform_event_auth(self, update):
- """Route a supported update to its event-specific auth-source extractor (reactor / editor);
- raises ``ValueError`` for updates without one so the boundary fails closed."""
- if getattr(update, "message_reaction", None) is not None:
- return self._source_from_reaction_for_auth(update)
- edited = getattr(update, "edited_message", None)
- if edited is not None:
- source = self._source_from_message_for_auth(edited)
- # Tolerates missing identities for pairing-flow callers; this boundary must not.
- if not source.user_id or not source.chat_id:
- raise ValueError("gateway_platform_event message_edited requires editor and chat identities")
- return source
- raise ValueError("gateway_platform_event source extraction has no extractor for this update type")
- def _normalize_platform_event(self, update) -> Optional[dict[str, Any]]:
- """Map a PTB update to a ``{platform, event_type, payload}`` envelope (hooks.md contracts), or
- ``None`` for types without one."""
- if getattr(update, "message_reaction", None) is not None:
- return self._normalize_reaction_event(update)
- if getattr(update, "edited_message", None) is not None:
- return self._normalize_message_edited_event(update)
- return None
- @staticmethod
- def _is_id_like(value: Any) -> bool:
- return not isinstance(value, bool) and isinstance(value, (str, int))
- def _normalize_reaction_event(self, update) -> Optional[dict[str, Any]]:
- """``message_reaction`` → ``reaction`` event: emojis (unicode), custom_emoji_ids, chat_id,
- message_id, thread_id (always None — reactions carry none)."""
- mr = getattr(update, "message_reaction", None)
- if mr is None:
- return None
- chat = getattr(mr, "chat", None)
- new_reaction = getattr(mr, "new_reaction", None) or []
- if not isinstance(new_reaction, (list, tuple)):
- return None
- chat_id = getattr(chat, "id", None) if chat is not None else None
- message_id = getattr(mr, "message_id", None)
- if not self._is_id_like(chat_id) or not self._is_id_like(message_id):
- return None
- emojis: list[str] = []
- custom_emoji_ids: list[str] = []
- for r in new_reaction[:64]:
- emoji = getattr(r, "emoji", None)
- if isinstance(emoji, str) and emoji:
- emojis.append(emoji[:64])
- custom_id = getattr(r, "custom_emoji_id", None)
- if self._is_id_like(custom_id):
- custom_emoji_ids.append(str(custom_id)[:128])
- return {
- "platform": "telegram",
- "event_type": "reaction",
- "payload": {
- "emojis": emojis, "custom_emoji_ids": custom_emoji_ids, "chat_id": str(chat_id)[:128],
- "message_id": str(message_id)[:128], "thread_id": None},
- }
- def _normalize_message_edited_event(self, update) -> Optional[dict[str, Any]]:
- """``edited_message`` → ``message_edited`` event (v1, additive): chat_id, message_id, thread_id
- (forum topic), text (edited text or caption, bounded), edited_at (ISO 8601 UTC or None)."""
- message = getattr(update, "edited_message", None)
- if message is None:
- return None
- chat = getattr(message, "chat", None)
- chat_id = getattr(chat, "id", None) if chat is not None else None
- message_id = getattr(message, "message_id", None)
- if not self._is_id_like(chat_id) or not self._is_id_like(message_id):
- return None
- text = getattr(message, "text", None) or getattr(message, "caption", None)
- if not isinstance(text, str):
- text = None
- thread_id = None
- thread_id_raw = getattr(message, "message_thread_id", None)
- if self._is_id_like(thread_id_raw) and bool(getattr(message, "is_topic_message", False)):
- thread_id = str(thread_id_raw)[:128]
- edited_at = None
- edit_date = getattr(message, "edit_date", None)
- try:
- if edit_date is not None and hasattr(edit_date, "isoformat"):
- edited_at = str(edit_date.isoformat())[:64]
- except Exception:
- edited_at = None
- return {
- "platform": "telegram",
- "event_type": "message_edited",
- "payload": {
- "chat_id": str(chat_id)[:128], "message_id": str(message_id)[:128], "thread_id": thread_id,
- "text": text[:8192] if text is not None else None, "edited_at": edited_at},
- }
- def _accept_update(self) -> None:
- """Handoff is irreversible for replay purposes, even if later work raises/cancels."""
- admission = getattr(self, "_update_admission", None)
- claim = admission.get() if admission is not None else None
- if claim is not None:
- claim.accepted = True
- def _fail_update_preparation(self) -> None:
- """A caught preparation error is not a successful no-op or an auth refusal."""
- admission = getattr(self, "_update_admission", None)
- claim = admission.get() if admission is not None else None
- if claim is not None:
- claim.failed = True
- async def handle_message(self, event: MessageEvent) -> None:
- self._accept_update()
- await apply_root_dm_topic_transfer(self, event) # opt-in dm_topic_auto_transfer (#139136)
- await super().handle_message(event)
- def set_topic_transfer_fn(self, fn: Optional[TopicTransferFn]) -> None:
- """Install the runner's root-DM topic-transfer hook (``dm_topic_auto_transfer``); None clears."""
- self._topic_transfer_fn = fn
- def _register_handlers(self, app) -> None:
- """Register every PTB handler on ``app`` (initial connect and the transient-init rebuild)."""
- table = getattr(app, "handlers", None)
- core_before = {g: len(hs) for g, hs in table.items()} if isinstance(table, dict) else {}
- app.add_handler(TelegramMessageHandler(filters.TEXT & ~filters.COMMAND, self._handle_text_message))
- app.add_handler(TelegramMessageHandler(filters.COMMAND, self._handle_command))
- app.add_handler(TelegramMessageHandler(
- filters.LOCATION | getattr(filters, "VENUE", filters.LOCATION), self._handle_location_message))
- app.add_handler(TelegramMessageHandler(
- filters.PHOTO | filters.VIDEO | filters.VIDEO_NOTE | filters.AUDIO | filters.VOICE | filters.Document.ALL | filters.Sticker.ALL,
- self._handle_media_message))
- # Topic-creation service messages carry the new topic id and no text, so no other handler sees
- # them. They are a routing signal only (never an agent turn): see _handle_forum_topic_created.
- created_filter = getattr(getattr(filters, "StatusUpdate", None), "FORUM_TOPIC_CREATED", None)
- if created_filter is not None:
- app.add_handler(TelegramMessageHandler(created_filter, self._handle_forum_topic_created))
- app.add_handler(CallbackQueryHandler(self._handle_callback_query))
- # Inline command picker; inert until the owner enables inline mode via BotFather /setinline.
- app.add_handler(InlineQueryHandler(self._handle_inline_query))
- # gateway_platform_event observer: group 99 observes alongside, never displaces, core handlers.
- app.add_handler(TypeHandler(Update, self._on_platform_update), group=99)
- # Everything appended above is core; a late plugin re-wire must land BEFORE these (#87770).
- if isinstance(table, dict):
- self._core_handler_ids = {id(h) for g, hs in table.items() for h in hs[core_before.get(g, 0):]}
- def _wire_plugin_handlers(self, native: Any = None) -> None:
- """PTB dispatches the FIRST matching handler per group and core registers catch-alls
- (``filters.COMMAND``, ``CallbackQueryHandler``), so a plugin handler appended after connect
- would never fire. Move whatever a late factory added ahead of the first core handler of its
- group, keeping the plugin handlers' own relative order."""
- handlers = getattr(native, "handlers", None)
- core_ids = getattr(self, "_core_handler_ids", None)
- if not isinstance(handlers, dict) or not core_ids:
- super()._wire_plugin_handlers(native) # first wire runs before core registers: nothing to hoist
- return
- before = {g: list(hs) for g, hs in handlers.items()}
- super()._wire_plugin_handlers(native)
- for group, current in handlers.items():
- prior = before.get(group, [])
- prior_ids = {id(h) for h in prior}
- added = [h for h in current if id(h) not in prior_ids]
- first_core = next((i for i, h in enumerate(prior) if id(h) in core_ids), None)
- if not added or first_core is None:
- continue
- current[:] = prior[:first_core] + added + prior[first_core:]
- async def _build_ptb_requests(self) -> tuple:
- """Build the (general, getUpdates) HTTPXRequest pair: fallback-IP transport, explicit proxy, or
- direct DNS; the getUpdates request is instrumented for polling-progress tracking."""
- # PTB's pool_timeout=1s default trips "Pool timeout" on flaky networks; safer defaults + env overrides.
- request_kwargs = {
- "connection_pool_size": env_int("HERMES_TELEGRAM_HTTP_POOL_SIZE", 512),
- "pool_timeout": env_float("HERMES_TELEGRAM_HTTP_POOL_TIMEOUT", 8.0),
- "connect_timeout": env_float("HERMES_TELEGRAM_HTTP_CONNECT_TIMEOUT", 10.0),
- "read_timeout": env_float("HERMES_TELEGRAM_HTTP_READ_TIMEOUT", 20.0),
- "write_timeout": env_float("HERMES_TELEGRAM_HTTP_WRITE_TIMEOUT", 20.0),
- # PTB routes file requests to media_write_timeout; httpx budgets it per socket write (stall
- # tolerance, not bandwidth), so 60s rides out congested-link buffer stalls.
- "media_write_timeout": 60.0,
- }
- # CLOSE_WAIT fd leak: PTB's httpx.AsyncClient has no keepalive tuning; inject platform_httpx_limits()
- # while preserving PTB's max_connections (httpx_kwargs is spread last, so `limits` here wins).
- # CLOSE_WAIT fd leak (#31599, same class as #18451): PTB's HTTPXRequest builds the underlying
- # httpx.AsyncClient with `limits = httpx.Limits(max_connections=connection_pool_size)` and *no*
- # keepalive tuning, so httpx's default keepalive_expiry=5.0 applies. Behind an HTTP proxy
- # (Cloudflare Warp etc.) a peer-initiated FIN can sit in CLOSE_WAIT longer than that, leaking fds in
- # the general request pool (_request[1]) which _drain_polling_connections never resets.
- from gateway.platforms._http_client_limits import platform_httpx_limits
- _base_limits = platform_httpx_limits()
- if _base_limits is not None:
- import httpx as _httpx
- _pool_limits = _httpx.Limits(
- max_connections=request_kwargs["connection_pool_size"],
- max_keepalive_connections=_base_limits.max_keepalive_connections, keepalive_expiry=_base_limits.keepalive_expiry)
- # A long-poll is continuously active, so keepalive expiry can't protect it from a server-side
- # close: never hand getUpdates a pooled socket from a previous poll.
- _updates_limits = _httpx.Limits(
- max_connections=request_kwargs["connection_pool_size"], max_keepalive_connections=0,
- keepalive_expiry=_base_limits.keepalive_expiry)
- else: # pragma: no cover — httpx always present alongside PTB
- _pool_limits = _updates_limits = None
- def _with_limits(httpx_kwargs: Optional[dict] = None) -> dict:
- """Merge tuned limits into httpx client kwargs (proxy/direct branches only; the fallback-IP
- branch must pass limits straight into the transport — httpx ignores client `limits` then)."""
- kwargs = dict(httpx_kwargs or {})
- if _pool_limits is not None and "limits" not in kwargs:
- kwargs["limits"] = _pool_limits
- return kwargs
- disable_fallback = os.getenv("HERMES_TELEGRAM_DISABLE_FALLBACK_IPS", "").strip().lower() in {"1", "true", "yes", "on"}
- fallback_ips = [] if disable_fallback else self._fallback_ips()
- if not fallback_ips and not disable_fallback:
- discovery_timeout = self._env_float_clamped("HERMES_TELEGRAM_FALLBACK_DISCOVERY_TIMEOUT", 5.0, min_value=0.0)
- logger.warning("[%s] Discovering Telegram API fallback IPs via DNS-over-HTTPS…", self.name)
- try:
- fallback_ips = await _await_with_thread_deadline(discover_fallback_ips(), timeout=discovery_timeout)
- except Exception as exc:
- logger.warning(
- "[%s] Telegram fallback-IP discovery failed after %.0fs; "
- "using seed IPv4 Telegram API IPs so a blackholed IPv6 hostname path cannot hang initialize() (#87015): %s",
- self.name, discovery_timeout, _redact_telegram_error_text(exc))
- fallback_ips = list(SEED_FALLBACK_IPS)
- else:
- logger.info("[%s] Auto-discovered Telegram fallback IPs: %s", self.name, ", ".join(fallback_ips))
- proxy_url = resolve_proxy_url(
- "TELEGRAM_PROXY", target_hosts=["api.telegram.org", *fallback_ips],
- configured=self.config.extra.get("proxy_url"))
- def _pair(general_httpx: dict, updates_httpx: dict, **extra) -> tuple:
- return (HTTPXRequest(**request_kwargs, **extra, httpx_kwargs={"verify": platform_ssl_context(), **general_httpx}),
- HTTPXRequest(**request_kwargs, **extra, httpx_kwargs={"verify": platform_ssl_context(), **updates_httpx}))
- if fallback_ips and not proxy_url and not disable_fallback:
- logger.info("[%s] Telegram fallback IPs active: %s", self.name, ", ".join(fallback_ips))
- _transport_kwargs: dict = {"socket_options": tcp_keepalive_socket_options(), "verify": platform_ssl_context()}
- # Keep request/update pools separate to reduce contention during polling reconnect + bot API
- # bootstrap/delete_webhook calls. httpx ignores the client-level `limits` kwarg when a custom
- # `transport` is supplied (#58790). Unlike the proxy/direct branches (which inject limits at the
- # client level via `_with_limits`), this branch MUST pass the tuned limits directly into
- # TelegramFallbackTransport so its inner AsyncHTTPTransport instances honour keepalive_expiry —
- # do not route this through `_with_limits`, httpx would discard it.
- if _pool_limits is not None:
- _transport_kwargs["limits"] = _pool_limits
- _updates_transport_kwargs = dict(_transport_kwargs)
- if _updates_limits is not None:
- _updates_transport_kwargs["limits"] = _updates_limits
- request, get_updates_request = _pair(
- {"transport": TelegramFallbackTransport(fallback_ips, **_transport_kwargs)},
- {"transport": TelegramFallbackTransport(fallback_ips, **_updates_transport_kwargs)})
- elif proxy_url:
- logger.info("[%s] Proxy detected; passing explicitly to HTTPXRequest: %s", self.name, proxy_url)
- request, get_updates_request = _pair(_with_limits(), {"limits": _updates_limits}, proxy=proxy_url)
- else:
- if disable_fallback:
- logger.info("[%s] Telegram fallback-IP transport disabled via env", self.name)
- request, get_updates_request = _pair(_with_limits(), {"limits": _updates_limits})
- return request, self._instrument_polling_request(get_updates_request)
- async def _initialize_app_with_retries(self, builder) -> None:
- """``app.initialize()`` with a bounded retry ladder; rebuilds ``self._app``/``self._bot`` from
- ``builder`` after each failed attempt; OSError when the per-attempt or total watchdog expires."""
- _max_connect = 8
- _init_timeout = env_float("HERMES_TELEGRAM_INIT_TIMEOUT", 30.0) # per attempt
- # Total watchdog: bounds the whole connect loop even if the retry loop silently stalls.
- _total_deadline = asyncio.get_running_loop().time() + _init_timeout * _max_connect + 120.0
- _timed_out = f"Telegram initialization timed out after {_max_connect} attempts ({_init_timeout:.0f}s each)"
- for _attempt in range(_max_connect):
- rebuild_app = False
- try:
- if asyncio.get_running_loop().time() >= _total_deadline:
- raise OSError(
- f"{_timed_out} — total connect watchdog deadline ({_init_timeout * _max_connect + 120.0:.0f}s) exceeded. "
- f"Check network connectivity to api.telegram.org or set HERMES_TELEGRAM_HTTP_CONNECT_TIMEOUT / "
- f"HERMES_TELEGRAM_INIT_TIMEOUT to a lower value.")
- logger.warning("[%s] Connecting to Telegram (attempt %d/%d)…", self.name, _attempt + 1, _max_connect)
- # On timeout the (possibly shielded) initialize() task is abandoned; release the half-built
- # app's httpx client so it isn't leaked across the ladder.
- await _await_with_thread_deadline(
- self._app.initialize(), timeout=_init_timeout, on_abandon=lambda app=self._app: _shutdown_abandoned_app(app),
- label="telegram-init")
- break
- except TimeoutError:
- rebuild_app = True
- if _attempt >= _max_connect - 1:
- raise OSError(
- f"{_timed_out}. Check network connectivity to api.telegram.org "
- f"or set HERMES_TELEGRAM_HTTP_CONNECT_TIMEOUT to a lower value.")
- wait = min(2 ** _attempt, 15)
- logger.warning(
- "[%s] Connect attempt %d/%d timed out after %.0fs — retrying in %ds", self.name, _attempt + 1,
- _max_connect, _init_timeout, wait)
- await asyncio.sleep(wait)
- except Exception as init_err:
- # OSError always retries; anything else only when it looks like a network error.
- rebuild_app = True
- if (not isinstance(init_err, OSError) and not self._looks_like_network_error(init_err)) or _attempt >= _max_connect - 1:
- raise
- wait = min(2 ** _attempt, 15)
- logger.warning(
- "[%s] Connect attempt %d/%d failed: %s — retrying in %ds", self.name, _attempt + 1, _max_connect, init_err, wait)
- await asyncio.sleep(wait)
- except BaseException:
- # CancelledError etc.: log for the operator, then reraise. LAST so the Exception handlers win.
- logger.warning(
- "[%s] Connect attempt %d/%d interrupted by %s — propagating", self.name, _attempt + 1, _max_connect,
- "CancelledError" if isinstance(sys.exc_info()[1], asyncio.CancelledError) else type(sys.exc_info()[1]).__name__)
- raise
- finally:
- # A failed attempt may leave the app half-initialized: rebuild a fresh Application from the
- # same builder for the next attempt and discard the old one.
- if rebuild_app and _attempt < _max_connect - 1:
- old_app = self._app
- self._app = builder.build()
- self._bot = self._app.bot
- # Same order as connect(): plugin handlers first (the wired-set is keyed per app, so
- # the rebuilt app gets them again), then core and the observer in lockstep.
- self._wire_plugin_handlers(self._app)
- self._register_handlers(self._app)
- with contextlib.suppress(Exception):
- await _shutdown_abandoned_app(old_app)
- def _cold_boot_drop_pending(self, *, is_reconnect: bool) -> bool:
- """Whether THIS connection asks Telegram to discard its queued updates.
- A watcher reconnect always preserves them (#46621); a cold boot follows
- ``platforms.telegram.extra.drop_pending_on_cold_boot`` (default true). The decision is logged
- on every cold boot — a command that never ran is otherwise invisible (#71811)."""
- drop_pending = self._drop_pending_on_cold_boot if not is_reconnect else False
- if not is_reconnect:
- logger.info(
- "[%s] Cold boot: %s Telegram updates queued while offline "
- "(platforms.telegram.extra.drop_pending_on_cold_boot: %s)",
- self.name, "dropping" if drop_pending else "preserving",
- "true" if self._drop_pending_on_cold_boot else "false")
- return drop_pending
- async def _start_webhook_mode(self, webhook_url: str, *, is_reconnect: bool) -> None:
- """Start PTB's webhook server (Telegram pushes updates; lets cloud platforms auto-wake suspended
- machines). SECURITY: TELEGRAM_WEBHOOK_SECRET is REQUIRED — without it the endpoint accepts forged
- updates (GHSA-3vpc-7q5r-276h); refuse to start rather than run fail-open."""
- webhook_port = env_int("TELEGRAM_WEBHOOK_PORT", 8443)
- # Default "" → tornado listens on IPv4 + IPv6; "0.0.0.0" is unreachable on IPv6-only networks.
- webhook_host = (os.getenv("TELEGRAM_WEBHOOK_HOST", "").strip() or str((self.config.extra or {}).get("webhook_host") or "").strip())
- webhook_secret = (_get_scoped_secret("TELEGRAM_WEBHOOK_SECRET") or "").strip()
- if not webhook_secret:
- raise RuntimeError(
- "TELEGRAM_WEBHOOK_SECRET is required when TELEGRAM_WEBHOOK_URL is set. Without it, the "
- "webhook endpoint accepts forged updates from anyone who can reach it — see "
- "https://github.com/NousResearch/hermes-agent/security/advisories/GHSA-3vpc-7q5r-276h.\n\n"
- "Generate a secret and set it in your .env:\n export TELEGRAM_WEBHOOK_SECRET=\"$(openssl rand -hex 32)\"\n\n"
- "Then register it with Telegram when setting the webhook via setWebhook's secret_token parameter.")
- from urllib.parse import urlparse
- webhook_path = urlparse(webhook_url).path or "/telegram"
- await self._app.updater.start_webhook(
- listen=webhook_host, port=webhook_port, url_path=webhook_path, webhook_url=webhook_url,
- secret_token=webhook_secret, allowed_updates=Update.ALL_TYPES,
- drop_pending_updates=self._cold_boot_drop_pending(is_reconnect=is_reconnect),
- )
- self._webhook_mode = True
- self._polling_progress_accepting = False
- self._send_path_degraded = False
- logger.info(
- "[%s] Webhook server listening on %s:%d%s", self.name, webhook_host or "* (all interfaces, IPv4+IPv6)",
- webhook_port, webhook_path)
- async def _start_polling_mode(self, *, is_reconnect: bool) -> None:
- """Clear any stale webhook and start resilient long polling."""
- # Best-effort: a transient Bot API error must not fail gateway startup — degrade to recovery.
- await self._delete_webhook_best_effort(require_success=not is_reconnect)
- loop = asyncio.get_running_loop()
- def _polling_error_callback(error: Exception) -> None:
- if self._teardown_started or self._recovery_in_flight():
- return
- if self._looks_like_polling_conflict(error):
- # Stop PTB's network_retry_loop synchronously BEFORE scheduling async recovery, else PTB's
- # retry and our stop->restart overlap and produce a fresh 409.
- self._disarm_ptb_retry_loop()
- self._spawn_polling_recovery(loop, self._handle_polling_conflict(error))
- elif self._looks_like_network_error(error):
- logger.warning(
- "[%s] Telegram network error, scheduling reconnect: %s", self.name, _redact_telegram_error_text(error))
- self._spawn_polling_recovery(loop, self._handle_polling_network_error(error))
- else:
- logger.error("[%s] Telegram polling error: %s", self.name, _redact_telegram_error_text(error), exc_info=True)
- self._polling_error_callback_ref = _polling_error_callback # reused by _handle_polling_conflict
- drop_pending = self._cold_boot_drop_pending(is_reconnect=is_reconnect)
- polling_started = await self._start_polling_resilient(
- drop_pending_updates=drop_pending, error_callback=_polling_error_callback, require_progress=not is_reconnect)
- if not polling_started:
- logger.warning(
- "[%s] Connected in degraded Telegram mode: gateway is alive, polling will be retried in the background", self.name)
- async def connect(self, *, is_reconnect: bool = False) -> bool:
- """Connect via long polling, or a webhook server if ``TELEGRAM_WEBHOOK_URL`` is set.
- ``is_reconnect``: False = cold boot (drop the Bot API queue unless
- ``extra.drop_pending_on_cold_boot`` is false); True = watcher reconnect (preserve
- queued updates, else every message sent during the outage is lost). Webhook env:
- TELEGRAM_WEBHOOK_URL, TELEGRAM_WEBHOOK_PORT (8443), TELEGRAM_WEBHOOK_HOST,
- TELEGRAM_WEBHOOK_SECRET."""
- # Explicit connect() is the only operation allowed to reopen polling after a completed teardown.
- self._polling_teardown_started = False
- self._webhook_mode = False # re-evaluated on every explicit connection
- if not TELEGRAM_AVAILABLE:
- logger.error("[%s] python-telegram-bot not installed. Run: pip install python-telegram-bot", self.name)
- self._set_fatal_error("missing_dependency", "python-telegram-bot not installed", retryable=False)
- return False
- if not self.config.token:
- logger.error("[%s] No bot token configured", self.name)
- self._set_fatal_error("missing_credentials", "No bot token configured", retryable=False)
- return False
- try:
- if not self._acquire_platform_lock('telegram-bot-token', self.config.token, 'Telegram bot token'):
- return False
- from plugins.platforms.telegram.update_admission import TelegramApplication, build_update_processor
- builder = Application.builder().token(self.config.token)
- builder.application_class(TelegramApplication, {"adapter": self})
- custom_base_url = self.config.extra.get("base_url")
- if custom_base_url:
- builder = builder.base_url(custom_base_url)
- builder = builder.base_file_url(self.config.extra.get("base_file_url", custom_base_url))
- logger.info("[%s] Using custom Telegram base_url: %s", self.name, custom_base_url)
- # Local-mode telegram-bot-api returns absolute server-side file paths; PTB needs local_mode=True
- # so download_*() reads from disk instead of a 404ing HTTP GET.
- if self.config.extra.get("local_mode"):
- builder = builder.local_mode(True)
- logger.info("[%s] Using Telegram local_mode (read files from disk)", self.name)
- request, get_updates_request = await self._build_ptb_requests()
- builder = builder.request(request).get_updates_request(get_updates_request)
- # PTB's default processor awaits each update inline, so one slow turn deafens every chat.
- # Concurrent across chats, FIFO within a chat; the builder keeps this instance, so the
- # connect-retry rebuild in _initialize_app_with_retries gets it too.
- builder = builder.concurrent_updates(build_update_processor(self.config.extra, self.name))
- self._app = builder.build()
- self._bot = self._app.bot
- # Plugin PTB handlers go BEFORE core: PTB dispatches the first matching handler per group.
- self._wire_plugin_handlers(self._app)
- self._register_handlers(self._app)
- await self._initialize_app_with_retries(builder)
- await self._app.start()
- # Profile-scoped like TELEGRAM_WEBHOOK_SECRET: under multiplex os.environ holds the DEFAULT
- # profile's URL, and registering it on a secondary bot pushes that bot's updates to the
- # default's listener (and stops polling for it).
- webhook_url = (_get_scoped_secret("TELEGRAM_WEBHOOK_URL") or "").strip()
- if webhook_url:
- await self._start_webhook_mode(webhook_url, is_reconnect=is_reconnect)
- else:
- await self._start_polling_mode(is_reconnect=is_reconnect)
- self._mark_connected()
- # WARNING, not INFO: "Connecting…" above is WARNING and reaches the terminal; an INFO success
- # line made healthy startups look stalled at "attempt 1/8".
- logger.warning("[%s] Connected to Telegram (%s mode)", self.name, "webhook" if self._webhook_mode else "polling")
- # Heartbeat only in polling mode: webhook mode has no long-poll socket to wedge in CLOSE-WAIT.
- # WARNING, not INFO: the "Connecting to Telegram (attempt N/8)…" line above is emitted at
- # WARNING and reaches the terminal (the gateway's default stderr handler is WARNING-only), but
- # this success line was INFO and went to the log file only. A healthy startup therefore looked
- # permanently stalled at "attempt 1/8" on the console — the logging illusion in #90835. Both
- # sides of the connect transition must share a terminal-visible level so a real hang is the
- # *absence* of this line, not ambiguity.
- if not self._webhook_mode:
- self._restart_task_attr("_polling_heartbeat_task", self._polling_heartbeat_loop())
- # Seed the live identity from PTB's initialize() cache; polling rides the heartbeat's get_me(),
- # webhook mode gets a low-frequency refresh loop (else a BotFather rename breaks routing).
- self._note_bot_username(getattr(self._bot, "username", None))
- self._bot_identity_checked_at = time.monotonic()
- if self._webhook_mode:
- self._restart_task_attr("_bot_identity_refresh_task", self._bot_identity_refresh_loop())
- # Command menu / DM topics / status indicator can stall for some tokens: defer to a cancellable
- # task so one slow call can't sink the (gateway-timed) connect while transport is live.
- # Command-menu registration, DM-topic setup, and the status indicator each make Bot API calls
- # that can stall for certain tokens. Running them here — inside the connect() coroutine that the
- # gateway wraps in a connect timeout — means one slow call blows the whole connect and the
- # adapter never comes up, even though polling/webhook is already live (#46298).
- self._start_post_connect_housekeeping()
- return True
- except Exception as e:
- self._release_platform_lock()
- safe_error = _redact_telegram_error_text(e)
- # Classify by exception TYPE (never message text): auth failures can never self-heal, so
- # marking them retryable put agents into a silent eternal reconnect loop.
- if self._looks_like_auth_error(e):
- message = (
- f"Telegram bot token rejected: {safe_error}. "
- "The token is invalid or was revoked — generate a new one "
- "with @BotFather and update TELEGRAM_BOT_TOKEN.")
- self._set_fatal_error("telegram_auth_error", message, retryable=False)
- else:
- self._set_fatal_error("telegram_connect_error", f"Telegram startup failed: {safe_error}", retryable=True)
- logger.error("[%s] Failed to connect to Telegram: %s", self.name, safe_error)
- return False
- async def _set_status_indicator(self, online: bool) -> None:
- """Set the bot's short description to the online/offline text (closest Bot API surface to
- presence). No-op unless ``extra.status_indicator``; failures are debug-logged."""
- if not getattr(self, "_status_indicator_enabled", False):
- return
- bot = self._bot
- if bot is None:
- return
- text = (self._status_online_text if online else self._status_offline_text)[:120] # Telegram cap
- try:
- await bot.set_my_short_description(short_description=text)
- logger.info("[%s] Set bot status indicator to %r", self.name, text)
- except Exception as e:
- logger.debug("[%s] Failed to set bot status indicator to %r: %s", self.name, text, _redact_telegram_error_text(e))
- @staticmethod
- def _collect_live_tasks(candidates, current_task) -> list:
- """Unique, unfinished tasks from ``candidates`` excluding ``current_task`` (so teardown never cancels itself)."""
- seen: set[int] = set()
- out: list[asyncio.Task] = []
- for task in candidates:
- if not task or task.done() or task is current_task or id(task) in seen:
- continue
- seen.add(id(task))
- out.append(task)
- return out
- def _clear_task_attrs_except(self, current_task, *attrs: str) -> None:
- for attr in attrs:
- if getattr(self, attr, None) is not current_task:
- setattr(self, attr, None)
- async def _cancel_pending_delivery_tasks(self) -> None:
- """Cancel every delayed-delivery task family before disconnect completes (media-group, photo-batch, text-batch flushes plus
- polling recovery all sit behind ``asyncio.sleep()`` and would dispatch ``handle_message`` into a torn-down session)."""
- current_task = asyncio.current_task()
- pending_tasks = self._collect_live_tasks(
- [
- *self._media_group_tasks.values(), *self._pending_photo_batch_tasks.values(), *self._pending_text_batch_tasks.values(),
- *getattr(self, "_committed_flush_tasks", {}).values(),
- getattr(self, "_polling_error_task", None), getattr(self, "_polling_progress_verifier_task", None),
- # Hold-queue redispatch must be cancellable+awaitable on teardown too.
- getattr(self, "_held_inbound_redispatch_task", None),
- ],
- current_task)
- awaitable_tasks = [t for t in pending_tasks if asyncio.isfuture(t) or asyncio.iscoroutine(t)]
- # Hold-queue redispatch must be cancellable+awaitable on teardown so it cannot dispatch
- # handle_message into a torn-down session (same lifecycle rule teknium called out on #72037 for
- # shielded flush dispatch).
- for task in pending_tasks:
- task.cancel()
- if awaitable_tasks:
- await asyncio.gather(*awaitable_tasks, return_exceptions=True)
- # Salvage buffered inbound events before clearing maps — unless permanent fatal, where no
- # reconnect can drain and hold would re-orphan them.
- if self._is_permanent_fatal():
- n_pending = len(self._pending_text_batches) + len(self._pending_photo_batches) + len(self._media_group_events)
- if n_pending:
- logger.warning("[Telegram] Non-retryable fatal teardown; discarding %d pending inbound batch(es)", n_pending)
- else:
- for events, where in (
- (self._pending_text_batches, "text-batch-teardown"), (self._pending_photo_batches, "photo-batch-teardown"),
- (self._media_group_events, "media-group-teardown")):
- for event in list(events.values()):
- self._hold_inbound_event(event, where=where)
- for d in (
- self._media_group_tasks, self._media_group_events, self._pending_photo_batch_tasks,
- self._pending_photo_batches, self._pending_text_batch_tasks, self._pending_text_batches):
- d.clear()
- self._clear_task_attrs_except(
- current_task, "_polling_error_task", "_polling_progress_verifier_task", "_held_inbound_redispatch_task")
- async def _await_disconnect_step(self, awaitable, timeout: float, step: str) -> bool:
- """Await one disconnect step; detach on timeout so teardown advances (``wait_for`` would wait for a
- PTB close that swallows ``CancelledError`` on a half-dead socket). Abandoned tasks are observed.
- ``asyncio.wait_for`` cancels an overdue child but then waits for it to exit. Detach at the deadline
- and continue — the abandoned task is observed via ``_consume_abandoned_task``. See #80598.
- """
- task = asyncio.ensure_future(awaitable)
- try:
- done, _pending = await asyncio.wait({task}, timeout=timeout if timeout > 0 else None)
- except asyncio.CancelledError:
- # asyncio.wait does NOT cancel its futures when itself cancelled; don't orphan the inner task.
- task.cancel()
- # Mirror the pattern used by GatewayRunner._await_adapter_cleanup_with_timeout. See #80598.
- task.add_done_callback(_consume_abandoned_task)
- raise
- if task in done:
- with contextlib.suppress(asyncio.CancelledError):
- await task
- return True
- task.cancel()
- task.add_done_callback(_consume_abandoned_task)
- logger.warning("[%s] %s timed out after %.1fs during disconnect; continuing teardown", self.name, step, timeout)
- return False
- def _restart_task_attr(self, attr: str, coro) -> None:
- """Cancel any live task stored at ``self.<attr>`` and start ``coro`` in its place."""
- prior = getattr(self, attr, None)
- if prior and not prior.done():
- prior.cancel()
- setattr(self, attr, asyncio.ensure_future(coro))
- async def _cancel_task_attr(self, attr: str, label: str) -> None:
- """Cancel + bounded-await the task stored at ``self.<attr>`` (may be missing: object.__new__ tests), then clear it."""
- task = getattr(self, attr, None)
- if task and not task.done():
- task.cancel()
- await self._await_disconnect_step(task, _DISCONNECT_STEP_TIMEOUT, label)
- setattr(self, attr, None)
- async def disconnect(self) -> None:
- """Stop polling/webhook, cancel pending delayed deliveries, and disconnect."""
- # Mark disconnected first so the drop guard short-circuits any flush that wins the race.
- self._mark_disconnected()
- self._polling_teardown_started = True
- self._polling_progress_accepting = False
- self._polling_generation = getattr(self, "_polling_generation", 0) + 1
- self._polling_progress_event = asyncio.Event()
- self._send_path_degraded = True
- # Release the bot-token lock immediately so a wedged close cannot block the reconnect watcher.
- # The rest of teardown is best-effort against a half-dead transport. See #80598.
- self._release_platform_lock()
- # Cancel and await both polling lifecycle owners right after the fence, before any other teardown
- # await lets them start a new generation.
- current_task = asyncio.current_task()
- lifecycle_tasks = self._collect_live_tasks(
- [getattr(self, "_polling_error_task", None), getattr(self, "_polling_progress_verifier_task", None)], current_task)
- for task in lifecycle_tasks:
- task.cancel()
- lifecycle_tasks = [t for t in lifecycle_tasks if asyncio.isfuture(t) or asyncio.iscoroutine(t)]
- if lifecycle_tasks:
- await self._await_disconnect_step(
- asyncio.gather(*lifecycle_tasks, return_exceptions=True), _DISCONNECT_STEP_TIMEOUT, "lifecycle-task cancel")
- self._clear_task_attrs_except(current_task, "_polling_error_task", "_polling_progress_verifier_task")
- # Cancellation callbacks may have run while awaited; the fence stays authoritative.
- self._polling_progress_accepting = False
- self._send_path_degraded = True
- # Cancel deferred post-connect housekeeping so it cannot fire into a half-torn-down bot client.
- # Cancel deferred post-connect housekeeping (command-menu / DM-topic / status-indicator Bot API
- # calls) so it cannot fire into a half-torn-down bot client (#46298). getattr guards the
- # object.__new__ test pattern where __init__ (which sets this attr) is never called.
- post_connect_task = getattr(self, "_post_connect_task", None)
- if post_connect_task and not post_connect_task.done():
- post_connect_task.cancel()
- await self._await_disconnect_step(
- asyncio.gather(post_connect_task, return_exceptions=True), _DISCONNECT_STEP_TIMEOUT, "post-connect cancel")
- self._post_connect_task = None
- # Cancel the heartbeat (and webhook-mode identity loop) before tearing down the app.
- await self._cancel_task_attr("_polling_heartbeat_task", "heartbeat cancel")
- await self._cancel_task_attr("_bot_identity_refresh_task", "identity-refresh cancel")
- # Mark the bot "Offline" while its HTTP client is still alive. Opt-in, non-fatal.
- with contextlib.suppress(Exception):
- await self._await_disconnect_step(self._set_status_indicator(online=False), _DISCONNECT_STEP_TIMEOUT, "status-indicator update")
- await self._await_disconnect_step(self._cancel_pending_delivery_tasks(), _DISCONNECT_STEP_TIMEOUT, "pending-delivery cancel")
- if self._app:
- try:
- # Bounded: a CLOSE-WAIT socket can wedge updater.stop() forever; fall through on timeout.
- if self._app.updater and self._app.updater.running:
- try:
- await self._await_disconnect_step(self._app.updater.stop(), _UPDATER_STOP_TIMEOUT, "updater.stop()")
- except Exception as stop_error:
- logger.warning(
- "[%s] updater.stop() failed during disconnect: %s", self.name, _redact_telegram_error_text(stop_error))
- # app.stop()/shutdown() can also block on a half-dead httpx pool.
- # Detach-on-timeout so disconnect always returns (#80598).
- if self._app.running:
- await self._await_disconnect_step(self._app.stop(), _DISCONNECT_STEP_TIMEOUT, "app.stop()")
- await self._await_disconnect_step(self._app.shutdown(), _DISCONNECT_STEP_TIMEOUT, "app.shutdown()")
- except Exception as e:
- logger.warning("[%s] Error during Telegram disconnect: %s", self.name, _redact_telegram_error_text(e))
- self._app = None
- self._bot = None
- # Land the last completed receipts before a replacement adapter reads them.
- flush = getattr(self, "_update_receipt_flush", None)
- if flush is not None and not flush.done():
- await self._await_disconnect_step(asyncio.shield(flush), _DISCONNECT_STEP_TIMEOUT, "update-receipt flush")
- logger.info("[%s] Disconnected from Telegram", self.name)
- def _should_thread_reply(self, reply_to: Optional[str], chunk_index: int) -> bool:
- """Whether this chunk (0 = first) should reply-thread to ``reply_to``, per reply_to_mode."""
- if not reply_to:
- return False
- mode = self._reply_to_mode
- if mode == "off":
- return False
- if mode == "all":
- return True
- return chunk_index == 0 # "first" (default)
- @staticmethod
- def _telegram_error_types() -> tuple:
- """``(NetworkError, BadRequest, TimedOut)`` from PTB, with import-failure fallbacks
- (``OSError``, ``None``, ``None``) so send() still classifies without the SDK."""
- try:
- from telegram.error import NetworkError as _NetErr
- except ImportError:
- _NetErr = OSError # type: ignore[misc,assignment]
- try:
- from telegram.error import BadRequest as _BadReq
- except ImportError:
- _BadReq = None # type: ignore[assignment,misc]
- try:
- from telegram.error import TimedOut as _TimedOut
- except (ImportError, AttributeError):
- _TimedOut = None # type: ignore[assignment,misc]
- return _NetErr, _BadReq, _TimedOut
- async def _send_chunk_markdown_or_plain(self, chunk: str, send_kwargs: dict[str, Any]):
- """MarkdownV2 first; on a parse/markdown rejection resend as stripped plain text."""
- try:
- return await _await_with_thread_deadline(
- self._bot.send_message(text=chunk, parse_mode=ParseMode.MARKDOWN_V2, **send_kwargs),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- except Exception as md_error:
- if "parse" in str(md_error).lower() or "markdown" in str(md_error).lower():
- logger.warning("[%s] MarkdownV2 parse failed, falling back to plain text: %s", self.name, md_error)
- return await _await_with_thread_deadline(
- self._bot.send_message(text=_strip_mdv2(chunk), parse_mode=None, **send_kwargs),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- raise
- async def _send_chunk_with_retries(
- self, chat_id: str, chunk: str, index: int, reply_to: Optional[str], metadata: Optional[dict[str, Any]],
- thread_id: Optional[str], used_thread_fallback: bool, error_types: tuple):
- """Deliver one chunk: routing, up to 3 attempts, thread-not-found / deleted-anchor / flood handling.
- Returns ``(msg, used_thread_fallback)`` on success or a ``SendResult`` to return verbatim (fail-loud DM-topic
- cases, flood cap); raises anything the caller's classifier should see."""
- _NetErr, _BadReq, _TimedOut = error_types
- retried_thread_not_found = False
- private_dm_topic_send, dm_topic_reply_to_off, reply_to_id = self._chunk_reply_routing(chat_id, reply_to, metadata, thread_id, index)
- if private_dm_topic_send and reply_to_id is None and not dm_topic_reply_to_off:
- return SendResult(success=False, error=self._dm_topic_missing_anchor_error(), retryable=False)
- thread_kwargs = self._thread_kwargs_for_send(
- chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
- if used_thread_fallback and thread_kwargs.get("message_thread_id") is not None:
- thread_kwargs = dict(thread_kwargs)
- thread_kwargs["message_thread_id"] = None
- effective_thread_id = thread_kwargs.get("message_thread_id")
- for _send_attempt in range(3):
- try:
- send_kwargs = {
- "chat_id": normalize_telegram_chat_id(chat_id), "reply_to_message_id": reply_to_id, **thread_kwargs,
- **self._link_preview_kwargs(), **self._notification_kwargs(metadata)}
- return await self._send_chunk_markdown_or_plain(chunk, send_kwargs), used_thread_fallback
- except _NetErr as send_err:
- # BadRequest subclasses NetworkError in PTB but is permanent; handle specific cases.
- if _BadReq and isinstance(send_err, _BadReq):
- if self._is_thread_not_found_error(send_err) and effective_thread_id is not None:
- if private_dm_topic_send or (metadata and metadata.get("telegram_dm_topic_created_for_send")):
- return SendResult(success=False, error=str(send_err), retryable=False)
- # One-off "thread not found" flakes recover on immediate retry: same thread_id once.
- if not retried_thread_not_found:
- retried_thread_not_found = True
- logger.warning("[%s] Thread %s not found, retrying once with same thread_id", self.name, effective_thread_id)
- continue
- # Thread is genuinely gone: retry without it and prune the stale binding.
- logger.warning("[%s] Thread %s not found, retrying without message_thread_id", self.name, effective_thread_id)
- self._prune_stale_dm_topic_binding(chat_id, effective_thread_id, metadata=metadata)
- used_thread_fallback = True
- effective_thread_id = None
- thread_kwargs = {"message_thread_id": None}
- continue
- if "message to be replied not found" in str(send_err).lower() and reply_to_id is not None:
- safe_send_error = _redact_telegram_error_text(send_err)
- if private_dm_topic_send:
- return SendResult(success=False, error=safe_send_error, retryable=False)
- # Reply target deleted; private-topic fallback sends drop anchor + topic id together.
- logger.warning("[%s] Reply target deleted, retrying without reply_to: %s", self.name, safe_send_error)
- reply_to_id = None
- if self._dm_topic_fallback(metadata):
- thread_kwargs = {}
- else:
- thread_kwargs = self._thread_kwargs_for_send(
- chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
- effective_thread_id = thread_kwargs.get("message_thread_id")
- continue
- raise # other BadRequest errors are permanent
- # TimedOut also subclasses NetworkError: a generic timeout may have reached Telegram (don't
- # retry); a wrapped ConnectTimeout or an httpx pool timeout is safe to retry.
- is_pool_timeout = self._looks_like_pool_timeout(send_err)
- if (
- _TimedOut and isinstance(send_err, _TimedOut)
- and not self._looks_like_connect_timeout(send_err) and not is_pool_timeout):
- raise
- if is_pool_timeout:
- await self._drain_general_connections_after_pool_timeout()
- if _send_attempt >= 2:
- raise
- wait = 2 ** _send_attempt
- logger.warning("[%s] Network error on send (attempt %d/3), retrying in %ds: %s",
- self.name, _send_attempt + 1, wait, _redact_telegram_error_text(send_err))
- await asyncio.sleep(wait)
- except Exception as send_err:
- retry_after = getattr(send_err, "retry_after", None)
- if retry_after is not None or "retry after" in str(send_err).lower():
- wait = float(retry_after) if retry_after is not None else 1.0
- safe_send_error = _redact_telegram_error_text(send_err)
- # Never sleep a long server RetryAfter verbatim — it once pinned send() for 97 minutes.
- # Mirror the edit path: a RetryAfter past a few seconds is not something to hold this
- # coroutine open for. Sleeping the server value verbatim pinned send() for 97 minutes in
- # production and froze inbound on every platform when it ran on the gateway boot path
- # (#91969).
- if wait > _FLOOD_INLINE_WAIT_CAP_SECS:
- logger.warning(
- "[%s] Telegram flood control on send (retry_after=%.1fs > %.0fs); failing closed instead of sleeping: %s",
- self.name, wait, _FLOOD_INLINE_WAIT_CAP_SECS, safe_send_error)
- return self._record_send_flood_cooldown(chat_id, wait)
- if _send_attempt < 2:
- logger.warning(
- "[%s] Telegram flood control on send (attempt %d/3), retrying in %.1fs: %s", self.name,
- _send_attempt + 1, wait, safe_send_error)
- await asyncio.sleep(wait)
- continue
- # Retries exhausted and still flooded. Fail closed the same way a long penalty
- # does: raising here handed the caller the platform's own wording instead of the
- # canonical result, so the delivery ledger did not recognise the row as a flood
- # refusal, armed no redelivery timer, and the reply waited for the next restart.
- logger.warning(
- "[%s] Telegram flood control on send persisted across %d attempts; failing "
- "closed so the delivery ledger owns the wait: %s",
- self.name, _send_attempt + 1, safe_send_error)
- return self._record_send_flood_cooldown(chat_id, wait)
- raise
- async def _retrigger_typing(self, chat_id: str, metadata: Optional[dict[str, Any]]) -> None:
- """Re-arm typing after an intermediate send (Telegram clears it when a message lands). Skipped on
- the FINAL reply (``metadata["notify"]``): the refresh loop is gone and no API cancels the bubble.
- Scheduled, never awaited: ``sendChatAction`` is a fire-and-forget UI hint, and awaiting its TLS
- round-trip on the send path after *every* streamed chunk pinned the event loop the ``getUpdates``
- long-polls live on until they rotted into CLOSE-WAIT while the adapter still reported connected
- (#111727). ``_keep_typing`` already refreshes every 2s, so one in-flight re-arm per chat, at most
- one per ``typing_retrigger_min_interval_seconds``, covers the gap a landed message leaves."""
- if (metadata or {}).get("notify") or not getattr(getattr(self, "config", None), "typing_indicator", True):
- return
- # __dict__.setdefault: tests build adapters via object.__new__() (no __init__).
- tasks: dict[str, asyncio.Task] = self.__dict__.setdefault("_telegram_typing_retrigger_tasks", {})
- sent_at: dict[str, float] = self.__dict__.setdefault("_telegram_typing_retrigger_at", {})
- key = str(chat_id)
- in_flight = tasks.get(key)
- if in_flight is not None and not in_flight.done():
- return
- loop = asyncio.get_running_loop()
- now = loop.time()
- # Stamped at scheduling time, not completion, so a burst of chunks cannot all pass while the
- # first round-trip is still open.
- if now - sent_at.get(key, float("-inf")) < getattr(self, "_telegram_typing_retrigger_interval", 2.0):
- return
- sent_at[key] = now
- async def _quiet() -> None:
- with contextlib.suppress(Exception):
- await self.send_typing(chat_id, metadata=metadata)
- task = loop.create_task(_quiet())
- tasks[key] = task
- task.add_done_callback(lambda done: tasks.get(key) is done and tasks.pop(key, None))
- # Shutdown cancels _background_tasks, so a detached re-arm cannot outlive the adapter.
- tracked = getattr(self, "_background_tasks", None)
- if isinstance(tracked, set):
- tracked.add(task)
- task.add_done_callback(tracked.discard)
- async def send(
- self, chat_id: str, content: str, reply_to: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Send a message to a Telegram chat."""
- if not self._bot:
- live = self._replacement_telegram_adapter()
- if live is not None:
- return await live.send(chat_id, content, reply_to, metadata)
- if self._is_permanent_fatal() or not await self._wait_for_reconnection():
- return SendResult(success=False, error="Not connected", retryable=not self._is_permanent_fatal())
- live = self._replacement_telegram_adapter()
- if not self._bot and live is not None:
- return await live.send(chat_id, content, reply_to, metadata)
- if not self._bot:
- return SendResult(success=False, error="Not connected", retryable=True)
- # getattr() — tests build adapters via object.__new__() (no __init__).
- if getattr(self, "_send_path_degraded", False):
- return SendResult(success=False, error="send_path_degraded", retryable=True)
- # Skip whitespace-only text to prevent Telegram 400 empty-text errors.
- if not content or not content.strip():
- return SendResult(success=True, message_id=None)
- # One chat at a time (held only around the API calls, never across the reconnect wait above), so
- # two concurrent split replies to one chat cannot interleave their chunks (#114396).
- async with self._chat_send_lock(chat_id):
- return await self._send_text_locked(chat_id, content, reply_to, metadata)
- async def _send_text_locked(
- self, chat_id: str, content: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> SendResult:
- """``send()`` body under the per-chat lock: rich fast-path, else MarkdownV2 chunks."""
- # Re-checked under the lock: a burst queued behind a refused send must not each fire once.
- cooldown = self._send_flood_cooldown_remaining(chat_id)
- if cooldown is not None:
- logger.warning(
- "[%s] Telegram flood control still active for chat %s (%.0fs left); refusing locally without an API call",
- self.name, chat_id, cooldown)
- return _flood_cap_result(cooldown)
- # Shared per-chat budget (#116312): a send WAITS for its slot (a send that waits
- # is delivered; one that is skipped would drop a message).
- slot_remaining = self._chat_outbound_slot_remaining(chat_id)
- if slot_remaining > 0:
- logger.debug(
- "[%s] pacing send for chat %s (shared send+edit budget: slot in %.1fs)",
- self.name, chat_id, slot_remaining)
- await asyncio.sleep(slot_remaining)
- self._hold_chat_outbound_slot(chat_id)
- error_types = self._telegram_error_types()
- chunks: list[str] = []
- delivered: list[str] = []
- try:
- # Bot API 10.1 rich fast-path; falls through to legacy MarkdownV2 on permanent/capability
- # errors or DM-topic skips; returns directly on success or transient failure (no legacy resend).
- if self._should_attempt_rich(content, metadata=metadata):
- rich_result = await self._try_send_rich(chat_id, content, reply_to, metadata)
- if rich_result is not None:
- if rich_result.success:
- await self._retrigger_typing(chat_id, metadata)
- return rich_result
- chunks = self.truncate_message(self.format_message(content), self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)
- if len(chunks) > 1:
- # truncate_message appends a raw " (1/2)" suffix; escape the MarkdownV2-special parentheses.
- chunks = [
- _separate_chunk_indicator_from_fence(re.sub(r" \((\d+)/(\d+)\)$", r" \\(\1/\2\\)", chunk))
- for chunk in chunks
- ]
- return await self._send_chunks(chat_id, chunks, delivered, reply_to, metadata, error_types)
- except Exception as e:
- classified = self._classify_send_exception(e, error_types)
- return self._with_partial_send(classified, chunks[len(delivered):], delivered, tail_certain=classified.retryable)
- def _classify_send_exception(self, e: Exception, error_types: tuple) -> SendResult:
- """The failed ``SendResult`` for an exception escaping the chunk loop. ``retryable`` doubles as
- "non-delivery is certain": a plain ``TimedOut`` may have reached Telegram, so it is neither re-sent
- by ``_send_with_retry`` nor resumed from; a wrapped ConnectTimeout / httpx pool timeout never left."""
- safe_error = _redact_telegram_error_text(e)
- logger.error("[%s] Failed to send Telegram message: %s", self.name, safe_error)
- err_str = str(e).lower()
- error_kind = classify_send_error(e)
- # Content exceeded 4096 chars: fail so the stream consumer enters fallback mode.
- if "message_too_long" in err_str or "too long" in err_str:
- logger.debug("[%s] send() content too long, falling back to new-message continuation", self.name)
- return SendResult(success=False, error="message_too_long", error_kind="too_long")
- _to = error_types[2]
- is_timeout = (_to and isinstance(e, _to)) or "timed out" in err_str
- return SendResult(
- success=False, error=safe_error,
- retryable=(self._looks_like_connect_timeout(e) or self._looks_like_pool_timeout(e) or not is_timeout),
- error_kind=error_kind)
- async def _send_chunks(
- self, chat_id: str, chunks: list[str], delivered: list[str], reply_to: Optional[str],
- metadata: Optional[dict[str, Any]], error_types: tuple) -> SendResult:
- """Deliver formatted ``chunks`` in order, appending each landed message id to ``delivered``
- (pre-seeded with the ids of an earlier partial send when resuming — the chunk index used for
- reply routing continues from there). A mid-loop refusal returns the ``partial_overflow`` result."""
- thread_id = self._metadata_thread_id(metadata)
- requested_thread_id = self._message_thread_id_for_send(thread_id)
- used_thread_fallback = False
- prior = len(delivered)
- for chunk in chunks:
- outcome = await self._send_chunk_with_retries(
- chat_id, chunk, len(delivered), reply_to, metadata, thread_id, used_thread_fallback, error_types)
- if isinstance(outcome, SendResult):
- # Every SendResult returned here is a DEFINITE non-delivery (flood cap, DM-topic refusal);
- # ambiguous timeouts raise instead, so the remainder is safe to resume from.
- return self._with_partial_send(outcome, chunks[len(delivered) - prior:], delivered)
- msg, used_thread_fallback = outcome
- delivered.append(str(msg.message_id))
- await self._retrigger_typing(chat_id, metadata)
- return SendResult(
- success=True, message_id=delivered[0] if delivered else None,
- raw_response={
- "message_ids": list(delivered), "requested_thread_id": requested_thread_id, "thread_fallback": used_thread_fallback})
- @staticmethod
- def _with_partial_send(
- result: SendResult, undelivered: list[str], delivered: list[str], *, tail_certain: bool = True) -> SendResult:
- """Mark a split-send failure that happened after earlier chunks landed with the ``partial_overflow``
- contract (the same key ``_edit_overflow_split`` sets and the stream consumer reads), so no caller
- re-sends the already-visible head. ``undelivered`` (the formatted remainder, for
- :meth:`_resume_partial_send`) is attached only when ``tail_certain``. No-op when nothing landed."""
- if not delivered:
- return result
- raw = dict(result.raw_response) if isinstance(result.raw_response, dict) else {}
- raw.update({
- "partial_overflow": True, "delivered_chunks": len(delivered), "total_chunks": len(delivered) + len(undelivered),
- "last_message_id": delivered[-1], "continuation_message_ids": tuple(delivered[1:])})
- if tail_certain and undelivered:
- raw["undelivered_chunks"] = tuple(undelivered)
- raw["delivered_message_ids"] = tuple(delivered)
- result.raw_response = raw
- return result
- async def _resume_partial_send(
- self, chat_id: str, result: SendResult, *, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> Optional[SendResult]:
- """Send only the chunks a partial ``send()`` could not deliver (``raw_response["undelivered_chunks"]``),
- continuing the message-id sequence; ``None`` when the tail was not certain-undelivered."""
- raw = result.raw_response if isinstance(result.raw_response, dict) else {}
- undelivered = raw.get("undelivered_chunks")
- if not undelivered or not self._bot:
- return None
- delivered = list(raw.get("delivered_message_ids") or ())
- prior = len(delivered)
- async with self._chat_send_lock(chat_id):
- cooldown = self._send_flood_cooldown_remaining(chat_id)
- if cooldown is not None:
- return self._with_partial_send(_flood_cap_result(cooldown), list(undelivered), delivered)
- error_types = self._telegram_error_types()
- try:
- return await self._send_chunks(chat_id, list(undelivered), delivered, reply_to, metadata, error_types)
- except Exception as e:
- classified = self._classify_send_exception(e, error_types)
- return self._with_partial_send(
- classified, list(undelivered)[len(delivered) - prior:], delivered, tail_certain=classified.retryable)
- async def send_or_update_status(
- self, chat_id: str, status_key: str, content: str, *, metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Send a status message, or edit the previous one with the same ``(chat_id, status_key)``; if the
- edit fails (deleted, too old, …) the cached id is dropped and a fresh message is sent.
- Issue #30045: progress/status callbacks (context-pressure, lifecycle, compression, etc.) used to
- append a fresh bubble on every call. With this method, the first call sends and the message id is
- remembered; subsequent calls with the same (chat_id, status_key) edit that same message in place.
- """
- key = (str(chat_id), str(status_key))
- cached_id = self._status_message_ids.get(key)
- if cached_id is not None:
- result = await self.edit_message(chat_id, cached_id, content, finalize=True, metadata=metadata)
- if result.success:
- # Only write back if nobody evicted/replaced this key during the await.
- if result.message_id and self._status_message_ids.get(key) == cached_id:
- self._status_message_ids[key] = str(result.message_id)
- return result
- self._status_message_ids.pop(key, None)
- result = await self.send(chat_id, content, metadata=metadata)
- if result.success and result.message_id:
- if len(self._status_message_ids) >= self._STATUS_MESSAGE_IDS_MAX:
- # FIFO trim: drop the oldest half to bound memory (mirrors the Slack adapter).
- for stale in list(self._status_message_ids)[: self._STATUS_MESSAGE_IDS_MAX // 2]:
- self._status_message_ids.pop(stale, None)
- self._status_message_ids[key] = str(result.message_id)
- return result
- async def _edit_text(self, chat_id: str, message_id: str, text: str, parse_mode: Any = None) -> None:
- """``editMessageText`` with normalized ids; ``parse_mode=None`` sends plain text."""
- kwargs: dict[str, Any] = {"chat_id": normalize_telegram_chat_id(chat_id), "message_id": int(message_id), "text": text}
- if parse_mode is not None:
- kwargs["parse_mode"] = parse_mode
- await _await_with_thread_deadline(
- self._bot.edit_message_text(**kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- async def _edit_markdown_or_plain(self, chat_id: str, message_id: str, formatted: str, plain: str, warn_fmt: str) -> bool:
- """MarkdownV2 edit with plain-text fallback. Returns True on a "not modified" no-op (caller may
- skip further work); the fallback edit's exceptions propagate."""
- try:
- await self._edit_text(chat_id, message_id, formatted, ParseMode.MARKDOWN_V2)
- except Exception as fmt_err:
- if "not modified" in str(fmt_err).lower():
- return True
- logger.warning(warn_fmt, self.name, _redact_telegram_error_text(fmt_err))
- await self._edit_text(chat_id, message_id, plain)
- return False
- async def edit_message(
- self, chat_id: str, message_id: str, content: str, *, finalize: bool = False, metadata: Optional[dict[str, Any]] = None,
- ) -> SendResult:
- """Edit a previously sent Telegram message.
- Telegram caps a message at 4096 UTF-16 codeunits. Streaming replies that outgrow it must NOT be truncated
- silently nor fail (the consumer would re-send a duplicate): edit with the first chunk, send the rest as
- continuations, and return the final chunk's id as the next edit target."""
- if not self._bot:
- return SendResult(success=False, error="Not connected")
- # Shared per-chat budget (#116312): an interim (preview) edit is SKIPPED when the slot is busy —
- # the text it would show is shown by the next edit anyway, so a burst of edits can't trip flood
- # control. A final edit is never gated (the completed answer is always delivered). Sends wait for
- # their slot; edits defer instead. Over-cap interim edits are exempt: the saturated-preview dedup
- # below already throttles them to one real edit per ~4096-char growth. Consumed only when the
- # edit actually fires. The skip is flagged in raw_response so the stream consumer does not
- # record never-shown text as the visible prefix (a later flood fallback would then drop the
- # tail the user never saw).
- if (
- not finalize
- and utf16_len(content) <= self.MAX_MESSAGE_LENGTH
- and self._chat_outbound_slot_remaining(chat_id) > 0
- ):
- logger.debug(
- "[%s] skipping interim edit for chat %s (shared send+edit budget: slot busy)",
- self.name, chat_id)
- return SendResult(success=True, message_id=message_id, raw_response={"skipped": True})
- self._hold_chat_outbound_slot(chat_id)
- # Rich finalize (Bot API 10.1): edit the preview IN PLACE via rich_message — no fresh send + delete.
- # Before the 4,096 pre-flight because the rich cap is 32,768; falls back to legacy on rejection.
- # Rich finalize (Bot API 10.1): when the completed content has constructs the legacy MarkdownV2 edit
- # degrades (tables → bullet lists, task lists, <details>, block math) and rich is available, edit
- # the preview IN PLACE via editMessageText's rich_message param. No fresh send + delete → no
- # duplicate preview (the problem #46206 reverted the fresh-final path for). Attempted before the
- # 4,096 overflow pre-flight because the rich text cap is 32,768 — a rich table that exceeds the
- # MarkdownV2 limit must not be split into legacy chunks. Falls back to the legacy edit path
- # (overflow split included) on capability/permanent rejection.
- if finalize and self._rich_eligible(content):
- rich_result = await self._try_edit_rich(chat_id, message_id, content, metadata=metadata)
- if rich_result is not None:
- return rich_result
- # Pre-flight: over-limit content is split-and-delivered on finalize; mid-stream we truncate instead
- # (splitting moves the edit target to a continuation → infinite duplication loop).
- # Pre-flight: if content already exceeds the limit, split-and-deliver without round-tripping a
- # doomed edit. During streaming (finalize=False) we truncate instead of splitting — splitting
- # creates continuation messages whose IDs become the new edit target, and on the next token chunk
- # the full accumulated text is re-edited into the continuation, triggering another split → infinite
- # duplication loop (#48648).
- _preview_key = (str(chat_id), str(message_id))
- _saturated_preview = False
- if finalize:
- self._last_overflow_preview.pop(_preview_key, None) # the final edit always delivers full content
- if utf16_len(content) > self.MAX_MESSAGE_LENGTH:
- if finalize:
- return await self._edit_overflow_split(chat_id, message_id, content, finalize=finalize, metadata=metadata)
- content = self._truncate_stream_overflow_preview(content)
- _saturated_preview = True
- # Saturated-preview dedup: past the cap every progressive edit truncates to the same text;
- # re-sending is a visual no-op that still burns flood budget (200s+ penalties).
- if self._last_overflow_preview.get(_preview_key) == content:
- return SendResult(success=True, message_id=message_id)
- elif not finalize:
- # Content shrank back under the cap — clear stale saturation state so dedup can't mask an edit.
- self._last_overflow_preview.pop(_preview_key, None)
- try:
- if not finalize:
- await self._edit_text(chat_id, message_id, content)
- if _saturated_preview:
- self._last_overflow_preview[_preview_key] = content
- return SendResult(success=True, message_id=message_id)
- await self._edit_markdown_or_plain(
- chat_id, message_id, self.format_message(content), _strip_mdv2(content) if content else content,
- "[%s] MarkdownV2 edit failed, falling back to plain text: %s")
- return SendResult(success=True, message_id=message_id)
- except Exception as e:
- err_str = str(e).lower()
- if "not modified" in err_str:
- return SendResult(success=True, message_id=message_id)
- # Reactive split: MarkdownV2 escapes can inflate the payload past the limit even when raw text fit.
- if "message_too_long" in err_str or "too long" in err_str:
- logger.debug(
- "[%s] edit_message overflow (%d UTF-16 > %d), splitting", self.name, utf16_len(content), self.MAX_MESSAGE_LENGTH)
- if finalize:
- return await self._edit_overflow_split(chat_id, message_id, content, finalize=finalize, metadata=metadata)
- # Mid-stream: truncate and retry instead of splitting (saturated-preview dedup as above).
- # See #48648.
- truncated = self._truncate_stream_overflow_preview(content)
- if self._last_overflow_preview.get(_preview_key) == truncated:
- return SendResult(success=True, message_id=message_id)
- await self._edit_text(chat_id, message_id, truncated)
- self._last_overflow_preview[_preview_key] = truncated
- return SendResult(success=True, message_id=message_id)
- # Flood control: short waits retry inline; long waits fail immediately so streaming falls back
- # to a normal final send instead of a clipped partial.
- retry_after = getattr(e, "retry_after", None)
- if retry_after is not None or "retry after" in err_str:
- wait = retry_after if retry_after else 1.0
- if wait > _FLOOD_INLINE_WAIT_CAP_SECS:
- # Log AFTER the cap check: "waiting 33.0s" followed by no wait misled an investigation.
- logger.warning(
- "[%s] Telegram flood control, refusing edit (retry_after %.1fs > %.0fs inline cap)",
- self.name, wait, _FLOOD_INLINE_WAIT_CAP_SECS)
- return _flood_cap_result(wait)
- logger.warning("[%s] Telegram flood control, waiting %.1fs", self.name, wait)
- await asyncio.sleep(wait)
- try:
- await self._edit_text(chat_id, message_id, content)
- return SendResult(success=True, message_id=message_id)
- except Exception as retry_err:
- safe_retry_error = _redact_telegram_error_text(retry_err)
- logger.error("[%s] Edit retry failed after flood wait: %s", self.name, safe_retry_error)
- retry_wait = getattr(retry_err, "retry_after", None)
- if retry_wait is not None or "retry after" in str(retry_err).lower():
- # Still flooded after the inline wait, and typically for much longer than the
- # first refusal asked for. Fail closed canonically so the ledger arms its
- # timer on this delay rather than storing the platform's raw wording, which
- # it would read as an ordinary failure and never redeliver.
- return _flood_cap_result(
- float(retry_wait) if retry_wait is not None else wait)
- return SendResult(success=False, error=safe_retry_error)
- safe_error = _redact_telegram_error_text(e)
- # Transient network errors must not permanently disable progress-message editing.
- _transient_markers = (
- "connecterror", "connect error", "connection error", "networkerror", "network error", "timed out", "readtimeout",
- "writetimeout", "server disconnected", "temporarily unavailable", "temporary failure", "httpx")
- if any(m in err_str for m in _transient_markers):
- logger.warning("[%s] Transient network error editing message %s (will retry): %s", self.name, message_id, safe_error)
- return SendResult(success=False, error=safe_error, retryable=True)
- logger.error("[%s] Failed to edit Telegram message %s: %s", self.name, message_id, safe_error)
- return SendResult(success=False, error=safe_error)
- def _truncate_stream_overflow_preview(self, content: str) -> str:
- """One-message preview for oversized streaming edits (edits must keep targeting the original id;
- final edits use ``_edit_overflow_split``).
- Splitting a mid-stream preview creates continuation messages and moves the active message id, so the
- next accumulated-token edit repeats the overflow cycle (#48648). Final edits still use
- ``_edit_overflow_split`` to deliver the complete response.
- """
- return self.truncate_message(content, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)[0]
- async def _send_overflow_continuation(
- self, chat_id: str, chunk: str, reply_to_id: Optional[int], thread_kwargs: dict[str, Any],
- thread_id: Optional[str], metadata: Optional[dict[str, Any]], finalize: bool):
- """Send one continuation chunk (MarkdownV2 then plain on finalize; raw when streaming); drops the
- reply anchor once on 'reply message not found'. Returns the sent message or None."""
- base = {**self._link_preview_kwargs(), **self._notification_kwargs(metadata)}
- for use_markdown in (True, False) if finalize else (False,):
- try:
- if use_markdown:
- text = _separate_chunk_indicator_from_fence(self.format_message(chunk))
- else:
- # Degrade to stripped text on finalize (raw ** / ``` would render literally); previews stay raw.
- text = _strip_mdv2(chunk) if finalize else chunk
- return await _await_with_thread_deadline(
- self._bot.send_message(
- chat_id=normalize_telegram_chat_id(chat_id), text=text, parse_mode=ParseMode.MARKDOWN_V2 if use_markdown else None,
- reply_to_message_id=reply_to_id, **thread_kwargs, **base),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- except Exception as send_err:
- if "reply message not found" in str(send_err).lower():
- # Private DM topic fallback needs anchor + topic id together; forum topics keep thread id.
- retry_thread_kwargs = (
- {} if self._dm_topic_fallback(metadata)
- else self._thread_kwargs_for_send(chat_id, thread_id, metadata, reply_to_message_id=None))
- try:
- return await _await_with_thread_deadline(
- self._bot.send_message(
- chat_id=normalize_telegram_chat_id(chat_id), text=_strip_mdv2(chunk) if finalize else chunk,
- **retry_thread_kwargs, **base),
- timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- except Exception as _retry_err:
- logger.warning(
- "[%s] Overflow continuation no-reply retry failed: %s", self.name, _redact_telegram_error_text(_retry_err))
- return None
- if use_markdown:
- continue # try plain text on next loop iteration
- logger.warning("[%s] Overflow continuation send failed: %s", self.name, _redact_telegram_error_text(send_err))
- return None
- return None
- async def _edit_overflow_split(
- self, chat_id: str, message_id: str, content: str, *, finalize: bool, metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Split an oversized edit across the existing message + continuations: edit ``message_id`` with
- chunk 1, send the rest as replies to the previous chunk, return ``message_id=<last-chunk-id>`` so
- the consumer keeps editing the newest message. ``success=False`` only if the first-chunk edit fails."""
- chunks = self.truncate_message(content, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)
- if len(chunks) <= 1:
- chunks = [content] # defensive: a single chunk just edits normally
- first_chunk = chunks[0]
- try:
- if finalize:
- await self._edit_markdown_or_plain(
- chat_id, message_id, _separate_chunk_indicator_from_fence(self.format_message(first_chunk)), _strip_mdv2(first_chunk),
- "[%s] Overflow split: MarkdownV2 first-chunk edit failed, falling back to plain text: %s")
- else:
- await self._edit_text(chat_id, message_id, first_chunk)
- except Exception as e:
- if "not modified" not in str(e).lower(): # identical first chunk still sends continuations
- logger.error("[%s] Overflow split: first-chunk edit failed: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
- return SendResult(success=False, error=_redact_telegram_error_text(e))
- # Continuations call self._bot.send_message directly to skip self.send's pre-chunking.
- continuation_ids: list[str] = []
- delivered_chunks = [first_chunk]
- prev_id = message_id
- thread_id = self._metadata_thread_id(metadata)
- for chunk in chunks[1:]:
- reply_to_id = int(prev_id) if prev_id else None
- thread_kwargs = self._thread_kwargs_for_send(chat_id, thread_id, metadata, reply_to_message_id=reply_to_id)
- sent_msg = await self._send_overflow_continuation(chat_id, chunk, reply_to_id, thread_kwargs, thread_id, metadata, finalize)
- if sent_msg is None:
- # Partial delivery: do NOT report success — the consumer would treat it as final delivery.
- logger.warning("[%s] Overflow split: stopped at %d/%d chunks delivered", self.name, 1 + len(continuation_ids), len(chunks))
- delivered_prefix = "".join(re.sub(r" \(\d+/\d+\)$", "", delivered) for delivered in delivered_chunks)
- return SendResult(
- success=False, message_id=prev_id, error="overflow_continuation_failed", retryable=True,
- raw_response={
- "partial_overflow": True, "delivered_chunks": 1 + len(continuation_ids),
- "total_chunks": len(chunks), "last_message_id": prev_id, "delivered_prefix": delivered_prefix,
- "continuation_message_ids": tuple(continuation_ids)},
- continuation_message_ids=tuple(continuation_ids))
- new_id = str(getattr(sent_msg, "message_id", "")) or prev_id
- continuation_ids.append(new_id)
- delivered_chunks.append(chunk)
- prev_id = new_id
- last_id = continuation_ids[-1] if continuation_ids else message_id
- logger.debug("[%s] Overflow split delivered %d chunks; last_id=%s", self.name, 1 + len(continuation_ids), last_id)
- return SendResult(success=True, message_id=last_id, continuation_message_ids=tuple(continuation_ids))
- async def delete_message(self, chat_id: str, message_id: str) -> bool:
- """Delete a bot-posted message (Bot API allows it within 48h); failures are non-fatal.
- Used by the stream consumer's fresh-final cleanup path (ported from openclaw/openclaw#72038) to
- remove long-lived preview messages after sending the completed reply as a fresh message. Telegram's
- Bot API ``deleteMessage`` works for bot-posted messages in the last 48 hours. Failures are non-fatal
- — the caller leaves the preview in place and logs at debug level.
- """
- if not self._bot:
- return False
- try:
- await self._bot.delete_message(chat_id=normalize_telegram_chat_id(chat_id), message_id=int(message_id))
- return True
- except Exception as e:
- logger.debug("[%s] Failed to delete Telegram message %s: %s", self.name, message_id, _redact_telegram_error_text(e))
- return False
- def supports_draft_streaming(self, chat_type: Optional[str] = None, metadata: Optional[dict[str, Any]] = None) -> bool:
- """sendMessageDraft works for private chats only (Bot API 9.5) and needs PTB >= 22.6; groups and
- older installs use the edit-based path. ``rich_drafts`` controls draft *format*, not availability."""
- if not self._bot or not hasattr(self._bot, "send_message_draft"):
- return False
- return (chat_type or "").lower() in {"dm", "private"}
- async def send_draft(self, chat_id: str, draft_id: int, content: str, metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Stream a partial message via ``sendRichMessageDraft`` (when rich is enabled and supported) else
- ``sendMessageDraft``; reusing ``draft_id`` animates the preview. The caller sends the final text."""
- if not self._bot:
- return SendResult(success=False, error="not_connected")
- # Rich draft fast-path; any failure degrades to the plain draft below. Drafts have no message_id.
- if self._should_attempt_rich_draft(content) and await self._try_send_rich_draft(chat_id, draft_id, content, metadata):
- return SendResult(success=True, message_id=None)
- if not hasattr(self._bot, "send_message_draft"):
- return SendResult(success=False, error="api_unavailable")
- # Drafts share the regular-send UTF-16 length contract.
- text = content if len(
- content) <= self.MAX_MESSAGE_LENGTH else self.truncate_message(content, self.MAX_MESSAGE_LENGTH, len_fn=utf16_len)[0]
- # Same MarkdownV2 conversion as ``send`` (MarkdownV2 then plain) so the draft doesn't snap at the end. Exception: a Rich
- # final with rich drafts disabled previews raw — the legacy formatter would turn pipe tables into bullets.
- plain_rich_preview = bool(
- getattr(self, "_rich_messages_enabled", False) and not getattr(self, "_rich_drafts_enabled", False)
- and self._needs_rich_rendering(text))
- draft_thread_kwargs = self._thread_kwargs_for_draft(chat_id, metadata)
- for use_markdown in ((False,) if plain_rich_preview else (True, False)):
- kwargs: dict[str, Any] = {
- "chat_id": normalize_telegram_chat_id(chat_id), "draft_id": int(draft_id),
- "text": self.format_message(text) if use_markdown else text}
- if use_markdown:
- kwargs["parse_mode"] = ParseMode.MARKDOWN_V2
- kwargs.update(draft_thread_kwargs)
- try:
- if await _await_with_thread_deadline(
- self._bot.send_message_draft(**kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False):
- return SendResult(success=True, message_id=None)
- return SendResult(success=False, error="draft_rejected")
- except Exception as e:
- # MarkdownV2 parse failure → retry once as plain text; anything else returns to the caller,
- # which falls back to edit-based streaming for this response.
- if use_markdown and self._is_bad_request_error(e):
- logger.debug(
- "[%s] sendMessageDraft MarkdownV2 rejected, retrying as plain text (chat=%s draft_id=%s): %s",
- self.name, chat_id, draft_id, _redact_telegram_error_text(e))
- continue
- logger.debug("[%s] sendMessageDraft failed (chat=%s draft_id=%s): %s", self.name, chat_id, draft_id, e)
- return SendResult(success=False, error=_redact_telegram_error_text(e))
- return SendResult(success=False, error="draft_rejected")
- async def _send_message_with_thread_fallback(self, **kwargs):
- """Send a control-style message (approval prompts, pickers), retrying once without
- message_thread_id on 'Message thread not found' (stale thread_id); ``send`` has its own.
- Used for control-style sends (approval prompts, model picker, update prompts) that can carry a stale
- thread_id from a DM reply chain. The streaming send loop has its own equivalent (PR #3390) at the
- body of ``send``; this helper applies the same retry pattern to the non-streaming control paths.
- """
- if not self._bot:
- raise RuntimeError("Not connected")
- message_thread_id = kwargs.get("message_thread_id")
- try:
- return await _await_with_thread_deadline(
- self._bot.send_message(**kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- except Exception as send_err:
- if (message_thread_id is not None and self._is_bad_request_error(send_err) and self._is_thread_not_found_error(send_err)):
- logger.warning(
- "[%s] Thread %s not found for control message, retrying without message_thread_id", self.name, message_thread_id)
- # Same prune as the streaming send path; control sends carry no gateway metadata.
- self._prune_stale_dm_topic_binding(kwargs.get("chat_id"), message_thread_id)
- retry_kwargs = dict(kwargs)
- retry_kwargs.pop("message_thread_id", None)
- return await _await_with_thread_deadline(
- self._bot.send_message(**retry_kwargs), timeout=_TEXT_SEND_DEADLINE, label="telegram-send", dump_on_blocked_loop=False)
- raise
- async def _send_control_message(
- self, chat_id: str, text: str, *, parse_mode: Any, thread_id: Optional[str], metadata: Optional[dict[str, Any]],
- reply_markup: Any = None, reply_to_mode: Optional[str] = None):
- """Send a control-style message (prompt/picker) with topic routing + thread fallback."""
- reply_to_id = self._reply_to_message_id_for_send(None, metadata, reply_to_mode=reply_to_mode)
- kwargs: dict[str, Any] = {
- "chat_id": normalize_telegram_chat_id(chat_id), "text": text, "parse_mode": parse_mode, **self._link_preview_kwargs()}
- if reply_markup is not None:
- kwargs["reply_markup"] = reply_markup
- kwargs["reply_to_message_id"] = reply_to_id
- kwargs.update(self._thread_kwargs_for_send(
- chat_id, thread_id, metadata, reply_to_message_id=reply_to_id, reply_to_mode=reply_to_mode))
- return await self._send_message_with_thread_fallback(**kwargs)
- async def _send_prompt(self, what: str, chat_id: str, metadata: Optional[dict[str, Any]], build, *,
- parse_mode: Any = None, thread_id: Any = None, reply_to_mode: Any = None) -> SendResult:
- """Shared control-prompt shell: not-connected guard, ``build()`` → ``(text, keyboard, on_sent)`` (or a
- SendResult to return as-is), routed send, state hook, redacted failure log."""
- if not self._bot:
- return SendResult(success=False, error="Not connected")
- try:
- built = build()
- if isinstance(built, SendResult):
- return built
- text, keyboard, on_sent = built
- msg = await self._send_control_message(
- chat_id, text, parse_mode=parse_mode if parse_mode is not None else ParseMode.MARKDOWN_V2,
- reply_markup=keyboard, thread_id=thread_id, metadata=metadata, reply_to_mode=reply_to_mode)
- if on_sent is not None:
- on_sent(msg)
- return SendResult(success=True, message_id=str(msg.message_id))
- except Exception as e:
- logger.warning("[%s] %s failed: %s", self.name, what, _redact_telegram_error_text(e))
- return SendResult(success=False, error=_redact_telegram_error_text(e))
- @staticmethod
- def _rows_of_two(buttons: list) -> list:
- """2-per-row layout keeps labels readable on mobile (a 4-button row truncates)."""
- return [buttons[i:i + 2] for i in range(0, len(buttons), 2)]
- async def send_update_prompt(
- self, chat_id: str, prompt: str, default: str = "", session_key: str = "", metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Send an inline-keyboard Yes/No prompt for the gateway ``/update`` watcher."""
- def build():
- default_hint = t("platform.telegram.prompt.default_hint", default=default) if default else ""
- text = self.format_message(f"☤ *{t('platform.telegram.prompt.update_header')}*\n\n{prompt}{default_hint}")
- keyboard = InlineKeyboardMarkup([[
- InlineKeyboardButton(t("platform.telegram.prompt.affirm"), callback_data="update_prompt:y"),
- InlineKeyboardButton(t("platform.telegram.prompt.negate"), callback_data="update_prompt:n")]])
- return text, keyboard, None
- return await self._send_prompt(
- "send_update_prompt", chat_id, metadata, build, thread_id=self._metadata_thread_id(metadata), reply_to_mode=self._reply_to_mode)
- # Template attrs for the shared _format_exec_approval core (HTML mode). Properties, not class
- # constants: the wording comes from the catalog for the language active at send time, and the
- # translated text is HTML-escaped BEFORE the <b> wrapper (a stray ``&``/``<`` would break the card).
- @property
- def _EA_HEADER(self) -> str:
- return f"⚠️ <b>{_html.escape(t('gateway.exec_approval.header'))}</b>\n\n"
- _EA_CODE_OPEN = "<pre>"
- _EA_CODE_CLOSE = "</pre>\n\n"
- @property
- def _EA_REASON_LABEL(self) -> str:
- return f"{_html.escape(t('gateway.exec_approval.reason_label'))}: "
- @property
- def _EA_SMART_DENY_LINE(self) -> str:
- return "\n\n" + _bold_label_html(t("gateway.exec_approval.smart_deny_line"))
- _EA_REASON_BUDGET = 500 # escaped chars; the reason shares the 4096 cap with the command
- def _ea_escape(self, text: str) -> str:
- return _html.escape(text)
- def _exec_approval_cmd_budget(self, description: str, smart_denied: bool) -> int:
- # Telegram rejects the whole card ("Message is too long") and the gateway then falls back to
- # the text /approve prompt, so budget the preview against what the framing leaves of the cap.
- fixed = utf16_len( # UTF-16 units, like the 4096 chunker in send()
- self._EA_HEADER + self._EA_CODE_OPEN + self._EA_CODE_CLOSE + self._EA_REASON_LABEL
- + self._ea_escape(description) + "..." + self._ea_deadline_line()
- + (self._EA_SMART_DENY_LINE if smart_denied else ""))
- return max(0, self.MAX_MESSAGE_LENGTH - fixed)
- @property
- def _EA_ACTION_LABELS(self) -> dict[str, str]:
- # Shorter than the base wording on purpose: two buttons share a row on mobile.
- return {choice: t(f"platform.telegram.approval.action_{choice}") for choice in ("once", "session", "always", "deny")}
- async def _send_exec_approval_prompt(self, prompt: ExecApprovalPrompt) -> SendResult:
- """Inline-keyboard approval prompt; buttons call ``resolve_gateway_approval()`` like the
- text ``/approve`` flow."""
- def build():
- # Short monotonic ids in callback_data map back to session_key.
- import itertools
- if not hasattr(self, "_approval_counter"):
- self._approval_counter = itertools.count(1)
- approval_id = next(self._approval_counter)
- buttons = [InlineKeyboardButton(label, callback_data=f"ea:{choice}:{approval_id}")
- for label, choice, _ in prompt.actions]
- def on_sent(_msg):
- self._approval_state[approval_id] = prompt.session_key
- if prompt.request_id:
- self._approval_request_ids[approval_id] = prompt.request_id
- return prompt.text, InlineKeyboardMarkup(self._rows_of_two(buttons)), on_sent
- return await self._send_prompt(
- "send_exec_approval", prompt.chat_id, prompt.metadata, build, parse_mode=ParseMode.HTML,
- thread_id=self._metadata_thread_id(prompt.metadata), reply_to_mode=self._reply_to_mode)
- async def send_slash_confirm(
- self, chat_id: str, title: str, message: str, session_key: str, confirm_id: str,
- metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Render a three-button slash-command confirmation prompt."""
- def build():
- keyboard = InlineKeyboardMarkup([
- [
- InlineKeyboardButton(t("platform.telegram.slash_confirm.approve_once"), callback_data=f"sc:once:{confirm_id}"),
- InlineKeyboardButton(t("platform.telegram.slash_confirm.always_approve"), callback_data=f"sc:always:{confirm_id}")],
- [InlineKeyboardButton(t("platform.telegram.slash_confirm.cancel"), callback_data=f"sc:cancel:{confirm_id}")],
- ])
- # Budget the MarkdownV2 rendering (escaping expands text), not the raw message.
- preview = self.format_message(self._ea_fit(
- message, self.MAX_MESSAGE_LENGTH - utf16_len(self.format_message("...")), escape=self.format_message))
- return preview, keyboard, lambda msg: self._slash_confirm_state.__setitem__(confirm_id, session_key)
- return await self._send_prompt(
- "send_slash_confirm", chat_id, metadata, build, thread_id=self._metadata_thread_id(metadata), reply_to_mode=self._reply_to_mode)
- async def send_clarify(
- self, chat_id: str, question: str, choices: Optional[list], clarify_id: str, session_key: str,
- metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Render a clarify prompt: numbered buttons per choice plus "✏️ Other (type answer)" (flips to
- text-capture mode); without choices, plain question and the gateway text-intercept captures."""
- def build():
- text = f"❓ {_html.escape(question)}"
- keyboard = None
- if choices:
- # Full option text in the body (mobile truncates button labels); buttons keep numeric labels.
- text += "\n\n" + "\n".join(f"{i + 1}. {_html.escape(str(c))}" for i, c in enumerate(choices))
- # Telegram caps callback_data at 64 bytes; keep "cl:<id>:<idx>" short.
- rows = [[InlineKeyboardButton(str(idx + 1), callback_data=f"cl:{clarify_id}:{idx}")] for idx in range(len(choices))]
- rows.append([InlineKeyboardButton(t("platform.telegram.prompt.other"), callback_data=f"cl:{clarify_id}:other")])
- keyboard = InlineKeyboardMarkup(rows)
- return text, keyboard, lambda msg: self._clarify_state.__setitem__(clarify_id, session_key)
- return await self._send_prompt(
- "send_clarify", chat_id, metadata, build, parse_mode=ParseMode.HTML, thread_id=self._metadata_thread_id(metadata))
- @staticmethod
- def _provider_get_label():
- try:
- from hermes_cli.providers import get_label
- except ImportError:
- def get_label(slug):
- return slug
- return get_label
- async def send_model_picker(
- self, chat_id: str, providers: list, current_model: str, current_provider: str, session_key: str,
- on_model_selected, metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Send an inline-keyboard model picker: provider → model drill-down, edited in place."""
- def build():
- keyboard, provider_page_info = self._build_provider_keyboard(providers, 0)
- text = self.format_message(
- self._provider_list_text(current_model, self._provider_get_label()(current_provider), provider_page_info)
- )
- def _remember(msg):
- self._model_picker_state[str(chat_id)] = {
- "msg_id": msg.message_id, "providers": providers, "session_key": session_key, "on_model_selected": on_model_selected,
- "current_model": current_model, "current_provider": current_provider, "provider_page": 0}
- return text, keyboard, _remember
- return await self._send_prompt(
- "send_model_picker", chat_id, metadata, build, thread_id=metadata.get("thread_id") if metadata else None,
- reply_to_mode=self._reply_to_mode)
- _PROVIDER_PAGE_SIZE = 10
- async def send_choice_picker(
- self, chat_id: str, title: str, choices: list, session_key: str, on_choice_selected,
- metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Flat inline-keyboard picker (one tap → one value) for /reasoning, /fast, etc. Each choice dict:
- ``{"value": str, "label": str, "is_current": bool}``."""
- def build():
- buttons = []
- for i, choice in enumerate(choices):
- label = str(choice.get("label") or choice.get("value") or "")
- if choice.get("is_current"):
- label = f"✓ {label}"
- buttons.append(InlineKeyboardButton(label, callback_data=f"cp:{i}"))
- if not buttons:
- return SendResult(success=False, error="No choices")
- keyboard = InlineKeyboardMarkup(self._rows_of_two(buttons))
- def _remember(msg):
- self._choice_picker_state[str(chat_id)] = {
- "msg_id": msg.message_id, "choices": choices, "session_key": session_key, "on_choice_selected": on_choice_selected}
- return self.format_message(title), keyboard, _remember
- return await self._send_prompt(
- "send_choice_picker", chat_id, metadata, build, thread_id=metadata.get("thread_id") if metadata else None,
- reply_to_mode=self._reply_to_mode)
- async def _edit_result_text(self, query, result_text: str) -> None:
- """Replace a picker message with ``result_text`` (MarkdownV2, then plain, then give up), keyboard removed."""
- try:
- await query.edit_message_text(text=self.format_message(result_text), parse_mode=ParseMode.MARKDOWN_V2, reply_markup=None)
- except Exception:
- with contextlib.suppress(Exception):
- await query.edit_message_text(text=result_text, parse_mode=None, reply_markup=None)
- async def _handle_choice_picker_callback(self, query, data: str, chat_id: str) -> None:
- """Handle choice picker button taps (cp:<index>)."""
- state = self._choice_picker_state.get(chat_id)
- if not state:
- await query.answer(text=_toast("platform.telegram.picker.expired_rerun"))
- return
- try:
- choice = state["choices"][int(data[3:])]
- except (ValueError, IndexError):
- await query.answer(text=_toast("platform.telegram.picker.invalid_selection"))
- return
- callback = state.get("on_choice_selected")
- if not callback:
- await query.answer(text=_toast("platform.telegram.picker.expired"))
- return
- try:
- result_text = await callback(chat_id, str(choice.get("value") or ""))
- except Exception as exc:
- logger.error("Choice picker selection failed: %s", exc)
- result_text = t("platform.telegram.picker.apply_error", error=str(exc))
- await self._edit_result_text(query, result_text)
- await query.answer()
- self._choice_picker_state.pop(chat_id, None)
- _MODEL_PAGE_SIZE = 8
- @staticmethod
- def _provider_button(p: dict) -> InlineKeyboardButton:
- count = p.get("total_models", len(p.get("models", [])))
- label = f"{p['name']} ({count})"
- if p.get("is_current"):
- label = f"✓ {label}"
- return InlineKeyboardButton(label, callback_data=f"mp:{p['slug']}")
- @staticmethod
- def _picker_nav_row(page: int, total_pages: int, prefix: str) -> list:
- """``◀ Prev | n/N | Next ▶`` row (``prefix`` = ``mpv``/``mg`` page callback)."""
- nav: list = []
- if page > 0:
- nav.append(InlineKeyboardButton(t("platform.telegram.picker.prev"), callback_data=f"{prefix}:{page - 1}"))
- nav.append(InlineKeyboardButton(f"{page + 1}/{total_pages}", callback_data="mx:noop"))
- if page < total_pages - 1:
- nav.append(InlineKeyboardButton(t("platform.telegram.picker.next"), callback_data=f"{prefix}:{page + 1}"))
- return nav
- @staticmethod
- def _picker_back_cancel_row() -> list:
- return [InlineKeyboardButton(t("platform.telegram.picker.back"), callback_data="mb"), InlineKeyboardButton(t("platform.telegram.picker.cancel"), callback_data="mx")]
- def _paged_keyboard(self, buttons: list, page_meta: dict, nav_prefix: str, tail_row: list) -> tuple:
- rows = self._rows_of_two(buttons)
- if page_meta["total_pages"] > 1:
- rows.append(self._picker_nav_row(page_meta["page"], page_meta["total_pages"], nav_prefix))
- rows.append(tail_row)
- return InlineKeyboardMarkup(rows), page_meta["page_info"]
- def _build_provider_keyboard(self, providers: list, page: int = 0) -> tuple:
- """Paginated top-level provider keyboard folding provider families (Kimi/Moonshot, MiniMax, xAI…)
- into one ``mpg:<gid>`` button via the shared ``group_providers`` fold; singles are ``mp:<slug>``."""
- try:
- from hermes_cli.models_catalog_static import group_providers
- except Exception:
- group_providers = None
- by_slug = {p.get("slug"): p for p in providers}
- buttons: list = []
- if group_providers is not None:
- for row in group_providers([p.get("slug") for p in providers]):
- if row["kind"] == "group":
- members = [by_slug[m] for m in row["members"] if m in by_slug]
- count = sum(m.get("total_models", len(m.get("models", []))) for m in members)
- label = f"{row['label']} ▸ ({count})"
- if any(m.get("is_current") for m in members):
- label = f"✓ {label}"
- buttons.append(InlineKeyboardButton(label, callback_data=f"mpg:{row['group_id']}"))
- else:
- p = by_slug.get(row["slug"])
- if p is not None:
- buttons.append(self._provider_button(p))
- else:
- buttons = [self._provider_button(p) for p in providers]
- page_buttons, page_meta = self._format_choice_page(buttons, page, self._PROVIDER_PAGE_SIZE)
- return self._paged_keyboard(page_buttons, page_meta, "mpv", [InlineKeyboardButton(t("platform.telegram.picker.cancel"), callback_data="mx")])
- def _build_model_keyboard(self, models: list, page: int, provider: Optional[dict] = None) -> tuple:
- """Build paginated model buttons. Returns (keyboard, page_info_text). Window plans: each button
- carries the plan word (``model-b · Large``) from the provider row; other accounts are unchanged."""
- from hermes_cli.model_picker_usage import picker_row_suffix
- page_models, page_meta = self._format_choice_page(models, page, self._MODEL_PAGE_SIZE)
- start = page_meta["start"]
- buttons: list = []
- for i, model_id in enumerate(page_models):
- short = model_id.split("/")[-1] if "/" in model_id else model_id
- suffix = picker_row_suffix(provider, model_id)
- room = 38 - (len(suffix) + 3 if suffix else 0)
- if len(short) > room:
- short = short[:max(room - 3, 8)] + "..."
- if suffix:
- short = f"{short} · {suffix}"
- buttons.append(InlineKeyboardButton(short, callback_data=f"mm:{start + i}"))
- return self._paged_keyboard(buttons, page_meta, "mg", self._picker_back_cancel_row())
- async def _picker_edit(self, query, text_md: str, keyboard) -> None:
- """Re-render the picker message in place (MarkdownV2) and ack the tap."""
- await query.edit_message_text(text=self.format_message(text_md), parse_mode=ParseMode.MARKDOWN_V2, reply_markup=keyboard)
- await query.answer()
- async def _picker_show_models(self, query, state: dict, page: int) -> None:
- """Render the model page for the provider currently selected in ``state``."""
- models = state.get("model_list", [])
- state["model_page"] = page
- pname = state.get("selected_provider_name", "")
- provider_slug = state.get("selected_provider", "")
- provider = next((p for p in state["providers"] if p["slug"] == provider_slug), None)
- keyboard, page_info = self._build_model_keyboard(models, page, provider)
- total = provider.get("total_models", len(models)) if provider else len(models)
- shown = len(models)
- extra = f"\n_{t('platform.telegram.picker.more_available', count=str(total - shown))}_" if total > shown else ""
- # Window plans: the limit line under the provider (absent for every other account).
- plan_line = f"\n⚠️ {provider['plan_line']}" if provider and provider.get("plan_line") else ""
- await self._picker_edit(
- query,
- f"⚙ *{t('platform.telegram.picker.title')}*\n\n"
- f"{t('platform.telegram.picker.provider_label', provider=f'*{pname}*')}{page_info}{plan_line}\n"
- f"{t('platform.telegram.picker.select_model')}{extra}",
- keyboard)
- @staticmethod
- def _provider_list_text(current_model: str, provider_label: str, page_info: str) -> str:
- model = f"`{current_model or t('platform.telegram.picker.unknown_model')}`"
- return (f"⚙ *{t('platform.telegram.picker.title')}*\n\n{t('platform.telegram.picker.current_model', model=model)}\n"
- f"{t('platform.telegram.picker.provider_label', provider=provider_label)}\n\n"
- f"{t('platform.telegram.picker.select_provider')}{page_info}")
- async def _picker_show_providers(self, query, state: dict, page: int, get_label) -> None:
- """Render the (folded, paginated) provider list."""
- keyboard, provider_page_info = self._build_provider_keyboard(state["providers"], page)
- try:
- provider_label = get_label(state["current_provider"])
- except Exception:
- provider_label = state["current_provider"]
- await self._picker_edit(query, self._provider_list_text(state["current_model"], provider_label, provider_page_info), keyboard)
- async def _picker_selection(self, query, state: dict, raw_idx: str) -> Optional[tuple]:
- """Resolve ``mm:``/``mc:`` index → ``(idx, model_id, provider_slug, callback)``; answers + None on error."""
- try:
- idx = int(raw_idx)
- except ValueError:
- await query.answer(text=_toast("platform.telegram.picker.invalid_selection"))
- return None
- model_list = state.get("model_list", [])
- if idx < 0 or idx >= len(model_list):
- await query.answer(text=_toast("platform.telegram.picker.invalid_model_index"))
- return None
- callback = state.get("on_model_selected")
- if not callback:
- await query.answer(text=_toast("platform.telegram.picker.expired"))
- return None
- return idx, model_list[idx], state.get("selected_provider", ""), callback
- async def _picker_switch(self, query, chat_id: str, model_id: str, provider_slug: str, callback) -> None:
- """Perform the model switch, render the result, and drop the picker state."""
- switch_failed = False
- try:
- result_text = await callback(chat_id, model_id, provider_slug)
- except Exception as exc:
- logger.error("Model picker switch failed: %s", exc)
- result_text = t("platform.telegram.picker.switch_error", error=str(exc))
- switch_failed = True
- await self._edit_result_text(query, result_text)
- await query.answer(text=_toast("platform.telegram.picker.switch_failed" if switch_failed else "platform.telegram.picker.switched"))
- self._model_picker_state.pop(chat_id, None)
- @staticmethod
- async def _parse_page(query, raw: str) -> Optional[int]:
- try:
- return int(raw)
- except ValueError:
- await query.answer(text=_toast("platform.telegram.picker.invalid_page"))
- return None
- async def _handle_model_picker_callback(self, query, data: str, chat_id: str) -> None:
- """Handle model picker callbacks (mp:/mpg:/mpv:/mm:/mc:/mb/mx/mg:)."""
- state = self._model_picker_state.get(chat_id)
- if not state:
- await query.answer(text=_toast("platform.telegram.picker.expired_model"))
- return
- get_label = self._provider_get_label()
- if data.startswith("mp:"): # provider selected: show model buttons (page 0)
- provider_slug = data[3:]
- provider = next((p for p in state["providers"] if p["slug"] == provider_slug), None)
- if not provider:
- await query.answer(text=_toast("platform.telegram.picker.provider_not_found"))
- return
- state["selected_provider"] = provider_slug
- state["selected_provider_name"] = provider.get("name", provider_slug)
- state["model_list"] = provider.get("models", [])
- await self._picker_show_models(query, state, 0)
- elif data.startswith("mg:"): # model page navigation
- page = await self._parse_page(query, data[3:])
- if page is not None:
- await self._picker_show_models(query, state, page)
- elif data.startswith("mpv:"): # provider page navigation
- page = await self._parse_page(query, data[4:])
- if page is not None:
- state["provider_page"] = page
- await self._picker_show_providers(query, state, page, get_label)
- elif data.startswith("mc:"): # expensive model confirmed: perform the switch
- sel = await self._picker_selection(query, state, data[3:])
- if sel is not None:
- _idx, model_id, provider_slug, callback = sel
- await self._picker_switch(query, chat_id, model_id, provider_slug, callback)
- elif data.startswith("mm:"): # model selected: warn if expensive, else perform the switch
- sel = await self._picker_selection(query, state, data[3:])
- if sel is None:
- return
- idx, model_id, provider_slug, callback = sel
- try:
- from hermes_cli.model_selection_guards import combined_selection_warning
- # Pricing lookup may hit models.dev on a cache miss — keep it off the event loop.
- warning = await asyncio.to_thread(combined_selection_warning, model_id, provider=provider_slug)
- except Exception:
- warning = None
- if warning is not None:
- keyboard = InlineKeyboardMarkup([
- [InlineKeyboardButton(t("platform.telegram.picker.switch_anyway"), callback_data=f"mc:{idx}")], self._picker_back_cancel_row()])
- await query.edit_message_text(
- text=self.format_message(f"⚠ *{warning.title}*\n\n{warning.message}"),
- parse_mode=ParseMode.MARKDOWN_V2, reply_markup=keyboard)
- await query.answer(text=_toast("platform.telegram.picker.confirm_selection"))
- return
- await self._picker_switch(query, chat_id, model_id, provider_slug, callback)
- elif data.startswith("mpg:"): # provider group selected: show member providers
- group_id = data[4:]
- try:
- from hermes_cli.models_catalog_static import PROVIDER_GROUPS
- _label, _desc, member_slugs = PROVIDER_GROUPS.get(group_id, ("", "", []))
- except Exception:
- _label, member_slugs = "", []
- by_slug = {p["slug"]: p for p in state["providers"]}
- members = [by_slug[m] for m in member_slugs if m in by_slug]
- if not members:
- await query.answer(text=_toast("platform.telegram.picker.group_not_found"))
- return
- rows = self._rows_of_two([self._provider_button(p) for p in members])
- rows.append(self._picker_back_cancel_row())
- await self._picker_edit(
- query,
- f"⚙ *{t('platform.telegram.picker.title')}*\n\n"
- f"{t('platform.telegram.picker.provider_family', family=f'*{_label or group_id}*')}\n\n"
- f"{t('platform.telegram.picker.select_provider')}",
- InlineKeyboardMarkup(rows))
- elif data == "mb": # back to provider list (folds groups)
- await self._picker_show_providers(query, state, int(state.get("provider_page", 0) or 0), get_label)
- elif data == "mx":
- self._model_picker_state.pop(chat_id, None)
- await query.edit_message_text(text=t("platform.telegram.picker.cancelled"), reply_markup=None)
- await query.answer()
- else:
- await query.answer() # e.g. page-counter button "mx:noop"
- async def _notify_clarify_expired(self, query, user_display: str) -> None:
- """Tell the user a clarify tap arrived too late (entry evicted or gateway restarted) — otherwise
- the tap leaves a misleading ✓ the agent never sees."""
- with contextlib.suppress(Exception):
- await query.answer(text=_toast("platform.telegram.prompt.expired_toast"))
- await self._edit_html_quiet(
- query, f"❓ {_html.escape(query.message.text or '')}\n\n<i>{_html.escape(t('platform.telegram.prompt.expired_body'))}</i>")
- @staticmethod
- async def _edit_html_quiet(query, text: str) -> None:
- """HTML edit with the keyboard removed; failures ignored (non-fatal)."""
- with contextlib.suppress(Exception):
- await query.edit_message_text(text=text, parse_mode=ParseMode.HTML, reply_markup=None)
- async def _edit_md_quiet(self, query, text_md: str) -> None:
- """MarkdownV2 edit with the keyboard removed; failures ignored (non-fatal)."""
- with contextlib.suppress(Exception):
- await query.edit_message_text(text=self.format_message(text_md), parse_mode=ParseMode.MARKDOWN_V2, reply_markup=None)
- async def _handle_inline_query(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
- """Answer ``@botname <query>`` with a searchable command/skill picker (the ``/`` menu is capped at
- 60 slots). Results are computed per keystroke, 50 per page; tapping sends ``/cmd`` text as the
- user, so dispatch flows through the normal command path. Inline queries arrive from ANY chat, so
- unauthorized users get an empty list (the skill catalog is not leaked)."""
- inline_query = getattr(update, "inline_query", None)
- if inline_query is None:
- return
- from_user = getattr(inline_query, "from_user", None)
- user_id = str(getattr(from_user, "id", "") or "").strip()
- try:
- # No chat context on inline queries — authorize on user identity alone, DM-shaped.
- authorized = bool(user_id) and self._is_callback_user_authorized(
- user_id, chat_id=user_id, chat_type="private", user_name=getattr(from_user, "username", None))
- except Exception:
- logger.debug("[%s] inline picker auth check failed", self.name, exc_info=True)
- authorized = False
- if not authorized:
- try:
- from plugins.platforms.telegram.inline_picker import CACHE_TIME_SECONDS as _deny_cache
- self._accept_update()
- await inline_query.answer([], cache_time=_deny_cache, is_personal=True)
- except Exception:
- self._fail_update_preparation()
- logger.debug("[%s] inline picker empty answer failed", self.name, exc_info=True)
- return
- try:
- from telegram import InlineQueryResultArticle, InputTextMessageContent
- from plugins.platforms.telegram.inline_picker import CACHE_TIME_SECONDS as _CACHE, build_inline_results
- # Per-keystroke catalog build resolves every skill path; keep it off the loop (#110707).
- results, next_offset = await asyncio.to_thread(
- build_inline_results,
- getattr(inline_query, "query", "") or "", offset=getattr(inline_query, "offset", "") or "")
- articles = [
- InlineQueryResultArticle(
- id=r["id"], title=r["title"], description=r["description"],
- input_message_content=InputTextMessageContent(r["message_text"]))
- for r in results
- ]
- # is_personal: catalogs differ per user (auth, disabled skills) — never share cached pages.
- self._accept_update()
- await inline_query.answer(articles, cache_time=_CACHE, is_personal=True, next_offset=next_offset)
- except Exception:
- self._fail_update_preparation()
- logger.debug("[%s] inline picker answer failed", self.name, exc_info=True)
- @staticmethod
- def _callback_ctx(query) -> dict[str, Any]:
- """Chat/thread/user context of a button tap, for the callback auth gate."""
- query_message = getattr(query, "message", None)
- query_chat = getattr(query_message, "chat", None)
- return {
- "chat_id": getattr(query_message, "chat_id", None), "chat_type": getattr(query_chat, "type", None),
- "thread_id": getattr(query_message, "message_thread_id", None), "user_name": getattr(query.from_user, "first_name", None)}
- async def _callback_authorized(self, query, cb: dict[str, Any], denial_text: str) -> bool:
- """Gate a button tap on the callback allowlist; answers ``denial_text`` when refused."""
- if self._is_callback_user_authorized(
- str(getattr(query.from_user, "id", "")), chat_id=cb["chat_id"],
- chat_type=str(cb["chat_type"]) if cb["chat_type"] is not None else None,
- thread_id=str(cb["thread_id"]) if cb["thread_id"] is not None else None, user_name=cb["user_name"]):
- return True
- await query.answer(text=denial_text)
- return False
- async def _handle_callback_query(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
- """Dispatch inline keyboard button clicks on the callback_data prefix."""
- query = update.callback_query
- if not query or not query.data:
- return
- self._accept_update()
- data = query.data
- cb = self._callback_ctx(query)
- # Model picker / generic choice picker (/reasoning, /fast) need a chat id.
- for prefixes, handler in (
- (("mp:", "mpg:", "mpv:", "mm:", "mc:", "mb", "mx", "mg:"), self._handle_model_picker_callback),
- (("cp:",), self._handle_choice_picker_callback)):
- if data.startswith(prefixes):
- chat_id = str(query.message.chat_id) if query.message else None
- # One auth gate for every chat-id picker: strangers in a shared group must not drive the owner's picker.
- if chat_id and await self._callback_authorized(query, cb, _unauthorized()):
- await handler(query, data, chat_id)
- return
- for prefix, handler in (
- ("gt:", self._handle_gmail_triage_callback), ("ea:", self._handle_exec_approval_callback),
- ("sc:", self._handle_slash_confirm_callback), ("cl:", self._handle_clarify_callback),
- ("update_prompt:", self._handle_update_prompt_callback)):
- if data.startswith(prefix):
- await handler(query, data, cb)
- return
- async def _claim_callback_state(self, query, cb: dict[str, Any], state: dict, key, denial: str, resolved: str, *, pop: bool = True):
- """Auth-gate a button tap, then claim its pending entry; None (after answering) when refused or expired."""
- if not await self._callback_authorized(query, cb, denial):
- return None
- session_key = state.pop(key, None) if pop else state.get(key)
- if not session_key:
- await query.answer(text=resolved)
- return session_key
- async def _handle_exec_approval_callback(self, query, data: str, cb: dict[str, Any]) -> None:
- """``ea:<choice>:<approval_id>`` — resolve a pending exec approval."""
- parts = data.split(":", 2)
- if len(parts) != 3:
- return
- choice = parts[1] # once, session, always, deny
- try:
- approval_id = int(parts[2])
- except (ValueError, IndexError):
- await query.answer(text=_toast("platform.telegram.approval.toast_invalid_data"))
- return
- session_key = await self._claim_callback_state(
- query, cb, self._approval_state, approval_id, _unauthorized(),
- _toast("platform.telegram.approval.toast_already_resolved"))
- if not session_key:
- return
- request_id = self._approval_request_ids.pop(approval_id, None)
- user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
- try:
- # Rendering happens after so the message reflects what actually occurred: a tap that lands after
- # the approval wait timed out (count == 0) must NOT claim "Approved" — the command was already
- # denied and will not run (#63501 regression follow-up: 60s waits made stale taps common).
- from tools.approval import resolve_gateway_approval_card
- count = resolve_gateway_approval_card(session_key, choice, request_id)
- logger.info(
- "Telegram button resolved %d approval(s) for session %s (choice=%s, user=%s)", count, session_key, choice, user_display)
- except Exception as exc:
- logger.error("Failed to resolve gateway approval from Telegram button: %s", exc)
- count = 0
- if count:
- label_key = {"once": "resolved_once", "session": "resolved_session", "always": "resolved_always", "deny": "resolved_deny"}.get(
- choice, "resolved_generic")
- label = t(f"platform.telegram.approval.{label_key}")
- edit_text = t("platform.telegram.approval.resolved_by_user", label=label, user=user_display)
- else:
- label = t("platform.telegram.approval.expired")
- edit_text = t("platform.telegram.approval.expired_detail", label=label)
- await query.answer(text=label[:_TOAST_LIMIT])
- await edit_keeping_prompt(self, query, edit_text)
- # Typing was paused when the approval was sent; the text /approve and /deny paths resume it too.
- if count and cb["chat_id"] is not None:
- self.resume_typing_for_chat(str(cb["chat_id"]))
- async def _handle_slash_confirm_callback(self, query, data: str, cb: dict[str, Any]) -> None:
- """``sc:<choice>:<confirm_id>`` — resolve a slash-command confirmation."""
- parts = data.split(":", 2)
- if len(parts) != 3:
- return
- choice = parts[1] # once, always, cancel
- confirm_id = parts[2]
- session_key = await self._claim_callback_state(
- query, cb, self._slash_confirm_state, confirm_id, _unauthorized(),
- _toast("platform.telegram.slash_confirm.already_resolved"))
- if not session_key:
- return
- label_key = {"once": "slash_confirm.resolved_once", "always": "slash_confirm.resolved_always", "cancel": "slash_confirm.resolved_cancel"}.get(
- choice, "approval.resolved_generic")
- user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
- label = t(f"platform.telegram.{label_key}")
- await query.answer(text=label[:_TOAST_LIMIT])
- decided = t("platform.telegram.approval.resolved_by_user", label=label, user=user_display)
- await edit_keeping_prompt(self, query, decided)
- # The runner stored a handler keyed by session_key; run it and send any returned text as a follow-up.
- try:
- from tools import slash_confirm as _slash_confirm_mod
- result_text = await _slash_confirm_mod.resolve(session_key, confirm_id, choice)
- if result_text and query.message:
- # Inherit the prompt's topic: forums use message_thread_id; private DM-topic lanes need
- # both the topic id and the prompt reply anchor.
- thread_id = getattr(query.message, "message_thread_id", None)
- chat_type = getattr(getattr(query.message, "chat", None), "type", None)
- prompt_message_id = getattr(query.message, "message_id", None)
- send_kwargs: dict[str, Any] = {
- "chat_id": int(query.message.chat_id), "text": self.format_message(result_text),
- "parse_mode": ParseMode.MARKDOWN_V2, **self._link_preview_kwargs()}
- is_private_chat = str(getattr(chat_type, "value", chat_type)).lower() in {
- "private", str(ChatType.PRIVATE).lower(), str(getattr(ChatType.PRIVATE, "value", ChatType.PRIVATE)).lower()}
- if thread_id is not None:
- meta: dict[str, Any] = {"thread_id": str(thread_id)}
- reply_to_id = None
- if is_private_chat and prompt_message_id is not None:
- reply_to_id = send_kwargs["reply_to_message_id"] = int(prompt_message_id)
- meta["telegram_dm_topic_reply_fallback"] = True
- send_kwargs.update(self._thread_kwargs_for_send(
- str(
- query.message.chat_id
- ), str(thread_id), meta, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode))
- await self._send_message_with_thread_fallback(**send_kwargs)
- except Exception as exc:
- logger.error("[%s] slash-confirm callback failed: %s", self.name, exc, exc_info=True)
- async def _handle_clarify_callback(self, query, data: str, cb: dict[str, Any]) -> None:
- """``cl:<clarify_id>:<idx|other>`` — resolve a clarify prompt or flip to text capture."""
- parts = data.split(":", 2)
- if len(parts) != 3:
- return
- clarify_id = parts[1]
- choice_token = parts[2]
- session_key = await self._claim_callback_state(
- query, cb, self._clarify_state, clarify_id, _unauthorized(),
- _toast("platform.telegram.slash_confirm.already_resolved"), pop=False)
- if not session_key:
- return
- user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
- if choice_token == "other":
- # Flip to text-capture: the gateway's text-intercept resolves the clarify with the next message.
- # Do NOT pop _clarify_state yet — still needed if the entry gets cleared by something else.
- flipped = False
- try:
- from tools.clarify_gateway import mark_awaiting_text
- flipped = mark_awaiting_text(clarify_id)
- except Exception as exc:
- logger.warning("[%s] mark_awaiting_text failed: %s", self.name, exc)
- if not flipped:
- # Entry evicted / gateway restarted — a typed answer would go nowhere.
- self._clarify_state.pop(clarify_id, None)
- await self._notify_clarify_expired(query, user_display)
- return
- await query.answer(text=_toast("platform.telegram.prompt.type_answer"))
- await self._edit_html_quiet(
- query,
- f"❓ {query.message.text or ''}\n\n"
- f"<i>{_html.escape(t('platform.telegram.prompt.awaiting_typed', user=user_display))}</i>")
- return
- # Numeric choice → resolve immediately with the chosen text
- try:
- idx = int(choice_token)
- except (ValueError, TypeError):
- await query.answer(text=_toast("platform.telegram.prompt.invalid_choice"))
- return
- resolved_text: Optional[str] = None
- try:
- from tools.clarify_gateway import get_pending
- entry = get_pending(clarify_id)
- if entry and entry.choices and 0 <= idx < len(entry.choices):
- resolved_text = entry.choices[idx]
- except Exception:
- resolved_text = None
- if resolved_text is None:
- # Race (timeout / session reset): echo the index so the agent sees an intentional response.
- resolved_text = f"choice {idx + 1}"
- self._clarify_state.pop(clarify_id, None)
- try:
- from tools.clarify_gateway import resolve_gateway_clarify
- resolved = resolve_gateway_clarify(clarify_id, resolved_text)
- except Exception as exc:
- logger.error("[%s] resolve_gateway_clarify failed: %s", self.name, exc)
- resolved = False
- if resolved:
- await query.answer(text=f"✓ {resolved_text[:60]}")
- await self._edit_html_quiet(
- query, f"❓ {_html.escape(query.message.text or '')}\n\n<b>{_html.escape(user_display)}:</b> {_html.escape(resolved_text)}")
- logger.info("Telegram clarify button resolved (id=%s, choice=%r, user=%s)", clarify_id, resolved_text, user_display)
- else:
- # Entry evicted / gateway restarted between ask and tap.
- await self._notify_clarify_expired(query, user_display)
- logger.warning("Telegram clarify button: resolve_gateway_clarify returned False (id=%s)", clarify_id)
- async def _handle_update_prompt_callback(self, query, data: str, cb: dict[str, Any]) -> None:
- """``update_prompt:<y|n>`` — forward the answer to the update process."""
- answer = data.split(":", 1)[1] # "y" or "n"
- if not await self._callback_authorized(query, cb, _unauthorized()):
- return
- await query.answer(text=_toast("platform.telegram.prompt.update_sent", answer=answer))
- word = t("platform.telegram.prompt.affirm_word" if answer == "y" else "platform.telegram.prompt.negate_word")
- await self._edit_md_quiet(query, f"☤ {t('platform.telegram.prompt.update_answered', answer=f'*{word}*')}")
- try:
- from hermes_constants import get_hermes_home
- response_path = get_hermes_home() / ".update_response"
- tmp = response_path.with_suffix(".tmp")
- tmp.write_text(answer, encoding="utf-8")
- tmp.replace(response_path)
- logger.info("Telegram update prompt answered '%s' by user %s", answer, getattr(query.from_user, "id", "unknown"))
- except Exception as exc:
- logger.error("Failed to write update response from callback: %s", exc)
- # `gt:<verb>` -> (script in ~/.hermes/scripts/gmail-triage/, extra-args, success-label, is_state). The callback
- # `arg` is always the first positional arg. is_state=True keeps the keyboard tappable (sticky sender rule);
- # False strips it on success (per-email one-shot).
- # The success label is a ``platform.telegram.gmail_triage.*`` catalog key, resolved per tap.
- _GT_VERB_DISPATCH = {
- "send": ("send-draft.sh", [], "done_send", False),
- "archive": ("archive.sh", [], "done_archive", False),
- "draft": ("draft-blank.sh", [], "done_draft", False),
- "spam": ("spam.sh", [], "done_spam", False),
- "mute": ("mute-add.sh", ["email"], "done_mute", True),
- "mute-domain": ("mute-add.sh", ["domain"], "done_mute_domain", True),
- "trust": ("trusted-ops-add.sh", ["email"], "done_trust", True),
- "trust-domain": ("trusted-ops-add.sh", ["domain"], "done_trust_domain", True),
- "vip": ("vip-add.sh", ["email"], "done_vip", True),
- "vip-domain": ("vip-add.sh", ["domain"], "done_vip_domain", True)}
- async def _handle_gmail_triage_callback(self, query, data: str, cb: dict[str, Any]) -> None:
- """Dispatch a gmail-triage inline-button callback (gt:verb:arg)."""
- parts = data.split(":", 2)
- if len(parts) != 3:
- await query.answer(text=_toast("platform.telegram.gmail_triage.invalid_data"))
- return
- verb, arg = parts[1], parts[2]
- if not await self._callback_authorized(query, cb, _unauthorized()):
- return
- entry = self._GT_VERB_DISPATCH.get(verb)
- if not entry:
- await query.answer(text=_toast("platform.telegram.gmail_triage.unknown_verb", verb=verb))
- return
- script_name, extra_args, success_key, is_state_verb = entry
- from hermes_constants import get_hermes_home
- script_path = get_hermes_home() / "scripts" / "gmail-triage" / script_name
- if not script_path.exists():
- await query.answer(text=_toast("platform.telegram.gmail_triage.script_missing", script=script_name))
- logger.error("[%s] gmail-triage script missing: %s", self.name, script_path)
- return
- success = False
- try:
- # A user script under HERMES_HOME the agent can write: scrubbed like cron and quick-command scripts.
- from tools.environments.local import build_subprocess_env
- proc = await asyncio.create_subprocess_exec(
- str(script_path), arg, *extra_args, stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
- env=build_subprocess_env(strip_launch_profile=True))
- _stdout_bytes, stderr_bytes = await asyncio.wait_for(proc.communicate(), timeout=60)
- if proc.returncode == 0:
- label = t(f"platform.telegram.gmail_triage.{success_key}")
- success = True
- logger.info("[%s] gmail-triage callback ok: verb=%s arg=%s", self.name, verb, arg)
- else:
- stderr_text = stderr_bytes.decode("utf-8", errors="replace").strip()
- last_line = stderr_text.splitlines()[-1] if stderr_text else t("platform.telegram.gmail_triage.exit_code", code=str(proc.returncode))
- label = t("platform.telegram.gmail_triage.failed", verb=verb, detail=last_line[:80])
- logger.error(
- "[%s] gmail-triage callback failed: verb=%s arg=%s rc=%s stderr=%s", self.name, verb, arg, proc.returncode, stderr_text)
- except TimeoutError:
- label = t("platform.telegram.gmail_triage.timed_out", verb=verb)
- logger.error("[%s] gmail-triage callback timed out: verb=%s arg=%s", self.name, verb, arg)
- except Exception as exc:
- label = t("platform.telegram.gmail_triage.error", verb=verb, error=str(exc))
- logger.error("[%s] gmail-triage callback exception: verb=%s arg=%s err=%s", self.name, verb, arg, exc, exc_info=True)
- await query.answer(text=label[:_TOAST_LIMIT])
- if not success:
- return
- original_text = (query.message.text or "") if query.message else ""
- user_display = getattr(query.from_user, "first_name", None) or t("platform.telegram.user_fallback")
- appended = f"{original_text}\n{t('platform.telegram.gmail_triage.appended_by', label=label, user=user_display)}"
- # Sticky state verbs keep the keyboard so further actions can stack; one-shots strip it (can't fire twice).
- with contextlib.suppress(Exception):
- await query.edit_message_text(text=appended, **({} if is_state_verb else {"reply_markup": None}))
- def _missing_media_path_error(self, label: str, path: str) -> str:
- """File-not-found error for MEDIA delivery; /workspace-style paths often exist only in the sandbox."""
- error = f"{label} file not found: {path}"
- if path.startswith(("/workspace/", "/output/", "/outputs/")):
- error += (
- " (path may only exist inside the Docker sandbox. "
- "Bind-mount a host directory and emit the host-visible path in MEDIA: for gateway file delivery.)")
- return error
- @staticmethod
- def _sniff_raster_format(image_path: str) -> Optional[str]:
- """Identify convertible raster formats by magic bytes.
- Replacement for stdlib ``imghdr`` (removed in Python 3.13). Returns
- one of ``png``/``gif``/``webp``/``bmp``/``tiff`` or None.
- """
- try:
- with open(image_path, "rb") as fh:
- head = fh.read(16)
- except OSError:
- return None
- if head.startswith(b"\x89PNG\r\n\x1a\n"):
- return "png"
- if head.startswith((b"GIF87a", b"GIF89a")):
- # GIFs are excluded: converting flattens animations to one frame.
- return None
- if head.startswith(b"RIFF") and head[8:12] == b"WEBP":
- return "webp"
- if head.startswith(b"BM"):
- return "bmp"
- if head.startswith((b"II*\x00", b"MM\x00*")):
- return "tiff"
- return None
- def _compress_image_to_jpeg(self, image_path: str) -> Optional[str]:
- """Pre-compress a large image to progressive JPEG before upload.
- Behind an HTTP proxy (e.g. tgapi.indevs.in) the PTB
- media_write_timeout (~20s) is easily exceeded by raw PNGs > 1-2MB.
- A progressive JPEG at ~85% quality keeps the upload well under the
- timeout while remaining visually equivalent for photos / info-graphics.
- Returns the path to a temporary JPEG, or None when the original can be
- used as-is (already small / already JPEG / Pillow not available). The
- caller is responsible for cleaning up the returned temp file.
- """
- import tempfile
- try:
- file_size = os.path.getsize(image_path)
- except OSError:
- return None
- ext = os.path.splitext(image_path)[1].lower()
- # Already JPEG — no gain in converting back
- if ext in (".jpg", ".jpeg"):
- return None
- # Skip tiny files; conversion cost > upload benefit
- if file_size < self._IMG_COMPRESS_THRESHOLD_BYTES:
- return None
- # Only convert raster image formats (png, gif, webp, bmp, tiff).
- # Magic-byte sniff instead of the stdlib imghdr module, which was
- # removed in Python 3.13.
- if self._sniff_raster_format(image_path) is None:
- return None
- try:
- from PIL import Image
- except Exception:
- # Pillow missing: fall back to uploading the original (may timeout)
- logger.warning("[%s] Pillow not available for image compression", self.name)
- return None
- try:
- # Close the source handle before returning: convert()/resize() produce new images, so
- # the original file object would otherwise stay open until garbage collection.
- with Image.open(image_path) as src:
- if src.mode in ("RGBA", "LA", "P"):
- # Alpha-capable modes: composite onto a white background so
- # transparency doesn't render as black in the JPEG.
- background = Image.new("RGB", src.size, (255, 255, 255))
- layer = src.convert("RGBA") if src.mode == "P" else src
- if layer.mode in ("RGBA", "LA"):
- background.paste(layer, mask=layer.split()[-1])
- else:
- background.paste(layer)
- img = background
- else:
- img = src.convert("RGB")
- max_w, max_h = img.size
- max_dim = max(max_w, max_h)
- if max_dim > self._IMG_MAX_DIMENSION:
- scale = self._IMG_MAX_DIMENSION / max_dim
- img = img.resize((int(max_w * scale), int(max_h * scale)), Image.LANCZOS)
- fd, tmp = tempfile.mkstemp(
- suffix=".jpg",
- prefix="tg_compress_",
- )
- os.close(fd)
- img.save(
- tmp,
- "JPEG",
- quality=self._IMG_JPEG_QUALITY,
- progressive=True,
- optimize=True,
- )
- logger.info(
- "[%s] Pre-compressed %s (%.1fKB → %s %.1fKB) for Telegram upload",
- self.name,
- image_path,
- file_size / 1024,
- tmp,
- os.path.getsize(tmp) / 1024,
- )
- return tmp
- except Exception as e:
- logger.warning(
- "[%s] Image compression failed, uploading original: %s",
- self.name,
- e,
- )
- return None
- def _telegram_media_too_large_note(self, label: str, file_size: Any, max_bytes: int) -> str:
- limit_mb = max(1, max_bytes // (1024 * 1024))
- try:
- size_text = f"{int(file_size or 0) / (1024 * 1024):.1f} MB"
- except (TypeError, ValueError):
- size_text = "unknown size"
- return f"[Telegram {label} skipped: file size {size_text} exceeds the {limit_mb} MB limit. Ask the user to send a smaller file.]"
- @staticmethod
- def _int_or_zero(value: Any) -> int:
- try:
- return int(value or 0)
- except (TypeError, ValueError):
- return 0
- def _telegram_media_size_allowed(self, source: Any, label: str) -> tuple[bool, Optional[str]]:
- """Validate Telegram media size before downloading into memory."""
- max_bytes = int(getattr(self, "_max_doc_bytes", 20 * 1024 * 1024) or 20 * 1024 * 1024)
- size = self._int_or_zero(getattr(source, "file_size", None))
- if size <= 0 or size <= max_bytes:
- return True, None
- return False, self._telegram_media_too_large_note(label, size, max_bytes)
- def _media_send_kwargs(
- self, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]]) -> tuple[Optional[int], dict[str, Any]]:
- """Return ``(reply_to_id, base_kwargs)`` shared by every native media send."""
- reply_to_id = self._reply_to_message_id_for_send(reply_to, metadata, reply_to_mode=self._reply_to_mode)
- thread_kwargs = self._thread_kwargs_for_send(
- chat_id, self._metadata_thread_id(metadata), metadata, reply_to_message_id=reply_to_id, reply_to_mode=self._reply_to_mode)
- return reply_to_id, {
- "chat_id": normalize_telegram_chat_id(chat_id), "reply_to_message_id": reply_to_id,
- "read_timeout": _MEDIA_SEND_READ_TIMEOUT, **thread_kwargs, **self._notification_kwargs(metadata)}
- async def _send_media(
- self, send_fn: Any, chat_id: str, reply_to: Optional[str], metadata: Optional[dict[str, Any]], media_label: str,
- reset_media: Optional[Any] = None, **media_kwargs: Any) -> Any:
- """Send one native media payload with thread routing + DM-topic anchor retry."""
- reply_to_id, kwargs = self._media_send_kwargs(chat_id, reply_to, metadata)
- return await self._send_with_dm_topic_reply_anchor_retry(
- send_fn, {**kwargs, **media_kwargs}, metadata, reply_to_id, media_label, reset_media=reset_media)
- @staticmethod
- def _caption_1024(caption: Optional[str]) -> Optional[str]:
- return caption[:1024] if caption else None
- async def _send_voice_bubble(self, audio_file, chat_id, reply_to, metadata, caption, duration_secs):
- """sendVoice with caption variants: MarkdownV2 when it fits 1024 chars, plain fallback when the
- Bot API rejects the entities; anything else is a real error."""
- # Render caption markdown (#32029): auto-TTS captions carry the agent's markdown reply, which showed
- # literal *asterisks* and [links](...) without a parse_mode. Format to MarkdownV2 when it fits the
- # 1024-char caption cap; fall back to the raw text (previous behaviour) when formatting would
- # overflow or the Bot API rejects the entities.
- _caption_variants: list[tuple] = []
- if caption:
- try:
- _formatted_caption = self.format_message(caption)
- if utf16_len(_formatted_caption) <= 1024:
- _caption_variants.append((_formatted_caption, ParseMode.MARKDOWN_V2))
- except Exception:
- logger.debug("[%s] voice caption MarkdownV2 formatting failed; sending plain caption", self.name, exc_info=True)
- _caption_variants.append((caption[:1024], None))
- else:
- _caption_variants.append((None, None))
- _last_parse_error: Optional[Exception] = None
- for _cap_text, _cap_parse_mode in _caption_variants:
- try:
- return await self._send_media(
- self._bot.send_voice, chat_id, reply_to, metadata, "voice", reset_media=lambda: audio_file.seek(0),
- voice=audio_file, caption=_cap_text, parse_mode=_cap_parse_mode, duration=duration_secs)
- except Exception as _cap_error:
- err = str(_cap_error).lower()
- if _cap_parse_mode is not None and ("parse" in err or "entit" in err):
- logger.warning(
- "[%s] voice caption MarkdownV2 rejected, retrying plain: %s", self.name, _redact_telegram_error_text(_cap_error))
- _last_parse_error = _cap_error
- audio_file.seek(0)
- continue
- raise
- raise _last_parse_error or RuntimeError("Telegram send_voice failed for all caption variants")
- async def send_voice(
- self, chat_id: str, audio_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
- metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
- """Send audio as a native Telegram voice message or audio file."""
- if not self._bot:
- return SendResult(success=False, error="Not connected")
- _transcoded_voice_path: Optional[str] = None
- try:
- if not os.path.exists(audio_path):
- return SendResult(success=False, error=self._missing_media_path_error("Audio", audio_path))
- # sendVoice only accepts Ogg/Opus: an explicit voice-bubble request (is_voice) transcodes via
- # ffmpeg; otherwise route by extension (.mp3/.m4a → sendAudio, others → document).
- if kwargs.get("is_voice") and os.path.splitext(audio_path)[1].lower() not in (".ogg", ".opus"):
- from gateway.platforms.base import transcode_to_ogg_opus
- _transcoded_voice_path = await asyncio.to_thread(transcode_to_ogg_opus, audio_path)
- if _transcoded_voice_path:
- audio_path = _transcoded_voice_path
- else:
- logger.warning(
- "[%s] voice transcode unavailable for %s — sending original format (install ffmpeg for voice bubbles)",
- self.name, os.path.basename(audio_path))
- # Telegram drops duration for long clips (~5 min+, shows 0:00).
- _duration_secs = await asyncio.to_thread(_probe_voice_duration_seconds, audio_path)
- with open(audio_path, "rb") as audio_file: # noqa: ASYNC230 -- file handle is streamed to the upload; a local open() is non-blocking in practice
- ext = os.path.splitext(audio_path)[1].lower()
- if ext in {".ogg", ".opus"}: # round playable voice bubble
- msg = await self._send_voice_bubble(audio_file, chat_id, reply_to, metadata, caption, _duration_secs)
- elif ext in {".mp3", ".m4a"}: # Bot API sendAudio only accepts MP3 / M4A
- msg = await self._send_media(
- self._bot.send_audio, chat_id, reply_to, metadata, "audio", reset_media=lambda: audio_file.seek(0),
- audio=audio_file, caption=self._caption_1024(caption), duration=_duration_secs)
- else: # formats Telegram can't play natively (.wav, .flac, ...)
- return await self.send_document(
- chat_id=chat_id, file_path=audio_path, caption=caption, reply_to=reply_to, metadata=metadata)
- return SendResult(success=True, message_id=str(msg.message_id))
- except Exception as e:
- logger.error(
- "[%s] Failed to send Telegram voice/audio, falling back to base adapter: %s", self.name,
- _redact_telegram_error_text(e), exc_info=True)
- return await super().send_voice(chat_id, audio_path, caption, reply_to, metadata=metadata)
- finally:
- if _transcoded_voice_path:
- with contextlib.suppress(OSError):
- os.unlink(_transcoded_voice_path)
- async def send_multiple_images(
- self, chat_id: str, images: list[tuple], metadata: Optional[dict[str, Any]] = None, human_delay: float = 0.0) -> SendResult:
- """Send images as Telegram albums (``send_media_group``, 10 per chunk). Animated GIFs can't join a
- media group (need ``send_animation``) so they go via the base per-image path, as does a failed chunk."""
- if not self._bot:
- return SendResult(success=False, error="Not connected")
- if not images:
- return SendResult(success=False, error="no images to send")
- try:
- from telegram import InputMediaPhoto
- except Exception as exc: # pragma: no cover - missing SDK
- logger.warning("[%s] InputMediaPhoto unavailable, falling back to per-image send: %s", self.name, exc)
- return await super().send_multiple_images(chat_id, images, metadata, human_delay)
- is_anim = lambda url: not url.startswith("file://") and self._is_animation_url(url)
- animations = [img for img in images if is_anim(img[0])]
- photos = [img for img in images if not is_anim(img[0])]
- delivered = False
- if animations:
- anim_result = await super().send_multiple_images(chat_id, animations, metadata, human_delay=human_delay)
- delivered = anim_result.success
- if not photos:
- return SendResult(success=delivered, error=None if delivered else "all images failed to send")
- from urllib.parse import unquote as _unquote
- CHUNK = 10 # Telegram's album limit
- chunks = [photos[i:i + CHUNK] for i in range(0, len(photos), CHUNK)]
- for chunk_idx, chunk in enumerate(chunks):
- if human_delay > 0 and chunk_idx > 0:
- await asyncio.sleep(human_delay)
- media: list[Any] = []
- opened_files: list[Any] = []
- temp_paths: list[str] = []
- try:
- for image_url, alt_text in chunk:
- source: Any = image_url
- if image_url.startswith("file://"):
- local_path = _unquote(image_url[7:])
- if not os.path.exists(local_path):
- logger.warning("[%s] Skipping missing image in media group: %s", self.name, local_path)
- continue
- # Pre-compress large raster images so the media-group upload stays under
- # media_write_timeout; the temp JPEG is removed after the send.
- compressed = self._compress_image_to_jpeg(local_path)
- if compressed:
- temp_paths.append(compressed)
- local_path = compressed
- source = open(local_path, "rb") # noqa: ASYNC230 -- file handle is streamed to the upload; a local open() is non-blocking in practice
- opened_files.append(source)
- media.append(InputMediaPhoto(media=source, caption=self._caption_1024(alt_text)))
- if not media:
- continue
- logger.info("[%s] Sending media group of %d photo(s) (chunk %d/%d)", self.name, len(media), chunk_idx + 1, len(chunks))
- reply_to_id, send_kwargs = self._media_send_kwargs(chat_id, None, metadata)
- def _reset_opened_files() -> None:
- for fh in opened_files:
- with contextlib.suppress(Exception):
- fh.seek(0)
- await self._send_with_dm_topic_reply_anchor_retry(
- self._bot.send_media_group, {**send_kwargs, "media": media}, metadata, reply_to_id,
- "media group", reset_media=_reset_opened_files)
- delivered = True
- except Exception as e:
- logger.warning(
- "[%s] send_media_group failed (chunk %d/%d), falling back to per-image: %s", self.name,
- chunk_idx + 1, len(chunks), _redact_telegram_error_text(e), exc_info=True)
- fallback = await super().send_multiple_images(chat_id, chunk, metadata, human_delay=human_delay)
- delivered = delivered or fallback.success
- finally:
- for fh in opened_files:
- with contextlib.suppress(Exception):
- fh.close()
- for tmp in temp_paths:
- with contextlib.suppress(OSError):
- os.remove(tmp)
- return SendResult(success=delivered, error=None if delivered else "all images failed to send")
- async def send_image_file(
- self, chat_id: str, image_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
- metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
- """Send a local image file natively as a Telegram photo."""
- # Pre-compress large raster images to progressive JPEG once; the photo send and the document
- # fallback both reuse the compressed file so either upload stays under media_write_timeout.
- compressed = self._compress_image_to_jpeg(image_path)
- actual_path = compressed or image_path
- doc_name = os.path.splitext(os.path.basename(image_path))[0] + ".jpg" if compressed else os.path.basename(image_path)
- async def _photo_failed(e: Exception) -> SendResult:
- error_str = str(e)
- # Dimension errors are expected for valid images Telegram refuses as photos → INFO.
- if "Photo_invalid_dimensions" in error_str or "PHOTO_INVALID_DIMENSIONS" in error_str:
- logger.info("[%s] Image dimensions exceed Telegram photo limits, sending as document: %s", self.name, image_path)
- else:
- logger.warning(
- "[%s] Failed to send Telegram local image as photo, trying document fallback: %s", self.name,
- _redact_telegram_error_text(e), exc_info=True)
- # Document has no dimension limit (50MB only); if even that fails, base adapter text.
- try:
- return await self.send_document(
- chat_id=chat_id, file_path=actual_path, caption=caption, file_name=doc_name,
- reply_to=reply_to, metadata=metadata)
- except Exception as doc_err:
- logger.error(
- "[%s] Failed to send Telegram local image as document, falling back to base adapter: %s",
- self.name, doc_err, exc_info=True)
- return await super(TelegramAdapter, self).send_image_file(chat_id, image_path, caption, reply_to, metadata=metadata)
- try:
- return await self._send_local_file(
- "Image", actual_path, chat_id, reply_to, metadata, "photo",
- lambda f: {"photo": f, "caption": self._caption_1024(caption)}, _photo_failed)
- finally:
- if compressed:
- with contextlib.suppress(OSError):
- os.remove(compressed)
- async def _send_local_file(
- self, label: str, path: str, chat_id, reply_to, metadata, media_key: str, build_kwargs, on_error,
- ) -> SendResult:
- """Shared shell for native local-file sends: existence check, open, send with routing, then
- ``await on_error(exc)`` on any failure. ``build_kwargs(f)`` supplies the media kwargs."""
- if not self._bot:
- return SendResult(success=False, error="Not connected")
- try:
- if not os.path.exists(path):
- return SendResult(success=False, error=self._missing_media_path_error(label, path))
- with open(path, "rb") as f: # noqa: ASYNC230 -- file handle is streamed to the upload; a local open() is non-blocking in practice
- msg = await self._send_media(
- getattr(self._bot, f"send_{media_key}"), chat_id, reply_to, metadata, media_key,
- reset_media=lambda: f.seek(0), **build_kwargs(f))
- return SendResult(success=True, message_id=str(msg.message_id))
- except Exception as e:
- return await on_error(e)
- async def _warn_then(self, media_key: str, e: Exception, fallback) -> SendResult:
- logger.warning("[%s] Failed to send %s: %s", self.name, media_key, _redact_telegram_error_text(e))
- return await fallback
- async def send_document(
- self, chat_id: str, file_path: str, caption: Optional[str] = None, file_name: Optional[str] = None,
- reply_to: Optional[str] = None, metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
- """Send a document/file natively as a Telegram file attachment."""
- return await self._send_local_file(
- "File", file_path, chat_id, reply_to, metadata, "document",
- lambda f: {"document": f, "filename": file_name or os.path.basename(file_path), "caption": self._caption_1024(caption)},
- lambda e: self._warn_then(
- "document", e, super(
- TelegramAdapter, self,
- ).send_document(chat_id, file_path, caption, file_name, reply_to, metadata=metadata)))
- async def send_video(
- self, chat_id: str, video_path: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
- metadata: Optional[dict[str, Any]] = None, **kwargs) -> SendResult:
- """Send a video natively as a Telegram video message.
- Real geometry and a JPEG thumbnail ride along explicitly. Telegram only runs its own video
- processing for uploads under roughly 10 MB; larger files come back as an unprocessed
- ``320x320`` video with ``duration=0`` and no thumbnail, which clients then draw as a square
- tile whatever the true aspect ratio (portrait reels and 16:9 clips alike).
- """
- geometry = await asyncio.to_thread(_probe_video_geometry, video_path)
- thumb_path = (
- await asyncio.to_thread(_video_thumbnail_jpeg, video_path, geometry.get("duration"))
- if geometry else None
- )
- try:
- def build_kwargs(f):
- payload = {"video": f, "caption": self._caption_1024(caption), **geometry}
- if thumb_path:
- # A path (not an open handle) so python-telegram-bot loads the bytes once and a
- # retry after a stale topic anchor still has a thumbnail to send.
- payload["thumbnail"] = thumb_path
- return payload
- return await self._send_local_file(
- "Video", video_path, chat_id, reply_to, metadata, "video", build_kwargs,
- lambda e: self._warn_then(
- "video", e, super(TelegramAdapter, self).send_video(chat_id, video_path, caption, reply_to, metadata=metadata),
- ))
- finally:
- if thumb_path:
- with contextlib.suppress(OSError):
- os.remove(thumb_path)
- async def send_image(
- self, chat_id: str, image_url: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
- metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Send a URL image as a Telegram photo: URL send (<5MB) → download+upload (≤10MB) → base text."""
- if not self._bot:
- return SendResult(success=False, error="Not connected")
- from tools.url_safety import is_safe_url
- if not is_safe_url(image_url):
- logger.warning("[%s] Blocked unsafe image URL (SSRF protection)", self.name)
- return await super().send_image(chat_id, image_url, caption, reply_to, metadata=metadata)
- photo_caption = self._caption_1024(caption)
- try:
- msg = await self._send_media(
- self._bot.send_photo, chat_id, reply_to, metadata, "URL photo", photo=image_url, caption=photo_caption)
- return SendResult(success=True, message_id=str(msg.message_id))
- except Exception as e:
- logger.warning(
- "[%s] URL-based send_photo failed, trying file upload: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
- try:
- from gateway.platforms.base import ssrf_redirect_guard
- from tools.url_safety import create_ssrf_safe_async_client
- async with create_ssrf_safe_async_client(timeout=30.0, event_hooks={"response": [ssrf_redirect_guard]}) as client:
- resp = await client.get(image_url)
- resp.raise_for_status()
- image_data = resp.content
- msg = await self._send_media(
- self._bot.send_photo, chat_id, reply_to, metadata, "uploaded photo", photo=image_data, caption=photo_caption)
- return SendResult(success=True, message_id=str(msg.message_id))
- except Exception as e2:
- logger.error("[%s] File upload send_photo also failed: %s", self.name, e2, exc_info=True)
- return await super().send_image(chat_id, image_url, caption, reply_to, metadata=metadata)
- async def send_animation(
- self, chat_id: str, animation_url: str, caption: Optional[str] = None, reply_to: Optional[str] = None,
- metadata: Optional[dict[str, Any]] = None) -> SendResult:
- """Send an animated GIF natively as a Telegram animation (auto-plays inline)."""
- if not self._bot:
- return SendResult(success=False, error="Not connected")
- try:
- msg = await self._send_media(
- self._bot.send_animation, chat_id, reply_to, metadata, "animation", animation=animation_url,
- caption=self._caption_1024(caption))
- return SendResult(success=True, message_id=str(msg.message_id))
- except Exception as e:
- logger.error(
- "[%s] Failed to send Telegram animation, falling back to photo: %s", self.name,
- _redact_telegram_error_text(e), exc_info=True)
- return await self.send_image(chat_id, animation_url, caption, reply_to, metadata=metadata)
- @staticmethod
- def _is_transient_typing_error(exc: Exception) -> bool:
- """Return True for Telegram typing errors worth cooling down."""
- if getattr(exc, "retry_after", None) is not None:
- return True
- status_code = getattr(exc, "status_code", None) or getattr(exc, "code", None)
- if isinstance(status_code, int) and (status_code == 429 or status_code >= 500):
- return True
- text = str(exc).lower()
- if any(marker in text for marker in ("too many requests", "rate limit", "timed out", "timeout", "temporar")):
- return True
- return isinstance(exc, (OSError, TimeoutError, ConnectionError, asyncio.TimeoutError))
- def _record_typing_cooldown(self, chat_id: str, exc: Exception) -> None:
- """Suppress Telegram typing refreshes for this chat after transient failures."""
- if not hasattr(self, "_telegram_typing_cooldown_until"):
- self._telegram_typing_cooldown_until = {}
- retry_after = getattr(exc, "retry_after", None)
- try:
- delay = float(retry_after) if retry_after is not None else self._telegram_typing_cooldown_seconds
- except (TypeError, ValueError):
- delay = self._telegram_typing_cooldown_seconds
- self._telegram_typing_cooldown_until[str(chat_id)] = asyncio.get_running_loop().time() + max(1.0, min(delay, 300.0))
- def _typing_in_cooldown(self, chat_id: str) -> bool:
- if not hasattr(self, "_telegram_typing_cooldown_until"):
- self._telegram_typing_cooldown_until = {}
- self._telegram_typing_cooldown_seconds = 30.0
- until = self._telegram_typing_cooldown_until.get(str(chat_id))
- if until is None:
- return False
- if asyncio.get_running_loop().time() < until:
- return True
- self._telegram_typing_cooldown_until.pop(str(chat_id), None)
- return False
- # --- per-chat send ordering + flood cooldown (#114396) ---------------------------------------------
- # Both keyed by the Bot-API-normalized chat id: the text path passes the raw chat_id while the media
- # funnel's send_kwargs carry the normalized value. ``__dict__.setdefault``: tests build adapters via
- # ``object.__new__()`` (no __init__).
- @contextlib.asynccontextmanager
- async def _chat_send_lock(self, chat_id: Any):
- """FIFO per-chat gate around outgoing API calls, reentrant within one asyncio task (media paths
- nest: send_voice → send_document, and ``super().send_*`` fallbacks reach ``send()``; a plain
- ``asyncio.Lock`` re-acquired by its holder would wedge that chat's sends for good)."""
- key = str(normalize_telegram_chat_id(chat_id))
- locks: dict[str, asyncio.Lock] = self.__dict__.setdefault("_telegram_chat_send_locks", {})
- owners: dict[str, asyncio.Task] = self.__dict__.setdefault("_telegram_chat_send_lock_owners", {})
- task = asyncio.current_task()
- if owners.get(key) is task:
- yield
- return
- lock = locks.setdefault(key, asyncio.Lock())
- async with lock:
- owners[key] = task
- try:
- yield
- finally:
- owners.pop(key, None)
- if not getattr(lock, "_waiters", None): # nobody queued: drop the entry (bounded dict)
- locks.pop(key, None)
- def _record_send_flood_cooldown(self, chat_id: Any, wait: float) -> SendResult:
- """A send refused with ``retry_after=wait`` arms a per-chat window during which ``send()`` fails
- closed locally (same ``flood_control:<s>`` result, so ledger recognition and redelivery timing are
- unchanged) instead of firing more requests into a penalty Telegram lengthens while it is hammered."""
- until: dict[str, float] = self.__dict__.setdefault("_telegram_send_cooldown_until", {})
- until[str(normalize_telegram_chat_id(chat_id))] = asyncio.get_running_loop().time() + max(1.0, min(float(wait), 300.0))
- return _flood_cap_result(wait)
- def _send_flood_cooldown_remaining(self, chat_id: Any) -> Optional[float]:
- """Seconds left in this chat's flood window, or ``None`` when sends may go out."""
- until: dict[str, float] = self.__dict__.setdefault("_telegram_send_cooldown_until", {})
- key = str(normalize_telegram_chat_id(chat_id))
- deadline = until.get(key)
- if deadline is None:
- return None
- remaining = deadline - asyncio.get_running_loop().time()
- if remaining > 0:
- return remaining
- until.pop(key, None)
- return None
- # --- shared per-chat send+edit pacing budget (#116312) -----------------------------------------
- # One slot per chat that sendMessage AND editMessageText both draw from (Telegram counts them
- # against the same per-chat allowance). ``_telegram_chat_outbound_slot_until`` maps the
- # normalized chat id to the loop-time when the next outbound call may fire.
- def _chat_outbound_slot_remaining(self, chat_id: Any) -> float:
- """Seconds until this chat's shared send+edit slot is open again (0 = may fire now)."""
- slot_until: dict[str, float] = self.__dict__.setdefault("_telegram_chat_outbound_slot_until", {})
- key = str(normalize_telegram_chat_id(chat_id))
- deadline = slot_until.get(key)
- if deadline is None:
- return 0.0
- remaining = deadline - asyncio.get_running_loop().time()
- if remaining <= 0:
- slot_until.pop(key, None) # expired — bounded dict, like the send locks
- return 0.0
- return remaining
- def _hold_chat_outbound_slot(self, chat_id: Any) -> None:
- """Arm/re-arm this chat's slot after an actual send/edit API call fires."""
- slot_until: dict[str, float] = self.__dict__.setdefault("_telegram_chat_outbound_slot_until", {})
- budget = getattr(self, "_telegram_chat_outbound_slot_secs", _TELEGRAM_CHAT_OUTBOUND_BUDGET_SECS)
- slot_until[str(normalize_telegram_chat_id(chat_id))] = (
- asyncio.get_running_loop().time() + max(0.0, budget))
- async def send_typing(self, chat_id: str, metadata: Optional[dict[str, Any]] = None) -> None:
- """Send typing indicator."""
- if not self._bot or self._typing_in_cooldown(chat_id):
- return
- _is_dm_topic: bool = False
- message_thread_id: Optional[int] = None
- async def _action(**kw) -> None:
- await self._bot.send_chat_action(chat_id=normalize_telegram_chat_id(chat_id), action="typing", **kw)
- self._telegram_typing_cooldown_until.pop(str(chat_id), None)
- try:
- _is_dm_topic = self._dm_topic_fallback(metadata)
- message_thread_id = self._message_thread_id_for_typing(self._metadata_thread_id(metadata))
- await _action(message_thread_id=message_thread_id)
- except Exception as e:
- # DM topic lanes: Telegram may reject message_thread_id — retry without it so the indicator at
- # least appears in the main DM view.
- if _is_dm_topic and message_thread_id is not None:
- try:
- await _action()
- return
- except Exception as fallback_exc:
- if self._is_transient_typing_error(fallback_exc):
- self._record_typing_cooldown(chat_id, fallback_exc)
- elif self._is_transient_typing_error(e):
- self._record_typing_cooldown(chat_id, e)
- logger.debug("[%s] Failed to send Telegram typing indicator: %s", self.name, _redact_telegram_error_text(e), exc_info=True)
- async def get_chat_info(self, chat_id: str) -> dict[str, Any]:
- """Get information about a Telegram chat."""
- if not self._bot:
- return {"name": "Unknown", "type": "dm"}
- try:
- chat = await self._bot.get_chat(normalize_telegram_chat_id(chat_id))
- chat_type = "dm"
- if chat.type == ChatType.GROUP:
- chat_type = "group"
- elif chat.type == ChatType.SUPERGROUP:
- chat_type = "forum" if chat.is_forum else "group"
- elif chat.type == ChatType.CHANNEL:
- chat_type = "channel"
- return {
- "name": chat.title or chat.full_name or str(chat_id), "type": chat_type, "username": chat.username,
- "is_forum": getattr(chat, "is_forum", False)}
- except Exception as e:
- logger.error("[%s] Failed to get Telegram chat info for %s: %s", self.name, chat_id, _redact_telegram_error_text(
- e), exc_info=True)
- return {"name": str(chat_id), "type": "dm", "error": str(e)}
- def format_message(self, content: str) -> str:
- """Convert standard markdown to Telegram MarkdownV2: code is stashed behind placeholders first (never
- modified), markdown constructs become MarkdownV2 syntax, everything else is escaped."""
- if not content:
- return content
- placeholders: dict = {}
- counter = [0]
- def _ph(value: str) -> str:
- """Stash *value* behind a placeholder token that survives escaping."""
- key = f"\x00PH{counter[0]}\x00"
- counter[0] += 1
- placeholders[key] = value
- return key
- def _ph_wrap(open_: str, close: str):
- return lambda m: _ph(f"{open_}{_escape_mdv2(m.group(1))}{close}")
- # 0) Rewrite GFM-style pipe tables into Telegram-friendly row groups
- # before the normal MarkdownV2 conversions run.
- text = _wrap_markdown_tables(content)
- # 1) Protect fenced code blocks (``` ... ```)
- # Per MarkdownV2 spec, \ and ` inside pre/code must be escaped.
- # A fence still opens on its own line — the opening ``` must be the
- # first triple-backtick run on that line and must end it — but the
- # line may carry arbitrary leading whitespace (list/blockquote-
- # nested code indents fences by 4+ spaces) or lead-in prose
- # ("Here is the code: ```"), both of which the line-start-only
- # anchor silently downgraded from <pre> to escaped literal text.
- # Requiring the rest of the opening line to be backtick-free is
- # what keeps *inline* triple backticks (e.g. "the syntax is
- # ```x``` inline") out of the match: a closing run can never sit
- # on the same line as the opener, and the tempered prefix cannot
- # skip past an earlier run to a later one. The closing fence must
- # sit on its own line (any indent), with optional trailing
- # whitespace and an optional ``\r`` so CRLF-terminated fences
- # (Windows-authored content) match too.
- def _protect_fenced(m):
- prefix = m.group(1) # lead-in text / indent before the opening fence
- opening = m.group(2) # opening ``` (with optional language) and newline
- body = m.group(3) # code body (may be empty)
- closing = m.group(4) # closing fence (with its indent)
- body = body.replace('\\', '\\\\').replace('`', '\\`')
- return prefix + _ph(opening + body + closing)
- text = re.sub(
- r'(?m)^((?:(?!```)[^\n])*)(```[^`\n]*\n)([\s\S]*?)(^[ \t]*```)[ \t]*\r?$',
- _protect_fenced,
- text,
- )
- # 2) Protect inline code (`...`)
- # Escape \ inside inline code per MarkdownV2 spec.
- text = re.sub(
- r'(`[^`]+`)',
- lambda m: _ph(m.group(0).replace('\\', '\\\\')),
- text,
- )
- # 3) Convert markdown links – escape the display text; inside the URL
- # only ')' and '\' need escaping per the MarkdownV2 spec.
- def _convert_link(m):
- url = m.group(2).replace('\\', '\\\\').replace(')', '\\)')
- return _ph(f'[{_escape_mdv2(m.group(1))}]({url})')
- text = re.sub(r'\[([^\]]+)\]\(([^()]*(?:\([^()]*\)[^()]*)*)\)', _convert_link, text)
- # 4) Headers (## Title) → bold *Title*, stripping redundant ** inside the header
- def _convert_header(m):
- inner = re.sub(r'\*\*(.+?)\*\*', r'\1', m.group(1).strip())
- return _ph(f'*{_escape_mdv2(inner)}*')
- text = re.sub(r'^#{1,6}\s+(.+)$', _convert_header, text, flags=re.MULTILINE)
- # 5) Bold **text** → *text*; 6) Italic *text* → _text_ ([^*\n]+ keeps matches on one line, or *
- # bullet lists corrupt); 7) Strikethrough ~~text~~ → ~text~; 8) Spoiler ||text|| kept as-is.
- text = re.sub(r'\*\*(.+?)\*\*', _ph_wrap('*', '*'), text)
- text = re.sub(r'\*([^*\n]+)\*', _ph_wrap('_', '_'), text)
- text = re.sub(r'~~(.+?)~~', _ph_wrap('~', '~'), text)
- text = re.sub(r'\|\|(.+?)\|\|', _ph_wrap('||', '||'), text)
- # 9) Blockquotes: protect leading > from escaping; expandable quotes (**> starts, trailing || ends).
- def _convert_blockquote(m):
- prefix, content = m.group(1), m.group(2) # prefix: >, >>, >>>, **>, **>> …
- if prefix.startswith('**') and content.endswith('||'):
- return _ph(f'{prefix} {_escape_mdv2(content[:-2])}||')
- return _ph(f'{prefix} {_escape_mdv2(content)}')
- text = re.sub(r'^((?:\*\*)?>{1,3}) (.+)$', _convert_blockquote, text, flags=re.MULTILINE)
- # 10) Escape remaining special characters in plain text
- text = _escape_mdv2(text)
- # 11) Restore placeholders in reverse insertion order so nested placeholders resolve.
- for key in reversed(list(placeholders.keys())):
- text = text.replace(key, placeholders[key])
- # 12) Safety net: escape bare ( ) { } that slipped through, but never inside ``` or ` spans.
- _safe_parts = []
- for _idx, _seg in enumerate(re.split(r'(```[\s\S]*?```|`[^`]+`)', text)):
- if _idx % 2 == 1:
- _safe_parts.append(_seg) # inside code — untouched
- else:
- _safe_parts.append(re.sub(r'[(){}]', lambda m, _seg=_seg: TelegramAdapter._escape_bare_bracket(m, _seg), _seg))
- return ''.join(_safe_parts)
- @staticmethod
- def _escape_bare_bracket(m, seg: str) -> str:
- """Escape a bare ( ) { } unless it is already escaped or delimits a ``[text](url)`` link."""
- s = m.start()
- ch = m.group(0)
- if s > 0 and seg[s - 1] == '\\': # already escaped
- return ch
- if ch == '(' and s > 0 and seg[s - 1] == ']': # opens a link [text](url)
- return ch
- if ch == ')': # closes a link URL? walk back matching depth
- before = seg[:s]
- if '](http' in before or '](' in before:
- depth = 0
- for j in range(s - 1, max(s - 2000, -1), -1):
- if seg[j] == '(':
- depth -= 1
- if depth < 0:
- if j > 0 and seg[j - 1] == ']':
- return ch
- break
- elif seg[j] == ')':
- depth += 1
- return '\\' + ch
- # ── Group mention gating ──────────────────────────────────────────────
- def _extra_bool(self, key: str, env_name: str, default: str, *fallback_keys: str) -> bool:
- """Boolean gate: scoped ``env_name`` → ``config.extra[key]`` (then ``fallback_keys``) → ``default``."""
- configured = _extra_or_secret(self.config.extra, key, env_name, None)
- for alt in fallback_keys:
- if configured is None:
- configured = self.config.extra.get(alt)
- if configured is None:
- configured = default
- if isinstance(configured, bool):
- return configured
- return str(configured).strip().lower() in {"true", "1", "yes", "on"}
- def _extra_str_set(self, key: str, env_name: str) -> set[str]:
- """Comma/list allowlist: scoped ``env_name`` → ``config.extra[key]`` → empty."""
- raw = _extra_or_secret(self.config.extra, key, env_name, "", blank_is_unset=False)
- raw = _decode_json_list_literal(raw)
- if isinstance(raw, list):
- return {str(part).strip() for part in raw if str(part).strip()}
- return {part.strip() for part in str(raw).split(",") if part.strip()}
- def _telegram_require_mention(self) -> bool:
- """Return whether group chats should require an explicit bot trigger."""
- return self._extra_bool("require_mention", "TELEGRAM_REQUIRE_MENTION", "false")
- def _telegram_observe_unmentioned_group_messages(self) -> bool:
- """Store skipped unmentioned group messages as context (observe chatter, dispatch only when addressed)."""
- return self._extra_bool(
- "observe_unmentioned_group_messages", "TELEGRAM_OBSERVE_UNMENTIONED_GROUP_MESSAGES", "false",
- "ingest_unmentioned_group_messages")
- def _telegram_guest_mode(self) -> bool:
- """Return whether non-allowlisted groups may trigger via direct @mention."""
- return self._extra_bool("guest_mode", "TELEGRAM_GUEST_MODE", "false")
- def _telegram_exclusive_bot_mentions(self) -> bool:
- """Return whether explicit @...bot mentions exclusively route group messages."""
- return self._extra_bool("exclusive_bot_mentions", "TELEGRAM_EXCLUSIVE_BOT_MENTIONS", "true")
- def _telegram_bots_require_mention(self) -> bool:
- """Whether another bot's message must explicitly @mention us (a quote-reply alone won't);
- breaks two-bot reply loops in groups while human replies stay unaffected."""
- return self._extra_bool(
- "bots_require_mention", "TELEGRAM_BOTS_REQUIRE_MENTION", "false"
- )
- def _telegram_free_response_chats(self) -> set[str]:
- return self._extra_str_set("free_response_chats", "TELEGRAM_FREE_RESPONSE_CHATS")
- def _telegram_free_response_topics(self) -> set[str]:
- """Topic-level free-response entries as ``<chat_id>:<thread_id>`` (General topic = ``1``)."""
- return self._extra_str_set("free_response_topics", "TELEGRAM_FREE_RESPONSE_TOPICS")
- def _telegram_is_free_response_topic(self, message: Message) -> bool:
- """True when the message's chat/topic pair is in ``free_response_topics``."""
- topics = self._telegram_free_response_topics()
- if not topics:
- return False
- chat_id = self._chat_id_str(message)
- if not chat_id:
- return False
- return f"{chat_id}:{self._topic_id_or_general(self._effective_message_thread_id(message))}" in topics
- def _telegram_allowed_chats(self) -> set[str]:
- """Group chat IDs the bot responds in (non-empty: others need ``guest_mode`` + @mention; DMs never
- filtered; empty = no restriction)."""
- return self._extra_str_set("allowed_chats", "TELEGRAM_ALLOWED_CHATS")
- def _telegram_group_allowed_chats(self) -> set[str]:
- """Return Telegram chats authorized at group scope."""
- return self._extra_str_set("group_allowed_chats", "TELEGRAM_GROUP_ALLOWED_CHATS")
- def _telegram_observe_allowed_chats(self) -> set[str]:
- """Chats where observed group context may use a shared source: ``group_allowed_chats`` ∩
- ``allowed_chats`` (when set)."""
- group_allowed = self._telegram_group_allowed_chats()
- if not group_allowed:
- return set()
- response_allowed = self._telegram_allowed_chats()
- return group_allowed & response_allowed if response_allowed else group_allowed
- def _telegram_allowed_topics(self) -> set[str]:
- """Forum topic IDs this bot handles (non-empty: other topics ignored; DMs never filtered; missing
- ``message_thread_id`` == General topic ``1``)."""
- return self._extra_str_set("allowed_topics", "TELEGRAM_ALLOWED_TOPICS")
- def _telegram_ignored_threads(self) -> set[int]:
- """Thread ids to skip: scoped ``TELEGRAM_IGNORED_THREADS`` → ``config.extra`` → none."""
- raw = _extra_or_secret(self.config.extra, "ignored_threads", "TELEGRAM_IGNORED_THREADS", "", blank_is_unset=False)
- raw = _decode_json_list_literal(raw)
- ignored: set[int] = set()
- for value in (raw if isinstance(raw, list) else str(raw).split(",")):
- text = str(value).strip()
- if not text:
- continue
- try:
- ignored.add(int(text))
- except (TypeError, ValueError):
- logger.warning("[%s] Ignoring invalid Telegram thread id: %r", self.name, value)
- return ignored
- def _compile_mention_patterns(self) -> list[re.Pattern]:
- """Compile optional regex wake-word patterns for group triggers."""
- # Scoped env → the profile's YAML → none. Only the env rung is a serialized string (JSON list,
- # newline- or comma-separated); a YAML string is one literal pattern and is left intact.
- env_raw = _scoped_gate_env("TELEGRAM_MENTION_PATTERNS", "").strip()
- if env_raw:
- try:
- patterns = json.loads(env_raw)
- except Exception:
- patterns = [part.strip() for part in env_raw.splitlines() if part.strip()]
- if not patterns:
- patterns = [part.strip() for part in env_raw.split(",") if part.strip()]
- else:
- patterns = self.config.extra.get("mention_patterns")
- if patterns is None:
- return [] # before touching ``self.name``: tests build bare adapters via object.__new__
- return compile_mention_patterns(patterns, log_prefix=self.name, platform_label="telegram", display_label="Telegram", logger_=logger)
- @staticmethod
- def _chat_type_str(chat) -> str:
- """PTB enum or plain-string ``chat.type`` → bare lowercase name (``supergroup``)."""
- return str(getattr(chat, "type", "")).split(".")[-1].lower() if chat else ""
- @staticmethod
- def _chat_id_str(message) -> str:
- return str(getattr(getattr(message, "chat", None), "id", ""))
- @classmethod
- def _topic_id_or_general(cls, thread_id) -> str:
- return str(thread_id) if thread_id is not None else cls._GENERAL_TOPIC_THREAD_ID
- def _is_group_chat(self, message: Message) -> bool:
- chat = getattr(message, "chat", None)
- return bool(chat) and self._chat_type_str(chat) in {"group", "supergroup"}
- @classmethod
- def _effective_message_thread_id(cls, message: Message) -> Optional[str]:
- """Routable thread id: forum General-topic messages arrive with ``message_thread_id=None`` but
- Telegram addresses that topic as ``1``; plain group/DM replies carry a reply-UI anchor that is NOT
- a routing id. Gating, skill binding and outbound routing must all agree on this value."""
- chat = getattr(message, "chat", None)
- chat_type = cls._chat_type_str(chat)
- raw = getattr(message, "message_thread_id", None)
- is_topic_message = bool(getattr(message, "is_topic_message", False))
- is_group = chat_type in ("group", "supergroup")
- is_forum_group = is_group and getattr(chat, "is_forum", False) is True
- if raw is not None:
- if is_forum_group or (is_group and is_topic_message) or (chat_type == "private" and is_topic_message):
- return str(raw)
- return None
- return cls._GENERAL_TOPIC_THREAD_ID if is_forum_group else None
- # Decides only whether a FOREIGN @handle is bot-shaped; our own handle is matched by identity, never
- # shape (collectible/Fragment bot usernames need not end in "bot").
- _FOREIGN_BOT_HANDLE_RE = re.compile(r"[a-z0-9_]{2,29}bot", re.IGNORECASE)
- _BOT_IDENTITY_TTL_SECONDS = 300.0 # how long an observed identity is trusted before re-check
- def _current_bot_username(self) -> str:
- """This bot's live @username (lowercased, no ``@``): the last observed handle beats PTB's
- ``get_me()`` cache, which keeps a stale handle after a BotFather rename."""
- observed = getattr(self, "_bot_username_observed", None)
- if observed:
- return observed
- return (getattr(self._bot, "username", None) or "").lstrip("@").lower()
- def _note_bot_username(self, username: Optional[str]) -> None:
- """Record the bot's current @username, logging real renames."""
- handle = (username or "").lstrip("@").lower()
- if not handle:
- return
- previous = getattr(self, "_bot_username_observed", None)
- if previous == handle:
- return
- self._bot_username_observed = handle
- self._bot_identity_checked_at = time.monotonic()
- if previous:
- logger.info(
- "[%s] Telegram bot username changed: @%s -> @%s (mention routing now follows the new handle)", self.name, previous, handle)
- def _observe_bot_identity_from_message(self, message: Message) -> None:
- """Learn our own handle from a message Telegram says we authored (own messages and
- ``reply_to_message``); only trusted when the user id matches this bot."""
- bot_id = getattr(self._bot, "id", None)
- if bot_id is None:
- return
- for candidate in (getattr(message, "from_user", None), getattr(getattr(message, "reply_to_message", None), "from_user", None)):
- if candidate is not None and getattr(candidate, "id", None) == bot_id:
- self._note_bot_username(getattr(candidate, "username", None))
- def _bot_identity_is_fresh(self) -> bool:
- """True when identity was re-read within the TTL. ``None`` (never checked) is always stale — do
- not fold it into ``0.0``: monotonic clocks have an arbitrary epoch."""
- checked_at = getattr(self, "_bot_identity_checked_at", None)
- return checked_at is not None and (time.monotonic() - checked_at) < self._BOT_IDENTITY_TTL_SECONDS
- async def _refresh_bot_identity(self, *, force: bool = False) -> None:
- """Re-read bot identity when the cache may be stale (``get_me()`` rewrites PTB's ``Bot._bot_user``
- in place). Best-effort: a failed probe keeps the last known handle."""
- bot = self._bot
- if bot is None or not callable(getattr(bot, "get_me", None)):
- return
- if not force and self._bot_identity_is_fresh():
- return
- try:
- me = await asyncio.wait_for(bot.get_me(), self._BOT_IDENTITY_PROBE_TIMEOUT)
- except asyncio.CancelledError:
- raise
- except Exception as exc:
- logger.debug(
- "[%s] Telegram identity refresh failed (keeping @%s): %s", self.name, self._current_bot_username() or "unknown", exc)
- return
- self._bot_identity_checked_at = time.monotonic()
- self._note_bot_username(getattr(me, "username", None))
- _BOT_IDENTITY_PROBE_TIMEOUT = 15.0
- def _is_reply_to_bot(self, message: Message) -> bool:
- if not self._bot or not getattr(message, "reply_to_message", None):
- return False
- reply_user = getattr(message.reply_to_message, "from_user", None)
- return bool(reply_user and getattr(reply_user, "id", None) == getattr(self._bot, "id", None))
- @staticmethod
- def _entity_sources(message: Message):
- """``(text, entities)`` pairs for the message text and caption."""
- yield getattr(message, "text", None) or "", getattr(message, "entities", None) or []
- yield getattr(message, "caption", None) or "", getattr(message, "caption_entities", None) or []
- @staticmethod
- def _entity_type(entity) -> str:
- return str(getattr(entity, "type", "")).split(".")[-1].lower()
- @classmethod
- def _entity_span(cls, source_text: str, entity) -> Optional[str]:
- """The entity's text, or None when its offsets are unusable."""
- # Telegram's official group-disambiguation form for slash commands (``/cmd@botname``) is emitted as
- # a single ``bot_command`` entity covering the whole span — there is no accompanying ``mention``
- # entity. Treat it as a direct address to this bot when the ``@botname`` suffix matches. This is the
- # form Telegram's own command menu autocomplete produces in groups, so dropping it at the mention
- # gate would break /new, /reset, /help, ... for every group that has ``require_mention`` enabled
- # (#15415).
- offset = int(getattr(entity, "offset", -1))
- length = int(getattr(entity, "length", 0))
- if offset < 0 or length <= 0:
- return None
- return cls._telegram_entity_text(source_text, offset, length)
- @classmethod
- def _extract_bot_mention_usernames(cls, message: Message, self_username: str = "") -> set[str]:
- """Explicit bot usernames mentioned in text/captions: foreign handles count only when bot-shaped
- (``...bot``), ``self_username`` opts our OWN handle in regardless of shape. Entity mentions are
- authoritative; the raw-text fallback is deliberately narrow."""
- mentioned_bot_usernames: set[str] = set()
- own = (self_username or "").lstrip("@").lower()
- def _is_bot_handle(handle: str) -> bool:
- if not handle:
- return False
- if own and handle == own:
- return True
- return bool(cls._FOREIGN_BOT_HANDLE_RE.fullmatch(handle))
- for source_text, entities in cls._entity_sources(message):
- for entity in entities:
- entity_type = cls._entity_type(entity)
- if entity_type not in {"mention", "bot_command"}:
- continue
- entity_text = cls._entity_span(source_text, entity)
- if entity_text is None:
- continue
- entity_text = entity_text.strip()
- if entity_type == "mention":
- handle = entity_text.lstrip("@").lower()
- if _is_bot_handle(handle):
- mentioned_bot_usernames.add(handle)
- continue
- # /cmd@botname is one bot_command entity; its suffix is an explicit bot address.
- at_index = entity_text.find("@")
- if at_index < 0:
- continue
- command_target = entity_text[at_index + 1:].strip().lower()
- if _is_bot_handle(command_target):
- mentioned_bot_usernames.add(command_target)
- # Entity-less fallback only: if Telegram supplied entities, trust them (no URL/code rescue).
- for raw_text, entities in cls._entity_sources(message):
- if not raw_text or entities:
- continue
- for match in re.finditer(r"(?i)(?<![A-Za-z0-9_`/])@([A-Za-z0-9_]{2,31})\b", raw_text):
- handle = match.group(1).lower()
- if _is_bot_handle(handle):
- mentioned_bot_usernames.add(handle)
- return mentioned_bot_usernames
- @staticmethod
- def _telegram_entity_text(source_text: str, offset: int, length: int) -> str:
- """Return a Telegram entity span using UTF-16 code-unit offsets."""
- if offset < 0 or length <= 0:
- return ""
- try:
- return source_text.encode("utf-16-le")[offset * 2:(offset + length) * 2].decode("utf-16-le")
- except UnicodeDecodeError:
- return ""
- def _message_mentions_bot(self, message: Message) -> bool:
- if not self._bot:
- return False
- bot_username = self._current_bot_username()
- bot_id = getattr(self._bot, "id", None)
- expected = f"@{bot_username}" if bot_username else None
- # Server-side MessageEntity values are authoritative: raw substrings like "foo@hermes_bot.example"
- # or handles inside URLs/code are not mentions.
- for source_text, entities in self._entity_sources(message):
- for entity in entities:
- entity_type = self._entity_type(entity)
- if entity_type == "mention" and expected:
- span = self._entity_span(source_text, entity)
- if span is not None and span.strip().lower() == expected:
- return True
- elif entity_type == "text_mention":
- user = getattr(entity, "user", None)
- if user and getattr(user, "id", None) == bot_id:
- return True
- elif entity_type == "bot_command" and expected:
- # ``/cmd@botname`` (what the group command menu produces) must count as a direct address.
- command_text = self._entity_span(source_text, entity)
- if command_text is None:
- continue
- at_index = command_text.find("@")
- if at_index >= 0 and command_text[at_index:].strip().lower() == expected:
- return True
- if bot_username:
- return bot_username in self._extract_bot_mention_usernames(message, bot_username)
- return False
- def _schedule_bot_identity_recheck(self) -> None:
- """Fire a TTL-guarded identity refresh in the background when routing is about to discard a
- message naming other bots but not us (the symptom of a stale handle after a rename)."""
- existing = getattr(self, "_bot_identity_refresh_task", None)
- if (existing is not None and not existing.done()) or self._bot_identity_is_fresh():
- return
- try:
- loop = asyncio.get_running_loop()
- except RuntimeError:
- return
- task = loop.create_task(self._refresh_bot_identity())
- self._bot_identity_refresh_task = task
- tracked = getattr(self, "_background_tasks", None)
- if isinstance(tracked, set):
- tracked.add(task)
- task.add_done_callback(tracked.discard)
- def _explicit_bot_mentions_exclude_self(self, message: Message) -> bool:
- """True when explicit bot handles target other bots, not this one (``@bot3 hi @bot4`` must not
- wake ``@bot1`` via reply/wake-word fallbacks)."""
- if not self._bot:
- return False
- bot_username = self._current_bot_username()
- if not bot_username:
- return False
- mentioned_bot_usernames = self._extract_bot_mention_usernames(message, bot_username)
- excludes_self = bool(mentioned_bot_usernames) and bot_username not in mentioned_bot_usernames
- if excludes_self:
- # Either truly for another bot, or our handle is stale after a rename — re-check out of band.
- self._schedule_bot_identity_recheck()
- return excludes_self
- def _message_matches_mention_patterns(self, message: Message) -> bool:
- if not self._mention_patterns:
- return False
- return any(
- pattern.search(candidate)
- for candidate in (getattr(message, "text", None), getattr(message, "caption", None)) if candidate
- for pattern in self._mention_patterns)
- def _is_guest_mention(self, message: Message) -> bool:
- """Guest-mode bypass: explicit bot mention (caller already verified group chat)."""
- return self._telegram_guest_mode() and self._message_mentions_bot(message)
- def _clean_bot_trigger_text(self, text: Optional[str]) -> Optional[str]:
- bot_username = self._current_bot_username()
- if not text or not bot_username:
- return text
- cleaned = re.sub(rf"(?i)@{re.escape(bot_username)}\b[,:\-]*\s*", "", text).strip()
- return cleaned or text
- def _topic_gates_pass(self, thread_id, *, warn_non_numeric: bool) -> Optional[bool]:
- """``allowed_topics`` / ``ignored_threads`` gates; False = blocked, None = undecided."""
- allowed_topics = self._telegram_allowed_topics()
- if allowed_topics and self._topic_id_or_general(thread_id) not in allowed_topics:
- return False
- if thread_id is not None:
- try:
- if int(thread_id) in self._telegram_ignored_threads():
- return False
- except (TypeError, ValueError):
- if not warn_non_numeric:
- return False
- logger.warning("[%s] Ignoring non-numeric Telegram message_thread_id: %r", self.name, thread_id)
- return None
- def _bot_sender_suppressed(self, message: Message) -> bool:
- """True when ``bots_require_mention`` vetoes this other-bot message: another bot must
- explicitly @mention us, its quote-replies and plain chatter do not count (two bots
- answering each other's replies never stop otherwise)."""
- return bool(
- self._telegram_bots_require_mention()
- and self._sender_is_other_bot(message)
- and not self._message_mentions_bot(message)
- )
- def _should_observe_unmentioned_group_message(self, message: Message) -> bool:
- """Return True when a group message should be stored but not dispatched."""
- if self._is_own_message(message) or not self._telegram_observe_unmentioned_group_messages() or not self._is_group_chat(message):
- return False
- if self._topic_gates_pass(getattr(message, "message_thread_id", None), warn_non_numeric=False) is False:
- return False
- chat_id_str = self._chat_id_str(message)
- if self._telegram_exclusive_bot_mentions() and self._explicit_bot_mentions_exclude_self(message):
- return False
- # Observed context is shared at chat/topic scope, so require an explicit chat allowlist.
- allowed = self._telegram_observe_allowed_chats()
- if not allowed or chat_id_str not in allowed:
- return False
- # Free-response chats/topics dispatch every message, so they are never observed.
- if chat_id_str in self._telegram_free_response_chats() or self._telegram_is_free_response_topic(message):
- return False
- # Only observe messages the require_mention gate would skip. The bot-to-bot loop breaker
- # in ``_should_process_message`` skips another bot's message too, so a sibling bot
- # addressing us by wake word (or quote-reply) is never dispatched and must still be
- # kept as observed context (#115119).
- if self._bot_sender_suppressed(message):
- return True
- if not self._telegram_require_mention() or self._is_reply_to_bot(message) or self._message_mentions_bot(message):
- return False
- return not self._message_matches_mention_patterns(message)
- def _telegram_group_observe_shared_source(self, source):
- """Return a chat/topic-scoped source for observed Telegram group context."""
- return dataclasses.replace(source, user_id=None, user_name=None, user_id_alt=None)
- def _telegram_group_observe_attributed_text(self, event: MessageEvent) -> str:
- user_id = event.source.user_id or "unknown"
- return f"[{event.source.user_name or user_id}|{user_id}]\n{event.text or ''}"
- def _telegram_group_observe_channel_prompt(self) -> str:
- username = self._current_bot_username() or "unknown"
- bot_id = getattr(getattr(self, "_bot", None), "id", None) or "unknown"
- return (
- "You are handling a Telegram group chat message.\n"
- f"- Your identity: user_id={bot_id}, @-mention name in this group=@{username}\n"
- "- observed Telegram group context may be provided in a separate context-only block "
- "before the current message; it is not necessarily addressed to you.\n"
- "- Treat only the current new message as a request explicitly directed at you, "
- "and use observed context only when the current message asks for it.")
- def _apply_telegram_group_observe_attribution(self, event: MessageEvent) -> MessageEvent:
- """Align triggered group turns with observed-history attribution."""
- if not self._telegram_observe_unmentioned_group_messages():
- return event
- raw_message = getattr(event, "raw_message", None)
- if not raw_message or not self._is_group_chat(raw_message):
- return event
- allowed = self._telegram_observe_allowed_chats()
- if not allowed or self._chat_id_str(raw_message) not in allowed:
- return event
- observe_prompt = self._telegram_group_observe_channel_prompt()
- channel_prompt = f"{event.channel_prompt}\n\n{observe_prompt}" if event.channel_prompt else observe_prompt
- if event.message_type == MessageType.COMMAND:
- # Commands keep the original source (user_id) so _check_slash_access can identify the sender.
- return dataclasses.replace(event, channel_prompt=channel_prompt)
- return dataclasses.replace(
- event, text=self._telegram_group_observe_attributed_text(event),
- source=self._telegram_group_observe_shared_source(event.source), channel_prompt=channel_prompt)
- def _media_message_type(self, msg: Message) -> MessageType:
- """Classify a Telegram media message into a MessageType (first present attachment wins)."""
- for attr, mtype in (
- ("sticker", MessageType.STICKER), ("photo", MessageType.PHOTO), ("video", MessageType.VIDEO),
- ("video_note", MessageType.VIDEO), ("audio", MessageType.AUDIO), ("voice", MessageType.VOICE)):
- if getattr(msg, attr, None):
- return mtype
- return MessageType.DOCUMENT
- _CACHED_KIND_TO_MESSAGE_TYPE = {"image": MessageType.PHOTO, "video": MessageType.VIDEO, "audio": MessageType.AUDIO}
- async def _download_observed_media(self, msg: Any, what: str):
- """Download ``msg``'s attachment into the media cache (bounded by ``_max_doc_bytes``). Returns ``(status, cached)``:
- ``"none"``, ``"oversized"`` (cached = raw file_size), ``"failed"``, ``"unreadable"`` or ``"ok"``."""
- from gateway.platforms.base import cache_media_bytes_async
- source, filename, mime, kind = self._observed_media_source(msg)
- if source is None:
- return "none", None
- file_size = getattr(source, "file_size", None)
- if not (0 < self._int_or_zero(file_size) <= getattr(self, "_max_doc_bytes", 20 * 1024 * 1024)):
- return "oversized", file_size
- try:
- file_obj = await source.get_file()
- data = bytes(await file_obj.download_as_bytearray())
- if not filename:
- filename = os.path.basename(getattr(file_obj, "file_path", "") or "")
- cached = await cache_media_bytes_async(data, filename=filename, mime_type=mime, default_kind=kind)
- except Exception as exc:
- logger.warning("[Telegram] Failed to cache %s: %s", what, _redact_telegram_error_text(exc), exc_info=True)
- return "failed", None
- if cached is None:
- return "unreadable", None
- return "ok", cached
- async def _cache_observed_media(self, msg: Message, event: MessageEvent) -> None:
- """Cache an unmentioned group attachment and annotate the observed text; oversized or unsupported
- attachments are noted in the transcript without downloading."""
- status, cached = await self._download_observed_media(msg, "observed group media")
- if status == "oversized":
- limit_mb = getattr(self, "_max_doc_bytes", 20 * 1024 * 1024) // (1024 * 1024)
- event.text = self._append_observed_note(
- event.text, f"[Observed Telegram attachment too large or unverifiable. Maximum: {limit_mb} MB.]")
- logger.info("[Telegram] Observed group attachment skipped (size=%s)", cached)
- return
- if status == "unreadable": # only images that fail validation reach here
- event.text = self._append_observed_note(event.text, "[Observed Telegram attachment could not be read, not cached.]")
- return
- if status == "ok":
- event.media_urls = []
- event.media_types = []
- self._attach_cached(event, cached, cached.context_note(), "[Telegram] Cached observed group %s at %s")
- def _attach_cached(self, event: MessageEvent, cached, note: str, log_fmt: str) -> None:
- """Append a cached attachment to the event (message type follows the kind only for the first one)."""
- event.media_urls.append(cached.path)
- event.media_types.append(cached.media_type)
- if len(event.media_urls) == 1 and cached.kind in self._CACHED_KIND_TO_MESSAGE_TYPE:
- event.message_type = self._CACHED_KIND_TO_MESSAGE_TYPE[cached.kind]
- event.text = self._append_observed_note(event.text, note)
- logger.info(log_fmt, cached.kind, cached.path)
- async def _cache_replied_media(self, msg: Any, event: MessageEvent) -> None:
- """Cache media from the message this turn replies to, if any."""
- reply_msg = getattr(msg, "reply_to_message", None)
- if reply_msg is None:
- return
- status, cached = await self._download_observed_media(reply_msg, "replied-to media")
- if status == "ok":
- self._attach_cached(
- event, cached, f"[Replied-to {cached.kind} '{cached.display_name}' saved at: {cached.path}]",
- "[Telegram] Cached replied-to %s at %s")
- def _observed_media_source(self, msg: Message):
- """Return (telegram_file_source, filename, mime, default_kind) or Nones."""
- if msg.photo:
- return msg.photo[-1], "", "", "image"
- if video := msg.video or getattr(msg, "video_note", None):
- return video, "", "video/mp4", "video"
- if msg.voice:
- return msg.voice, "voice.ogg", "audio/ogg", "audio"
- if msg.audio:
- return msg.audio, getattr(msg.audio, "file_name", "") or "", "", "audio"
- if msg.document:
- doc = msg.document
- return doc, doc.file_name or "", (doc.mime_type or "").lower(), None
- return None, "", "", None
- @staticmethod
- def _append_observed_note(existing: Optional[str], note: str) -> str:
- if not note:
- return existing or ""
- return f"{existing}\n\n{note}" if existing else note
- async def _surface_media_cache_failure(
- self, msg: Message, event: MessageEvent, kind: str, exc: Exception, display_name: Optional[str] = None) -> None:
- """Surface a failed media download to BOTH the user (reply asking to retry) and the agent (observed
- note) — otherwise the turn dispatches silently with empty media_urls.
- This (1) replies to the user in Telegram so they know to retry, and (2) appends an agent-visible
- notice to event.text via the existing observed-note channel so the agent knows an attachment was
- attempted and failed — never a silent empty turn. No new event fields (the structured-event refactor
- is out of scope per #23045).
- """
- # Inbound media fails before handle_message binds the routed profile.
- with self._media_delivery_scope(event.source):
- named = t("platform.telegram.media.named_suffix", name=display_name) if display_name else ""
- # ``kind`` is the English label the agent-visible note below keeps; the chat reply is localized.
- notice = self.warning_text(t(
- "platform.telegram.media.download_failed",
- kind=t(_MEDIA_KIND_KEYS[kind]) if kind in _MEDIA_KIND_KEYS else kind,
- name=named, error=exc.__class__.__name__))
- if notice:
- try:
- self._accept_update()
- await msg.reply_text(notice)
- except Exception as reply_err:
- logger.warning("[Telegram] Failed to notify user about %s cache failure: %s", kind, reply_err, exc_info=True)
- # The agent-visible note is execution evidence, not a channel diagnostic; it stays in both modes.
- event.text = self._append_observed_note(
- event.text,
- f"[The user attempted to send a {kind}{named} but it could not be downloaded ({exc.__class__.__name__}); they have been asked to retry.]"
- if notice else f"[The user attempted to send a {kind}{named} but it could not be downloaded.]",
- )
- def _observe_unmentioned_group_message(
- self, message: Message, msg_type: MessageType, update_id: Optional[int] = None, event: Optional[MessageEvent] = None) -> None:
- """Append skipped group chatter to the target session without dispatching."""
- store = getattr(self, "_session_store", None)
- if not store:
- return
- adapter_name = getattr(self, "name", "telegram")
- try:
- event = event or self._build_message_event(message, msg_type, update_id=update_id)
- session_entry = store.get_or_create_session(self._telegram_group_observe_shared_source(event.source))
- entry = {
- "role": "user", "content": self._telegram_group_observe_attributed_text(event),
- "timestamp": datetime.now(tz=UTC).isoformat(), "observed": True}
- if event.message_id:
- entry["message_id"] = str(event.message_id)
- self._accept_update()
- store.append_to_transcript(session_entry.session_id, entry)
- logger.info(
- "[%s] Telegram group message observed (no bot trigger): chat=%s from=%s", adapter_name,
- getattr(getattr(message, "chat", None), "id", "unknown"), event.source.user_id or "unknown")
- except Exception as exc:
- self._fail_update_preparation()
- logger.warning("[%s] Failed to observe Telegram group message: %s", adapter_name, exc)
- def _is_own_message(self, message: Message) -> bool:
- """True when sent by this bot itself (echoed getUpdates must not count as incoming unread)."""
- if not self._bot:
- return False
- from_user = getattr(message, "from_user", None)
- if from_user is None:
- return False
- bot_id = getattr(self._bot, "id", None)
- user_id = getattr(from_user, "id", None)
- return bot_id is not None and user_id is not None and bot_id == user_id
- def _sender_is_other_bot(self, message: Message) -> bool:
- """True when the sender is a bot other than this one (this bot's own echoes are already
- filtered by ``_is_own_message``)."""
- sender = getattr(message, "from_user", None)
- if sender is None or not getattr(sender, "is_bot", False):
- return False
- bot_id = getattr(self._bot, "id", None)
- sender_id = getattr(sender, "id", None)
- return bot_id is None or sender_id is None or sender_id != bot_id
- def _should_process_message(self, message: Message, *, is_command: bool = False) -> bool:
- """Apply Telegram group trigger rules: DMs unrestricted; group messages pass ``allowed_chats`` (hard gate; only
- the ``guest_mode`` @mention bypass crosses it) and then any of free_response chat/topic, ``require_mention``
- off, reply to the bot, @mention (incl. ``/cmd@botname``), or a wake-word match."""
- # Learn the live handle BEFORE any mention gate routes on it, then drop our own echoed messages.
- # Filter out the bot's own messages (returned by getUpdates in some environments like
- # groups/supergroups where the bot can see its own messages). Without this, outbound messages are
- # counted as incoming unread in the Hermes inbox (#52363). Otherwise a BotFather rename leaves the
- # stale handle in place and the exclusive-mention gate reads a message addressed to us as one
- # addressed to some other bot.
- self._observe_bot_identity_from_message(message)
- if self._is_own_message(message):
- return False
- if not self._is_group_chat(message):
- return True
- thread_id = self._effective_message_thread_id(message)
- if self._topic_gates_pass(thread_id, warn_non_numeric=True) is False:
- return False
- chat_id_str = self._chat_id_str(message)
- if self._telegram_exclusive_bot_mentions() and self._explicit_bot_mentions_exclude_self(message):
- return False
- # Resolve once; _message_mentions_bot is not re-called below in guest mode.
- guest_mention = self._is_guest_mention(message)
- # allowed_chats whitelist: outside chats pass only via the guest-mode explicit mention.
- allowed = self._telegram_allowed_chats()
- if allowed and chat_id_str not in allowed:
- return guest_mention
- if guest_mention or chat_id_str in self._telegram_free_response_chats() or self._telegram_is_free_response_topic(message):
- return True
- # Bot-to-bot loop breaker: another bot must explicitly @mention us; its quote-reply or
- # plain chatter does not count (two bots answering each other's replies never stop otherwise).
- if self._bot_sender_suppressed(message):
- return False
- if not self._telegram_require_mention() or self._is_reply_to_bot(message):
- return True
- if not self._telegram_guest_mode() and self._message_mentions_bot(message):
- return True
- return self._message_matches_mention_patterns(message)
- async def _ensure_forum_commands(self, message) -> None:
- """Lazy-register bot commands for forum supergroups (topics don't inherit AllGroupChats scope;
- Telegram resolves via BotCommandScopeChat)."""
- async with self._forum_lock:
- try:
- chat = getattr(message, "chat", None)
- if not chat or not getattr(chat, "is_forum", False):
- return
- chat_id = int(chat.id)
- if chat_id in self._forum_command_registered:
- return
- from telegram import BotCommandScopeChat
- from hermes_cli.commands_platforms import telegram_menu_commands, telegram_menu_max_commands
- menu_commands, _ = await asyncio.to_thread(
- telegram_menu_commands, max_commands=telegram_menu_max_commands())
- bot_commands = self._bot_commands(menu_commands)
- await self._bot.set_my_commands(bot_commands, scope=BotCommandScopeChat(chat_id=chat_id))
- self._forum_command_registered.add(chat_id)
- logger.info("[%s] Lazy-registered %d commands for forum chat %s", self.name, len(bot_commands), chat_id)
- except Exception as e:
- logger.warning("[%s] Forum command lazy-registration failed: %s", self.name, _redact_telegram_error_text(e))
- def _effective_update_message(self, update: Update) -> Optional[Message]:
- """Message-like payload for normal messages and channel posts (``update.channel_post``)."""
- return getattr(update, "effective_message", None) or getattr(update, "message", None)
- def _log_blocked_user(self, msg, *, level=logging.WARNING, what: str = "unauthorized user") -> None:
- logger.log(
- level, "[Telegram] Blocked %s %s in chat %s", what, getattr(getattr(msg, "from_user", None), "id", None),
- getattr(getattr(msg, "chat", None), "id", None))
- def _gate_or_observe(self, msg, update, msg_type: MessageType) -> bool:
- """Group trigger gate; observes unmentioned chatter when configured. True = proceed."""
- if self._should_process_message(msg):
- return True
- if self._should_observe_unmentioned_group_message(msg):
- self._observe_unmentioned_group_message(msg, msg_type, update_id=update.update_id)
- return False
- async def _build_triggered_event(self, msg, update, msg_type: MessageType) -> MessageEvent:
- """Event for an addressed text/command: trigger text cleaned (sole addressee only), replied-to
- media cached, attribution applied."""
- from plugins.platforms.telegram.telegram_context import group_trigger_text
- event = self._build_message_event(msg, msg_type, update_id=update.update_id)
- event.text = group_trigger_text(self, msg, event.text)
- await self._cache_replied_media(msg, event)
- return self._apply_telegram_group_observe_attribution(event)
- async def _handle_text_message(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
- """Handle incoming text; buffers client-split chunks into one MessageEvent."""
- msg = self._effective_update_message(update)
- if not msg or not msg.text:
- return
- # Auth check first: blocked users must not reach batching, the observed transcript, or the agent.
- if not self._is_user_authorized_from_message(msg):
- self._log_blocked_user(msg)
- return
- if not self._gate_or_observe(msg, update, MessageType.TEXT):
- return
- await self._ensure_forum_commands(update.message)
- self._enqueue_text_event(await self._build_triggered_event(msg, update, MessageType.TEXT))
- async def _handle_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
- """Handle incoming command messages."""
- msg = self._effective_update_message(update)
- if not msg or not msg.text:
- return
- if not self._should_process_message(msg, is_command=True):
- return
- if not self._is_user_authorized_from_message(msg):
- self._log_blocked_user(msg)
- return
- await self._ensure_forum_commands(msg)
- event = await self._build_triggered_event(msg, update, MessageType.COMMAND)
- # A >4096-char command paste arrives as a near-limit COMMAND chunk plus TEXT continuations; dispatching
- # immediately would orphan them. Near-limit commands go through text batching.
- if len(event.text or "") >= self._SPLIT_THRESHOLD:
- self._enqueue_text_event(event)
- return
- await self.handle_message(event)
- async def _handle_location_message(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
- """Handle incoming location/venue pin messages."""
- msg = self._effective_update_message(update)
- if not msg:
- return
- if not self._is_user_authorized_from_message(msg):
- self._log_blocked_user(msg)
- return
- if not self._gate_or_observe(msg, update, MessageType.LOCATION):
- return
- venue = getattr(msg, "venue", None)
- location = getattr(venue, "location", None) if venue else getattr(msg, "location", None)
- if not location:
- return
- lat = getattr(location, "latitude", None)
- lon = getattr(location, "longitude", None)
- if lat is None or lon is None:
- return
- parts = ["[The user shared a location pin.]"]
- if venue:
- title = getattr(venue, "title", None)
- address = getattr(venue, "address", None)
- if title:
- parts.append(f"Venue: {title}")
- if address:
- parts.append(f"Address: {address}")
- parts += [
- f"latitude: {lat}", f"longitude: {lon}", f"Map: https://www.google.com/maps/search/?api=1&query={lat},{lon}",
- "Ask what they'd like to find nearby (restaurants, cafes, etc.) and any preferences."]
- event = self._build_message_event(msg, MessageType.LOCATION, update_id=update.update_id)
- event.text = "\n".join(parts)
- await self.handle_message(self._apply_telegram_group_observe_attribution(event))
- # -- Text message aggregation (handles Telegram client-side splits) --
- def _text_batch_key(self, event: MessageEvent) -> str:
- """Session-scoped batching key; topic recovery first so DM-topic batches coalesce on the recovered lane."""
- self._apply_topic_recovery(event)
- return super()._text_batch_key(event)
- def _enqueue_text_event(self, event: MessageEvent) -> None:
- """Buffer a text chunk, or hold it while delayed delivery must be dropped."""
- if self._should_drop_delayed_delivery():
- self._hold_inbound_event(event, where="text-enqueue")
- return
- super()._enqueue_text_event(event)
- self._accept_update()
- async def _flush_buffered(self, pending: dict, tasks: dict, key: str, delay: float, where: str, log_fn=None) -> None:
- """Shared delayed-flush body: sleep, pop, hold if teardown started, else dispatch. A cancel after
- the pop but before durable dispatch re-holds the event (never lose it)."""
- current_task = asyncio.current_task()
- committed = self.__dict__.setdefault("_committed_flush_tasks", {})
- event = None
- try:
- await asyncio.sleep(delay)
- # Superseded flush (a newer chunk re-armed the timer while our sleep was already done):
- # CancelledError only lands at the next await, so check synchronously before the pop.
- owner = tasks.get(key)
- if owner is not None and owner is not current_task:
- return
- event = pending.pop(key, None)
- if not event:
- return
- # Committed: leave the per-key slot so a newer chunk re-arms without cancelling this
- # dispatch mid-flight (e.g. during a root-DM createForumTopic, whose ID would be lost).
- tasks.pop(key, None)
- prior = committed.get((where, key))
- committed[(where, key)] = current_task
- if prior is not None and not prior.done():
- await asyncio.wait({prior}) # dispatch in order, after the earlier batch settles
- if self._should_drop_delayed_delivery():
- self._hold_inbound_event(event, where=f"{where}-flush")
- event = None
- return
- if log_fn is not None:
- log_fn(event)
- await self.handle_message(event)
- event = None
- except asyncio.CancelledError:
- if event is not None:
- self._hold_inbound_event(event, where=f"{where}-flush-cancelled")
- raise
- finally:
- if committed.get((where, key)) is current_task:
- committed.pop((where, key), None)
- if tasks.get(key) is current_task:
- tasks.pop(key, None)
- def _text_batch_delay_for(self, pending: Optional[MessageEvent]) -> float:
- """Adaptive delay: near-split-point last chunk → long delay (continuation almost certain);
- short/medium totals → capped fast delays; else configured cap (all min()'d with the operator cap)."""
- last_len = getattr(pending, "_last_chunk_len", 0) if pending else 0
- total_len = len(getattr(pending, "text", "") or "") if pending else 0
- if last_len >= self._SPLIT_THRESHOLD:
- return self._text_batch_split_delay_seconds
- if total_len <= self._TEXT_BATCH_FAST_LEN:
- return min(self._text_batch_delay_seconds, self._TEXT_BATCH_FAST_DELAY_S)
- if total_len <= self._TEXT_BATCH_SHORT_LEN:
- return min(self._text_batch_delay_seconds, self._TEXT_BATCH_SHORT_DELAY_S)
- return self._text_batch_delay_seconds
- async def _flush_text_batch(self, key: str) -> None:
- """Telegram keeps its own flush body: a cancel after the pop must HOLD the event and re-raise
- (PTB already acked the update; the hold queue redispatches after reconnect) rather than shield
- the dispatch — teardown must be able to stop a flush from reaching a torn-down session."""
- await self._flush_buffered(
- self._pending_text_batches, self._pending_text_batch_tasks, key,
- self._text_batch_delay_for(self._pending_text_batches.get(key)), "text",
- lambda ev: logger.info("[Telegram] Flushing text batch %s (%d chars)", key, len(ev.text or "")))
- # -- Photo batching --
- def _photo_batch_key(self, event: MessageEvent, msg: Message) -> str:
- """Return a batching key for Telegram photos/albums."""
- session_key = self._event_session_key(event)
- media_group_id = getattr(msg, "media_group_id", None)
- return f"{session_key}:album:{media_group_id}" if media_group_id else f"{session_key}:photo-burst"
- async def _flush_photo_batch(self, batch_key: str) -> None:
- """Send a buffered photo burst/album as a single MessageEvent."""
- await self._flush_buffered(
- self._pending_photo_batches, self._pending_photo_batch_tasks, batch_key, self._media_batch_delay_seconds, "photo",
- lambda ev: logger.info("[Telegram] Flushing photo batch %s with %d image(s)", batch_key, len(ev.media_urls)))
- def _merge_into_pending(self, pending: dict, key: str, event: MessageEvent) -> None:
- """Merge ``event`` into ``pending[key]`` (media, caption, reply expectation) or seed it."""
- if (existing := pending.get(key)) is None:
- pending[key] = event
- return
- existing.absorb_reply_expected(event)
- existing.media_urls.extend(event.media_urls)
- existing.media_types.extend(event.media_types)
- if event.text:
- existing.text = self._merge_caption(existing.text, event.text)
- def _enqueue_photo_event(self, batch_key: str, event: MessageEvent) -> None:
- """Merge photo events into a pending batch and schedule flush."""
- if self._should_drop_delayed_delivery():
- self._hold_inbound_event(event, where="photo-enqueue")
- return
- self._merge_into_pending(self._pending_photo_batches, batch_key, event)
- self._accept_update()
- prior_task = self._pending_photo_batch_tasks.get(batch_key)
- if prior_task and not prior_task.done():
- prior_task.cancel()
- self._pending_photo_batch_tasks[batch_key] = asyncio.create_task(self._flush_photo_batch(batch_key))
- async def _route_photo_event(self, msg, event: MessageEvent) -> None:
- """Album items debounce on media_group_id; singles go through the photo burst batcher."""
- if self._drop_unresolved(event): # identity FIRST: the batch lane is derived from it
- return
- media_group_id = getattr(msg, "media_group_id", None)
- if media_group_id:
- await self._queue_media_group_event(str(media_group_id), event)
- else:
- self._enqueue_photo_event(self._photo_batch_key(event, msg), event)
- @staticmethod
- def _ext_from_path(file_path: Optional[str], candidates, default: str) -> str:
- """First extension in ``candidates`` that ``file_path`` ends with (case-insensitive), else default."""
- if file_path:
- lowered = file_path.lower()
- for candidate in candidates:
- if lowered.endswith(candidate):
- return candidate
- return default
- async def _cache_inbound_av(self, msg, event: MessageEvent, source: Any, label: str, kind: str, ext: str, mime: str) -> bool:
- """Download a voice/audio/video attachment into the local cache. Returns True when the event was
- already dispatched (oversized attachment), so the caller must return."""
- try:
- allowed, note = self._telegram_media_size_allowed(source, label)
- if not allowed:
- event.text = self._append_observed_note(event.text, note or "")
- logger.info("[Telegram] Skipped oversized user %s (size=%s)", kind, getattr(source, "file_size", None))
- await self.handle_message(event)
- return True
- file_obj = await source.get_file()
- data = await file_obj.download_as_bytearray()
- if kind == "video":
- ext = self._ext_from_path(getattr(file_obj, "file_path", None), SUPPORTED_VIDEO_TYPES, ext)
- cached_path = await cache_video_from_bytes_async(bytes(data), ext=ext)
- mime = SUPPORTED_VIDEO_TYPES.get(ext, "video/mp4")
- else:
- cached_path = await cache_audio_from_bytes_async(bytes(data), ext=ext)
- event.media_urls = [cached_path]
- event.media_types = [mime]
- logger.info("[Telegram] Cached user %s at %s", kind, cached_path)
- except Exception as e:
- logger.warning("[Telegram] Failed to cache %s: %s", kind, _redact_telegram_error_text(e), exc_info=True)
- await self._surface_media_cache_failure(msg, event, label, e)
- return False
- async def _dispatch_with_text(self, event: MessageEvent, text: str) -> bool:
- """Replace the event text with a user-facing note and dispatch it; returns True (handled)."""
- event.text = text
- await self.handle_message(event)
- return True
- @staticmethod
- def _set_cached_media(event: MessageEvent, path: str, mime: str, mtype: MessageType, log_fmt: str) -> None:
- event.media_urls = [path]
- event.media_types = [mime]
- event.message_type = mtype
- logger.info(log_fmt, path)
- async def _cache_inbound_document(self, msg, event: MessageEvent) -> bool:
- """Cache a document attachment (image → photo path, video, else generic media + text injection).
- Returns True when the event was already dispatched/routed so the caller must return."""
- doc = msg.document
- try:
- original_filename = doc.file_name or ""
- ext = os.path.splitext(original_filename)[1].lower() if original_filename else ""
- doc_mime = (doc.mime_type or "").lower() # some clients send "IMAGE/PNG"
- if not ext and doc_mime:
- ext = _TELEGRAM_IMAGE_MIME_TO_EXT.get(doc_mime, "")
- if not ext:
- ext = {v: k for k, v in SUPPORTED_DOCUMENT_TYPES.items()}.get(doc_mime, "")
- display = original_filename or doc_mime or ext or 'unknown'
- # Size check before the image branch so image documents can't bypass the limit.
- if not doc.file_size or doc.file_size > self._max_doc_bytes:
- logger.info("[Telegram] Document too large: %s bytes", doc.file_size)
- return await self._dispatch_with_text(
- event, f"The document is too large or its size could not be verified. Maximum: {self._max_doc_bytes // (1024 * 1024)} MB.")
- # Screenshots/photos sent as documents take the image cache + batching path.
- if ext in _TELEGRAM_IMAGE_EXTENSIONS or doc_mime.startswith("image/"):
- file_obj = await doc.get_file()
- image_bytes = await file_obj.download_as_bytearray()
- image_ext = ext if ext in _TELEGRAM_IMAGE_EXTENSIONS else _TELEGRAM_IMAGE_MIME_TO_EXT.get(doc_mime, ".jpg")
- try:
- cached_path = await cache_image_from_bytes_async(bytes(image_bytes), ext=image_ext)
- except ValueError as e:
- logger.warning("[Telegram] Failed to cache image document: %s", _redact_telegram_error_text(e), exc_info=True)
- return await self._dispatch_with_text(event, f"Image document '{display}' could not be read as an image.")
- self._set_cached_media(
- event, cached_path, doc_mime if doc_mime.startswith(
- "image/"
- ) else _TELEGRAM_IMAGE_EXT_TO_MIME.get(image_ext, "image/jpeg"),
- MessageType.PHOTO, "[Telegram] Cached user image-document at %s")
- await self._route_photo_event(msg, event)
- return True
- if not ext and doc.mime_type:
- ext = {v: k for k, v in SUPPORTED_VIDEO_TYPES.items()}.get(doc.mime_type, "")
- if not ext and doc.mime_type:
- # .jpg and .jpeg both map to image/jpeg; keep the first ext seen.
- image_mime_to_ext: dict[str, str] = {}
- for _ext, _mime in SUPPORTED_IMAGE_DOCUMENT_TYPES.items():
- image_mime_to_ext.setdefault(_mime, _ext)
- ext = image_mime_to_ext.get(doc.mime_type, "")
- if ext in SUPPORTED_VIDEO_TYPES:
- file_obj = await doc.get_file()
- video_bytes = await file_obj.download_as_bytearray()
- self._set_cached_media(
- event, await cache_video_from_bytes_async(bytes(video_bytes), ext=ext), SUPPORTED_VIDEO_TYPES[ext], MessageType.VIDEO,
- "[Telegram] Cached user video document at %s")
- await self.handle_message(event)
- return True
- # Any file type is accepted (authorization is the gate, not the extension); unknown types get
- # application/octet-stream. Image documents already returned above.
- file_obj = await doc.get_file()
- raw_bytes = bytes(await file_obj.download_as_bytearray())
- from gateway.platforms.base import cache_media_bytes_async
- cached = await cache_media_bytes_async(raw_bytes, filename=original_filename or f"document{ext or '.bin'}", mime_type=doc_mime)
- if cached is None:
- return await self._dispatch_with_text(event, f"Document '{display}' could not be cached.")
- event.media_urls = [cached.path]
- event.media_types = [cached.media_type]
- event.media_text_inlined = [False] # flipped below once the text is actually injected
- if cached.kind == "audio":
- event.message_type = MessageType.AUDIO
- logger.info("[Telegram] Cached user %s at %s (%s)", cached.kind, cached.path, cached.media_type)
- # Inject text-readable content (≤100 KB). Gate on extension/MIME, NOT a blind UTF-8 decode:
- # PDF/zip/docx have decodable ASCII headers. Binary files are surfaced as a cached path only.
- MAX_TEXT_INJECT_BYTES = 100 * 1024
- _is_text = ext in _TEXT_INJECT_EXTENSIONS or (doc_mime or "").startswith("text/")
- if _is_text and len(raw_bytes) <= MAX_TEXT_INJECT_BYTES:
- try:
- text_content = raw_bytes.decode("utf-8")
- display_name = re.sub(r'[^\w.\- ]', '_', original_filename or f"document{ext or '.txt'}")
- injection = f"[Content of {display_name}]:\n{text_content}"
- event.text = f"{injection}\n\n{event.text}" if event.text else injection
- event.media_text_inlined = [True]
- except UnicodeDecodeError:
- pass # binary — agent has the cached path
- except Exception as e:
- logger.warning("[Telegram] Failed to cache document: %s", _redact_telegram_error_text(e), exc_info=True)
- await self._surface_media_cache_failure(msg, event, "attachment", e, display_name=getattr(doc, "file_name", None) or None)
- return False
- async def _handle_media_message(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
- """Handle incoming media messages, downloading images to local cache."""
- msg = update.message
- if not msg:
- return
- if not self._is_user_authorized_from_message(msg):
- self._log_blocked_user(msg, level=logging.INFO, what="media from unauthorized user")
- return
- if not self._should_process_message(msg):
- if self._should_observe_unmentioned_group_message(msg):
- _event = self._build_message_event(msg, self._media_message_type(msg), update_id=update.update_id)
- if msg.caption:
- _event.text = self._clean_bot_trigger_text(expand_link_entities(msg))
- await self._cache_observed_media(msg, _event)
- self._observe_unmentioned_group_message(msg, _event.message_type, update_id=update.update_id, event=_event)
- return
- event = self._build_message_event(msg, self._media_message_type(msg), update_id=update.update_id)
- if msg.caption:
- from plugins.platforms.telegram.telegram_context import group_trigger_text
- event.text = group_trigger_text(self, msg, expand_link_entities(msg))
- # Stickers: _handle_sticker overwrites event.text with its vision description, so observe attribution must run after it.
- if msg.sticker:
- await self._handle_sticker(msg, event)
- await self.handle_message(self._apply_telegram_group_observe_attribution(event))
- return
- event = self._apply_telegram_group_observe_attribution(event)
- # Cache photo locally: Telegram's file URLs expire (~1 hour) before vision may run.
- if msg.photo:
- try:
- file_obj = await msg.photo[-1].get_file() # PhotoSize list sorted by size; largest last
- image_bytes = await file_obj.download_as_bytearray()
- ext = self._ext_from_path(file_obj.file_path, [".png", ".webp", ".gif", ".jpeg", ".jpg"], ".jpg")
- self._set_cached_media(
- event, await cache_image_from_bytes_async(bytes(image_bytes), ext=ext), f"image/{ext.lstrip('.')}", event.message_type,
- "[Telegram] Cached user photo at %s")
- await self._route_photo_event(msg, event)
- return
- except Exception as e:
- logger.warning("[Telegram] Failed to cache photo: %s", _redact_telegram_error_text(e), exc_info=True)
- await self._surface_media_cache_failure(msg, event, "photo", e)
- # Voice/audio cached for STT transcription; video for vision.
- if msg.voice:
- if await self._cache_inbound_av(msg, event, msg.voice, "voice message", "voice", ".ogg", "audio/ogg"):
- return
- elif msg.audio:
- if await self._cache_inbound_av(msg, event, msg.audio, "audio file", "audio", ".mp3", "audio/mp3"):
- return
- elif video := msg.video or getattr(msg, "video_note", None):
- if await self._cache_inbound_av(msg, event, video, "video file" if msg.video else "video note", "video", ".mp4", "video/mp4"):
- return
- elif msg.document and await self._cache_inbound_document(msg, event):
- return
- media_group_id = getattr(msg, "media_group_id", None)
- if media_group_id:
- await self._queue_media_group_event(str(media_group_id), event)
- return
- await self.handle_message(event)
- async def _queue_media_group_event(self, media_group_id: str, event: MessageEvent) -> None:
- """Debounce album items (shared media_group_id) into one MessageEvent so the second image isn't
- treated as a new message interrupting the first."""
- if self._should_drop_delayed_delivery():
- self._hold_inbound_event(event, where="media-group-enqueue")
- return
- self._merge_into_pending(self._media_group_events, media_group_id, event)
- self._accept_update()
- prior_task = self._media_group_tasks.get(media_group_id)
- if prior_task:
- prior_task.cancel()
- self._media_group_tasks[media_group_id] = asyncio.create_task(self._flush_media_group_event(media_group_id))
- async def _flush_media_group_event(self, media_group_id: str) -> None:
- await self._flush_buffered(
- self._media_group_events, self._media_group_tasks, media_group_id, self.MEDIA_GROUP_WAIT_SECONDS, "media-group")
- async def _handle_sticker(self, msg: Message, event: MessageEvent) -> None:
- """Describe a sticker via vision, cached by file_unique_id; animated/video stickers get an emoji placeholder."""
- from gateway.sticker_cache import (
- get_cached_description, cache_sticker_description_async, build_sticker_injection,
- build_animated_sticker_injection, STICKER_VISION_PROMPT)
- sticker = msg.sticker
- emoji = sticker.emoji or ""
- set_name = sticker.set_name or ""
- if sticker.is_animated or sticker.is_video:
- event.text = build_animated_sticker_injection(emoji)
- return
- cached = get_cached_description(sticker.file_unique_id)
- if cached:
- event.text = build_sticker_injection(cached["description"], cached.get("emoji", emoji), cached.get("set_name", set_name))
- logger.info("[Telegram] Sticker cache hit: %s", sticker.file_unique_id)
- return
- fallback = f"a sticker with emoji {emoji}" if emoji else "a sticker"
- try:
- file_obj = await sticker.get_file()
- image_bytes = await file_obj.download_as_bytearray()
- cached_path = await cache_image_from_bytes_async(bytes(image_bytes), ext=".webp")
- logger.info("[Telegram] Analyzing sticker at %s", cached_path)
- from tools.vision_tools import vision_analyze_tool
- self._accept_update() # Cancellation cannot undo an already submitted auxiliary LLM request.
- result = json.loads(await vision_analyze_tool(image_url=cached_path, user_prompt=STICKER_VISION_PROMPT))
- if result.get("success"):
- description = result.get("analysis", "a sticker")
- await cache_sticker_description_async(sticker.file_unique_id, description, emoji, set_name)
- event.text = build_sticker_injection(description, emoji, set_name)
- else:
- event.text = build_sticker_injection(fallback, emoji, set_name)
- except Exception as e:
- logger.warning("[Telegram] Sticker analysis error: %s", _redact_telegram_error_text(e), exc_info=True)
- event.text = build_sticker_injection(fallback, emoji, set_name)
- def _reload_dm_topics_from_config(self) -> None:
- """Re-read dm_topics from config.yaml so externally created topics work without restart."""
- try:
- from hermes_cli.config import load_config_readonly # canonical loader: managed overlay + ${VAR}
- dm_topics = load_config_readonly().get("platforms", {}).get("telegram", {}).get("extra", {}).get("dm_topics", [])
- if not dm_topics:
- self._dm_topics_config = []
- self._dm_topic_chat_ids = set()
- return
- self._dm_topics_config = dm_topics
- self._dm_topic_chat_ids = {str(chat_entry["chat_id"]) for chat_entry in dm_topics if "chat_id" in chat_entry}
- for chat_entry in dm_topics:
- cid = chat_entry.get("chat_id")
- if not cid:
- continue
- for t in chat_entry.get("topics", []):
- tid = t.get("thread_id")
- name = t.get("name")
- if tid and name and f"{cid}:{name}" not in self._dm_topics:
- self._dm_topics[f"{cid}:{name}"] = int(tid)
- logger.info("[%s] Hot-loaded DM topic from config: %s -> thread_id=%s", self.name, f"{cid}:{name}", tid)
- except Exception as e:
- logger.debug("[%s] Failed to reload dm_topics from config: %s", self.name, e)
- def _get_dm_topic_info(self, chat_id: str, thread_id: Optional[str]) -> Optional[dict[str, Any]]:
- """Return the DM topic config dict (name, skill, ...) for this thread_id, or None."""
- if not thread_id:
- return None
- thread_id_int = int(thread_id)
- def _lookup() -> Optional[dict[str, Any]]:
- for key, cached_tid in self._dm_topics.items():
- if cached_tid == thread_id_int and key.startswith(f"{chat_id}:"):
- topic_name = key.split(":", 1)[1]
- for chat_entry in self._dm_topics_config:
- if str(chat_entry.get("chat_id")) == chat_id:
- for t in chat_entry.get("topics", []):
- if t.get("name") == topic_name:
- return t
- return {"name": topic_name}
- return None
- found = _lookup()
- if found is not None:
- return found
- self._reload_dm_topics_from_config() # cache miss — topics may have been added externally
- return _lookup()
- def _cache_dm_topic_from_message(self, chat_id: str, thread_id: str, topic_name: str) -> None:
- """Cache a thread_id -> topic_name mapping discovered from an incoming message."""
- cache_key = f"{chat_id}:{topic_name}"
- if cache_key not in self._dm_topics:
- self._dm_topics[cache_key] = int(thread_id)
- logger.info("[%s] Cached DM topic from message: %s -> thread_id=%s", self.name, cache_key, thread_id)
- @classmethod
- def _flatten_rich_inline_text(cls, value: Any) -> str:
- """Best-effort plaintext flattener for Bot API rich-message inline nodes."""
- if value is None:
- return ""
- if isinstance(value, str):
- return value
- if isinstance(value, list):
- return "".join(cls._flatten_rich_inline_text(item) for item in value)
- if isinstance(value, dict):
- for key in ("text", "children"):
- if value.get(key) is not None:
- return cls._flatten_rich_inline_text(value[key])
- return ""
- @classmethod
- def _flatten_rich_blocks(cls, blocks: Any) -> str:
- """Best-effort plaintext flattener for Bot API rich-message blocks."""
- if not isinstance(blocks, list):
- return ""
- lines: list[str] = []
- for block in blocks:
- if not isinstance(block, dict):
- continue
- if block.get("type") == "list":
- for item in block.get("items", []):
- if not isinstance(item, dict):
- continue
- item_lines = cls._flatten_rich_blocks(item.get("blocks")).splitlines()
- if not item_lines:
- continue
- label = item.get("label")
- lines.append(f"{label} {item_lines[0]}".strip() if label else item_lines[0])
- lines.extend(item_lines[1:])
- continue
- text = cls._flatten_rich_inline_text(block.get("text"))
- if text:
- lines.extend(text.splitlines())
- return "\n".join(line.rstrip() for line in lines if line)
- @classmethod
- def _extract_rich_reply_text(cls, reply_to_message: Any) -> Optional[str]:
- """Return plaintext echoed by Telegram's rich_message reply payload."""
- try:
- getter = getattr(getattr(reply_to_message, "api_kwargs", None), "get", None)
- if not callable(getter):
- return None
- rich_getter = getattr(getter("rich_message"), "get", None)
- if not callable(rich_getter):
- return None
- return cls._flatten_rich_blocks(rich_getter("blocks")).strip() or None
- except Exception:
- return None
- def _resolve_topic_binding(self, message: Message, chat_type: str, thread_id_str: Optional[str]) -> tuple:
- """Return ``(chat_topic, topic_skill)`` for a DM topic or bound forum topic (else Nones)."""
- chat = message.chat
- chat_topic = None
- topic_skill = None
- if chat_type == "dm" and thread_id_str:
- topic_info = self._get_dm_topic_info(str(chat.id), thread_id_str)
- if topic_info:
- chat_topic = topic_info.get("name")
- topic_skill = topic_info.get("skill")
- # forum_topic_created service messages also reveal topic names
- if hasattr(message, "forum_topic_created") and message.forum_topic_created:
- created_name = message.forum_topic_created.name
- if created_name:
- self._cache_dm_topic_from_message(str(chat.id), thread_id_str, created_name)
- if not chat_topic:
- chat_topic = created_name
- elif chat_type == "group" and thread_id_str:
- # Forum topic skill binding via config.extra['group_topics']; accepts both
- # [{"chat_id": ..., "topics": [...]}] and legacy {"-100...": [{"thread_id": 12}]}.
- group_topics_config = self.config.extra.get("group_topics", [])
- if isinstance(group_topics_config, dict):
- group_topics_iter = [{"chat_id": cfg_chat_id, "topics": topics} for cfg_chat_id, topics in group_topics_config.items()]
- elif isinstance(group_topics_config, list):
- group_topics_iter = [entry for entry in group_topics_config if isinstance(entry, dict)]
- else:
- group_topics_iter = []
- for chat_entry in group_topics_iter:
- if str(chat_entry.get("chat_id", "")) != str(chat.id):
- continue
- topics = chat_entry.get("topics", [])
- for topic in (topics if isinstance(topics, list) else []):
- if not isinstance(topic, dict):
- continue
- tid = topic.get("thread_id")
- if tid is not None and str(tid) == thread_id_str:
- chat_topic = topic.get("name")
- topic_skill = topic.get("skill")
- break
- break
- return chat_topic, topic_skill
- def _reply_context(self, message: Message) -> tuple:
- """``(reply_to_id, reply_to_text)`` for the replied-to message: Telegram's native partial quote
- first, then text/caption, rich echo, then the sent index."""
- if not message.reply_to_message:
- return None, None
- reply_to_id = str(message.reply_to_message.message_id)
- quote = getattr(message, "quote", None)
- quote_text = getattr(quote, "text", None) if quote is not None else None
- if quote_text:
- return reply_to_id, quote_text
- reply_to_text = message.reply_to_message.text or message.reply_to_message.caption or None
- if not reply_to_text:
- reply_to_text = self._extract_rich_reply_text(message.reply_to_message)
- if not reply_to_text:
- try:
- from gateway import rich_sent_store
- reply_to_text = rich_sent_store.lookup(str(message.chat.id), reply_to_id)
- except Exception:
- # Extract reply context if this message is a reply. Prefer Telegram's native partial quote
- # (message.quote, TextQuote) so a user replying to a single selected substring of a prior
- # multi-section message doesn't get the whole replied-to message injected into the agent's
- # context — which can cause the agent to act on unrelated actionable-looking text the user
- # didn't quote (#22619). Fall back to the full replied-to message text / caption when no
- # native quote is present.
- reply_to_text = None
- return reply_to_id, reply_to_text
- def _build_message_event(self, message: Message, msg_type: MessageType, update_id: Optional[int] = None) -> MessageEvent:
- """Build a MessageEvent from a Telegram message. ``update_id`` lets ``/restart`` record the
- triggering offset so the new gateway process advances past it."""
- chat = message.chat
- user = message.from_user
- telegram_chat_type = self._chat_type_str(chat) # str() so PTB enums and plain-string mocks both work
- chat_type = "group" if telegram_chat_type in {"group", "supergroup"} else ("channel" if telegram_chat_type == "channel" else "dm")
- # Shared normalizer so gating and session routing agree (reply-UI anchors dropped, General → "1").
- # Resolve routable thread id for DM topics and forum group topics via the shared normalizer, so
- # gating and session routing agree on one value. Only real topic/forum messages keep a thread id;
- # ordinary reply-UI anchors are dropped (they are not durable session threads and sends against them
- # hit 'Message thread not found', #3206), while forum General-topic messages
- # (message_thread_id=None) normalize to the General-topic id so replies route back to General
- # (#22423).
- thread_id_str = self._effective_message_thread_id(message)
- # A new private-chat topic's opening message can arrive with no thread id at all (#136567); the
- # creation it follows is then the only source of the topic id. Adopt it before anything downstream
- # (gating, batching, session routing, topic recovery) reads the event source.
- thread_id_str = self._adopt_created_topic_thread(message, chat_type, thread_id_str)
- chat_topic, topic_skill = self._resolve_topic_binding(message, chat_type, thread_id_str)
- has_full_name = hasattr(chat, "full_name")
- if user:
- user_name = user.full_name
- elif has_full_name and chat_type == "dm":
- user_name = chat.full_name
- else:
- user_name = chat.title if chat_type == "channel" else None
- source = self.build_source(
- chat_id=str(chat.id), chat_name=chat.title or (chat.full_name if has_full_name else None), chat_type=chat_type,
- user_id=(str(user.id) if user else (str(chat.id) if chat_type in {"dm", "channel"} else None)),
- user_name=user_name, thread_id=thread_id_str, chat_topic=chat_topic, message_id=str(message.message_id),
- is_bot=bool(getattr(user, "is_bot", False)) if user else False)
- reply_to_id, reply_to_text = self._reply_context(message)
- from gateway.platforms.base import resolve_channel_prompt # per-channel/topic ephemeral prompt
- from plugins.platforms.telegram.telegram_context import group_identity_prompt, group_reply_expected
- _chat_id_str = str(chat.id)
- channel_prompt = resolve_channel_prompt(self.config.extra, thread_id_str or _chat_id_str, _chat_id_str if thread_id_str else None)
- return MessageEvent(
- text=expand_link_entities(message), message_type=msg_type, source=source, raw_message=message,
- message_id=str(message.message_id), platform_update_id=update_id,
- reply_to_message_id=reply_to_id, reply_to_text=reply_to_text, auto_skill=topic_skill,
- channel_prompt=group_identity_prompt(self, message, channel_prompt), reply_expected=group_reply_expected(self, message, chat_type),
- edit_revision=self._edit_revision(message, update_id), timestamp=message.date)
- @staticmethod
- def _edit_revision(message, update_id: Optional[int]) -> Optional[str]:
- """Identity of one edit of a message, None for a new send. ``edit_date`` has whole-second
- resolution, so two edits in one second would share it: the update id is unique per edit and
- stable on redelivery; without one, the edited text tells same-second edits apart."""
- edit_date = getattr(message, "edit_date", None)
- if not hasattr(edit_date, "timestamp"):
- return None
- if update_id is not None:
- return f"u{update_id}"
- text = (getattr(message, "text", None) or getattr(message, "caption", None) or "").encode()
- return f"{int(edit_date.timestamp())}.{hashlib.sha256(text).hexdigest()[:16]}"
- # -- Message reactions (processing lifecycle) --
- def _reactions_enabled(self) -> bool:
- """Reactions: scoped ``TELEGRAM_REACTIONS`` → ``extra.reactions`` (YAML, per profile) → off.
- An explicit env var wins over YAML, so the stock ``reactions: false`` every install
- materializes cannot silently kill a documented ``TELEGRAM_REACTIONS=true`` (#109032). Under
- multiplex a scoped miss falls to the profile's own YAML, never another profile's env (#72348).
- """
- configured = _extra_or_secret(self.config.extra, "reactions", "TELEGRAM_REACTIONS", None)
- if configured is None:
- return False
- return str(configured).lower() not in {"false", "0", "no"}
- async def _set_reaction(self, chat_id: str, message_id: str, emoji: Optional[str]) -> bool:
- """Set a single emoji reaction (``None`` clears all bot-set reactions, the documented Bot API way)."""
- if not self._bot:
- return False
- try:
- await self._bot.set_message_reaction(chat_id=normalize_telegram_chat_id(chat_id), message_id=int(message_id), reaction=emoji)
- return True
- except Exception as e:
- if emoji is None:
- logger.debug("[%s] clear reactions failed: %s", self.name, _redact_telegram_error_text(e))
- else:
- logger.debug("[%s] set_message_reaction failed (%s): %s", self.name, emoji, _redact_telegram_error_text(e))
- return False
- async def _clear_reactions(self, chat_id: str, message_id: str) -> bool:
- """Clear all bot-set reactions."""
- return await self._set_reaction(chat_id, message_id, None)
- async def on_processing_start(self, event: MessageEvent) -> None:
- """Add an in-progress reaction when message processing begins."""
- if not self._reactions_enabled():
- return
- chat_id = getattr(event.source, "chat_id", None)
- message_id = getattr(event, "message_id", None)
- if chat_id and message_id:
- await self._set_reaction(chat_id, message_id, "\U0001f440")
- async def on_processing_complete(self, event: MessageEvent, outcome: ProcessingOutcome) -> None:
- """Swap the in-progress reaction for a final success/failure reaction (set_message_reaction
- replaces, not adds); CANCELLED explicitly clears the 👀."""
- if not self._reactions_enabled():
- return
- chat_id = getattr(event.source, "chat_id", None)
- message_id = getattr(event, "message_id", None)
- if not (chat_id and message_id):
- return
- if outcome == ProcessingOutcome.CANCELLED:
- await self._clear_reactions(chat_id, message_id)
- else:
- await self._set_reaction(chat_id, message_id, "\U0001f44d" if outcome == ProcessingOutcome.SUCCESS else "\U0001f44e")
- # -- Plugin registration glue: register(ctx) plus the hook implementations (adapter factory, YAML→env/extra
- # config, setup wizard, standalone sender).
- # ────────────────────────────────────────────────────────────────────────── Plugin migration glue (#41112 /
- # #3823) Added when the Telegram adapter (+ its telegram_network satellite) moved from gateway/platforms/
- # into this bundled plugin. Mirrors the Discord (#24356) / Slack migrations: a register(ctx) entry point
- # plus hook implementations that replace the per-platform core touchpoints (the Platform.TELEGRAM branch in
- # gateway/run.py, the telegram_cfg YAML→env/extra block in gateway/config.py, the _setup_telegram wizard +
- # _PLATFORMS["telegram"] static dict in hermes_cli/{setup,gateway}.py, and the _send_telegram dispatch in
- # tools/send_message_tool.py). Telegram uses the generic token connected check, so no is_connected override
- # is needed. ──────────────────────────────────────────────────────────────────────────
- def _resolve_notifications_mode() -> str:
- """Notification mode (all/important) from env, else config.yaml display.platforms.telegram.notifications."""
- mode = os.getenv("HERMES_TELEGRAM_NOTIFICATIONS", "")
- if not mode:
- try:
- from gateway.config import load_gateway_config
- from gateway.run import cfg_get
- _raw = cfg_get(load_gateway_config(), "display", "platforms", "telegram", "notifications")
- if _raw not in {None, ""}:
- mode = str(_raw).strip().lower()
- except Exception:
- pass
- mode = mode or "important"
- if mode not in {"all", "important"}:
- logger.warning("Unknown telegram notifications mode '%s', defaulting to 'important' (valid: all, important)", mode)
- mode = "important"
- return mode
- def _build_adapter(config):
- """Construct TelegramAdapter and apply the notification mode."""
- adapter = TelegramAdapter(config)
- try:
- adapter._notifications_mode = _resolve_notifications_mode()
- except Exception:
- adapter._notifications_mode = "important"
- return adapter
- def _is_connected(config) -> bool:
- """Connected when a bot token is configured (env or PlatformConfig.token); the SDK being importable is
- not enough or the plugin-enable pass would enable Telegram on any machine with it installed."""
- token = getattr(config, "token", None)
- if not token:
- import hermes_cli.gateway as gateway_mod
- token = gateway_mod.get_env_value("TELEGRAM_BOT_TOKEN") or ""
- return bool(str(token).strip())
- async def _standalone_send(pconfig, chat_id, message, *, thread_id=None, media_files=None, force_document=False):
- """Out-of-process delivery (standalone_sender_fn) so deliver=telegram cron jobs succeed without the
- gateway; delegates to the REST ``_send_telegram`` sender."""
- token = getattr(pconfig, "token", None)
- if not token:
- from agent.secret_scope import get_secret # profile-scoped: never borrow another profile's token
- token = get_secret("TELEGRAM_BOT_TOKEN", "") or ""
- disable_link_previews = bool(getattr(pconfig, "extra", {}) and pconfig.extra.get("disable_link_previews"))
- from tools.send_message_tool import _send_telegram
- return await _send_telegram(
- token, chat_id, message, media_files=media_files, thread_id=thread_id,
- disable_link_previews=disable_link_previews, force_document=force_document)
- def interactive_setup() -> None:
- """Configure Telegram credentials and allowlist via the CLI setup wizard (lazy import)."""
- setup_platforms._setup_telegram()
- def _apply_yaml_config(yaml_cfg: dict, telegram_cfg: dict) -> dict | None:
- """Translate config.yaml telegram: keys into TELEGRAM_* env vars and PlatformConfig.extra. Env vars
- take precedence over YAML. Returns extras to merge into PlatformConfig.extra, or None.
- Implements the apply_yaml_config_fn contract (#24849). Mirrors the legacy telegram_cfg block from
- gateway/config.py::load_gateway_config().
- """
- import json as _json
- from gateway.platforms._shared import yaml_env_setter
- extras: dict = {}
- # Under multiplex a secondary profile's settings must NOT hit the process-global env (first-writer-wins
- # would pin them for every profile, #72348); yaml_env_setter skips the write under its scope and the
- # values flow via extra/secret scope instead.
- _set_env = yaml_env_setter()
- def _bridge_lower(key: str, env: str) -> None:
- if key in telegram_cfg:
- extras.setdefault(key, telegram_cfg[key])
- _set_env(env, str(telegram_cfg[key]).lower())
- def _bridge_gate(key: str, env: str, value: Any, *, seed_extra: bool = False) -> None:
- """CSV allowlist gate: list → comma-joined; env write skipped under multiplex secret scope."""
- if value is None:
- return
- if seed_extra:
- extras.setdefault(key, value)
- _set_env(env, value)
- for _topic_key in ("disable_topic_auto_rename", "dm_topic_auto_transfer"):
- if _topic_key in telegram_cfg:
- extras.setdefault(_topic_key, telegram_cfg[_topic_key])
- _effective_rm = telegram_cfg.get("require_mention", yaml_cfg.get("require_mention"))
- if _effective_rm is not None:
- _set_env("TELEGRAM_REQUIRE_MENTION", str(_effective_rm).lower())
- if "mention_patterns" in telegram_cfg:
- _set_env("TELEGRAM_MENTION_PATTERNS", _json.dumps(telegram_cfg["mention_patterns"]))
- for key, env in (
- ("exclusive_bot_mentions", "TELEGRAM_EXCLUSIVE_BOT_MENTIONS"), ("allow_bots", "TELEGRAM_ALLOW_BOTS"),
- ("bots_require_mention", "TELEGRAM_BOTS_REQUIRE_MENTION"),
- ("guest_mode", "TELEGRAM_GUEST_MODE", ), ("observe_unmentioned_group_messages", "TELEGRAM_OBSERVE_UNMENTIONED_GROUP_MESSAGES")):
- _bridge_lower(key, env)
- # No extras seed for allowed_chats / allowed_topics / group_allowed_chats: the shared-key loop already
- # bridges them with their original type and this merge would clobber it.
- for key, env, seed in (
- ("free_response_chats", "TELEGRAM_FREE_RESPONSE_CHATS", True), ("free_response_topics", "TELEGRAM_FREE_RESPONSE_TOPICS", False),
- ("allowed_chats", "TELEGRAM_ALLOWED_CHATS", False), ("allowed_topics", "TELEGRAM_ALLOWED_TOPICS", False),
- ("ignored_threads", "TELEGRAM_IGNORED_THREADS", True)):
- _bridge_gate(key, env, telegram_cfg.get(key), seed_extra=seed)
- _bridge_lower("reactions", "TELEGRAM_REACTIONS")
- if "proxy_url" in telegram_cfg:
- # Seeded into extra so ``_build_ptb_requests`` keeps a secondary's route without the env bridge.
- extras.setdefault("proxy_url", str(telegram_cfg["proxy_url"]).strip())
- _set_env("TELEGRAM_PROXY", str(telegram_cfg["proxy_url"]).strip())
- _telegram_extra = telegram_cfg.get("extra") if isinstance(telegram_cfg.get("extra"), dict) else {}
- _telegram_rtm = telegram_cfg["reply_to_mode"] if "reply_to_mode" in telegram_cfg else _telegram_extra.get("reply_to_mode")
- if _telegram_rtm is not None:
- _set_env("TELEGRAM_REPLY_TO_MODE", "off" if _telegram_rtm is False else str(_telegram_rtm).lower())
- _bridge_gate("allow_from", "TELEGRAM_ALLOWED_USERS", telegram_cfg.get("allow_from"))
- _bridge_gate("group_allow_from", "TELEGRAM_GROUP_ALLOWED_USERS", telegram_cfg.get("group_allow_from") or _telegram_extra.get("group_allow_from"))
- _bridge_gate(
- "group_allowed_chats", "TELEGRAM_GROUP_ALLOWED_CHATS",
- telegram_cfg.get("group_allowed_chats") or _telegram_extra.get("group_allowed_chats"))
- for _key in ("guest_mode", "disable_link_previews", "observe_unmentioned_group_messages", "free_response_topics"):
- if _key in telegram_cfg:
- extras.setdefault(_key, telegram_cfg[_key])
- # Pass through telegram-specific extra keys but EXCLUDE generic shared-config keys: _merge_platform_map
- # already applied top-level-over-nested precedence and re-emitting them via dict.update() would undo it.
- _GENERIC_MERGE_KEYS = {
- "reply_prefix", "reply_in_thread", "reply_to_mode", "unauthorized_dm_behavior", "notice_delivery",
- "require_mention", "channel_skill_bindings", "channel_prompts", "gateway_restart_notification", "allow_from",
- "allow_admin_from", "dm_policy", "group_policy"}
- for _k, _v in _telegram_extra.items():
- if _k not in _GENERIC_MERGE_KEYS:
- extras.setdefault(_k, _v)
- return extras or None
- def register(ctx) -> None:
- """Plugin entry point — called by the Hermes plugin system."""
- from .adapter_hint import PLATFORM_HINT
- from .dashboard_card import DASHBOARD_CARD
- from . import gateway_env # env -> GatewayConfig bridge (gateway.config_env)
- ctx.register_platform(
- name="telegram", label="Telegram", adapter_factory=_build_adapter, check_fn=telegram_deps_present,
- **DASHBOARD_CARD,
- ensure_deps_fn=check_telegram_requirements, is_connected=_is_connected, required_env=["TELEGRAM_BOT_TOKEN"],
- install_hint=t("platform.telegram.install_hint"), setup_fn=interactive_setup, apply_yaml_config_fn=_apply_yaml_config, cron_deliver_env_var="TELEGRAM_HOME_CHANNEL",
- standalone_sender_fn=_standalone_send, max_message_length=4096, emoji="✈️", allow_update_command=True,
- platform_hint=PLATFORM_HINT,
- display_tier="high",
- # Mobile inbox: quiet tool_progress / busy-ack, but keep interim commentary and heartbeats.
- display_overrides={"tool_progress": "off", "busy_ack_detail": False},
- target_ref_syntax_fn=parse_send_target,
- env_overrides=gateway_env.ENV_OVERRIDES,
- env_enable_credentials=gateway_env.ENV_ENABLE_CREDENTIALS, **gateway_env.AUTHZ,
- )
|