conversation_compression.py 244 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439
  1. """Context compression: feasibility probe, warning replay, compress, image fix.
  2. Thread-safety contract for extension points: with ``compression.context_timeout_seconds > 0`` (default) the
  3. whole pass, context engines and memory providers included, runs on a pooled daemon thread. Calls may arrive
  4. on any pooled thread (never rely on thread-affinity/locals); the message list is a private deep snapshot
  5. (in-place mutation is invisible to the live conversation unless the pass commits); state is published ONLY
  6. on an admitted :class:`CompressionCommitFence` commit (work still running after a host timeout is discarded);
  7. one pass per session at a time (durable lock) but sessions run concurrently, so shared engines must be thread-safe.
  8. """
  9. from __future__ import annotations
  10. import concurrent.futures
  11. import contextlib
  12. import contextvars
  13. import copy
  14. import dataclasses
  15. import inspect
  16. import json
  17. import logging
  18. import math
  19. import os
  20. import tempfile
  21. import time
  22. import uuid
  23. import threading
  24. from datetime import datetime
  25. from pathlib import Path
  26. from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
  27. from agent.auxiliary_client import AuxiliaryExplicitCancellation
  28. from agent.context_compressor_summary import _accepts_keyword_argument
  29. from agent.context_engine import automatic_compaction_status_message, sanitize_memory_context
  30. from agent.conversation_compression_codex import _compress_context_via_codex_app_server
  31. from agent.conversation_compression_deadline import resolve_context_compression_timeouts
  32. from agent.conversation_compression_telemetry import (
  33. _emit_aborted_attempt_telemetry, _emit_blocked_attempt_telemetry, _emit_bypassed_attempt_telemetry,
  34. _emit_compression_attempt_telemetry, _record_committed_attempt_effect, _snapshot_compression_effect_input,
  35. )
  36. from agent.memory_provider import PRE_COMPRESS_CHECKPOINT_API_VERSION
  37. from agent.model_metadata import estimate_messages_tokens_rough, estimate_request_tokens_rough
  38. from agent.session_activity import ActivityProvenance, normalize_activity_provenance
  39. from agent.usage_anchor import set_usage_anchor
  40. from hermes_state_ids import new_session_id as mint_session_id
  41. from hermes_state_pidns import holder_namespace_token
  42. from agent.conversation_compression_watermark import held_archive_watermark
  43. logger = logging.getLogger(__name__)
  44. @contextlib.contextmanager
  45. def _swallow(message: str, *, exc_info: bool = False):
  46. """Run a best-effort block; on Exception log ``message`` at DEBUG and continue."""
  47. try:
  48. yield
  49. except Exception as exc:
  50. logger.debug(message, exc_info=True) if exc_info else logger.debug(message, exc)
  51. # Terminal outcomes from host/hygiene timeout or cooldown writers. Detached heartbeat workers must not clobber these
  52. # (timeout unobservable). Seeing one latches the heartbeat silent so a later UNKNOWN rewrite can't re-arm a zombie.
  53. _TERMINAL_COMPRESSION_PROVENANCES = frozenset(
  54. {ActivityProvenance.AGENT_COMPRESSION_TIMEOUT, ActivityProvenance.AGENT_COMPRESSION_COOLDOWN}
  55. )
  56. # Split failures are usually transient lease/DB conditions, so use the FIRST
  57. # timeout-ladder rung (60s), not the 600s summary-provider cooldown.
  58. # Cooldown armed when a compression SPLIT fails (session_split_failed / rotation rollback, #97948 symptom
  59. # B).
  60. _SPLIT_FAILURE_COOLDOWN_SECONDS = 60
  61. # Marker tui_gateway/server.py::_status_update matches to tag kind="compacting" for drivers' "Summarizing…" UI. Keep
  62. # the phrase intact when rewording. Idle/preflight/retry lines lack it; is_compaction_progress_status covers those.
  63. COMPACTION_STATUS_MARKER = "Compacting context"
  64. COMPACTION_STATUS = f"🗜️ {COMPACTION_STATUS_MARKER} — summarizing earlier conversation so I can continue..."
  65. # Periodic heartbeat re-emitted while a long compression is still running so remote transports with
  66. # idle-turn watchdogs (#98371) see progress. Same marker as COMPACTION_STATUS so consumers classify it alike.
  67. COMPACTION_HEARTBEAT_STATUS = f"🗜️ {COMPACTION_STATUS_MARKER} — still summarizing earlier conversation so I can continue..."
  68. COMPACTION_DONE_STATUS = "✓ Context compaction complete — continuing turn..."
  69. def _strip_marker_for_comparison(msgs: Any) -> Any:
  70. """Copy ``msgs`` with the ``_db_persisted`` marker removed for no-op comparison.
  71. Live dicts carry the marker while ``compress()`` output is swept, so a raw ``==`` would misclassify an
  72. identical no-op copy as progress. Non-list inputs and non-dict entries pass through unchanged."""
  73. from agent.context_compressor import _DB_PERSISTED_MARKER
  74. if not isinstance(msgs, list):
  75. return msgs
  76. return [{k: v for k, v in m.items() if k != _DB_PERSISTED_MARKER} if isinstance(m, dict) else m for m in msgs]
  77. def _emit_compaction_done(agent: Any) -> None:
  78. """Emit the structured terminal edge for a started compaction."""
  79. status_callback = getattr(agent, "status_callback", None)
  80. if not status_callback:
  81. return
  82. with _swallow('status_callback error in compaction completion', exc_info=True):
  83. status_callback("compacted", COMPACTION_DONE_STATUS)
  84. # Every ROUTINE compression status line lives here: suppressed on chat platforms
  85. # by _TELEGRAM_NOISY_STATUS_RE (gateway/run.py); update that regex + telegram
  86. # noise test when rewording. Failure notices and /compress feedback: NOT here.
  87. PRE_API_COMPRESSION_STATUS_TEMPLATE = (
  88. "📦 Pre-API compression: ~{tokens:,} tokens near the context/output limit. Compacting before the next model call."
  89. )
  90. PREFLIGHT_COMPRESSION_STATUS_TEMPLATE = (
  91. "📦 Preflight compression: ~{tokens:,} tokens >= {threshold:,} threshold. This may take a moment."
  92. )
  93. IDLE_COMPACTION_STATUS_TEMPLATE = (
  94. "💤 Resumed after {idle_seconds}s idle — compacting ~{tokens:,} tokens before continuing."
  95. )
  96. COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE = (
  97. "🗜️ Context too large (~{tokens:,} tokens) — compressing ({attempt}/{cap})..."
  98. )
  99. COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE = "🗜️ Compressed {before} → {after} messages, retrying..."
  100. COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE = "🗜️ Compressed ~{before:,} → ~{after:,} tokens, retrying..."
  101. COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE = (
  102. "🗜️ Context reduced to {new_ctx:,} tokens (was {old_ctx:,}), retrying..."
  103. )
  104. # FAILURE-class notice: compression blocked, so the session grows until the provider limit kills it. Must stay visible
  105. # on gateways: never add it to ROUTINE_COMPRESSION_STATUS_SAMPLES or _TELEGRAM_NOISY_STATUS_RE.
  106. # FAILURE-CLASS notice — a deliberate carve-out from routine-compression silence (#16775 class): the context
  107. # is over the compression threshold but compression is blocked (summary-LLM cooldown / anti-thrash breaker),
  108. # so the session will keep growing until the hard provider token limit kills it. Do NOT add it to
  109. # ROUTINE_COMPRESSION_STATUS_SAMPLES or the gateway noise regex (_TELEGRAM_NOISY_STATUS_RE); it is pinned
  110. # un-swallowed in tests/gateway/test_telegram_noise_filter.py::VISIBLE_COMPRESSION_MESSAGES.
  111. CONTEXT_OVERFLOW_BLOCKED_WARNING_TEMPLATE = (
  112. "⚠ Context is over the compression threshold (~{tokens:,} tokens >= {threshold:,}) "
  113. "but compression is currently blocked ({reason}). The model may stop responding. Run /new to start a fresh "
  114. "session or /compress to retry immediately."
  115. )
  116. # Formatted from the same constants the emission sites use, so noise-filter tests exercise the ACTUAL wording.
  117. ROUTINE_COMPRESSION_STATUS_SAMPLES = (
  118. COMPACTION_STATUS, COMPACTION_HEARTBEAT_STATUS, COMPACTION_DONE_STATUS,
  119. PRE_API_COMPRESSION_STATUS_TEMPLATE.format(tokens=123456),
  120. PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(tokens=120000, threshold=100000),
  121. IDLE_COMPACTION_STATUS_TEMPLATE.format(idle_seconds=3600, tokens=120000),
  122. COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE.format(tokens=250000, attempt=1, cap=3),
  123. COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE.format(before=30, after=12),
  124. COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE.format(before=250000, after=120000),
  125. COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE.format(new_ctx=120000, old_ctx=250000),
  126. )
  127. def is_compaction_progress_status(text: str | None) -> bool:
  128. """True for in-progress auto-compaction lifecycle lines (not the done edge).
  129. The gateway re-tags matches as ``kind="compacting"`` for the whole pause; matching only the marker left
  130. idle/preflight/retry lines looking hung. ``COMPACTION_DONE_STATUS`` is emitted as ``kind="compacted"`` and
  131. must not match here."""
  132. body = text.strip() if isinstance(text, str) else ""
  133. if not body:
  134. return False
  135. if COMPACTION_STATUS_MARKER in body:
  136. return True
  137. if body == COMPACTION_DONE_STATUS:
  138. return False
  139. lowered = body.lower()
  140. # The failure-class overflow warning mentions compression but is a blocked notice, not progress.
  141. if "compaction complete" in lowered or "compression is currently blocked" in lowered:
  142. return False
  143. return "compact" in lowered or "compress" in lowered or "context reduced to" in lowered
  144. def _refresh_agent_tool_definitions(agent) -> bool:
  145. """Rebuild agent.tools at the compaction commit boundary (the only moment config reaches a forever-session's
  146. frozen tool schemas; the prompt cache is already invalid). Returns True when tools were added."""
  147. from agent.runtime_context import current
  148. if current() is not None:
  149. return False
  150. from tools.mcp_tool_agent import refresh_agent_mcp_tools
  151. added = refresh_agent_mcp_tools(agent, content_aware=True)
  152. if added:
  153. logger.info("Compaction tool refresh added tools: %s", sorted(added))
  154. return bool(added)
  155. _COMPRESSOR_ATTEMPT_STATE_FIELDS = (
  156. "_previous_summary", "_summary_has_user_turn", "compression_count", "_last_compression_savings_pct",
  157. "_ineffective_compression_count", "_anti_thrash_recovery_deadline", "_fallback_compression_streak",
  158. "_verify_compaction_cleared_threshold", "_last_compression_made_progress", "_summary_failure_cooldown_until",
  159. "_cooldown_persist_failed", "_last_summary_error", "_consecutive_timeout_failures", "_consecutive_truncation_failures",
  160. "_last_summary_dropped_count",
  161. "_last_summary_fallback_used", "_last_compress_aborted", "_last_summary_auth_failure",
  162. "_last_summary_network_failure", "_last_summary_empty_content_failure", "_last_summary_truncated_failure",
  163. "_last_summary_overload_failure", "_consecutive_overload_aborts", "_last_summary_overload_degraded",
  164. "_last_aux_model_failure_error", "_last_aux_model_failure_model", "_last_aux_resolved_model",
  165. "_summary_model_fallen_back", "summary_model",
  166. "_last_compression_telemetry", "_active_compression_telemetry", "_compression_telemetry_seed",
  167. "_proactive_prune_rearm_tokens",
  168. )
  169. _COMPRESSOR_COOLDOWN_STATE_FIELDS = ("_summary_failure_cooldown_until", "_last_summary_error", "_cooldown_persist_failed")
  170. def _snapshot_compressor_attempt_state(compressor: Any) -> dict[str, Any]:
  171. """Copy only the mutable bookkeeping owned by one compression attempt.
  172. The allow-list avoids copying clients, DB handles, locks and plugin resources; missing fields are ignored
  173. so legacy/third-party compressors keep working."""
  174. try:
  175. values = vars(compressor)
  176. except TypeError:
  177. return {}
  178. selected = {name: values[name] for name in _COMPRESSOR_ATTEMPT_STATE_FIELDS if name in values}
  179. # Copy the collection as one object so aliases between fields (notably
  180. # _active_compression_telemetry and _last_compression_telemetry) survive.
  181. return copy.deepcopy(selected)
  182. # Attempt ownership: stall-fallback detaches a timed-out worker and reuses the compressor, so its late unwind could
  183. # restore a stale snapshot or clear the fallback's cancel check. Generation guards ATTRIBUTE writes; fence, COMMITs.
  184. # --------------------------------------------------------------------------- Attempt ownership (#96634
  185. # follow-up). The stall-fallback path deliberately DETACHES a timed-out primary worker (fence cancel wins;
  186. # the future stays on the shared pool) and immediately starts a fallback attempt against the SAME
  187. # ContextCompressor. Two races follow from that overlap: 1. The late primary's unwind still calls
  188. # _restore_compressor_attempt_state with the PRIMARY's pre-attempt snapshot. Landing after the fallback's
  189. # commit, it rolls _previous_summary / cooldown / provenance / telemetry back to pre-primary values —
  190. # silently discarding fallback-owned state. 2. _compression_cancelled_check is one shared attribute: the
  191. # late primary's ``finally`` clears the callback the fallback just installed, so the fallback's F4
  192. # cancellation consult reads None. Both are fixed with a monotonic per-compressor attempt generation,
  193. # claimed under one module lock. Restores and callback set/clear are keyed to the claiming generation and
  194. # no-op when a newer attempt owns the compressor. The commit fence still owns COMMIT admission; the
  195. # generation owns compressor-ATTRIBUTE writes — two different boundaries.
  196. # ---------------------------------------------------------------------------
  197. _COMPRESSOR_ATTEMPT_LOCK = threading.Lock()
  198. # The calling attempt's generation rides a ContextVar (not a compressor attribute) so compressor code
  199. # deep in the call stack can tell ITS OWN attempt apart from whichever attempt currently owns the
  200. # compressor. A shared attribute can only answer "who owns now", never "am I stale". Set/reset inside
  201. # _run_summary_dispatch around compress_fn, which always runs in the calling attempt's own thread, so
  202. # worker and fallback threads each see their own generation. Callers outside the dispatch machinery
  203. # (manual compress, legacy paths) read None and keep unguarded historical behavior.
  204. _COMPRESSOR_ATTEMPT_GENERATION: contextvars.ContextVar[Any] = contextvars.ContextVar(
  205. "hermes_compressor_attempt_generation", default=None
  206. )
  207. def _compressor_attempt_serial_lock(compressor: Any) -> Any:
  208. """Per-compressor lock serializing the durable cooldown rollback against claims. The process-wide
  209. claim lock must stay cheap (every compressor in a gateway shares it), so the slow SQLite write in
  210. ``_restore_compressor_attempt_state`` is fenced by THIS lock instead; ``_claim_compressor_attempt``
  211. takes it first so a claim on that compressor waits for the restore while other compressors proceed.
  212. A slotted/frozen compressor that cannot hold the attribute gets a no-op (its guard is off anyway)."""
  213. with _COMPRESSOR_ATTEMPT_LOCK:
  214. lock = getattr(compressor, "_compression_attempt_serial_lock", None)
  215. if lock is None:
  216. lock = threading.Lock()
  217. try:
  218. compressor._compression_attempt_serial_lock = lock
  219. except Exception:
  220. return contextlib.nullcontext()
  221. return lock
  222. def _claim_compressor_attempt(compressor: Any) -> int:
  223. """Claim the compressor for a new attempt; return its monotonic generation id.
  224. Restores or cancelled-check mutations stamped with an OLDER generation no-op, so a detached late attempt
  225. cannot clobber its successor's state."""
  226. with _compressor_attempt_serial_lock(compressor), _COMPRESSOR_ATTEMPT_LOCK:
  227. generation = int(getattr(compressor, "_compression_attempt_generation", 0) or 0) + 1
  228. try:
  229. compressor._compression_attempt_generation = generation
  230. except Exception:
  231. return 0 # slotted/frozen compressor: gen 0 disables the guard (per-instance, so never mixed)
  232. return generation
  233. def _compressor_attempt_is_current(compressor: Any, generation: int) -> bool:
  234. """True when *generation* still owns the compressor (or guard disabled)."""
  235. if not generation:
  236. return True
  237. with _COMPRESSOR_ATTEMPT_LOCK:
  238. return int(getattr(compressor, "_compression_attempt_generation", 0) or 0) == generation
  239. def _mark_compressor_working_attempt(compressor: Any, generation: int) -> None:
  240. """Publish the generation of the attempt that is ACTUALLY running summary work.
  241. The entry claim is taken before the breaker gates and the per-session lock, so no-op
  242. entries (lock sit-outs, transient gates) bump ``_compression_attempt_generation``
  243. without doing any work. Candidate supersession must key on this separate marker,
  244. published only when the summary dispatch begins, or those no-op claims discard a
  245. completed candidate and compression livelocks. Slotted/frozen compressors that
  246. cannot hold the attribute keep the entry-generation check as a conservative fallback.
  247. """
  248. with _COMPRESSOR_ATTEMPT_LOCK:
  249. with contextlib.suppress(Exception):
  250. compressor._compression_working_attempt_generation = generation
  251. def _raise_if_stale_attempt(compressor: Any) -> None:
  252. """Unwind the CALLING attempt (its generation rides the ContextVar) as a cancellation when a newer
  253. attempt has since begun summary work on *compressor*, so none of the shared-state writes that
  254. follow the call site can land."""
  255. if not _caller_attempt_is_current(compressor):
  256. raise AuxiliaryExplicitCancellation()
  257. def _caller_attempt_is_current(compressor: Any) -> bool:
  258. """Working-attempt check for the calling attempt's own generation (ContextVar; None → unguarded)."""
  259. return _working_attempt_is_current(compressor, _COMPRESSOR_ATTEMPT_GENERATION.get())
  260. def _working_attempt_is_current(compressor: Any, generation: Any) -> bool:
  261. """True when *generation* is still the last attempt that began summary work.
  262. Without a published marker (attribute-less compressor, or the attempt never reached
  263. dispatch) supersession falls back to the entry-generation ownership check; a compressor
  264. that no attempt has ever claimed cannot have been superseded."""
  265. if not generation:
  266. return True
  267. with _COMPRESSOR_ATTEMPT_LOCK:
  268. marker = getattr(compressor, "_compression_working_attempt_generation", None)
  269. if marker is None:
  270. entry_generation = int(getattr(compressor, "_compression_attempt_generation", 0) or 0)
  271. return not entry_generation or entry_generation == generation
  272. return int(marker) == int(generation)
  273. def _install_compression_cancelled_check(compressor: Any, check: Any, generation: int) -> None:
  274. """Install the F4 cancellation consult, stamped with its owner attempt."""
  275. with _COMPRESSOR_ATTEMPT_LOCK:
  276. with contextlib.suppress(Exception):
  277. compressor._compression_cancelled_check = check
  278. compressor._compression_cancelled_check_owner = generation
  279. def _clear_compression_cancelled_check_if_owner(compressor: Any, generation: int) -> bool:
  280. """Clear the cancellation consult only when *generation* installed it.
  281. Prevents a detached late primary from tearing down a newer fallback's callback. Returns True when cleared."""
  282. with _COMPRESSOR_ATTEMPT_LOCK:
  283. owner = getattr(compressor, "_compression_cancelled_check_owner", None)
  284. if owner is not None and generation and owner != generation:
  285. return False
  286. with contextlib.suppress(Exception):
  287. compressor._compression_cancelled_check = None
  288. compressor._compression_cancelled_check_owner = None
  289. return True
  290. def _rollback_durable_cooldown(
  291. compressor: Any, snapshot: dict[str, Any], authoritative: Optional[bool], durable_state: Optional[dict[str, Any]]
  292. ) -> None:
  293. """Recreate/clear the durable cooldown row from the attempt snapshot.
  294. Authoritative captures use the exact raw-row restore API (verifies read-back, propagates failure); the
  295. legacy path re-derives deadline/error best-effort."""
  296. session_db = vars(compressor).get("_session_db")
  297. session_id = vars(compressor).get("_session_id")
  298. if session_db is None or not session_id:
  299. return
  300. if authoritative is True:
  301. restorer = getattr(type(session_db), "restore_compression_failure_cooldown_row", None)
  302. if not callable(restorer) or durable_state is None:
  303. raise RuntimeError("exact compression cooldown rollback API is unavailable")
  304. restorer(session_db, session_id, copy.deepcopy(durable_state))
  305. return
  306. with _swallow('compression cooldown persistence rollback failed', exc_info=True):
  307. deadline = float(snapshot["_summary_failure_cooldown_until"] or 0.0)
  308. remaining = max(0.0, deadline - time.monotonic())
  309. if remaining > 0:
  310. recorder = getattr(type(session_db), "record_compression_failure_cooldown", None)
  311. if callable(recorder):
  312. recorder(session_db, session_id, time.time() + remaining, snapshot.get("_last_summary_error"))
  313. else:
  314. clearer = getattr(type(session_db), "clear_compression_failure_cooldown", None)
  315. if callable(clearer):
  316. clearer(session_db, session_id)
  317. def _restore_compressor_attempt_state(
  318. compressor: Any, snapshot: dict[str, Any], *, durable_cooldown_authoritative: Optional[bool] = None,
  319. durable_cooldown_state: Optional[dict[str, Any]] = None, attempt_generation: Optional[int] = None,
  320. ) -> None:
  321. """Restore the per-attempt snapshot after a pre-commit hard cancel.
  322. A restore stamped with a stale ``attempt_generation`` no-ops so a timed-out primary's late unwind cannot
  323. roll back state owned by the fallback attempt.
  324. ``attempt_generation`` (when provided) is the claim the calling attempt took via
  325. :func:`_claim_compressor_attempt`. See #96634.
  326. """
  327. if attempt_generation is not None and not _compressor_attempt_is_current(compressor, attempt_generation):
  328. logger.warning(
  329. "Skipping stale compressor attempt-state restore: attempt "
  330. "generation %s no longer owns the compressor (current: %s). A newer (stall-fallback) attempt's state is preserved.",
  331. attempt_generation, getattr(compressor, "_compression_attempt_generation", None),
  332. )
  333. return
  334. restored = copy.deepcopy(snapshot)
  335. # Re-validate AND run the durable rollback under this compressor's serial lock: the slow DB
  336. # write used to sit between the first ownership check and this re-check, so a fallback
  337. # claiming mid-restore could have its freshly written cooldown row overwritten by this
  338. # attempt's stale snapshot row. _claim_compressor_attempt takes the same per-compressor
  339. # lock, so the write is serialized against claims on THIS compressor without stalling
  340. # every other compressor behind the process-wide claim lock. The row still lands BEFORE
  341. # the in-memory restore so the next refresh cannot overwrite the rollback.
  342. with _compressor_attempt_serial_lock(compressor):
  343. with _COMPRESSOR_ATTEMPT_LOCK:
  344. lost = attempt_generation and int(getattr(compressor, "_compression_attempt_generation", 0) or 0) != attempt_generation
  345. if lost:
  346. logger.warning(
  347. "Skipping stale compressor attempt-state restore at write "
  348. "time: attempt generation %s lost the compressor mid-restore.", attempt_generation,
  349. )
  350. return
  351. # Success clears the durable cooldown pre-commit; recreate/clear that row BEFORE
  352. # restoring in-memory values or the next refresh overwrites the rollback. Never
  353. # turn unknown durable state / unpersisted local cooldowns into DB writes.
  354. if (
  355. "_summary_failure_cooldown_until" in snapshot
  356. and durable_cooldown_authoritative is not False
  357. and (durable_cooldown_authoritative is True or not bool(snapshot.get("_cooldown_persist_failed", False)))
  358. ):
  359. _rollback_durable_cooldown(compressor, snapshot, durable_cooldown_authoritative, durable_cooldown_state)
  360. for name, value in restored.items():
  361. setattr(compressor, name, value)
  362. def _capture_authoritative_cooldown_under_lease(
  363. compressor: Any, attempt_snapshot: dict[str, Any]
  364. ) -> tuple[Optional[bool], Optional[dict[str, Any]]]:
  365. """Refresh and snapshot built-in durable cooldown state under the lease.
  366. Third-party compressors are not invoked: plugin code must not run under the lease. Returns ``False`` on
  367. durable read failure (rollback must not mistake unknown state for an empty row) and ``None`` when the
  368. legacy API is absent."""
  369. try:
  370. from agent.context_compressor import ContextCompressor
  371. if not isinstance(compressor, ContextCompressor):
  372. return None, None
  373. values = vars(compressor)
  374. session_db = values.get("_session_db")
  375. session_id = values.get("_session_id")
  376. if session_db is None or not session_id:
  377. # Unbound compressors have no durable row to mutate or restore.
  378. return None, None
  379. raw_reader = getattr(type(session_db), "get_compression_failure_cooldown_row", None)
  380. if not callable(raw_reader):
  381. return False, None
  382. # Read the raw persisted row: the active getter filters expired rows and is not a lossless rollback snapshot.
  383. durable_state = raw_reader(session_db, session_id)
  384. if not isinstance(durable_state, dict):
  385. raise TypeError("raw compression cooldown snapshot must be a mapping")
  386. ContextCompressor.get_active_compression_failure_cooldown(compressor, refresh=True)
  387. except Exception as exc:
  388. logger.debug("authoritative compression cooldown capture failed: %s", exc)
  389. return False, None
  390. authoritative = getattr(compressor, "_last_cooldown_refresh_was_authoritative", None)
  391. if authoritative is not True:
  392. return authoritative, None
  393. values = vars(compressor)
  394. for name in _COMPRESSOR_COOLDOWN_STATE_FIELDS:
  395. if name in values:
  396. attempt_snapshot[name] = copy.deepcopy(values[name])
  397. return True, copy.deepcopy(durable_state)
  398. class CompressionCommitFence:
  399. """Fence timeout cancellation against post-summary session mutation.
  400. The sync worker thread cannot be killed; the fence makes the commit boundary deterministic: cancellation
  401. wins before mutation starts, or waits for an already-started commit to finish completely."""
  402. def __init__(self, total_ceiling_seconds: float | None = None) -> None:
  403. self._lock = threading.Lock()
  404. self._cancelled = False
  405. self._commit_started = False
  406. # Readable WITHOUT the lock (begin_commit holds it until finish_commit): hosts see a hung commit.
  407. # Lock-free commit-phase marker (#76354 review F1). ``begin_commit`` RETAINS ``self._lock`` until
  408. # ``finish_commit``, so any host-side observation that needs the lock (``try_cancel_before_commit``)
  409. # blocks/space-outs for the whole commit. This Event is set inside ``begin_commit`` while the lock
  410. # is held but is READABLE WITHOUT the lock, so a host can observe "a commit was admitted and may be
  411. # in flight" even while the commit itself is hung — which is exactly when the overrun warning must
  412. # be able to fire.
  413. self._commit_phase = threading.Event()
  414. # Set on ANY host unwind without the fence lock so FUTURE commits are blocked; bool store is atomic.
  415. # Lock-free admission revocation (#76354 review F2). Set by :meth:`revoke_commit_admission` on ANY
  416. # host unwind (KeyboardInterrupt, cancellation, unexpected exception) without touching the fence
  417. # lock, so a host that cannot afford to block behind an in-flight commit can still guarantee no
  418. # FUTURE commit is admitted.
  419. self._admission_revoked = False
  420. # Holder-scoped release published by the worker once it owns the durable lock (no ABA on a NEW holder).
  421. # Holder-qualified durable-lock release hook (#76354 review F4; transplanted from PR #71569 by
  422. # @ciabata-git). The worker publishes an idempotent, holder-scoped release callable once it owns the
  423. # durable compression lock; a timed-out host invokes it to free the lease without racing a NEW
  424. # holder (DB release is holder-qualified, so a stale release can never delete a replacement's row —
  425. # no ABA).
  426. self._lock_release_guard = threading.Lock()
  427. self._cancelled_lock_release: Optional[Callable[[], None]] = None
  428. self._cancelled_lock_release_requested = False
  429. # Touched per streamed token so waiters tell SLOW-but-alive from HUNG (no fixed wall-clock kill).
  430. self._last_progress = time.monotonic()
  431. self._progress_observed = False
  432. self._deadline: float | None = None
  433. self._retain_cancelled_lock_until_worker_done = False
  434. # Set once the active-row watermark is captured: later rows survive as tail, so hosts may keep admission.
  435. self._commit_watermark_fenced = False
  436. if total_ceiling_seconds is not None:
  437. self.set_total_ceiling_seconds(total_ceiling_seconds)
  438. def set_total_ceiling_seconds(self, seconds: float) -> None:
  439. """Arm the wall-clock deadline shared by the host and worker."""
  440. seconds = float(seconds)
  441. if seconds <= 0:
  442. raise ValueError("total compression ceiling must be positive")
  443. self._deadline = time.monotonic() + seconds
  444. def set_precommit_deadline_monotonic(self, deadline: float) -> None:
  445. """Arm one absolute pre-commit deadline shared across route attempts (primary + stall retry)."""
  446. deadline = float(deadline)
  447. if deadline <= 0:
  448. raise ValueError("compression deadline must be positive")
  449. self._deadline = deadline
  450. def extend_deadline(self, seconds: float) -> bool:
  451. """Push the armed deadline back by *seconds* (time llama-server spent reading the prompt). A cancelled
  452. attempt or a passed deadline stays final, so a late progress chunk can't revive it; while a commit or a
  453. cancellation holds the lock there is nothing to extend."""
  454. if seconds <= 0 or not self._lock.acquire(blocking=False):
  455. return False
  456. try:
  457. alive = self._deadline is not None and self._deadline > time.monotonic() and not self.is_cancelled
  458. if alive:
  459. self._deadline += seconds
  460. return alive
  461. finally:
  462. self._lock.release()
  463. def touch_progress(self) -> None:
  464. """Record forward progress (a streamed token); a bare float store is atomic, so no lock."""
  465. self._last_progress = time.monotonic()
  466. self._progress_observed = True
  467. @property
  468. def progress_observed(self) -> bool:
  469. """Whether semantic provider progress was reported for this attempt."""
  470. return self._progress_observed
  471. @property
  472. def deadline_exceeded(self) -> bool:
  473. deadline = self._deadline
  474. return deadline is not None and time.monotonic() >= deadline
  475. @property
  476. def deadline_monotonic(self) -> float | None:
  477. """Armed deadline (absolute monotonic); the worker's stream consumer stops when the host stops waiting.
  478. :meth:`set_total_ceiling_seconds` documents this deadline as "shared by the host and worker", but
  479. until #99692 only the host could read it — ``deadline_exceeded`` answers "is it past?" for a caller
  480. that is already polling, which is useless to a worker blocked inside a provider stream. Publishing
  481. the instant itself lets the worker's stream consumer stop at exactly the moment the host stops
  482. waiting (see ``auxiliary_client.aux_stream_deadline``).
  483. """
  484. return self._deadline
  485. def seconds_since_progress(self) -> float:
  486. """Seconds since the worker last reported forward progress."""
  487. return max(0.0, time.monotonic() - self._last_progress)
  488. def cancel_before_commit(self, cancel_event: Any = None) -> bool:
  489. """Cancel a pending commit (``True``), or block until an active commit finishes (``False``)."""
  490. with self._lock:
  491. if not self._commit_started:
  492. self._cancelled = True
  493. if cancel_event is not None:
  494. cancel_event.set()
  495. return not self._commit_started
  496. def try_cancel_before_commit(self) -> Optional[bool]:
  497. """Non-blocking :meth:`cancel_before_commit`; ``None`` while an active commit owns the fence."""
  498. if not self._lock.acquire(blocking=False):
  499. return None
  500. try:
  501. if not self._commit_started:
  502. self._cancelled = True
  503. return not self._commit_started
  504. finally:
  505. self._lock.release()
  506. def begin_commit(self, cancel_event: Any = None) -> bool:
  507. """Atomically admit commit unless a hard cancellation already won."""
  508. self._lock.acquire()
  509. if self.is_cancelled or self._admission_revoked or (cancel_event is not None and bool(cancel_event.is_set())):
  510. self._cancelled = True
  511. self._lock.release()
  512. if self._admission_revoked:
  513. # A revoke that lost the fence-lock race deferred its lease release; commit refused: release now.
  514. self.release_cancelled_compression_lock()
  515. return False
  516. self._commit_started = True
  517. # Set under the fence lock so commit_in_flight is never True for a commit that lost to cancellation.
  518. self._commit_phase.set()
  519. return True
  520. def finish_commit(self) -> None:
  521. """Leave a commit boundary entered by :meth:`begin_commit`."""
  522. self._commit_phase.clear()
  523. self._lock.release()
  524. if self._admission_revoked:
  525. # A revoke during THIS commit deferred its lease release (never free mid-mutation); release now.
  526. self.release_cancelled_compression_lock()
  527. @property
  528. def commit_in_flight(self) -> bool:
  529. """Lock-free read: an admitted commit is in progress (hosts reach the overrun loop on a hung commit)."""
  530. return self._commit_phase.is_set()
  531. @property
  532. def is_cancelled(self) -> bool:
  533. """True after cancellation won before the commit boundary."""
  534. return self._cancelled or self._admission_revoked or self.deadline_exceeded
  535. def retain_compression_lock_until_worker_done(self) -> None:
  536. """Prevent a timed-out live worker from overlapping a retry."""
  537. self._retain_cancelled_lock_until_worker_done = True
  538. def mark_commit_watermark_fenced(self) -> None:
  539. """Record a watermark-bounded commit (later rows survive as tail); a detached worker may keep admission.
  540. Called by the compression worker right after it captures ``get_active_message_watermark()`` under
  541. the durable compression lock (#75316/#87484). A watermark-fenced commit archives ONLY rows at or
  542. below the watermark; rows appended later — e.g. the user turn the host released at the turn-hold
  543. boundary (#97963) — are cloned as live concurrent tail. That is exactly the property a host needs
  544. before letting a detached worker keep its commit admission.
  545. """
  546. self._commit_watermark_fenced = True
  547. @property
  548. def commit_watermark_fenced(self) -> bool:
  549. """Lock-free read: the worker's commit is watermark-bounded."""
  550. return self._commit_watermark_fenced
  551. def allow_cancelled_lock_release(self) -> None:
  552. """Undo :meth:`retain_compression_lock_until_worker_done` once a bounded join proved the worker exited."""
  553. self._retain_cancelled_lock_until_worker_done = False
  554. def revoke_commit_admission(self) -> None:
  555. """Revoke FUTURE commit admission without blocking on the fence lock.
  556. An in-flight commit is never abandoned (``begin_commit`` re-checks the flag under the lock). The lease
  557. release must not run mid-commit: released now if the lock is free, else deferred to
  558. ``finish_commit``/refusal (holder-qualified)."""
  559. self._admission_revoked = True
  560. if self._lock.acquire(blocking=False):
  561. try:
  562. self.release_cancelled_compression_lock()
  563. finally:
  564. self._lock.release()
  565. # ── Holder-qualified durable-lease cancellation: release is DELETE WHERE
  566. # holder = ?, so a stale release can never free a NEW holder's lease (no ABA).
  567. # ── Holder-qualified durable-lease cancellation (#76354 F4) ────────── Transplanted from PR #71569
  568. # (@ciabata-git): the worker publishes an idempotent, holder-scoped release hook once it owns the
  569. # durable compression lock, and the host invokes it after winning cancellation. ABA safety comes from
  570. # SessionDB.release_compression_lock being holder-qualified (DELETE ... WHERE holder = ?), so a stale
  571. # release can never free a NEW holder's lease.
  572. def begin_lock_setup(self) -> bool:
  573. """Hold the fence across lock acquisition + release-hook publication so a timeout cannot win between."""
  574. self._lock.acquire()
  575. if self.is_cancelled or self._admission_revoked:
  576. self._lock.release()
  577. return False
  578. return True
  579. def finish_lock_setup(self) -> None:
  580. """Leave a lock setup boundary entered by :meth:`begin_lock_setup`."""
  581. self._lock.release()
  582. def register_cancelled_lock_release(self, release: Callable[[], None]) -> bool:
  583. """Publish the worker's holder-qualified release; if cleanup was already requested, run it and return True."""
  584. with self._lock_release_guard:
  585. self._cancelled_lock_release = release
  586. requested = self._cancelled_lock_release_requested
  587. if requested:
  588. release()
  589. return requested
  590. def clear_cancelled_lock_release(self, release: Callable[[], None]) -> None:
  591. """Forget ``release`` after the worker's normal cleanup finishes."""
  592. with self._lock_release_guard:
  593. if self._cancelled_lock_release is release:
  594. self._cancelled_lock_release = None
  595. def release_cancelled_compression_lock(self) -> None:
  596. """After cancellation won: release the worker's lock (a request ahead of hook publication is retained)."""
  597. if self._retain_cancelled_lock_until_worker_done:
  598. return
  599. with self._lock_release_guard:
  600. self._cancelled_lock_release_requested = True
  601. release = self._cancelled_lock_release
  602. if release is not None:
  603. release()
  604. # Unlike explicit_interrupt, a /stop after the stall window arms the durable backoff (no automatic re-entry).
  605. # Distinct from ``explicit_interrupt``: a /stop that arrived after the summary stream had already crossed
  606. # the no-progress stall window (#96775). Ordinary early /stop stays cooldown-neutral; this class arms the
  607. # durable backoff so the next automatic turn does not re-enter the same stalled strategy.
  608. STALL_INTERRUPTED_FAILURE_CLASS = "stall_interrupted"
  609. # Daemon pool so a fence-cancelled hung worker cannot block interpreter exit; never shut down per call.
  610. _compress_timeout_executor = None
  611. _compress_timeout_executor_lock = threading.Lock()
  612. # Overrun waits proceed in bounded slices so each window logs (escalating); clamped to ceiling for tiny values.
  613. _COMMIT_OVERRUN_WAIT_SLICE_SECONDS = 30.0
  614. # A worker exiting within the grace proves no provider call is in flight, so its lease may be released even
  615. # on the total-ceiling path; one that doesn't exit is orphaned behind the poison fence and keeps its lease.
  616. # Bounded grace given to a fence-cancelled compression worker to actually exit before the host moves on
  617. # (#97488).
  618. _CANCELLED_WORKER_TEARDOWN_GRACE_SECONDS = 5.0
  619. def _join_cancelled_worker(future: Any, grace_seconds: float) -> bool:
  620. """Best-effort bounded join of a fence-cancelled compression worker.
  621. Returns True when the future settled within ``grace_seconds`` (thread provably exited); False for a
  622. still-running worker, which the caller must treat as an orphan behind the poison fence."""
  623. grace = 0.0
  624. with contextlib.suppress(TypeError, ValueError):
  625. grace = max(float(grace_seconds), 0.0)
  626. try:
  627. future.result(timeout=grace)
  628. return True
  629. except concurrent.futures.TimeoutError:
  630. # Aliases builtin TimeoutError (3.11+): also raised when the worker DIED with a timeout-class
  631. # error. That worker has exited, so report it settled or the caller orphans the lease (#63892).
  632. return future.done()
  633. except concurrent.futures.CancelledError:
  634. # Never started; nothing can be in flight.
  635. return True
  636. except Exception:
  637. # The host already chose the fallback result; the fence keeps the failed attempt from touching state.
  638. logger.debug("cancelled compression worker exited with an exception", exc_info=True)
  639. return True
  640. # The executor queue is unbounded and a queued job would run stale, so admission is capped at the worker
  641. # count (fail fast, continue uncompressed). Slots free via done-callback; a never-returning worker loses one.
  642. # Bounded admission for the shared compress-timeout pool (#76354 review F6). The stdlib executor queue is
  643. # unbounded: with all four workers wedged in hung summaries, a fifth compression would queue silently, wait
  644. # out its whole timeout without ever starting, and remain eligible to run as a stale job whenever a worker
  645. # recovered. Admission is therefore capped at the worker count — when every worker slot is occupied (running
  646. # OR admitted-not-started) submission FAILS FAST and the caller continues without compression. Recovery
  647. # contract when all workers are wedged: new compressions fail fast (no queue growth, conversation continues
  648. # uncompressed, a warning is logged each attempt); wedged workers are fence-cancelled so they cannot publish
  649. # anything when they eventually return, and each recovery frees its admission slot via the future
  650. # done-callback, restoring normal service. If a worker NEVER returns, its slot is lost for the process
  651. # lifetime — bounded, observable degradation instead of an unbounded stale-job queue.
  652. _COMPRESS_EXECUTOR_MAX_WORKERS = 4
  653. _compress_admission_lock = threading.Lock()
  654. _compress_admitted_count = 0
  655. def _try_admit_compression_job() -> bool:
  656. """Reserve one bounded compression-pool admission slot (F6)."""
  657. global _compress_admitted_count
  658. with _compress_admission_lock:
  659. if _compress_admitted_count >= _COMPRESS_EXECUTOR_MAX_WORKERS:
  660. return False
  661. _compress_admitted_count += 1
  662. return True
  663. def _release_compression_admission(_future=None) -> None:
  664. """Free an admission slot (future done-callback or failed submit)."""
  665. global _compress_admitted_count
  666. with _compress_admission_lock:
  667. if _compress_admitted_count > 0:
  668. _compress_admitted_count -= 1
  669. def _get_compress_timeout_executor():
  670. """Return the process-wide compress-timeout DaemonThreadPoolExecutor."""
  671. global _compress_timeout_executor
  672. executor = _compress_timeout_executor
  673. if executor is not None:
  674. return executor
  675. from tools.daemon_pool import DaemonThreadPoolExecutor
  676. with _compress_timeout_executor_lock:
  677. if _compress_timeout_executor is None:
  678. # Small pool sized for live compress + cancelled workers winding down, not asyncio's min(32, cpu+4).
  679. _compress_timeout_executor = DaemonThreadPoolExecutor(
  680. max_workers=_COMPRESS_EXECUTOR_MAX_WORKERS, thread_name_prefix="compress-ctx-timeout"
  681. )
  682. return _compress_timeout_executor
  683. def compression_attempt_stalled(
  684. *, commit_fence: Optional[CompressionCommitFence], started_at: float, idle_timeout_seconds: Optional[float] = None
  685. ) -> bool:
  686. """Return whether a pre-commit cancel landed after the stall window.
  687. An early ``/stop`` stays cooldown-neutral; an interrupt after the inactivity budget counts as a stall so
  688. the next automatic turn does not blindly retry.
  689. When the fence (or, without a fence, the attempt clock) has already sat idle for the configured
  690. compression inactivity budget, the interrupt is a stalled attempt — the same condition the host timeout
  691. uses — and the next automatic turn must not blindly retry that strategy (#96775).
  692. """
  693. idle = idle_timeout_seconds
  694. if idle is None:
  695. idle, _ceiling = resolve_context_compression_timeouts()
  696. try:
  697. idle = float(idle)
  698. except (TypeError, ValueError):
  699. return False
  700. if idle <= 0:
  701. return False
  702. if commit_fence is not None:
  703. try:
  704. return float(commit_fence.seconds_since_progress()) >= idle
  705. except Exception:
  706. return False
  707. try:
  708. return (time.monotonic() - float(started_at)) >= idle
  709. except (TypeError, ValueError):
  710. return False
  711. def _stall_source_fingerprint(agent: Any, messages: Any, approx_tokens: Optional[int]) -> str:
  712. """Identity of the stalled source context + summary strategy."""
  713. compressor = getattr(agent, "context_compressor", None)
  714. model = getattr(compressor, "summary_model", None) or getattr(agent, "model", None) or ""
  715. n_messages = len(messages) if isinstance(messages, list) else 0
  716. tokens = 0
  717. with contextlib.suppress(TypeError, ValueError):
  718. tokens = int(approx_tokens or 0)
  719. return f"msgs={n_messages}:tokens={tokens}:model={model}"
  720. def _record_stall_interrupted_backoff(
  721. agent: Any, *, commit_fence: Optional[CompressionCommitFence], started_at: float, messages: Any,
  722. approx_tokens: Optional[int],
  723. ) -> bool:
  724. """Persist a stall-interrupted cooldown after snapshot restore.
  725. Must run *after* ``_restore_compressor_attempt_state`` so rollback cannot wipe the new row. Returns True
  726. when the backoff was recorded."""
  727. if not compression_attempt_stalled(commit_fence=commit_fence, started_at=started_at):
  728. return False
  729. compressor = getattr(agent, "context_compressor", None)
  730. # Same timeout cooldown ladder as summary-LLM timeouts (#62452): avoid re-burning the full idle budget
  731. # every turn.
  732. record = getattr(compressor, "record_timeout_failure", None)
  733. if not callable(record):
  734. return False
  735. error = f"{STALL_INTERRUPTED_FAILURE_CLASS}:{_stall_source_fingerprint(agent, messages, approx_tokens)}"
  736. try:
  737. record(error, failure_kind="stall_interrupted")
  738. except Exception:
  739. logger.debug("stall-interrupted compression cooldown persist failed", exc_info=True)
  740. return False
  741. logger.info(
  742. "Recorded stall-interrupted compression backoff (session=%s, %s)", getattr(agent, "session_id", None) or "none",
  743. error,
  744. )
  745. return True
  746. def resolve_compression_fallback_route() -> Optional[dict]:
  747. """Return the first usable ``auxiliary.compression.fallback_chain`` entry.
  748. The aux client applies the chain only from its exception handler, so a silent stall never reaches it; this
  749. pins the route onto one bounded retry instead. Only the first complete entry: if it errors, the aux
  750. client's own exception path walks the rest. ``None`` when none is usable (skip compression)."""
  751. try:
  752. from agent.auxiliary_client import _fallback_entry_api_key, _get_auxiliary_task_config
  753. chain = _get_auxiliary_task_config("compression").get("fallback_chain")
  754. except Exception:
  755. logger.debug("compression fallback_chain lookup failed", exc_info=True)
  756. return None
  757. if not isinstance(chain, list):
  758. return None
  759. for index, entry in enumerate(chain):
  760. if not isinstance(entry, dict):
  761. continue
  762. provider = str(entry.get("provider") or "").strip()
  763. model = str(entry.get("model") or "").strip()
  764. # Both are required to name a route. _resolve_fallback_entry applies
  765. # the same rule when the aux client walks this chain itself.
  766. if not provider or not model:
  767. continue
  768. try:
  769. api_key = _fallback_entry_api_key(entry)
  770. except Exception:
  771. logger.debug("compression fallback_chain[%d] api key resolution failed", index, exc_info=True)
  772. api_key = None
  773. from agent.auxiliary_client import _coerce_positive_timeout
  774. timeout = _coerce_positive_timeout(entry.get("timeout"))
  775. return {
  776. "label": f"fallback_chain[{index}]({provider})",
  777. "provider": provider,
  778. "model": model,
  779. "base_url": str(entry.get("base_url") or "").strip() or None,
  780. "api_key": api_key or None,
  781. "api_mode": str(entry.get("api_mode") or entry.get("transport") or "").strip() or None,
  782. "timeout": timeout,
  783. }
  784. return None
  785. def _stall_retry_routes(escalate_deterministic: bool) -> list:
  786. """Pinned routes for the stall retry, in order: the configured chain entry, then (only once a
  787. stall-class backoff has already burned a window in this session) the deterministic fallback summary."""
  788. routes = [route for route in (resolve_compression_fallback_route(),) if route is not None]
  789. if escalate_deterministic:
  790. from agent.context_compressor import DETERMINISTIC_SUMMARY_ROUTE
  791. routes.append(dict(DETERMINISTIC_SUMMARY_ROUTE))
  792. return routes
  793. def _prior_timeout_failures(agent: Any) -> int:
  794. """Timeout-class failures this session that no healthy summary has cleared yet (type-pinned)."""
  795. count = getattr(getattr(agent, "context_compressor", None), "_consecutive_timeout_failures", 0)
  796. return count if isinstance(count, int) and not isinstance(count, bool) else 0
  797. def request_exceeds_model_window(agent: Any, request_tokens: Any) -> Optional[bool]:
  798. """Whether a ~``request_tokens`` request cannot be sent at all (above the model's context window).
  799. ``None`` when either side is unknown (no compressor / unresolvable window / no estimate), so callers
  800. keep their conservative default instead of treating "unknown" as "fits"."""
  801. window = getattr(getattr(agent, "context_compressor", None), "context_length", None)
  802. if isinstance(window, bool) or not isinstance(window, int) or window <= 0:
  803. return None
  804. if isinstance(request_tokens, bool) or not isinstance(request_tokens, int):
  805. return None
  806. return request_tokens > window
  807. def _retry_compression_on_fallback_chain(
  808. *, worker: Callable[[CompressionCommitFence], tuple[list, str]], messages: list,
  809. system_prompt_fallback: Any, idle_timeout_seconds: float, total_ceiling_seconds: float,
  810. precommit_deadline_monotonic: float,
  811. on_commit_overrun: Optional[Callable[[float, float], None]] = None,
  812. on_timeout_cause: Optional[Callable[[bool, bool], None]] = None, telemetry_agent: Any = None,
  813. new_fence: Optional[Callable[[], CompressionCommitFence]] = None, escalate_deterministic: bool = False,
  814. ) -> Optional[tuple[list, str]]:
  815. """Re-run an aborted compression with the summary route pinned: once on the configured chain entry,
  816. then — when ``escalate_deterministic`` (a stall backoff already burned one idle window this session,
  817. #112420) — once with the summary LLM skipped so compress() commits its deterministic fallback summary.
  818. Returns ``(messages, system_prompt)`` on real compression, else ``None`` and the caller degrades as
  819. before. The entry's ``timeout`` sets the idle window. Re-runs the whole worker, so pre-compression
  820. callbacks must be idempotent.
  821. A summary-model retry spends only the primary's REMAINING pre-commit budget
  822. (``precommit_deadline_monotonic``, shared with the primary attempt), so a stalled primary plus a
  823. stalled fallback cannot each burn a full ceiling; when the budget is spent the retry is skipped. The
  824. entry's own ``timeout`` (when declared) still sets the retry's idle window (#62452), clamped to that
  825. remaining budget. The deterministic route makes no provider call, so it keeps its own budget: it is
  826. the over-window last resort (#114594) and must not be starved by the routes that stalled before it.
  827. Known limitation (accepted, #96634 review): the retry re-runs the COMPLETE worker, which repeats
  828. memory/plugin pre-compression callbacks. Built-in callbacks are idempotent (re-reads and overwrites of
  829. attempt-scoped state); third-party plugin callbacks are advised to be. Splitting the worker to resume
  830. mid-pipeline would couple this path to every host's callback ordering — deliberately out of scope.
  831. """
  832. # An explicit stop is not a stalled route. The retry worker would abort on
  833. # the same event anyway, but starting one at all makes /stop look ignored.
  834. hard_cancel = getattr(telemetry_agent, "_hard_interrupt_requested", None)
  835. if callable(getattr(hard_cancel, "is_set", None)) and hard_cancel.is_set():
  836. return None
  837. for route in _stall_retry_routes(escalate_deterministic):
  838. recovered = _run_pinned_compression_retry(
  839. route, worker=worker, messages=messages, system_prompt_fallback=system_prompt_fallback,
  840. idle_timeout_seconds=idle_timeout_seconds, total_ceiling_seconds=total_ceiling_seconds,
  841. precommit_deadline_monotonic=precommit_deadline_monotonic,
  842. on_commit_overrun=on_commit_overrun, on_timeout_cause=on_timeout_cause, telemetry_agent=telemetry_agent,
  843. new_fence=new_fence,
  844. )
  845. if recovered is not None:
  846. return recovered
  847. return None
  848. def _run_pinned_compression_retry(
  849. route: dict, *, worker: Callable[[CompressionCommitFence], tuple[list, str]], messages: list,
  850. system_prompt_fallback: Any, idle_timeout_seconds: float, total_ceiling_seconds: float,
  851. precommit_deadline_monotonic: float,
  852. on_commit_overrun: Optional[Callable[[float, float], None]], on_timeout_cause: Optional[Callable[[bool, bool], None]],
  853. telemetry_agent: Any, new_fence: Optional[Callable[[], CompressionCommitFence]],
  854. ) -> Optional[tuple[list, str]]:
  855. """One bounded re-run of ``worker`` with ``route`` pinned; ``None`` when it produced no compression."""
  856. deterministic = route.get("deterministic") is True
  857. if not deterministic and time.monotonic() >= precommit_deadline_monotonic:
  858. logger.info(
  859. "Skipping compression stall fallback on %s: the shared pre-commit ceiling is spent", route["label"],
  860. )
  861. return None
  862. # The aborted fence refuses all commits; mint a fresh one via the host factory
  863. # so a /stop during the retry serializes against THIS attempt's commit boundary.
  864. retry_fence = None
  865. if new_fence is not None:
  866. try:
  867. retry_fence = new_fence()
  868. except Exception:
  869. logger.warning(
  870. "compression stall-fallback fence factory failed; the retry "
  871. "will run on an unpublished fence (a /stop mid-retry cannot serialize against its commit boundary)",
  872. exc_info=True,
  873. )
  874. if not isinstance(retry_fence, CompressionCommitFence):
  875. logger.warning(
  876. "compression stall-fallback retry running on an unpublished fence; "
  877. "hard-interrupt admission will read the aborted attempt's fence rather than the retry's commit boundary"
  878. )
  879. retry_fence = CompressionCommitFence()
  880. idle = float(route.get("timeout") or idle_timeout_seconds)
  881. if deterministic:
  882. # No provider call to stall on: its own budget, not the leftover the stalled routes left.
  883. ceiling, deadline = max(float(total_ceiling_seconds), idle), None
  884. else:
  885. ceiling, deadline = precommit_deadline_monotonic - time.monotonic(), precommit_deadline_monotonic
  886. if ceiling <= 0:
  887. retry_fence.revoke_commit_admission()
  888. logger.info(
  889. "Skipping compression stall fallback on %s: its setup spent the shared pre-commit ceiling",
  890. route["label"],
  891. )
  892. return None
  893. idle = min(idle, ceiling)
  894. if deterministic:
  895. logger.warning(
  896. "Context compression stalled on every summary route — committing the %s (no summary model) "
  897. "before continuing without compression", route["label"],
  898. )
  899. else:
  900. logger.warning(
  901. "Context compression stalled on the configured summary route — "
  902. "retrying once on %s (%s) within %.1fs of the shared pre-commit budget before continuing "
  903. "without compression", route["label"], route["model"], ceiling,
  904. )
  905. compressor = getattr(telemetry_agent, "context_compressor", None)
  906. streak_before = getattr(compressor, "_fallback_compression_streak", 0)
  907. try:
  908. from agent.context_compressor import pin_summary_route
  909. with pin_summary_route(route):
  910. result_msgs, result_prompt = run_compress_context_with_progress_timeout(
  911. worker=worker, messages=messages, system_prompt_fallback=system_prompt_fallback,
  912. idle_timeout_seconds=idle, total_ceiling_seconds=ceiling, on_commit_overrun=on_commit_overrun,
  913. on_timeout_cause=on_timeout_cause, fence=retry_fence, telemetry_agent=telemetry_agent,
  914. stall_fallback=False, precommit_deadline_monotonic=deadline,
  915. )
  916. except Exception:
  917. # The primary already failed; a failing fallback must degrade, never
  918. # turn "continue without compression" into a raised turn.
  919. logger.warning("Context compression fallback attempt on %s failed", route["label"], exc_info=True)
  920. return None
  921. if result_msgs is messages:
  922. # Aborted or no-op: the worker hands back the caller's own list.
  923. logger.warning(
  924. "Context compression fallback attempt on %s produced no compression; continuing without compression",
  925. route["label"],
  926. )
  927. return None
  928. # A pinned summary call that failed still commits (static fallback summary under the default
  929. # abort_on_summary_failure=false); the streak bump is the post-commit tell. Never call that "recovered".
  930. streak_after = getattr(compressor, "_fallback_compression_streak", 0)
  931. if deterministic or (isinstance(streak_after, int) and isinstance(streak_before, int) and streak_after > streak_before):
  932. logger.warning(
  933. "Context compression committed a deterministic fallback summary on %s after the primary summary route "
  934. "stalled (no summary model produced output)", route["label"],
  935. )
  936. else:
  937. logger.info("Context compression recovered on %s after the primary summary route stalled", route["label"])
  938. return result_msgs, result_prompt
  939. def _await_worker_within_budget(
  940. future: Any, fence: CompressionCommitFence, *, idle: float, ceiling: float, wait_started: float
  941. ) -> tuple[bool, Any]:
  942. """Poll ``future`` under the idle budget + ceiling; ``(True, result)`` when it settled."""
  943. while True:
  944. # The fence deadline moves: prompt reading on llama-server pushes it back.
  945. remaining_ceiling = (fence.deadline_monotonic or wait_started + ceiling) - time.monotonic()
  946. if remaining_ceiling <= 0:
  947. return False, None
  948. # Charge idle budget from LAST PROGRESS, not slice start, or silence could approach 2x the budget.
  949. since_progress = fence.seconds_since_progress()
  950. wait_slice = min(max(idle - since_progress, 0.005), remaining_ceiling)
  951. try:
  952. return True, future.result(timeout=wait_slice)
  953. except concurrent.futures.TimeoutError:
  954. # Aliases builtin TimeoutError (3.11+): also fires when the WORKER died with one (#63892).
  955. # A settled future never unsettles — re-waiting spun ~2k iter/s; take the stall path now.
  956. if future.done():
  957. exc = future.exception()
  958. if exc is None:
  959. return True, future.result()
  960. logger.info("Context compression worker exited with %r — taking the stall path", exc)
  961. return False, None
  962. waited = time.monotonic() - wait_started
  963. since_progress = fence.seconds_since_progress()
  964. if not fence.deadline_exceeded and since_progress < idle:
  965. logger.info(
  966. "Context compression still streaming after %.0fs (last progress %.1fs ago) — extending wait (ceiling %.0fs)",
  967. waited, since_progress, ceiling,
  968. )
  969. continue
  970. return False, None
  971. def _await_in_flight_commit(
  972. future: Any, *, ceiling: float, wait_started: float, on_commit_overrun: Optional[Callable[[float, float], None]]
  973. ) -> Any:
  974. """begin_commit won the race: the SessionDB mutation cannot be fence-cancelled, so wait
  975. in bounded slices, logging (escalating) + surfacing once via ``on_commit_overrun``
  976. WHILE the commit hangs. Never silently hung or abandoned.
  977. """
  978. overrun_surfaced = False
  979. overrun_reports = 0
  980. while True:
  981. waited = time.monotonic() - wait_started
  982. remaining = ceiling - waited
  983. if remaining <= 0:
  984. # Bounded increments so each overrun window is visible in logs rather than one silent unbounded block.
  985. remaining = min(_COMMIT_OVERRUN_WAIT_SLICE_SECONDS, max(ceiling, 0.05))
  986. overrun_reports += 1
  987. log = logger.warning if overrun_reports <= 2 else logger.error
  988. log(
  989. "Context compression SessionDB commit still running "
  990. "%.1fs past the total ceiling (waited %.1fs, ceiling %.1fs); commit cannot be abandoned mid-flight — "
  991. "continuing to wait (check SessionDB health if this persists)", waited - ceiling, waited,
  992. ceiling,
  993. )
  994. if not overrun_surfaced and on_commit_overrun is not None:
  995. overrun_surfaced = True
  996. with _swallow('compress_context commit-overrun callback failed', exc_info=True):
  997. on_commit_overrun(waited, ceiling)
  998. try:
  999. return future.result(timeout=remaining)
  1000. except concurrent.futures.TimeoutError:
  1001. # Aliases builtin TimeoutError (3.11+): also fires when the commit worker died with one (#63892).
  1002. # A settled future never unsettles — this ceiling-less loop spun forever; re-raise the worker's error.
  1003. if future.done():
  1004. return future.result()
  1005. # Commit-phase progress is informative only — the commit must complete; loop
  1006. # and re-report with the updated overrun window.
  1007. continue
  1008. def _cancel_or_join_worker(fence: CompressionCommitFence) -> bool:
  1009. """Cancel pre-commit; ``False`` when an admitted commit owns the fence (caller waits)."""
  1010. while True:
  1011. # begin_commit holds the fence lock until finish_commit, so try_cancel spins
  1012. # forever on a hung commit; lock-free marker makes the overrun loop reachable.
  1013. if fence.commit_in_flight:
  1014. return False
  1015. cancelled = fence.try_cancel_before_commit()
  1016. if cancelled is not None:
  1017. return cancelled
  1018. # Fence is held only transiently here, but that window rides SessionDB write
  1019. # patience (seconds). 25ms keeps sub-tick latency without a 1kHz spin.
  1020. time.sleep(0.025)
  1021. def _release_cancelled_worker(
  1022. future: Any, fence: CompressionCommitFence, *, total_exhausted: bool, ceiling: float
  1023. ) -> None:
  1024. """Idle-timeout unwind: free the worker's durable lease via the holder-qualified hook.
  1025. Total-ceiling only: bounded grace for the worker to exit (it checks the fence between provider phases; an
  1026. uninterruptible call is orphaned). Idle-stall skips the join: the worker is hung, the fallback needs a
  1027. prompt return, the fence guards."""
  1028. if total_exhausted:
  1029. grace = min(_CANCELLED_WORKER_TEARDOWN_GRACE_SECONDS, ceiling)
  1030. if _join_cancelled_worker(future, grace):
  1031. # Worker provably exited: no provider call can outlive this attempt, so lease
  1032. # retention is unneeded and a retry cannot overlap.
  1033. fence.allow_cancelled_lock_release()
  1034. else:
  1035. logger.warning(
  1036. "Cancelled compression worker did not exit within %.1fs "
  1037. "grace — orphaning it behind the poison fence (late result will be discarded); retaining the session "
  1038. "compression lease until it exits so no new attempt overlaps it", grace,
  1039. )
  1040. fence.release_cancelled_compression_lock()
  1041. def run_compress_context_with_progress_timeout(
  1042. *, worker: Callable[[CompressionCommitFence], tuple[list, str]], messages: list,
  1043. system_prompt_fallback: Any, idle_timeout_seconds: float, total_ceiling_seconds: float,
  1044. on_timeout: Optional[Callable[[float, float, float], None]] = None,
  1045. on_timeout_cause: Optional[Callable[[bool, bool], None]] = None,
  1046. on_commit_overrun: Optional[Callable[[float, float], None]] = None,
  1047. fence: Optional[CompressionCommitFence] = None, telemetry_agent: Any = None, stall_fallback: bool = True,
  1048. new_fence: Optional[Callable[[], CompressionCommitFence]] = None,
  1049. fallback_worker: Optional[Callable[[CompressionCommitFence], tuple[list, str]]] = None,
  1050. request_exceeds_window: bool = False,
  1051. precommit_deadline_monotonic: Optional[float] = None,
  1052. ) -> tuple[list, str]:
  1053. """Run ``worker(fence)`` under a sync progress-aware (idle + ceiling) timeout.
  1054. Budgets bound the PRE-commit phase only: an admitted commit always completes (overrun logged, surfaced
  1055. once via ``on_commit_overrun``). A pre-commit cancel returns ``(messages, system_prompt_fallback)`` (lazy
  1056. callable), detaching the worker; a stall first retries the chain once on ``new_fence``, then on_timeout.
  1057. ``request_exceeds_window``: the request this compaction must shrink is above the model's context
  1058. window, so "continue without compression" is not an option — a stall escalates to the deterministic
  1059. fallback summary on the FIRST timeout instead of waiting for a prior stall in the session (#114594).
  1060. ``precommit_deadline_monotonic``: an absolute deadline inherited from an earlier attempt (the stall
  1061. retry passes the primary's), clamping this attempt's ceiling so the attempts share one budget."""
  1062. if idle_timeout_seconds <= 0:
  1063. raise ValueError(
  1064. "run_compress_context_with_progress_timeout requires "
  1065. "idle_timeout_seconds > 0; call compress_context directly to disable"
  1066. )
  1067. def _resolve_fallback_prompt() -> str:
  1068. return system_prompt_fallback() if callable(system_prompt_fallback) else system_prompt_fallback
  1069. ceiling = max(float(total_ceiling_seconds), float(idle_timeout_seconds))
  1070. idle = float(idle_timeout_seconds)
  1071. # An over-window request cannot be sent uncompressed, so a summary that keeps streaming while
  1072. # reclaiming nothing must not hold the host (and the Desktop UI) to the full ceiling: bound the
  1073. # pre-commit wait to one inactivity budget (``compression.context_timeout_seconds``) and let the
  1074. # first-stall deterministic fallback below carry the compaction (#116472: a 600s trickle froze
  1075. # the Desktop for 10 minutes per turn).
  1076. if request_exceeds_window:
  1077. ceiling = idle
  1078. wait_started = time.monotonic()
  1079. if precommit_deadline_monotonic is None:
  1080. precommit_deadline_monotonic = wait_started + ceiling
  1081. else:
  1082. ceiling = min(ceiling, float(precommit_deadline_monotonic) - wait_started)
  1083. if ceiling <= 0:
  1084. if fence is not None:
  1085. fence.revoke_commit_admission()
  1086. return messages, _resolve_fallback_prompt()
  1087. idle = min(idle, ceiling)
  1088. fence = fence if fence is not None else CompressionCommitFence()
  1089. fence.set_precommit_deadline_monotonic(precommit_deadline_monotonic)
  1090. # Read BEFORE this attempt runs: the host's ``stalled`` record and the cancelled worker's
  1091. # ``stall_interrupted`` record both land during the unwind below, and this stall must not count as
  1092. # One prior timeout-class failure = the route already burned a full idle window. An over-window request
  1093. # cannot be sent unchanged, so it earns the deterministic fallback on its first stall (#114594).
  1094. escalate_deterministic = stall_fallback and (
  1095. request_exceeds_window or _prior_timeout_failures(telemetry_agent) >= 1
  1096. )
  1097. # Sync mirror of gateway hygiene's run_in_executor + wait_for loop: offload,
  1098. # poll idle budget + ceiling, fence-cancel on timeout so no late commit lands.
  1099. from tools.thread_context import propagate_context_to_thread
  1100. executor = _get_compress_timeout_executor()
  1101. # Refuse rather than queue when the pool is full: a queued job would wait out
  1102. # its budget unstarted and run stale later. Skip compression this cycle.
  1103. # A queued job would silently wait out its whole budget without starting and stay eligible to run as a
  1104. # stale cancelled job when a worker recovers. Fail fast: continue without compression this cycle. See
  1105. # #76354.
  1106. if not _try_admit_compression_job():
  1107. logger.warning(
  1108. "Context compression pool saturated (%d workers busy) — refusing new compression this cycle and continuing without "
  1109. "compression. Wedged workers are fence-cancelled and free their "
  1110. "slot when they return; if this persists, check the summary provider health.",
  1111. _COMPRESS_EXECUTOR_MAX_WORKERS,
  1112. )
  1113. # Saturation refusals must hit the same telemetry stream as other failures, or
  1114. # a wedged pool looks like compression simply stopped being attempted.
  1115. if telemetry_agent is not None:
  1116. _emit_compression_attempt_telemetry(
  1117. telemetry_agent, started_at=time.monotonic(), commit_status="aborted", split_status="aborted",
  1118. failure_class="pool_saturated", include_last_telemetry=False,
  1119. )
  1120. return messages, _resolve_fallback_prompt()
  1121. def _fence_gated_worker(worker_fence: CompressionCommitFence):
  1122. # An admitted job may start after the host stopped waiting; check the fence
  1123. # BEFORE summary work so a stale job never burns an LLM call.
  1124. if worker_fence.deadline_exceeded:
  1125. raise concurrent.futures.TimeoutError("compression deadline expired before worker start")
  1126. if worker_fence.is_cancelled:
  1127. logger.info("Skipping stale compression job: fence cancelled before start")
  1128. return messages, ""
  1129. return worker(worker_fence)
  1130. # Bare pool workers start with an empty ContextVar map; propagate the
  1131. # parent conversation/approval context into the worker.
  1132. try:
  1133. future = executor.submit(propagate_context_to_thread(_fence_gated_worker), fence)
  1134. except BaseException:
  1135. _release_compression_admission()
  1136. raise
  1137. future.add_done_callback(_release_compression_admission)
  1138. # EVERY host unwind must revoke commit admission or a detached worker could
  1139. # later mutate durable state; handled_exit marks paths that settle it themselves
  1140. handled_exit = False
  1141. def _is_unchanged_snapshot(result: Any) -> bool:
  1142. # A worker that observed the deadline/cancel returns the SAME messages object it was handed.
  1143. return result[0] is messages
  1144. def _recover_from_stall() -> tuple[list[dict[str, Any]], str]:
  1145. """One stall-fallback ladder for every host-side stall exit (idle timeout, settled-at-deadline
  1146. no-op, post-cancel unchanged commit): retry chain, then on_timeout, then the degraded prompt."""
  1147. # Sample before the retry chain so the reported wait is the stall itself, not stall + retry.
  1148. # #76354 S3 analogue: silence is charged from the LAST PROGRESS event, not from the start of
  1149. # this wait slice, or progress early in a previous slice would let silence approach 2x idle.
  1150. waited = time.monotonic() - wait_started
  1151. since_progress = fence.seconds_since_progress()
  1152. # Lease is free, so run the fallback BEFORE on_timeout: that callback records
  1153. # the summary-failure cooldown, which would no-op the retry's summary call.
  1154. if stall_fallback:
  1155. recovered = _retry_compression_on_fallback_chain(
  1156. worker=fallback_worker or worker, messages=messages, system_prompt_fallback=system_prompt_fallback,
  1157. idle_timeout_seconds=idle, total_ceiling_seconds=ceiling,
  1158. # Prompt reading moves the fence deadline after the primary armed it.
  1159. precommit_deadline_monotonic=fence.deadline_monotonic or precommit_deadline_monotonic,
  1160. on_commit_overrun=on_commit_overrun,
  1161. on_timeout_cause=on_timeout_cause, telemetry_agent=telemetry_agent, new_fence=new_fence,
  1162. escalate_deterministic=escalate_deterministic,
  1163. )
  1164. if recovered is not None:
  1165. return recovered
  1166. if on_timeout is not None:
  1167. with _swallow('compress_context timeout callback failed', exc_info=True):
  1168. on_timeout(idle, waited, since_progress)
  1169. else:
  1170. logger.warning(
  1171. "Context compression made no progress for %.1fs (total wait %.1fs, ceiling %.1fs); continuing without "
  1172. "compression", since_progress, waited, ceiling,
  1173. )
  1174. return messages, _resolve_fallback_prompt()
  1175. try:
  1176. settled, result = _await_worker_within_budget(
  1177. future, fence, idle=idle, ceiling=ceiling, wait_started=wait_started
  1178. )
  1179. if settled:
  1180. handled_exit = True
  1181. # The deadline is visible to the worker as well as the host. A cooperative summary call can
  1182. # observe it, unwind, and return the unchanged snapshot just BEFORE future.result() times out.
  1183. # Treat that settled no-op exactly like the host-side stall path; otherwise whether the
  1184. # deterministic fallback runs depends on a thread-scheduling race at the deadline.
  1185. if stall_fallback and fence.deadline_exceeded and _is_unchanged_snapshot(result):
  1186. if on_timeout_cause is not None:
  1187. with _swallow('compress_context timeout-cause callback failed', exc_info=True):
  1188. on_timeout_cause(True, fence.progress_observed)
  1189. return _recover_from_stall()
  1190. return result
  1191. # F6: a not-yet-started future must not linger as a stale queued job.
  1192. # cancel() is a no-op for a running worker (fence handles that path).
  1193. future.cancel()
  1194. total_exhausted = fence.deadline_exceeded
  1195. # #97488 teardown (total-ceiling path only): give the cancelled worker a bounded grace to actually
  1196. # exit before this host moves on. The worker checks the poison fence between provider phases, so a
  1197. # cooperative worker exits quickly; an uninterruptible provider call is orphaned behind the fence
  1198. # after the grace elapses (its late result is discarded and cannot touch session state). The
  1199. # idle-stall path intentionally skips the join: its worker is by definition silent/hung, the
  1200. # stall-fallback retry below needs a prompt host return (pinned by the #76354 S3 latency contract),
  1201. # and the fence poison + attempt-generation supersession already protect state against its late
  1202. # unwind.
  1203. if total_exhausted:
  1204. # A total-ceiling candidate may be unwinding a healthy provider call; keep its
  1205. # lease until it exits so no other attempt overlaps the unchanged source.
  1206. fence.retain_compression_lock_until_worker_done()
  1207. if on_timeout_cause is not None:
  1208. with _swallow('compress_context timeout-cause callback failed', exc_info=True):
  1209. on_timeout_cause(total_exhausted, fence.progress_observed)
  1210. if not _cancel_or_join_worker(fence):
  1211. result = _await_in_flight_commit(
  1212. future, ceiling=max(ceiling, fence.deadline_monotonic - wait_started), wait_started=wait_started,
  1213. on_commit_overrun=on_commit_overrun)
  1214. handled_exit = True
  1215. # The cancelled worker can race the host into its commit section while unwinding a stalled
  1216. # summary. When that commit is only the unchanged snapshot, returning it here skips the
  1217. # stall-fallback ladder entirely (the over-window first-stall test then flakes). The worker is
  1218. # settled and its lease is free at this point, so retry exactly as the pre-commit cancel path
  1219. # does. A real compression result remains authoritative and returns immediately.
  1220. if stall_fallback and _is_unchanged_snapshot(result):
  1221. return _recover_from_stall()
  1222. return result
  1223. # Idle-timeout: cancel won pre-commit. Also free the worker's durable lease via
  1224. # the holder-qualified hook so a NEW compressor can acquire at once (no ABA).
  1225. handled_exit = True
  1226. _release_cancelled_worker(future, fence, total_exhausted=total_exhausted, ceiling=ceiling)
  1227. # Leave the future on the shared pool: fence cancel won, so a late
  1228. # commit cannot land (same detachment model as gateway hygiene).
  1229. return _recover_from_stall()
  1230. finally:
  1231. if not handled_exit:
  1232. # Any unwind while waiting: revoke commit admission and release the worker's
  1233. # lease before the host unwinds, so the detached worker can never publish.
  1234. fence.revoke_commit_admission()
  1235. from agent.conversation_compression_checkpoint import _checkpoint_blocked, _checkpoint_incapable
  1236. from agent.conversation_compression_feasibility import lazy_compression_feasibility_check
  1237. def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool:
  1238. """Whether the live in-memory SessionDB class structurally predates locks.
  1239. Only the exact old ``hermes_state.SessionDB`` class (hot-reload skew) may fail open; proxies, lookalikes,
  1240. non-callables and descriptor failures fail closed."""
  1241. try:
  1242. from hermes_state import SessionDB
  1243. missing = object()
  1244. return (
  1245. type(lock_db) is SessionDB
  1246. and inspect.getattr_static(SessionDB, "try_acquire_compression_lock", missing) is missing
  1247. )
  1248. except Exception:
  1249. return False
  1250. def _refresh_persisted_compression_guards(compressor: Any, *, include_cooldown: bool = True) -> None:
  1251. """Refresh durable automatic-compression guards on a built-in compressor."""
  1252. method_calls = [("_load_fallback_compression_streak", {}), ("_load_ineffective_compression_count", {})]
  1253. if include_cooldown:
  1254. method_calls.insert(0, ("get_active_compression_failure_cooldown", {"refresh": True}))
  1255. for method_name, kwargs in method_calls:
  1256. method = getattr(type(compressor), method_name, None)
  1257. if not callable(method):
  1258. continue
  1259. try:
  1260. method(compressor, **kwargs)
  1261. except Exception as exc:
  1262. logger.debug("compression guard refresh failed (%s): %s", method_name, exc)
  1263. def _session_was_rotated_by_compression(session_db: Any, session_id: str) -> bool:
  1264. """Return whether another path already rotated this compression parent."""
  1265. getter = getattr(type(session_db), "get_session", None)
  1266. if not callable(getter):
  1267. return False
  1268. session = getter(session_db, session_id)
  1269. return bool(session and session.get("ended_at") is not None and session.get("end_reason") == "compression")
  1270. def _existing_system_prompt(agent: Any, system_message: str) -> str:
  1271. """Cached system prompt, or a fresh build when nothing is cached (abort paths)."""
  1272. return getattr(agent, "_cached_system_prompt", None) or agent._build_system_prompt(system_message)
  1273. def _restore_messages_snapshot(messages: list, snapshot: Optional[list]) -> None:
  1274. """Put the pre-compression deep snapshot back into the live list if it drifted."""
  1275. if snapshot is not None and messages != snapshot:
  1276. messages[:] = copy.deepcopy(snapshot)
  1277. def _restore_prune_rearm_tokens(compressor: Any, snapshot: dict) -> None:
  1278. """Restore ONLY the prune runway from the attempt snapshot.
  1279. compress() zeroes it in memory while the durable copy only clears on a successful commit; a kept
  1280. transcript keeps its cached prefix, and 0 would let the next prune break that cache."""
  1281. if "_proactive_prune_rearm_tokens" in snapshot:
  1282. compressor._proactive_prune_rearm_tokens = snapshot["_proactive_prune_rearm_tokens"]
  1283. def compression_skipped_due_to_lock(agent: Any) -> bool:
  1284. """Type-pinned read of the per-session lock-skip signal.
  1285. ``agent._compression_skipped_due_to_lock`` is a holder string or ``True`` when a pass no-oped because the
  1286. lock was held, ``None`` otherwise. Pinning avoids MagicMock auto-attributes hijacking mocked agents into
  1287. the lock-skip branch."""
  1288. _sig = getattr(agent, "_compression_skipped_due_to_lock", None)
  1289. return _sig is True or isinstance(_sig, str)
  1290. def _get_context_compression_timeout_state(
  1291. agent: Any, *, create: bool
  1292. ) -> Optional[tuple[Any, Optional[threading.local]]]:
  1293. """Return the stable lock and thread-local timeout state for an agent."""
  1294. try:
  1295. attributes = vars(agent)
  1296. except TypeError:
  1297. return None
  1298. lock = attributes.setdefault("_context_compression_timeout_state_lock", threading.Lock())
  1299. with lock:
  1300. state = attributes.get("_context_compression_timeout_state")
  1301. if create and not isinstance(state, threading.local):
  1302. state = threading.local()
  1303. attributes["_context_compression_timeout_state"] = state
  1304. return lock, state if isinstance(state, threading.local) else None
  1305. def _set_context_compression_timeout_outcome(agent: Any, timed_out: bool) -> None:
  1306. """Write this thread's owned-compression timeout outcome.
  1307. The ``agent._last_compression_timed_out`` mirror stays authoritative for minimal agent doubles that do not
  1308. support ``vars()``."""
  1309. lock, state = _get_context_compression_timeout_state(agent, create=True) or (None, None)
  1310. if state is None:
  1311. agent._last_compression_timed_out = timed_out
  1312. return
  1313. with lock:
  1314. state.timed_out = timed_out
  1315. agent._last_compression_timed_out = timed_out
  1316. def reset_context_compression_timeout_outcome(agent: Any) -> None:
  1317. """Clear the current thread's owned-compression timeout outcome."""
  1318. _set_context_compression_timeout_outcome(agent, False)
  1319. def mark_context_compression_timed_out(agent: Any) -> None:
  1320. """Mark the current owned compression as host-timed-out."""
  1321. _set_context_compression_timeout_outcome(agent, True)
  1322. def context_compression_timed_out(agent: Any) -> bool:
  1323. """Return whether this thread's owned compression hit its host timeout.
  1324. Thread-local so overlapping automatic/manual entrypoints cannot hide each other's timeout; attribute
  1325. fallback for minimal doubles; reads type-pinned."""
  1326. locked_state = _get_context_compression_timeout_state(agent, create=False)
  1327. if locked_state is not None:
  1328. lock, state = locked_state
  1329. with lock:
  1330. if isinstance(state, threading.local):
  1331. return getattr(state, "timed_out", None) is True
  1332. return getattr(agent, "_last_compression_timed_out", None) is True
  1333. def _automatic_compression_gate_blocks(
  1334. agent: Any, bypass_cooldown: bool, *, include_cooldown: bool = True,
  1335. ) -> Optional[str]:
  1336. """Refresh durable guards, then evaluate the automatic breaker gate; return its block reason or None. The gate and
  1337. the reason both get ``ignore_cooldown`` when ``bypass_cooldown`` and they accept it (older engines get the no-arg
  1338. call), so a bypassed cooldown is never the reason. When blocked, the transient-block signal is published."""
  1339. compressor = agent.context_compressor
  1340. _refresh_persisted_compression_guards(compressor, include_cooldown=include_cooldown)
  1341. blocked = getattr(type(compressor), "_automatic_compression_blocked", None)
  1342. if not callable(blocked):
  1343. return None
  1344. ignore = bypass_cooldown and _accepts_keyword_argument(blocked, "ignore_cooldown")
  1345. if not (blocked(compressor, ignore_cooldown=True) if ignore else blocked(compressor)):
  1346. return None
  1347. reason, reason_fn = None, getattr(compressor, "_compression_block_reason", None)
  1348. if callable(reason_fn): # engines predating it name no reason
  1349. with _swallow('compression block-reason read failed', exc_info=True):
  1350. reason = (
  1351. reason_fn(ignore_cooldown=True) if ignore and _accepts_keyword_argument(reason_fn, "ignore_cooldown")
  1352. else reason_fn()
  1353. )
  1354. reason = reason if isinstance(reason, str) and reason else "unknown"
  1355. _mark_compression_blocked_transient(agent, compressor, reason)
  1356. return reason
  1357. def compression_blocked_transiently(agent: Any) -> bool:
  1358. """Type-pinned read of the transient-block signal.
  1359. Set when an automatic pass no-ops on a TRANSIENT guard (summary-failure cooldown or structural backoff).
  1360. Consumers must defer, not count it toward ``compression_exhausted``, or an overflow auto-reset wipes a
  1361. session that was merely cooling down. The permanent ``ineffective`` breaker never sets it.
  1362. See #97488.
  1363. Consumers (the overflow-recovery loops in ``conversation_loop``) must treat such a no-op as a temporary
  1364. defer, NOT as evidence the session is incompressible: counting it toward ``compression_exhausted`` lets
  1365. a real upstream ``context_length_exceeded`` auto-reset (wipe) a session whose compression was merely
  1366. cooling down (#97488). The permanent ``ineffective`` breaker intentionally does NOT set this signal — a
  1367. genuinely incompressible session must still be able to exhaust.
  1368. """
  1369. _sig = getattr(agent, "_compression_blocked_transient", None)
  1370. return isinstance(_sig, str) and bool(_sig)
  1371. def _mark_compression_blocked_transient(agent: Any, compressor: Any, reason: str) -> None:
  1372. """Publish the transient-block signal when the active guard is transient.
  1373. ``reason`` comes from ``_compression_block_reason``: ``cooldown:*`` and ``structural_backoff:*`` are
  1374. transient; ``ineffective`` stays unmarked."""
  1375. if reason.startswith("cooldown") or reason.startswith("structural_backoff"):
  1376. logger.info(
  1377. "Skipping automatic compression re-entry: transient guard "
  1378. "active (%s, session=%s, last failure: %s) — will retry after "
  1379. "the backoff lapses; /compress forces an immediate retry", reason,
  1380. getattr(agent, "session_id", None) or "none",
  1381. getattr(compressor, "_last_summary_error", None) or "unknown",
  1382. )
  1383. with contextlib.suppress(Exception):
  1384. agent._compression_blocked_transient = reason
  1385. def _rebind_session_context(session_id: str) -> None:
  1386. """Point the worker thread's session ContextVar and log context at ``session_id``."""
  1387. try:
  1388. from gateway.session_context import set_current_session_id
  1389. set_current_session_id(session_id)
  1390. except Exception:
  1391. os.environ["HERMES_SESSION_ID"] = session_id
  1392. with contextlib.suppress(Exception):
  1393. from hermes_logging import set_session_context
  1394. set_session_context(session_id)
  1395. def _adopt_live_compression_child(
  1396. agent: Any, session_db: Any, parent_session_id: str
  1397. ) -> Optional[list[dict[str, Any]]]:
  1398. """Move a stale compression contender onto the live continuation tip.
  1399. Resolve and load first, then mutate the agent, so ambiguous lineage or an unreadable handoff fails closed.
  1400. Uses the transitive ``get_compression_tip`` walk; a tip is adopted only while its row is still live.
  1401. Resolution uses the canonical transitive walk ``get_compression_tip`` so a lineage with >=2 compression
  1402. hops (root -> mid -> tip) recovers to the live tip — the depth-1 ``find_live_compression_child`` lookup
  1403. this used to call finds no live *direct* child in that shape and skipped recovery (#82001). The tip walk
  1404. returns the input id when no continuation exists, and a resolved tip is adopted only while its row is
  1405. still live — both cases fail closed exactly as before.
  1406. """
  1407. resolver = getattr(type(session_db), "get_compression_tip", None)
  1408. row_getter = getattr(type(session_db), "get_session", None)
  1409. loader = getattr(type(session_db), "get_messages_as_conversation", None)
  1410. if not callable(resolver) or not callable(row_getter) or not callable(loader):
  1411. return None
  1412. tip = resolver(session_db, parent_session_id)
  1413. if not tip or str(tip) == str(parent_session_id):
  1414. return None
  1415. child_session_id = str(tip)
  1416. child = row_getter(session_db, child_session_id)
  1417. if not isinstance(child, dict) or child.get("ended_at") is not None:
  1418. return None
  1419. recovered = loader(session_db, child_session_id)
  1420. if not (isinstance(recovered, list) and recovered):
  1421. return None
  1422. # Revalidate after loading: the tip may have rotated or a competing
  1423. # continuation may have appeared between the two DB reads.
  1424. confirmed = resolver(session_db, parent_session_id)
  1425. if not confirmed or str(confirmed) != child_session_id:
  1426. return None
  1427. agent.session_id = child_session_id
  1428. _rebind_session_context(child_session_id)
  1429. _hand_off_metrics_segment(parent_session_id, child_session_id)
  1430. agent._session_db_created = True
  1431. # The turn skips restore/rebuild while this slot is set, so it may hold only the child's own
  1432. # prompt, and only when that prompt matches the current runtime (otherwise None -> rebuild).
  1433. # Turn-start adoption runs before _restore_primary_runtime on purpose; a reject here is re-checked by the normal restore.
  1434. from agent.conversation_loop import _stored_prompt_matches_runtime
  1435. child_prompt = child.get("system_prompt")
  1436. agent._cached_system_prompt = (
  1437. child_prompt if child_prompt and _stored_prompt_matches_runtime(agent, child_prompt) else None
  1438. )
  1439. agent._last_flushed_db_idx = len(recovered)
  1440. agent._flushed_db_message_session_id = child_session_id
  1441. agent._flushed_db_message_ids = {id(message) for message in recovered if isinstance(message, dict)}
  1442. on_session_start = getattr(agent.context_compressor, "on_session_start", None)
  1443. if callable(on_session_start):
  1444. with _swallow('context engine compression-child adoption failed: %s'):
  1445. on_session_start(
  1446. child_session_id, boundary_reason="compression", old_session_id=parent_session_id,
  1447. session_db=session_db, platform=getattr(agent, "platform", None) or "cli",
  1448. conversation_id=getattr(agent, "_gateway_session_key", None),
  1449. )
  1450. else:
  1451. bind_state = getattr(agent.context_compressor, "bind_session_state", None)
  1452. if callable(bind_state):
  1453. with contextlib.suppress(Exception):
  1454. bind_state(session_db=session_db, session_id=child_session_id)
  1455. with _swallow('memory manager compression-child adoption failed: %s'):
  1456. if agent._memory_manager:
  1457. agent._memory_manager.on_session_switch(
  1458. child_session_id, parent_session_id=parent_session_id, reset=False, reason="compression"
  1459. )
  1460. return recovered
  1461. def _reopen_orphaned_parent(session_db: Any, session_id: str) -> None:
  1462. """Reopen a compression-ended parent that has no continuation and no lease holder."""
  1463. orphan_reopener = getattr(type(session_db), "reopen_orphaned_compression_session", None)
  1464. if not callable(orphan_reopener):
  1465. return
  1466. try:
  1467. if orphan_reopener(session_db, session_id):
  1468. logger.warning("compression recovery: reopened orphaned session=%s with no continuation", session_id)
  1469. except Exception as exc:
  1470. logger.warning("orphaned compression session reopen failed for %s: %s", session_id, exc)
  1471. def recover_rotated_compression_session(agent: Any) -> Optional[list[dict[str, Any]]]:
  1472. """Recover a stale live agent before a new turn writes to its old parent."""
  1473. session_db = getattr(agent, "_session_db", None)
  1474. session_id = getattr(agent, "session_id", None) or ""
  1475. if session_db is None or not session_id:
  1476. return None
  1477. try:
  1478. if not _session_was_rotated_by_compression(session_db, session_id):
  1479. return None
  1480. # Rotation holds the parent lease until the child handoff is durable; wait
  1481. # briefly rather than observe the parent-ended/child-empty intermediate state.
  1482. holder_getter = getattr(session_db, "get_compression_lock_holder", None)
  1483. for attempt in range(21):
  1484. recovered = _adopt_live_compression_child(agent, session_db, session_id)
  1485. if recovered is not None:
  1486. return recovered
  1487. holder = holder_getter(session_id) if callable(holder_getter) else None
  1488. if not holder:
  1489. _reopen_orphaned_parent(session_db, session_id)
  1490. return None
  1491. if attempt == 20:
  1492. return None
  1493. time.sleep(0.05)
  1494. return None
  1495. except Exception as exc:
  1496. logger.warning(
  1497. "compression session recovery failed for session=%s (%s: %s)", session_id, type(exc).__name__, exc
  1498. )
  1499. return None
  1500. def _compression_lock_holder(agent: Any) -> str:
  1501. """Build a unique lock holder id: ``pid[:pidns]:tid:agent-instance:uuid`` (pidns: see ``hermes_state_pidns``).
  1502. pid+tid tell crashed holders apart; instance id and per-acquire uuid disambiguate co-resident/pooled agents."""
  1503. return f"pid={os.getpid()}{holder_namespace_token()}:tid={threading.get_ident()}:agent={id(agent):x}:nonce={uuid.uuid4().hex[:8]}"
  1504. def _supported_compression_kwargs(
  1505. compress_fn: Any, *, current_tokens: Optional[int], focus_topic: Optional[str], force: bool,
  1506. memory_context: str, bypass_cooldown: bool = False,
  1507. ) -> dict:
  1508. """Return only compression kwargs accepted by an engine callable.
  1509. Inspecting first keeps older plugin signatures compatible without catching ``TypeError`` and running a
  1510. stateful compressor twice."""
  1511. candidates = {"current_tokens": current_tokens, "focus_topic": focus_topic, "force": force}
  1512. if bypass_cooldown:
  1513. candidates["bypass_cooldown"] = True
  1514. if memory_context:
  1515. candidates["memory_context"] = memory_context
  1516. try:
  1517. parameters = inspect.signature(compress_fn).parameters
  1518. except (TypeError, ValueError):
  1519. # current_tokens has always been in the ContextEngine ABC; use the oldest call
  1520. # shape when the callable has no inspectable signature.
  1521. return {"current_tokens": current_tokens}
  1522. if any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values()):
  1523. return candidates
  1524. return {name: value for name, value in candidates.items() if name in parameters}
  1525. class _CompressionActivityHeartbeat:
  1526. """Refresh the agent inactivity tracker while compression blocks in an aux call."""
  1527. def __init__(
  1528. self, agent: Any, interval_seconds: float | None = None, *, emit_client_status: bool = False,
  1529. commit_fence: Optional[CompressionCommitFence] = None,
  1530. ) -> None:
  1531. self._agent = agent
  1532. self._commit_fence = commit_fence
  1533. # Latched once host cancel/timeout wins or a terminal stamp is observed,
  1534. # so a later UNKNOWN rewrite cannot re-arm a detached zombie heartbeat.
  1535. self._suppressed = False
  1536. if interval_seconds is None:
  1537. interval_seconds = getattr(agent, "_compression_activity_heartbeat_interval", 60.0)
  1538. try:
  1539. interval_seconds = float(interval_seconds or 60.0)
  1540. if not math.isfinite(interval_seconds):
  1541. interval_seconds = 60.0
  1542. except (TypeError, ValueError):
  1543. interval_seconds = 60.0
  1544. self._interval_seconds = max(0.1, interval_seconds)
  1545. # Only a compression that opened a VISIBLE compaction phase (the
  1546. # routine start status was emitted) keeps it alive with heartbeats;
  1547. # quiet context engines emit neither (#98371 follow-up).
  1548. self._emit_client_status = emit_client_status
  1549. self._stop = threading.Event()
  1550. self._thread = threading.Thread(target=self._run, name="compression-activity-heartbeat", daemon=True)
  1551. def start(self) -> _CompressionActivityHeartbeat:
  1552. # A new compression episode always republishes agent.compression even
  1553. # if a prior timeout/cooldown stamp is still on the agent.
  1554. self._suppressed = False
  1555. self._touch("context compression started", allow_terminal_overwrite=True)
  1556. self._thread.start()
  1557. return self
  1558. def stop(self, desc: str = "context compression completed") -> None:
  1559. self._stop.set()
  1560. if self._thread.is_alive() and threading.current_thread() is not self._thread:
  1561. self._thread.join(timeout=1.0)
  1562. # Host timeout already owns the terminal stamp; a detached worker's
  1563. # late stop must not republish agent.compression / "completed".
  1564. if self._should_suppress():
  1565. return
  1566. # Force persist: /compress never hits run_conversation's turn-end clear, so
  1567. # durable labels would stay "in progress" for the 60s persist window.
  1568. self._touch(desc, force_persist=True)
  1569. def _should_suppress(self) -> bool:
  1570. if not self._suppressed and self._commit_fence is not None and self._commit_fence.is_cancelled:
  1571. self._suppressed = True
  1572. return self._suppressed
  1573. def _touch(self, desc: str, *, allow_terminal_overwrite: bool = False, force_persist: bool = False) -> None:
  1574. with _swallow('compression activity heartbeat touch failed', exc_info=True):
  1575. if not allow_terminal_overwrite:
  1576. if self._should_suppress():
  1577. return
  1578. current = normalize_activity_provenance(getattr(self._agent, "_last_activity_provenance", None))
  1579. if current in _TERMINAL_COMPRESSION_PROVENANCES:
  1580. self._suppressed = True
  1581. return
  1582. touch = getattr(self._agent, "_touch_activity", None)
  1583. if callable(touch):
  1584. # Re-check after reading provenance: host may cancel/stamp
  1585. # TIMEOUT between the earlier guard and the write.
  1586. if not allow_terminal_overwrite and self._should_suppress():
  1587. return
  1588. touch(desc, provenance=ActivityProvenance.AGENT_COMPRESSION, force_persist=force_persist)
  1589. def _emit_progress_status(self) -> None:
  1590. """Re-publish the compacting status so remote transports see progress.
  1591. Compression can stream for minutes with no deltas, tool events, or
  1592. status lines reaching remote transports. Idle-progress watchdogs on
  1593. those clients (e.g. the Android relay app's 180s turn watchdog)
  1594. treat the silence as a dead turn and fire ``session.interrupt`` —
  1595. killing a healthy compression mid-flight and rolling back its work,
  1596. which retriggers on the next prompt and loops forever on sessions
  1597. near the context ceiling (#98371).
  1598. Routed through ``agent._emit_status`` like every other compaction
  1599. status: same "lifecycle" key (the TUI gateway re-tags it to
  1600. ``compacting``; Telegram edits one bubble per key), same chat-platform
  1601. filter, same CLI print path.
  1602. """
  1603. if not self._emit_client_status:
  1604. return
  1605. emit = getattr(self._agent, "_emit_status", None)
  1606. if not callable(emit):
  1607. return
  1608. try:
  1609. emit(COMPACTION_HEARTBEAT_STATUS)
  1610. except Exception:
  1611. logger.debug(
  1612. "status emit error in compression heartbeat", exc_info=True
  1613. )
  1614. def _run(self) -> None:
  1615. while not self._stop.wait(self._interval_seconds):
  1616. if self._should_suppress():
  1617. return
  1618. self._touch("context compression in progress")
  1619. self._emit_progress_status()
  1620. def _direct_messages_for_pre_compress_memory(messages: Any) -> list[dict[str, Any]]:
  1621. """Return direct user/assistant evidence safe for memory checkpointing.
  1622. Summaries, tool rows and system messages are omitted; assistant prose is kept with ``tool_calls``
  1623. stripped, and pure tool-call wrappers are dropped."""
  1624. # Deferred import: context_compressor → turn_context → this module would form an import cycle.
  1625. from agent.context_compressor import COMPRESSED_SUMMARY_METADATA_KEY
  1626. direct_messages: list[dict[str, Any]] = []
  1627. for message in messages or []:
  1628. if not isinstance(message, dict):
  1629. continue
  1630. role = message.get("role")
  1631. if role not in {"user", "assistant"} or message.get(COMPRESSED_SUMMARY_METADATA_KEY):
  1632. continue
  1633. if role == "assistant" and message.get("tool_calls"):
  1634. content = message.get("content")
  1635. has_prose = bool(content.strip() if isinstance(content, str) else content)
  1636. if not has_prose:
  1637. continue
  1638. message = {k: v for k, v in message.items() if k != "tool_calls"}
  1639. direct_messages.append(message)
  1640. return direct_messages
  1641. class _CompressionLockLeaseRefresher:
  1642. def __init__(
  1643. self, db: Any, session_id: str, holder: str, ttl_seconds: float, refresh_interval_seconds: float | None = None
  1644. ) -> None:
  1645. self._db = db
  1646. self._session_id = session_id
  1647. self._holder = holder
  1648. self._ttl_seconds = ttl_seconds
  1649. if refresh_interval_seconds is None:
  1650. refresh_interval_seconds = max(1.0, min(60.0, ttl_seconds / 2.0))
  1651. self._refresh_interval_seconds = max(0.1, float(refresh_interval_seconds))
  1652. # Tolerate transient refresh failures for at most one TTL so the lease cannot
  1653. # outlive its TTL; floor 1 so interval >= ttl still tolerates one blip.
  1654. self._max_consecutive_failures = max(1, int(self._ttl_seconds / self._refresh_interval_seconds))
  1655. self._stop = threading.Event()
  1656. self._thread = threading.Thread(target=self._run, name="compression-lock-refresh", daemon=True)
  1657. def start(self) -> _CompressionLockLeaseRefresher:
  1658. self._thread.start()
  1659. return self
  1660. def stop(self) -> None:
  1661. self._stop.set()
  1662. # join() timing out mid-UPDATE is safe: daemon thread, and a late refresh on a
  1663. # released lock is a rowcount-0 no-op. stop() does not guarantee quiescence.
  1664. if self._thread.is_alive() and threading.current_thread() is not self._thread:
  1665. self._thread.join(timeout=1.0)
  1666. def _run(self) -> None:
  1667. # A single falsy refresh (transient DB blip) must not kill the lease; only
  1668. # ttl/interval consecutive failures do, so a stuck refresher never outlives TTL.
  1669. consecutive_failures = 0
  1670. # Refresh immediately: work between try_acquire() and start() is charged to the
  1671. # first lease, so on a short TTL it could expire before tick #1.
  1672. first = True
  1673. while first or not self._stop.wait(self._refresh_interval_seconds):
  1674. if first:
  1675. first = False
  1676. if self._stop.is_set():
  1677. break
  1678. refreshed = False
  1679. with _swallow("compression lock refresh raised: %s"):
  1680. refreshed = self._db.refresh_compression_lock(self._session_id, self._holder, ttl_seconds=self._ttl_seconds)
  1681. if refreshed:
  1682. consecutive_failures = 0
  1683. continue
  1684. consecutive_failures += 1
  1685. if consecutive_failures >= self._max_consecutive_failures:
  1686. logger.debug(
  1687. "compression lock refresh failed %d times in a row; stopping lease refresher for session %s",
  1688. consecutive_failures, self._session_id,
  1689. )
  1690. break
  1691. def _emit_feasibility_notice(agent: Any, msg: str) -> None:
  1692. """Store + emit a feasibility verdict once per distinct text: every main-runtime change re-probes, and an
  1693. unchanged verdict must not re-warn on each `/model --once` restore or fallback cycle (#114707)."""
  1694. if getattr(agent, "_last_feasibility_notice", None) == msg:
  1695. return
  1696. agent._last_feasibility_notice = msg
  1697. agent._compression_warning = msg
  1698. agent._emit_diagnostic_status(msg)
  1699. def _lower_threshold_to_aux_context(
  1700. agent: Any, *, aux_model: str, aux_context: int, aux_provider: str, aux_base_url: str
  1701. ) -> None:
  1702. """Lower the live threshold to the aux model's window and tell the user how to fix config.
  1703. The summariser sends one user prompt (no system/tools), so threshold == aux_context is safe.
  1704. Retention is recalibrated through its selected policy: lean is window-relative;
  1705. only legacy follows the lowered threshold."""
  1706. compressor = agent.context_compressor
  1707. old_threshold = compressor.threshold_tokens
  1708. new_threshold = compressor.threshold_tokens = aux_context
  1709. # Durable ceiling: update_model() recomputes threshold_tokens from the main model on every window
  1710. # correction and re-applies this through _apply_threshold_tokens_cap() (#114707).
  1711. compressor._aux_context_ceiling = aux_context
  1712. summary_target_ratio = getattr(compressor, "summary_target_ratio", None)
  1713. if getattr(compressor, "tail_mode", None) == "lean":
  1714. # Keep the window-relative policy owned by the compressor property.
  1715. compressor._tail_token_budget = None
  1716. elif isinstance(summary_target_ratio, (int, float)):
  1717. compressor.tail_token_budget = int(new_threshold * summary_target_ratio)
  1718. main_ctx = compressor.context_length
  1719. if main_ctx:
  1720. compressor.threshold_percent = new_threshold / main_ctx
  1721. safe_pct = int((aux_context / main_ctx) * 100) if main_ctx else 50
  1722. # Mirror the compressor's threshold math (percent floor, output reservation, 64K floor): a suggestion it
  1723. # would override is silently ignored and this warning reappears every session. External engines: keep it plain.
  1724. # The "lower the threshold" suggestion must survive the built-in trigger recomputation (#67422):
  1725. # _effective_threshold_percent() raises sub-75% values back up for main windows under 512K, and
  1726. # _compute_threshold_tokens() further applies the output-token reservation, the 64K floor, and the
  1727. # degenerate-window guard. Recommending a value those would override is silently ignored and this
  1728. # warning would reappear every session — so mirror the compressor's own math and only offer the option
  1729. # when the recomputed trigger actually fits the auxiliary model's context.
  1730. from agent.context_compressor import ContextCompressor as _CC
  1731. recomputed_threshold = None
  1732. if main_ctx and isinstance(compressor, _CC):
  1733. recomputed_threshold = _CC._compute_threshold_tokens(
  1734. main_ctx, _CC._effective_threshold_percent(main_ctx, safe_pct / 100),
  1735. getattr(compressor, "max_tokens", None),
  1736. )
  1737. threshold_suggestion_viable = recomputed_threshold is None or recomputed_threshold <= aux_context
  1738. # "model (provider)" labels for both sides; empty/"auto" provider falls back to the client's base_url hostname.
  1739. _main_model = getattr(agent, "model", "") or "?"
  1740. _main_provider = getattr(agent, "provider", "") or ""
  1741. _aux_provider_label = aux_provider if aux_provider and aux_provider != "auto" else ""
  1742. if not _aux_provider_label:
  1743. try:
  1744. from urllib.parse import urlparse
  1745. _aux_provider_label = urlparse(aux_base_url).hostname or aux_base_url
  1746. except Exception:
  1747. _aux_provider_label = aux_base_url or "auto"
  1748. _main_label = f"{_main_model} ({_main_provider})" if _main_provider else _main_model
  1749. _aux_label = f"{aux_model} ({_aux_provider_label})"
  1750. msg = (
  1751. f"⚠ Compression model {_aux_label} context is {aux_context:,} tokens, but the main model "
  1752. f"{_main_label}'s compression threshold was {old_threshold:,} tokens. "
  1753. f"Auto-lowered this session's threshold to {new_threshold:,} tokens so compression can run.\n"
  1754. )
  1755. if threshold_suggestion_viable:
  1756. msg += (
  1757. f" To make this permanent, edit config.yaml — either:\n 1. Use a larger compression model:\n"
  1758. f" auxiliary:\n compression:\n model: <model-with-{old_threshold:,}+-context>\n"
  1759. f" 2. Lower the compression threshold:\n compression:\n threshold: 0.{safe_pct:02d}"
  1760. )
  1761. else:
  1762. msg += (
  1763. f" To make this permanent, use a larger compression model in config.yaml:\n auxiliary:\n"
  1764. f" compression:\n model: <model-with-{old_threshold:,}+-context>\n"
  1765. f" (Lowering compression.threshold cannot help here — with {_main_label}'s {main_ctx:,}-token window, "
  1766. f"Hermes's small-context floor and output reservation would recompute the trigger to "
  1767. f"{recomputed_threshold:,} tokens, still above the compression model's {aux_context:,}.)"
  1768. )
  1769. _emit_feasibility_notice(agent, msg)
  1770. logger.warning(
  1771. "Auxiliary compression model %s has %d token context, below the main model's compression threshold of %d "
  1772. "tokens — auto-lowered session threshold to %d to keep compression working.", aux_model, aux_context,
  1773. old_threshold, new_threshold,
  1774. )
  1775. def _aux_inherits_main_route(agent: Any, aux_model: str, aux_base_url: str) -> bool:
  1776. """True when the auxiliary compression client is the main model on the main endpoint."""
  1777. from hermes_cli.route_identity import normalize_route_base_url
  1778. if str(aux_model or "").strip().lower() != str(getattr(agent, "model", "") or "").strip().lower():
  1779. return False
  1780. main_base = normalize_route_base_url(str(getattr(agent, "base_url", "") or ""))
  1781. return not main_base or normalize_route_base_url(aux_base_url) == main_base
  1782. def _configured_compression_context_length(agent: Any) -> Optional[int]:
  1783. """``auxiliary.compression.context_length`` as configured now (a session's side tasks follow its
  1784. profile, so the value captured at agent start can belong to an earlier summariser)."""
  1785. try:
  1786. from agent.auxiliary_task_config import _get_auxiliary_task_config
  1787. raw = _get_auxiliary_task_config("compression").get("context_length")
  1788. except Exception:
  1789. logger.debug("Compression context_length unreadable; using the value from agent start", exc_info=True)
  1790. return getattr(agent, "_aux_compression_context_length_config", None)
  1791. try:
  1792. return int(raw) if raw is not None else None
  1793. except (TypeError, ValueError):
  1794. return None
  1795. def _configured_custom_providers(agent: Any) -> Optional[list]:
  1796. """The provider entries as configured now, for measuring the compression model (a side task reads
  1797. them from config.yaml); the list captured at agent start inside a bypass session (its side tasks
  1798. never read the profile) or when config cannot be read."""
  1799. from agent.auxiliary_task_config import _SESSION_AUXILIARY
  1800. if _SESSION_AUXILIARY.get() is not None:
  1801. return getattr(agent, "_custom_providers", None)
  1802. try:
  1803. from hermes_cli.config import get_compatible_custom_providers, load_config_readonly
  1804. return get_compatible_custom_providers(load_config_readonly())
  1805. except Exception:
  1806. logger.debug("Provider entries unreadable; using the ones from agent start", exc_info=True)
  1807. return getattr(agent, "_custom_providers", None)
  1808. def check_compression_model_feasibility(agent: Any) -> None:
  1809. """Warn at session start if the aux compression context is below the threshold.
  1810. Called from ``AIAgent.__init__`` (CLI sees it via ``_vprint``); the gateway wires ``status_callback``
  1811. later, so ``replay_compression_warning`` resends it."""
  1812. if not agent.compression_enabled:
  1813. return
  1814. try:
  1815. from agent.auxiliary_client import (
  1816. _resolve_task_provider_model, _try_configured_fallback_for_unavailable_client,
  1817. get_text_auxiliary_client,
  1818. )
  1819. from agent.model_metadata import MINIMUM_CONTEXT_LENGTH, get_model_context_length
  1820. # Provider may be "auto"; fall back to the client's base_url hostname so the
  1821. # user can tell where the compression model is actually called.
  1822. try:
  1823. _aux_cfg_provider, _, _, _, _ = _resolve_task_provider_model("compression")
  1824. except Exception:
  1825. _aux_cfg_provider = ""
  1826. client, aux_model = get_text_auxiliary_client("compression", main_runtime=agent._current_main_runtime())
  1827. if client is None or not aux_model:
  1828. fb_client, fb_model, fb_label = _try_configured_fallback_for_unavailable_client(
  1829. "compression", _aux_cfg_provider
  1830. )
  1831. if fb_client is not None and fb_model:
  1832. client, aux_model = fb_client, fb_model
  1833. if "(" in fb_label and fb_label.endswith(")"):
  1834. _aux_cfg_provider = fb_label.rsplit("(", 1)[1][:-1]
  1835. if client is None or not aux_model:
  1836. if _aux_cfg_provider and _aux_cfg_provider != "auto":
  1837. msg = (
  1838. f"⚠ Configured auxiliary compression provider '{_aux_cfg_provider}' is unavailable, "
  1839. "so older messages in long chats will be cut without a summary. Sign in to that "
  1840. "provider again, or change auxiliary.compression in your config."
  1841. )
  1842. else:
  1843. msg = (
  1844. "⚠ No auxiliary LLM provider configured: Hermes has no helper model for summarising "
  1845. "long chats, so older messages will be cut without a summary. Run `hermes setup` to add one."
  1846. )
  1847. _emit_feasibility_notice(agent, msg)
  1848. logger.warning("No auxiliary LLM provider for compression — summaries will be unavailable.")
  1849. return
  1850. aux_base_url = str(getattr(client, "base_url", ""))
  1851. # client.api_key may be a callable (Entra bearer); the resolver only needs a key
  1852. # for live catalogue probes, so pass "" rather than mint a JWT for a lookup.
  1853. _raw_aux_key = getattr(client, "api_key", "")
  1854. aux_api_key = "" if (callable(_raw_aux_key) and not isinstance(_raw_aux_key, str)) else str(_raw_aux_key or "")
  1855. # Resolve each model with its own provider so provider-specific paths (Bedrock table, OpenRouter API)
  1856. # hit the correct client, not the main model's.
  1857. _aux_provider = (
  1858. _aux_cfg_provider if _aux_cfg_provider and _aux_cfg_provider != "auto" else getattr(agent, "provider", "")
  1859. )
  1860. _aux_cfg_ctx = _configured_compression_context_length(agent)
  1861. if _aux_cfg_ctx is None and _aux_inherits_main_route(agent, aux_model, aux_base_url):
  1862. # Same model on the same route: reuse the main model's already-resolved window (which honours
  1863. # model.context_length / provider pins). Re-resolving from scratch lost the pin and auto-lowered
  1864. # the session threshold to a catch-all catalog value (#89500, #45519).
  1865. aux_context = int(agent.context_compressor.context_length)
  1866. else:
  1867. aux_context = get_model_context_length(
  1868. aux_model, base_url=aux_base_url, api_key=aux_api_key, config_context_length=_aux_cfg_ctx,
  1869. provider=_aux_provider, custom_providers=_configured_custom_providers(agent),
  1870. )
  1871. # Aux model must meet MINIMUM_CONTEXT_LENGTH like the main model, else it cannot summarise a full window.
  1872. if aux_context and aux_context < MINIMUM_CONTEXT_LENGTH:
  1873. raise ValueError(
  1874. f"Auxiliary compression model {aux_model} has a context "
  1875. f"window of {aux_context:,} tokens, which is below the "
  1876. f"minimum {MINIMUM_CONTEXT_LENGTH:,} required by Hermes "
  1877. f"Agent. Choose a compression model with at least "
  1878. f"{MINIMUM_CONTEXT_LENGTH // 1000}K context (set "
  1879. f"auxiliary.compression.model in config.yaml), or set "
  1880. f"auxiliary.compression.context_length to override the "
  1881. f"detected value if it is wrong."
  1882. )
  1883. if aux_context < agent.context_compressor.threshold_tokens:
  1884. _lower_threshold_to_aux_context(
  1885. agent, aux_model=aux_model, aux_context=aux_context, aux_provider=_aux_cfg_provider,
  1886. aux_base_url=aux_base_url,
  1887. )
  1888. elif getattr(agent, "_last_feasibility_notice", None) is not None:
  1889. # Symmetric un-clamp: the summariser fits again, so the stale "auto-lowered" notice must not be
  1890. # replayed (``replay_compression_warning``) for a session that is no longer clamped (#114707).
  1891. agent._last_feasibility_notice = None
  1892. agent._compression_warning = None
  1893. except ValueError:
  1894. # Hard rejections (aux below minimum context) must propagate so the session refuses to start.
  1895. raise
  1896. except Exception as exc:
  1897. logger.debug("Compression feasibility check failed (non-fatal): %s", exc)
  1898. def replay_compression_warning(agent: Any) -> None:
  1899. """Re-send the stored compression warning through ``status_callback``.
  1900. Called once at the start of the first ``run_conversation()``, when the gateway callback (absent during
  1901. ``__init__``) is finally wired."""
  1902. msg = getattr(agent, "_compression_warning", None)
  1903. if msg and agent.status_callback:
  1904. # Replayed as a classified diagnostic so every sink applies its own policy snapshot.
  1905. from gateway.warning_notifications import DiagnosticText
  1906. with contextlib.suppress(Exception):
  1907. agent.status_callback("lifecycle", DiagnosticText(msg))
  1908. def conversation_history_after_compression(
  1909. agent: Any, messages: list, previous_history: Optional[list] = None
  1910. ) -> Optional[list]:
  1911. """Return the correct flush baseline after a compression boundary.
  1912. Session rotation returns ``None`` so the child gets the full compacted list. In-place compaction returns a
  1913. shallow copy of the already-persisted rows (else the identity flush re-appends them). Aborted/no-op
  1914. attempts keep the baseline: marking all persisted drops unflushed turns; clearing re-appends rows."""
  1915. if bool(getattr(agent, "_last_compression_attempt_recorded", False)):
  1916. attempt_in_place = getattr(agent, "_last_compression_attempt_in_place", None)
  1917. if attempt_in_place is True:
  1918. return list(messages)
  1919. if attempt_in_place is False:
  1920. return None
  1921. return previous_history
  1922. if bool(getattr(agent, "_last_compaction_in_place", False)):
  1923. return list(messages)
  1924. return None
  1925. _SYNTHETIC_USER_PREFIXES = (
  1926. "[System: Your previous response was truncated", "[System: The previous response was cut off",
  1927. "[System: Your previous tool call", "[Your active task list was preserved across context compression]",
  1928. "[IMPORTANT: Background process ",
  1929. )
  1930. def _message_text(message: Any) -> str:
  1931. content = message.get("content") if isinstance(message, dict) else None
  1932. if isinstance(content, str):
  1933. return content
  1934. if isinstance(content, list):
  1935. return "\n".join(
  1936. str(part.get("text") or part.get("content") or "") for part in content if isinstance(part, dict)
  1937. )
  1938. return ""
  1939. _SYNTHETIC_USER_FLAGS = (
  1940. "_todo_snapshot_synthetic", "_empty_recovery_synthetic", "_verification_stop_synthetic", "_pre_verify_synthetic",
  1941. "_dropped_toolcall_nudge",
  1942. )
  1943. def _is_real_user_message(message: Any) -> bool:
  1944. """Distinguish human intent from user-role runtime scaffolding.
  1945. A compaction summary flipped to ``role="user"`` for alternation is scaffolding and must not short-circuit
  1946. anchor restoration."""
  1947. if not isinstance(message, dict) or message.get("role") != "user":
  1948. return False
  1949. if any(message.get(flag) for flag in _SYNTHETIC_USER_FLAGS):
  1950. return False
  1951. text = _message_text(message).strip()
  1952. if not text or text.startswith(_SYNTHETIC_USER_PREFIXES):
  1953. return False
  1954. from agent.context_compressor import ContextCompressor
  1955. return not ContextCompressor._is_synthetic_compression_user_turn(message)
  1956. _STEER_FALLBACK_OPEN = "[OUT-OF-BAND USER MESSAGE"
  1957. _STEER_FALLBACK_CLOSE = "[/OUT-OF-BAND USER MESSAGE]"
  1958. def _steer_markers() -> tuple[str, str]:
  1959. """``(open, close)`` steer markers from prompt_builder, or the stable fallback literals."""
  1960. try:
  1961. from agent.prompt_builder import STEER_MARKER_CLOSE, STEER_MARKER_OPEN
  1962. return STEER_MARKER_OPEN, STEER_MARKER_CLOSE
  1963. except Exception:
  1964. return _STEER_FALLBACK_OPEN, _STEER_FALLBACK_CLOSE
  1965. def _message_contains_busy_steer(message: Any) -> bool:
  1966. """Return whether *message* carries a busy-steer marker.
  1967. Steer follow-ups are now their own ``role=user`` rows (caught by ``_is_real_user_message``); in
  1968. transcripts persisted before that they ride inside ``role=tool`` results, so those still count."""
  1969. text = _message_text(message)
  1970. if not text:
  1971. return False
  1972. open_marker, close_marker = _steer_markers()
  1973. return open_marker in text and close_marker in text
  1974. def _extract_steer_text_from_message(message: Any) -> Optional[str]:
  1975. """Extract the inner user text from a steer marker, or None."""
  1976. text = _message_text(message)
  1977. if not text:
  1978. return None
  1979. open_marker, close_marker = _steer_markers()
  1980. start = text.find(open_marker)
  1981. if start == -1:
  1982. # Fallback: marker wording may evolve; look for the stable prefix, then skip to the end of the opening line.
  1983. start = text.find(_STEER_FALLBACK_OPEN)
  1984. if start == -1:
  1985. return None
  1986. nl = text.find("\n", start)
  1987. start = nl + 1 if nl != -1 else start + len(_STEER_FALLBACK_OPEN)
  1988. else:
  1989. start += len(open_marker)
  1990. end = text.find(close_marker, start)
  1991. if end == -1:
  1992. end = text.find(_STEER_FALLBACK_CLOSE, start)
  1993. if end == -1:
  1994. return None
  1995. return text[start:end].strip() or None
  1996. def _compressed_has_busy_steer(messages: list) -> bool:
  1997. """Whether *messages* already carries a steer marker in a ``role=tool`` row.
  1998. Only tool rows count, so a summary merely quoting the marker text is not mistaken for live intent."""
  1999. return any(
  2000. isinstance(msg, dict) and msg.get("role") == "tool" and _message_contains_busy_steer(msg) for msg in messages
  2001. )
  2002. def _strip_stale_todo_snapshot(content: Any) -> Any:
  2003. """Remove a previously merged todo-snapshot block from message content.
  2004. Snapshots are appended to the trailing user turn, so a surviving header is stale; stripping before
  2005. re-injection prevents accumulation across boundaries."""
  2006. from tools.todo_tool import TODO_INJECTION_HEADER
  2007. if isinstance(content, str):
  2008. idx = content.find(TODO_INJECTION_HEADER)
  2009. if idx == -1:
  2010. return content
  2011. return content[:idx].rstrip()
  2012. if isinstance(content, list):
  2013. cleaned = []
  2014. for part in content:
  2015. text = str(part.get("text") or "") if isinstance(part, dict) and part.get("type") == "text" else ""
  2016. idx = text.find(TODO_INJECTION_HEADER) if text else -1
  2017. if idx == -1:
  2018. cleaned.append(part)
  2019. elif stripped := text[:idx].rstrip():
  2020. cleaned.append({**part, "text": stripped})
  2021. return cleaned
  2022. return content
  2023. def _todo_snapshot_is_only_content(content: Any, stripped: Any) -> bool:
  2024. """Return whether stripping the snapshot leaves no structured content.
  2025. Text snapshots trail a string; structured ones occupy their own text part, so only an empty remainder
  2026. proves the row was scaffolding alone. Text extraction is deliberately not used: image, audio and other
  2027. non-text parts must survive."""
  2028. if isinstance(content, str) and isinstance(stripped, str):
  2029. return not stripped.strip()
  2030. if isinstance(content, list) and isinstance(stripped, list):
  2031. return not stripped
  2032. return False
  2033. def _replace_message_content(message: dict, content: Any) -> None:
  2034. """Rewrite message content without allowing an old API sidecar to replay."""
  2035. from agent.turn_context import drop_stale_api_content
  2036. message["content"] = content
  2037. drop_stale_api_content(message)
  2038. # Compaction re-injects the todo list verbatim but prunes skills to markers, so
  2039. # couple them: tell the model to reload pruned skills BEFORE acting on tasks.
  2040. # Lives after TODO_INJECTION_HEADER so it strips with the snapshot next time.
  2041. _PRUNED_SKILL_RELOAD_NOTICE_HEADER = "[Skills pruned during compression — reload before acting on these tasks]"
  2042. def _pruned_skill_reload_notice(compressed: list) -> str:
  2043. """Reload notice for skills whose bodies were pruned, or ``""``.
  2044. Scans ``[SKILL_PRUNED: ...]`` markers in the post-compression transcript; first-seen order, deduplicated,
  2045. capped at ``_MAX_PRUNED_SKILL_MARKERS``."""
  2046. from agent.context_compressor import _MAX_PRUNED_SKILL_MARKERS, _extract_pruned_skill_names
  2047. names: list = []
  2048. for message in compressed:
  2049. if not isinstance(message, dict):
  2050. continue
  2051. for name in _extract_pruned_skill_names(_message_text(message)):
  2052. if name not in names:
  2053. names.append(name)
  2054. del names[_MAX_PRUNED_SKILL_MARKERS:]
  2055. if not names:
  2056. return ""
  2057. calls = "; ".join(f"skill_view(name='{name}')" for name in names)
  2058. return (
  2059. f"{_PRUNED_SKILL_RELOAD_NOTICE_HEADER}\n"
  2060. "The task list above crossed the compression boundary verbatim, but "
  2061. "the skill instructions that governed it were pruned. Before "
  2062. f"executing any preserved task that depends on these skills, reload "
  2063. f"them first: {calls}. After reloading, re-check that each pending "
  2064. "task is still justified — findings recorded before the boundary may have invalidated it."
  2065. )
  2066. def _merge_anchor_into_user_message(target: dict, anchor: dict) -> None:
  2067. """Fold the human anchor into an existing user-role scaffolding turn.
  2068. Used only when any insertion would create consecutive user turns. Anchor text leads, scaffolding follows,
  2069. and synthetic flags are cleared."""
  2070. anchor_content = anchor.get("content")
  2071. target_content = target.get("content")
  2072. if isinstance(anchor_content, list) or isinstance(target_content, list):
  2073. def _parts(content: Any) -> list:
  2074. return list(content) if isinstance(content, list) else [{"type": "text", "text": str(content or "")}]
  2075. _replace_message_content(target, _parts(anchor_content) + _parts(target_content))
  2076. else:
  2077. merged = f"{anchor_content or ''}\n\n{target_content or ''}".strip()
  2078. _replace_message_content(target, merged)
  2079. for flag in _SYNTHETIC_USER_FLAGS:
  2080. target.pop(flag, None)
  2081. # The anchor's text leads the composite, so the fold keeps the anchor's uid and records the
  2082. # scaffolding turn's (merge witness).
  2083. from agent.message_metadata import record_absorbed_message
  2084. record_absorbed_message(target, anchor, dropped_leads=True)
  2085. CompressedUserTurnOutcome = Literal["inserted", "merged", "already_present", "placeholder_appended"]
  2086. def _insert_real_user_anchor(messages: list, anchor: dict) -> CompressedUserTurnOutcome:
  2087. """Insert the latest human turn without breaking role alternation."""
  2088. from agent.context_compressor import _DB_PERSISTED_MARKER, ContextCompressor
  2089. def _role(msg: Any) -> Optional[str]:
  2090. return msg.get("role") if isinstance(msg, dict) else None
  2091. def _place(index: int) -> CompressedUserTurnOutcome:
  2092. anchor[_DB_PERSISTED_MARKER] = True
  2093. messages.insert(index, anchor)
  2094. return "inserted"
  2095. # Preferred anchor: the summary boundary — first assistant message not preceded
  2096. # by a user turn. Left neighbour is then non-user, right is an assistant.
  2097. for index, message in enumerate(messages):
  2098. if _role(message) == "assistant" and (index == 0 or _role(messages[index - 1]) != "user"):
  2099. return _place(index)
  2100. # Every assistant is user-preceded (or there are none). Appending is safe whenever the transcript does not already
  2101. # end with a user turn. Never merge into a summary either: its prefix must stay at message start for summary
  2102. # detection; repair_message_sequence merges adjacent user turns summary-first.
  2103. if (
  2104. not messages
  2105. or _role(messages[-1]) != "user"
  2106. or ContextCompressor._is_context_summary_content(_message_text(messages[-1]))
  2107. ):
  2108. return _place(len(messages))
  2109. # Trailing user-role scaffolding (e.g. the todo snapshot): merge instead
  2110. # of inserting a consecutive same-role message (#55677 strict templates).
  2111. _merge_anchor_into_user_message(messages[-1], anchor)
  2112. messages[-1][_DB_PERSISTED_MARKER] = True
  2113. return "merged"
  2114. def _ensure_compressed_has_user_turn(original_messages: list, compressed: list) -> CompressedUserTurnOutcome:
  2115. """Preserve human intent, not merely a synthetic user-role placeholder."""
  2116. if any(_is_real_user_message(message) for message in compressed) or _compressed_has_busy_steer(compressed):
  2117. return "already_present"
  2118. # Post-commit contract (#98450, mirrors _sync_micro_compact_to_db): archive_and_compact just durably
  2119. # wrote every dict in `compressed` as the new active set, but compress() returned marker-swept COPIES
  2120. # (_strip_persistence_markers, #57491). These exact dict instances become the live message list the
  2121. # caller keeps, so without the stamp the next _persist_session → _flush_messages_to_session_db_unlocked
  2122. # walk treats the whole compacted transcript as unpersisted and re-INSERTs it — the live set doubles on
  2123. # every compaction (~58K → ~512K tokens in production).
  2124. from agent.context_compressor import (
  2125. ContextCompressor, COMPRESSION_CONTINUATION_USER_CONTENT, _fresh_compaction_message_copy,
  2126. )
  2127. if any(ContextCompressor._has_merged_inflight_replay(message) for message in compressed):
  2128. # The in-flight request was restated onto the summary carrier (#100818); an anchor would duplicate it.
  2129. return "already_present"
  2130. # One reversed scan over BOTH kinds: scanning steer then user would let an older
  2131. # consumed steer outrank a newer real user request and replay it.
  2132. # One reversed positional scan: the anchor is whichever intent-bearing row is LAST in the original
  2133. # transcript — a real ``role=user`` turn or a steer marker riding inside a ``role=tool`` result. See
  2134. # #100053.
  2135. for message in reversed(original_messages):
  2136. if _is_real_user_message(message):
  2137. return _insert_real_user_anchor(compressed, _fresh_compaction_message_copy(message))
  2138. if not isinstance(message, dict) or message.get("role") != "tool":
  2139. continue
  2140. steer_text = _extract_steer_text_from_message(message)
  2141. if steer_text:
  2142. return _insert_real_user_anchor(compressed, {"role": "user", "content": steer_text})
  2143. from agent.message_metadata import append_message
  2144. append_message(compressed, {"role": "user", "content": COMPRESSION_CONTINUATION_USER_CONTENT})
  2145. return "placeholder_appended"
  2146. def _messages_match_scoped_identity(left: Any, right: Any) -> bool:
  2147. """Compare the live turn identity we care about for rotation stamping."""
  2148. if (
  2149. not isinstance(left, dict)
  2150. or not isinstance(right, dict)
  2151. or left.get("role") != right.get("role")
  2152. or left.get("content") != right.get("content")
  2153. ):
  2154. return False
  2155. left_timestamp = left.get("timestamp")
  2156. right_timestamp = right.get("timestamp")
  2157. if left_timestamp is not None and right_timestamp is not None:
  2158. return left_timestamp == right_timestamp
  2159. return True
  2160. def _stamp_scoped_twins(targets: list, source: dict, *, exact_counts_stamped: bool = False) -> None:
  2161. """Stamp ``_db_persisted`` on every unstamped scoped twin of ``source`` in ``targets``.
  2162. Exact-timestamp twins are preferred: when the source carries a timestamp and any exact twin was stamped
  2163. (or, with ``exact_counts_stamped``, merely exists), the broad scoped pass is skipped so a content-equal
  2164. old duplicate is left alone."""
  2165. from agent.context_compressor import _DB_PERSISTED_MARKER
  2166. source_timestamp = source.get("timestamp")
  2167. exact_hit = False
  2168. if source_timestamp is not None:
  2169. for target in targets:
  2170. if (
  2171. not isinstance(target, dict)
  2172. or target.get("timestamp") != source_timestamp
  2173. or not _messages_match_scoped_identity(target, source)
  2174. ):
  2175. continue
  2176. if target.get(_DB_PERSISTED_MARKER):
  2177. exact_hit = exact_hit or exact_counts_stamped
  2178. continue
  2179. target[_DB_PERSISTED_MARKER] = True
  2180. exact_hit = True
  2181. if exact_hit:
  2182. return
  2183. for target in targets:
  2184. if (
  2185. isinstance(target, dict)
  2186. and not target.get(_DB_PERSISTED_MARKER)
  2187. and _messages_match_scoped_identity(target, source)
  2188. ):
  2189. target[_DB_PERSISTED_MARKER] = True
  2190. _PENDING_CONTEXT_ENGINE_NOTIFICATION = "_pending_context_engine_compression_notification"
  2191. def _hand_off_metrics_segment(old_session_id: str, new_session_id: str) -> None:
  2192. """Shared metrics count one conversation across the rotation (a no-op when collection is off)."""
  2193. with _swallow('shared-metrics segment hand-off failed', exc_info=True):
  2194. from hermes_cli.observability.relay_shared_metrics import rotate_segment
  2195. rotate_segment(old_session_id, new_session_id)
  2196. def _notify_context_engine_compression_complete(agent: Any, *, new_session_id: str, old_session_id: str) -> bool:
  2197. """Notify the active context engine after a durable compression commit."""
  2198. # Opt-in relay session-span segmentation. Observer semantics — failure must
  2199. # never undo or delay the committed compression.
  2200. with _swallow('relay segment rotation notification failed', exc_info=True):
  2201. from agent import relay_runtime
  2202. relay_runtime.SESSION_COORDINATOR.notify_session_compacted(
  2203. profile_key=relay_runtime.current_profile_key(), session_id=new_session_id, old_session_id=old_session_id
  2204. )
  2205. _hand_off_metrics_segment(old_session_id, new_session_id)
  2206. callback = getattr(agent.context_compressor, "on_session_start", None)
  2207. if not callable(callback):
  2208. return False
  2209. try:
  2210. callback(
  2211. new_session_id, boundary_reason="compression", old_session_id=old_session_id,
  2212. platform=getattr(agent, "platform", None) or "cli",
  2213. conversation_id=getattr(agent, "_gateway_session_key", None),
  2214. )
  2215. return True
  2216. except Exception:
  2217. # Context-engine hooks are observers. A callback failure must not undo
  2218. # history that the core or an outer host transaction already committed.
  2219. logger.debug("context engine on_session_start (compression) failed", exc_info=True)
  2220. return False
  2221. def _queue_context_engine_compression_notification(agent: Any, *, new_session_id: str, old_session_id: str) -> None:
  2222. """Stage exactly one existing hook call for an outer host transaction."""
  2223. if callable(getattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, None)):
  2224. raise RuntimeError("a compression notification is already pending")
  2225. def _notify() -> bool:
  2226. return _notify_context_engine_compression_complete(
  2227. agent, new_session_id=new_session_id, old_session_id=old_session_id
  2228. )
  2229. setattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, _notify)
  2230. def finalize_context_engine_compression_notification(agent: Any, *, committed: bool) -> bool:
  2231. """Emit or discard a deferred notification; repeated calls are no-ops."""
  2232. pending = getattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, None)
  2233. setattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, None)
  2234. if not committed or not callable(pending):
  2235. return False
  2236. return bool(pending())
  2237. class _CompactionLifecycle:
  2238. """Owns the one-shot terminal edge of the compaction status lifecycle.
  2239. ``commit_status`` is rebound to "committed" only on success and read at ``complete()`` time, so abort
  2240. paths keep the terminal edge suppressed."""
  2241. def __init__(self, agent: Any, status_emitted: bool) -> None:
  2242. self._agent = agent
  2243. self.status_emitted = status_emitted
  2244. self._done_emitted = False
  2245. self.commit_status = "aborted"
  2246. def complete(self, *, force_terminal: bool = False) -> None:
  2247. if self._done_emitted:
  2248. return
  2249. self._done_emitted = True
  2250. # Suppressed start → no terminal edge. Non-compacting aborts (lock contender,
  2251. # cancelled fence) opt in via force_terminal so clients can retire their phase.
  2252. # Failure warnings go through _emit_warning and are never suppressed here.
  2253. if self.status_emitted and (self.commit_status == "committed" or force_terminal):
  2254. _emit_compaction_done(self._agent)
  2255. class _CompressionLease:
  2256. """The per-attempt durable compression lock plus its lifecycle plumbing.
  2257. ``holder`` is None when no durable lock is owned (legacy DB, no session db); ``watermark`` is the highest
  2258. active row the compaction already represents: MAX(id) at lease start, advanced to the newest row of an
  2259. adopted durable snapshot (None = archive everything, no concurrent-tail preservation this cycle)."""
  2260. def __init__(
  2261. self, agent: Any, *, db: Any, sid: str, ttl: float, refresh_interval: Any,
  2262. commit_fence: Optional[CompressionCommitFence], lifecycle: _CompactionLifecycle,
  2263. ) -> None:
  2264. self._agent = agent
  2265. self.db = db
  2266. self.sid = sid
  2267. self.ttl = ttl
  2268. self._refresh_interval = refresh_interval
  2269. self._commit_fence = commit_fence
  2270. self._lifecycle = lifecycle
  2271. self.holder: Optional[str] = None
  2272. self.watermark: Optional[int] = None
  2273. self._refresher: Optional[_CompressionLockLeaseRefresher] = None
  2274. self._released = False
  2275. self._release_guard = threading.Lock()
  2276. # Fence lock acquisition + release-hook publication together so a host timeout
  2277. # cannot win between acquiring the lock and having a way to release it.
  2278. self._lock_setup_entered = False
  2279. @property
  2280. def status_emitted(self) -> bool:
  2281. """True when the routine compaction start status was shown (heartbeats may follow it)."""
  2282. return self._lifecycle.status_emitted
  2283. def begin_lock_setup(self) -> bool:
  2284. if self._commit_fence is None:
  2285. return True
  2286. self._lock_setup_entered = self._commit_fence.begin_lock_setup()
  2287. return self._lock_setup_entered
  2288. def finish_lock_setup(self) -> None:
  2289. if not self._lock_setup_entered or self._commit_fence is None:
  2290. return
  2291. self._lock_setup_entered = False
  2292. self._commit_fence.finish_lock_setup()
  2293. def start_refresher(self) -> None:
  2294. if self.holder is None:
  2295. return
  2296. candidate = _CompressionLockLeaseRefresher(self.db, self.sid, self.holder, self.ttl, self._refresh_interval)
  2297. # Cancellation may release the holder between hook publication and this
  2298. # start; serialize with the release path so no refresher starts on a freed lock.
  2299. with self._release_guard:
  2300. if not self._released:
  2301. self._refresher = candidate.start()
  2302. def release_holder_only(self) -> None:
  2303. """Stop this holder's refresher and release only its durable lock.
  2304. Holder-qualified and idempotent: safe for the host after a timeout because a newer holder's lease can
  2305. never be deleted by this stale release."""
  2306. with self._release_guard:
  2307. if self._released:
  2308. return
  2309. self._released = True
  2310. if getattr(self._agent, "_active_compression_lock_holder", None) == self.holder:
  2311. self._agent._active_compression_lock_holder = None
  2312. if self._refresher is not None:
  2313. with _swallow('compression lock refresher stop failed: %s'):
  2314. self._refresher.stop()
  2315. if self.db is not None and self.sid and self.holder:
  2316. with _swallow('compression lock release failed: %s'):
  2317. self.db.release_compression_lock(self.sid, self.holder)
  2318. def release(self) -> None:
  2319. """Finish lifecycle cleanup and release the OLD session lock once."""
  2320. try:
  2321. self._lifecycle.complete()
  2322. finally:
  2323. try:
  2324. self.release_holder_only()
  2325. finally:
  2326. try:
  2327. if self._commit_fence is not None:
  2328. self._commit_fence.clear_cancelled_lock_release(self.release_holder_only)
  2329. finally:
  2330. self.finish_lock_setup()
  2331. def _resolve_lock_api(lock_db: Any) -> tuple[Any, Optional[Exception]]:
  2332. """Return ``(try_acquire_compression_lock, lookup_error)`` for ``lock_db``.
  2333. ``(None, None)`` = no db or legacy SessionDB without the lock API (fail open); ``(None, exc)`` = lookup
  2334. itself failed (caller fails closed)."""
  2335. if lock_db is None:
  2336. return None, None
  2337. try:
  2338. if _lock_api_is_absent_on_session_db(lock_db):
  2339. return None, None
  2340. try_acquire = lock_db.try_acquire_compression_lock
  2341. except Exception as exc:
  2342. return None, exc
  2343. if not callable(try_acquire):
  2344. return None, TypeError("compression lock API is present but not callable")
  2345. return try_acquire, None
  2346. def _abort_lease(
  2347. agent: Any, lifecycle: _CompactionLifecycle, system_message: str, attempt: _Attempt,
  2348. failure_class: str, prompt: Optional[str] = None,
  2349. ) -> tuple[None, str]:
  2350. """Sit-out return for lease acquisition: prompt, aborted telemetry, terminal status edge."""
  2351. if prompt is None:
  2352. prompt = _existing_system_prompt(agent, system_message)
  2353. _emit_aborted_attempt_telemetry(agent, attempt.started_at, failure_class, attempt.seed)
  2354. lifecycle.complete(force_terminal=True)
  2355. return None, prompt
  2356. def _try_acquire_durable_lock(lease: _CompressionLease, try_acquire: Any, commit_fence: Any) -> bool:
  2357. """Acquire the durable lock for ``lease.holder`` and capture the start watermark.
  2358. Watermark = MAX(id) of active rows at START: appends aren't blocked during summary; later rows are
  2359. concurrent tail that archive_and_compact re-sequences. Capture is safety-additive (fallback archives
  2360. everything), so its failure never aborts. An acquire that raises is not version skew: fail closed and
  2361. release holder-qualified best-effort (safe if never acquired)."""
  2362. try:
  2363. acquired = try_acquire(lease.sid, lease.holder, ttl_seconds=lease.ttl)
  2364. if acquired:
  2365. try:
  2366. lease.watermark = lease.db.get_active_message_watermark(lease.sid)
  2367. # A captured watermark makes the commit safe against later rows on BOTH commit
  2368. # paths; tell the fence so a host may keep this attempt's admission.
  2369. if commit_fence is not None:
  2370. with contextlib.suppress(AttributeError): # test doubles without the method
  2371. commit_fence.mark_commit_watermark_fenced()
  2372. except Exception as _wm_err:
  2373. logger.warning(
  2374. "compression watermark capture failed for session=%s (%s) — concurrent appends this cycle "
  2375. "will be archived with the snapshot", lease.sid, _wm_err,
  2376. )
  2377. lease.watermark = None
  2378. return acquired
  2379. except Exception as _lock_err:
  2380. with _swallow('compression lock cleanup after failed acquire failed: %s'):
  2381. lease.db.release_compression_lock(lease.sid, lease.holder)
  2382. lease.holder = None
  2383. logger.warning(
  2384. "compression lock acquisition raised unexpectedly for session=%s (%s: %s) — skipping compression this cycle",
  2385. lease.sid, type(_lock_err).__name__, _lock_err,
  2386. )
  2387. return False
  2388. def _sit_out_lock_contention(
  2389. agent: Any, lease: _CompressionLease, lifecycle: _CompactionLifecycle, system_message: str,
  2390. approx_tokens: Optional[int], attempt: _Attempt,
  2391. ) -> tuple[None, str]:
  2392. """Another path holds the lock: publish the lock-skip signal, warn once, sit out."""
  2393. existing = None
  2394. with contextlib.suppress(Exception):
  2395. existing = lease.db.get_compression_lock_holder(lease.sid)
  2396. logger.warning(
  2397. "compression skipped: another path is compressing session=%s "
  2398. "(holder=%s) — returning messages unchanged to avoid session fork", lease.sid, existing,
  2399. )
  2400. lease.holder = None # don't release a lock we don't own
  2401. # Distinguish lock-contention no-op from "nothing to compress" so manual
  2402. # /compress can show a clear status instead of "No changes".
  2403. agent._compression_skipped_due_to_lock = existing or True
  2404. # Surface to the user once — quiet for downstream auto-compress loops
  2405. if getattr(agent, "_last_compression_lock_warning_sid", None) != lease.sid:
  2406. agent._last_compression_lock_warning_sid = lease.sid
  2407. with contextlib.suppress(Exception):
  2408. agent._emit_warning(
  2409. "⚠ Skipping concurrent compression — another path is already compressing this session. Will retry "
  2410. "after it finishes."
  2411. )
  2412. # The record comes from this attempt's seed: the compressor's telemetry belongs to the lock holder when both
  2413. # share a compressor, so writing it here would erase the running attempt's record.
  2414. return _abort_lease(agent, lifecycle, system_message, attempt, "lock_contended", _existing_system_prompt(
  2415. agent, system_message))
  2416. def _acquire_compression_lease(
  2417. agent: Any, *, commit_fence: Optional[CompressionCommitFence], lifecycle: _CompactionLifecycle,
  2418. system_message: str, approx_tokens: Optional[int], attempt: _Attempt,
  2419. ) -> tuple[Optional[_CompressionLease], Optional[str]]:
  2420. """Take the per-session compression lock; ``(None, prompt)`` means sit out.
  2421. Two AIAgents sharing a session_id (e.g. background review fork) would both rotate and orphan a child.
  2422. Keyed on the OLD id (what rivals read from SessionEntry). Loser sits out: messages unchanged, caller sees
  2423. no-op. Only structural absence of the lock API (version skew) fails open; once resolved, any exception
  2424. fails closed since unlocked runs can fork lineage."""
  2425. _lock_db = getattr(agent, "_session_db", None)
  2426. _lock_sid = agent.session_id or ""
  2427. # Clear stale lock-skip so this call's outcome alone is visible; else a manual
  2428. # /compress after an auto lock-skip falsely reports "already in progress".
  2429. agent._compression_skipped_due_to_lock = None
  2430. _try_acquire_lock, _lock_lookup_error = _resolve_lock_api(_lock_db)
  2431. _lock_ttl = 300.0
  2432. with contextlib.suppress(TypeError, ValueError):
  2433. _lock_ttl = float(getattr(agent, "_compression_lock_ttl_seconds", 300.0) or 300.0)
  2434. lease = _CompressionLease(
  2435. agent, db=_lock_db, sid=_lock_sid, ttl=_lock_ttl,
  2436. refresh_interval=getattr(agent, "_compression_lock_refresh_interval", None), commit_fence=commit_fence,
  2437. lifecycle=lifecycle,
  2438. )
  2439. if _lock_db is not None and _lock_sid:
  2440. lease.holder = _compression_lock_holder(agent)
  2441. if _lock_lookup_error is not None:
  2442. # Attribute lookup itself failed for a reason other than a missing
  2443. # lock API. It is unsafe to proceed without a lock in that case.
  2444. lease.holder = None
  2445. logger.warning(
  2446. "compression lock lookup raised unexpectedly for session=%s (%s: %s) — skipping compression this cycle",
  2447. _lock_sid, type(_lock_lookup_error).__name__, _lock_lookup_error,
  2448. )
  2449. _lock_acquired = False
  2450. elif _try_acquire_lock is None:
  2451. # Lock API absent on this instance: log once, proceed unlocked so version skew
  2452. # cannot stall the outer auto-compression loop forever.
  2453. lease.holder = None
  2454. if getattr(agent, "_last_compression_lock_error_sid", None) != _lock_sid:
  2455. agent._last_compression_lock_error_sid = _lock_sid
  2456. logger.warning(
  2457. "compression lock subsystem unavailable for session=%s — proceeding without lock. This usually means a stale "
  2458. "in-memory module after an update; restart the process (or `hermes update`) to resync.",
  2459. _lock_sid,
  2460. )
  2461. _lock_acquired = True # acquired-but-unlocked compatibility path
  2462. else:
  2463. if not lease.begin_lock_setup():
  2464. logger.info(
  2465. "Compression commit cancelled before lock acquisition (session=%s).", agent.session_id or "none"
  2466. )
  2467. agent._last_compaction_in_place = False
  2468. return _abort_lease(agent, lifecycle, system_message, attempt, "commit_fence_cancelled")
  2469. _lock_acquired = _try_acquire_durable_lock(lease, _try_acquire_lock, commit_fence)
  2470. if not _lock_acquired:
  2471. lease.finish_lock_setup()
  2472. return _sit_out_lock_contention(
  2473. agent, lease, lifecycle, system_message, approx_tokens, attempt
  2474. )
  2475. if lease.holder is not None:
  2476. agent._active_compression_lock_holder = lease.holder
  2477. if commit_fence is not None and commit_fence.register_cancelled_lock_release(lease.release_holder_only):
  2478. # Cancellation won during lock setup (hook ran synchronously, lease gone): abort before any summary work.
  2479. logger.info(
  2480. "Compression commit cancelled before summary dispatch (session=%s).", agent.session_id or "none"
  2481. )
  2482. agent._last_compaction_in_place = False
  2483. _existing_sp = _existing_system_prompt(agent, system_message)
  2484. _emit_aborted_attempt_telemetry(agent, attempt.started_at, "commit_fence_cancelled", attempt.seed)
  2485. lease.release()
  2486. return None, _existing_sp
  2487. return lease, None
  2488. def _adopt_if_parent_rotated(
  2489. agent: Any, lease: _CompressionLease, messages: list, system_message: str, *, attempt: _Attempt,
  2490. approx_tokens: Optional[int],
  2491. ) -> Optional[tuple[list, str]]:
  2492. """Sit out (or adopt the live child) when the parent was already rotated.
  2493. A late contender can take the parent lock after the winner released it and rotated; holding the lock does
  2494. not prove this agent still owns a live parent. Returns the ``compress_context`` result to hand back, or
  2495. None to proceed. Each returning exit logs its own attempt record."""
  2496. if lease.db is None or not lease.sid:
  2497. return None
  2498. try:
  2499. _parent_already_rotated = _session_was_rotated_by_compression(lease.db, lease.sid)
  2500. except Exception as _session_err:
  2501. logger.warning(
  2502. "compression session ownership lookup failed for session=%s (%s: %s) - skipping compression this cycle",
  2503. lease.sid, type(_session_err).__name__, _session_err,
  2504. )
  2505. lease.release()
  2506. _emit_bypassed_attempt_telemetry(
  2507. agent, attempt.started_at, commit_status="aborted", failure_class="session_ownership_unreadable",
  2508. approx_tokens=approx_tokens, attempt_seed=attempt.seed,
  2509. )
  2510. return messages, _existing_system_prompt(agent, system_message)
  2511. if not _parent_already_rotated:
  2512. return None
  2513. recovered_messages = _adopt_live_compression_child(agent, lease.db, lease.sid)
  2514. lease.release()
  2515. _emit_bypassed_attempt_telemetry(
  2516. agent, attempt.started_at, commit_status="skipped", failure_class="session_ownership_lost",
  2517. approx_tokens=approx_tokens, attempt_seed=attempt.seed,
  2518. )
  2519. _existing_sp = _existing_system_prompt(agent, system_message)
  2520. if recovered_messages is not None:
  2521. logger.warning("compression recovery: stale session=%s adopted live child=%s", lease.sid, agent.session_id)
  2522. return recovered_messages, _existing_sp
  2523. logger.warning(
  2524. "compression skipped: session=%s was already rotated by "
  2525. "another compression path, but no unique live child could be adopted", lease.sid,
  2526. )
  2527. return messages, _existing_sp
  2528. def _adopt_grown_durable_parent(agent: Any, lease: _CompressionLease, messages: list) -> Optional[list]:
  2529. """Return the durable parent transcript when it outgrew the in-memory snapshot.
  2530. Rotation only (in-place never loses rows). The snapshot predates the lease: if durable grew, a writer
  2531. committed a turn — ADOPT it (aborting wedged busy sessions forever). Length check only: in-memory edits of
  2532. past turns are legal. Adoption advances ``lease.watermark`` to the snapshot's newest row, so publication
  2533. clones only rows the snapshot does not already carry."""
  2534. if lease.db is None or not lease.sid:
  2535. return None
  2536. durable_loader = getattr(type(lease.db), "get_messages_as_conversation", None)
  2537. if not callable(durable_loader):
  2538. return None
  2539. durable_parent = durable_loader(lease.db, lease.sid)
  2540. if not (isinstance(durable_parent, list) and len(durable_parent) > len(messages)):
  2541. return None
  2542. # In-memory carries this turn's un-persisted user tail; flush it via the normal
  2543. # rotation-boundary path before adopting, else skip adoption (would drop input).
  2544. # The in-memory transcript carries the CURRENT turn's un-persisted user tail (anchored by
  2545. # _persist_user_message_idx) that the durable snapshot read above does not contain yet. Flush that tail
  2546. # through the normal rotation-boundary path (conversation_history = the already-durable prefix, #68196
  2547. # boundary) BEFORE adopting, then re-read the durable parent so the adopted snapshot includes the live
  2548. # input. If the flush fails (or the anchor is unknown), skip adoption entirely: replacing the in-memory
  2549. # transcript with a snapshot that lacks the user's input would silently drop it from the summarized and
  2550. # rotated history (#adopt-live-tail).
  2551. _preflush_idx = getattr(agent, "_persist_user_message_idx", None)
  2552. # No un-persisted tail means the transcript is fully durable: adopting the longer parent cannot drop input.
  2553. _preflush_ok = True
  2554. if isinstance(_preflush_idx, int) and 0 <= _preflush_idx < len(messages):
  2555. _preflush_ok = False
  2556. with contextlib.suppress(Exception):
  2557. _preflush_ok = agent._flush_messages_to_session_db(messages, conversation_history=messages[:_preflush_idx])
  2558. if not _preflush_ok:
  2559. logger.warning(
  2560. "compression: session=%s grew before lease (%d → %d msgs) but the pre-adoption flush of the "
  2561. "live tail failed; skipping durable-snapshot adoption so un-persisted user input is kept",
  2562. lease.sid, len(messages), len(durable_parent),
  2563. )
  2564. return None
  2565. # Re-read after the flush so the adopted snapshot carries the just-persisted tail.
  2566. durable_parent = durable_loader(lease.db, lease.sid, include_row_ids=True)
  2567. if not (isinstance(durable_parent, list) and len(durable_parent) > len(messages)):
  2568. return None
  2569. adopted_row_ids = [rid for m in durable_parent if isinstance(rid := m.pop("_row_id", None), int)]
  2570. if lease.watermark is not None and adopted_row_ids:
  2571. lease.watermark = max(lease.watermark, *adopted_row_ids)
  2572. logger.info(
  2573. "compression: session=%s grew before lease (%d → %d msgs); adopting durable snapshot", lease.sid, len(messages),
  2574. len(durable_parent),
  2575. )
  2576. return durable_parent
  2577. def _pre_compress_memory_context(agent: Any, messages: list, checkpoint_required: bool) -> str:
  2578. """Provider ``on_pre_compress()`` insights to surface in the summary ("" if none).
  2579. Raw messages stay the API v1 provider contract; normalized evidence goes only to API v2+ checkpoint
  2580. providers inside MemoryManager.on_pre_compress(). Raises :class:`CompressionCheckpointUnavailable` when a
  2581. required checkpoint cannot be taken."""
  2582. memory_context = ""
  2583. memory_manager = getattr(agent, "_memory_manager", None)
  2584. evidence_messages = _direct_messages_for_pre_compress_memory(messages)
  2585. # A detached review/side-question fork (no memory manager, session DB or persistence) cannot checkpoint by design.
  2586. if checkpoint_required and getattr(agent, "_persist_disabled", False) and memory_manager is None:
  2587. checkpoint_required = getattr(agent, "_session_db", None) is not None
  2588. if checkpoint_required:
  2589. supports_checkpoint = getattr(memory_manager, "supports_pre_compress_checkpoint", None)
  2590. if memory_manager is None or not callable(supports_checkpoint):
  2591. raise _checkpoint_incapable(f"no active provider implements checkpoint API v{PRE_COMPRESS_CHECKPOINT_API_VERSION}")
  2592. try:
  2593. compatible = bool(supports_checkpoint(PRE_COMPRESS_CHECKPOINT_API_VERSION))
  2594. except Exception as exc:
  2595. raise _checkpoint_blocked("provider capability probe failed") from exc
  2596. if not compatible:
  2597. raise _checkpoint_incapable(f"active provider does not implement checkpoint API v{PRE_COMPRESS_CHECKPOINT_API_VERSION}")
  2598. try:
  2599. _maybe_ctx = memory_manager.on_pre_compress(
  2600. messages, evidence_messages=evidence_messages, require_checkpoint=True,
  2601. checkpoint_api_version=PRE_COMPRESS_CHECKPOINT_API_VERSION,
  2602. )
  2603. except Exception as exc:
  2604. logger.warning("Required pre-compress checkpoint failed (%s)", type(exc).__name__)
  2605. raise _checkpoint_blocked(f"provider checkpoint API v{PRE_COMPRESS_CHECKPOINT_API_VERSION} failed") from exc
  2606. if isinstance(_maybe_ctx, str):
  2607. memory_context = sanitize_memory_context(_maybe_ctx)
  2608. elif memory_manager:
  2609. with contextlib.suppress(Exception):
  2610. _maybe_ctx = memory_manager.on_pre_compress(messages, evidence_messages=evidence_messages)
  2611. if isinstance(_maybe_ctx, str):
  2612. memory_context = sanitize_memory_context(_maybe_ctx)
  2613. return memory_context
  2614. def _resolve_compress_call(
  2615. agent: Any, *, approx_tokens: Optional[int], focus_topic: Optional[str], force: bool, memory_context: str,
  2616. bypass_cooldown: bool,
  2617. ) -> tuple[Callable[..., Any], dict[str, Any]]:
  2618. """Bind ``compress()`` and only the kwargs its signature accepts."""
  2619. compress_fn = agent.context_compressor.compress
  2620. compress_kwargs = _supported_compression_kwargs(
  2621. compress_fn, current_tokens=approx_tokens, focus_topic=focus_topic, force=force, memory_context=memory_context,
  2622. bypass_cooldown=bypass_cooldown,
  2623. )
  2624. if memory_context.strip() and "memory_context" not in compress_kwargs:
  2625. engine_name = getattr(agent.context_compressor, "name", type(agent.context_compressor).__name__)
  2626. if getattr(agent, "_last_memory_context_unsupported_engine", None) != engine_name:
  2627. agent._last_memory_context_unsupported_engine = engine_name
  2628. logger.warning(
  2629. "context engine %s does not accept memory_context; continuing without provider-supplied summary context",
  2630. engine_name,
  2631. )
  2632. return compress_fn, compress_kwargs
  2633. def _run_summary_dispatch(
  2634. agent: Any, messages: list, compress_fn: Callable[..., Any], compress_kwargs: dict[str, Any], *,
  2635. commit_fence: Optional[CompressionCommitFence], attempt_generation: Any, hard_cancel_event: Any,
  2636. ) -> list:
  2637. """Run the compressor under the fence's progress hook, deadline and interrupt guard."""
  2638. # Publish progress to the commit fence so hosts extend deadlines while tokens
  2639. # flow. Any active hook (even no-op) selects the streamed path: the timeout is
  2640. # inactivity-based and a byte-trickling provider hits the stream total ceiling.
  2641. from agent.auxiliary_client import aux_interrupt_protection, aux_progress_hook
  2642. from agent.conversation_compression_deadline import compression_deadline_scope
  2643. _progress_hook = commit_fence.touch_progress if commit_fence is not None else (lambda: None)
  2644. # Return leg: cancel frees the owner but the provider daemon streams on to its own larger
  2645. # ceiling; share the host deadline (live: prompt reading pushes it back) so orphans stop with it.
  2646. # A LATE successful summary must not undo the host's timeout cooldown: the
  2647. # compressor checks cancellation before clearing; removed in finally (no leak).
  2648. if commit_fence is not None:
  2649. # Install a cancellation check the compressor consults BEFORE clearing the failure cooldown; removed
  2650. # in the finally below so it cannot leak into later attempts (e.g. a manual /compress force-clear).
  2651. # See #76354.
  2652. _install_compression_cancelled_check(
  2653. agent.context_compressor, lambda: commit_fence.is_cancelled, attempt_generation
  2654. )
  2655. def _compression_cancel_requested() -> bool:
  2656. return bool(
  2657. (hard_cancel_event is not None and hard_cancel_event.is_set())
  2658. or (commit_fence is not None and commit_fence.is_cancelled)
  2659. )
  2660. _attempt_ctx_token = _COMPRESSOR_ATTEMPT_GENERATION.set(attempt_generation)
  2661. try:
  2662. # F6: never start expensive summary work for an already-cancelled
  2663. # fence (a stale queued job admitted after host departure).
  2664. if commit_fence is not None and commit_fence.is_cancelled:
  2665. logger.info(
  2666. "Compression cancelled before summary dispatch (session=%s) — skipping summary work.",
  2667. agent.session_id or "none",
  2668. )
  2669. compressed = messages
  2670. else:
  2671. with (
  2672. aux_progress_hook(_progress_hook), compression_deadline_scope(commit_fence),
  2673. aux_interrupt_protection(cancel_check=_compression_cancel_requested),
  2674. ):
  2675. # This attempt is now doing real summary work: publish it as the working attempt so later
  2676. # no-op entry claims (lock sit-outs, gates, the cancelled-fence skip above) cannot supersede
  2677. # the candidate this run produces (#112482).
  2678. _mark_compressor_working_attempt(agent.context_compressor, attempt_generation)
  2679. compressed = compress_fn(messages, **compress_kwargs)
  2680. # Freeze a hard stop that arrived after the last provider attempt but before session state rotates.
  2681. if hard_cancel_event is not None and hard_cancel_event.is_set():
  2682. raise AuxiliaryExplicitCancellation()
  2683. finally:
  2684. _COMPRESSOR_ATTEMPT_GENERATION.reset(_attempt_ctx_token)
  2685. if commit_fence is not None:
  2686. _clear_compression_cancelled_check_if_owner(agent.context_compressor, attempt_generation)
  2687. return compressed
  2688. def _fold_todo_snapshot(agent: Any, compressed: list) -> None:
  2689. """Strip stale todo snapshots from ``compressed`` and fold the live one in (in place)."""
  2690. todo_snapshot = agent._todo_store.format_for_injection()
  2691. # Non-empty store (even all done) is authoritative: drop the old snapshot. A
  2692. # truly empty store may be un-rehydrated post-compaction: keep the snapshot.
  2693. _todo_has_items = getattr(agent._todo_store, "has_items", None)
  2694. # Store may implement only format_for_injection(); unknown authority must
  2695. # preserve the pending snapshot rather than risk deleting it.
  2696. _todo_store_is_authoritative = False
  2697. with contextlib.suppress(Exception):
  2698. _todo_store_is_authoritative = bool(_todo_has_items()) if callable(_todo_has_items) else False
  2699. if _todo_store_is_authoritative:
  2700. for _todo_idx in range(len(compressed) - 1, -1, -1):
  2701. _todo_message = compressed[_todo_idx]
  2702. if not isinstance(_todo_message, dict) or _todo_message.get("role") != "user":
  2703. continue
  2704. _todo_content = _todo_message.get("content")
  2705. _todo_stripped = _strip_stale_todo_snapshot(_todo_content)
  2706. if _todo_stripped == _todo_content:
  2707. continue
  2708. if _todo_message.get("_todo_snapshot_synthetic") and _todo_snapshot_is_only_content(
  2709. _todo_content, _todo_stripped
  2710. ):
  2711. compressed.pop(_todo_idx)
  2712. if _todo_idx < len(compressed):
  2713. # A standalone snapshot can drift from the tail; deleting it may expose two
  2714. # assistant rows, so use the normal replay repair to keep metadata consistent.
  2715. agent._repair_message_sequence(compressed)
  2716. else:
  2717. _replace_message_content(_todo_message, _todo_stripped)
  2718. # No longer todo-only scaffolding; other synthetic flags stay authoritative and
  2719. # _is_real_user_message() recomputes provenance from content + flags.
  2720. _todo_message.pop("_todo_snapshot_synthetic", None)
  2721. break
  2722. if todo_snapshot:
  2723. # If this boundary pruned skill bodies, the policy behind the todos is gone:
  2724. # add a reload notice after TODO_INJECTION_HEADER so both strip together.
  2725. # Retention parity (#84718): the snapshot below re-injects the imperative verbatim. If this same
  2726. # boundary pruned skill bodies to [SKILL_PRUNED: ...] markers, the policy that governed those tasks
  2727. # is gone — couple a reload instruction to the snapshot so the imperative never crosses the boundary
  2728. # alone.
  2729. _reload_notice = _pruned_skill_reload_notice(compressed)
  2730. if _reload_notice:
  2731. todo_snapshot = f"{todo_snapshot}\n\n{_reload_notice}"
  2732. # Fold the snapshot into a trailing REAL user msg (no synthetic user/user pair);
  2733. # strip old snapshots first. Scaffolding tails must not absorb it (provenance).
  2734. # Any snapshot merged at an earlier boundary is stripped first so repeated compactions refresh
  2735. # rather than accumulate todo state (#26981). Scaffolding tails (continuation marker, summary
  2736. # handoff, a bare stale snapshot row) must never absorb the snapshot: merging would upgrade them to
  2737. # "real user" evidence and break zero-user provenance (#69292), so those keep the flagged standalone
  2738. # append and the real-user preservation pass continues to see todo scaffolding, not human intent.
  2739. from agent.context_compressor import _append_text_to_content
  2740. merged = False
  2741. _tail = compressed[-1] if compressed and isinstance(compressed[-1], dict) else None
  2742. if _tail is not None and _tail.get("role") == "user":
  2743. _stripped = _strip_stale_todo_snapshot(_tail.get("content"))
  2744. _probe = {key: value for key, value in _tail.items() if key != "content"}
  2745. _probe["content"] = _stripped
  2746. if _is_real_user_message(_probe):
  2747. _snapshot_text = f"\n\n{todo_snapshot}" if isinstance(_stripped, str) and _stripped else todo_snapshot
  2748. _replace_message_content(_tail, _append_text_to_content(_stripped, _snapshot_text))
  2749. merged = True
  2750. elif (
  2751. _stripped != _tail.get("content") and not _message_text({"role": "user", "content": _stripped}).strip()
  2752. ):
  2753. # The tail was nothing but an earlier snapshot row —
  2754. # refresh it in place instead of stacking a duplicate.
  2755. _replace_message_content(_tail, todo_snapshot)
  2756. _tail["_todo_snapshot_synthetic"] = True
  2757. merged = True
  2758. if not merged:
  2759. compressed.append({"role": "user", "content": todo_snapshot, "_todo_snapshot_synthetic": True})
  2760. def _rebuild_system_prompt_at_boundary(agent: Any, system_message: str) -> str:
  2761. """Refresh tool schemas and rebuild the system prompt at the commit boundary."""
  2762. if getattr(agent, "_retain_seeded_system_prompt", False) is True:
  2763. # Gateway hygiene / gateway /compress run a detached agent with a reduced toolset and no live
  2764. # surface: its builder output drops the skills index, external provider blocks and tool guidance,
  2765. # and the commit below would persist that over the live session's snapshot (restored verbatim by
  2766. # the next fresh agent). Keep the seeded bytes; the live agent's own compaction propagates updates.
  2767. # Returning here also deliberately skips _refresh_agent_tool_definitions: its MCP refresh persists
  2768. # the agent's tool names, which would overwrite the session's saved tools[] with the memory-only set.
  2769. if agent._cached_system_prompt:
  2770. from agent.system_prompt import reconstruct_static_prefix
  2771. reconstruct_static_prefix(agent, system_message=system_message, log_label="compression seeded-prompt")
  2772. return agent._cached_system_prompt
  2773. cached_system_prompt = agent._cached_system_prompt
  2774. agent._invalidate_system_prompt()
  2775. # Refresh dynamic tool schemas at the same admitted-commit boundary that rebuilds the system prompt
  2776. # (maintainer-directed, #95681 arc): forever-sessions (Bot Mode chats, gateway channels) never
  2777. # restart, so compaction is the ONLY point where a config change — image model swap, delegation
  2778. # depth, code_execution mode — can reach agent.tools. The prompt cache is already broken here, so
  2779. # the refresh is free; when nothing changed the snapshot is byte-equal and we keep the existing list
  2780. # object (identity matters to provider-side tool-block caching on some backends).
  2781. try:
  2782. _refresh_agent_tool_definitions(agent)
  2783. except Exception:
  2784. logger.warning(
  2785. "compaction tool-definition refresh failed; keeping the session's existing tool snapshot", exc_info=True
  2786. )
  2787. # ALWAYS rebuild the prompt here: keeping old bytes meant prompt-builder changes
  2788. # never reached long sessions. Equal bytes keep KV; preserve object identity.
  2789. # ALWAYS rebuild the prompt at the admitted-commit boundary (maintainer-directed, #95681 arc). The
  2790. # previous "keep-prompt" containment branch put the OLD bytes back whenever the reloaded memory blocks
  2791. # were already embedded — which meant prompt-builder changes (guidance diets, new blocks, renames) NEVER
  2792. # reached a long-lived session. The cache argument for keeping bytes was hollow: when nothing changed,
  2793. # the rebuild is byte-identical and local KV prefixes survive on equality; when something changed, the
  2794. # cache was stale by definition and propagation is the point. Preserve OBJECT identity on byte-equality
  2795. # for backends that key on it.
  2796. rebuilt_system_prompt = agent._build_system_prompt(system_message)
  2797. if cached_system_prompt is not None and rebuilt_system_prompt == cached_system_prompt:
  2798. new_system_prompt = agent._cached_system_prompt = cached_system_prompt
  2799. from agent.system_prompt import reconstruct_static_prefix
  2800. reconstruct_static_prefix(agent, system_message=system_message, log_label="compression keep-prompt")
  2801. else:
  2802. new_system_prompt = agent._cached_system_prompt = rebuilt_system_prompt
  2803. if cached_system_prompt is not None:
  2804. # The rebuild itself stays mandatory; only the first drift per session is INFO — a long session
  2805. # compacting many times logged this on every compact (19x/day in #112420).
  2806. log = logger.debug if getattr(agent, "_compaction_prompt_drift_logged", False) is True else logger.info
  2807. log(
  2808. "Compaction rebuilt a drifted system prompt (session=%s, %d -> %d chars): builder output changed "
  2809. "since the stored snapshot (update, config change, or memory/skills growth)",
  2810. agent.session_id or "none", len(cached_system_prompt), len(new_system_prompt),
  2811. )
  2812. agent._compaction_prompt_drift_logged = True
  2813. return new_system_prompt
  2814. def _salvage_or_refuse_grown_transcript(
  2815. agent: Any, original_messages: list, compressed: list, *, system_message: str, attempt: _Attempt,
  2816. ) -> tuple[Optional[list], Optional[str]]:
  2817. """Anti-growth guard at the COMMIT SITE (in-place commits before the gateway can inspect).
  2818. Compares like-for-like rough estimates; on growth tries one mechanical salvage pass, else treats the
  2819. attempt as a refused no-op. Returns ``(compressed, None)`` to proceed or ``(None, prompt)`` when refused
  2820. (caller releases the lease)."""
  2821. # Anti-growth guard at the COMMIT SITE: never persist a compression that makes the transcript larger
  2822. # (observed: 379K -> 687K when the generated summary plus retained reasoning exceeded what it replaced).
  2823. # Compare like-for-like (both rough estimates of the same message shape) so an "actual vs estimate"
  2824. # measurement mismatch cannot produce a false verdict. The gateway has a rotation-path-only guard
  2825. # (#83339), but in-place compaction commits inside this method via archive_and_compact — before the
  2826. # gateway can inspect the result — so the guard must live here to protect both paths. On growth, treat
  2827. # the attempt as a no-op: the original transcript stays untouched and durable.
  2828. _rough_in = estimate_messages_tokens_rough(original_messages)
  2829. _rough_out = estimate_messages_tokens_rough(compressed)
  2830. if _rough_out > _rough_in:
  2831. # Todo refresh and user-turn anchoring run after the compressor's own size check
  2832. # and can tip a break-even candidate; give it one mechanical salvage pass.
  2833. from agent.context_compressor import salvage_grown_transcript
  2834. _salvaged = salvage_grown_transcript(original_messages, compressed, budget=_rough_in)
  2835. if _salvaged is not None:
  2836. _salv_est = estimate_messages_tokens_rough(_salvaged)
  2837. if _salv_est < _rough_in:
  2838. logger.info(
  2839. "Compression salvage recovered a shrinking transcript (session=%s, ~%s -> ~%s tokens)",
  2840. agent.session_id or "none", f"{_rough_in:,}", f"{_salv_est:,}",
  2841. )
  2842. compressed = _salvaged
  2843. _rough_out = _salv_est
  2844. if _rough_out > _rough_in:
  2845. logger.warning(
  2846. "Compression refused: compressed transcript would be larger than the original (session=%s, ~%s -> ~%s "
  2847. "tokens); keeping the original transcript unchanged", agent.session_id or "none",
  2848. f"{_rough_in:,}",
  2849. f"{_rough_out:,}",
  2850. )
  2851. # Flag the refusal on compressor state so /compress feedback reports it instead
  2852. # of comparing list lengths (adoption can change the count), claiming success.
  2853. with contextlib.suppress(Exception):
  2854. agent.context_compressor._last_compress_refused_would_grow = True
  2855. with contextlib.suppress(Exception):
  2856. agent._emit_warning(
  2857. "⚠️ Compression refused: the generated summary would have GROWN the conversation instead of "
  2858. "shrinking it. No messages were dropped — conversation continues unchanged."
  2859. )
  2860. _existing_sp = _existing_system_prompt(agent, system_message)
  2861. _emit_aborted_attempt_telemetry(agent, attempt.started_at, "would_grow", attempt.seed)
  2862. # Count the refusal as an ineffective-compaction strike so the anti-thrash
  2863. # breaker latches; otherwise auto-compress retries the same summary every turn.
  2864. with _swallow('could not record rejected-compaction strike', exc_info=True):
  2865. # Without this, the unchanged transcript stays over the compression threshold and automatic
  2866. # compression retries the identical summary request on every turn (#88568). Manual /compress
  2867. # keeps bypassing the latch (force=True skips the guards).
  2868. agent.context_compressor.record_rejected_compaction()
  2869. _restore_prune_rearm_tokens(agent.context_compressor, attempt.snapshot)
  2870. return None, _existing_sp
  2871. return compressed, None
  2872. def _parent_deliberately_ended(session_db: Any, session_id: str) -> bool:
  2873. """True when the parent row was ended by a non-automatic reason. Fails OPEN: an
  2874. unreadable row must not turn a cheap guard into a new way to lose compression."""
  2875. reader = getattr(session_db, "get_session", None)
  2876. if not callable(reader):
  2877. return False
  2878. try:
  2879. from hermes_state_common import is_automatic_end_reason
  2880. row = reader(session_id) or {}
  2881. return row.get("ended_at") is not None and not is_automatic_end_reason(row.get("end_reason"))
  2882. except Exception:
  2883. return False
  2884. def _carry_session_state_to_child(agent: Any, old_session_id: str, old_title: Any) -> None:
  2885. """Migrate /goal, /heartbeat, /loop state, rejected-thinking fingerprints and the title from the parent to the child.
  2886. Each lookup is a flat per-session read with no parent walk, so state would silently die at the boundary. The title
  2887. is carried unchanged (renumbering per rotation made one session look like many); its provenance is read BEFORE the
  2888. transfer clears the ancestor's row, then restored so an inherited auto-title stays upgradeable.
  2889. """
  2890. # /goal, /heartbeat, /loop: carried locally, or atomically by the owner on a RuntimeSessionStore.
  2891. from agent.compression_automation import carry_compression_automation
  2892. carry_compression_automation(agent, old_session_id)
  2893. with _swallow('Could not carry rejected thinking on compression: %s'):
  2894. from agent.anthropic_thinking_replay import carry_rejected_thinking_to_session
  2895. carry_rejected_thinking_to_session(agent, old_session_id)
  2896. if not old_title:
  2897. return
  2898. _src = None
  2899. with _swallow('Could not read title provenance: %s'):
  2900. _src = agent._session_db.get_session_title_source(old_session_id)
  2901. try:
  2902. agent._session_db.set_session_title(agent.session_id, old_title)
  2903. except Exception as e:
  2904. logger.debug("Could not propagate title on compression: %s", e)
  2905. return
  2906. # set_session_title() records "user"; restore the original authority.
  2907. if _src is not None:
  2908. with _swallow('Could not propagate title provenance: %s'):
  2909. agent._session_db.set_session_title_source(agent.session_id, _src)
  2910. def _compression_child_source(agent: Any, parent_session_id: str) -> str:
  2911. """The parent row's persisted source: a compression child is the same conversation, so a ``--source tool``,
  2912. ``oneshot`` or inherited ``kanban`` label must not degrade to the bare ``agent.platform`` (#112550)."""
  2913. parent = None
  2914. with contextlib.suppress(Exception):
  2915. parent = agent._session_db.get_session(parent_session_id)
  2916. if parent and parent.get("source"):
  2917. return parent["source"]
  2918. from agent.session_source import session_source_for
  2919. return session_source_for(getattr(agent, "platform", None))
  2920. def _publish_rotated_compaction(
  2921. agent: Any, messages: list, compressed: list, *, new_system_prompt: str, lease: _CompressionLease,
  2922. old_session_id: str, compressed_user_turn_outcome: str,
  2923. ) -> None:
  2924. """Rotate the session: flush the parent, publish the child, re-point the agent.
  2925. Flushes current-turn msgs to the OLD session, passing the durable prefix (messages[:persist idx]) so
  2926. preflight, which runs before rows are marker-stamped, can't re-append them."""
  2927. current_idx = getattr(agent, "_persist_user_message_idx", None)
  2928. persisted_history = (
  2929. messages[:current_idx] if isinstance(current_idx, int) and 0 <= current_idx <= len(messages) else None
  2930. )
  2931. # The flush is durable and NOT rolled back on abort: a deliberately-ended parent
  2932. # fails publish forever, so check that before writing. Automatic end stamps are
  2933. # healed by publish (don't abort); the lease is re-acquirable (don't check it).
  2934. if _parent_deliberately_ended(agent._session_db, old_session_id):
  2935. raise RuntimeError(f"Compression parent already ended: {old_session_id}")
  2936. # Foreign-tail ceiling: the flush below writes OUR rows (already in handoff);
  2937. # rows above the lease watermark (lease start, or the newest adopted-snapshot row) up to this MAX(id)
  2938. # are foreign appends.
  2939. # No trustworthy ceiling means the clone could duplicate the handoff: skip tail preservation this rotation.
  2940. _foreign_tail_ceiling = None
  2941. with contextlib.suppress(Exception):
  2942. _foreign_tail_ceiling = agent._session_db.get_active_message_watermark(agent.session_id)
  2943. with contextlib.suppress(Exception): # best-effort — don't block compression on a flush error
  2944. agent._flush_messages_to_session_db(messages, conversation_history=persisted_history)
  2945. # Publish closure + child + handoff in one transaction so no reader sees an empty child. Child stays on the
  2946. # parent's profile ("default" persists as NULL) and keeps a live /yolo; publish COALESCEs from the parent row.
  2947. _profile_for_child = None
  2948. with contextlib.suppress(Exception):
  2949. from hermes_cli.profiles import get_active_profile_name
  2950. _profile_for_child = get_active_profile_name()
  2951. _profile_for_child = None if _profile_for_child == "default" else _profile_for_child
  2952. old_title = agent._session_db.get_session_title(agent.session_id)
  2953. new_session_id = mint_session_id()
  2954. from agent.context_compressor import _DB_PERSISTED_MARKER
  2955. from tools.approval_yolo import with_session_yolo
  2956. agent._session_db.publish_compression_child(
  2957. parent_session_id=old_session_id, child_session_id=new_session_id,
  2958. source=_compression_child_source(agent, old_session_id), model=agent.model,
  2959. model_config=with_session_yolo(agent._session_init_model_config, old_session_id),
  2960. system_prompt=new_system_prompt, messages=compressed,
  2961. cwd=getattr(agent, "working_directory", None), profile_name=_profile_for_child,
  2962. compression_lock_holder=lease.holder, require_compression_lease=lease.holder is not None,
  2963. require_lease_refresh=lease.holder is not None, lease_ttl_seconds=lease.ttl,
  2964. watermark=(lease.watermark if _foreign_tail_ceiling is not None else None),
  2965. watermark_ceiling=_foreign_tail_ceiling,
  2966. )
  2967. # `already_present` stamping is done by run_agent's _sync_persisted_markers;
  2968. # this branch covers inserted/merged only; direct callers must use that wrapper.
  2969. if compressed_user_turn_outcome in {"inserted", "merged"}:
  2970. # Stamp the anchor source row itself, not the (drifted, possibly out-of-range)
  2971. # persist index; don't match the HANDOFF row — for `merged` it is a superset.
  2972. _compressed_anchor_source = next((m for m in reversed(messages) if _is_real_user_message(m)), None)
  2973. if isinstance(_compressed_anchor_source, dict):
  2974. _compressed_anchor_source[_DB_PERSISTED_MARKER] = True
  2975. _session_messages = getattr(agent, "_session_messages", None)
  2976. if isinstance(_session_messages, list) and _session_messages is not messages:
  2977. # Adoption may leave _session_messages on the pre-adoption list with an out-of-range idx; stamp every
  2978. # scoped twin against the ANCHOR SOURCE, as the wrapper. An already-stamped exact twin still
  2979. # suppresses the broad pass here, or a content-equal old duplicate would get stamped.
  2980. _stamp_scoped_twins(_session_messages, _compressed_anchor_source, exact_counts_stamped=True)
  2981. for _handoff_message in compressed:
  2982. if isinstance(_handoff_message, dict):
  2983. _handoff_message[_DB_PERSISTED_MARKER] = True
  2984. agent.session_id = new_session_id
  2985. agent._db_flush_scan_prefix = None
  2986. _rebind_session_context(agent.session_id)
  2987. agent._session_db_created = True
  2988. _carry_session_state_to_child(agent, old_session_id, old_title)
  2989. def _warn_summary_or_aux_fallback(agent: Any) -> None:
  2990. """Surface a failed summary, or a recovered-but-broken aux compression model, once."""
  2991. summary_error = getattr(agent.context_compressor, "_last_summary_error", None)
  2992. if summary_error:
  2993. if getattr(agent, "_last_compression_summary_warning", None) != summary_error:
  2994. agent._last_compression_summary_warning = summary_error
  2995. agent._emit_warning(f"⚠ Compression summary failed: {summary_error}. Inserted a fallback context marker.")
  2996. else:
  2997. # Aux model may have errored and been recovered on main; tell the user their
  2998. # auxiliary.compression.model is broken even though compression succeeded.
  2999. _aux_fail_model = getattr(agent.context_compressor, "_last_aux_model_failure_model", None)
  3000. _aux_fail_err = getattr(agent.context_compressor, "_last_aux_model_failure_error", None)
  3001. # Dedup on (model, error) so we don't spam on every compaction
  3002. _aux_key = (_aux_fail_model, _aux_fail_err)
  3003. if _aux_fail_model and getattr(agent, "_last_aux_fallback_warning_key", None) != _aux_key:
  3004. agent._last_aux_fallback_warning_key = _aux_key
  3005. logger.warning(
  3006. "Configured compression model %r failed (%s); recovered using the main model.",
  3007. _aux_fail_model, _aux_fail_err or "unknown error",
  3008. )
  3009. agent._emit_warning(
  3010. f"ℹ Configured compression model '{_aux_fail_model}' failed, so Hermes summarised "
  3011. "with your main model instead. Check auxiliary.compression.model in your config."
  3012. )
  3013. def _reset_read_dedup_caches(task_id: str, *, session_id: str = "") -> None:
  3014. """Advance the file-read and skill_view repeat-read dedup to a fresh generation after a boundary.
  3015. The mtime map is kept: the first read of each unchanged key returns full content compaction may have
  3016. omitted; later reads return stubs, and stub-hit counters restart at the same boundary (#84857).
  3017. The computer_use screenshot dedup is session-keyed and forgets its last frame for the same reason.
  3018. """
  3019. with contextlib.suppress(Exception):
  3020. from tools.file_tools_read_tracking import reset_file_dedup
  3021. reset_file_dedup(task_id)
  3022. if session_id:
  3023. with contextlib.suppress(Exception):
  3024. from tools.computer_use.tool import reset_screenshot_dedup
  3025. reset_screenshot_dedup(session_id)
  3026. with contextlib.suppress(Exception):
  3027. from tools.skills_tool import reset_skill_view_dedup
  3028. reset_skill_view_dedup(task_id)
  3029. def _finish_compaction_boundary(
  3030. agent: Any, compressed: list, *, new_system_prompt: str, old_session_id: Optional[str], in_place: bool,
  3031. compacted_in_place: bool, session_commit_succeeded: bool, defer_context_engine_notification: bool,
  3032. compression_made_progress: bool, compression_used_fallback: bool, compression_feasibility_skip: bool,
  3033. task_id: str,
  3034. ) -> int:
  3035. """Post-commit bookkeeping: notify engines/providers/hooks, re-arm usage tracking.
  3036. Returns the rough post-compression token estimate (diagnostics only)."""
  3037. # old_session_id is bound only on rotation; _boundary_parent is the id the
  3038. # boundary notifications attribute prior state to (old id, or same id in-place).
  3039. _old_sid = old_session_id
  3040. _boundary_parent = _old_sid or agent.session_id or ""
  3041. # The heartbeat's terminal stamp landed on the PARENT before the id re-pointed;
  3042. # clear labels (keep last_activity_at) so the archived row isn't falsely fresh.
  3043. if _old_sid and session_commit_succeeded:
  3044. with _swallow("failed to clear archived compression parent's activity labels (ignored)", exc_info=True):
  3045. _labels_db = getattr(agent, "_session_db", None)
  3046. if callable(_clear_labels := getattr(type(_labels_db), "clear_session_activity_labels", None)):
  3047. _clear_labels(_labels_db, _old_sid)
  3048. # Plugin engines use boundary_reason="compression" to keep lineage/checkpoint
  3049. # state. Fires in BOTH modes: in-place passes the same id, the boundary is real.
  3050. if session_commit_succeeded and (bool(_old_sid) or compacted_in_place):
  3051. notify = (
  3052. _queue_context_engine_compression_notification
  3053. if defer_context_engine_notification
  3054. else _notify_context_engine_compression_complete
  3055. )
  3056. notify(agent, new_session_id=agent.session_id or "", old_session_id=_boundary_parent)
  3057. # Providers refresh cached per-session state; reset=False, conversation goes on.
  3058. # Fires in BOTH modes so buffers don't double-count dropped turns in-place.
  3059. with _swallow('memory manager on_session_switch (compression): %s'):
  3060. if (bool(_old_sid) or in_place) and agent._memory_manager:
  3061. agent._memory_manager.on_session_switch(
  3062. agent.session_id or "", parent_session_id=_boundary_parent, reset=False, reason="compression"
  3063. )
  3064. # Route via _emit_status so the warning reaches gateway platforms; store it on
  3065. # _compression_warning so a late-bound status_callback can replay it.
  3066. compressor = agent.context_compressor
  3067. _cc = compressor.compression_count
  3068. if _cc >= 2:
  3069. _cc_msg = (
  3070. f"{agent.log_prefix}⚠️ Session compressed {_cc} times — accuracy may degrade. Consider /new to start fresh."
  3071. )
  3072. agent._compression_warning = _cc_msg
  3073. agent._emit_diagnostic_status(_cc_msg)
  3074. # session:compress lets hooks ingest the old session before it's lost;
  3075. # in_place=True tells them the same id was compacted rather than rotated.
  3076. if getattr(agent, "event_callback", None):
  3077. with _swallow('event_callback error on session:compress: %s'):
  3078. agent.event_callback(
  3079. "session:compress",
  3080. {
  3081. "platform": agent.platform or "", "session_id": agent.session_id,
  3082. "old_session_id": _old_sid or "", "in_place": in_place,
  3083. "compression_count": compressor.compression_count,
  3084. },
  3085. )
  3086. # Rotation-independent flag: the gateway uses it (not an id diff) to re-baseline
  3087. # transcript handling (history_offset=0 + rewrite on the same id) in-place.
  3088. agent._last_compression_attempt_in_place = compacted_in_place
  3089. agent._last_compaction_in_place = compacted_in_place
  3090. # Diagnostics only, not provider usage: schema-heavy rough estimates can stay
  3091. # above threshold even after the next real request fits.
  3092. _compressed_est = estimate_request_tokens_rough(
  3093. compressed, system_prompt=new_system_prompt or "", tools=agent.tools or None
  3094. )
  3095. compressor.last_compression_rough_tokens = _compressed_est
  3096. compressor.last_prompt_tokens = -1
  3097. compressor.last_completion_tokens = 0
  3098. compressor.awaiting_real_usage_after_compression = True
  3099. # Transcript rewritten: invalidate the usage anchor's base snapshot explicitly
  3100. # (its structural check would fail closed anyway); estimate until re-anchored.
  3101. set_usage_anchor(agent, None)
  3102. # Arm the effectiveness verdict only after a completed rewrite crosses the
  3103. # boundary so later usage isn't charged to an attempt that changed nothing.
  3104. if compression_made_progress:
  3105. record_boundary = getattr(type(compressor), "record_completed_compaction", None)
  3106. if callable(record_boundary):
  3107. record_boundary(
  3108. compressor, used_fallback=compression_used_fallback, feasibility_skip=compression_feasibility_skip
  3109. )
  3110. else:
  3111. compressor._verify_compaction_cleared_threshold = True
  3112. _reset_read_dedup_caches(task_id, session_id=agent.session_id or "")
  3113. return _compressed_est
  3114. def _candidate_rejected(
  3115. agent: Any, compressed: Any, messages: list, messages_before_compression: list, *,
  3116. attempt: _Attempt,
  3117. ) -> bool:
  3118. """Reject an unusable compression candidate before any session mutation.
  3119. Order matters: compressor-reported abort, no progress, empty transcript, superseded attempt. Each branch surfaces
  3120. its own warning/telemetry; the caller releases the lease and returns the input unchanged when True.
  3121. """
  3122. # Aborted compression returns input unchanged: surface the error, skip rotation
  3123. # (no session ended); auto-compress callers detect no-op via equal lengths.
  3124. if getattr(agent.context_compressor, "_last_compress_aborted", False):
  3125. _summary_error = getattr(agent.context_compressor, "_last_summary_error", None)
  3126. _err = _summary_error or "unknown error"
  3127. if getattr(agent, "_last_compression_summary_warning", None) != _err:
  3128. agent._last_compression_summary_warning = _err
  3129. agent._emit_warning(
  3130. f"⚠ Compression aborted: {_err}. "
  3131. "No messages were dropped — conversation continues unchanged. "
  3132. "Run /compress to retry, or /new to start a fresh session."
  3133. )
  3134. _emit_aborted_attempt_telemetry(
  3135. agent, attempt.started_at, _summary_error and "summary_generation_aborted", attempt.seed
  3136. )
  3137. return True
  3138. # Compare semantic state, not identity: engines may return an equal copy or
  3139. # mutate the live list. ``==`` first (subclass __eq__), then marker-insensitive.
  3140. # Neither case may rotate or rewrite the session. The raw ``==`` leg runs FIRST so a list subclass
  3141. # returned by an engine keeps its ``__eq__`` semantics (tests seam on this); the marker-insensitive leg
  3142. # (#92231) then covers the cold-resume shape where the stamped snapshot differs from the marker-swept
  3143. # compress() output only by ``_db_persisted``.
  3144. if compressed == messages_before_compression or (
  3145. _strip_marker_for_comparison(compressed) == _strip_marker_for_comparison(messages_before_compression)
  3146. ):
  3147. logger.info(
  3148. "Compression made no progress (session=%s) — skipping boundary rewrite.", agent.session_id or "none"
  3149. )
  3150. # Unchanged output would fail identically next turn; arm structural backoff so
  3151. # auto-compress stops re-firing each turn (success lifts it, force overrides).
  3152. with _swallow('no-progress backoff arm failed', exc_info=True):
  3153. if callable(_recorder := getattr(agent.context_compressor, "_record_structural_no_op", None)):
  3154. _recorder("compaction returned the transcript unchanged (no_progress)")
  3155. _emit_aborted_attempt_telemetry(agent, attempt.started_at, "no_progress", attempt.seed)
  3156. return True
  3157. if not compressed:
  3158. logger.error(
  3159. "context compression returned an empty transcript; refusing to rotate session=%s so the parent remains resumable",
  3160. agent.session_id or "none",
  3161. )
  3162. with contextlib.suppress(Exception):
  3163. agent._emit_warning(
  3164. "⚠ Compression returned an empty transcript. No session split was performed; conversation continues unchanged."
  3165. )
  3166. _emit_aborted_attempt_telemetry(agent, attempt.started_at, "empty_transcript", attempt.seed)
  3167. return True
  3168. # A newer WORKING attempt supersedes us; discard the late candidate. No-op
  3169. # entry claims (sit-outs that never ran a summary) do not: keying on them
  3170. # discards a completed candidate and livelocks compression. Without a
  3171. # published working marker, fence poison alone misses a successor that
  3172. # minted its own fence — fall back to the entry-generation check.
  3173. if not _working_attempt_is_current(agent.context_compressor, attempt.generation):
  3174. _working_gen = getattr(
  3175. agent.context_compressor, "_compression_working_attempt_generation", None
  3176. )
  3177. logger.warning(
  3178. "Discarding late compression candidate: attempt generation "
  3179. "%s was superseded by a newer working attempt (current working: %s) (session=%s).",
  3180. attempt.generation,
  3181. _working_gen,
  3182. agent.session_id or "none",
  3183. )
  3184. agent._last_compaction_in_place = False
  3185. _emit_aborted_attempt_telemetry(agent, attempt.started_at, "attempt_superseded", attempt.seed)
  3186. return True
  3187. return False
  3188. @dataclasses.dataclass
  3189. class _CommitOutcome:
  3190. """Result of the SessionDB commit phase (in-place or rotation)."""
  3191. compressed: list
  3192. commit_started_at: float
  3193. refused_prompt: Optional[str] = None
  3194. old_session_id: Optional[str] = None
  3195. split_status: str = "not_applicable"
  3196. session_commit_succeeded: bool = False
  3197. compacted_in_place: bool = False
  3198. made_progress: bool = False
  3199. def _held_watermark(agent: Any, watermark: Optional[int], messages: list, verbatim_tail: Optional[list]) -> Optional[int]:
  3200. """The in-place commit's cap; see :func:`held_archive_watermark`."""
  3201. return held_archive_watermark(agent._session_db, agent.session_id, watermark, messages, verbatim_tail)
  3202. _ATTEMPT_TELEMETRY_FIELDS = (
  3203. "_last_compression_telemetry", "_active_compression_telemetry", "_compression_telemetry_seed",
  3204. )
  3205. @contextlib.contextmanager
  3206. def _hold_attempt_telemetry(compressor: Any):
  3207. """Restore the live attempt's telemetry after a call that may reset it (the memory flush)."""
  3208. held = tuple(getattr(compressor, name, None) for name in _ATTEMPT_TELEMETRY_FIELDS)
  3209. try:
  3210. yield
  3211. finally:
  3212. for name, value in zip(_ATTEMPT_TELEMETRY_FIELDS, held):
  3213. with contextlib.suppress(Exception):
  3214. setattr(compressor, name, value)
  3215. def _commit_compaction(
  3216. agent: Any, messages: list, compressed: list, *, in_place: bool, lease: _CompressionLease,
  3217. new_system_prompt: str, system_message: str, compressed_user_turn_outcome: str,
  3218. messages_before_compression: Optional[list], made_progress: bool, attempt: _Attempt,
  3219. verbatim_tail: Optional[list] = None, carried_messages: Optional[list] = None,
  3220. ) -> _CommitOutcome:
  3221. """Persist the compacted transcript: anti-growth guard, memory extraction, then the
  3222. in-place archive or the parent->child rotation.
  3223. Failures roll the live list back and arm the split-failure cooldown; a refused (would-grow) candidate returns
  3224. ``refused_prompt`` so the caller hands back the input unchanged. ``verbatim_tail`` (``/compress here N``) is
  3225. re-inserted after the compacted head by either commit and stamped once durable.
  3226. """
  3227. session_commit_succeeded = False
  3228. compacted_in_place = False
  3229. commit_started_at = time.monotonic()
  3230. split_status = "not_applicable"
  3231. old_session_id: Optional[str] = None # bound only once rotation begins
  3232. if agent._session_db:
  3233. split_status = "pending"
  3234. try:
  3235. # Pop _compaction_tail tags before the size estimate / rotation: they must not
  3236. # inflate anti-growth or reach the provider. Track ids: salvage may subset list.
  3237. _tail_tagged_ids = {id(m) for m in compressed if isinstance(m, dict) and m.pop("_compaction_tail", None)}
  3238. original_messages = messages_before_compression if messages_before_compression is not None else messages
  3239. compressed, _refused_sp = _salvage_or_refuse_grown_transcript(
  3240. agent, original_messages, compressed, system_message=system_message, attempt=attempt,
  3241. )
  3242. if compressed is None:
  3243. return _CommitOutcome(
  3244. compressed=messages, refused_prompt=_refused_sp, commit_started_at=commit_started_at
  3245. )
  3246. # Publish memory only after every refusal gate, from the immutable pre-compression snapshot.
  3247. # The engine's session-end reset nulls the in-flight attempt telemetry the success emit
  3248. # still needs (#118580), so hold it across the call.
  3249. with _hold_attempt_telemetry(agent.context_compressor):
  3250. agent.commit_memory_session(original_messages)
  3251. from agent.context_compressor import PROACTIVE_PRUNE_REARM_MODEL_CONFIG_KEY, stamp_db_persisted_markers
  3252. from hermes_cli.partial_compress import rejoin_compressed_head_and_tail
  3253. if in_place:
  3254. # In-place compaction: same session_id; soft-archive old turns (active=0, still
  3255. # searchable) + insert `compressed` atomically; no pre-flush (tail already in).
  3256. # Tail rows tagged by compress() are archived as superseded duplicates, not
  3257. # compacted=1. Count against the FINAL list — salvage may have dropped rows.
  3258. tail_count = sum(1 for m in compressed if id(m) in _tail_tagged_ids)
  3259. _tail_held = messages[max(0, len(messages) - tail_count):]
  3260. # The rewind takes the newest `tail_count` durable rows as the tail's originals, so a tail row
  3261. # with none (this turn's user row, which the CLI and gateway persist after preflight; unflushed
  3262. # scaffolding) would flag a summarized row superseded instead: gone from display and search.
  3263. # Only while a turn holds the session: between turns (manual /compress, gateway hygiene) the
  3264. # anchor is the last turn's, and the rows it points at are durable, just unmarked.
  3265. _turn_idx = getattr(agent, "_persist_user_message_idx", None)
  3266. if (getattr(agent, "_active_session_turn_lease_holder", None) is not None
  3267. and isinstance(_turn_idx, int) and 0 <= _turn_idx < len(messages)):
  3268. from agent.context_compressor import _DB_PERSISTED_MARKER
  3269. tail_count -= sum(
  3270. 1 for m in messages[max(_turn_idx, len(messages) - tail_count):]
  3271. if isinstance(m, dict) and not m.get(_DB_PERSISTED_MARKER)
  3272. and not isinstance(m.get("_row_id"), int))
  3273. # A tail dict stands for every row a repair retired into it (merged users, dropped orphan
  3274. # tool rows): counted once, the oldest carried original stays compacted=1 beside its live
  3275. # copy and is recalled twice. Only rows still active: a live list keeps the ids after an
  3276. # earlier compaction archived them.
  3277. from agent.conversation_compression_archive import ABSORBED_ROW_IDS, _positive_id
  3278. tail_count += len({
  3279. r for m in _tail_held if isinstance(m, dict)
  3280. for r in map(_positive_id, m.get(ABSORBED_ROW_IDS) or ())
  3281. if r is not None and agent._session_db.get_message_role(agent.session_id, r) is not None})
  3282. persisted = compressed
  3283. if verbatim_tail:
  3284. # The kept exchanges are durable rows under the watermark, so the archive below covers
  3285. # them too. Store them after the head in the same transaction, with the seam the caller
  3286. # would build, and count their originals as carried duplicates like compress()'s tail.
  3287. persisted = rejoin_compressed_head_and_tail(compressed, verbatim_tail)
  3288. tail_count += len(verbatim_tail)
  3289. from agent.conversation_compression_archive import coverage_for_commit
  3290. covered_ids, unresolved_held = coverage_for_commit(
  3291. agent._session_db, agent.session_id,
  3292. original_messages,
  3293. verbatim_tail)
  3294. agent._session_db.archive_and_compact(
  3295. agent.session_id, persisted, model_config_patch={PROACTIVE_PRUNE_REARM_MODEL_CONFIG_KEY: None},
  3296. watermark=_held_watermark(agent, lease.watermark, messages, verbatim_tail),
  3297. lock_holder=lease.holder, tail_count=tail_count, carried_messages=carried_messages,
  3298. covered_ids=covered_ids, unresolved_held=unresolved_held,
  3299. )
  3300. compressed = persisted
  3301. split_status = "in_place_committed"
  3302. # compress() returned marker-swept copies; stamp them as persisted or the next
  3303. # flush re-INSERTs the whole compacted transcript, doubling the live set. Reset
  3304. # the flush identity set so next turn diffs against the COMPACTED transcript.
  3305. # The verbatim tail is stamped as well: a seam fold drops its first row from
  3306. # `compressed`, and the stamps tell the caller the tail is already in the list.
  3307. stamp_db_persisted_markers([*compressed, *(verbatim_tail or ())])
  3308. agent._flushed_db_message_ids = set()
  3309. # Rotation-independent signal; the gateway reads this (not an id diff) to
  3310. # re-baseline transcript handling.
  3311. compacted_in_place = True
  3312. # In-place still updates the current row's prompt; rotation published it atomically above.
  3313. agent._session_db.update_system_prompt(agent.session_id, new_system_prompt)
  3314. agent._last_flushed_db_idx = 0
  3315. else:
  3316. # Bind old_session_id first: it is the rollback key in the handler below.
  3317. # ── Rotation (legacy): end this session, fork a continuation ─ Flush any un-persisted
  3318. # current-turn messages to the OLD session before ending it, so they survive in the
  3319. # preserved parent transcript (#47202). (In-place skips this — see above.) Pass the
  3320. # already-durable prefix as conversation_history so the flush skips it by identity (#68196).
  3321. # Preflight compression runs BEFORE the normal turn flush has stamped the cold-resumed
  3322. # history dicts with _DB_PERSISTED_MARKER, so without a boundary
  3323. # _flush_messages_to_session_db treats every restored row as new and re-appends the whole
  3324. # transcript to the parent. turn_context anchors _persist_user_message_idx at the
  3325. # current-turn user message before preflight runs, so messages[:idx] is exactly the
  3326. # persisted prefix; only the current turn's new messages get written. Bound to
  3327. # old_session_id, hoisted above the flush: the ``except`` handler below keys its in-memory
  3328. # rollback off this name, so anything that fails from here on rolls the transcript back
  3329. # instead of leaving the failed attempt's compacted snapshot in place.
  3330. old_session_id = agent.session_id
  3331. if verbatim_tail:
  3332. # Publish the kept exchanges with the head, as the in-place branch stores them: the
  3333. # gateway no longer rewrites a published child, so a head-only handoff loses the tail.
  3334. compressed = rejoin_compressed_head_and_tail(compressed, verbatim_tail)
  3335. _publish_rotated_compaction(
  3336. agent, messages, compressed, new_system_prompt=new_system_prompt, lease=lease,
  3337. old_session_id=old_session_id, compressed_user_turn_outcome=compressed_user_turn_outcome,
  3338. )
  3339. if verbatim_tail:
  3340. stamp_db_persisted_markers(verbatim_tail)
  3341. split_status = "rotated_committed"
  3342. agent._last_flushed_db_idx = len(compressed)
  3343. agent._flushed_db_message_session_id = agent.session_id
  3344. session_commit_succeeded = True
  3345. except Exception as e:
  3346. # Rotation: atomic publication failed (including lease loss) — keep the parent live and discard the stale
  3347. # compacted snapshot. In-place: archive_and_compact is atomic so old rows stay active, but marker-swept
  3348. # `compressed` would re-INSERT on top of them (doubling each try); gate on split_status (set right after
  3349. # commit). Either way the deepcopy keeps markers/identity and only the prune runway rolls back (the full
  3350. # snapshot restore is for pre-commit cancels; telemetry keeps the failed values). _db_flush_scan_prefix is
  3351. # intentionally NOT cleared: the scan is identity-based and the deepcopy replaces every row.
  3352. rotation_rollback = not in_place and old_session_id and agent.session_id == old_session_id
  3353. if rotation_rollback or (
  3354. in_place and split_status != "in_place_committed" and messages_before_compression is not None
  3355. ):
  3356. if rotation_rollback:
  3357. old_session_id = None
  3358. # In-place sibling of the rotation rollback above (#99477). archive_and_compact() is atomic,
  3359. # so a raise before it returned means EVERY pre-compaction row is still ``active = 1`` in
  3360. # state.db — nothing was archived and the compacted set was never inserted. But
  3361. # ``compressed`` is the marker-swept output of compress() (_strip_persistence_markers,
  3362. # #57491) and the post-commit ``stamp_db_persisted_markers`` never ran, so handing it back
  3363. # makes the next append-only flush treat the whole compacted transcript as new and INSERT it
  3364. # ON TOP of the rows it was supposed to replace. The active set then holds the summary AND
  3365. # the turns it summarized; the next resume reloads both, the token count goes UP, preflight
  3366. # fires again, and each failed attempt appends another copy of the protected head + tail
  3367. # (#99477: ~15 real turns stored as 3,814 rows, the first user message repeated 893 times).
  3368. # Gate on ``split_status`` rather than ``compacted_in_place``: it is assigned on the
  3369. # statement immediately after the atomic commit returns, so a committed compaction can never
  3370. # be rolled back into a live/durable mismatch of the opposite sign. The deepcopy carries
  3371. # each row's _DB_PERSISTED_MARKER from the pre-compression snapshot, so the restored
  3372. # transcript is correctly skipped by the flush, and replacing every dict breaks
  3373. # _db_flush_scan_prefix identity (same reasoning as the rotation branch — no explicit clear
  3374. # needed).
  3375. messages[:] = copy.deepcopy(messages_before_compression)
  3376. compressed = messages
  3377. made_progress = False
  3378. _restore_prune_rearm_tokens(agent.context_compressor, attempt.snapshot)
  3379. split_status = "aborted" if old_session_id is None and not in_place else "failed_not_indexed"
  3380. # If rotation rolled back to the parent, agent.session_id is the indexed parent
  3381. # and old_session_id was cleared: recovery, not an un-indexed orphan.
  3382. if old_session_id is None and not in_place:
  3383. logger.warning(
  3384. "Compression rotation aborted and rolled back to the parent session (%s): %s",
  3385. agent.session_id or "?", e,
  3386. )
  3387. else:
  3388. logger.warning("Session DB compression split failed — new session will NOT be indexed: %s", e)
  3389. # Arm the failure cooldown so the next turn can't rerun the doomed compression;
  3390. # try/except so a stub compressor can't mask the original error in this handler.
  3391. with _swallow('could not record split-failure cooldown', exc_info=True):
  3392. # See #97948.
  3393. agent.context_compressor._record_compression_failure_cooldown(
  3394. _SPLIT_FAILURE_COOLDOWN_SECONDS, f"session_split_failed: {e}"
  3395. )
  3396. return _CommitOutcome(
  3397. compressed=compressed, commit_started_at=commit_started_at, old_session_id=old_session_id,
  3398. split_status=split_status, session_commit_succeeded=session_commit_succeeded,
  3399. compacted_in_place=compacted_in_place, made_progress=made_progress,
  3400. )
  3401. @dataclasses.dataclass
  3402. class _SummaryPhase:
  3403. """Outcome of the summary phase; ``abort_prompt`` set means hand ``messages`` back."""
  3404. messages: list
  3405. compressed: Any = None
  3406. messages_before_compression: Optional[list] = None
  3407. approx_tokens: Optional[int] = None
  3408. pre_msg_count: int = 0
  3409. abort_prompt: Optional[str] = None
  3410. def _run_summary_phase(
  3411. agent: Any, messages: list, *, lease: _CompressionLease, in_place: bool, checkpoint_required: bool,
  3412. approx_tokens: Optional[int], focus_topic: Optional[str], force: bool, bypass_cooldown: bool,
  3413. commit_fence: Optional[CompressionCommitFence], hard_cancel_event: Any, system_message: str,
  3414. attempt: _Attempt,
  3415. ) -> _SummaryPhase:
  3416. """Adopt a grown durable parent, gather memory context and run the summarizer.
  3417. A hard cancel restores the compressor snapshot + live list, records a stall backoff while the lease is
  3418. still held, and aborts; any other failure restores the live list, releases the lease and re-raises."""
  3419. pre_msg_count = len(messages)
  3420. _activity_heartbeat: Optional[_CompressionActivityHeartbeat] = None
  3421. messages_before_compression = None
  3422. def _stop_heartbeat(desc: str) -> None:
  3423. nonlocal _activity_heartbeat
  3424. if _activity_heartbeat is not None:
  3425. _activity_heartbeat.stop(desc)
  3426. _activity_heartbeat = None
  3427. try:
  3428. lease.start_refresher()
  3429. if not in_place:
  3430. _adopted_parent = _adopt_grown_durable_parent(agent, lease, messages)
  3431. if _adopted_parent is not None:
  3432. messages = _adopted_parent
  3433. pre_msg_count = len(messages)
  3434. # Estimate was for the stale snapshot; force re-derivation from adopted rows.
  3435. approx_tokens = 0
  3436. # Adopted list is fully durable: re-anchor persist idx at the end so the post-
  3437. # compression flush skips it; run_agent marker sync realigns _session_messages.
  3438. agent._persist_user_message_idx = len(messages)
  3439. memory_context = _pre_compress_memory_context(agent, messages, checkpoint_required)
  3440. compress_fn, compress_kwargs = _resolve_compress_call(
  3441. agent, approx_tokens=approx_tokens, focus_topic=focus_topic, force=force, memory_context=memory_context,
  3442. bypass_cooldown=bypass_cooldown,
  3443. )
  3444. messages_before_compression = copy.deepcopy(messages)
  3445. _activity_heartbeat = _CompressionActivityHeartbeat(
  3446. agent, commit_fence=commit_fence, emit_client_status=lease.status_emitted,
  3447. ).start()
  3448. # Start this attempt's telemetry clean (#118580) only now, under the lease: done at entry, a call that stops
  3449. # at a gate or the lease erased a running attempt's record and seed.
  3450. for _name, _value in zip(_ATTEMPT_TELEMETRY_FIELDS, (None, None, dict(attempt.seed))):
  3451. with contextlib.suppress(Exception):
  3452. setattr(agent.context_compressor, _name, _value)
  3453. compressed = _run_summary_dispatch(
  3454. agent, messages, compress_fn, compress_kwargs, commit_fence=commit_fence,
  3455. attempt_generation=attempt.generation, hard_cancel_event=hard_cancel_event,
  3456. )
  3457. except AuxiliaryExplicitCancellation:
  3458. try:
  3459. attempt.restore_compressor(agent.context_compressor)
  3460. except BaseException as _rollback_exc:
  3461. # Compensation failure must surface, but it must not strand the
  3462. # session lease or retain an in-memory transcript mutation.
  3463. _restore_messages_snapshot(messages, messages_before_compression)
  3464. _stop_heartbeat("context compression rollback failed")
  3465. lease.release()
  3466. _emit_aborted_attempt_telemetry(
  3467. agent, attempt.started_at, f"rollback:{type(_rollback_exc).__name__}", attempt.seed
  3468. )
  3469. raise
  3470. _restore_messages_snapshot(messages, messages_before_compression)
  3471. # Record after restore so rollback cannot wipe a stall backoff, and
  3472. # while the lease is still held so the next turn cannot race it.
  3473. _stall_backoff = _record_stall_interrupted_backoff(
  3474. agent, commit_fence=commit_fence, started_at=attempt.started_at, messages=messages,
  3475. approx_tokens=approx_tokens,
  3476. )
  3477. _stop_heartbeat("context compression cancelled")
  3478. lease.release()
  3479. # Only the user's stop is ``explicit_interrupt`` (counted skipped); a host timeout cancels the fence
  3480. # without setting the stop event, and that is a real failure (hygiene 30 s budget, 600 s ceiling).
  3481. _host_cancel = commit_fence is not None and commit_fence.is_cancelled and not (
  3482. hard_cancel_event is not None and hard_cancel_event.is_set()
  3483. )
  3484. _emit_aborted_attempt_telemetry(
  3485. agent, attempt.started_at,
  3486. STALL_INTERRUPTED_FAILURE_CLASS if _stall_backoff
  3487. else "commit_fence_cancelled" if _host_cancel else "explicit_interrupt", attempt.seed,
  3488. )
  3489. return _SummaryPhase(messages=messages, abort_prompt=_existing_system_prompt(agent, system_message))
  3490. except BaseException as _compress_exc:
  3491. _restore_messages_snapshot(messages, messages_before_compression)
  3492. _stop_heartbeat("context compression failed")
  3493. lease.release()
  3494. _emit_aborted_attempt_telemetry(
  3495. agent, attempt.started_at, f"exception:{type(_compress_exc).__name__}", attempt.seed
  3496. )
  3497. raise
  3498. finally:
  3499. _stop_heartbeat("context compression completed")
  3500. return _SummaryPhase(
  3501. messages=messages, compressed=compressed, messages_before_compression=messages_before_compression,
  3502. approx_tokens=approx_tokens, pre_msg_count=pre_msg_count,
  3503. )
  3504. @dataclasses.dataclass
  3505. class _Attempt:
  3506. """Per-attempt ownership state threaded through the compress_context phases."""
  3507. snapshot: dict
  3508. generation: int
  3509. started_at: float
  3510. # This attempt's id, start session and trigger. Its emits pass it explicitly: the agent's copy is replaced
  3511. # when a stall fallback begins while this attempt is still unwinding.
  3512. seed: dict[str, Any]
  3513. durable_cooldown_authoritative: Optional[bool] = None
  3514. durable_cooldown_state: Optional[dict[str, Any]] = None
  3515. def restore_compressor(self, compressor: Any) -> None:
  3516. """Roll the compressor back to this attempt's snapshot (durable cooldown included)."""
  3517. _restore_compressor_attempt_state(
  3518. compressor, self.snapshot, durable_cooldown_authoritative=self.durable_cooldown_authoritative,
  3519. durable_cooldown_state=self.durable_cooldown_state, attempt_generation=self.generation,
  3520. )
  3521. def _begin_compression_attempt(
  3522. agent: Any, *, force: bool, defer_notification: bool, trigger: Optional[str] = None,
  3523. approx_tokens: Optional[int] = None, overflow_reason: Optional[str] = None,
  3524. ) -> _Attempt:
  3525. """Snapshot + claim the compressor, reset per-attempt agent signals, build this attempt's telemetry seed
  3526. (written to the compressor only at dispatch, in ``_run_summary_phase``).
  3527. The claim stops a late-unwinding sibling (stall-fallback overlap) from restoring its snapshot over ours or
  3528. clearing our cancellation consult. Signals are cleared at the VERY TOP, before codex/breaker
  3529. early-returns, so a stale value cannot make a later no-op look like lock contention;
  3530. ``_last_compression_attempt_in_place=None`` means aborted/no boundary for
  3531. ``conversation_history_after_compression()``."""
  3532. snapshot = _snapshot_compressor_attempt_state(agent.context_compressor)
  3533. generation = _claim_compressor_attempt(agent.context_compressor)
  3534. if defer_notification and callable(getattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, None)):
  3535. raise RuntimeError("a compression notification is already pending")
  3536. agent._last_compression_attempt_recorded = True
  3537. agent._last_compression_attempt_in_place = None
  3538. agent._compression_skipped_due_to_lock = None
  3539. # Clear the lock-skip signal at the VERY TOP, before the codex route and the breaker gates below can
  3540. # early-return (per-attempt state rule, #58630/#69853). A stale ``True``/holder value from a prior
  3541. # lock-skip must never make a later breaker/codex no-op look like lock contention to the automatic-path
  3542. # consumers (compression_deferred, #49874) — the second clear before lock acquisition below stays for
  3543. # the same reason it was added in #69870 and is simply idempotent now.
  3544. # Transient-block signal (#97488): cleared with the same per-attempt rule; set by the breaker gates
  3545. # below when a TRANSIENT guard (cooldown / structural backoff) no-ops this pass.
  3546. agent._compression_blocked_transient = None
  3547. started_at = time.monotonic()
  3548. attempt_id = uuid.uuid4().hex
  3549. trigger = trigger or ("manual" if force else "auto")
  3550. seed = {
  3551. "attempt_id": attempt_id, "session_id": getattr(agent, "session_id", None) or "", "trigger_source": trigger,
  3552. "overflow_reason": overflow_reason,
  3553. }
  3554. with contextlib.suppress(Exception):
  3555. agent._compression_attempt_id = attempt_id
  3556. from hermes_cli.observability.shared_metrics_events import begin_compression_attempt
  3557. begin_compression_attempt(trigger, approx_tokens or getattr(agent.context_compressor, "last_prompt_tokens", None))
  3558. return _Attempt(snapshot, generation, started_at, seed)
  3559. def _route_codex_compaction(
  3560. agent: Any, messages: list, system_message: str, *, commit_fence: Optional[CompressionCommitFence],
  3561. attempt: _Attempt, approx_tokens: Optional[int], task_id: str, force: bool,
  3562. ) -> tuple[list, str]:
  3563. """Codex owns the real thread: run its own compact under the commit fence bracket."""
  3564. if commit_fence is not None and not commit_fence.begin_commit(getattr(agent, "_hard_interrupt_requested", None)):
  3565. attempt.restore_compressor(agent.context_compressor)
  3566. _emit_bypassed_attempt_telemetry(
  3567. agent, attempt.started_at, commit_status="aborted", failure_class="commit_fence_cancelled",
  3568. approx_tokens=approx_tokens, route="codex_app_server", attempt_seed=attempt.seed,
  3569. )
  3570. return messages, _existing_system_prompt(agent, system_message)
  3571. try:
  3572. return _compress_context_via_codex_app_server(
  3573. agent, messages, system_message, approx_tokens=approx_tokens, task_id=task_id, force=force,
  3574. started_at=attempt.started_at, attempt_seed=attempt.seed,
  3575. )
  3576. finally:
  3577. if commit_fence is not None:
  3578. commit_fence.finish_commit()
  3579. def _refresh_main_credential(agent: Any) -> None:
  3580. """Re-resolve a rotating Anthropic OAuth token so the summary call never carries a revoked one."""
  3581. if getattr(agent, "api_mode", None) != "anthropic_messages":
  3582. return
  3583. refresh = getattr(agent, "_try_refresh_anthropic_client_credentials", None)
  3584. if callable(refresh):
  3585. with _swallow("pre-compression credential refresh failed: %s"):
  3586. refresh()
  3587. def _announce_compression_start(
  3588. agent: Any, *, message_count: int, approx_tokens: Optional[int], focus_topic: Optional[str], force: bool
  3589. ) -> _CompactionLifecycle:
  3590. """Log the attempt, emit the (engine-customisable) compacting status, return the lifecycle."""
  3591. logger.info(
  3592. "context compression started: session=%s messages=%d tokens=~%s model=%s focus=%r", agent.session_id or "none",
  3593. message_count, f"{approx_tokens:,}" if approx_tokens else "unknown", agent.model, focus_topic,
  3594. )
  3595. status = COMPACTION_STATUS
  3596. if not force:
  3597. status = automatic_compaction_status_message(
  3598. agent.context_compressor, phase="compress", default_message=status, approx_tokens=approx_tokens,
  3599. message_count=message_count, model=agent.model, focus_topic=focus_topic,
  3600. )
  3601. if status:
  3602. agent._emit_status(status)
  3603. return _CompactionLifecycle(agent, bool(status))
  3604. def compress_context(
  3605. agent: Any, messages: list, system_message: str, *, approx_tokens: Optional[int] = None,
  3606. task_id: str = "default", focus_topic: Optional[str] = None, force: bool = False,
  3607. bypass_cooldown: bool = False, defer_context_engine_notification: bool = False,
  3608. commit_fence: Optional[CompressionCommitFence] = None, verbatim_tail: Optional[list] = None,
  3609. trigger: Optional[str] = None, snapshot_is_current: Optional[Callable[[], bool]] = None,
  3610. overflow_reason: Optional[str] = None,
  3611. ) -> tuple[list, str]:
  3612. """Compress conversation context and split the session in SQLite.
  3613. ``force`` (manual /compress) clears the summary-failure cooldown; ``bypass_cooldown`` (provider-proven
  3614. overflow) skips it once, breakers still apply. ``commit_fence`` stops a timed-out worker mutating session
  3615. state. Returns ``(messages, system_prompt)``; on abort input is unchanged, NOT split.
  3616. Args: agent: The owning :class:`AIAgent`. messages: Current message history (will be summarised).
  3617. system_message: Current system prompt; used when compression needs a rebuilt cached prompt.
  3618. approx_tokens: Pre-compression token estimate, logged for ops. task_id: Tool task scope (used for
  3619. clearing file-read dedup state). focus_topic: Optional focus string for guided compression — the
  3620. summariser will prioritise preserving information related to this topic. Inspired by Claude Code's
  3621. ``/compact <focus>``. force: If True, bypass any active summary-failure cooldown. Set by the manual
  3622. ``/compress`` slash command so users can retry immediately after an auto-compress abort. Auto-compress
  3623. callers use the default ``False``. bypass_cooldown: If True, the automatic breaker gates ignore ONLY the
  3624. summary-failure cooldown for this attempt (#100661). Set by the provider-proven overflow recovery path:
  3625. the provider already rejected the request, so deferring until the cooldown lapses wedges the session.
  3626. Unlike ``force`` it does not clear the cooldown, and the ineffective/structural breakers still apply; a
  3627. failed attempt records its cooldown normally. defer_context_engine_notification: Delay the existing
  3628. context-engine hook until a manual host commits its outer history transaction. commit_fence: Optional
  3629. cooperative fence for executor callers that may time out. It prevents a late worker from mutating
  3630. session state after its caller has moved on. verbatim_tail: The exchanges ``/compress here N`` keeps
  3631. after ``messages``; an in-place commit stores them after the compacted head and returns head + tail.
  3632. trigger: Why this attempt runs (``"overflow"``, ``"pre_api"``, ``"post_tool"``, ``"idle"``, ...); defaults
  3633. to manual/auto from ``force``. Feeds attempt telemetry only; shared metrics bucket it coarsely.
  3634. snapshot_is_current: Optional host snapshot validation after durable lease admission. This protects
  3635. edits completed before admission; it is not a substitute for the host's final publication fence.
  3636. overflow_reason: The error classifier's reason for the provider rejection an ``"overflow"`` attempt
  3637. recovers from (``"context_overflow"``, ``"payload_too_large"``, ``"long_context_tier"``). Telemetry only.
  3638. """
  3639. attempt = _begin_compression_attempt(
  3640. agent, force=force, defer_notification=defer_context_engine_notification, trigger=trigger,
  3641. approx_tokens=approx_tokens, overflow_reason=overflow_reason,
  3642. )
  3643. # Codex owns the real thread; route compaction to its own compact (config
  3644. # compression.codex_app_server_auto). Memory handoff is Hermes-only: no native
  3645. # summary prompt to inject into. `is True`: MagicMock attributes are truthy.
  3646. checkpoint_required = getattr(agent, "compression_checkpoint_required", False) is True
  3647. if getattr(agent, "api_mode", None) == "codex_app_server":
  3648. if checkpoint_required:
  3649. blocked = _checkpoint_blocked(
  3650. "codex_app_server owns the authoritative thread and does not expose a truthful pre-compaction transcript boundary"
  3651. )
  3652. _emit_bypassed_attempt_telemetry(
  3653. agent, attempt.started_at, commit_status="aborted", failure_class=f"exception:{type(blocked).__name__}",
  3654. approx_tokens=approx_tokens, route="codex_app_server", attempt_seed=attempt.seed,
  3655. )
  3656. raise blocked
  3657. return _route_codex_compaction(
  3658. agent, messages, system_message, commit_fence=commit_fence, attempt=attempt, approx_tokens=approx_tokens,
  3659. task_id=task_id, force=force,
  3660. )
  3661. # All automatic entrypoints honor compressor cooldown/breaker state; hygiene's
  3662. # fresh AIAgent loads the persisted streak via bind_session_state() first.
  3663. if not force and (block_reason := _automatic_compression_gate_blocks(agent, bypass_cooldown)):
  3664. _emit_blocked_attempt_telemetry(agent, attempt.started_at, approx_tokens, attempt.seed, block_reason)
  3665. return messages, _existing_system_prompt(agent, system_message)
  3666. _pre_msg_count = len(messages)
  3667. # In-place keeps the SAME session_id (no rotation/child/renumber/re-sync). A
  3668. # missing attribute must default True, not rotation, which can wedge sessions.
  3669. in_place = bool(getattr(agent, "compression_in_place", True))
  3670. # Announce BEFORE the lazy feasibility probe: its live catalog / provider lookups are
  3671. # network-bound (connect timeouts stack up through proxies), and until this status lands
  3672. # the Desktop working row is a bare spinner with no "Summarizing thread" label (#111294).
  3673. lifecycle = _announce_compression_start(
  3674. agent, message_count=_pre_msg_count, approx_tokens=approx_tokens, focus_topic=focus_topic, force=force
  3675. )
  3676. # After the announce: an expired token's refresh is a blocking network call. Before the probe and summary call.
  3677. _refresh_main_credential(agent)
  3678. # Lazy feasibility probe (~400ms cold) on first attempt, not __init__; it sets
  3679. # _compression_warning so status replay still surfaces the warning. Marked checked
  3680. # only after the probe completes (transient failures are swallowed inside). A hard
  3681. # rejection propagates; retire the announced phase first so the client is not left compacting.
  3682. try:
  3683. lazy_compression_feasibility_check(agent)
  3684. except Exception:
  3685. lifecycle.complete(force_terminal=True)
  3686. raise
  3687. lease, _abort_prompt = _acquire_compression_lease(
  3688. agent, commit_fence=commit_fence, lifecycle=lifecycle, system_message=system_message,
  3689. approx_tokens=approx_tokens, attempt=attempt,
  3690. )
  3691. if lease is None:
  3692. return messages, _abort_prompt
  3693. # Publish the holder-qualified release hook before a timeout can win the
  3694. # fence. If no durable lock was acquired there is no hook to publish.
  3695. lease.finish_lock_setup()
  3696. try:
  3697. current = snapshot_is_current is None or snapshot_is_current()
  3698. except BaseException:
  3699. lease.release()
  3700. raise
  3701. if not current:
  3702. _emit_aborted_attempt_telemetry(agent, attempt.started_at, "snapshot_stale", attempt.seed)
  3703. lease.release()
  3704. return messages, _existing_system_prompt(agent, system_message)
  3705. _adopted = _adopt_if_parent_rotated(
  3706. agent, lease, messages, system_message, attempt=attempt, approx_tokens=approx_tokens,
  3707. )
  3708. if _adopted is not None:
  3709. return _adopted
  3710. # Snapshot durable cooldown only once we own the lease. Runs for force=True
  3711. # too but skips the automatic breaker gate: manual compression retries now.
  3712. attempt.durable_cooldown_authoritative, attempt.durable_cooldown_state = (
  3713. _capture_authoritative_cooldown_under_lease(agent.context_compressor, attempt.snapshot)
  3714. )
  3715. if attempt.durable_cooldown_authoritative is False:
  3716. # Durable cooldown read failed under a built-in compressor: force=True could
  3717. # clear an unknown newer row before cancellation could restore it. Abort.
  3718. lease.release()
  3719. _emit_bypassed_attempt_telemetry(
  3720. agent, attempt.started_at, commit_status="aborted", failure_class="cooldown_state_unreadable",
  3721. approx_tokens=approx_tokens, attempt_seed=attempt.seed,
  3722. )
  3723. return messages, _existing_system_prompt(agent, system_message)
  3724. # Another path may have compacted this session in place since construction;
  3725. # re-read breaker state under the lock, not the bind_session_state() snapshot.
  3726. block_reason = None if force else _automatic_compression_gate_blocks(agent, bypass_cooldown, include_cooldown=False)
  3727. if block_reason:
  3728. lease.release()
  3729. _emit_blocked_attempt_telemetry(agent, attempt.started_at, approx_tokens, attempt.seed, block_reason)
  3730. return messages, _existing_system_prompt(agent, system_message)
  3731. # Interrupts/redirects must not tear a summary in half. Use the explicit stop
  3732. # Event (message fields race) + fence timeout so pool slots free promptly.
  3733. # Explicit stop surfaces set a separate Event atomically; never infer cause from the racy message
  3734. # fields. A host timeout also cancels the attempt's commit fence. Feed BOTH into the protected
  3735. # auxiliary-call seam so the compression owner unwinds promptly while an isolated provider stream
  3736. # finishes or closes in its daemon worker. Otherwise four timed-out streams retain all four shared
  3737. # compression-pool slots until the auxiliary stream's longer absolute ceiling expires. See #23975.
  3738. _hard_cancel_event = getattr(agent, "_hard_interrupt_requested", None)
  3739. phase = _run_summary_phase(
  3740. agent, messages, lease=lease, in_place=in_place, checkpoint_required=checkpoint_required,
  3741. approx_tokens=approx_tokens, focus_topic=focus_topic, force=force, bypass_cooldown=bypass_cooldown,
  3742. commit_fence=commit_fence, hard_cancel_event=_hard_cancel_event, system_message=system_message, attempt=attempt,
  3743. )
  3744. if phase.abort_prompt is not None:
  3745. return phase.messages, phase.abort_prompt
  3746. messages, compressed = phase.messages, phase.compressed
  3747. messages_before_compression = phase.messages_before_compression
  3748. approx_tokens, _pre_msg_count = phase.approx_tokens, phase.pre_msg_count
  3749. effect_messages_before = _snapshot_compression_effect_input(messages_before_compression, verbatim_tail)
  3750. _commit_fence_entered = False
  3751. try:
  3752. # Capture the verdict before rotation callbacks reset compressor fields on rebind.
  3753. _compression_made_progress, _compression_used_fallback, _compression_feasibility_skip = (
  3754. bool(getattr(agent.context_compressor, name, False))
  3755. for name in ("_last_compression_made_progress", "_last_summary_fallback_used", "_last_feasibility_skip")
  3756. )
  3757. if _candidate_rejected(
  3758. agent, compressed, messages, messages_before_compression, attempt=attempt,
  3759. ):
  3760. # Engines may mutate their input in place; a refused candidate must not leak that mutation.
  3761. _restore_messages_snapshot(messages, messages_before_compression)
  3762. return messages, _existing_system_prompt(agent, system_message)
  3763. if commit_fence is not None:
  3764. _commit_fence_entered = commit_fence.begin_commit(_hard_cancel_event)
  3765. if not _commit_fence_entered:
  3766. attempt.restore_compressor(agent.context_compressor)
  3767. _restore_messages_snapshot(messages, messages_before_compression)
  3768. logger.info(
  3769. "Compression commit cancelled before session mutation (session=%s).", agent.session_id or "none"
  3770. )
  3771. agent._last_compaction_in_place = False
  3772. _stall_backoff = _record_stall_interrupted_backoff(
  3773. agent, commit_fence=commit_fence, started_at=attempt.started_at, messages=messages,
  3774. approx_tokens=approx_tokens,
  3775. )
  3776. _existing_sp = _existing_system_prompt(agent, system_message)
  3777. _emit_aborted_attempt_telemetry(
  3778. agent, attempt.started_at,
  3779. STALL_INTERRUPTED_FAILURE_CLASS if _stall_backoff else "commit_fence_cancelled", attempt.seed,
  3780. )
  3781. return messages, _existing_sp
  3782. _warn_summary_or_aux_fallback(agent)
  3783. # A just-delivered reply the engine folded away must stay live or the
  3784. # next render drops it from the surface (#118900). It runs FIRST: the
  3785. # todo fold rewrites the trailing user row, and both later passes place themselves around the tail, so the
  3786. # reply has to be back in its chronological slot before they look.
  3787. from agent.conversation_compression_reply_anchor import _ensure_compressed_keeps_last_assistant_reply
  3788. # `/compress here N` hands only the HEAD in as `messages` and carries the kept tail
  3789. # separately: the head's last assistant is an OLD reply the user explicitly asked to
  3790. # fold, not the just-delivered one (which lives in the verbatim tail), so the guard
  3791. # must not undo the compression it was asked for.
  3792. reinserted_reply = None if verbatim_tail else _ensure_compressed_keeps_last_assistant_reply(
  3793. messages, compressed, session_id=agent.session_id,
  3794. )
  3795. if reinserted_reply is not None:
  3796. logger.info(
  3797. "Compression: engine folded away the just-delivered assistant reply; reinserted it into the "
  3798. "active set (session=%s).", agent.session_id or "none",
  3799. )
  3800. _fold_todo_snapshot(agent, compressed)
  3801. compressed_user_turn_outcome = _ensure_compressed_has_user_turn(messages, compressed)
  3802. new_system_prompt = _rebuild_system_prompt_at_boundary(agent, system_message)
  3803. commit = _commit_compaction(
  3804. agent, messages, compressed, in_place=in_place, lease=lease, new_system_prompt=new_system_prompt,
  3805. system_message=system_message, compressed_user_turn_outcome=compressed_user_turn_outcome,
  3806. messages_before_compression=messages_before_compression, made_progress=_compression_made_progress,
  3807. attempt=attempt, verbatim_tail=verbatim_tail,
  3808. # The reinserted copy keeps the original's _row_id/timestamp (production flush stamps
  3809. # both); carry exactly that one row so the commit rewinds the durable original instead
  3810. # of archiving it compacted=1 next to a fresh twin (display would show it twice). The
  3811. # todo fold / user-anchor rows above keep their own class and are not carried.
  3812. carried_messages=[reinserted_reply] if reinserted_reply is not None else None,
  3813. )
  3814. if commit.refused_prompt is not None:
  3815. _restore_messages_snapshot(messages, messages_before_compression)
  3816. return messages, commit.refused_prompt
  3817. compressed = commit.compressed
  3818. split_status = commit.split_status
  3819. _compressed_est = _finish_compaction_boundary(
  3820. agent, compressed, new_system_prompt=new_system_prompt, old_session_id=commit.old_session_id,
  3821. in_place=in_place, compacted_in_place=commit.compacted_in_place,
  3822. session_commit_succeeded=commit.session_commit_succeeded,
  3823. defer_context_engine_notification=defer_context_engine_notification,
  3824. compression_made_progress=commit.made_progress, compression_used_fallback=_compression_used_fallback,
  3825. compression_feasibility_skip=_compression_feasibility_skip, task_id=task_id,
  3826. )
  3827. logger.info(
  3828. "context compression done: session=%s messages=%d->%d rough_tokens=~%s awaiting_real_usage=true",
  3829. agent.session_id or "none", _pre_msg_count, len(compressed), f"{_compressed_est:,}",
  3830. )
  3831. lifecycle.commit_status = (
  3832. "committed" if split_status in {"not_applicable", "in_place_committed", "rotated_committed"} else "aborted"
  3833. )
  3834. # A rewrite that survived a later bookkeeping failure is still what the model sees: count it as saved.
  3835. rewritten = commit.compacted_in_place or compressed is not messages
  3836. if rewritten or lifecycle.commit_status == "committed":
  3837. _record_committed_attempt_effect(agent, effect_messages_before, compressed, verbatim_tail, split_status)
  3838. _emit_compression_attempt_telemetry(
  3839. agent, started_at=attempt.started_at, commit_status=lifecycle.commit_status, split_status=split_status,
  3840. failure_class=("session_split_failed" if split_status in {"failed_not_indexed", "aborted"} else None),
  3841. commit_started_at=commit.commit_started_at, attempt_seed=attempt.seed, history_rewritten=rewritten,
  3842. )
  3843. return compressed, new_system_prompt
  3844. finally:
  3845. # Release the old lock after rotation bookkeeping so a waking contender sees the new session id.
  3846. try:
  3847. lease.release()
  3848. finally:
  3849. if _commit_fence_entered:
  3850. commit_fence.finish_commit()
  3851. # 4 MB leaves headroom under Anthropic's 5 MB; shrinking loses quality but only
  3852. # runs after a confirmed provider rejection, so the alternative is failure.
  3853. _IMAGE_SHRINK_TARGET_BYTES = 4 * 1024 * 1024
  3854. _IMAGE_SUFFIX_BY_MIME = {
  3855. "image/png": ".png", "image/gif": ".gif", "image/webp": ".webp", "image/jpeg": ".jpg", "image/jpg": ".jpg",
  3856. "image/bmp": ".bmp",
  3857. }
  3858. def _data_url_mime(header: str, default: str = "image/jpeg") -> str:
  3859. """``image/*`` mime from a ``data:`` URL header, else ``default``."""
  3860. if header.startswith("data:"):
  3861. candidate = header[len("data:") :].split(";", 1)[0].strip()
  3862. if candidate.startswith("image/"):
  3863. return candidate
  3864. return default
  3865. def _decode_pixels(data_url: str) -> Optional[tuple]:
  3866. """``(width, height)`` of a base64 data URL; None when Pillow is missing or the payload is corrupt."""
  3867. try:
  3868. import base64, io
  3869. _, _, data_d = data_url.partition(",")
  3870. if not data_d or not data_url.startswith("data:"):
  3871. return None
  3872. from PIL import Image
  3873. with Image.open(io.BytesIO(base64.b64decode(data_d))) as _img:
  3874. return _img.size
  3875. except Exception:
  3876. return None
  3877. def _shrink_data_url(url: str, *, max_dimension: int, resize_fn: Any) -> tuple:
  3878. """Return ``(resized_url, unshrinkable)`` for a data URL.
  3879. ``resized_url`` is None when no rewrite applied. ``unshrinkable`` is True only when the image violated a
  3880. constraint and resizing failed to satisfy that same constraint, so the caller knows a retry is pointless.
  3881. The accept gate MUST use the axis that triggered the shrink: a pixel downscale can re-encode to MORE bytes
  3882. (PNG non-monotonic); a byte-only reject wedges."""
  3883. target_bytes = _IMAGE_SHRINK_TARGET_BYTES
  3884. if not isinstance(url, str) or not url.startswith("data:"):
  3885. return None, False
  3886. triggered_by = "bytes" if len(url) > target_bytes else None # over byte budget
  3887. if triggered_by is None:
  3888. # Bytes fine; check pixels against the provider cap (tiny bytes, huge pixels).
  3889. dims = _decode_pixels(url)
  3890. if dims is None or max(dims) <= max_dimension:
  3891. return None, False
  3892. triggered_by = "dimension"
  3893. try:
  3894. header, _, data = url.partition(",")
  3895. mime = _data_url_mime(header)
  3896. import base64 as _b64
  3897. raw = _b64.b64decode(data)
  3898. tmp = tempfile.NamedTemporaryFile(
  3899. prefix="hermes_shrink_", suffix=_IMAGE_SUFFIX_BY_MIME.get(mime, ".jpg"), delete=False
  3900. )
  3901. try:
  3902. tmp.write(raw)
  3903. tmp.close()
  3904. resized = resize_fn(
  3905. Path(tmp.name), mime_type=mime, max_base64_bytes=target_bytes, max_dimension=max_dimension
  3906. )
  3907. finally:
  3908. with contextlib.suppress(Exception):
  3909. Path(tmp.name).unlink(missing_ok=True)
  3910. if not resized:
  3911. return None, True # Pillow couldn't help
  3912. new_dims = _decode_pixels(resized)
  3913. if triggered_by == "bytes":
  3914. # Byte budget is binding — bytes must shrink; and the resizer may return an
  3915. # over-cap blob (long side freezes at the 64px short-side floor) → still 400.
  3916. ok = len(resized) < len(url) and (new_dims is None or max(new_dims) <= max_dimension)
  3917. elif new_dims is not None:
  3918. # Dimension cap is binding: accept a byte-larger re-encode if now within cap.
  3919. ok = max(new_dims) <= max_dimension
  3920. else:
  3921. # Can't verify dimensions: fall back to the bytes-must-shrink gate so we never
  3922. # accept an unverifiable byte-larger blob.
  3923. ok = len(resized) < len(url)
  3924. return (resized, False) if ok else (None, True)
  3925. except Exception as exc:
  3926. logger.warning("image-shrink recovery: re-encode failed — %s", exc)
  3927. return None, triggered_by is not None
  3928. def _source_to_data_url(source: Any) -> Optional[str]:
  3929. """Anthropic ``{"type": "base64", ...}`` image source → data URL, else None."""
  3930. if not isinstance(source, dict) or source.get("type") != "base64":
  3931. return None
  3932. data = source.get("data")
  3933. if not isinstance(data, str) or not data:
  3934. return None
  3935. media_type = str(source.get("media_type") or "image/jpeg").strip()
  3936. return f"data:{media_type if media_type.startswith('image/') else 'image/jpeg'};base64,{data}"
  3937. def _write_data_url_to_source(source: dict, data_url: str) -> dict:
  3938. """Return a NEW source dict carrying the re-encoded payload.
  3939. Copy-on-write: parts may be shared with the persistent history, so mutating in place would store the
  3940. degraded image; the caller replaces the part."""
  3941. header, _, data = data_url.partition(",")
  3942. return {**source, "type": "base64", "media_type": _data_url_mime(header), "data": data}
  3943. def try_shrink_image_parts_in_messages(api_messages: list, *, max_dimension: int = 8000) -> bool:
  3944. """Re-encode oversized native image parts to recover from image-too-large errors.
  3945. Mutates ``api_messages`` in place. Returns True if any part was replaced, False if nothing to shrink or
  3946. Pillow could not help. Targets data-URL parts over 4 MB or ``max_dimension`` (Anthropic's per-side pixel
  3947. cap, parsed from the rejection by the caller); http(s) image URLs are left untouched."""
  3948. if not api_messages:
  3949. return False
  3950. try:
  3951. from tools.vision_tools import _resize_image_for_vision
  3952. except Exception as exc:
  3953. logger.warning("image-shrink recovery: vision_tools unavailable — %s", exc)
  3954. return False
  3955. changed_count = 0
  3956. # Track over-target parts that could not be shrunk: if any remain, a retry
  3957. # re-sends the same payload and wastes the single retry budget.
  3958. unshrinkable_oversized = 0
  3959. def _shrink(url: Any) -> tuple:
  3960. return _shrink_data_url(url, max_dimension=max_dimension, resize_fn=_resize_image_for_vision)
  3961. for msg in api_messages:
  3962. if not isinstance(msg, dict):
  3963. continue
  3964. content = msg.get("content")
  3965. if not isinstance(content, list):
  3966. continue
  3967. # Copy-on-write: part/source dicts can alias stored history, so build a new
  3968. # content list and reassign msg["content"] on the per-call copy.
  3969. new_content: list | None = None
  3970. for part_idx, part in enumerate(content):
  3971. if not isinstance(part, dict):
  3972. continue
  3973. ptype = part.get("type")
  3974. replacement = None
  3975. if ptype == "image":
  3976. source = part.get("source")
  3977. resized, unshrinkable = _shrink(_source_to_data_url(source) or "")
  3978. if resized and isinstance(source, dict):
  3979. replacement = {**part, "source": _write_data_url_to_source(source, resized)}
  3980. elif ptype in {"image_url", "input_image"}:
  3981. image_value = part.get("image_url")
  3982. # OpenAI chat.completions: {"image_url": {"url": "data:..."}}
  3983. # OpenAI Responses: {"image_url": "data:..."}
  3984. if isinstance(image_value, dict):
  3985. resized, unshrinkable = _shrink(image_value.get("url", ""))
  3986. if resized:
  3987. replacement = {**part, "image_url": {**image_value, "url": resized}}
  3988. elif isinstance(image_value, str):
  3989. resized, unshrinkable = _shrink(image_value)
  3990. if resized:
  3991. replacement = {**part, "image_url": resized}
  3992. else:
  3993. continue
  3994. else:
  3995. continue
  3996. if replacement is not None:
  3997. if new_content is None:
  3998. new_content = list(content)
  3999. new_content[part_idx] = replacement
  4000. changed_count += 1
  4001. elif unshrinkable:
  4002. unshrinkable_oversized += 1
  4003. if new_content is not None:
  4004. msg["content"] = new_content
  4005. target_mb = _IMAGE_SHRINK_TARGET_BYTES / (1024 * 1024)
  4006. if changed_count:
  4007. logger.info("image-shrink recovery: re-encoded %d image part(s) to fit under %.0f MB", changed_count, target_mb)
  4008. if unshrinkable_oversized:
  4009. # An unshrinkable oversized image makes retry pointless; signal no progress even
  4010. # if others shrank so the caller surfaces the original error.
  4011. logger.warning(
  4012. "image-shrink recovery: %d oversized image part(s) could not be "
  4013. "shrunk under %.0f MB — not retrying (would re-send rejected payload)", unshrinkable_oversized,
  4014. target_mb,
  4015. )
  4016. return False
  4017. return changed_count > 0
  4018. __all__ = [
  4019. "COMPACTION_DONE_STATUS",
  4020. "COMPACTION_HEARTBEAT_STATUS",
  4021. "COMPACTION_STATUS",
  4022. "COMPACTION_STATUS_MARKER",
  4023. "check_compression_model_feasibility",
  4024. "compress_context",
  4025. "is_compaction_progress_status",
  4026. "replay_compression_warning",
  4027. "try_shrink_image_parts_in_messages",
  4028. ]